@delali/narsil 0.3.1-next.157 → 0.3.1-next.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
- package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +66 -18
- package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1199 -1092
- package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -16
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -227
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
- package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +95 -52
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +33 -13
- package/dist/server.mjs +81 -100
- package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +88 -72
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +89 -59
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +66 -50
- package/package.json +233 -8
- package/dist/analysis-change-CKEMM2MC.mjs +0 -55
- package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
- package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-25MTVDYD.mjs +0 -432
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-AI5UI46D.mjs +0 -12
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
- package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
- package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
- package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MY2XKH22.mjs +0 -40
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
- package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
- package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
- package/dist/browser/chunk-V43N6TZO.mjs +0 -857
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
- package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-5ESVMFDJ.mjs +0 -20
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NDSQLPMQ.mjs +0 -25
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-HYXPVJVS.mjs +0 -32
- package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
- package/dist/browser/retention-TY3L3BFN.mjs +0 -211
- package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
- package/dist/browser/start-2UZTOBG5.mjs +0 -26
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-ULXAN5B4.mjs +0 -196
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-2ROUFMQX.mjs +0 -543
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-4B7JIGQD.mjs +0 -106
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BNL5OPRV.mjs +0 -634
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-C7XQYG6U.mjs +0 -50
- package/dist/chunk-CQZXZ75V.mjs +0 -562
- package/dist/chunk-CWS55TEX.mjs +0 -242
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FEZVDOWX.mjs +0 -6996
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GPWMUPCA.mjs +0 -1126
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JSV6OQ4C.mjs +0 -74
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KCQYFO4K.mjs +0 -62
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LDNCLRZU.mjs +0 -218
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MW6BR4QP.mjs +0 -316
- package/dist/chunk-MWD5VPN7.mjs +0 -21
- package/dist/chunk-ODYKO5TO.mjs +0 -101
- package/dist/chunk-OG7UGJ6F.mjs +0 -63
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-OYABFKTW.mjs +0 -1974
- package/dist/chunk-P5LANSH6.mjs +0 -121
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PIH5SJ4N.mjs +0 -7481
- package/dist/chunk-PSZI4O24.mjs +0 -253
- package/dist/chunk-PV2CG7EJ.mjs +0 -886
- package/dist/chunk-PWGYQXML.mjs +0 -470
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QLDABJZM.mjs +0 -20
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RCX4WNLC.mjs +0 -151
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RIHIVFIQ.mjs +0 -240
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RWURUDSP.mjs +0 -2639
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SOI6LTHM.mjs +0 -216
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UD5HIS4U.mjs +0 -382
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-UJDJB3NE.mjs +0 -913
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VSLSGASN.mjs +0 -30
- package/dist/chunk-VZ2ONHEE.mjs +0 -3715
- package/dist/chunk-W7OSSKH5.mjs +0 -1427
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-V5DRIECI.mjs +0 -34
- package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
- package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-GPKLTSKA.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-JYHFTVPS.mjs +0 -63
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
- package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
- package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
- package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
- package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
- package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
- package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
- package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
- package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
- package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
- package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
- package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
- package/dist/release-SMR4WJD4.mjs +0 -32
- package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
- package/dist/restore-queue-LZH7LMLR.mjs +0 -55
- package/dist/retention-3K7ARGAG.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-O7ICP7NJ.mjs +0 -783
- package/dist/start-S4PFP6RS.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-YLJ3ZPQ3.mjs +0 -50
- package/dist/write-2OBJGLFS.mjs +0 -233
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
|
@@ -1,548 +1,201 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
import './chunk-
|
|
8
|
-
|
|
9
|
-
import './chunk-
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
import './chunk-
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
import {
|
|
16
|
-
import
|
|
17
|
-
import './chunk-
|
|
18
|
-
import {
|
|
19
|
-
import './chunk-
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
|
|
23
|
-
import './chunk-
|
|
24
|
-
import './chunk-
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import './chunk-
|
|
28
|
-
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import './chunk-
|
|
32
|
-
import {
|
|
33
|
-
import './chunk-
|
|
34
|
-
import { decode
|
|
1
|
+
import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES, scheduleSyncRetention } from './chunk-FE3MRIBN.mjs';
|
|
2
|
+
export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-FE3MRIBN.mjs';
|
|
3
|
+
import { openPagedSegmentScan, countsAsLatestHolder, buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, chunksOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, bytesThatSegmentsHoldInMemory, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, createDetachedIndex, checkPagedSegmentFully, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readMetadataEnvelope, snapshotCheckpointFor, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, readSegmentManifest, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, growPartitionsTo, createMarkerWriter, metadataConfigOf, durableMetadataConfigOf } from './chunk-SXIRM4MX.mjs';
|
|
4
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedSegmentFully, countsAsLatestHolder, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, openPagedSegmentScan, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-SXIRM4MX.mjs';
|
|
5
|
+
import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-TSLV5P2J.mjs';
|
|
6
|
+
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-TSLV5P2J.mjs';
|
|
7
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-2JRERUNZ.mjs';
|
|
8
|
+
export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-2JRERUNZ.mjs';
|
|
9
|
+
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-4CAMH7FO.mjs';
|
|
10
|
+
import { readSnapshotArchive, decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-OOO54OVC.mjs';
|
|
11
|
+
export { readSnapshotArchive } from './chunk-OOO54OVC.mjs';
|
|
12
|
+
import { createDurableDirectory } from './chunk-TW543F33.mjs';
|
|
13
|
+
import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, readSegmentFile, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-W34VJXY7.mjs';
|
|
14
|
+
export { checkPagedContents, checkPagedSelections, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-W34VJXY7.mjs';
|
|
15
|
+
import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-U6B7DUQW.mjs';
|
|
16
|
+
import './chunk-5FLRXF73.mjs';
|
|
17
|
+
import './chunk-JDDXV6UF.mjs';
|
|
18
|
+
import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-7MWYORQC.mjs';
|
|
19
|
+
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
20
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
|
|
21
|
+
import { encode } from './chunk-77AUVDQ5.mjs';
|
|
22
|
+
export { bindStorageHost } from './chunk-77AUVDQ5.mjs';
|
|
23
|
+
import { sha256 } from './chunk-LNGE7SLY.mjs';
|
|
24
|
+
import { setNestedValue } from './chunk-QBVTYY4J.mjs';
|
|
25
|
+
import { rewriteSpillOf } from './chunk-IB4BYLE3.mjs';
|
|
26
|
+
import { deserializePartition } from './chunk-2RIS76RJ.mjs';
|
|
27
|
+
import { decodeVectorIndexParts, decodeVectorFilePayload, isDead, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-LAWIJRMJ.mjs';
|
|
28
|
+
export { createHNSWIndex, osqBitsOf } from './chunk-LAWIJRMJ.mjs';
|
|
29
|
+
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
30
|
+
import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-6EZW2GQL.mjs';
|
|
31
|
+
import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createRequestId, createPartitionRouter, buildSegmentPayloads } from './chunk-TINW4BSH.mjs';
|
|
32
|
+
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
33
|
+
import './chunk-AB27FWZM.mjs';
|
|
34
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
35
|
+
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
36
|
+
import './chunk-7T4LAYT4.mjs';
|
|
37
|
+
import './chunk-JV74RKCY.mjs';
|
|
38
|
+
import { flattenSchema, parseVectorDimension, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, MAX_DOCUMENT_NESTING_DEPTH, requireValidFieldTypes } from './chunk-R4CE4IVU.mjs';
|
|
39
|
+
import './chunk-U2BKYEC7.mjs';
|
|
40
|
+
import { detectRuntime, spawnNodeWorker, resolveWorkerEntry } from './chunk-OUWZKNO3.mjs';
|
|
41
|
+
import { STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-3B775F3C.mjs';
|
|
42
|
+
import './chunk-E2F6LZR3.mjs';
|
|
43
|
+
import { MIN_M } from './chunk-ZXX5GZEX.mjs';
|
|
44
|
+
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
45
|
+
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
46
|
+
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
35
47
|
|
|
36
|
-
// src/persistence/
|
|
37
|
-
function
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EACCES", "EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
63
|
-
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
64
|
-
function errnoOf(err) {
|
|
65
|
-
return err?.code;
|
|
66
|
-
}
|
|
67
|
-
function integerOrNull(text) {
|
|
68
|
-
const value = Number.parseInt(text ?? "", 10);
|
|
69
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
70
|
-
}
|
|
71
|
-
function parseHolder(text) {
|
|
72
|
-
const [pidText, threadText, startText, nonceText] = text.split("\n");
|
|
73
|
-
const pid = integerOrNull(pidText);
|
|
74
|
-
if (pid === null || pid <= 0) return null;
|
|
75
|
-
const nonce = nonceText?.trim() ?? "";
|
|
76
|
-
return {
|
|
77
|
-
pid,
|
|
78
|
-
threadId: integerOrNull(threadText),
|
|
79
|
-
processStartMs: integerOrNull(startText),
|
|
80
|
-
nonce: nonce.length > 0 ? nonce : null
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
function formatHolder(holder) {
|
|
84
|
-
return `${holder.pid}
|
|
85
|
-
${holder.threadId ?? ""}
|
|
86
|
-
${holder.processStartMs ?? ""}
|
|
87
|
-
${holder.nonce ?? ""}
|
|
88
|
-
`;
|
|
89
|
-
}
|
|
90
|
-
async function readHolder(fs, lockPath) {
|
|
91
|
-
try {
|
|
92
|
-
return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
|
|
93
|
-
} catch (err) {
|
|
94
|
-
if (errnoOf(err) === "ENOENT") return "absent";
|
|
95
|
-
throw err;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
async function settledHolder(fs, lockPath) {
|
|
99
|
-
const first = await readHolder(fs, lockPath);
|
|
100
|
-
if (first !== "unreadable") return first;
|
|
101
|
-
await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
|
|
102
|
-
return readHolder(fs, lockPath);
|
|
103
|
-
}
|
|
104
|
-
function heldElsewhere(root, holder, lockPath) {
|
|
105
|
-
return new NarsilError(
|
|
106
|
-
ErrorCodes.CONFIG_INVALID,
|
|
107
|
-
`The durability directory "${root}" is in use by ${holder}, and only one engine may write a write-ahead log directory. Point this engine at another directory, or shut the other engine down first. Where no engine uses the directory, delete "${lockPath}"`,
|
|
108
|
-
{ directory: root }
|
|
109
|
-
);
|
|
110
|
-
}
|
|
111
|
-
function holderLabel(holder, self) {
|
|
112
|
-
if (holder.pid !== self.pid) return `process ${holder.pid}`;
|
|
113
|
-
return holder.threadId === self.threadId ? "another copy of Narsil in this thread" : "another thread of this process";
|
|
114
|
-
}
|
|
115
|
-
async function createLockFileExclusively(fs, lockPath, content) {
|
|
116
|
-
try {
|
|
117
|
-
await fs.writeFile(lockPath, content, { flag: "wx" });
|
|
118
|
-
return true;
|
|
119
|
-
} catch (err) {
|
|
120
|
-
if (errnoOf(err) === "EEXIST") return false;
|
|
121
|
-
throw err;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
async function createLockFile(fs, lockPath, draftPath, content) {
|
|
125
|
-
await fs.writeFile(draftPath, content, { flag: "w" });
|
|
126
|
-
try {
|
|
127
|
-
await fs.link(draftPath, lockPath);
|
|
128
|
-
return true;
|
|
129
|
-
} catch (err) {
|
|
130
|
-
const code = errnoOf(err);
|
|
131
|
-
if (code === "EEXIST") return false;
|
|
132
|
-
if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
|
|
133
|
-
return createLockFileExclusively(fs, lockPath, content);
|
|
48
|
+
// src/persistence/paged/adopt.ts
|
|
49
|
+
function yieldToEventLoop() {
|
|
50
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
51
|
+
}
|
|
52
|
+
function applyNewerTombstones(prepared) {
|
|
53
|
+
for (let at = 0; at < prepared.length; at++) {
|
|
54
|
+
if (!prepared[at].isNew) continue;
|
|
55
|
+
for (let newer = at + 1; newer < prepared.length; newer++) {
|
|
56
|
+
const access = prepared[newer].segment.paged;
|
|
57
|
+
if (access === null) continue;
|
|
58
|
+
for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async function applyNewerTombstonesInTurns(prepared) {
|
|
63
|
+
let applied = 0;
|
|
64
|
+
for (let at = 0; at < prepared.length; at++) {
|
|
65
|
+
if (!prepared[at].isNew) continue;
|
|
66
|
+
for (let newer = at + 1; newer < prepared.length; newer++) {
|
|
67
|
+
const access = prepared[newer].segment.paged;
|
|
68
|
+
if (access === null) continue;
|
|
69
|
+
for (const docId of access.snapshot.tombstones) {
|
|
70
|
+
prepared[at].segment.tombstoneDocument(docId);
|
|
71
|
+
applied += 1;
|
|
72
|
+
if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
|
|
73
|
+
}
|
|
134
74
|
}
|
|
135
|
-
throw err;
|
|
136
|
-
} finally {
|
|
137
|
-
await fs.unlink(draftPath).catch(() => void 0);
|
|
138
75
|
}
|
|
139
76
|
}
|
|
140
|
-
async function
|
|
141
|
-
const
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
if (await createLockFile(fs, lockPath, draftPath, content)) return;
|
|
145
|
-
const holder = await settledHolder(fs, lockPath);
|
|
146
|
-
if (typeof holder === "object" && await holderIsLive()) {
|
|
147
|
-
throw heldElsewhere(root, holderLabel(holder, self), lockPath);
|
|
77
|
+
async function prepareDiskAdoption(directory, manager, partitions) {
|
|
78
|
+
for (const entry of partitions) {
|
|
79
|
+
for (const ref of entry.segments) {
|
|
80
|
+
if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
148
81
|
}
|
|
149
|
-
await removeStaleLock();
|
|
150
|
-
}
|
|
151
|
-
throw heldElsewhere(root, "another process", lockPath);
|
|
152
|
-
}
|
|
153
|
-
async function describeSelf() {
|
|
154
|
-
return {
|
|
155
|
-
pid: process.pid,
|
|
156
|
-
threadId: await nodeThreadId(),
|
|
157
|
-
processStartMs: Math.round(Date.now() - process.uptime() * 1e3),
|
|
158
|
-
nonce: globalThis.crypto.randomUUID()
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
async function lockDirectory(root) {
|
|
162
|
-
const fs = await getFs();
|
|
163
|
-
const pathMod = await getPath();
|
|
164
|
-
const resolvedRoot = pathMod.resolve(root);
|
|
165
|
-
const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
|
|
166
|
-
const self = await describeSelf();
|
|
167
|
-
async function holds() {
|
|
168
|
-
const holder = await readHolder(fs, lockPath);
|
|
169
|
-
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
170
|
-
}
|
|
171
|
-
async function removeOwnLockFile() {
|
|
172
|
-
if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
|
|
173
|
-
}
|
|
174
|
-
let directoryKey = null;
|
|
175
|
-
let closeDirectory = () => void 0;
|
|
176
|
-
try {
|
|
177
|
-
await fs.mkdir(resolvedRoot, { recursive: true });
|
|
178
|
-
const stat = await fs.stat(resolvedRoot, { bigint: true });
|
|
179
|
-
const key = `${stat.dev}:${stat.ino}`;
|
|
180
|
-
if (lockedDirectories.has(key)) throw heldElsewhere(root, "another engine in this process", lockPath);
|
|
181
|
-
lockedDirectories.add(key);
|
|
182
|
-
directoryKey = key;
|
|
183
|
-
await acquire(fs, root, lockPath, self);
|
|
184
|
-
closeDirectory = holdDirectoryOpen(resolvedRoot);
|
|
185
|
-
} catch (err) {
|
|
186
|
-
if (directoryKey !== null) lockedDirectories.delete(directoryKey);
|
|
187
|
-
await removeOwnLockFile();
|
|
188
|
-
if (err instanceof NarsilError) throw err;
|
|
189
|
-
throw new NarsilError(
|
|
190
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
191
|
-
`The engine cannot lock the durability directory "${root}"`,
|
|
192
|
-
{ directory: root, cause: err instanceof Error ? err.message : String(err) }
|
|
193
|
-
);
|
|
194
82
|
}
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
}
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
// src/persistence/durability/list-recursive.ts
|
|
210
|
-
async function listRecursive(dir, base, pathMod, fs) {
|
|
211
|
-
const results = [];
|
|
212
|
-
let entries;
|
|
213
|
-
try {
|
|
214
|
-
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
215
|
-
} catch (err) {
|
|
216
|
-
if (err.code === "ENOENT") {
|
|
217
|
-
return results;
|
|
83
|
+
const prepared = [];
|
|
84
|
+
const droppedKeys = [];
|
|
85
|
+
for (const entry of partitions) {
|
|
86
|
+
if (entry.partitionId >= manager.partitionCount) continue;
|
|
87
|
+
const partition = manager.getPartition(entry.partitionId);
|
|
88
|
+
const held = /* @__PURE__ */ new Map();
|
|
89
|
+
if (isCompositePartition(partition)) {
|
|
90
|
+
for (const segment of partition.frozenSegments()) {
|
|
91
|
+
if (segment.paged !== null) held.set(segment.paged.key, segment);
|
|
92
|
+
}
|
|
218
93
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
94
|
+
const list = [];
|
|
95
|
+
for (const ref of entry.segments) {
|
|
96
|
+
const existing = held.get(ref.key);
|
|
97
|
+
if (existing !== void 0) {
|
|
98
|
+
held.delete(ref.key);
|
|
99
|
+
list.push({ segment: existing, isNew: false });
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
103
|
+
list.push({ segment: createPagedFrozenSegment(snapshot), isNew: true });
|
|
227
104
|
}
|
|
105
|
+
droppedKeys.push(...held.keys());
|
|
106
|
+
await applyNewerTombstonesInTurns(list);
|
|
107
|
+
prepared.push({ partitionId: entry.partitionId, prepared: list });
|
|
228
108
|
}
|
|
229
|
-
return
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
// src/persistence/durability/durable-filesystem.ts
|
|
233
|
-
function validateRelativeKey(key) {
|
|
234
|
-
if (key.length === 0) {
|
|
235
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
236
|
-
}
|
|
237
|
-
if (key.includes("\0")) {
|
|
238
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
function wrapFsyncError(err, key) {
|
|
242
|
-
throw new NarsilError(
|
|
243
|
-
ErrorCodes.PERSISTENCE_FSYNC_FAILED,
|
|
244
|
-
`fsync failed for "${key}"; the write is not acknowledged and the node must recover from the durable log`,
|
|
245
|
-
{ key, cause: err instanceof Error ? err.message : String(err) }
|
|
246
|
-
);
|
|
109
|
+
return { partitions: prepared, droppedKeys };
|
|
247
110
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
try {
|
|
257
|
-
return await fs.open(filePath, "wx+");
|
|
258
|
-
} catch (err) {
|
|
259
|
-
if (err.code !== "EEXIST") {
|
|
260
|
-
throw err;
|
|
111
|
+
function keysThatSegmentsStillRead(manager) {
|
|
112
|
+
const keys = /* @__PURE__ */ new Set();
|
|
113
|
+
for (const partition of manager.getAllPartitions()) {
|
|
114
|
+
if (!isCompositePartition(partition)) continue;
|
|
115
|
+
for (const segment of partition.frozenSegments()) {
|
|
116
|
+
if (segment.paged !== null) keys.add(segment.paged.key);
|
|
261
117
|
}
|
|
262
|
-
return fs.open(filePath, "r+");
|
|
263
118
|
}
|
|
119
|
+
return keys;
|
|
264
120
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
try {
|
|
270
|
-
;
|
|
271
|
-
({ bytesWritten } = await handle.write(buffer, offset, buffer.length - offset));
|
|
272
|
-
} catch (err) {
|
|
273
|
-
throw new NarsilError(
|
|
274
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
275
|
-
`Write to "${key}" failed; the write is not acknowledged`,
|
|
276
|
-
{
|
|
277
|
-
key,
|
|
278
|
-
written: offset,
|
|
279
|
-
total: buffer.length,
|
|
280
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
281
|
-
}
|
|
282
|
-
);
|
|
283
|
-
}
|
|
284
|
-
if (bytesWritten <= 0) {
|
|
285
|
-
throw new NarsilError(
|
|
286
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
287
|
-
`Write to "${key}" made no progress; the file would be truncated and the write is not acknowledged`,
|
|
288
|
-
{ key, written: offset, total: buffer.length }
|
|
289
|
-
);
|
|
121
|
+
function closeUnusedSegments(adoption) {
|
|
122
|
+
for (const { prepared } of adoption.partitions) {
|
|
123
|
+
for (const { segment, isNew } of prepared) {
|
|
124
|
+
if (isNew) segment.paged?.close();
|
|
290
125
|
}
|
|
291
|
-
offset += bytesWritten;
|
|
292
126
|
}
|
|
293
127
|
}
|
|
294
|
-
|
|
295
|
-
|
|
128
|
+
function actionPartitionsOf(adoption) {
|
|
129
|
+
return adoption.partitions.map(({ partitionId, prepared }) => ({
|
|
130
|
+
partitionId,
|
|
131
|
+
segments: prepared.map(({ segment, isNew }) => {
|
|
132
|
+
const access = segment.paged;
|
|
133
|
+
if (access !== null && isNew) return { snapshot: access.snapshot };
|
|
134
|
+
return { segmentId: segment.segmentId };
|
|
135
|
+
})
|
|
136
|
+
}));
|
|
137
|
+
}
|
|
138
|
+
function swapPreparedPartitions(manager, partitions, changed, onSwapped) {
|
|
139
|
+
for (const { partitionId, prepared } of partitions) {
|
|
140
|
+
const kept = [];
|
|
141
|
+
for (const { segment } of prepared) {
|
|
142
|
+
if (segment.arrays.documentCount > 0) kept.push(segment);
|
|
143
|
+
else segment.paged?.close();
|
|
144
|
+
}
|
|
145
|
+
adoptDiskSegments(manager, partitionId, kept, changed);
|
|
146
|
+
onSwapped?.(partitionId, kept);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
function adoptDiskSegments(manager, partitionId, segments, changed) {
|
|
150
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
151
|
+
const before = composite.frozenSegments();
|
|
152
|
+
const kept = [];
|
|
153
|
+
for (const docId of changed) {
|
|
154
|
+
if (docPartitionMap.get(docId) !== partitionId) continue;
|
|
155
|
+
const document = composite.getRef(docId);
|
|
156
|
+
if (document !== void 0) kept.push([docId, document]);
|
|
157
|
+
}
|
|
158
|
+
const dropped = before.filter((segment) => !segments.includes(segment));
|
|
159
|
+
for (const segment of segments) {
|
|
160
|
+
if (before.includes(segment)) continue;
|
|
161
|
+
for (const docId of changed) segment.tombstoneDocument(docId);
|
|
162
|
+
}
|
|
163
|
+
composite.replaceFrozenSegments(segments);
|
|
164
|
+
composite.live.clear();
|
|
165
|
+
for (const [docId, pid] of docPartitionMap) {
|
|
166
|
+
if (pid === partitionId) docPartitionMap.delete(docId);
|
|
167
|
+
}
|
|
168
|
+
composite.live.beginBatch();
|
|
296
169
|
try {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
const code = err.code;
|
|
301
|
-
if (code === "EISDIR" || code === "EPERM" || code === "EINVAL" || code === "ENOTSUP") {
|
|
302
|
-
return;
|
|
170
|
+
for (const [docId, document] of kept) {
|
|
171
|
+
composite.live.insert(docId, document, manager.config.schema, manager.language, insertOptions);
|
|
172
|
+
docPartitionMap.set(docId, partitionId);
|
|
303
173
|
}
|
|
304
|
-
wrapFsyncError(err, dirPath);
|
|
305
174
|
} finally {
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
return { path: resolvedPath, resolvedBase, pathMod };
|
|
324
|
-
}
|
|
325
|
-
async function ensureDir(filePath, pathMod, fs) {
|
|
326
|
-
await fs.mkdir(pathMod.dirname(filePath), { recursive: true });
|
|
327
|
-
}
|
|
328
|
-
return {
|
|
329
|
-
get root() {
|
|
330
|
-
return root;
|
|
331
|
-
},
|
|
332
|
-
lock() {
|
|
333
|
-
return lockDirectory(root);
|
|
334
|
-
},
|
|
335
|
-
async identity() {
|
|
336
|
-
const fs = await getFs();
|
|
337
|
-
const pathMod = await getPath();
|
|
338
|
-
try {
|
|
339
|
-
const stat = await fs.stat(pathMod.resolve(root), { bigint: true });
|
|
340
|
-
return `${stat.dev}:${stat.ino}`;
|
|
341
|
-
} catch (err) {
|
|
342
|
-
if (err.code === "ENOENT") return null;
|
|
343
|
-
throw err;
|
|
344
|
-
}
|
|
345
|
-
},
|
|
346
|
-
async appendHandle(key) {
|
|
347
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
348
|
-
const fs = await getFs();
|
|
349
|
-
await ensureDir(filePath, pathMod, fs);
|
|
350
|
-
const handle = await fs.open(filePath, "a");
|
|
351
|
-
return {
|
|
352
|
-
async append(bytes) {
|
|
353
|
-
await writeFully(handle, bytes, key);
|
|
354
|
-
},
|
|
355
|
-
async sync() {
|
|
356
|
-
try {
|
|
357
|
-
await handle.sync();
|
|
358
|
-
} catch (err) {
|
|
359
|
-
wrapFsyncError(err, key);
|
|
360
|
-
}
|
|
361
|
-
},
|
|
362
|
-
async truncate(byteLength) {
|
|
363
|
-
await handle.truncate(byteLength);
|
|
364
|
-
try {
|
|
365
|
-
await handle.sync();
|
|
366
|
-
} catch (err) {
|
|
367
|
-
wrapFsyncError(err, key);
|
|
368
|
-
}
|
|
369
|
-
},
|
|
370
|
-
async size() {
|
|
371
|
-
const stat = await handle.stat();
|
|
372
|
-
return stat.size;
|
|
373
|
-
},
|
|
374
|
-
async close() {
|
|
375
|
-
await handle.close();
|
|
376
|
-
}
|
|
377
|
-
};
|
|
378
|
-
},
|
|
379
|
-
async markerHandle(key) {
|
|
380
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
381
|
-
const fs = await getFs();
|
|
382
|
-
await ensureDir(filePath, pathMod, fs);
|
|
383
|
-
const handle = await openReadWriteCreating(filePath, fs);
|
|
384
|
-
return {
|
|
385
|
-
async read() {
|
|
386
|
-
const stat = await handle.stat();
|
|
387
|
-
if (stat.size === 0) {
|
|
388
|
-
return new Uint8Array(0);
|
|
389
|
-
}
|
|
390
|
-
const bytes = new Uint8Array(stat.size);
|
|
391
|
-
await handle.read(bytes, 0, stat.size, 0);
|
|
392
|
-
return bytes;
|
|
393
|
-
},
|
|
394
|
-
async writeSlot(offset, slot) {
|
|
395
|
-
try {
|
|
396
|
-
await handle.write(slot, 0, slot.length, offset);
|
|
397
|
-
} catch (err) {
|
|
398
|
-
throw new NarsilError(
|
|
399
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
400
|
-
`Write to "${key}" failed; the write is not acknowledged`,
|
|
401
|
-
{ key, cause: err instanceof Error ? err.message : String(err) }
|
|
402
|
-
);
|
|
403
|
-
}
|
|
404
|
-
try {
|
|
405
|
-
await handle.sync();
|
|
406
|
-
} catch (err) {
|
|
407
|
-
wrapFsyncError(err, key);
|
|
408
|
-
}
|
|
409
|
-
},
|
|
410
|
-
async close() {
|
|
411
|
-
await handle.close();
|
|
412
|
-
}
|
|
413
|
-
};
|
|
414
|
-
},
|
|
415
|
-
async syncDirectoryOf(key) {
|
|
416
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
417
|
-
const fs = await getFs();
|
|
418
|
-
await fsyncDirectory(pathMod.dirname(filePath), fs);
|
|
419
|
-
},
|
|
420
|
-
async atomicWrite(key, data) {
|
|
421
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
422
|
-
const fs = await getFs();
|
|
423
|
-
await ensureDir(filePath, pathMod, fs);
|
|
424
|
-
const dir = pathMod.dirname(filePath);
|
|
425
|
-
const tempName = `${pathMod.basename(filePath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
426
|
-
const tempPath = pathMod.join(dir, tempName);
|
|
427
|
-
const buffers = Array.isArray(data) ? data : [data];
|
|
428
|
-
let handle = null;
|
|
429
|
-
try {
|
|
430
|
-
handle = await fs.open(tempPath, "w");
|
|
431
|
-
for (const buffer of buffers) {
|
|
432
|
-
await writeFully(handle, buffer, tempPath);
|
|
433
|
-
}
|
|
434
|
-
try {
|
|
435
|
-
await handle.sync();
|
|
436
|
-
} catch (err) {
|
|
437
|
-
wrapFsyncError(err, tempPath);
|
|
438
|
-
}
|
|
439
|
-
await handle.close();
|
|
440
|
-
handle = null;
|
|
441
|
-
await fs.rename(tempPath, filePath);
|
|
442
|
-
await fsyncDirectory(dir, fs);
|
|
443
|
-
} catch (err) {
|
|
444
|
-
if (handle !== null) {
|
|
445
|
-
await handle.close().catch(() => void 0);
|
|
446
|
-
}
|
|
447
|
-
await fs.unlink(tempPath).catch(() => void 0);
|
|
448
|
-
throw err;
|
|
449
|
-
}
|
|
450
|
-
},
|
|
451
|
-
async read(key) {
|
|
452
|
-
const { path: filePath } = await resolve(key);
|
|
453
|
-
const fs = await getFs();
|
|
454
|
-
try {
|
|
455
|
-
const data = await fs.readFile(filePath);
|
|
456
|
-
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
457
|
-
} catch (err) {
|
|
458
|
-
if (err.code === "ENOENT") {
|
|
459
|
-
return null;
|
|
460
|
-
}
|
|
461
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read "${key}"`, {
|
|
462
|
-
key,
|
|
463
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
464
|
-
});
|
|
465
|
-
}
|
|
466
|
-
},
|
|
467
|
-
async readRange(key, offset, length) {
|
|
468
|
-
const { path: filePath } = await resolve(key);
|
|
469
|
-
const fs = await getFs();
|
|
470
|
-
let handle = null;
|
|
471
|
-
try {
|
|
472
|
-
handle = await fs.open(filePath, "r");
|
|
473
|
-
const bytes = new Uint8Array(length);
|
|
474
|
-
let filled = 0;
|
|
475
|
-
while (filled < length) {
|
|
476
|
-
const { bytesRead } = await handle.read(bytes, filled, length - filled, offset + filled);
|
|
477
|
-
if (bytesRead === 0) break;
|
|
478
|
-
filled += bytesRead;
|
|
479
|
-
}
|
|
480
|
-
if (filled < length) {
|
|
481
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
|
|
482
|
-
key,
|
|
483
|
-
offset,
|
|
484
|
-
length
|
|
485
|
-
});
|
|
486
|
-
}
|
|
487
|
-
return bytes;
|
|
488
|
-
} finally {
|
|
489
|
-
await handle?.close();
|
|
490
|
-
}
|
|
491
|
-
},
|
|
492
|
-
async byteLength(key) {
|
|
493
|
-
const { path: filePath } = await resolve(key);
|
|
494
|
-
const fs = await getFs();
|
|
495
|
-
try {
|
|
496
|
-
return (await fs.stat(filePath)).size;
|
|
497
|
-
} catch (err) {
|
|
498
|
-
if (err.code === "ENOENT") {
|
|
499
|
-
return null;
|
|
500
|
-
}
|
|
501
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read the length of "${key}"`, {
|
|
502
|
-
key,
|
|
503
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
504
|
-
});
|
|
505
|
-
}
|
|
506
|
-
},
|
|
507
|
-
async remove(key) {
|
|
508
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
509
|
-
const fs = await getFs();
|
|
510
|
-
try {
|
|
511
|
-
await fs.unlink(filePath);
|
|
512
|
-
} catch (err) {
|
|
513
|
-
if (err.code === "ENOENT") {
|
|
514
|
-
return;
|
|
515
|
-
}
|
|
516
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, {
|
|
517
|
-
key,
|
|
518
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
519
|
-
});
|
|
175
|
+
composite.live.endBatch();
|
|
176
|
+
}
|
|
177
|
+
for (const segment of dropped) segment.paged?.close();
|
|
178
|
+
}
|
|
179
|
+
function adoptSegmentsOnWorker(manager, action) {
|
|
180
|
+
const partitions = action.partitions.map(({ partitionId, segments }) => {
|
|
181
|
+
const partition = manager.getPartition(partitionId);
|
|
182
|
+
const held = isCompositePartition(partition) ? partition.frozenSegments() : [];
|
|
183
|
+
const prepared = segments.map((item) => {
|
|
184
|
+
if ("snapshot" in item) return { segment: createPagedFrozenSegment(item.snapshot), isNew: true };
|
|
185
|
+
const existing = held.find((segment) => segment.segmentId === item.segmentId);
|
|
186
|
+
if (existing === void 0) {
|
|
187
|
+
throw new NarsilError(
|
|
188
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
189
|
+
`Partition ${partitionId} of "${action.indexName}" holds no segment "${item.segmentId}" to keep`,
|
|
190
|
+
{ indexName: action.indexName, partitionId, segmentId: item.segmentId }
|
|
191
|
+
);
|
|
520
192
|
}
|
|
521
|
-
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
await ensureDir(target.path, target.pathMod, fs);
|
|
528
|
-
await fs.rename(source.path, target.path);
|
|
529
|
-
const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
|
|
530
|
-
for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
|
|
531
|
-
},
|
|
532
|
-
async list(prefix) {
|
|
533
|
-
const fs = await getFs();
|
|
534
|
-
const pathMod = await getPath();
|
|
535
|
-
const resolvedBase = pathMod.resolve(root);
|
|
536
|
-
const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
|
|
537
|
-
const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
|
|
538
|
-
const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
|
|
539
|
-
return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
|
|
540
|
-
},
|
|
541
|
-
async pathOf(key) {
|
|
542
|
-
const { path: filePath } = await resolve(key);
|
|
543
|
-
return filePath;
|
|
544
|
-
}
|
|
545
|
-
};
|
|
193
|
+
return { segment: existing, isNew: false };
|
|
194
|
+
});
|
|
195
|
+
applyNewerTombstones(prepared);
|
|
196
|
+
return { partitionId, prepared };
|
|
197
|
+
});
|
|
198
|
+
swapPreparedPartitions(manager, partitions, new Set(action.changed));
|
|
546
199
|
}
|
|
547
200
|
|
|
548
201
|
// src/persistence/durability/sync/layout.ts
|
|
@@ -656,6 +309,7 @@ async function reclaimWalBeyondCount(directory, indexName, partitionCount, parti
|
|
|
656
309
|
await partition.walWriter.close();
|
|
657
310
|
} catch (err) {
|
|
658
311
|
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
312
|
+
throw err;
|
|
659
313
|
}
|
|
660
314
|
partitions.delete(partitionId);
|
|
661
315
|
}
|
|
@@ -693,170 +347,6 @@ function parseSegmentStartSeqNo(key, prefix) {
|
|
|
693
347
|
function formatSegmentStart(startSeqNo) {
|
|
694
348
|
return startSeqNo.toString().padStart(16, "0");
|
|
695
349
|
}
|
|
696
|
-
function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes, schedulesGraphBuild = true) {
|
|
697
|
-
if (entry.document === null) {
|
|
698
|
-
return;
|
|
699
|
-
}
|
|
700
|
-
const document = decode(entry.document);
|
|
701
|
-
restoreVectorFields(document, vectorFieldPaths);
|
|
702
|
-
const documentId = entry.documentId;
|
|
703
|
-
if (manager.has(documentId)) {
|
|
704
|
-
removeDocumentVectors(documentId, vecIndexes);
|
|
705
|
-
manager.remove(documentId);
|
|
706
|
-
}
|
|
707
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths);
|
|
708
|
-
manager.insert(documentId, partitionDoc);
|
|
709
|
-
try {
|
|
710
|
-
insertDocumentVectors(documentId, extractedVectors, vecIndexes, manager.partitionIdOf(documentId));
|
|
711
|
-
} catch (err) {
|
|
712
|
-
manager.remove(documentId);
|
|
713
|
-
throw err;
|
|
714
|
-
}
|
|
715
|
-
if (!schedulesGraphBuild) return;
|
|
716
|
-
for (const fieldPath of extractedVectors.keys()) {
|
|
717
|
-
const vecIndex = vecIndexes.get(fieldPath);
|
|
718
|
-
if (vecIndex) {
|
|
719
|
-
vecIndex.scheduleBuild();
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
|
|
724
|
-
removeDocumentVectors(entry.documentId, vecIndexes);
|
|
725
|
-
if (!manager.has(entry.documentId)) {
|
|
726
|
-
return;
|
|
727
|
-
}
|
|
728
|
-
manager.remove(entry.documentId, removedSpellings);
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
// src/persistence/durability/marker-writer.ts
|
|
732
|
-
function markerKey(indexName, partitionId) {
|
|
733
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
734
|
-
}
|
|
735
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
736
|
-
const key = markerKey(indexName, partitionId);
|
|
737
|
-
const handle = await directory.markerHandle(key);
|
|
738
|
-
const existing = await handle.read();
|
|
739
|
-
const current = readCommitMarker(existing);
|
|
740
|
-
const createdFresh = existing.length === 0;
|
|
741
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
742
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
743
|
-
return {
|
|
744
|
-
created: createdFresh,
|
|
745
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
746
|
-
async commit(state) {
|
|
747
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
748
|
-
writeSeq += 1;
|
|
749
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
750
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
751
|
-
lastSlot = slotIndex;
|
|
752
|
-
},
|
|
753
|
-
async close() {
|
|
754
|
-
await handle.close();
|
|
755
|
-
}
|
|
756
|
-
};
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
// src/persistence/durability/recovery.ts
|
|
760
|
-
async function removeLeftoverRewriteFiles(directory, names) {
|
|
761
|
-
for (const indexName of names) {
|
|
762
|
-
for (const place of ["rebuilt", "received"]) {
|
|
763
|
-
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
}
|
|
767
|
-
async function listPersistedIndexes(directory) {
|
|
768
|
-
const metaKeys = await directory.list("");
|
|
769
|
-
const names = [];
|
|
770
|
-
for (const key of metaKeys) {
|
|
771
|
-
if (key.endsWith("/meta")) {
|
|
772
|
-
names.push(key.slice(0, -"/meta".length));
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
return names;
|
|
776
|
-
}
|
|
777
|
-
async function loadMetadata(directory, indexName) {
|
|
778
|
-
const bytes = await directory.read(`${indexName}/meta`);
|
|
779
|
-
if (bytes === null) {
|
|
780
|
-
return null;
|
|
781
|
-
}
|
|
782
|
-
const { metadata } = await readMetadataEnvelope(bytes);
|
|
783
|
-
return metadata;
|
|
784
|
-
}
|
|
785
|
-
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
786
|
-
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
787
|
-
const read = await readWalSegments(directory, indexName, partitionId);
|
|
788
|
-
if (read === null) {
|
|
789
|
-
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
790
|
-
}
|
|
791
|
-
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
792
|
-
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
793
|
-
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
794
|
-
}
|
|
795
|
-
let highestSeqNo = fromSeqNoExclusive;
|
|
796
|
-
let highestPrimaryTerm = 0;
|
|
797
|
-
const replay = async (entries) => {
|
|
798
|
-
for (const entry of entries) {
|
|
799
|
-
if (entry.seqNo <= highestSeqNo) {
|
|
800
|
-
continue;
|
|
801
|
-
}
|
|
802
|
-
if (turnIsSpent()) await yieldTurn();
|
|
803
|
-
applyEntry(entry, deps);
|
|
804
|
-
highestSeqNo = entry.seqNo;
|
|
805
|
-
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
806
|
-
highestPrimaryTerm = entry.primaryTerm;
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
};
|
|
810
|
-
await replay(durableEntries);
|
|
811
|
-
if (activeTail !== null) {
|
|
812
|
-
await replay(activeTail.entries);
|
|
813
|
-
}
|
|
814
|
-
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
815
|
-
if (activeTail !== null) {
|
|
816
|
-
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
817
|
-
}
|
|
818
|
-
return { highestSeqNo, highestPrimaryTerm };
|
|
819
|
-
}
|
|
820
|
-
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
821
|
-
const lastReplayed = tail.entries.at(-1);
|
|
822
|
-
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
823
|
-
const handle = await directory.appendHandle(tail.key);
|
|
824
|
-
try {
|
|
825
|
-
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
826
|
-
else await handle.sync();
|
|
827
|
-
} finally {
|
|
828
|
-
await handle.close();
|
|
829
|
-
}
|
|
830
|
-
if (lastReplayed === void 0) return;
|
|
831
|
-
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
832
|
-
try {
|
|
833
|
-
await writer.commit({
|
|
834
|
-
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
835
|
-
durableByteLength: tail.cleanEnd,
|
|
836
|
-
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
837
|
-
});
|
|
838
|
-
} finally {
|
|
839
|
-
await writer.close();
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
843
|
-
for (const { key, startSeqNo } of segments) {
|
|
844
|
-
if (startSeqNo > activeSegmentSeqNo) {
|
|
845
|
-
await directory.remove(key);
|
|
846
|
-
}
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
|
-
function applyEntry(entry, deps) {
|
|
850
|
-
deps.writeRecords?.replayed(entry);
|
|
851
|
-
if (entry.operation === "DELETE") {
|
|
852
|
-
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
853
|
-
return;
|
|
854
|
-
}
|
|
855
|
-
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
856
|
-
}
|
|
857
|
-
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
858
|
-
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
859
|
-
}
|
|
860
350
|
|
|
861
351
|
// src/persistence/durability/checkpoint-capture.ts
|
|
862
352
|
function whileNoMutationApplies(partitions, capture2) {
|
|
@@ -865,9 +355,9 @@ function whileNoMutationApplies(partitions, capture2) {
|
|
|
865
355
|
const released = new Promise((settle) => {
|
|
866
356
|
release = settle;
|
|
867
357
|
});
|
|
868
|
-
function captureNow() {
|
|
358
|
+
async function captureNow() {
|
|
869
359
|
try {
|
|
870
|
-
resolve(capture2());
|
|
360
|
+
resolve(await capture2());
|
|
871
361
|
} catch (error) {
|
|
872
362
|
reject(error);
|
|
873
363
|
} finally {
|
|
@@ -876,13 +366,13 @@ function whileNoMutationApplies(partitions, capture2) {
|
|
|
876
366
|
}
|
|
877
367
|
let waiting = partitions.length;
|
|
878
368
|
if (waiting === 0) {
|
|
879
|
-
captureNow();
|
|
369
|
+
void captureNow();
|
|
880
370
|
return;
|
|
881
371
|
}
|
|
882
372
|
for (const partition of partitions) {
|
|
883
373
|
partition.appendChain = partition.appendChain.then(() => {
|
|
884
374
|
waiting -= 1;
|
|
885
|
-
if (waiting === 0) captureNow();
|
|
375
|
+
if (waiting === 0) void captureNow();
|
|
886
376
|
return released;
|
|
887
377
|
});
|
|
888
378
|
}
|
|
@@ -922,9 +412,11 @@ function noWholePartitions() {
|
|
|
922
412
|
return { contents: [], captured: [] };
|
|
923
413
|
}
|
|
924
414
|
function wholePartitionsWhereMostChanged(manager, rules) {
|
|
925
|
-
const { priorCheckpoint, priorPartitionIds,
|
|
415
|
+
const { priorCheckpoint, priorPartitionIds, aWorkerCanSerialise, captureRecords } = rules;
|
|
926
416
|
return (targets) => {
|
|
927
417
|
const whole = { contents: [], captured: [] };
|
|
418
|
+
const everyPartition = rules.everyPartition || requiresLayoutCheckpoint(manager) || priorPartitionIds.length > 0 && priorPartitionIds.length !== manager.partitionCount;
|
|
419
|
+
if (rules.usesLogForUnchangedLayout && !everyPartition) return whole;
|
|
928
420
|
const changed = targets.filter(
|
|
929
421
|
(target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
|
|
930
422
|
);
|
|
@@ -947,45 +439,57 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
947
439
|
return whole;
|
|
948
440
|
};
|
|
949
441
|
}
|
|
950
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured
|
|
442
|
+
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
951
443
|
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
952
|
-
return whileNoMutationApplies(
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
const
|
|
959
|
-
|
|
960
|
-
const
|
|
961
|
-
const
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
partitionId
|
|
968
|
-
lastSeqNo
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
977
|
-
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
978
|
-
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
979
|
-
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
|
|
444
|
+
return whileNoMutationApplies(
|
|
445
|
+
[...indexState.partitions.values()],
|
|
446
|
+
() => captureAppliedPrefix(manager, (cutoffs) => {
|
|
447
|
+
onCaptured?.();
|
|
448
|
+
const partitionCount = manager.partitionCount;
|
|
449
|
+
const previous = priorCheckpointThatLimitsTheRecords;
|
|
450
|
+
const changedLayout = requiresLayoutCheckpoint(manager) || previous !== void 0 && previous.length > 0 && previous.length !== partitionCount;
|
|
451
|
+
const bounded = changedLayout ? void 0 : previous;
|
|
452
|
+
const documentCount = manager.countDocuments();
|
|
453
|
+
const targets = [];
|
|
454
|
+
let recordsLeftInLog = 0;
|
|
455
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
456
|
+
const partition = indexState.partitions.get(partitionId);
|
|
457
|
+
if (partition !== void 0 && partition.failed !== null) continue;
|
|
458
|
+
const appliedSeqNo = Math.min(partition?.appliedSeqNo ?? 0, cutoffs.get(partitionId) ?? Infinity);
|
|
459
|
+
const lastSeqNo = bounded === void 0 ? appliedSeqNo : Math.min(appliedSeqNo, snapshotCheckpointFor(bounded, partitionId) + LOG_RECORDS_PER_CHECKPOINT);
|
|
460
|
+
recordsLeftInLog += appliedSeqNo - lastSeqNo;
|
|
461
|
+
const target = {
|
|
462
|
+
partitionId,
|
|
463
|
+
lastSeqNo,
|
|
464
|
+
primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
|
|
465
|
+
};
|
|
466
|
+
if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
|
|
467
|
+
targets.push(target);
|
|
980
468
|
}
|
|
981
|
-
const
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
469
|
+
const vectorPlans = /* @__PURE__ */ new Map();
|
|
470
|
+
try {
|
|
471
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
472
|
+
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
473
|
+
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
474
|
+
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, true));
|
|
475
|
+
}
|
|
476
|
+
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
477
|
+
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
478
|
+
return {
|
|
479
|
+
partitionCount,
|
|
480
|
+
targets,
|
|
481
|
+
documentCount,
|
|
482
|
+
vectorPlans,
|
|
483
|
+
wholePartitions,
|
|
484
|
+
recordsLeftInLog,
|
|
485
|
+
unreplayedCutoffs: cutoffs
|
|
486
|
+
};
|
|
487
|
+
} catch (error) {
|
|
488
|
+
for (const plan of vectorPlans.values()) plan.release();
|
|
489
|
+
throw error;
|
|
490
|
+
}
|
|
491
|
+
})
|
|
492
|
+
);
|
|
989
493
|
}
|
|
990
494
|
async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
|
|
991
495
|
if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
|
|
@@ -1005,6 +509,20 @@ async function writeCapturedVectors(store, indexName, capture2, priorManifest, r
|
|
|
1005
509
|
return { vectors: written.refs, layouts: written.layouts, written: written.written };
|
|
1006
510
|
}
|
|
1007
511
|
|
|
512
|
+
// src/persistence/durability/checkpoint-prior.ts
|
|
513
|
+
function priorForLayout(manifest, partitionCount) {
|
|
514
|
+
const checkpoints = new Map(manifest?.checkpoint.map((entry) => [entry.partitionId, entry]));
|
|
515
|
+
const partitions = new Map(manifest?.partitions.map((entry) => [entry.partitionId, entry]));
|
|
516
|
+
const prior = { checkpoint: [], partitions: [] };
|
|
517
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
518
|
+
prior.checkpoint.push(
|
|
519
|
+
checkpoints.get(partitionId) ?? { partitionId, lastSeqNo: 0, primaryTerm: SINGLE_NODE_PRIMARY_TERM }
|
|
520
|
+
);
|
|
521
|
+
prior.partitions.push(partitions.get(partitionId) ?? { partitionId, nextSegmentId: 0, segments: [] });
|
|
522
|
+
}
|
|
523
|
+
return prior;
|
|
524
|
+
}
|
|
525
|
+
|
|
1008
526
|
// src/persistence/durability/checkpoint-worker-entry.ts
|
|
1009
527
|
function resolveCheckpointWorkerEntry() {
|
|
1010
528
|
return resolveWorkerEntry(
|
|
@@ -1325,6 +843,7 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1325
843
|
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1326
844
|
everyPartition: fromMemory,
|
|
1327
845
|
aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
|
|
846
|
+
usesLogForUnchangedLayout: textOnDisk,
|
|
1328
847
|
...writeRecords === null ? {} : {
|
|
1329
848
|
captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
1330
849
|
}
|
|
@@ -1334,20 +853,28 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1334
853
|
input.indexState,
|
|
1335
854
|
manager,
|
|
1336
855
|
vectorIndexes,
|
|
1337
|
-
|
|
856
|
+
wholePartitionsOf,
|
|
1338
857
|
fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1339
858
|
priorManifest?.vectors ?? [],
|
|
1340
859
|
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
|
|
1341
860
|
textOnDisk ? () => {
|
|
1342
861
|
changesSinceCapture = manager.openCheckpointChangeLog();
|
|
1343
|
-
} : void 0
|
|
1344
|
-
input.pinsRetiredVectors
|
|
862
|
+
} : void 0
|
|
1345
863
|
);
|
|
1346
864
|
try {
|
|
1347
865
|
await openNewLogSegments(input.indexState, input.failPartition);
|
|
1348
|
-
return await finishBoundedCheckpoint(
|
|
1349
|
-
|
|
1350
|
-
|
|
866
|
+
return await finishBoundedCheckpoint(
|
|
867
|
+
input,
|
|
868
|
+
manager,
|
|
869
|
+
store,
|
|
870
|
+
{ ...metadata, partitionCount: capture2.partitionCount },
|
|
871
|
+
priorManifest,
|
|
872
|
+
capture2,
|
|
873
|
+
fromMemory,
|
|
874
|
+
() => {
|
|
875
|
+
return changesSinceCapture;
|
|
876
|
+
}
|
|
877
|
+
);
|
|
1351
878
|
} finally {
|
|
1352
879
|
for (const plan of capture2.vectorPlans.values()) plan.release();
|
|
1353
880
|
changesSinceCapture?.close();
|
|
@@ -1355,24 +882,26 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1355
882
|
}
|
|
1356
883
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1357
884
|
const { indexName, indexState, hooks } = input;
|
|
1358
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-NDSQLPMQ.mjs');
|
|
1359
885
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1360
886
|
if (adoption === null) return [];
|
|
1361
|
-
const adopted = await whileNoMutationApplies(
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
887
|
+
const adopted = await whileNoMutationApplies(
|
|
888
|
+
[...indexState.partitions.values()],
|
|
889
|
+
() => manager.publication.write(() => {
|
|
890
|
+
if (changes.partitionsReplaced) return false;
|
|
891
|
+
const changed = changes.takeWrittenDocIds();
|
|
892
|
+
swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
|
|
893
|
+
hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
|
|
894
|
+
});
|
|
895
|
+
hooks.queueDiskAdoption?.({
|
|
896
|
+
type: "adoptDiskSegments",
|
|
897
|
+
indexName,
|
|
898
|
+
partitions: actionPartitionsOf(adoption),
|
|
899
|
+
changed: [...changed],
|
|
900
|
+
requestId: createRequestId()
|
|
901
|
+
});
|
|
902
|
+
return true;
|
|
903
|
+
})
|
|
904
|
+
);
|
|
1376
905
|
if (!adopted) {
|
|
1377
906
|
closeUnusedSegments(adoption);
|
|
1378
907
|
return [];
|
|
@@ -1394,7 +923,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1394
923
|
targets,
|
|
1395
924
|
compactionThreshold: input.compactionThreshold,
|
|
1396
925
|
canOffload: input.canOffload,
|
|
1397
|
-
prior: priorManifest,
|
|
926
|
+
prior: priorForLayout(priorManifest, capture2.partitionCount),
|
|
1398
927
|
wholePartitions: capture2.wholePartitions.contents,
|
|
1399
928
|
capturedPartitions: capture2.wholePartitions.captured,
|
|
1400
929
|
keepsWriteRecords: writeRecords !== null
|
|
@@ -1410,7 +939,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1410
939
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1411
940
|
if (!input.holdsCheckpointFiles()) {
|
|
1412
941
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1413
|
-
const stillRead = metadata.textStorage === "disk" ?
|
|
942
|
+
const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1414
943
|
const keptForSync = await input.keepsCheckpointKey();
|
|
1415
944
|
await removeCheckpointGarbage(
|
|
1416
945
|
store,
|
|
@@ -1423,9 +952,12 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1423
952
|
if (checkpointMetadata === void 0) {
|
|
1424
953
|
return;
|
|
1425
954
|
}
|
|
1426
|
-
const bytes = await writeMetadataEnvelope(
|
|
955
|
+
const bytes = await writeMetadataEnvelope(
|
|
956
|
+
{ ...checkpointMetadata, partitionCount: capture2.partitionCount },
|
|
957
|
+
{ checksum: true }
|
|
958
|
+
);
|
|
1427
959
|
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
1428
|
-
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount,
|
|
960
|
+
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture2.partitionCount);
|
|
1429
961
|
});
|
|
1430
962
|
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
1431
963
|
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
@@ -1433,69 +965,36 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1433
965
|
await reclaimWalBeyondCount(
|
|
1434
966
|
input.directory,
|
|
1435
967
|
input.indexName,
|
|
1436
|
-
|
|
968
|
+
capture2.partitionCount,
|
|
1437
969
|
input.indexState.partitions,
|
|
1438
970
|
input.markFatal
|
|
1439
971
|
);
|
|
972
|
+
await input.directory.remove(transitionKey(input.indexName));
|
|
1440
973
|
const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
|
|
1441
974
|
noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
|
|
1442
|
-
const
|
|
1443
|
-
|
|
975
|
+
const eligibleBeyond = recordsBeyond(input.indexState, targets, capture2.unreplayedCutoffs);
|
|
976
|
+
const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
|
|
977
|
+
return anotherIsDue ? eligibleBeyond : capture2.recordsLeftInLog;
|
|
1444
978
|
}
|
|
1445
|
-
function recordsBeyond(indexState, targets) {
|
|
979
|
+
function recordsBeyond(indexState, targets, cutoffs) {
|
|
1446
980
|
let records = 0;
|
|
1447
981
|
for (const [partitionId, partition] of indexState.partitions) {
|
|
1448
982
|
if (partition.failed !== null) continue;
|
|
1449
|
-
records += Math.max(
|
|
983
|
+
records += Math.max(
|
|
984
|
+
0,
|
|
985
|
+
Math.min(partition.appliedSeqNo, cutoffs?.get(partitionId) ?? Infinity) - snapshotCheckpointFor(targets, partitionId)
|
|
986
|
+
);
|
|
1450
987
|
}
|
|
1451
988
|
return records;
|
|
1452
989
|
}
|
|
1453
990
|
|
|
1454
|
-
// src/persistence/durability/
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
|
|
1459
|
-
vectorIndex.scheduleBuild();
|
|
1460
|
-
vectorIndex.startShuffledRebuildWhenDue();
|
|
1461
|
-
}
|
|
1462
|
-
};
|
|
1463
|
-
return {
|
|
1464
|
-
track(indexName, load, startsGraphRepairs = true) {
|
|
1465
|
-
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
1466
|
-
loads.set(
|
|
1467
|
-
indexName,
|
|
1468
|
-
loaded.catch(() => void 0)
|
|
1469
|
-
);
|
|
1470
|
-
return loaded;
|
|
1471
|
-
},
|
|
1472
|
-
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
1473
|
-
startGraphRepairs
|
|
1474
|
-
};
|
|
991
|
+
// src/persistence/durability/deferred-mutations.ts
|
|
992
|
+
var deferred = /* @__PURE__ */ new WeakSet();
|
|
993
|
+
function deferMutationPublication(record) {
|
|
994
|
+
deferred.add(record);
|
|
1475
995
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
await checkpointInTurn(turns, {
|
|
1479
|
-
joinsInFlight: false,
|
|
1480
|
-
mayStart: turn.mayStart,
|
|
1481
|
-
perform: async () => {
|
|
1482
|
-
swapped = turn.swap();
|
|
1483
|
-
if (swapped) await turn.checkpoint();
|
|
1484
|
-
},
|
|
1485
|
-
afterSettling: turn.afterSettling
|
|
1486
|
-
});
|
|
1487
|
-
return swapped;
|
|
1488
|
-
}
|
|
1489
|
-
async function checkpointInTurn(turns, turn) {
|
|
1490
|
-
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
1491
|
-
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
1492
|
-
if (!turn.mayStart()) return;
|
|
1493
|
-
const run = turn.perform().finally(() => {
|
|
1494
|
-
turns.checkpointInFlight = null;
|
|
1495
|
-
turn.afterSettling();
|
|
1496
|
-
});
|
|
1497
|
-
turns.checkpointInFlight = run;
|
|
1498
|
-
return run;
|
|
996
|
+
function hasDeferredPublication(record) {
|
|
997
|
+
return deferred.has(record);
|
|
1499
998
|
}
|
|
1500
999
|
|
|
1501
1000
|
// src/persistence/durability/held-checkpoints.ts
|
|
@@ -1553,24 +1052,6 @@ function createHeldCheckpoints(hooks) {
|
|
|
1553
1052
|
};
|
|
1554
1053
|
}
|
|
1555
1054
|
|
|
1556
|
-
// src/persistence/durability/manager-state.ts
|
|
1557
|
-
function closePartitionWriter(partition) {
|
|
1558
|
-
return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
|
|
1559
|
-
}
|
|
1560
|
-
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
1561
|
-
indexState.unloading = true;
|
|
1562
|
-
try {
|
|
1563
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
1564
|
-
await pendingMetadataWrite;
|
|
1565
|
-
for (const partition of indexState.partitions.values()) {
|
|
1566
|
-
await closeWalWriter(closePartitionWriter(partition));
|
|
1567
|
-
}
|
|
1568
|
-
} catch (error) {
|
|
1569
|
-
indexState.unloading = false;
|
|
1570
|
-
throw error;
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
|
|
1574
1055
|
// src/persistence/durability/restore-journal.ts
|
|
1575
1056
|
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
1576
1057
|
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
@@ -2211,6 +1692,8 @@ function createFilePins(remove) {
|
|
|
2211
1692
|
}
|
|
2212
1693
|
};
|
|
2213
1694
|
}
|
|
1695
|
+
|
|
1696
|
+
// src/persistence/durability/sync/published.ts
|
|
2214
1697
|
var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
|
|
2215
1698
|
var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
|
|
2216
1699
|
function hexOf(bytes) {
|
|
@@ -2879,13 +2362,13 @@ function createWalWriter(directory, config) {
|
|
|
2879
2362
|
for (const frame of frames) {
|
|
2880
2363
|
totalBytes += frame.length;
|
|
2881
2364
|
}
|
|
2882
|
-
const
|
|
2365
|
+
const joined2 = new Uint8Array(totalBytes);
|
|
2883
2366
|
let offset = 0;
|
|
2884
2367
|
for (const frame of frames) {
|
|
2885
|
-
|
|
2368
|
+
joined2.set(frame, offset);
|
|
2886
2369
|
offset += frame.length;
|
|
2887
2370
|
}
|
|
2888
|
-
return
|
|
2371
|
+
return joined2;
|
|
2889
2372
|
}
|
|
2890
2373
|
function framesThatFit(frames, start) {
|
|
2891
2374
|
const room = segmentMaxBytes - activeBytes;
|
|
@@ -2906,16 +2389,16 @@ function createWalWriter(directory, config) {
|
|
|
2906
2389
|
{ indexName: config.indexName, partitionId: config.partitionId }
|
|
2907
2390
|
);
|
|
2908
2391
|
}
|
|
2909
|
-
const
|
|
2910
|
-
if (activeBytes +
|
|
2392
|
+
const joined2 = joinFrames([...frames]);
|
|
2393
|
+
if (activeBytes + joined2.length > MAX_FILE_BYTES) {
|
|
2911
2394
|
throw new NarsilError(
|
|
2912
2395
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2913
|
-
`A log record of ${
|
|
2914
|
-
{ indexName: config.indexName, partitionId: config.partitionId, bytes:
|
|
2396
|
+
`A log record of ${joined2.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
|
|
2397
|
+
{ indexName: config.indexName, partitionId: config.partitionId, bytes: joined2.length }
|
|
2915
2398
|
);
|
|
2916
2399
|
}
|
|
2917
|
-
await activeHandle.append(
|
|
2918
|
-
activeBytes +=
|
|
2400
|
+
await activeHandle.append(joined2);
|
|
2401
|
+
activeBytes += joined2.length;
|
|
2919
2402
|
const lastSeqNo = entries[entries.length - 1].seqNo;
|
|
2920
2403
|
if (lastSeqNo > highestAppendedSeqNo) {
|
|
2921
2404
|
highestAppendedSeqNo = lastSeqNo;
|
|
@@ -3036,6 +2519,109 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
|
|
|
3036
2519
|
}
|
|
3037
2520
|
};
|
|
3038
2521
|
}
|
|
2522
|
+
|
|
2523
|
+
// src/serialization/payload-v2.ts
|
|
2524
|
+
function validateHnswMetric(value) {
|
|
2525
|
+
if (value === void 0 || value === null) return void 0;
|
|
2526
|
+
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
2527
|
+
return value;
|
|
2528
|
+
}
|
|
2529
|
+
return void 0;
|
|
2530
|
+
}
|
|
2531
|
+
function storedDocumentsOf(raw) {
|
|
2532
|
+
requireValidFieldTypes(raw.schema ?? {});
|
|
2533
|
+
const documents = {};
|
|
2534
|
+
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
2535
|
+
documents[docId] = {
|
|
2536
|
+
fields: doc.fields,
|
|
2537
|
+
fieldLengths: doc.field_lengths ?? {}
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
2540
|
+
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
2541
|
+
return documents;
|
|
2542
|
+
}
|
|
2543
|
+
function documentsOfPayloadV2(data) {
|
|
2544
|
+
const raw = decode(data);
|
|
2545
|
+
return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
|
|
2546
|
+
}
|
|
2547
|
+
function deserializePayloadV2(data) {
|
|
2548
|
+
const raw = decode(data);
|
|
2549
|
+
const documents = storedDocumentsOf(raw);
|
|
2550
|
+
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
2551
|
+
const invertedIndex = {};
|
|
2552
|
+
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
2553
|
+
const count = list.ids.length;
|
|
2554
|
+
const postings = new Array(count);
|
|
2555
|
+
for (let i = 0; i < count; i++) {
|
|
2556
|
+
postings[i] = {
|
|
2557
|
+
docId: list.ids[i],
|
|
2558
|
+
termFrequency: list.tf[i],
|
|
2559
|
+
field: fieldNames[list.fi[i]] ?? "",
|
|
2560
|
+
positions: list.pos ? list.pos[i] : []
|
|
2561
|
+
};
|
|
2562
|
+
}
|
|
2563
|
+
invertedIndex[token] = {
|
|
2564
|
+
docFrequency: list.df,
|
|
2565
|
+
postings
|
|
2566
|
+
};
|
|
2567
|
+
}
|
|
2568
|
+
const numeric = {};
|
|
2569
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
2570
|
+
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
2571
|
+
}
|
|
2572
|
+
const boolean = {};
|
|
2573
|
+
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
2574
|
+
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
2575
|
+
}
|
|
2576
|
+
const geopoint = {};
|
|
2577
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
2578
|
+
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
2579
|
+
}
|
|
2580
|
+
const vectorData = {};
|
|
2581
|
+
for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
|
|
2582
|
+
vectorData[field] = {
|
|
2583
|
+
dimension: data2.dimension,
|
|
2584
|
+
vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
2585
|
+
hnswGraph: data2.hnsw_graph ? {
|
|
2586
|
+
entryPoint: data2.hnsw_graph.entry_point,
|
|
2587
|
+
maxLayer: data2.hnsw_graph.max_layer,
|
|
2588
|
+
m: data2.hnsw_graph.m,
|
|
2589
|
+
efConstruction: data2.hnsw_graph.ef_construction,
|
|
2590
|
+
metric: validateHnswMetric(data2.hnsw_graph.metric),
|
|
2591
|
+
nodes: data2.hnsw_graph.nodes
|
|
2592
|
+
} : null,
|
|
2593
|
+
codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
|
|
2594
|
+
};
|
|
2595
|
+
}
|
|
2596
|
+
return {
|
|
2597
|
+
indexName: raw.index_name,
|
|
2598
|
+
partitionId: raw.partition_id,
|
|
2599
|
+
totalPartitions: raw.total_partitions,
|
|
2600
|
+
language: raw.language ?? "english",
|
|
2601
|
+
schema: raw.schema ?? {},
|
|
2602
|
+
docCount: raw.doc_count ?? 0,
|
|
2603
|
+
avgDocLength: raw.avg_doc_length ?? 0,
|
|
2604
|
+
documents,
|
|
2605
|
+
invertedIndex,
|
|
2606
|
+
fieldIndexes: {
|
|
2607
|
+
numeric,
|
|
2608
|
+
boolean,
|
|
2609
|
+
enum: raw.field_indexes?.enum ?? {},
|
|
2610
|
+
geopoint,
|
|
2611
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
2612
|
+
},
|
|
2613
|
+
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
2614
|
+
vectorData,
|
|
2615
|
+
statistics: {
|
|
2616
|
+
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
2617
|
+
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
2618
|
+
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
2619
|
+
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
2620
|
+
}
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
|
|
2624
|
+
// src/persistence/durability/earlier-layout/bundle.ts
|
|
3039
2625
|
var SNAPSHOT_BUNDLE_VERSION = 2;
|
|
3040
2626
|
function invalid(message) {
|
|
3041
2627
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Snapshot bundle ${message}`);
|
|
@@ -3095,7 +2681,7 @@ async function loadSnapshotBundle(bytes, deps) {
|
|
|
3095
2681
|
while (deps.manager.partitionCount < bundle.partitions.length) deps.manager.addPartition();
|
|
3096
2682
|
if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
|
|
3097
2683
|
for (let i = 0; i < bundle.partitions.length; i += 1) {
|
|
3098
|
-
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
|
|
2684
|
+
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]), deserializePartition);
|
|
3099
2685
|
}
|
|
3100
2686
|
for (const [fieldPath, parts] of Object.entries(bundle.vectorIndexes)) {
|
|
3101
2687
|
const rebuildsGraph = parts.some((part) => part.graphs.some((graph) => graph.m < MIN_M));
|
|
@@ -3403,6 +2989,8 @@ function validateMergedPartition(merged) {
|
|
|
3403
2989
|
}
|
|
3404
2990
|
}
|
|
3405
2991
|
}
|
|
2992
|
+
|
|
2993
|
+
// src/persistence/durability/earlier-layout/segment.ts
|
|
3406
2994
|
function normalizeTombstones(raw) {
|
|
3407
2995
|
if (raw === void 0) return [];
|
|
3408
2996
|
if (!Array.isArray(raw) || !raw.every((value) => typeof value === "string")) {
|
|
@@ -3441,7 +3029,6 @@ var decoder = new TextDecoder();
|
|
|
3441
3029
|
async function countPagedPartition(store, partition) {
|
|
3442
3030
|
const { directory } = store;
|
|
3443
3031
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
3444
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-NDSQLPMQ.mjs');
|
|
3445
3032
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
3446
3033
|
const scans = [];
|
|
3447
3034
|
try {
|
|
@@ -3493,6 +3080,8 @@ async function countCheckpointDocuments(store, indexName) {
|
|
|
3493
3080
|
const bundle = await store.read(snapshotBundleKey(indexName));
|
|
3494
3081
|
return bundle === null ? 0 : countSnapshotBundleDocuments(await decodeSnapshotBundle(bundle));
|
|
3495
3082
|
}
|
|
3083
|
+
|
|
3084
|
+
// src/persistence/durability/earlier-layout/load.ts
|
|
3496
3085
|
function removeWhatTheSegmentReplaces(manager, segment) {
|
|
3497
3086
|
manager.beginBatchRemove();
|
|
3498
3087
|
try {
|
|
@@ -3508,20 +3097,21 @@ function removeWhatTheSegmentReplaces(manager, segment) {
|
|
|
3508
3097
|
}
|
|
3509
3098
|
function attachDocuments(manager, partitionId, segment) {
|
|
3510
3099
|
if (segment.documents.length === 0) return;
|
|
3511
|
-
const
|
|
3100
|
+
const outputs = buildSegmentPayloads(
|
|
3512
3101
|
segment.documents,
|
|
3513
3102
|
manager.config.schema,
|
|
3514
3103
|
manager.language,
|
|
3515
3104
|
resolvePartitionInsertOptions(manager.config, manager.analysis),
|
|
3516
3105
|
manager.config.trackPositions ?? true
|
|
3517
3106
|
);
|
|
3518
|
-
const
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3107
|
+
const prepared = outputs.map(({ payload, documents: members }) => {
|
|
3108
|
+
const documents = members.map((entry) => entry.document);
|
|
3109
|
+
return { payload, documents, snapshot: freezeSegmentShared(payload, documents) };
|
|
3110
|
+
});
|
|
3111
|
+
for (const { payload, documents, snapshot } of prepared) {
|
|
3112
|
+
if (snapshot === null) manager.mergeSegment(partitionId, payload, documents);
|
|
3113
|
+
else manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
|
|
3523
3114
|
}
|
|
3524
|
-
manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
|
|
3525
3115
|
}
|
|
3526
3116
|
async function loadPartition(store, indexName, partition, deps) {
|
|
3527
3117
|
if (deps.storedTermsAreCurrent) {
|
|
@@ -3542,7 +3132,7 @@ async function loadPartition(store, indexName, partition, deps) {
|
|
|
3542
3132
|
totalPartitions: deps.manager.partitionCount,
|
|
3543
3133
|
language: deps.manager.language.name
|
|
3544
3134
|
});
|
|
3545
|
-
deps.manager.deserializePartition(partition.partitionId, merged);
|
|
3135
|
+
deps.manager.deserializePartition(partition.partitionId, merged, deserializePartition);
|
|
3546
3136
|
}
|
|
3547
3137
|
async function readGraph(store, ref) {
|
|
3548
3138
|
if (ref.graphKey === null) return null;
|
|
@@ -3741,13 +3331,68 @@ async function recoverCheckpoint(deps) {
|
|
|
3741
3331
|
return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
|
|
3742
3332
|
}
|
|
3743
3333
|
|
|
3744
|
-
// src/persistence/durability/
|
|
3745
|
-
function
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3334
|
+
// src/persistence/durability/layout-transition/recovery.ts
|
|
3335
|
+
async function settleStoredLayout(store, metadata) {
|
|
3336
|
+
const transition = await readTransition(store, metadata.indexName);
|
|
3337
|
+
const decoded = await readSegmentManifest(store, metadata.indexName);
|
|
3338
|
+
const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
|
|
3339
|
+
if (transition !== null && partitionCount !== transition.oldPartitionCount && partitionCount !== transition.newPartitionCount) {
|
|
3340
|
+
invalidTransition("matches neither the manifest nor the metadata");
|
|
3341
|
+
}
|
|
3342
|
+
if (transition !== null && decoded === null && partitionCount !== transition.oldPartitionCount) {
|
|
3343
|
+
invalidTransition("has no old checkpoint or matching metadata");
|
|
3344
|
+
}
|
|
3345
|
+
const changedMetadata = partitionCount !== metadata.partitionCount;
|
|
3346
|
+
metadata.partitionCount = partitionCount;
|
|
3347
|
+
return {
|
|
3348
|
+
transition,
|
|
3349
|
+
changedMetadata,
|
|
3350
|
+
replaysOldLayout: transition !== null && partitionCount === transition.oldPartitionCount
|
|
3351
|
+
};
|
|
3352
|
+
}
|
|
3353
|
+
async function checkRetiredHead(directory, indexName, head) {
|
|
3354
|
+
const wal = await readWalSegments(directory, indexName, head.partitionId);
|
|
3355
|
+
if (wal === null) return;
|
|
3356
|
+
if (wal.highestReadFromWal > head.lastSeqNo || wal.marker.state.highestDurableSeqNo > head.lastSeqNo || wal.activeTail?.entries.some((entry) => entry.seqNo > head.lastSeqNo)) {
|
|
3357
|
+
invalidTransition("has records above a retired partition head");
|
|
3358
|
+
}
|
|
3359
|
+
}
|
|
3360
|
+
async function replayAcrossLayout(directory, indexName, transition, checkpoint, deps) {
|
|
3361
|
+
const terms = [];
|
|
3362
|
+
for (const head of transition.oldHeads) {
|
|
3363
|
+
const from = checkpointLastSeqNo(checkpoint, head.partitionId);
|
|
3364
|
+
if (from > head.lastSeqNo) invalidTransition("has an old head below its checkpoint");
|
|
3365
|
+
const replayed2 = await replayWalUpTo(directory, indexName, head.partitionId, from, head.lastSeqNo, deps);
|
|
3366
|
+
if (replayed2.highestSeqNo < head.lastSeqNo) invalidTransition("is missing an admitted old record");
|
|
3367
|
+
terms[head.partitionId] = Math.max(
|
|
3368
|
+
head.primaryTerm,
|
|
3369
|
+
replayed2.highestPrimaryTerm,
|
|
3370
|
+
checkpointPrimaryTerm(checkpoint, head.partitionId)
|
|
3371
|
+
);
|
|
3372
|
+
if (head.partitionId >= transition.newPartitionCount) await checkRetiredHead(directory, indexName, head);
|
|
3373
|
+
}
|
|
3374
|
+
const spill = await rewriteSpillOf({ directory, writeRecords: deps.writeRecords ?? null }, deps.manager);
|
|
3375
|
+
await createRebalancer().rebalance(
|
|
3376
|
+
deps.manager,
|
|
3377
|
+
transition.newPartitionCount,
|
|
3378
|
+
createPartitionRouter(),
|
|
3379
|
+
void 0,
|
|
3380
|
+
spill
|
|
3381
|
+
);
|
|
3382
|
+
const replayed = [];
|
|
3383
|
+
for (let partitionId = 0; partitionId < transition.newPartitionCount; partitionId++) {
|
|
3384
|
+
const head = transition.oldHeads[partitionId];
|
|
3385
|
+
const next = await replayWal(directory, indexName, partitionId, head?.lastSeqNo ?? 0, deps);
|
|
3386
|
+
replayed.push({
|
|
3387
|
+
partitionId,
|
|
3388
|
+
lastSeqNo: next.highestSeqNo,
|
|
3389
|
+
primaryTerm: Math.max(terms[partitionId] ?? SINGLE_NODE_PRIMARY_TERM, next.highestPrimaryTerm)
|
|
3390
|
+
});
|
|
3391
|
+
}
|
|
3392
|
+
return { checkpoint: replayed, removeUnread: async () => spill?.removeUnread(deps.manager) };
|
|
3750
3393
|
}
|
|
3394
|
+
|
|
3395
|
+
// src/persistence/durability/stored-analysis.ts
|
|
3751
3396
|
function promotePendingAnalysis(metadata) {
|
|
3752
3397
|
const pending = metadata.pendingAnalysis;
|
|
3753
3398
|
if (pending === void 0) return;
|
|
@@ -3761,9 +3406,13 @@ function promotePendingAnalysis(metadata) {
|
|
|
3761
3406
|
}
|
|
3762
3407
|
async function settleStoredAnalysis(store, metadata, metadataOnly) {
|
|
3763
3408
|
if (metadataOnly && metadata.pendingAnalysis === void 0) return false;
|
|
3764
|
-
const
|
|
3409
|
+
const decoded = await readSegmentManifest(store, metadata.indexName);
|
|
3410
|
+
const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
|
|
3411
|
+
const changedLayout = partitionCount !== metadata.partitionCount;
|
|
3412
|
+
metadata.partitionCount = partitionCount;
|
|
3413
|
+
const stored = decoded === null ? null : decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
|
|
3765
3414
|
const current = metadata.analysisGeneration ?? 0;
|
|
3766
|
-
if (stored === null || stored === current) return
|
|
3415
|
+
if (stored === null || stored === current) return changedLayout;
|
|
3767
3416
|
if (stored === metadata.pendingAnalysis?.generation) {
|
|
3768
3417
|
promotePendingAnalysis(metadata);
|
|
3769
3418
|
return true;
|
|
@@ -3787,15 +3436,16 @@ async function writeCountedMetadata(deps, indexName, documentCount) {
|
|
|
3787
3436
|
if (metadata === void 0) return;
|
|
3788
3437
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
3789
3438
|
await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
3790
|
-
})
|
|
3439
|
+
});
|
|
3791
3440
|
}
|
|
3792
3441
|
async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
3793
3442
|
const { directory, hooks } = deps;
|
|
3794
3443
|
const store = directoryCheckpointStore(directory);
|
|
3795
3444
|
const metadata = await loadMetadata(directory, indexName);
|
|
3796
3445
|
if (metadata === null) {
|
|
3797
|
-
return;
|
|
3446
|
+
return null;
|
|
3798
3447
|
}
|
|
3448
|
+
const layout = metadataOnly ? { transition: null, changedMetadata: false, replaysOldLayout: false } : await settleStoredLayout(store, metadata);
|
|
3799
3449
|
let derivedCount;
|
|
3800
3450
|
if (metadataOnly && metadata.documentCount === void 0) {
|
|
3801
3451
|
try {
|
|
@@ -3808,15 +3458,15 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3808
3458
|
const promoted = await settleStoredAnalysis(store, metadata, metadataOnly);
|
|
3809
3459
|
await hooks.createIndexFromMetadata(metadata, !metadataOnly);
|
|
3810
3460
|
if (metadataOnly) {
|
|
3811
|
-
if (derivedCount !== void 0 || promoted) {
|
|
3461
|
+
if (derivedCount !== void 0 || promoted || layout.changedMetadata) {
|
|
3812
3462
|
await writeCountedMetadata(deps, indexName, derivedCount ?? metadata.documentCount ?? 0);
|
|
3813
3463
|
}
|
|
3814
|
-
return;
|
|
3464
|
+
return null;
|
|
3815
3465
|
}
|
|
3816
3466
|
const manager = hooks.getManager(indexName);
|
|
3817
3467
|
const current = hooks.buildMetadata(indexName);
|
|
3818
3468
|
if (manager === void 0 || current === void 0) {
|
|
3819
|
-
return;
|
|
3469
|
+
return null;
|
|
3820
3470
|
}
|
|
3821
3471
|
const writeRecords = hooks.writeRecords ?? null;
|
|
3822
3472
|
writeRecords?.dropIndex(indexName);
|
|
@@ -3827,7 +3477,13 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3827
3477
|
writeRecords,
|
|
3828
3478
|
defersGraphBuilds: true
|
|
3829
3479
|
};
|
|
3480
|
+
let removeUnread = async () => void 0;
|
|
3830
3481
|
const replay = async (checkpoint) => {
|
|
3482
|
+
if (layout.replaysOldLayout && layout.transition !== null) {
|
|
3483
|
+
const replayed2 = await replayAcrossLayout(directory, indexName, layout.transition, checkpoint, replayDeps);
|
|
3484
|
+
removeUnread = replayed2.removeUnread;
|
|
3485
|
+
return replayed2.checkpoint;
|
|
3486
|
+
}
|
|
3831
3487
|
const replayed = [];
|
|
3832
3488
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
3833
3489
|
const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
|
|
@@ -3871,9 +3527,19 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3871
3527
|
for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
|
|
3872
3528
|
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
3873
3529
|
}
|
|
3874
|
-
if (recovered.upgradedDocumentCount !== null || promoted) {
|
|
3530
|
+
if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
|
|
3875
3531
|
await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
|
|
3876
3532
|
}
|
|
3533
|
+
return layout.transition === null ? null : removeUnread;
|
|
3534
|
+
}
|
|
3535
|
+
async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkpoint) {
|
|
3536
|
+
const removeUnread = await recoverPersistedIndex(deps, indexName, metadataOnly);
|
|
3537
|
+
if (removeUnread === null) return;
|
|
3538
|
+
try {
|
|
3539
|
+
await checkpoint();
|
|
3540
|
+
} finally {
|
|
3541
|
+
await removeUnread();
|
|
3542
|
+
}
|
|
3877
3543
|
}
|
|
3878
3544
|
|
|
3879
3545
|
// src/persistence/durability/manager.ts
|
|
@@ -3950,21 +3616,25 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3950
3616
|
}
|
|
3951
3617
|
}
|
|
3952
3618
|
}
|
|
3953
|
-
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false) {
|
|
3619
|
+
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false, requested = false) {
|
|
3954
3620
|
const indexState = indexes.get(indexName);
|
|
3955
3621
|
if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
|
|
3956
|
-
return
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3622
|
+
return holdScheduledCheckpoint(
|
|
3623
|
+
checkpointInTurn(indexState, {
|
|
3624
|
+
joinsInFlight: !afterInFlight && !pinsRetiredVectors,
|
|
3625
|
+
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
3626
|
+
perform: () => performCheckpoint(indexName, indexState, fromMemory),
|
|
3627
|
+
afterSettling: () => releaseStalledWrites(indexState)
|
|
3628
|
+
}),
|
|
3629
|
+
requested
|
|
3630
|
+
);
|
|
3962
3631
|
}
|
|
3963
3632
|
async function requestedCheckpoint(indexName, fromMemory = false, pinsVectors = false) {
|
|
3964
3633
|
heldCheckpoints.refuse(indexName, indexes.get(indexName));
|
|
3965
|
-
|
|
3634
|
+
await requireCheckpointLayout(hooks.getManager(indexName));
|
|
3635
|
+
return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors, true);
|
|
3966
3636
|
}
|
|
3967
|
-
async function performCheckpoint(indexName, indexState, fromMemory = false
|
|
3637
|
+
async function performCheckpoint(indexName, indexState, fromMemory = false) {
|
|
3968
3638
|
await directoryWatch.verify();
|
|
3969
3639
|
if (fatalError !== null) throw fatalError;
|
|
3970
3640
|
await runDurableCheckpoint({
|
|
@@ -3975,7 +3645,6 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3975
3645
|
compactionThreshold,
|
|
3976
3646
|
canOffload: canOffloadCheckpoint,
|
|
3977
3647
|
fromMemory,
|
|
3978
|
-
pinsRetiredVectors,
|
|
3979
3648
|
mutationThreshold: checkpointMutationThreshold,
|
|
3980
3649
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
3981
3650
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
@@ -4007,9 +3676,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4007
3676
|
const recordedByIndex = /* @__PURE__ */ new Map();
|
|
4008
3677
|
for (let i = 0; i < records.length; i++) {
|
|
4009
3678
|
if (!outcomes[i].ok) continue;
|
|
4010
|
-
const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0 };
|
|
3679
|
+
const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0, deferred: false };
|
|
4011
3680
|
recorded.records += 1;
|
|
4012
3681
|
recorded.documentBytes += records[i].document?.byteLength ?? 0;
|
|
3682
|
+
recorded.deferred ||= hasDeferredPublication(records[i]);
|
|
4013
3683
|
recordedByIndex.set(records[i].indexName, recorded);
|
|
4014
3684
|
}
|
|
4015
3685
|
for (const [indexName, recorded] of recordedByIndex) {
|
|
@@ -4021,7 +3691,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4021
3691
|
markFatal(toError(err));
|
|
4022
3692
|
});
|
|
4023
3693
|
}
|
|
4024
|
-
await stallWhileCheckpointsFallBehind(indexState);
|
|
3694
|
+
if (!recorded.deferred) await stallWhileCheckpointsFallBehind(indexState);
|
|
4025
3695
|
}
|
|
4026
3696
|
if (recordedByIndex.size === 0) return outcomes;
|
|
4027
3697
|
if (mode === "sync") await directoryWatch.verify();
|
|
@@ -4056,7 +3726,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4056
3726
|
syncStorage
|
|
4057
3727
|
};
|
|
4058
3728
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
4059
|
-
|
|
3729
|
+
const recovered = recoverWithLayoutCheckpoint(
|
|
3730
|
+
recoveryDeps,
|
|
3731
|
+
indexName,
|
|
3732
|
+
metadataOnly,
|
|
3733
|
+
() => performCheckpoint(indexName, getOrCreateIndexState(indexName), true)
|
|
3734
|
+
);
|
|
3735
|
+
return loads.track(indexName, recovered, startsGraphRepairs);
|
|
4060
3736
|
}
|
|
4061
3737
|
return {
|
|
4062
3738
|
isActive: () => true,
|
|
@@ -4138,7 +3814,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4138
3814
|
},
|
|
4139
3815
|
async checkpointAll() {
|
|
4140
3816
|
for (const indexName of [...indexes.keys()]) {
|
|
4141
|
-
await
|
|
3817
|
+
await requestedCheckpoint(indexName);
|
|
4142
3818
|
}
|
|
4143
3819
|
},
|
|
4144
3820
|
...unloading,
|
|
@@ -4164,27 +3840,47 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4164
3840
|
}
|
|
4165
3841
|
|
|
4166
3842
|
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
4167
|
-
function capture(input, prior) {
|
|
3843
|
+
function capture(input, prior, cutoffs) {
|
|
4168
3844
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
4169
3845
|
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
4170
3846
|
const targets = [];
|
|
4171
3847
|
const contents = [];
|
|
3848
|
+
const positions = /* @__PURE__ */ new Map();
|
|
4172
3849
|
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
4173
|
-
const lastSeqNo =
|
|
3850
|
+
const lastSeqNo = Math.min(
|
|
3851
|
+
input.appliedSeqNoByPartition.get(partitionId) ?? 0,
|
|
3852
|
+
cutoffs.get(partitionId) ?? Infinity
|
|
3853
|
+
);
|
|
3854
|
+
positions.set(partitionId, lastSeqNo);
|
|
4174
3855
|
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
4175
3856
|
if (unchanged) continue;
|
|
4176
3857
|
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
4177
3858
|
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
4178
3859
|
}
|
|
4179
3860
|
const plans = /* @__PURE__ */ new Map();
|
|
4180
|
-
|
|
4181
|
-
const
|
|
4182
|
-
|
|
3861
|
+
try {
|
|
3862
|
+
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
3863
|
+
const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
|
|
3864
|
+
plans.set(
|
|
3865
|
+
fieldPath,
|
|
3866
|
+
vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), false, true)
|
|
3867
|
+
);
|
|
3868
|
+
}
|
|
3869
|
+
return {
|
|
3870
|
+
targets,
|
|
3871
|
+
plans,
|
|
3872
|
+
documentCount: input.manager.countDocuments(),
|
|
3873
|
+
contents,
|
|
3874
|
+
positions,
|
|
3875
|
+
partitionCount: input.manager.partitionCount
|
|
3876
|
+
};
|
|
3877
|
+
} catch (error) {
|
|
3878
|
+
for (const plan of plans.values()) plan.release();
|
|
3879
|
+
throw error;
|
|
4183
3880
|
}
|
|
4184
|
-
return { targets, plans, documentCount: input.manager.countDocuments(), contents };
|
|
4185
3881
|
}
|
|
4186
3882
|
async function writeCaptured(input, store, prior, captured) {
|
|
4187
|
-
const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior,
|
|
3883
|
+
const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, captured.partitionCount);
|
|
4188
3884
|
const written = await writeCheckpointSegments({
|
|
4189
3885
|
store,
|
|
4190
3886
|
metadata: input.metadata,
|
|
@@ -4210,9 +3906,17 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
4210
3906
|
const store = adapterCheckpointStore(input.adapter);
|
|
4211
3907
|
for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
4212
3908
|
const prior = await readCurrentManifest(store, input.indexName);
|
|
4213
|
-
const captured = capture(input, prior);
|
|
4214
|
-
|
|
4215
|
-
|
|
3909
|
+
const captured = await captureAppliedPrefix(input.manager, (cutoffs) => capture(input, prior, cutoffs));
|
|
3910
|
+
try {
|
|
3911
|
+
await writeCaptured(input, store, prior, captured);
|
|
3912
|
+
return {
|
|
3913
|
+
documentCount: captured.documentCount,
|
|
3914
|
+
positions: captured.positions,
|
|
3915
|
+
partitionCount: captured.partitionCount
|
|
3916
|
+
};
|
|
3917
|
+
} finally {
|
|
3918
|
+
for (const plan of captured.plans.values()) plan.release();
|
|
3919
|
+
}
|
|
4216
3920
|
}
|
|
4217
3921
|
|
|
4218
3922
|
// src/persistence/durability/snapshot-only/index-state.ts
|
|
@@ -4225,6 +3929,20 @@ function createSnapshotIndexState() {
|
|
|
4225
3929
|
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
4226
3930
|
};
|
|
4227
3931
|
}
|
|
3932
|
+
function metadataKey(indexName) {
|
|
3933
|
+
return `${indexName}/meta`;
|
|
3934
|
+
}
|
|
3935
|
+
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
3936
|
+
let recordsLeft = 0;
|
|
3937
|
+
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
3938
|
+
if (partitionId >= partitionCount) {
|
|
3939
|
+
state.appliedSeqNoByPartition.delete(partitionId);
|
|
3940
|
+
continue;
|
|
3941
|
+
}
|
|
3942
|
+
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
3943
|
+
}
|
|
3944
|
+
state.mutationsSinceCheckpoint = recordsLeft;
|
|
3945
|
+
}
|
|
4228
3946
|
|
|
4229
3947
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
4230
3948
|
function toError5(error) {
|
|
@@ -4264,9 +3982,6 @@ function createPendingRestores(onFatalError) {
|
|
|
4264
3982
|
}
|
|
4265
3983
|
|
|
4266
3984
|
// src/persistence/durability/snapshot-only/index.ts
|
|
4267
|
-
function metadataKey(indexName) {
|
|
4268
|
-
return `${indexName}/meta`;
|
|
4269
|
-
}
|
|
4270
3985
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
4271
3986
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
4272
3987
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
@@ -4312,10 +4027,8 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4312
4027
|
await settleRestore(indexName);
|
|
4313
4028
|
const manager = hooks.getManager(indexName);
|
|
4314
4029
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
4315
|
-
if (manager === void 0 || metadata === void 0)
|
|
4316
|
-
|
|
4317
|
-
}
|
|
4318
|
-
const documentCount = await writeSnapshotOnlyCheckpoint({
|
|
4030
|
+
if (manager === void 0 || metadata === void 0) return;
|
|
4031
|
+
const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
|
|
4319
4032
|
adapter,
|
|
4320
4033
|
indexName,
|
|
4321
4034
|
metadata,
|
|
@@ -4328,11 +4041,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4328
4041
|
await queueMetadataWrite(indexName, async () => {
|
|
4329
4042
|
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
|
|
4330
4043
|
if (checkpointMetadata === void 0) return;
|
|
4044
|
+
checkpointMetadata.partitionCount = partitionCount;
|
|
4331
4045
|
const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
4332
4046
|
await adapter.save(metadataKey(indexName), metadataBytes);
|
|
4333
|
-
hooks.recordCheckpoint?.(indexName, documentCount,
|
|
4047
|
+
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
|
|
4334
4048
|
});
|
|
4335
|
-
indexState
|
|
4049
|
+
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
4336
4050
|
if (publisher !== void 0) {
|
|
4337
4051
|
const partitionIds = [];
|
|
4338
4052
|
for (let i = 0; i < manager.partitionCount; i += 1) {
|
|
@@ -4341,15 +4055,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4341
4055
|
await publisher.publishPartitions(indexName, partitionIds);
|
|
4342
4056
|
}
|
|
4343
4057
|
}
|
|
4344
|
-
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
|
|
4058
|
+
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, requested = false) {
|
|
4345
4059
|
const indexState = indexes.get(indexName);
|
|
4346
4060
|
if (indexState === void 0 || indexState.unloading) return;
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4061
|
+
if (requested) await requireCheckpointLayout(hooks.getManager(indexName));
|
|
4062
|
+
return holdScheduledCheckpoint(
|
|
4063
|
+
checkpointInTurn(indexState, {
|
|
4064
|
+
joinsInFlight: !afterInFlight,
|
|
4065
|
+
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
4066
|
+
perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
|
|
4067
|
+
afterSettling: () => void 0
|
|
4068
|
+
}),
|
|
4069
|
+
requested
|
|
4070
|
+
);
|
|
4353
4071
|
}
|
|
4354
4072
|
function queueMetadataWrite(indexName, write) {
|
|
4355
4073
|
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
@@ -4379,7 +4097,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4379
4097
|
let derivedCount;
|
|
4380
4098
|
if (metadataOnly && metadata.documentCount === void 0) {
|
|
4381
4099
|
try {
|
|
4382
|
-
derivedCount = await
|
|
4100
|
+
derivedCount = await countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
|
|
4383
4101
|
metadata.documentCount = derivedCount;
|
|
4384
4102
|
} catch {
|
|
4385
4103
|
derivedCount = void 0;
|
|
@@ -4423,9 +4141,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4423
4141
|
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
|
|
4424
4142
|
}
|
|
4425
4143
|
}
|
|
4426
|
-
function countStoredBundleDocuments(indexName) {
|
|
4427
|
-
return countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
|
|
4428
|
-
}
|
|
4429
4144
|
async function recordMutation(record) {
|
|
4430
4145
|
const indexState = getOrCreateIndexState(record.indexName);
|
|
4431
4146
|
const buffered = indexState.applyChain.then(async () => {
|
|
@@ -4505,10 +4220,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4505
4220
|
});
|
|
4506
4221
|
},
|
|
4507
4222
|
checkpoint(indexName) {
|
|
4508
|
-
return checkpointIndex(indexName);
|
|
4223
|
+
return checkpointIndex(indexName, false, false, true);
|
|
4509
4224
|
},
|
|
4510
4225
|
checkpointFromMemory(indexName) {
|
|
4511
|
-
return checkpointIndex(indexName, true);
|
|
4226
|
+
return checkpointIndex(indexName, true, true, true);
|
|
4512
4227
|
},
|
|
4513
4228
|
async checkpointAfterSwitch(indexName, swap) {
|
|
4514
4229
|
const indexState = getOrCreateIndexState(indexName);
|
|
@@ -4527,7 +4242,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4527
4242
|
},
|
|
4528
4243
|
async checkpointAll() {
|
|
4529
4244
|
for (const indexName of [...indexes.keys()]) {
|
|
4530
|
-
await checkpointIndex(indexName);
|
|
4245
|
+
await checkpointIndex(indexName, false, false, true);
|
|
4531
4246
|
}
|
|
4532
4247
|
},
|
|
4533
4248
|
stageRestore(indexName) {
|
|
@@ -4587,7 +4302,13 @@ function buildMetadata(indexName, hooks, documentCount) {
|
|
|
4587
4302
|
if (config === void 0) {
|
|
4588
4303
|
return void 0;
|
|
4589
4304
|
}
|
|
4590
|
-
|
|
4305
|
+
const manager = hooks.getManager(indexName);
|
|
4306
|
+
return metadataFromConfig(
|
|
4307
|
+
indexName,
|
|
4308
|
+
config,
|
|
4309
|
+
manager === void 0 ? void 0 : metadataPartitionCount(manager),
|
|
4310
|
+
documentCount
|
|
4311
|
+
);
|
|
4591
4312
|
}
|
|
4592
4313
|
function createDurabilityIntegration(tier, hooks) {
|
|
4593
4314
|
const managerHooks = {
|
|
@@ -4612,22 +4333,30 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4612
4333
|
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
4613
4334
|
};
|
|
4614
4335
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
4615
|
-
function partitionFor(indexName, docId) {
|
|
4616
|
-
const partitionManager = hooks.getManager(indexName);
|
|
4617
|
-
const partitionCount = partitionManager?.partitionCount ?? 1;
|
|
4618
|
-
return fnv1a(docId) % Math.max(1, partitionCount);
|
|
4619
|
-
}
|
|
4620
4336
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4337
|
+
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
4338
|
+
const partitionId = fnv1a(docId) % partitionCount;
|
|
4339
|
+
let queued;
|
|
4340
|
+
try {
|
|
4341
|
+
const record = {
|
|
4342
|
+
indexName,
|
|
4343
|
+
partitionId,
|
|
4344
|
+
operation,
|
|
4345
|
+
documentId: docId,
|
|
4346
|
+
document,
|
|
4347
|
+
async apply() {
|
|
4348
|
+
queued = await apply() ?? void 0;
|
|
4349
|
+
if (queued !== void 0) deferMutationPublication(record);
|
|
4350
|
+
}
|
|
4351
|
+
};
|
|
4352
|
+
const seqNo = await manager.recordMutation(record);
|
|
4353
|
+
queued?.commit(partitionId, partitionCount, seqNo);
|
|
4354
|
+
return { indexName, partitionId, seqNo };
|
|
4355
|
+
} catch (error) {
|
|
4356
|
+
queued?.fail(error);
|
|
4357
|
+
throw error;
|
|
4358
|
+
}
|
|
4629
4359
|
});
|
|
4630
|
-
return { indexName, partitionId, seqNo };
|
|
4631
4360
|
}
|
|
4632
4361
|
return {
|
|
4633
4362
|
manager,
|
|
@@ -4638,29 +4367,48 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4638
4367
|
const records = [];
|
|
4639
4368
|
const outcomes = new Array(mutations.length);
|
|
4640
4369
|
const recordPositions = [];
|
|
4370
|
+
const queued = [];
|
|
4641
4371
|
for (let i = 0; i < mutations.length; i++) {
|
|
4642
4372
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
4643
4373
|
const mutation = mutations[i];
|
|
4644
4374
|
try {
|
|
4645
|
-
|
|
4375
|
+
const record = {
|
|
4646
4376
|
indexName,
|
|
4647
|
-
partitionId:
|
|
4377
|
+
partitionId: 0,
|
|
4648
4378
|
operation: mutation.document === null ? "DELETE" : "INDEX",
|
|
4649
4379
|
documentId: mutation.docId,
|
|
4650
4380
|
document: mutation.document === null ? null : encodeDocument(mutation.document),
|
|
4651
|
-
apply
|
|
4652
|
-
|
|
4381
|
+
async apply() {
|
|
4382
|
+
queued[i] = await mutation.apply() ?? void 0;
|
|
4383
|
+
if (queued[i] !== void 0) deferMutationPublication(record);
|
|
4384
|
+
}
|
|
4385
|
+
};
|
|
4386
|
+
records.push(record);
|
|
4653
4387
|
recordPositions.push(i);
|
|
4654
4388
|
} catch (error) {
|
|
4655
4389
|
outcomes[i] = { ok: false, error };
|
|
4656
4390
|
}
|
|
4657
4391
|
}
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
4663
|
-
|
|
4392
|
+
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
4393
|
+
let recorded;
|
|
4394
|
+
try {
|
|
4395
|
+
for (const record of records) {
|
|
4396
|
+
record.partitionId = fnv1a(record.documentId) % partitionCount;
|
|
4397
|
+
}
|
|
4398
|
+
recorded = await manager.recordMutations(records);
|
|
4399
|
+
} catch (error) {
|
|
4400
|
+
for (const entry of queued) entry?.fail(error);
|
|
4401
|
+
throw error;
|
|
4402
|
+
}
|
|
4403
|
+
for (let i = 0; i < recorded.length; i++) {
|
|
4404
|
+
const outcome = recorded[i];
|
|
4405
|
+
const entry = queued[recordPositions[i]];
|
|
4406
|
+
if (outcome.ok) entry?.commit(records[i].partitionId, partitionCount, outcome.seqNo);
|
|
4407
|
+
else entry?.fail(outcome.error);
|
|
4408
|
+
outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
|
|
4409
|
+
}
|
|
4410
|
+
return outcomes;
|
|
4411
|
+
});
|
|
4664
4412
|
},
|
|
4665
4413
|
recordRemove(indexName, docId, apply) {
|
|
4666
4414
|
return recordMutation(indexName, docId, "DELETE", null, apply);
|
|
@@ -4668,6 +4416,172 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4668
4416
|
};
|
|
4669
4417
|
}
|
|
4670
4418
|
|
|
4419
|
+
// src/engine/invalidation.ts
|
|
4420
|
+
function createInvalidationFromConfig(config, tierKind, deps, indexRegistry) {
|
|
4421
|
+
if (config?.invalidation === void 0) {
|
|
4422
|
+
return null;
|
|
4423
|
+
}
|
|
4424
|
+
if (tierKind !== "snapshot") {
|
|
4425
|
+
throw new NarsilError(
|
|
4426
|
+
ErrorCodes.CONFIG_INVALID,
|
|
4427
|
+
"Invalidation requires a shared persistence adapter. The write-ahead log tier owns its directory exclusively, so configure a non-filesystem persistence adapter to share data between instances"
|
|
4428
|
+
);
|
|
4429
|
+
}
|
|
4430
|
+
return createInvalidationIntegration({
|
|
4431
|
+
adapter: config.invalidation,
|
|
4432
|
+
instanceId: generateId(),
|
|
4433
|
+
...deps,
|
|
4434
|
+
listBroadcastIndexNames() {
|
|
4435
|
+
const indexes = [];
|
|
4436
|
+
for (const [name, entry] of indexRegistry) {
|
|
4437
|
+
if (entry.config.defaultScoring === "broadcast") indexes.push(name);
|
|
4438
|
+
}
|
|
4439
|
+
return indexes;
|
|
4440
|
+
}
|
|
4441
|
+
});
|
|
4442
|
+
}
|
|
4443
|
+
function createInvalidationIntegration(deps) {
|
|
4444
|
+
const { adapter, instanceId, reloadIndex, getManager, listBroadcastIndexNames, onError } = deps;
|
|
4445
|
+
const foreignStats = /* @__PURE__ */ new Map();
|
|
4446
|
+
const reloadChains = /* @__PURE__ */ new Map();
|
|
4447
|
+
let broadcastTimer = null;
|
|
4448
|
+
let stopped = false;
|
|
4449
|
+
function toError6(err) {
|
|
4450
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
4451
|
+
}
|
|
4452
|
+
function pruneStaleForeignStats(entries) {
|
|
4453
|
+
const cutoff = Date.now() - FOREIGN_STATS_TTL_MS;
|
|
4454
|
+
for (const [sourceInstanceId, entry] of entries) {
|
|
4455
|
+
if (entry.receivedAt < cutoff) {
|
|
4456
|
+
entries.delete(sourceInstanceId);
|
|
4457
|
+
}
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
function scheduleReload(indexName) {
|
|
4461
|
+
const chain = reloadChains.get(indexName) ?? Promise.resolve();
|
|
4462
|
+
const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(toError6(err)));
|
|
4463
|
+
reloadChains.set(indexName, next);
|
|
4464
|
+
}
|
|
4465
|
+
function storeForeignStats(indexName, sourceInstanceId, stats) {
|
|
4466
|
+
let entries = foreignStats.get(indexName);
|
|
4467
|
+
if (entries === void 0) {
|
|
4468
|
+
entries = /* @__PURE__ */ new Map();
|
|
4469
|
+
foreignStats.set(indexName, entries);
|
|
4470
|
+
}
|
|
4471
|
+
entries.set(sourceInstanceId, { stats, receivedAt: Date.now() });
|
|
4472
|
+
pruneStaleForeignStats(entries);
|
|
4473
|
+
}
|
|
4474
|
+
function handleEvent(event) {
|
|
4475
|
+
try {
|
|
4476
|
+
if (event.type === "partition") {
|
|
4477
|
+
if (event.sourceInstanceId === instanceId) {
|
|
4478
|
+
return;
|
|
4479
|
+
}
|
|
4480
|
+
scheduleReload(event.indexName);
|
|
4481
|
+
return;
|
|
4482
|
+
}
|
|
4483
|
+
if (event.instanceId === instanceId) {
|
|
4484
|
+
return;
|
|
4485
|
+
}
|
|
4486
|
+
storeForeignStats(event.indexName, event.instanceId, event.stats);
|
|
4487
|
+
} catch (err) {
|
|
4488
|
+
onError(toError6(err));
|
|
4489
|
+
}
|
|
4490
|
+
}
|
|
4491
|
+
async function publishOwnStatistics() {
|
|
4492
|
+
for (const indexName of listBroadcastIndexNames()) {
|
|
4493
|
+
const manager = getManager(indexName);
|
|
4494
|
+
if (manager === void 0) {
|
|
4495
|
+
continue;
|
|
4496
|
+
}
|
|
4497
|
+
const aggregate = manager.statisticsMessage();
|
|
4498
|
+
try {
|
|
4499
|
+
await adapter.publish({
|
|
4500
|
+
type: "statistics",
|
|
4501
|
+
indexName,
|
|
4502
|
+
instanceId,
|
|
4503
|
+
stats: {
|
|
4504
|
+
totalDocs: aggregate.totalDocuments,
|
|
4505
|
+
docFrequencies: aggregate.docFrequencies,
|
|
4506
|
+
totalFieldLengths: aggregate.totalFieldLengths
|
|
4507
|
+
}
|
|
4508
|
+
});
|
|
4509
|
+
} catch (err) {
|
|
4510
|
+
onError(toError6(err));
|
|
4511
|
+
}
|
|
4512
|
+
}
|
|
4513
|
+
}
|
|
4514
|
+
return {
|
|
4515
|
+
instanceId,
|
|
4516
|
+
async publishPartitions(indexName, partitions) {
|
|
4517
|
+
if (stopped || partitions.length === 0) {
|
|
4518
|
+
return;
|
|
4519
|
+
}
|
|
4520
|
+
try {
|
|
4521
|
+
await adapter.publish({
|
|
4522
|
+
type: "partition",
|
|
4523
|
+
indexName,
|
|
4524
|
+
partitions,
|
|
4525
|
+
timestamp: Date.now(),
|
|
4526
|
+
sourceInstanceId: instanceId
|
|
4527
|
+
});
|
|
4528
|
+
} catch (err) {
|
|
4529
|
+
onError(toError6(err));
|
|
4530
|
+
}
|
|
4531
|
+
},
|
|
4532
|
+
broadcastStats(indexName, tokens) {
|
|
4533
|
+
const manager = getManager(indexName);
|
|
4534
|
+
if (manager === void 0) {
|
|
4535
|
+
return void 0;
|
|
4536
|
+
}
|
|
4537
|
+
const own = manager.getAggregateStats(tokens);
|
|
4538
|
+
const entries = foreignStats.get(indexName);
|
|
4539
|
+
if (entries === void 0 || entries.size === 0) {
|
|
4540
|
+
return mergePartitionStats([own]);
|
|
4541
|
+
}
|
|
4542
|
+
pruneStaleForeignStats(entries);
|
|
4543
|
+
const combined = [own];
|
|
4544
|
+
for (const [, entry] of entries) {
|
|
4545
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
4546
|
+
for (const token of tokens) {
|
|
4547
|
+
if (Object.hasOwn(entry.stats.docFrequencies, token))
|
|
4548
|
+
docFrequencies[token] = entry.stats.docFrequencies[token];
|
|
4549
|
+
}
|
|
4550
|
+
combined.push({
|
|
4551
|
+
totalDocuments: entry.stats.totalDocs,
|
|
4552
|
+
docFrequencies,
|
|
4553
|
+
totalFieldLengths: entry.stats.totalFieldLengths
|
|
4554
|
+
});
|
|
4555
|
+
}
|
|
4556
|
+
return mergePartitionStats(combined);
|
|
4557
|
+
},
|
|
4558
|
+
async start() {
|
|
4559
|
+
await adapter.subscribe(handleEvent);
|
|
4560
|
+
if (broadcastTimer !== null || stopped) {
|
|
4561
|
+
return;
|
|
4562
|
+
}
|
|
4563
|
+
broadcastTimer = setInterval(() => {
|
|
4564
|
+
void publishOwnStatistics();
|
|
4565
|
+
}, STATISTICS_BROADCAST_INTERVAL_MS);
|
|
4566
|
+
if (typeof broadcastTimer.unref === "function") {
|
|
4567
|
+
broadcastTimer.unref();
|
|
4568
|
+
}
|
|
4569
|
+
},
|
|
4570
|
+
async shutdown() {
|
|
4571
|
+
stopped = true;
|
|
4572
|
+
if (broadcastTimer !== null) {
|
|
4573
|
+
clearInterval(broadcastTimer);
|
|
4574
|
+
broadcastTimer = null;
|
|
4575
|
+
}
|
|
4576
|
+
try {
|
|
4577
|
+
await adapter.shutdown();
|
|
4578
|
+
} catch (err) {
|
|
4579
|
+
onError(toError6(err));
|
|
4580
|
+
}
|
|
4581
|
+
}
|
|
4582
|
+
};
|
|
4583
|
+
}
|
|
4584
|
+
|
|
4671
4585
|
// src/engine/durability-wiring.ts
|
|
4672
4586
|
function createDurabilityFromTier(tier, wiring) {
|
|
4673
4587
|
return createDurabilityIntegration(tier, {
|
|
@@ -4698,7 +4612,11 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4698
4612
|
keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
|
|
4699
4613
|
onHistoryMoved(indexName) {
|
|
4700
4614
|
if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
|
|
4701
|
-
|
|
4615
|
+
try {
|
|
4616
|
+
scheduleSyncRetention(wiring.engineCore(), indexName);
|
|
4617
|
+
} catch (error) {
|
|
4618
|
+
wiring.emitFatalError(error instanceof Error ? error : new Error(String(error)));
|
|
4619
|
+
}
|
|
4702
4620
|
}
|
|
4703
4621
|
});
|
|
4704
4622
|
}
|
|
@@ -4713,4 +4631,1489 @@ async function reloadIndexAndCopies(orchestrator, durability, indexName) {
|
|
|
4713
4631
|
}
|
|
4714
4632
|
}
|
|
4715
4633
|
|
|
4716
|
-
export
|
|
4634
|
+
// src/engine/export/lines.ts
|
|
4635
|
+
var NEWLINE = 10;
|
|
4636
|
+
function joined(parts, total) {
|
|
4637
|
+
if (parts.length === 1) return parts[0];
|
|
4638
|
+
const bytes = new Uint8Array(total);
|
|
4639
|
+
let offset = 0;
|
|
4640
|
+
for (const part of parts) {
|
|
4641
|
+
bytes.set(part, offset);
|
|
4642
|
+
offset += part.byteLength;
|
|
4643
|
+
}
|
|
4644
|
+
return bytes;
|
|
4645
|
+
}
|
|
4646
|
+
async function* exportLines(source, maxLineBytes) {
|
|
4647
|
+
const decoder3 = new TextDecoder("utf-8", { fatal: true });
|
|
4648
|
+
let parts = [];
|
|
4649
|
+
let pending = 0;
|
|
4650
|
+
let overflowing = false;
|
|
4651
|
+
let line = 0;
|
|
4652
|
+
const finishLine = () => {
|
|
4653
|
+
line += 1;
|
|
4654
|
+
if (overflowing) {
|
|
4655
|
+
overflowing = false;
|
|
4656
|
+
parts = [];
|
|
4657
|
+
pending = 0;
|
|
4658
|
+
return { line, refused: "too-long" };
|
|
4659
|
+
}
|
|
4660
|
+
const bytes = joined(parts, pending);
|
|
4661
|
+
parts = [];
|
|
4662
|
+
pending = 0;
|
|
4663
|
+
try {
|
|
4664
|
+
return { line, text: decoder3.decode(bytes) };
|
|
4665
|
+
} catch {
|
|
4666
|
+
return { line, refused: "not-utf8" };
|
|
4667
|
+
}
|
|
4668
|
+
};
|
|
4669
|
+
for await (const chunk of chunksOf(source)) {
|
|
4670
|
+
let start = 0;
|
|
4671
|
+
for (; ; ) {
|
|
4672
|
+
const newline = chunk.indexOf(NEWLINE, start);
|
|
4673
|
+
const end = newline < 0 ? chunk.byteLength : newline;
|
|
4674
|
+
const piece = chunk.subarray(start, end);
|
|
4675
|
+
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
4676
|
+
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
4677
|
+
pending += piece.byteLength;
|
|
4678
|
+
} else {
|
|
4679
|
+
overflowing = true;
|
|
4680
|
+
parts = [];
|
|
4681
|
+
}
|
|
4682
|
+
if (newline < 0) break;
|
|
4683
|
+
yield finishLine();
|
|
4684
|
+
start = newline + 1;
|
|
4685
|
+
}
|
|
4686
|
+
}
|
|
4687
|
+
if (pending > 0 || overflowing) yield finishLine();
|
|
4688
|
+
}
|
|
4689
|
+
|
|
4690
|
+
// src/serialization/base64.ts
|
|
4691
|
+
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
4692
|
+
var PAD = 61;
|
|
4693
|
+
var BYTES_PER_GROUP = 3;
|
|
4694
|
+
var CHARACTERS_PER_GROUP = 4;
|
|
4695
|
+
var ASCII_LIMIT = 128;
|
|
4696
|
+
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
4697
|
+
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
4698
|
+
for (let at = 0; at < ALPHABET.length; at++) {
|
|
4699
|
+
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
4700
|
+
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
4701
|
+
}
|
|
4702
|
+
var asciiDecoder = new TextDecoder();
|
|
4703
|
+
function encodeBase64(bytes) {
|
|
4704
|
+
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
4705
|
+
let write = 0;
|
|
4706
|
+
let read = 0;
|
|
4707
|
+
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
4708
|
+
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
4709
|
+
out[write++] = ENCODE[group >>> 18];
|
|
4710
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
4711
|
+
out[write++] = ENCODE[group >>> 6 & 63];
|
|
4712
|
+
out[write++] = ENCODE[group & 63];
|
|
4713
|
+
}
|
|
4714
|
+
const left = bytes.byteLength - read;
|
|
4715
|
+
if (left > 0) {
|
|
4716
|
+
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
4717
|
+
out[write++] = ENCODE[group >>> 18];
|
|
4718
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
4719
|
+
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
4720
|
+
out[write++] = PAD;
|
|
4721
|
+
}
|
|
4722
|
+
return asciiDecoder.decode(out);
|
|
4723
|
+
}
|
|
4724
|
+
function sextetAt(text, at) {
|
|
4725
|
+
const code = text.charCodeAt(at);
|
|
4726
|
+
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
4727
|
+
}
|
|
4728
|
+
function decodeExactBase64(text) {
|
|
4729
|
+
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
4730
|
+
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
4731
|
+
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
4732
|
+
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
4733
|
+
let write = 0;
|
|
4734
|
+
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
4735
|
+
const a2 = sextetAt(text, at);
|
|
4736
|
+
const b2 = sextetAt(text, at + 1);
|
|
4737
|
+
const c2 = sextetAt(text, at + 2);
|
|
4738
|
+
const d = sextetAt(text, at + 3);
|
|
4739
|
+
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
4740
|
+
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
4741
|
+
out[write++] = group >>> 16;
|
|
4742
|
+
out[write++] = group >>> 8 & 255;
|
|
4743
|
+
out[write++] = group & 255;
|
|
4744
|
+
}
|
|
4745
|
+
if (padding === 0) return out;
|
|
4746
|
+
const a = sextetAt(text, fullEnd);
|
|
4747
|
+
const b = sextetAt(text, fullEnd + 1);
|
|
4748
|
+
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
4749
|
+
if ((a | b | c) < 0) return null;
|
|
4750
|
+
if (padding === 2 && (b & 15) !== 0) return null;
|
|
4751
|
+
if (padding === 1 && (c & 3) !== 0) return null;
|
|
4752
|
+
out[write++] = a << 2 | b >>> 4;
|
|
4753
|
+
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
4754
|
+
return out;
|
|
4755
|
+
}
|
|
4756
|
+
|
|
4757
|
+
// src/engine/export/tags.ts
|
|
4758
|
+
var TAG_PREFIX = "$";
|
|
4759
|
+
var DATE_TAG = "$date";
|
|
4760
|
+
var BYTES_TAG = "$bytes";
|
|
4761
|
+
var NUMBER_TAG = "$number";
|
|
4762
|
+
var OBJECT_TAG = "$object";
|
|
4763
|
+
var MILLISECONDS_PER_SECOND = 1e3;
|
|
4764
|
+
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
4765
|
+
var MAX_NANOSECONDS = 999999999;
|
|
4766
|
+
var MAX_DATE_MILLISECONDS = 864e13;
|
|
4767
|
+
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
4768
|
+
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
4769
|
+
["NaN", Number.NaN],
|
|
4770
|
+
["Infinity", Number.POSITIVE_INFINITY],
|
|
4771
|
+
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
4772
|
+
]);
|
|
4773
|
+
function refuse(path, reason) {
|
|
4774
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
4775
|
+
field: path
|
|
4776
|
+
});
|
|
4777
|
+
}
|
|
4778
|
+
function isRecord2(value) {
|
|
4779
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4780
|
+
}
|
|
4781
|
+
function startsWithTagPrefix(key) {
|
|
4782
|
+
return key.startsWith(TAG_PREFIX);
|
|
4783
|
+
}
|
|
4784
|
+
function soleTagKey(record) {
|
|
4785
|
+
let sole = null;
|
|
4786
|
+
for (const key of Object.keys(record)) {
|
|
4787
|
+
if (sole !== null) return null;
|
|
4788
|
+
sole = key;
|
|
4789
|
+
}
|
|
4790
|
+
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
4791
|
+
}
|
|
4792
|
+
function dateOf(value, path) {
|
|
4793
|
+
if (!Array.isArray(value) || value.length !== 2)
|
|
4794
|
+
refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
4795
|
+
const seconds = value[0];
|
|
4796
|
+
const nanoseconds = value[1];
|
|
4797
|
+
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
4798
|
+
refuse(
|
|
4799
|
+
path,
|
|
4800
|
+
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
4801
|
+
);
|
|
4802
|
+
}
|
|
4803
|
+
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
4804
|
+
refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
4805
|
+
}
|
|
4806
|
+
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
4807
|
+
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
4808
|
+
refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
4809
|
+
return new Date(milliseconds);
|
|
4810
|
+
}
|
|
4811
|
+
function bytesOf(value, path) {
|
|
4812
|
+
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
4813
|
+
if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
4814
|
+
return bytes;
|
|
4815
|
+
}
|
|
4816
|
+
function numberOf(value, path) {
|
|
4817
|
+
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
4818
|
+
if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
4819
|
+
return number;
|
|
4820
|
+
}
|
|
4821
|
+
function requireDepth(path, depth) {
|
|
4822
|
+
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
4823
|
+
}
|
|
4824
|
+
function untagMap(record, path, depth) {
|
|
4825
|
+
requireDepth(path, depth);
|
|
4826
|
+
for (const key of Object.keys(record)) {
|
|
4827
|
+
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
4828
|
+
}
|
|
4829
|
+
return record;
|
|
4830
|
+
}
|
|
4831
|
+
function untagList(list, path, depth) {
|
|
4832
|
+
requireDepth(path, depth);
|
|
4833
|
+
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
4834
|
+
return list;
|
|
4835
|
+
}
|
|
4836
|
+
function untagTag(tag, value, path, depth) {
|
|
4837
|
+
switch (tag) {
|
|
4838
|
+
case DATE_TAG:
|
|
4839
|
+
return dateOf(value, path);
|
|
4840
|
+
case BYTES_TAG:
|
|
4841
|
+
return bytesOf(value, path);
|
|
4842
|
+
case NUMBER_TAG:
|
|
4843
|
+
return numberOf(value, path);
|
|
4844
|
+
case OBJECT_TAG:
|
|
4845
|
+
if (!isRecord2(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
4846
|
+
return untagMap(value, path, depth);
|
|
4847
|
+
default:
|
|
4848
|
+
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
4849
|
+
}
|
|
4850
|
+
}
|
|
4851
|
+
function untag(value, path, depth) {
|
|
4852
|
+
if (typeof value !== "object" || value === null) return value;
|
|
4853
|
+
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
4854
|
+
const record = value;
|
|
4855
|
+
const tag = soleTagKey(record);
|
|
4856
|
+
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
4857
|
+
}
|
|
4858
|
+
function untagDocumentLine(text, parsed) {
|
|
4859
|
+
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
4860
|
+
return untagMap(parsed, "", 0);
|
|
4861
|
+
}
|
|
4862
|
+
|
|
4863
|
+
// src/engine/export/import.ts
|
|
4864
|
+
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
4865
|
+
var DOCUMENT_EXPORT_VERSION = 1;
|
|
4866
|
+
function invalidHeader(message) {
|
|
4867
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
4868
|
+
}
|
|
4869
|
+
function isRecord3(value) {
|
|
4870
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4871
|
+
}
|
|
4872
|
+
function configOfHeader(line) {
|
|
4873
|
+
if (line === void 0) invalidHeader("is missing");
|
|
4874
|
+
if (!("text" in line)) {
|
|
4875
|
+
invalidHeader(
|
|
4876
|
+
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
4877
|
+
);
|
|
4878
|
+
}
|
|
4879
|
+
let raw;
|
|
4880
|
+
try {
|
|
4881
|
+
raw = JSON.parse(line.text);
|
|
4882
|
+
} catch {
|
|
4883
|
+
invalidHeader("is not JSON");
|
|
4884
|
+
}
|
|
4885
|
+
if (!isRecord3(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
4886
|
+
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
4887
|
+
}
|
|
4888
|
+
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
4889
|
+
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
4890
|
+
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
4891
|
+
return {
|
|
4892
|
+
schema: configuration.schema,
|
|
4893
|
+
language: configuration.language,
|
|
4894
|
+
...fields,
|
|
4895
|
+
...embedding === void 0 ? {} : {
|
|
4896
|
+
embedding: {
|
|
4897
|
+
fields: embedding.fields,
|
|
4898
|
+
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
4899
|
+
}
|
|
4900
|
+
}
|
|
4901
|
+
};
|
|
4902
|
+
}
|
|
4903
|
+
async function prepareIndex(engine, indexName, config) {
|
|
4904
|
+
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
4905
|
+
await engine.createIndex(indexName, config);
|
|
4906
|
+
return;
|
|
4907
|
+
}
|
|
4908
|
+
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
4909
|
+
const exported = flattenSchema(config.schema);
|
|
4910
|
+
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
4911
|
+
for (const field of fields) {
|
|
4912
|
+
if (existing[field] === exported[field]) continue;
|
|
4913
|
+
throw new NarsilError(
|
|
4914
|
+
ErrorCodes.CONFIG_INVALID,
|
|
4915
|
+
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
4916
|
+
{ indexName, field }
|
|
4917
|
+
);
|
|
4918
|
+
}
|
|
4919
|
+
}
|
|
4920
|
+
function highestReportedAt(refused) {
|
|
4921
|
+
let highest = 0;
|
|
4922
|
+
for (let at = 1; at < refused.length; at++) {
|
|
4923
|
+
if (refused[at].line > refused[highest].line) highest = at;
|
|
4924
|
+
}
|
|
4925
|
+
return highest;
|
|
4926
|
+
}
|
|
4927
|
+
function refuse2(state, line, code, message) {
|
|
4928
|
+
state.result.refusedCount += 1;
|
|
4929
|
+
const refused = state.result.refused;
|
|
4930
|
+
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
4931
|
+
refused.push({ line, code, message });
|
|
4932
|
+
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
4933
|
+
return;
|
|
4934
|
+
}
|
|
4935
|
+
if (line >= refused[state.highestReported].line) return;
|
|
4936
|
+
refused[state.highestReported] = { line, code, message };
|
|
4937
|
+
state.highestReported = highestReportedAt(refused);
|
|
4938
|
+
}
|
|
4939
|
+
async function flush(state) {
|
|
4940
|
+
if (state.batch.length === 0) return;
|
|
4941
|
+
const batch = state.batch;
|
|
4942
|
+
state.batch = [];
|
|
4943
|
+
state.batchIds = /* @__PURE__ */ new Set();
|
|
4944
|
+
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
4945
|
+
const inserts = [];
|
|
4946
|
+
const updates = [];
|
|
4947
|
+
for (const entry of batch) {
|
|
4948
|
+
if (await state.engine.has(state.indexName, entry.docId))
|
|
4949
|
+
updates.push({ docId: entry.docId, document: entry.document });
|
|
4950
|
+
else inserts.push(entry.document);
|
|
4951
|
+
}
|
|
4952
|
+
for (const outcome of [
|
|
4953
|
+
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
4954
|
+
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
4955
|
+
]) {
|
|
4956
|
+
if (outcome === null) continue;
|
|
4957
|
+
state.result.imported += outcome.succeeded.length;
|
|
4958
|
+
for (const failure of outcome.failed) {
|
|
4959
|
+
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
4960
|
+
refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
4961
|
+
}
|
|
4962
|
+
}
|
|
4963
|
+
}
|
|
4964
|
+
async function takeLine(state, entry) {
|
|
4965
|
+
if (!("text" in entry)) {
|
|
4966
|
+
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
4967
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
4968
|
+
return;
|
|
4969
|
+
}
|
|
4970
|
+
if (entry.text.length === 0) return;
|
|
4971
|
+
let document;
|
|
4972
|
+
try {
|
|
4973
|
+
document = JSON.parse(entry.text);
|
|
4974
|
+
} catch {
|
|
4975
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
4976
|
+
return;
|
|
4977
|
+
}
|
|
4978
|
+
if (!isRecord3(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
4979
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
4980
|
+
return;
|
|
4981
|
+
}
|
|
4982
|
+
try {
|
|
4983
|
+
untagDocumentLine(entry.text, document);
|
|
4984
|
+
} catch (error) {
|
|
4985
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
4986
|
+
refuse2(state, entry.line, error.code, error.message);
|
|
4987
|
+
return;
|
|
4988
|
+
}
|
|
4989
|
+
if (state.batchIds.has(document.id)) await flush(state);
|
|
4990
|
+
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
4991
|
+
state.batchIds.add(document.id);
|
|
4992
|
+
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
4993
|
+
}
|
|
4994
|
+
async function readDocumentExport(engine, indexName, source) {
|
|
4995
|
+
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
4996
|
+
const header = await lines.next();
|
|
4997
|
+
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
4998
|
+
const state = {
|
|
4999
|
+
engine,
|
|
5000
|
+
indexName,
|
|
5001
|
+
batch: [],
|
|
5002
|
+
batchIds: /* @__PURE__ */ new Set(),
|
|
5003
|
+
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
5004
|
+
highestReported: 0
|
|
5005
|
+
};
|
|
5006
|
+
for await (const entry of lines) {
|
|
5007
|
+
await takeLine(state, entry);
|
|
5008
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5009
|
+
}
|
|
5010
|
+
await flush(state);
|
|
5011
|
+
state.result.refused.sort((a, b) => a.line - b.line);
|
|
5012
|
+
return state.result;
|
|
5013
|
+
}
|
|
5014
|
+
|
|
5015
|
+
// src/engine/export/json-line.ts
|
|
5016
|
+
function refuse3(path, kind, docId) {
|
|
5017
|
+
throw new NarsilError(
|
|
5018
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5019
|
+
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
5020
|
+
{ docId, field: path }
|
|
5021
|
+
);
|
|
5022
|
+
}
|
|
5023
|
+
function encodeNumber(value, path, target) {
|
|
5024
|
+
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
5025
|
+
if (!target.tagged) refuse3(path, String(value), target.docId);
|
|
5026
|
+
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
5027
|
+
}
|
|
5028
|
+
function encodeDate(value, path, target) {
|
|
5029
|
+
const milliseconds = value.getTime();
|
|
5030
|
+
if (!target.tagged || Number.isNaN(milliseconds)) refuse3(path, "a date", target.docId);
|
|
5031
|
+
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
5032
|
+
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
5033
|
+
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
5034
|
+
}
|
|
5035
|
+
function encodeBytes(value, path, target) {
|
|
5036
|
+
if (!target.tagged) refuse3(path, "bytes", target.docId);
|
|
5037
|
+
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
5038
|
+
}
|
|
5039
|
+
function encodeList(value, path, target) {
|
|
5040
|
+
target.parts.push("[");
|
|
5041
|
+
for (let at = 0; at < value.length; at++) {
|
|
5042
|
+
if (at > 0) target.parts.push(",");
|
|
5043
|
+
encodeValue(value[at], `${path}[${at}]`, target);
|
|
5044
|
+
}
|
|
5045
|
+
target.parts.push("]");
|
|
5046
|
+
}
|
|
5047
|
+
function encodeMap(value, path, target) {
|
|
5048
|
+
const proto = Object.getPrototypeOf(value);
|
|
5049
|
+
if (proto !== Object.prototype && proto !== null) refuse3(path, "a value other than a plain object", target.docId);
|
|
5050
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
5051
|
+
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
5052
|
+
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
5053
|
+
for (let at = 0; at < entries.length; at++) {
|
|
5054
|
+
const [key, entry] = entries[at];
|
|
5055
|
+
if (at > 0) target.parts.push(",");
|
|
5056
|
+
target.parts.push(JSON.stringify(key), ":");
|
|
5057
|
+
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
5058
|
+
}
|
|
5059
|
+
target.parts.push(wrapped ? "}}" : "}");
|
|
5060
|
+
}
|
|
5061
|
+
function encodeValue(value, path, target) {
|
|
5062
|
+
if (value === null) {
|
|
5063
|
+
target.parts.push("null");
|
|
5064
|
+
return;
|
|
5065
|
+
}
|
|
5066
|
+
switch (typeof value) {
|
|
5067
|
+
case "string":
|
|
5068
|
+
target.parts.push(JSON.stringify(value));
|
|
5069
|
+
return;
|
|
5070
|
+
case "boolean":
|
|
5071
|
+
target.parts.push(value ? "true" : "false");
|
|
5072
|
+
return;
|
|
5073
|
+
case "number":
|
|
5074
|
+
target.parts.push(encodeNumber(value, path, target));
|
|
5075
|
+
return;
|
|
5076
|
+
case "object":
|
|
5077
|
+
break;
|
|
5078
|
+
default:
|
|
5079
|
+
refuse3(path, `a ${typeof value}`, target.docId);
|
|
5080
|
+
}
|
|
5081
|
+
if (Array.isArray(value)) encodeList(value, path, target);
|
|
5082
|
+
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
5083
|
+
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
5084
|
+
else encodeMap(value, path, target);
|
|
5085
|
+
}
|
|
5086
|
+
function encodeExactJson(value, docId) {
|
|
5087
|
+
const target = { docId, tagged: false, parts: [] };
|
|
5088
|
+
encodeValue(value, "", target);
|
|
5089
|
+
return target.parts.join("");
|
|
5090
|
+
}
|
|
5091
|
+
function encodeDocumentJson(document, docId) {
|
|
5092
|
+
const target = { docId, tagged: true, parts: [] };
|
|
5093
|
+
encodeValue(document, "", target);
|
|
5094
|
+
return target.parts.join("");
|
|
5095
|
+
}
|
|
5096
|
+
|
|
5097
|
+
// src/engine/export/paged-lines.ts
|
|
5098
|
+
async function coldFieldOf(checkpoint, field) {
|
|
5099
|
+
const { directory } = checkpoint;
|
|
5100
|
+
if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
|
|
5101
|
+
if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
|
|
5102
|
+
const fieldType = checkpoint.manifest.schema[field.fieldPath];
|
|
5103
|
+
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
5104
|
+
if (dimension === null) return null;
|
|
5105
|
+
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
5106
|
+
return openColdVectorField({
|
|
5107
|
+
fieldPath: field.fieldPath,
|
|
5108
|
+
dimension,
|
|
5109
|
+
metric: "cosine",
|
|
5110
|
+
codeBits: null,
|
|
5111
|
+
files: await Promise.all(field.files.map(withPath)),
|
|
5112
|
+
graphParts: await Promise.all(field.graphParts.map(withPath)),
|
|
5113
|
+
resolvePartition: () => void 0
|
|
5114
|
+
});
|
|
5115
|
+
}
|
|
5116
|
+
async function openFileVectorReads(checkpoint, wholeFileReads) {
|
|
5117
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5118
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
5119
|
+
const opened = await coldFieldOf(checkpoint, field);
|
|
5120
|
+
if (opened !== null) fields.set(field.fieldPath, opened);
|
|
5121
|
+
}
|
|
5122
|
+
const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
|
|
5123
|
+
return {
|
|
5124
|
+
async vectorOf(fieldPath, docId) {
|
|
5125
|
+
const field = fields.get(fieldPath);
|
|
5126
|
+
if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
|
|
5127
|
+
const number = field.numberOf(docId);
|
|
5128
|
+
return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
|
|
5129
|
+
},
|
|
5130
|
+
close() {
|
|
5131
|
+
for (const field of fields.values()) field.close();
|
|
5132
|
+
fallback?.close();
|
|
5133
|
+
}
|
|
5134
|
+
};
|
|
5135
|
+
}
|
|
5136
|
+
async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
|
|
5137
|
+
const { directory } = checkpoint;
|
|
5138
|
+
if (directory === void 0) return;
|
|
5139
|
+
const scans = [];
|
|
5140
|
+
try {
|
|
5141
|
+
for (const ref of partition.segments) {
|
|
5142
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
5143
|
+
scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
|
|
5144
|
+
}
|
|
5145
|
+
for (let at = 0; at < scans.length; at++) {
|
|
5146
|
+
const scan = scans[at];
|
|
5147
|
+
for (let ordinal = 0; ordinal < scan.count; ordinal++) {
|
|
5148
|
+
const docId = scan.idAt(ordinal);
|
|
5149
|
+
if (!countsAsLatestHolder(scans, at, docId)) continue;
|
|
5150
|
+
const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
|
|
5151
|
+
document.id = docId;
|
|
5152
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
5153
|
+
const vector = await vectors.vectorOf(field.fieldPath, docId);
|
|
5154
|
+
if (vector !== null) setNestedValue(document, field.fieldPath, vector);
|
|
5155
|
+
}
|
|
5156
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
5157
|
+
`;
|
|
5158
|
+
}
|
|
5159
|
+
}
|
|
5160
|
+
} finally {
|
|
5161
|
+
for (const scan of scans) scan.close();
|
|
5162
|
+
}
|
|
5163
|
+
}
|
|
5164
|
+
|
|
5165
|
+
// src/engine/export/write.ts
|
|
5166
|
+
var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
|
|
5167
|
+
var FLOAT32_BYTES = 4;
|
|
5168
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
5169
|
+
async function locateVectors(checkpoint) {
|
|
5170
|
+
const located = /* @__PURE__ */ new Map();
|
|
5171
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
5172
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
5173
|
+
for (const ref of field.files) {
|
|
5174
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
5175
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
|
|
5176
|
+
what: `vector file "${ref.key}"`,
|
|
5177
|
+
allowsCompression: false
|
|
5178
|
+
});
|
|
5179
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
5180
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
5181
|
+
if (isDead(ref.dead, position)) continue;
|
|
5182
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
5183
|
+
}
|
|
5184
|
+
}
|
|
5185
|
+
located.set(field.fieldPath, byDocId);
|
|
5186
|
+
}
|
|
5187
|
+
return located;
|
|
5188
|
+
}
|
|
5189
|
+
async function readVector(checkpoint, location) {
|
|
5190
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
5191
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
5192
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
5193
|
+
}
|
|
5194
|
+
async function readSegment(checkpoint, ref) {
|
|
5195
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
5196
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
5197
|
+
what: `segment "${ref.key}"`,
|
|
5198
|
+
allowsCompression: true
|
|
5199
|
+
});
|
|
5200
|
+
return decodeKeywordSegment(payload, false);
|
|
5201
|
+
}
|
|
5202
|
+
var decoder2 = new TextDecoder();
|
|
5203
|
+
function idAt(contents, ordinal) {
|
|
5204
|
+
const ids = contents.arrays.idTable;
|
|
5205
|
+
return decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
5206
|
+
}
|
|
5207
|
+
async function winnersOf(checkpoint, partition) {
|
|
5208
|
+
const winners = /* @__PURE__ */ new Map();
|
|
5209
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
5210
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
5211
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
5212
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
5213
|
+
}
|
|
5214
|
+
return winners;
|
|
5215
|
+
}
|
|
5216
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
5217
|
+
const contents = await readSegment(checkpoint, ref);
|
|
5218
|
+
const table = contents.arrays.documentTable;
|
|
5219
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5220
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
5221
|
+
return {
|
|
5222
|
+
count: contents.arrays.documentCount,
|
|
5223
|
+
schema: contents.schema,
|
|
5224
|
+
tombstones: new Set(contents.tombstones),
|
|
5225
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
5226
|
+
holds: (docId) => ids.has(docId),
|
|
5227
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
5228
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
5229
|
+
table.discardedFields
|
|
5230
|
+
),
|
|
5231
|
+
close: () => void 0
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
5235
|
+
const located = await locateVectors(checkpoint);
|
|
5236
|
+
return {
|
|
5237
|
+
async vectorOf(fieldPath, docId) {
|
|
5238
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
5239
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
5240
|
+
},
|
|
5241
|
+
close: () => located.clear()
|
|
5242
|
+
};
|
|
5243
|
+
}
|
|
5244
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
5245
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
5246
|
+
try {
|
|
5247
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
5248
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
5249
|
+
}
|
|
5250
|
+
} finally {
|
|
5251
|
+
vectors.close();
|
|
5252
|
+
}
|
|
5253
|
+
}
|
|
5254
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
5255
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
5256
|
+
for (const ref of partition.segments) {
|
|
5257
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
5258
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
return false;
|
|
5262
|
+
}
|
|
5263
|
+
async function* documentLines(checkpoint) {
|
|
5264
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
5265
|
+
yield* pagedDocumentLines(checkpoint);
|
|
5266
|
+
return;
|
|
5267
|
+
}
|
|
5268
|
+
const vectors = await locateVectors(checkpoint);
|
|
5269
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
5270
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
5271
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
5272
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
5273
|
+
const table = contents.arrays.documentTable;
|
|
5274
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
5275
|
+
const docId = idAt(contents, ordinal);
|
|
5276
|
+
if (winners.get(docId) !== at) continue;
|
|
5277
|
+
const stored = decodeStoredDocument(
|
|
5278
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
5279
|
+
table.discardedFields
|
|
5280
|
+
);
|
|
5281
|
+
const document = { id: docId, ...stored };
|
|
5282
|
+
document.id = docId;
|
|
5283
|
+
for (const [fieldPath, located] of vectors) {
|
|
5284
|
+
const location = located.get(docId);
|
|
5285
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
5286
|
+
}
|
|
5287
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
5288
|
+
`;
|
|
5289
|
+
}
|
|
5290
|
+
}
|
|
5291
|
+
}
|
|
5292
|
+
}
|
|
5293
|
+
async function* exportLines2(header, checkpoint) {
|
|
5294
|
+
yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
|
|
5295
|
+
`;
|
|
5296
|
+
yield* documentLines(checkpoint);
|
|
5297
|
+
}
|
|
5298
|
+
function documentExportStream(header, checkpoint) {
|
|
5299
|
+
const lines = exportLines2(header, checkpoint);
|
|
5300
|
+
const encoder = new TextEncoder();
|
|
5301
|
+
let released = false;
|
|
5302
|
+
const release = () => {
|
|
5303
|
+
if (released) return;
|
|
5304
|
+
released = true;
|
|
5305
|
+
checkpoint.release();
|
|
5306
|
+
};
|
|
5307
|
+
return new ReadableStream({
|
|
5308
|
+
async pull(controller) {
|
|
5309
|
+
try {
|
|
5310
|
+
let text = "";
|
|
5311
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
5312
|
+
const step = await lines.next();
|
|
5313
|
+
if (step.done) {
|
|
5314
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
5315
|
+
release();
|
|
5316
|
+
controller.close();
|
|
5317
|
+
return;
|
|
5318
|
+
}
|
|
5319
|
+
text += step.value;
|
|
5320
|
+
}
|
|
5321
|
+
controller.enqueue(encoder.encode(text));
|
|
5322
|
+
} catch (error) {
|
|
5323
|
+
release();
|
|
5324
|
+
controller.error(error);
|
|
5325
|
+
}
|
|
5326
|
+
},
|
|
5327
|
+
async cancel() {
|
|
5328
|
+
release();
|
|
5329
|
+
await lines.return(void 0);
|
|
5330
|
+
}
|
|
5331
|
+
});
|
|
5332
|
+
}
|
|
5333
|
+
|
|
5334
|
+
// src/query-suggestions/archive-history.ts
|
|
5335
|
+
function inDayOrder(left, right) {
|
|
5336
|
+
if (left.day !== right.day) return left.day < right.day ? -1 : 1;
|
|
5337
|
+
return left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
|
|
5338
|
+
}
|
|
5339
|
+
async function* historyArchiveEntries(core, indexName) {
|
|
5340
|
+
const runtime = attachedRuntimeOf(core);
|
|
5341
|
+
const files = runtime?.host.files ?? historyFilesOf(core.durability?.manager.historyStorage?.() ?? null);
|
|
5342
|
+
await runtime?.saveDay(indexName);
|
|
5343
|
+
const names = [];
|
|
5344
|
+
for (const key of await listHistoryFolder(files, indexName)) {
|
|
5345
|
+
const name = historyFileOf(indexName, key);
|
|
5346
|
+
if (name !== null) names.push(name);
|
|
5347
|
+
}
|
|
5348
|
+
names.sort(inDayOrder);
|
|
5349
|
+
let position = 0;
|
|
5350
|
+
for (const name of names) {
|
|
5351
|
+
const bytes = await files.read(name.key);
|
|
5352
|
+
if (bytes === null || bytes === "too-long") continue;
|
|
5353
|
+
yield { name: `${QUERY_HISTORY_ARCHIVE_PREFIX}${position}`, bytes };
|
|
5354
|
+
position += 1;
|
|
5355
|
+
}
|
|
5356
|
+
}
|
|
5357
|
+
|
|
5358
|
+
// src/persistence/archive/validate.ts
|
|
5359
|
+
function requireDimension(key, found, expected) {
|
|
5360
|
+
if (expected === found) return;
|
|
5361
|
+
throw new NarsilError(
|
|
5362
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5363
|
+
`The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
|
|
5364
|
+
{ key, found, expected }
|
|
5365
|
+
);
|
|
5366
|
+
}
|
|
5367
|
+
function requireSchemaOfManifest(key, schema, manifest) {
|
|
5368
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
5369
|
+
const listed = manifest.schema[field];
|
|
5370
|
+
if (listed === void 0 || listed === type) continue;
|
|
5371
|
+
throw new NarsilError(
|
|
5372
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5373
|
+
`The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
|
|
5374
|
+
{ key, field }
|
|
5375
|
+
);
|
|
5376
|
+
}
|
|
5377
|
+
}
|
|
5378
|
+
async function validateStagedCheckpoint(store, manifest) {
|
|
5379
|
+
for (const partition of manifest.partitions) {
|
|
5380
|
+
for (const ref of partition.segments) {
|
|
5381
|
+
const { directory } = store;
|
|
5382
|
+
if (directory !== void 0) {
|
|
5383
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
5384
|
+
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
5385
|
+
continue;
|
|
5386
|
+
}
|
|
5387
|
+
}
|
|
5388
|
+
const contents = await readSegmentFile(store, ref);
|
|
5389
|
+
requireSchemaOfManifest(ref.key, contents.schema, manifest);
|
|
5390
|
+
decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
5391
|
+
}
|
|
5392
|
+
}
|
|
5393
|
+
for (const field of manifest.vectors) {
|
|
5394
|
+
const fieldType = manifest.schema[field.fieldPath];
|
|
5395
|
+
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
5396
|
+
for (const file of field.files) {
|
|
5397
|
+
const { payload } = await readVectorFile(store, file, field.fieldPath);
|
|
5398
|
+
requireDimension(file.key, payload.dimension, dimension);
|
|
5399
|
+
}
|
|
5400
|
+
await readGraphParts(store, field);
|
|
5401
|
+
}
|
|
5402
|
+
}
|
|
5403
|
+
|
|
5404
|
+
// src/engine/snapshot/restore-setup.ts
|
|
5405
|
+
function prepareRestoredIndex(configuration, deps, reject) {
|
|
5406
|
+
const configFields = restoredConfigFields(configuration, reject);
|
|
5407
|
+
const embedding = restoredEmbedding(configuration.embedding, reject);
|
|
5408
|
+
const languageName = configuration.language ?? "";
|
|
5409
|
+
const language = getLanguage(languageName);
|
|
5410
|
+
const schema = validateSchema(configuration.schema);
|
|
5411
|
+
validatePatternValueLimit(configFields.patternValueLimit);
|
|
5412
|
+
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
5413
|
+
validateVectorPromotion(configFields.vectorPromotion);
|
|
5414
|
+
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
5415
|
+
validateTextStorage(configFields.textStorage, deps.filesystemDurability);
|
|
5416
|
+
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
5417
|
+
validateRequiredFieldsInSchema(configFields.required, schema);
|
|
5418
|
+
}
|
|
5419
|
+
const config = {
|
|
5420
|
+
schema,
|
|
5421
|
+
language: languageName,
|
|
5422
|
+
...configFields,
|
|
5423
|
+
...embedding !== void 0 ? {
|
|
5424
|
+
embedding: {
|
|
5425
|
+
fields: embedding.fields,
|
|
5426
|
+
...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
|
|
5427
|
+
}
|
|
5428
|
+
} : {}
|
|
5429
|
+
};
|
|
5430
|
+
resolveIndexAnalysis(config);
|
|
5431
|
+
const adapterName = embedding?.adapter ?? null;
|
|
5432
|
+
let embeddingAdapter = null;
|
|
5433
|
+
if (embedding !== void 0) {
|
|
5434
|
+
const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
|
|
5435
|
+
if (candidate) {
|
|
5436
|
+
validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
|
|
5437
|
+
embeddingAdapter = candidate;
|
|
5438
|
+
}
|
|
5439
|
+
}
|
|
5440
|
+
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
5441
|
+
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
5442
|
+
const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
|
|
5443
|
+
return {
|
|
5444
|
+
config,
|
|
5445
|
+
language,
|
|
5446
|
+
embeddingAdapter,
|
|
5447
|
+
adapterName,
|
|
5448
|
+
storedRevision,
|
|
5449
|
+
analysisGeneration,
|
|
5450
|
+
storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
|
|
5451
|
+
synonyms,
|
|
5452
|
+
querySuggestions: storedStateOf(configuration)
|
|
5453
|
+
};
|
|
5454
|
+
}
|
|
5455
|
+
function restoredSynonyms(raw, config, language, reject) {
|
|
5456
|
+
if (raw === void 0 || raw === null) return null;
|
|
5457
|
+
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
5458
|
+
}
|
|
5459
|
+
async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
5460
|
+
const loadingSchemaCode = loadSchemaCodeFor(prepared.config.schema);
|
|
5461
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
5462
|
+
const detached = createDetachedIndex(
|
|
5463
|
+
indexName,
|
|
5464
|
+
prepared.config,
|
|
5465
|
+
prepared.language,
|
|
5466
|
+
prepared.synonyms?.lookup ?? null,
|
|
5467
|
+
deps.executor
|
|
5468
|
+
);
|
|
5469
|
+
while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
|
|
5470
|
+
return detached;
|
|
5471
|
+
}
|
|
5472
|
+
function restoredRegistryEntry(prepared, manager, deps) {
|
|
5473
|
+
return {
|
|
5474
|
+
config: prepared.config,
|
|
5475
|
+
language: prepared.language,
|
|
5476
|
+
embeddingAdapter: prepared.embeddingAdapter,
|
|
5477
|
+
embeddingAdapterName: prepared.adapterName,
|
|
5478
|
+
vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
|
|
5479
|
+
indexUuid: null,
|
|
5480
|
+
heldPartitions: null,
|
|
5481
|
+
documentCount: manager.countDocuments(),
|
|
5482
|
+
partitionCount: manager.partitionCount,
|
|
5483
|
+
analysisRevision: prepared.storedRevision,
|
|
5484
|
+
arrayPositionGap: prepared.storedArrayPositionGap,
|
|
5485
|
+
synonyms: prepared.synonyms?.encoded ?? null,
|
|
5486
|
+
analysisGeneration: prepared.analysisGeneration,
|
|
5487
|
+
pendingAnalysis: null,
|
|
5488
|
+
durableAnalysis: null,
|
|
5489
|
+
sync: null,
|
|
5490
|
+
querySuggestions: prepared.querySuggestions
|
|
5491
|
+
};
|
|
5492
|
+
}
|
|
5493
|
+
|
|
5494
|
+
// src/engine/snapshot/replace.ts
|
|
5495
|
+
function releaseTimedOut(indexName, timeoutMs) {
|
|
5496
|
+
return new NarsilError(
|
|
5497
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
5498
|
+
`The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
|
|
5499
|
+
{ indexName, timeoutMs }
|
|
5500
|
+
);
|
|
5501
|
+
}
|
|
5502
|
+
async function withinLimit(indexName, release, work, deps) {
|
|
5503
|
+
release.settled = work.catch(() => {
|
|
5504
|
+
release.reloads = true;
|
|
5505
|
+
});
|
|
5506
|
+
let timer;
|
|
5507
|
+
const limit = new Promise((_, reject) => {
|
|
5508
|
+
timer = setTimeout(() => {
|
|
5509
|
+
release.timedOut = true;
|
|
5510
|
+
reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
|
|
5511
|
+
}, deps.releaseTimeoutMs);
|
|
5512
|
+
});
|
|
5513
|
+
try {
|
|
5514
|
+
await Promise.race([work, limit]);
|
|
5515
|
+
} finally {
|
|
5516
|
+
clearTimeout(timer);
|
|
5517
|
+
}
|
|
5518
|
+
}
|
|
5519
|
+
function vectorIndexesOf2(indexName, deps) {
|
|
5520
|
+
const manager = deps.executor.getManager(indexName);
|
|
5521
|
+
return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
|
|
5522
|
+
}
|
|
5523
|
+
async function holdVectorCopies(vectorIndexes, release) {
|
|
5524
|
+
for (const vectorIndex of vectorIndexes) {
|
|
5525
|
+
if (release.held.includes(vectorIndex)) continue;
|
|
5526
|
+
release.held.push(vectorIndex);
|
|
5527
|
+
await vectorIndex.holdWorkerCopies();
|
|
5528
|
+
}
|
|
5529
|
+
}
|
|
5530
|
+
async function releaseCopies(indexName, release, deps) {
|
|
5531
|
+
release.reloads = await deps.copies.releaseIndexCopies(indexName);
|
|
5532
|
+
await holdVectorCopies(vectorIndexesOf2(indexName, deps), release);
|
|
5533
|
+
}
|
|
5534
|
+
async function endRelease(indexName, release, deps) {
|
|
5535
|
+
await release.settled;
|
|
5536
|
+
for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
|
|
5537
|
+
await deps.copies.endCopyRelease(indexName, release.reloads);
|
|
5538
|
+
}
|
|
5539
|
+
function persistedCheckpoint(indexName, partitionCount, deps) {
|
|
5540
|
+
const durable = deps.durability?.manager;
|
|
5541
|
+
if (durable === void 0) return [];
|
|
5542
|
+
return Array.from({ length: partitionCount }, (_, partitionId) => ({
|
|
5543
|
+
partitionId,
|
|
5544
|
+
lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
|
|
5545
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
5546
|
+
}));
|
|
5547
|
+
}
|
|
5548
|
+
async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
5549
|
+
const durable = deps.durability?.manager;
|
|
5550
|
+
const staging = restored.staging;
|
|
5551
|
+
if (durable === void 0 || staging === null) return;
|
|
5552
|
+
const previous = persistedCheckpoint(indexName, previousPartitions, deps);
|
|
5553
|
+
if (previousPartitions > 0) await durable.unloadIndex(indexName);
|
|
5554
|
+
const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
|
|
5555
|
+
try {
|
|
5556
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
5557
|
+
await staging.commit();
|
|
5558
|
+
} catch (error) {
|
|
5559
|
+
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
|
|
5560
|
+
throw error;
|
|
5561
|
+
}
|
|
5562
|
+
await staging.finish();
|
|
5563
|
+
const relocate = await staging.relocation().catch(() => null);
|
|
5564
|
+
if (relocate === null) return;
|
|
5565
|
+
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
5566
|
+
const { manager } = restored.detached;
|
|
5567
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
5568
|
+
const partition = manager.getPartition(partitionId);
|
|
5569
|
+
if (!isCompositePartition(partition)) continue;
|
|
5570
|
+
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
5571
|
+
}
|
|
5572
|
+
}
|
|
5573
|
+
function activate(indexName, restored, entry, deps) {
|
|
5574
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
5575
|
+
deps.indexRegistry.set(indexName, entry);
|
|
5576
|
+
restored.detached.attach();
|
|
5577
|
+
deps.clearAnalysisStale(indexName);
|
|
5578
|
+
const { prepared } = restored;
|
|
5579
|
+
if (prepared.storedRevision !== prepared.language.revision || arrayPositionsAreStale(prepared.config, prepared.storedArrayPositionGap)) {
|
|
5580
|
+
deps.markAnalysisStale({
|
|
5581
|
+
indexName,
|
|
5582
|
+
language: prepared.language.name,
|
|
5583
|
+
storedRevision: prepared.storedRevision,
|
|
5584
|
+
currentRevision: prepared.language.revision
|
|
5585
|
+
});
|
|
5586
|
+
}
|
|
5587
|
+
if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
|
|
5588
|
+
deps.forgetWatermark(indexName);
|
|
5589
|
+
}
|
|
5590
|
+
async function replaceExisting(indexName, restored, entry, deps) {
|
|
5591
|
+
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
5592
|
+
try {
|
|
5593
|
+
await deps.abandonAnalysisChange(indexName);
|
|
5594
|
+
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
5595
|
+
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
5596
|
+
await deps.abandonAnalysisRebuild(indexName);
|
|
5597
|
+
const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
|
|
5598
|
+
const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf2(indexName, deps);
|
|
5599
|
+
await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
|
|
5600
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
5601
|
+
await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
|
|
5602
|
+
if (previous === void 0) activate(indexName, restored, entry, deps);
|
|
5603
|
+
else await previous.publication.write(() => activate(indexName, restored, entry, deps));
|
|
5604
|
+
await deps.querySuggestions?.afterActivate(entry);
|
|
5605
|
+
});
|
|
5606
|
+
} finally {
|
|
5607
|
+
const ending = endRelease(indexName, release, deps).catch(() => void 0);
|
|
5608
|
+
if (!release.timedOut) await ending;
|
|
5609
|
+
}
|
|
5610
|
+
}
|
|
5611
|
+
async function createFresh(indexName, restored, entry, deps) {
|
|
5612
|
+
deps.copies.desyncIndex(indexName);
|
|
5613
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
5614
|
+
deps.indexRegistry.set(indexName, entry);
|
|
5615
|
+
try {
|
|
5616
|
+
await deps.indexState.registerReplacing(indexName, async () => {
|
|
5617
|
+
await commitToStorage(indexName, restored, 0, deps);
|
|
5618
|
+
activate(indexName, restored, entry, deps);
|
|
5619
|
+
await deps.querySuggestions?.afterActivate(entry);
|
|
5620
|
+
});
|
|
5621
|
+
} catch (error) {
|
|
5622
|
+
if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
|
|
5623
|
+
throw error;
|
|
5624
|
+
} finally {
|
|
5625
|
+
await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
|
|
5626
|
+
}
|
|
5627
|
+
}
|
|
5628
|
+
async function stageMetadata(indexName, staging, entry) {
|
|
5629
|
+
const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
|
|
5630
|
+
await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
|
|
5631
|
+
}
|
|
5632
|
+
async function installRestoredIndex(indexName, restored, deps, history = null) {
|
|
5633
|
+
const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
|
|
5634
|
+
const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
|
|
5635
|
+
if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
|
|
5636
|
+
let replaced;
|
|
5637
|
+
try {
|
|
5638
|
+
if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
|
|
5639
|
+
await deps.querySuggestions?.beforeInstall(entry, deps.indexRegistry.get(indexName));
|
|
5640
|
+
if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
|
|
5641
|
+
replaced = deps.indexRegistry.get(indexName);
|
|
5642
|
+
if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
|
|
5643
|
+
else await createFresh(indexName, restored, entry, deps);
|
|
5644
|
+
} catch (error) {
|
|
5645
|
+
if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
|
|
5646
|
+
else deps.executor.discardDetachedIndex(restored.detached);
|
|
5647
|
+
throw error;
|
|
5648
|
+
}
|
|
5649
|
+
await restored.staging?.removeRetired().catch(() => void 0);
|
|
5650
|
+
const durable = deps.durability?.manager;
|
|
5651
|
+
if (restored.staging === null && durable !== void 0) {
|
|
5652
|
+
await durable.persistMetadata(indexName);
|
|
5653
|
+
await durable.checkpoint(indexName);
|
|
5654
|
+
}
|
|
5655
|
+
if (previousSync !== null) {
|
|
5656
|
+
const storage = durable?.syncStorage?.();
|
|
5657
|
+
if (storage !== void 0)
|
|
5658
|
+
await oneSyncChangeAtATime(deps.indexRegistry, indexName, () => storage.release(indexName));
|
|
5659
|
+
emitSyncChange(deps.indexRegistry, { kind: "epoch", indexName });
|
|
5660
|
+
}
|
|
5661
|
+
await deps.querySuggestions?.afterInstall(entry, history);
|
|
5662
|
+
if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
|
|
5663
|
+
await deps.announceIndexCreated(indexName, restored.prepared.config);
|
|
5664
|
+
for (const vectorIndex of restored.detached.manager.getVectorIndexes().values()) {
|
|
5665
|
+
vectorIndex.scheduleBuild();
|
|
5666
|
+
vectorIndex.startShuffledRebuildWhenDue();
|
|
5667
|
+
}
|
|
5668
|
+
}
|
|
5669
|
+
|
|
5670
|
+
// src/engine/snapshot/archive-restore.ts
|
|
5671
|
+
function rejectConfiguration(message) {
|
|
5672
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
|
|
5673
|
+
}
|
|
5674
|
+
function renamed(manifest, sourceName, indexName) {
|
|
5675
|
+
const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
|
|
5676
|
+
return {
|
|
5677
|
+
...manifest,
|
|
5678
|
+
partitions: manifest.partitions.map((partition) => ({
|
|
5679
|
+
...partition,
|
|
5680
|
+
segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
|
|
5681
|
+
})),
|
|
5682
|
+
vectors: manifest.vectors.map((field) => ({
|
|
5683
|
+
...field,
|
|
5684
|
+
files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
|
|
5685
|
+
graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
|
|
5686
|
+
}))
|
|
5687
|
+
};
|
|
5688
|
+
}
|
|
5689
|
+
function requireSameSchema(prepared, manifest) {
|
|
5690
|
+
const configured = flattenSchema(prepared.config.schema);
|
|
5691
|
+
const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
|
|
5692
|
+
for (const field of keys) {
|
|
5693
|
+
if (configured[field] === manifest.schema[field]) continue;
|
|
5694
|
+
throw new NarsilError(
|
|
5695
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5696
|
+
`The archive's configuration gives field "${field}" another type than its manifest does`,
|
|
5697
|
+
{ field }
|
|
5698
|
+
);
|
|
5699
|
+
}
|
|
5700
|
+
}
|
|
5701
|
+
function partitionCountOf(manifest) {
|
|
5702
|
+
let count = 1;
|
|
5703
|
+
for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
|
|
5704
|
+
return count;
|
|
5705
|
+
}
|
|
5706
|
+
function missingPart(what) {
|
|
5707
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
|
|
5708
|
+
}
|
|
5709
|
+
function keepsTextOnDisk(prepared, staging) {
|
|
5710
|
+
return prepared.config.textStorage === "disk" && staging.directory !== void 0;
|
|
5711
|
+
}
|
|
5712
|
+
async function stageArchive(indexName, reader, staging, deps, staged) {
|
|
5713
|
+
await readSnapshotArchive(reader, {
|
|
5714
|
+
history: async (chunks) => {
|
|
5715
|
+
if (staged.history === void 0) {
|
|
5716
|
+
staged.history = await deps.querySuggestions?.historySink(staged.prepared?.querySuggestions ?? null) ?? null;
|
|
5717
|
+
}
|
|
5718
|
+
await staged.history?.add(chunks);
|
|
5719
|
+
},
|
|
5720
|
+
config: async (bytes) => {
|
|
5721
|
+
staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
|
|
5722
|
+
},
|
|
5723
|
+
manifest: async (manifest2, sourceName) => {
|
|
5724
|
+
if (staged.prepared === null) missingPart("configuration");
|
|
5725
|
+
requireSameSchema(staged.prepared, manifest2);
|
|
5726
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
5727
|
+
const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
|
|
5728
|
+
requireRestoreMemory(
|
|
5729
|
+
indexName,
|
|
5730
|
+
archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
|
|
5731
|
+
replacesIndex
|
|
5732
|
+
);
|
|
5733
|
+
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
5734
|
+
},
|
|
5735
|
+
file: async (entryName, _ref, chunks) => {
|
|
5736
|
+
if (staged.manifest === null) missingPart("manifest");
|
|
5737
|
+
await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
|
|
5738
|
+
}
|
|
5739
|
+
});
|
|
5740
|
+
const { prepared, manifest } = staged;
|
|
5741
|
+
if (prepared === null) missingPart("configuration");
|
|
5742
|
+
if (manifest === null) missingPart("manifest");
|
|
5743
|
+
const { directory } = staging;
|
|
5744
|
+
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
5745
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
5746
|
+
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
5747
|
+
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
5748
|
+
}
|
|
5749
|
+
await validateStagedCheckpoint(staging, manifest);
|
|
5750
|
+
return { prepared, manifest };
|
|
5751
|
+
}
|
|
5752
|
+
async function loadStagedIndex(indexName, staged, staging, store, deps) {
|
|
5753
|
+
const { prepared, manifest } = staged;
|
|
5754
|
+
await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
|
|
5755
|
+
const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
|
|
5756
|
+
try {
|
|
5757
|
+
await loadSegmentedCheckpoint(
|
|
5758
|
+
store,
|
|
5759
|
+
indexName,
|
|
5760
|
+
manifest,
|
|
5761
|
+
{ manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
|
|
5762
|
+
true
|
|
5763
|
+
);
|
|
5764
|
+
} catch (error) {
|
|
5765
|
+
deps.executor.discardDetachedIndex(detached);
|
|
5766
|
+
throw error;
|
|
5767
|
+
}
|
|
5768
|
+
return { prepared, detached, staging, checkpoint: manifest.checkpoint };
|
|
5769
|
+
}
|
|
5770
|
+
async function restoreArchive(indexName, reader, deps) {
|
|
5771
|
+
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
5772
|
+
const store = staging?.store ?? memoryCheckpointStore();
|
|
5773
|
+
const archive = { prepared: null, manifest: null, history: void 0 };
|
|
5774
|
+
try {
|
|
5775
|
+
const staged = await stageArchive(indexName, reader, store, deps, archive);
|
|
5776
|
+
const restored = await loadStagedIndex(indexName, staged, staging, store, deps);
|
|
5777
|
+
await installRestoredIndex(indexName, restored, deps, archive.history ?? null);
|
|
5778
|
+
} catch (error) {
|
|
5779
|
+
await archive.history?.abandon().catch(() => void 0);
|
|
5780
|
+
throw error;
|
|
5781
|
+
} finally {
|
|
5782
|
+
await staging?.discard().catch(() => void 0);
|
|
5783
|
+
await reader.close();
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
|
|
5787
|
+
// src/engine/snapshot/earlier-restore.ts
|
|
5788
|
+
function invalidSnapshot(message, details) {
|
|
5789
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
|
|
5790
|
+
}
|
|
5791
|
+
async function decodeEarlierSnapshot(data) {
|
|
5792
|
+
const payloadBytes = await unpackIndexSnapshotEnvelope(data);
|
|
5793
|
+
let decoded;
|
|
5794
|
+
try {
|
|
5795
|
+
decoded = decode(payloadBytes);
|
|
5796
|
+
} catch (err) {
|
|
5797
|
+
invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
|
|
5798
|
+
bytes: data.length
|
|
5799
|
+
});
|
|
5800
|
+
}
|
|
5801
|
+
if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
|
|
5802
|
+
invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
|
|
5803
|
+
}
|
|
5804
|
+
const envelope = decoded;
|
|
5805
|
+
if (envelope.version !== 1 && envelope.version !== 2) {
|
|
5806
|
+
invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
|
|
5807
|
+
version: envelope.version
|
|
5808
|
+
});
|
|
5809
|
+
}
|
|
5810
|
+
if (!envelope.schema || typeof envelope.schema !== "object")
|
|
5811
|
+
invalidSnapshot("Invalid snapshot: missing or invalid schema");
|
|
5812
|
+
if (!envelope.language || typeof envelope.language !== "string") {
|
|
5813
|
+
invalidSnapshot("Invalid snapshot: missing or invalid language");
|
|
5814
|
+
}
|
|
5815
|
+
if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
|
|
5816
|
+
return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
|
|
5817
|
+
}
|
|
5818
|
+
function vectorsOfSnapshot(envelope) {
|
|
5819
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
5820
|
+
for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
|
|
5821
|
+
for (const part of decodeVectorIndexParts(rawParts)) {
|
|
5822
|
+
const values = bytesToVectors(part.vectors);
|
|
5823
|
+
for (let i = 0; i < part.docIds.length; i++) {
|
|
5824
|
+
let fields = byDocument.get(part.docIds[i]);
|
|
5825
|
+
if (fields === void 0) {
|
|
5826
|
+
fields = /* @__PURE__ */ new Map();
|
|
5827
|
+
byDocument.set(part.docIds[i], fields);
|
|
5828
|
+
}
|
|
5829
|
+
fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
|
|
5830
|
+
}
|
|
5831
|
+
}
|
|
5832
|
+
}
|
|
5833
|
+
return byDocument;
|
|
5834
|
+
}
|
|
5835
|
+
function footprintOf(envelope, prepared, replacesIndex) {
|
|
5836
|
+
let partitionBytes = 0;
|
|
5837
|
+
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
5838
|
+
const vectorCounts = /* @__PURE__ */ new Map();
|
|
5839
|
+
for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
|
|
5840
|
+
let count = 0;
|
|
5841
|
+
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
5842
|
+
vectorCounts.set(fieldPath, count);
|
|
5843
|
+
}
|
|
5844
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
|
|
5845
|
+
}
|
|
5846
|
+
function documentsOfPartition(envelope, payload) {
|
|
5847
|
+
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
5848
|
+
}
|
|
5849
|
+
function insertRestoredDocument(manager, docId, fields, vectors) {
|
|
5850
|
+
manager.insert(docId, fields, { skipClone: true });
|
|
5851
|
+
if (vectors === void 0) return;
|
|
5852
|
+
try {
|
|
5853
|
+
insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
5854
|
+
} catch (err) {
|
|
5855
|
+
manager.remove(docId);
|
|
5856
|
+
throw err;
|
|
5857
|
+
}
|
|
5858
|
+
}
|
|
5859
|
+
function insertSnapshotDocuments(envelope, manager) {
|
|
5860
|
+
const vectors = vectorsOfSnapshot(envelope);
|
|
5861
|
+
for (const payload of envelope.partitions ?? []) {
|
|
5862
|
+
for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
|
|
5863
|
+
insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
|
|
5864
|
+
}
|
|
5865
|
+
}
|
|
5866
|
+
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
|
|
5867
|
+
}
|
|
5868
|
+
async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
5869
|
+
const entry = restoredRegistryEntry(prepared, built.manager, deps);
|
|
5870
|
+
const archive = await buildArchiveFromMemory({
|
|
5871
|
+
indexName,
|
|
5872
|
+
metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
|
|
5873
|
+
manager: built.manager,
|
|
5874
|
+
vectorIndexes: built.manager.getVectorIndexes(),
|
|
5875
|
+
appliedSeqNo: () => 0,
|
|
5876
|
+
partitionId: null
|
|
5877
|
+
});
|
|
5878
|
+
try {
|
|
5879
|
+
for (const ref of manifestFileRefs(archive.manifest))
|
|
5880
|
+
await staging.store.write(ref.key, [await archive.readFile(ref)]);
|
|
5881
|
+
} finally {
|
|
5882
|
+
archive.release();
|
|
5883
|
+
}
|
|
5884
|
+
await validateStagedCheckpoint(staging.store, archive.manifest);
|
|
5885
|
+
return archive.manifest;
|
|
5886
|
+
}
|
|
5887
|
+
async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
|
|
5888
|
+
const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
|
|
5889
|
+
let manifest;
|
|
5890
|
+
try {
|
|
5891
|
+
insertSnapshotDocuments(envelope, built.manager);
|
|
5892
|
+
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
|
|
5893
|
+
manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
|
|
5894
|
+
} catch (err) {
|
|
5895
|
+
deps.executor.discardDetachedIndex(built);
|
|
5896
|
+
throw err;
|
|
5897
|
+
}
|
|
5898
|
+
deps.executor.discardDetachedIndex(built);
|
|
5899
|
+
return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
|
|
5900
|
+
}
|
|
5901
|
+
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
5902
|
+
const envelope = await decodeEarlierSnapshot(data);
|
|
5903
|
+
const prepared = prepareRestoredIndex(envelope, deps);
|
|
5904
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
5905
|
+
requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
|
|
5906
|
+
const analysedNow = {
|
|
5907
|
+
...prepared,
|
|
5908
|
+
storedRevision: prepared.language.revision,
|
|
5909
|
+
storedArrayPositionGap: ARRAY_POSITION_GAP
|
|
5910
|
+
};
|
|
5911
|
+
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
5912
|
+
try {
|
|
5913
|
+
await installRestoredIndex(
|
|
5914
|
+
indexName,
|
|
5915
|
+
await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
|
|
5916
|
+
deps
|
|
5917
|
+
);
|
|
5918
|
+
} finally {
|
|
5919
|
+
await staging?.discard().catch(() => void 0);
|
|
5920
|
+
}
|
|
5921
|
+
}
|
|
5922
|
+
|
|
5923
|
+
// src/engine/snapshot/index.ts
|
|
5924
|
+
var NRSL_MAGIC_BYTES = 4;
|
|
5925
|
+
async function checkpointForArchive(deps) {
|
|
5926
|
+
const durable = deps.durability?.manager;
|
|
5927
|
+
if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
|
|
5928
|
+
return buildArchiveFromMemory({
|
|
5929
|
+
indexName: deps.indexName,
|
|
5930
|
+
metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
|
|
5931
|
+
manager: deps.manager,
|
|
5932
|
+
vectorIndexes: deps.manager.getVectorIndexes(),
|
|
5933
|
+
appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
|
|
5934
|
+
partitionId: null
|
|
5935
|
+
});
|
|
5936
|
+
}
|
|
5937
|
+
async function* queryHistoryEntries(core, indexName) {
|
|
5938
|
+
yield* historyArchiveEntries(core, indexName);
|
|
5939
|
+
}
|
|
5940
|
+
async function createSnapshot(deps, coreWithAddedHistory) {
|
|
5941
|
+
const trailingEntries = coreWithAddedHistory === null ? void 0 : () => queryHistoryEntries(coreWithAddedHistory, deps.indexName);
|
|
5942
|
+
for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
|
|
5943
|
+
const checkpoint = await checkpointForArchive(deps);
|
|
5944
|
+
try {
|
|
5945
|
+
if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
|
|
5946
|
+
const config = await encodeIndexConfiguration(deps.entry);
|
|
5947
|
+
return archiveStream({
|
|
5948
|
+
...checkpoint,
|
|
5949
|
+
config,
|
|
5950
|
+
sourceIndexName: deps.indexName,
|
|
5951
|
+
...trailingEntries !== void 0 ? { trailingEntries } : {}
|
|
5952
|
+
});
|
|
5953
|
+
}
|
|
5954
|
+
} catch (error) {
|
|
5955
|
+
checkpoint.release();
|
|
5956
|
+
throw error;
|
|
5957
|
+
}
|
|
5958
|
+
checkpoint.release();
|
|
5959
|
+
}
|
|
5960
|
+
throw new NarsilError(
|
|
5961
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
5962
|
+
`The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
|
|
5963
|
+
{ indexName: deps.indexName }
|
|
5964
|
+
);
|
|
5965
|
+
}
|
|
5966
|
+
function requireSnapshotSource(data) {
|
|
5967
|
+
if (data instanceof Uint8Array) return data;
|
|
5968
|
+
if (data !== null && typeof data === "object") {
|
|
5969
|
+
if (typeof data.getReader === "function") return data;
|
|
5970
|
+
if (typeof data[Symbol.asyncIterator] === "function") {
|
|
5971
|
+
return data;
|
|
5972
|
+
}
|
|
5973
|
+
}
|
|
5974
|
+
throw new NarsilError(
|
|
5975
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5976
|
+
"Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
|
|
5977
|
+
);
|
|
5978
|
+
}
|
|
5979
|
+
async function readEarlierSnapshotFile(reader) {
|
|
5980
|
+
const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
|
|
5981
|
+
if (length > MAX_FILE_BYTES) {
|
|
5982
|
+
throw new NarsilError(
|
|
5983
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5984
|
+
`The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
|
|
5985
|
+
{ bytes: length }
|
|
5986
|
+
);
|
|
5987
|
+
}
|
|
5988
|
+
const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
|
|
5989
|
+
if (bytes.byteLength < length) {
|
|
5990
|
+
throw new NarsilError(
|
|
5991
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5992
|
+
"The snapshot ends before the payload its header declares"
|
|
5993
|
+
);
|
|
5994
|
+
}
|
|
5995
|
+
const trailing = await reader.readExactly(1);
|
|
5996
|
+
if (trailing !== null && trailing.byteLength > 0) {
|
|
5997
|
+
throw new NarsilError(
|
|
5998
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5999
|
+
"The snapshot holds bytes after the payload its header declares"
|
|
6000
|
+
);
|
|
6001
|
+
}
|
|
6002
|
+
return bytes;
|
|
6003
|
+
}
|
|
6004
|
+
async function restoreFromSnapshot(indexName, data, deps) {
|
|
6005
|
+
const reader = createByteReader(requireSnapshotSource(data));
|
|
6006
|
+
const head = await reader.peek(NRSL_MAGIC_BYTES);
|
|
6007
|
+
if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
|
|
6008
|
+
let bytes;
|
|
6009
|
+
try {
|
|
6010
|
+
bytes = await readEarlierSnapshotFile(reader);
|
|
6011
|
+
} finally {
|
|
6012
|
+
await reader.close();
|
|
6013
|
+
}
|
|
6014
|
+
await restoreEarlierSnapshot(indexName, bytes, deps);
|
|
6015
|
+
return;
|
|
6016
|
+
}
|
|
6017
|
+
if (head.byteLength === 0) {
|
|
6018
|
+
await reader.close();
|
|
6019
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
|
|
6020
|
+
}
|
|
6021
|
+
await restoreArchive(indexName, reader, deps);
|
|
6022
|
+
}
|
|
6023
|
+
|
|
6024
|
+
// src/engine/sync/status.ts
|
|
6025
|
+
function syncStatusOf(sync) {
|
|
6026
|
+
if (sync === null) return { enabled: false, active: false, epoch: null };
|
|
6027
|
+
return {
|
|
6028
|
+
enabled: true,
|
|
6029
|
+
...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
|
|
6030
|
+
...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {},
|
|
6031
|
+
active: sync.active,
|
|
6032
|
+
epoch: sync.epoch
|
|
6033
|
+
};
|
|
6034
|
+
}
|
|
6035
|
+
|
|
6036
|
+
// src/engine/sync/methods.ts
|
|
6037
|
+
async function updateSyncOf(core, indexName, setting) {
|
|
6038
|
+
core.guardShutdown();
|
|
6039
|
+
core.requireIndex(indexName);
|
|
6040
|
+
await updateIndexSync(core, indexName, setting);
|
|
6041
|
+
}
|
|
6042
|
+
function syncOf(core, indexName) {
|
|
6043
|
+
core.guardShutdown();
|
|
6044
|
+
return syncStatusOf(core.requireIndex(indexName).sync);
|
|
6045
|
+
}
|
|
6046
|
+
|
|
6047
|
+
// src/persistence/paged/rewrite-spill.ts
|
|
6048
|
+
function openRewriteSpill(storage, manager, target) {
|
|
6049
|
+
const config = target?.config ?? manager.config;
|
|
6050
|
+
return createRewriteSpill(storage, manager.indexName, {
|
|
6051
|
+
schema: flattenSchema(config.schema),
|
|
6052
|
+
trackPositions: config.trackPositions ?? true,
|
|
6053
|
+
partialWordAnalysis: target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis,
|
|
6054
|
+
textAnalysis: {
|
|
6055
|
+
language: target?.language ?? manager.language,
|
|
6056
|
+
options: resolvePartitionInsertOptions(config, target?.analysis ?? manager.analysis)
|
|
6057
|
+
}
|
|
6058
|
+
});
|
|
6059
|
+
}
|
|
6060
|
+
function createRewriteSpill(storage, indexName, analysis) {
|
|
6061
|
+
const { directory, writeRecords } = storage;
|
|
6062
|
+
const prefix = `${indexName}/rebuilt/${generateId()}/`;
|
|
6063
|
+
const opened = [];
|
|
6064
|
+
let nextFile = 0;
|
|
6065
|
+
async function removeFiles() {
|
|
6066
|
+
for (const key of await directory.list(prefix)) await directory.remove(key);
|
|
6067
|
+
}
|
|
6068
|
+
function freezeLive(partition) {
|
|
6069
|
+
if (!isCompositePartition(partition)) return null;
|
|
6070
|
+
return partition.freezeLiveTail((payload, documents) => {
|
|
6071
|
+
const snapshot = freezeSegmentShared(payload, documents, void 0, analysis.partialWordAnalysis);
|
|
6072
|
+
return snapshot === null ? createFrozenSegment(payload, documents) : createSharedFrozenSegment(snapshot);
|
|
6073
|
+
});
|
|
6074
|
+
}
|
|
6075
|
+
return {
|
|
6076
|
+
newPartition(partitionId) {
|
|
6077
|
+
const live = createPartitionIndex(partitionId, analysis.trackPositions);
|
|
6078
|
+
return createCompositePartition(partitionId, analysis.trackPositions, live, analysis.textAnalysis);
|
|
6079
|
+
},
|
|
6080
|
+
async spillIfFull(partition, partitionId) {
|
|
6081
|
+
if (!isCompositePartition(partition)) return;
|
|
6082
|
+
if (partition.live.estimateMemoryBytes() < REWRITE_SPILL_BYTES) return;
|
|
6083
|
+
const frozen = freezeLive(partition);
|
|
6084
|
+
if (frozen === null) return;
|
|
6085
|
+
const key = `${prefix}${nextFile}`;
|
|
6086
|
+
nextFile += 1;
|
|
6087
|
+
const written = await writePagedSegment({
|
|
6088
|
+
directory,
|
|
6089
|
+
key,
|
|
6090
|
+
schema: analysis.schema,
|
|
6091
|
+
tombstones: [],
|
|
6092
|
+
selections: [{ segment: frozen, keep: (ordinal) => !frozen.isTombstoned(ordinal) }],
|
|
6093
|
+
partialWordAnalysis: analysis.partialWordAnalysis,
|
|
6094
|
+
textAnalysis: analysis.textAnalysis,
|
|
6095
|
+
...writeRecords === null ? {} : { writeRecords: writeRecords.captureSegments(indexName, partitionId, [frozen]) }
|
|
6096
|
+
});
|
|
6097
|
+
const listing = await pagedListingOf(directory, { key, bytes: written.bytes, checksum: written.checksum });
|
|
6098
|
+
const segment = createPagedFrozenSegment(openPagedSnapshot(listing));
|
|
6099
|
+
opened.push(segment);
|
|
6100
|
+
partition.swapFrozenSegments([frozen.segmentId], segment);
|
|
6101
|
+
},
|
|
6102
|
+
async abandon() {
|
|
6103
|
+
for (const segment of opened.splice(0)) segment.paged?.close();
|
|
6104
|
+
await removeFiles();
|
|
6105
|
+
},
|
|
6106
|
+
async removeUnread(manager) {
|
|
6107
|
+
for (const partition of manager.getAllPartitions()) {
|
|
6108
|
+
if (!isCompositePartition(partition)) continue;
|
|
6109
|
+
for (const segment of partition.frozenSegments()) {
|
|
6110
|
+
if (segment.paged?.key.startsWith(prefix)) return;
|
|
6111
|
+
}
|
|
6112
|
+
}
|
|
6113
|
+
for (const segment of opened.splice(0)) segment.paged?.close();
|
|
6114
|
+
await removeFiles();
|
|
6115
|
+
}
|
|
6116
|
+
};
|
|
6117
|
+
}
|
|
6118
|
+
|
|
6119
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, actionPartitionsOf, adoptSegmentsOnWorker, checkpointForArchive, closeUnusedSegments, createDurabilityFromTier, createInvalidationFromConfig, createSnapshot, documentExportStream, keysThatSegmentsStillRead, openFileVectorReads, openRewriteSpill, pagedPartitionLines, prepareDiskAdoption, readDocumentExport, reloadIndexAndCopies, restoreFromSnapshot, swapPreparedPartitions, syncOf, updateSyncOf };
|