@delali/narsil 0.3.1-next.156 → 0.3.1-next.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-3V7DNNZL.mjs → chunk-4RNSFVWI.mjs} +20997 -19899
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/browser/{durability-config-TIYKW6XX.mjs → durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{geo-index-MRH5JMIR.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/{react-native/chunks/restore-hooks-IBGIY3DR.mjs → browser/restore-queue-S4BWOGMU.mjs} +98 -36
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-SSTPBZZX.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-Da4ywVVl.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-K5QFDYST.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{browser/chunk-23XRXIBW.mjs → chunk-7MWYORQC.mjs} +10 -3
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-XAXKV65G.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-I2KPOLOZ.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-ZKP7G7TV.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-2UV4QBHW.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-25KDNIZZ.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-CRE3G7F3.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-FRBFVQFN.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-6I6AYAXS.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{browser/chunk-SRMZXJSL.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/chunk-QBVTYY4J.mjs +54 -0
- package/dist/{react-native/chunks/chunk-GXX3DQXP.mjs → chunk-R4CE4IVU.mjs} +352 -165
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-FAKFUDZZ.mjs → chunk-SHGN3QIF.mjs} +80 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{react-native/chunks/chunk-EJ33LCHV.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-6MO2XJF2.mjs → chunk-TSLV5P2J.mjs} +397 -79
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-FYOVBEMM.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-DRv7UVeY.d.ts → client-CTp8RRiK.d.ts} +16 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +70 -18
- package/dist/{config-DMwffLKn.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CoSig5Ox.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1192 -1077
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CFxPYqEk.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-HHWGMNIS.mjs → geo-index-2ZMMWWSK.mjs} +190 -92
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -15
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-YA4IAYSY.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-PXW5XAYU.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -58
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -226
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/chunk-FTQ7JNMU.mjs +898 -0
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/{chunk-TBC3XUL2.mjs → react-native/chunks/chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-KV46CT4D.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/chunk-OK3WUA6N.mjs +54 -0
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-DZ7H7D2C.mjs → chunk-TQ26BKBH.mjs} +6 -0
- package/dist/react-native/chunks/{chunk-6VBXNFLE.mjs → chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/react-native/chunks/{geo-index-PX3BIHAH.mjs → geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-UQ4OAXKU.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +127 -50
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +35 -13
- package/dist/server.mjs +90 -101
- package/dist/{browser/durability-wiring-A7BVG2DW.mjs → storage-leaf-HMIIOUYN.mjs} +2348 -902
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +120 -70
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +121 -57
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-qrJojgST.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-BZCGjfLn.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +137 -50
- package/package.json +233 -8
- package/dist/analysis-change-5BQOQ4BF.mjs +0 -55
- package/dist/analysis-change-entry-TXX6JSMX.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-435CUEPZ.mjs +0 -933
- package/dist/browser/analysis-change-entry-HZXKPPGV.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-2CFI5VVV.mjs +0 -241
- package/dist/browser/chunk-2TI4SSUP.mjs +0 -315
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3SSRJMFN.mjs +0 -100
- package/dist/browser/chunk-4NGKC4H4.mjs +0 -2635
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-6LWDPLTB.mjs +0 -177
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-7U3WVJUP.mjs +0 -150
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BHFQATSC.mjs +0 -432
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-EA63E3D2.mjs +0 -857
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-GPPOG6PT.mjs +0 -9
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-IZXVW2ZJ.mjs +0 -217
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K4Q6KX3R.mjs +0 -2528
- package/dist/browser/chunk-KZWSX6BH.mjs +0 -645
- package/dist/browser/chunk-MGI4BCXC.mjs +0 -63
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MOG4OVDK.mjs +0 -51
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NQAME3RK.mjs +0 -21
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-STOWQFT3.mjs +0 -1829
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WI4MR75E.mjs +0 -86
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-2EHCIV4K.mjs +0 -759
- package/dist/browser/engine-hooks-E4UD67JD.mjs +0 -17
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-DXYB6756.mjs +0 -19
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NIYMEA5R.mjs +0 -23
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-hooks-YPN4ZFCY.mjs +0 -114
- package/dist/browser/restore-queue-G3RQGL3Y.mjs +0 -55
- package/dist/browser/retention-ZWVMNC3Q.mjs +0 -210
- package/dist/browser/snapshot-VEQNIE6C.mjs +0 -963
- package/dist/browser/start-BM442C6R.mjs +0 -25
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-2EVMZWDE.mjs +0 -194
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3HYC4JRF.mjs +0 -50
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-46KY2PJD.mjs +0 -905
- package/dist/chunk-4KKXNJM6.mjs +0 -242
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7LXHHCXZ.mjs +0 -1397
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-BU4SAIC4.mjs +0 -196
- package/dist/chunk-BWZ3P4MU.mjs +0 -20
- package/dist/chunk-CSLGFPFQ.mjs +0 -218
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-DP5C7YFQ.mjs +0 -543
- package/dist/chunk-DWEPMFJE.mjs +0 -240
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-EFBDHH5P.mjs +0 -7081
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FNGAM5B7.mjs +0 -2639
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HFPMMRQT.mjs +0 -206
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I53G4JAU.mjs +0 -1974
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JDJBQLRO.mjs +0 -1126
- package/dist/chunk-JWRUUCWD.mjs +0 -151
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KF4EXSM4.mjs +0 -21
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-L6V6BST3.mjs +0 -1435
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MQOJIO5B.mjs +0 -316
- package/dist/chunk-NF4DT3IH.mjs +0 -85
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-P5ZI2P2W.mjs +0 -7481
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PWC4VB3Q.mjs +0 -106
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QOW2JPUI.mjs +0 -634
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RDXO6JST.mjs +0 -63
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SA5LOGO3.mjs +0 -470
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TGS5UXBE.mjs +0 -216
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-TZ3355VS.mjs +0 -121
- package/dist/chunk-TZTRKM33.mjs +0 -74
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VTTRFF2C.mjs +0 -3711
- package/dist/chunk-WX4LAO6M.mjs +0 -656
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-XXHN7J46.mjs +0 -101
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-YTWZVGCB.mjs +0 -382
- package/dist/chunk-Z2KAFCRT.mjs +0 -30
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-FTMYY3TI.mjs +0 -34
- package/dist/durability-wiring-VIF5MOSY.mjs +0 -4064
- package/dist/engine-hooks-EGUGNZYE.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-XKDPJQDP.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-FQ4JWNQJ.mjs +0 -62
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-LT5QBQ3H.mjs +0 -926
- package/dist/react-native/chunks/analysis-change-entry-SCVWK6OU.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3JWDECAZ.mjs +0 -858
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4JYO3JD2.mjs +0 -242
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +0 -85
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DNXOMPAQ.mjs +0 -191
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-G43IPIDG.mjs +0 -151
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MIP4SOR6.mjs +0 -22803
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-MUWD2H2S.mjs +0 -21
- package/dist/react-native/chunks/chunk-NB3LP33V.mjs +0 -315
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PDRWWPKV.mjs +0 -2532
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-POPDAGH6.mjs +0 -2636
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-QUKQNI3H.mjs +0 -647
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TP6KCKER.mjs +0 -63
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-VWUDIOT6.mjs +0 -218
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZLWVW734.mjs +0 -100
- package/dist/react-native/chunks/chunk-ZXZFN7FI.mjs +0 -432
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-KVZYUGRY.mjs +0 -772
- package/dist/react-native/chunks/durability-wiring-34OYO32E.mjs +0 -4942
- package/dist/react-native/chunks/engine-hooks-M7JV46NH.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-H6AQREWI.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-LBZPGOWH.mjs +0 -36
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +0 -33
- package/dist/react-native/chunks/restore-queue-ZZX54AH7.mjs +0 -55
- package/dist/react-native/chunks/retention-EW4TTDFB.mjs +0 -223
- package/dist/react-native/chunks/snapshot-REYVIN3X.mjs +0 -975
- package/dist/react-native/chunks/start-BQK6KN3R.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-FQAYCZF5.mjs +0 -207
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-hooks-CDERDEF4.mjs +0 -153
- package/dist/restore-queue-7NNSTROA.mjs +0 -55
- package/dist/retention-QSVBBNRM.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-RH4NSKIZ.mjs +0 -782
- package/dist/start-MCKCOGIW.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-O4SBJJJS.mjs +0 -50
- package/dist/write-3FK7NG3Y.mjs +0 -232
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
|
@@ -1,546 +1,201 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
import './chunk-
|
|
8
|
-
|
|
9
|
-
import './chunk-
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
import {
|
|
16
|
-
import './chunk-
|
|
17
|
-
import
|
|
18
|
-
import './chunk-
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import './chunk-
|
|
22
|
-
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import './chunk-
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
|
|
29
|
-
import './chunk-
|
|
30
|
-
import {
|
|
31
|
-
import './chunk-
|
|
32
|
-
import {
|
|
1
|
+
import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES, scheduleSyncRetention } from './chunk-FE3MRIBN.mjs';
|
|
2
|
+
export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-FE3MRIBN.mjs';
|
|
3
|
+
import { openPagedSegmentScan, countsAsLatestHolder, buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, chunksOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, bytesThatSegmentsHoldInMemory, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, createDetachedIndex, checkPagedSegmentFully, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readMetadataEnvelope, snapshotCheckpointFor, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, readSegmentManifest, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, growPartitionsTo, createMarkerWriter, metadataConfigOf, durableMetadataConfigOf } from './chunk-SXIRM4MX.mjs';
|
|
4
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedSegmentFully, countsAsLatestHolder, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, openPagedSegmentScan, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-SXIRM4MX.mjs';
|
|
5
|
+
import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-TSLV5P2J.mjs';
|
|
6
|
+
export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-TSLV5P2J.mjs';
|
|
7
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-2JRERUNZ.mjs';
|
|
8
|
+
export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-2JRERUNZ.mjs';
|
|
9
|
+
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-4CAMH7FO.mjs';
|
|
10
|
+
import { readSnapshotArchive, decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-OOO54OVC.mjs';
|
|
11
|
+
export { readSnapshotArchive } from './chunk-OOO54OVC.mjs';
|
|
12
|
+
import { createDurableDirectory } from './chunk-TW543F33.mjs';
|
|
13
|
+
import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, readSegmentFile, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-W34VJXY7.mjs';
|
|
14
|
+
export { checkPagedContents, checkPagedSelections, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-W34VJXY7.mjs';
|
|
15
|
+
import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-U6B7DUQW.mjs';
|
|
16
|
+
import './chunk-5FLRXF73.mjs';
|
|
17
|
+
import './chunk-JDDXV6UF.mjs';
|
|
18
|
+
import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-7MWYORQC.mjs';
|
|
19
|
+
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
20
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
|
|
21
|
+
import { encode } from './chunk-77AUVDQ5.mjs';
|
|
22
|
+
export { bindStorageHost } from './chunk-77AUVDQ5.mjs';
|
|
23
|
+
import { sha256 } from './chunk-LNGE7SLY.mjs';
|
|
24
|
+
import { setNestedValue } from './chunk-QBVTYY4J.mjs';
|
|
25
|
+
import { rewriteSpillOf } from './chunk-IB4BYLE3.mjs';
|
|
26
|
+
import { deserializePartition } from './chunk-2RIS76RJ.mjs';
|
|
27
|
+
import { decodeVectorIndexParts, decodeVectorFilePayload, isDead, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-LAWIJRMJ.mjs';
|
|
28
|
+
export { createHNSWIndex, osqBitsOf } from './chunk-LAWIJRMJ.mjs';
|
|
29
|
+
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
30
|
+
import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-6EZW2GQL.mjs';
|
|
31
|
+
import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createRequestId, createPartitionRouter, buildSegmentPayloads } from './chunk-TINW4BSH.mjs';
|
|
32
|
+
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
33
|
+
import './chunk-AB27FWZM.mjs';
|
|
34
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
35
|
+
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
36
|
+
import './chunk-7T4LAYT4.mjs';
|
|
37
|
+
import './chunk-JV74RKCY.mjs';
|
|
38
|
+
import { flattenSchema, parseVectorDimension, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, MAX_DOCUMENT_NESTING_DEPTH, requireValidFieldTypes } from './chunk-R4CE4IVU.mjs';
|
|
39
|
+
import './chunk-U2BKYEC7.mjs';
|
|
40
|
+
import { detectRuntime, spawnNodeWorker, resolveWorkerEntry } from './chunk-OUWZKNO3.mjs';
|
|
41
|
+
import { STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-3B775F3C.mjs';
|
|
42
|
+
import './chunk-E2F6LZR3.mjs';
|
|
43
|
+
import { MIN_M } from './chunk-ZXX5GZEX.mjs';
|
|
44
|
+
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
45
|
+
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
46
|
+
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
33
47
|
|
|
34
|
-
// src/persistence/
|
|
35
|
-
function
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EACCES", "EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
61
|
-
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
62
|
-
function errnoOf(err) {
|
|
63
|
-
return err?.code;
|
|
64
|
-
}
|
|
65
|
-
function integerOrNull(text) {
|
|
66
|
-
const value = Number.parseInt(text ?? "", 10);
|
|
67
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
68
|
-
}
|
|
69
|
-
function parseHolder(text) {
|
|
70
|
-
const [pidText, threadText, startText, nonceText] = text.split("\n");
|
|
71
|
-
const pid = integerOrNull(pidText);
|
|
72
|
-
if (pid === null || pid <= 0) return null;
|
|
73
|
-
const nonce = nonceText?.trim() ?? "";
|
|
74
|
-
return {
|
|
75
|
-
pid,
|
|
76
|
-
threadId: integerOrNull(threadText),
|
|
77
|
-
processStartMs: integerOrNull(startText),
|
|
78
|
-
nonce: nonce.length > 0 ? nonce : null
|
|
79
|
-
};
|
|
80
|
-
}
|
|
81
|
-
function formatHolder(holder) {
|
|
82
|
-
return `${holder.pid}
|
|
83
|
-
${holder.threadId ?? ""}
|
|
84
|
-
${holder.processStartMs ?? ""}
|
|
85
|
-
${holder.nonce ?? ""}
|
|
86
|
-
`;
|
|
87
|
-
}
|
|
88
|
-
async function readHolder(fs, lockPath) {
|
|
89
|
-
try {
|
|
90
|
-
return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
|
|
91
|
-
} catch (err) {
|
|
92
|
-
if (errnoOf(err) === "ENOENT") return "absent";
|
|
93
|
-
throw err;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
async function settledHolder(fs, lockPath) {
|
|
97
|
-
const first = await readHolder(fs, lockPath);
|
|
98
|
-
if (first !== "unreadable") return first;
|
|
99
|
-
await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
|
|
100
|
-
return readHolder(fs, lockPath);
|
|
101
|
-
}
|
|
102
|
-
function heldElsewhere(root, holder, lockPath) {
|
|
103
|
-
return new NarsilError(
|
|
104
|
-
ErrorCodes.CONFIG_INVALID,
|
|
105
|
-
`The durability directory "${root}" is in use by ${holder}, and only one engine may write a write-ahead log directory. Point this engine at another directory, or shut the other engine down first. Where no engine uses the directory, delete "${lockPath}"`,
|
|
106
|
-
{ directory: root }
|
|
107
|
-
);
|
|
108
|
-
}
|
|
109
|
-
function holderLabel(holder, self) {
|
|
110
|
-
if (holder.pid !== self.pid) return `process ${holder.pid}`;
|
|
111
|
-
return holder.threadId === self.threadId ? "another copy of Narsil in this thread" : "another thread of this process";
|
|
112
|
-
}
|
|
113
|
-
async function createLockFileExclusively(fs, lockPath, content) {
|
|
114
|
-
try {
|
|
115
|
-
await fs.writeFile(lockPath, content, { flag: "wx" });
|
|
116
|
-
return true;
|
|
117
|
-
} catch (err) {
|
|
118
|
-
if (errnoOf(err) === "EEXIST") return false;
|
|
119
|
-
throw err;
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
async function createLockFile(fs, lockPath, draftPath, content) {
|
|
123
|
-
await fs.writeFile(draftPath, content, { flag: "w" });
|
|
124
|
-
try {
|
|
125
|
-
await fs.link(draftPath, lockPath);
|
|
126
|
-
return true;
|
|
127
|
-
} catch (err) {
|
|
128
|
-
const code = errnoOf(err);
|
|
129
|
-
if (code === "EEXIST") return false;
|
|
130
|
-
if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
|
|
131
|
-
return createLockFileExclusively(fs, lockPath, content);
|
|
48
|
+
// src/persistence/paged/adopt.ts
|
|
49
|
+
function yieldToEventLoop() {
|
|
50
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
51
|
+
}
|
|
52
|
+
function applyNewerTombstones(prepared) {
|
|
53
|
+
for (let at = 0; at < prepared.length; at++) {
|
|
54
|
+
if (!prepared[at].isNew) continue;
|
|
55
|
+
for (let newer = at + 1; newer < prepared.length; newer++) {
|
|
56
|
+
const access = prepared[newer].segment.paged;
|
|
57
|
+
if (access === null) continue;
|
|
58
|
+
for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async function applyNewerTombstonesInTurns(prepared) {
|
|
63
|
+
let applied = 0;
|
|
64
|
+
for (let at = 0; at < prepared.length; at++) {
|
|
65
|
+
if (!prepared[at].isNew) continue;
|
|
66
|
+
for (let newer = at + 1; newer < prepared.length; newer++) {
|
|
67
|
+
const access = prepared[newer].segment.paged;
|
|
68
|
+
if (access === null) continue;
|
|
69
|
+
for (const docId of access.snapshot.tombstones) {
|
|
70
|
+
prepared[at].segment.tombstoneDocument(docId);
|
|
71
|
+
applied += 1;
|
|
72
|
+
if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
|
|
73
|
+
}
|
|
132
74
|
}
|
|
133
|
-
throw err;
|
|
134
|
-
} finally {
|
|
135
|
-
await fs.unlink(draftPath).catch(() => void 0);
|
|
136
75
|
}
|
|
137
76
|
}
|
|
138
|
-
async function
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
if (await createLockFile(fs, lockPath, draftPath, content)) return;
|
|
143
|
-
const holder = await settledHolder(fs, lockPath);
|
|
144
|
-
if (typeof holder === "object" && await holderIsLive()) {
|
|
145
|
-
throw heldElsewhere(root, holderLabel(holder, self), lockPath);
|
|
77
|
+
async function prepareDiskAdoption(directory, manager, partitions) {
|
|
78
|
+
for (const entry of partitions) {
|
|
79
|
+
for (const ref of entry.segments) {
|
|
80
|
+
if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
146
81
|
}
|
|
147
|
-
await removeStaleLock();
|
|
148
|
-
}
|
|
149
|
-
throw heldElsewhere(root, "another process", lockPath);
|
|
150
|
-
}
|
|
151
|
-
async function describeSelf() {
|
|
152
|
-
return {
|
|
153
|
-
pid: process.pid,
|
|
154
|
-
threadId: await nodeThreadId(),
|
|
155
|
-
processStartMs: Math.round(Date.now() - process.uptime() * 1e3),
|
|
156
|
-
nonce: globalThis.crypto.randomUUID()
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
async function lockDirectory(root) {
|
|
160
|
-
const fs = await getFs();
|
|
161
|
-
const pathMod = await getPath();
|
|
162
|
-
const resolvedRoot = pathMod.resolve(root);
|
|
163
|
-
const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
|
|
164
|
-
const self = await describeSelf();
|
|
165
|
-
async function holds() {
|
|
166
|
-
const holder = await readHolder(fs, lockPath);
|
|
167
|
-
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
168
|
-
}
|
|
169
|
-
async function removeOwnLockFile() {
|
|
170
|
-
if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
|
|
171
|
-
}
|
|
172
|
-
let directoryKey = null;
|
|
173
|
-
let closeDirectory = () => void 0;
|
|
174
|
-
try {
|
|
175
|
-
await fs.mkdir(resolvedRoot, { recursive: true });
|
|
176
|
-
const stat = await fs.stat(resolvedRoot, { bigint: true });
|
|
177
|
-
const key = `${stat.dev}:${stat.ino}`;
|
|
178
|
-
if (lockedDirectories.has(key)) throw heldElsewhere(root, "another engine in this process", lockPath);
|
|
179
|
-
lockedDirectories.add(key);
|
|
180
|
-
directoryKey = key;
|
|
181
|
-
await acquire(fs, root, lockPath, self);
|
|
182
|
-
closeDirectory = holdDirectoryOpen(resolvedRoot);
|
|
183
|
-
} catch (err) {
|
|
184
|
-
if (directoryKey !== null) lockedDirectories.delete(directoryKey);
|
|
185
|
-
await removeOwnLockFile();
|
|
186
|
-
if (err instanceof NarsilError) throw err;
|
|
187
|
-
throw new NarsilError(
|
|
188
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
189
|
-
`The engine cannot lock the durability directory "${root}"`,
|
|
190
|
-
{ directory: root, cause: err instanceof Error ? err.message : String(err) }
|
|
191
|
-
);
|
|
192
82
|
}
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
}
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
// src/persistence/durability/list-recursive.ts
|
|
208
|
-
async function listRecursive(dir, base, pathMod, fs) {
|
|
209
|
-
const results = [];
|
|
210
|
-
let entries;
|
|
211
|
-
try {
|
|
212
|
-
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
213
|
-
} catch (err) {
|
|
214
|
-
if (err.code === "ENOENT") {
|
|
215
|
-
return results;
|
|
83
|
+
const prepared = [];
|
|
84
|
+
const droppedKeys = [];
|
|
85
|
+
for (const entry of partitions) {
|
|
86
|
+
if (entry.partitionId >= manager.partitionCount) continue;
|
|
87
|
+
const partition = manager.getPartition(entry.partitionId);
|
|
88
|
+
const held = /* @__PURE__ */ new Map();
|
|
89
|
+
if (isCompositePartition(partition)) {
|
|
90
|
+
for (const segment of partition.frozenSegments()) {
|
|
91
|
+
if (segment.paged !== null) held.set(segment.paged.key, segment);
|
|
92
|
+
}
|
|
216
93
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
94
|
+
const list = [];
|
|
95
|
+
for (const ref of entry.segments) {
|
|
96
|
+
const existing = held.get(ref.key);
|
|
97
|
+
if (existing !== void 0) {
|
|
98
|
+
held.delete(ref.key);
|
|
99
|
+
list.push({ segment: existing, isNew: false });
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
103
|
+
list.push({ segment: createPagedFrozenSegment(snapshot), isNew: true });
|
|
225
104
|
}
|
|
105
|
+
droppedKeys.push(...held.keys());
|
|
106
|
+
await applyNewerTombstonesInTurns(list);
|
|
107
|
+
prepared.push({ partitionId: entry.partitionId, prepared: list });
|
|
226
108
|
}
|
|
227
|
-
return
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// src/persistence/durability/durable-filesystem.ts
|
|
231
|
-
function validateRelativeKey(key) {
|
|
232
|
-
if (key.length === 0) {
|
|
233
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
234
|
-
}
|
|
235
|
-
if (key.includes("\0")) {
|
|
236
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
function wrapFsyncError(err, key) {
|
|
240
|
-
throw new NarsilError(
|
|
241
|
-
ErrorCodes.PERSISTENCE_FSYNC_FAILED,
|
|
242
|
-
`fsync failed for "${key}"; the write is not acknowledged and the node must recover from the durable log`,
|
|
243
|
-
{ key, cause: err instanceof Error ? err.message : String(err) }
|
|
244
|
-
);
|
|
109
|
+
return { partitions: prepared, droppedKeys };
|
|
245
110
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
try {
|
|
255
|
-
return await fs.open(filePath, "wx+");
|
|
256
|
-
} catch (err) {
|
|
257
|
-
if (err.code !== "EEXIST") {
|
|
258
|
-
throw err;
|
|
111
|
+
function keysThatSegmentsStillRead(manager) {
|
|
112
|
+
const keys = /* @__PURE__ */ new Set();
|
|
113
|
+
for (const partition of manager.getAllPartitions()) {
|
|
114
|
+
if (!isCompositePartition(partition)) continue;
|
|
115
|
+
for (const segment of partition.frozenSegments()) {
|
|
116
|
+
if (segment.paged !== null) keys.add(segment.paged.key);
|
|
259
117
|
}
|
|
260
|
-
return fs.open(filePath, "r+");
|
|
261
118
|
}
|
|
119
|
+
return keys;
|
|
262
120
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
try {
|
|
268
|
-
;
|
|
269
|
-
({ bytesWritten } = await handle.write(buffer, offset, buffer.length - offset));
|
|
270
|
-
} catch (err) {
|
|
271
|
-
throw new NarsilError(
|
|
272
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
273
|
-
`Write to "${key}" failed; the write is not acknowledged`,
|
|
274
|
-
{
|
|
275
|
-
key,
|
|
276
|
-
written: offset,
|
|
277
|
-
total: buffer.length,
|
|
278
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
279
|
-
}
|
|
280
|
-
);
|
|
281
|
-
}
|
|
282
|
-
if (bytesWritten <= 0) {
|
|
283
|
-
throw new NarsilError(
|
|
284
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
285
|
-
`Write to "${key}" made no progress; the file would be truncated and the write is not acknowledged`,
|
|
286
|
-
{ key, written: offset, total: buffer.length }
|
|
287
|
-
);
|
|
121
|
+
function closeUnusedSegments(adoption) {
|
|
122
|
+
for (const { prepared } of adoption.partitions) {
|
|
123
|
+
for (const { segment, isNew } of prepared) {
|
|
124
|
+
if (isNew) segment.paged?.close();
|
|
288
125
|
}
|
|
289
|
-
offset += bytesWritten;
|
|
290
126
|
}
|
|
291
127
|
}
|
|
292
|
-
|
|
293
|
-
|
|
128
|
+
function actionPartitionsOf(adoption) {
|
|
129
|
+
return adoption.partitions.map(({ partitionId, prepared }) => ({
|
|
130
|
+
partitionId,
|
|
131
|
+
segments: prepared.map(({ segment, isNew }) => {
|
|
132
|
+
const access = segment.paged;
|
|
133
|
+
if (access !== null && isNew) return { snapshot: access.snapshot };
|
|
134
|
+
return { segmentId: segment.segmentId };
|
|
135
|
+
})
|
|
136
|
+
}));
|
|
137
|
+
}
|
|
138
|
+
function swapPreparedPartitions(manager, partitions, changed, onSwapped) {
|
|
139
|
+
for (const { partitionId, prepared } of partitions) {
|
|
140
|
+
const kept = [];
|
|
141
|
+
for (const { segment } of prepared) {
|
|
142
|
+
if (segment.arrays.documentCount > 0) kept.push(segment);
|
|
143
|
+
else segment.paged?.close();
|
|
144
|
+
}
|
|
145
|
+
adoptDiskSegments(manager, partitionId, kept, changed);
|
|
146
|
+
onSwapped?.(partitionId, kept);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
function adoptDiskSegments(manager, partitionId, segments, changed) {
|
|
150
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
151
|
+
const before = composite.frozenSegments();
|
|
152
|
+
const kept = [];
|
|
153
|
+
for (const docId of changed) {
|
|
154
|
+
if (docPartitionMap.get(docId) !== partitionId) continue;
|
|
155
|
+
const document = composite.getRef(docId);
|
|
156
|
+
if (document !== void 0) kept.push([docId, document]);
|
|
157
|
+
}
|
|
158
|
+
const dropped = before.filter((segment) => !segments.includes(segment));
|
|
159
|
+
for (const segment of segments) {
|
|
160
|
+
if (before.includes(segment)) continue;
|
|
161
|
+
for (const docId of changed) segment.tombstoneDocument(docId);
|
|
162
|
+
}
|
|
163
|
+
composite.replaceFrozenSegments(segments);
|
|
164
|
+
composite.live.clear();
|
|
165
|
+
for (const [docId, pid] of docPartitionMap) {
|
|
166
|
+
if (pid === partitionId) docPartitionMap.delete(docId);
|
|
167
|
+
}
|
|
168
|
+
composite.live.beginBatch();
|
|
294
169
|
try {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const code = err.code;
|
|
299
|
-
if (code === "EISDIR" || code === "EPERM" || code === "EINVAL" || code === "ENOTSUP") {
|
|
300
|
-
return;
|
|
170
|
+
for (const [docId, document] of kept) {
|
|
171
|
+
composite.live.insert(docId, document, manager.config.schema, manager.language, insertOptions);
|
|
172
|
+
docPartitionMap.set(docId, partitionId);
|
|
301
173
|
}
|
|
302
|
-
wrapFsyncError(err, dirPath);
|
|
303
174
|
} finally {
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
return { path: resolvedPath, resolvedBase, pathMod };
|
|
322
|
-
}
|
|
323
|
-
async function ensureDir(filePath, pathMod, fs) {
|
|
324
|
-
await fs.mkdir(pathMod.dirname(filePath), { recursive: true });
|
|
325
|
-
}
|
|
326
|
-
return {
|
|
327
|
-
get root() {
|
|
328
|
-
return root;
|
|
329
|
-
},
|
|
330
|
-
lock() {
|
|
331
|
-
return lockDirectory(root);
|
|
332
|
-
},
|
|
333
|
-
async identity() {
|
|
334
|
-
const fs = await getFs();
|
|
335
|
-
const pathMod = await getPath();
|
|
336
|
-
try {
|
|
337
|
-
const stat = await fs.stat(pathMod.resolve(root), { bigint: true });
|
|
338
|
-
return `${stat.dev}:${stat.ino}`;
|
|
339
|
-
} catch (err) {
|
|
340
|
-
if (err.code === "ENOENT") return null;
|
|
341
|
-
throw err;
|
|
342
|
-
}
|
|
343
|
-
},
|
|
344
|
-
async appendHandle(key) {
|
|
345
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
346
|
-
const fs = await getFs();
|
|
347
|
-
await ensureDir(filePath, pathMod, fs);
|
|
348
|
-
const handle = await fs.open(filePath, "a");
|
|
349
|
-
return {
|
|
350
|
-
async append(bytes) {
|
|
351
|
-
await writeFully(handle, bytes, key);
|
|
352
|
-
},
|
|
353
|
-
async sync() {
|
|
354
|
-
try {
|
|
355
|
-
await handle.sync();
|
|
356
|
-
} catch (err) {
|
|
357
|
-
wrapFsyncError(err, key);
|
|
358
|
-
}
|
|
359
|
-
},
|
|
360
|
-
async truncate(byteLength) {
|
|
361
|
-
await handle.truncate(byteLength);
|
|
362
|
-
try {
|
|
363
|
-
await handle.sync();
|
|
364
|
-
} catch (err) {
|
|
365
|
-
wrapFsyncError(err, key);
|
|
366
|
-
}
|
|
367
|
-
},
|
|
368
|
-
async size() {
|
|
369
|
-
const stat = await handle.stat();
|
|
370
|
-
return stat.size;
|
|
371
|
-
},
|
|
372
|
-
async close() {
|
|
373
|
-
await handle.close();
|
|
374
|
-
}
|
|
375
|
-
};
|
|
376
|
-
},
|
|
377
|
-
async markerHandle(key) {
|
|
378
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
379
|
-
const fs = await getFs();
|
|
380
|
-
await ensureDir(filePath, pathMod, fs);
|
|
381
|
-
const handle = await openReadWriteCreating(filePath, fs);
|
|
382
|
-
return {
|
|
383
|
-
async read() {
|
|
384
|
-
const stat = await handle.stat();
|
|
385
|
-
if (stat.size === 0) {
|
|
386
|
-
return new Uint8Array(0);
|
|
387
|
-
}
|
|
388
|
-
const bytes = new Uint8Array(stat.size);
|
|
389
|
-
await handle.read(bytes, 0, stat.size, 0);
|
|
390
|
-
return bytes;
|
|
391
|
-
},
|
|
392
|
-
async writeSlot(offset, slot) {
|
|
393
|
-
try {
|
|
394
|
-
await handle.write(slot, 0, slot.length, offset);
|
|
395
|
-
} catch (err) {
|
|
396
|
-
throw new NarsilError(
|
|
397
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
398
|
-
`Write to "${key}" failed; the write is not acknowledged`,
|
|
399
|
-
{ key, cause: err instanceof Error ? err.message : String(err) }
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
try {
|
|
403
|
-
await handle.sync();
|
|
404
|
-
} catch (err) {
|
|
405
|
-
wrapFsyncError(err, key);
|
|
406
|
-
}
|
|
407
|
-
},
|
|
408
|
-
async close() {
|
|
409
|
-
await handle.close();
|
|
410
|
-
}
|
|
411
|
-
};
|
|
412
|
-
},
|
|
413
|
-
async syncDirectoryOf(key) {
|
|
414
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
415
|
-
const fs = await getFs();
|
|
416
|
-
await fsyncDirectory(pathMod.dirname(filePath), fs);
|
|
417
|
-
},
|
|
418
|
-
async atomicWrite(key, data) {
|
|
419
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
420
|
-
const fs = await getFs();
|
|
421
|
-
await ensureDir(filePath, pathMod, fs);
|
|
422
|
-
const dir = pathMod.dirname(filePath);
|
|
423
|
-
const tempName = `${pathMod.basename(filePath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
424
|
-
const tempPath = pathMod.join(dir, tempName);
|
|
425
|
-
const buffers = Array.isArray(data) ? data : [data];
|
|
426
|
-
let handle = null;
|
|
427
|
-
try {
|
|
428
|
-
handle = await fs.open(tempPath, "w");
|
|
429
|
-
for (const buffer of buffers) {
|
|
430
|
-
await writeFully(handle, buffer, tempPath);
|
|
431
|
-
}
|
|
432
|
-
try {
|
|
433
|
-
await handle.sync();
|
|
434
|
-
} catch (err) {
|
|
435
|
-
wrapFsyncError(err, tempPath);
|
|
436
|
-
}
|
|
437
|
-
await handle.close();
|
|
438
|
-
handle = null;
|
|
439
|
-
await fs.rename(tempPath, filePath);
|
|
440
|
-
await fsyncDirectory(dir, fs);
|
|
441
|
-
} catch (err) {
|
|
442
|
-
if (handle !== null) {
|
|
443
|
-
await handle.close().catch(() => void 0);
|
|
444
|
-
}
|
|
445
|
-
await fs.unlink(tempPath).catch(() => void 0);
|
|
446
|
-
throw err;
|
|
447
|
-
}
|
|
448
|
-
},
|
|
449
|
-
async read(key) {
|
|
450
|
-
const { path: filePath } = await resolve(key);
|
|
451
|
-
const fs = await getFs();
|
|
452
|
-
try {
|
|
453
|
-
const data = await fs.readFile(filePath);
|
|
454
|
-
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
455
|
-
} catch (err) {
|
|
456
|
-
if (err.code === "ENOENT") {
|
|
457
|
-
return null;
|
|
458
|
-
}
|
|
459
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read "${key}"`, {
|
|
460
|
-
key,
|
|
461
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
462
|
-
});
|
|
463
|
-
}
|
|
464
|
-
},
|
|
465
|
-
async readRange(key, offset, length) {
|
|
466
|
-
const { path: filePath } = await resolve(key);
|
|
467
|
-
const fs = await getFs();
|
|
468
|
-
let handle = null;
|
|
469
|
-
try {
|
|
470
|
-
handle = await fs.open(filePath, "r");
|
|
471
|
-
const bytes = new Uint8Array(length);
|
|
472
|
-
let filled = 0;
|
|
473
|
-
while (filled < length) {
|
|
474
|
-
const { bytesRead } = await handle.read(bytes, filled, length - filled, offset + filled);
|
|
475
|
-
if (bytesRead === 0) break;
|
|
476
|
-
filled += bytesRead;
|
|
477
|
-
}
|
|
478
|
-
if (filled < length) {
|
|
479
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
|
|
480
|
-
key,
|
|
481
|
-
offset,
|
|
482
|
-
length
|
|
483
|
-
});
|
|
484
|
-
}
|
|
485
|
-
return bytes;
|
|
486
|
-
} finally {
|
|
487
|
-
await handle?.close();
|
|
488
|
-
}
|
|
489
|
-
},
|
|
490
|
-
async byteLength(key) {
|
|
491
|
-
const { path: filePath } = await resolve(key);
|
|
492
|
-
const fs = await getFs();
|
|
493
|
-
try {
|
|
494
|
-
return (await fs.stat(filePath)).size;
|
|
495
|
-
} catch (err) {
|
|
496
|
-
if (err.code === "ENOENT") {
|
|
497
|
-
return null;
|
|
498
|
-
}
|
|
499
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read the length of "${key}"`, {
|
|
500
|
-
key,
|
|
501
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
502
|
-
});
|
|
503
|
-
}
|
|
504
|
-
},
|
|
505
|
-
async remove(key) {
|
|
506
|
-
const { path: filePath, pathMod } = await resolve(key);
|
|
507
|
-
const fs = await getFs();
|
|
508
|
-
try {
|
|
509
|
-
await fs.unlink(filePath);
|
|
510
|
-
} catch (err) {
|
|
511
|
-
if (err.code === "ENOENT") {
|
|
512
|
-
return;
|
|
513
|
-
}
|
|
514
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, {
|
|
515
|
-
key,
|
|
516
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
517
|
-
});
|
|
175
|
+
composite.live.endBatch();
|
|
176
|
+
}
|
|
177
|
+
for (const segment of dropped) segment.paged?.close();
|
|
178
|
+
}
|
|
179
|
+
function adoptSegmentsOnWorker(manager, action) {
|
|
180
|
+
const partitions = action.partitions.map(({ partitionId, segments }) => {
|
|
181
|
+
const partition = manager.getPartition(partitionId);
|
|
182
|
+
const held = isCompositePartition(partition) ? partition.frozenSegments() : [];
|
|
183
|
+
const prepared = segments.map((item) => {
|
|
184
|
+
if ("snapshot" in item) return { segment: createPagedFrozenSegment(item.snapshot), isNew: true };
|
|
185
|
+
const existing = held.find((segment) => segment.segmentId === item.segmentId);
|
|
186
|
+
if (existing === void 0) {
|
|
187
|
+
throw new NarsilError(
|
|
188
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
189
|
+
`Partition ${partitionId} of "${action.indexName}" holds no segment "${item.segmentId}" to keep`,
|
|
190
|
+
{ indexName: action.indexName, partitionId, segmentId: item.segmentId }
|
|
191
|
+
);
|
|
518
192
|
}
|
|
519
|
-
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
await ensureDir(target.path, target.pathMod, fs);
|
|
526
|
-
await fs.rename(source.path, target.path);
|
|
527
|
-
const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
|
|
528
|
-
for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
|
|
529
|
-
},
|
|
530
|
-
async list(prefix) {
|
|
531
|
-
const fs = await getFs();
|
|
532
|
-
const pathMod = await getPath();
|
|
533
|
-
const resolvedBase = pathMod.resolve(root);
|
|
534
|
-
const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
|
|
535
|
-
const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
|
|
536
|
-
const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
|
|
537
|
-
return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
|
|
538
|
-
},
|
|
539
|
-
async pathOf(key) {
|
|
540
|
-
const { path: filePath } = await resolve(key);
|
|
541
|
-
return filePath;
|
|
542
|
-
}
|
|
543
|
-
};
|
|
193
|
+
return { segment: existing, isNew: false };
|
|
194
|
+
});
|
|
195
|
+
applyNewerTombstones(prepared);
|
|
196
|
+
return { partitionId, prepared };
|
|
197
|
+
});
|
|
198
|
+
swapPreparedPartitions(manager, partitions, new Set(action.changed));
|
|
544
199
|
}
|
|
545
200
|
|
|
546
201
|
// src/persistence/durability/sync/layout.ts
|
|
@@ -654,6 +309,7 @@ async function reclaimWalBeyondCount(directory, indexName, partitionCount, parti
|
|
|
654
309
|
await partition.walWriter.close();
|
|
655
310
|
} catch (err) {
|
|
656
311
|
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
312
|
+
throw err;
|
|
657
313
|
}
|
|
658
314
|
partitions.delete(partitionId);
|
|
659
315
|
}
|
|
@@ -692,136 +348,6 @@ function formatSegmentStart(startSeqNo) {
|
|
|
692
348
|
return startSeqNo.toString().padStart(16, "0");
|
|
693
349
|
}
|
|
694
350
|
|
|
695
|
-
// src/persistence/durability/marker-writer.ts
|
|
696
|
-
function markerKey(indexName, partitionId) {
|
|
697
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
698
|
-
}
|
|
699
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
700
|
-
const key = markerKey(indexName, partitionId);
|
|
701
|
-
const handle = await directory.markerHandle(key);
|
|
702
|
-
const existing = await handle.read();
|
|
703
|
-
const current = readCommitMarker(existing);
|
|
704
|
-
const createdFresh = existing.length === 0;
|
|
705
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
706
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
707
|
-
return {
|
|
708
|
-
created: createdFresh,
|
|
709
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
710
|
-
async commit(state) {
|
|
711
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
712
|
-
writeSeq += 1;
|
|
713
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
714
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
715
|
-
lastSlot = slotIndex;
|
|
716
|
-
},
|
|
717
|
-
async close() {
|
|
718
|
-
await handle.close();
|
|
719
|
-
}
|
|
720
|
-
};
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
// src/persistence/durability/recovery.ts
|
|
724
|
-
async function removeLeftoverRewriteFiles(directory, names) {
|
|
725
|
-
for (const indexName of names) {
|
|
726
|
-
for (const place of ["rebuilt", "received"]) {
|
|
727
|
-
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
728
|
-
}
|
|
729
|
-
}
|
|
730
|
-
}
|
|
731
|
-
async function listPersistedIndexes(directory) {
|
|
732
|
-
const metaKeys = await directory.list("");
|
|
733
|
-
const names = [];
|
|
734
|
-
for (const key of metaKeys) {
|
|
735
|
-
if (key.endsWith("/meta")) {
|
|
736
|
-
names.push(key.slice(0, -"/meta".length));
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
return names;
|
|
740
|
-
}
|
|
741
|
-
async function loadMetadata(directory, indexName) {
|
|
742
|
-
const bytes = await directory.read(`${indexName}/meta`);
|
|
743
|
-
if (bytes === null) {
|
|
744
|
-
return null;
|
|
745
|
-
}
|
|
746
|
-
const { metadata } = await readMetadataEnvelope(bytes);
|
|
747
|
-
return metadata;
|
|
748
|
-
}
|
|
749
|
-
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
750
|
-
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
751
|
-
const read = await readWalSegments(directory, indexName, partitionId);
|
|
752
|
-
if (read === null) {
|
|
753
|
-
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
754
|
-
}
|
|
755
|
-
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
756
|
-
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
757
|
-
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
758
|
-
}
|
|
759
|
-
let highestSeqNo = fromSeqNoExclusive;
|
|
760
|
-
let highestPrimaryTerm = 0;
|
|
761
|
-
const replay = async (entries) => {
|
|
762
|
-
for (const entry of entries) {
|
|
763
|
-
if (entry.seqNo <= highestSeqNo) {
|
|
764
|
-
continue;
|
|
765
|
-
}
|
|
766
|
-
if (turnIsSpent()) await yieldTurn();
|
|
767
|
-
applyEntry(entry, deps);
|
|
768
|
-
highestSeqNo = entry.seqNo;
|
|
769
|
-
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
770
|
-
highestPrimaryTerm = entry.primaryTerm;
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
};
|
|
774
|
-
await replay(durableEntries);
|
|
775
|
-
if (activeTail !== null) {
|
|
776
|
-
await replay(activeTail.entries);
|
|
777
|
-
}
|
|
778
|
-
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
779
|
-
if (activeTail !== null) {
|
|
780
|
-
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
781
|
-
}
|
|
782
|
-
return { highestSeqNo, highestPrimaryTerm };
|
|
783
|
-
}
|
|
784
|
-
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
785
|
-
const lastReplayed = tail.entries.at(-1);
|
|
786
|
-
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
787
|
-
const handle = await directory.appendHandle(tail.key);
|
|
788
|
-
try {
|
|
789
|
-
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
790
|
-
else await handle.sync();
|
|
791
|
-
} finally {
|
|
792
|
-
await handle.close();
|
|
793
|
-
}
|
|
794
|
-
if (lastReplayed === void 0) return;
|
|
795
|
-
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
796
|
-
try {
|
|
797
|
-
await writer.commit({
|
|
798
|
-
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
799
|
-
durableByteLength: tail.cleanEnd,
|
|
800
|
-
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
801
|
-
});
|
|
802
|
-
} finally {
|
|
803
|
-
await writer.close();
|
|
804
|
-
}
|
|
805
|
-
}
|
|
806
|
-
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
807
|
-
for (const { key, startSeqNo } of segments) {
|
|
808
|
-
if (startSeqNo > activeSegmentSeqNo) {
|
|
809
|
-
await directory.remove(key);
|
|
810
|
-
}
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
function applyEntry(entry, deps) {
|
|
814
|
-
deps.writeRecords?.replayed(entry);
|
|
815
|
-
if (entry.operation === "DELETE") {
|
|
816
|
-
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
817
|
-
return;
|
|
818
|
-
}
|
|
819
|
-
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
820
|
-
}
|
|
821
|
-
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
822
|
-
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
823
|
-
}
|
|
824
|
-
|
|
825
351
|
// src/persistence/durability/checkpoint-capture.ts
|
|
826
352
|
function whileNoMutationApplies(partitions, capture2) {
|
|
827
353
|
return new Promise((resolve, reject) => {
|
|
@@ -829,9 +355,9 @@ function whileNoMutationApplies(partitions, capture2) {
|
|
|
829
355
|
const released = new Promise((settle) => {
|
|
830
356
|
release = settle;
|
|
831
357
|
});
|
|
832
|
-
function captureNow() {
|
|
358
|
+
async function captureNow() {
|
|
833
359
|
try {
|
|
834
|
-
resolve(capture2());
|
|
360
|
+
resolve(await capture2());
|
|
835
361
|
} catch (error) {
|
|
836
362
|
reject(error);
|
|
837
363
|
} finally {
|
|
@@ -840,13 +366,13 @@ function whileNoMutationApplies(partitions, capture2) {
|
|
|
840
366
|
}
|
|
841
367
|
let waiting = partitions.length;
|
|
842
368
|
if (waiting === 0) {
|
|
843
|
-
captureNow();
|
|
369
|
+
void captureNow();
|
|
844
370
|
return;
|
|
845
371
|
}
|
|
846
372
|
for (const partition of partitions) {
|
|
847
373
|
partition.appendChain = partition.appendChain.then(() => {
|
|
848
374
|
waiting -= 1;
|
|
849
|
-
if (waiting === 0) captureNow();
|
|
375
|
+
if (waiting === 0) void captureNow();
|
|
850
376
|
return released;
|
|
851
377
|
});
|
|
852
378
|
}
|
|
@@ -886,9 +412,11 @@ function noWholePartitions() {
|
|
|
886
412
|
return { contents: [], captured: [] };
|
|
887
413
|
}
|
|
888
414
|
function wholePartitionsWhereMostChanged(manager, rules) {
|
|
889
|
-
const { priorCheckpoint, priorPartitionIds,
|
|
415
|
+
const { priorCheckpoint, priorPartitionIds, aWorkerCanSerialise, captureRecords } = rules;
|
|
890
416
|
return (targets) => {
|
|
891
417
|
const whole = { contents: [], captured: [] };
|
|
418
|
+
const everyPartition = rules.everyPartition || requiresLayoutCheckpoint(manager) || priorPartitionIds.length > 0 && priorPartitionIds.length !== manager.partitionCount;
|
|
419
|
+
if (rules.usesLogForUnchangedLayout && !everyPartition) return whole;
|
|
892
420
|
const changed = targets.filter(
|
|
893
421
|
(target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
|
|
894
422
|
);
|
|
@@ -911,45 +439,57 @@ function wholePartitionsWhereMostChanged(manager, rules) {
|
|
|
911
439
|
return whole;
|
|
912
440
|
};
|
|
913
441
|
}
|
|
914
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured
|
|
442
|
+
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
|
|
915
443
|
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
916
|
-
return whileNoMutationApplies(
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
const
|
|
923
|
-
|
|
924
|
-
const
|
|
925
|
-
const
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
partitionId
|
|
932
|
-
lastSeqNo
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
941
|
-
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
942
|
-
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
943
|
-
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
|
|
444
|
+
return whileNoMutationApplies(
|
|
445
|
+
[...indexState.partitions.values()],
|
|
446
|
+
() => captureAppliedPrefix(manager, (cutoffs) => {
|
|
447
|
+
onCaptured?.();
|
|
448
|
+
const partitionCount = manager.partitionCount;
|
|
449
|
+
const previous = priorCheckpointThatLimitsTheRecords;
|
|
450
|
+
const changedLayout = requiresLayoutCheckpoint(manager) || previous !== void 0 && previous.length > 0 && previous.length !== partitionCount;
|
|
451
|
+
const bounded = changedLayout ? void 0 : previous;
|
|
452
|
+
const documentCount = manager.countDocuments();
|
|
453
|
+
const targets = [];
|
|
454
|
+
let recordsLeftInLog = 0;
|
|
455
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
456
|
+
const partition = indexState.partitions.get(partitionId);
|
|
457
|
+
if (partition !== void 0 && partition.failed !== null) continue;
|
|
458
|
+
const appliedSeqNo = Math.min(partition?.appliedSeqNo ?? 0, cutoffs.get(partitionId) ?? Infinity);
|
|
459
|
+
const lastSeqNo = bounded === void 0 ? appliedSeqNo : Math.min(appliedSeqNo, snapshotCheckpointFor(bounded, partitionId) + LOG_RECORDS_PER_CHECKPOINT);
|
|
460
|
+
recordsLeftInLog += appliedSeqNo - lastSeqNo;
|
|
461
|
+
const target = {
|
|
462
|
+
partitionId,
|
|
463
|
+
lastSeqNo,
|
|
464
|
+
primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
|
|
465
|
+
};
|
|
466
|
+
if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
|
|
467
|
+
targets.push(target);
|
|
944
468
|
}
|
|
945
|
-
const
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
469
|
+
const vectorPlans = /* @__PURE__ */ new Map();
|
|
470
|
+
try {
|
|
471
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
472
|
+
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
473
|
+
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
474
|
+
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, true));
|
|
475
|
+
}
|
|
476
|
+
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
477
|
+
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
478
|
+
return {
|
|
479
|
+
partitionCount,
|
|
480
|
+
targets,
|
|
481
|
+
documentCount,
|
|
482
|
+
vectorPlans,
|
|
483
|
+
wholePartitions,
|
|
484
|
+
recordsLeftInLog,
|
|
485
|
+
unreplayedCutoffs: cutoffs
|
|
486
|
+
};
|
|
487
|
+
} catch (error) {
|
|
488
|
+
for (const plan of vectorPlans.values()) plan.release();
|
|
489
|
+
throw error;
|
|
490
|
+
}
|
|
491
|
+
})
|
|
492
|
+
);
|
|
953
493
|
}
|
|
954
494
|
async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
|
|
955
495
|
if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
|
|
@@ -969,8 +509,22 @@ async function writeCapturedVectors(store, indexName, capture2, priorManifest, r
|
|
|
969
509
|
return { vectors: written.refs, layouts: written.layouts, written: written.written };
|
|
970
510
|
}
|
|
971
511
|
|
|
972
|
-
// src/persistence/durability/checkpoint-
|
|
973
|
-
function
|
|
512
|
+
// src/persistence/durability/checkpoint-prior.ts
|
|
513
|
+
function priorForLayout(manifest, partitionCount) {
|
|
514
|
+
const checkpoints = new Map(manifest?.checkpoint.map((entry) => [entry.partitionId, entry]));
|
|
515
|
+
const partitions = new Map(manifest?.partitions.map((entry) => [entry.partitionId, entry]));
|
|
516
|
+
const prior = { checkpoint: [], partitions: [] };
|
|
517
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
518
|
+
prior.checkpoint.push(
|
|
519
|
+
checkpoints.get(partitionId) ?? { partitionId, lastSeqNo: 0, primaryTerm: SINGLE_NODE_PRIMARY_TERM }
|
|
520
|
+
);
|
|
521
|
+
prior.partitions.push(partitions.get(partitionId) ?? { partitionId, nextSegmentId: 0, segments: [] });
|
|
522
|
+
}
|
|
523
|
+
return prior;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// src/persistence/durability/checkpoint-worker-entry.ts
|
|
527
|
+
function resolveCheckpointWorkerEntry() {
|
|
974
528
|
return resolveWorkerEntry(
|
|
975
529
|
import.meta.url,
|
|
976
530
|
/\/src\/persistence\/durability\/[^/]+$/,
|
|
@@ -1281,13 +835,15 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1281
835
|
const store = directoryCheckpointStore(input.directory);
|
|
1282
836
|
const writeRecords = input.hooks.writeRecords ?? null;
|
|
1283
837
|
const { indexName } = input;
|
|
1284
|
-
const
|
|
838
|
+
const storedManifest = await readCurrentManifest(store, input.indexName);
|
|
839
|
+
const priorManifest = storedManifest === null ? null : manifestWithinPartitionCount(storedManifest, manager.partitionCount);
|
|
1285
840
|
const textOnDisk = metadata.textStorage === "disk";
|
|
1286
841
|
const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
|
|
1287
842
|
priorCheckpoint: priorManifest?.checkpoint ?? [],
|
|
1288
843
|
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1289
844
|
everyPartition: fromMemory,
|
|
1290
845
|
aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
|
|
846
|
+
usesLogForUnchangedLayout: textOnDisk,
|
|
1291
847
|
...writeRecords === null ? {} : {
|
|
1292
848
|
captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
1293
849
|
}
|
|
@@ -1297,20 +853,28 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1297
853
|
input.indexState,
|
|
1298
854
|
manager,
|
|
1299
855
|
vectorIndexes,
|
|
1300
|
-
|
|
856
|
+
wholePartitionsOf,
|
|
1301
857
|
fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1302
858
|
priorManifest?.vectors ?? [],
|
|
1303
859
|
writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
|
|
1304
860
|
textOnDisk ? () => {
|
|
1305
861
|
changesSinceCapture = manager.openCheckpointChangeLog();
|
|
1306
|
-
} : void 0
|
|
1307
|
-
input.pinsRetiredVectors
|
|
862
|
+
} : void 0
|
|
1308
863
|
);
|
|
1309
864
|
try {
|
|
1310
865
|
await openNewLogSegments(input.indexState, input.failPartition);
|
|
1311
|
-
return await finishBoundedCheckpoint(
|
|
1312
|
-
|
|
1313
|
-
|
|
866
|
+
return await finishBoundedCheckpoint(
|
|
867
|
+
input,
|
|
868
|
+
manager,
|
|
869
|
+
store,
|
|
870
|
+
{ ...metadata, partitionCount: capture2.partitionCount },
|
|
871
|
+
priorManifest,
|
|
872
|
+
capture2,
|
|
873
|
+
fromMemory,
|
|
874
|
+
() => {
|
|
875
|
+
return changesSinceCapture;
|
|
876
|
+
}
|
|
877
|
+
);
|
|
1314
878
|
} finally {
|
|
1315
879
|
for (const plan of capture2.vectorPlans.values()) plan.release();
|
|
1316
880
|
changesSinceCapture?.close();
|
|
@@ -1318,24 +882,26 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1318
882
|
}
|
|
1319
883
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1320
884
|
const { indexName, indexState, hooks } = input;
|
|
1321
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-NIYMEA5R.mjs');
|
|
1322
885
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1323
886
|
if (adoption === null) return [];
|
|
1324
|
-
const adopted = await whileNoMutationApplies(
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
887
|
+
const adopted = await whileNoMutationApplies(
|
|
888
|
+
[...indexState.partitions.values()],
|
|
889
|
+
() => manager.publication.write(() => {
|
|
890
|
+
if (changes.partitionsReplaced) return false;
|
|
891
|
+
const changed = changes.takeWrittenDocIds();
|
|
892
|
+
swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
|
|
893
|
+
hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
|
|
894
|
+
});
|
|
895
|
+
hooks.queueDiskAdoption?.({
|
|
896
|
+
type: "adoptDiskSegments",
|
|
897
|
+
indexName,
|
|
898
|
+
partitions: actionPartitionsOf(adoption),
|
|
899
|
+
changed: [...changed],
|
|
900
|
+
requestId: createRequestId()
|
|
901
|
+
});
|
|
902
|
+
return true;
|
|
903
|
+
})
|
|
904
|
+
);
|
|
1339
905
|
if (!adopted) {
|
|
1340
906
|
closeUnusedSegments(adoption);
|
|
1341
907
|
return [];
|
|
@@ -1357,7 +923,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1357
923
|
targets,
|
|
1358
924
|
compactionThreshold: input.compactionThreshold,
|
|
1359
925
|
canOffload: input.canOffload,
|
|
1360
|
-
prior: priorManifest,
|
|
926
|
+
prior: priorForLayout(priorManifest, capture2.partitionCount),
|
|
1361
927
|
wholePartitions: capture2.wholePartitions.contents,
|
|
1362
928
|
capturedPartitions: capture2.wholePartitions.captured,
|
|
1363
929
|
keepsWriteRecords: writeRecords !== null
|
|
@@ -1373,7 +939,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1373
939
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1374
940
|
if (!input.holdsCheckpointFiles()) {
|
|
1375
941
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1376
|
-
const stillRead = metadata.textStorage === "disk" ?
|
|
942
|
+
const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1377
943
|
const keptForSync = await input.keepsCheckpointKey();
|
|
1378
944
|
await removeCheckpointGarbage(
|
|
1379
945
|
store,
|
|
@@ -1386,9 +952,12 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1386
952
|
if (checkpointMetadata === void 0) {
|
|
1387
953
|
return;
|
|
1388
954
|
}
|
|
1389
|
-
const bytes = await writeMetadataEnvelope(
|
|
955
|
+
const bytes = await writeMetadataEnvelope(
|
|
956
|
+
{ ...checkpointMetadata, partitionCount: capture2.partitionCount },
|
|
957
|
+
{ checksum: true }
|
|
958
|
+
);
|
|
1390
959
|
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
1391
|
-
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount,
|
|
960
|
+
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture2.partitionCount);
|
|
1392
961
|
});
|
|
1393
962
|
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
1394
963
|
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
@@ -1396,69 +965,36 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1396
965
|
await reclaimWalBeyondCount(
|
|
1397
966
|
input.directory,
|
|
1398
967
|
input.indexName,
|
|
1399
|
-
|
|
968
|
+
capture2.partitionCount,
|
|
1400
969
|
input.indexState.partitions,
|
|
1401
970
|
input.markFatal
|
|
1402
971
|
);
|
|
972
|
+
await input.directory.remove(transitionKey(input.indexName));
|
|
1403
973
|
const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
|
|
1404
974
|
noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
|
|
1405
|
-
const
|
|
1406
|
-
|
|
975
|
+
const eligibleBeyond = recordsBeyond(input.indexState, targets, capture2.unreplayedCutoffs);
|
|
976
|
+
const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
|
|
977
|
+
return anotherIsDue ? eligibleBeyond : capture2.recordsLeftInLog;
|
|
1407
978
|
}
|
|
1408
|
-
function recordsBeyond(indexState, targets) {
|
|
979
|
+
function recordsBeyond(indexState, targets, cutoffs) {
|
|
1409
980
|
let records = 0;
|
|
1410
981
|
for (const [partitionId, partition] of indexState.partitions) {
|
|
1411
982
|
if (partition.failed !== null) continue;
|
|
1412
|
-
records += Math.max(
|
|
983
|
+
records += Math.max(
|
|
984
|
+
0,
|
|
985
|
+
Math.min(partition.appliedSeqNo, cutoffs?.get(partitionId) ?? Infinity) - snapshotCheckpointFor(targets, partitionId)
|
|
986
|
+
);
|
|
1413
987
|
}
|
|
1414
988
|
return records;
|
|
1415
989
|
}
|
|
1416
990
|
|
|
1417
|
-
// src/persistence/durability/
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
|
|
1422
|
-
vectorIndex.scheduleBuild();
|
|
1423
|
-
vectorIndex.startShuffledRebuildWhenDue();
|
|
1424
|
-
}
|
|
1425
|
-
};
|
|
1426
|
-
return {
|
|
1427
|
-
track(indexName, load, startsGraphRepairs = true) {
|
|
1428
|
-
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
1429
|
-
loads.set(
|
|
1430
|
-
indexName,
|
|
1431
|
-
loaded.catch(() => void 0)
|
|
1432
|
-
);
|
|
1433
|
-
return loaded;
|
|
1434
|
-
},
|
|
1435
|
-
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
1436
|
-
startGraphRepairs
|
|
1437
|
-
};
|
|
991
|
+
// src/persistence/durability/deferred-mutations.ts
|
|
992
|
+
var deferred = /* @__PURE__ */ new WeakSet();
|
|
993
|
+
function deferMutationPublication(record) {
|
|
994
|
+
deferred.add(record);
|
|
1438
995
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
await checkpointInTurn(turns, {
|
|
1442
|
-
joinsInFlight: false,
|
|
1443
|
-
mayStart: turn.mayStart,
|
|
1444
|
-
perform: async () => {
|
|
1445
|
-
swapped = turn.swap();
|
|
1446
|
-
if (swapped) await turn.checkpoint();
|
|
1447
|
-
},
|
|
1448
|
-
afterSettling: turn.afterSettling
|
|
1449
|
-
});
|
|
1450
|
-
return swapped;
|
|
1451
|
-
}
|
|
1452
|
-
async function checkpointInTurn(turns, turn) {
|
|
1453
|
-
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
1454
|
-
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
1455
|
-
if (!turn.mayStart()) return;
|
|
1456
|
-
const run = turn.perform().finally(() => {
|
|
1457
|
-
turns.checkpointInFlight = null;
|
|
1458
|
-
turn.afterSettling();
|
|
1459
|
-
});
|
|
1460
|
-
turns.checkpointInFlight = run;
|
|
1461
|
-
return run;
|
|
996
|
+
function hasDeferredPublication(record) {
|
|
997
|
+
return deferred.has(record);
|
|
1462
998
|
}
|
|
1463
999
|
|
|
1464
1000
|
// src/persistence/durability/held-checkpoints.ts
|
|
@@ -1516,24 +1052,6 @@ function createHeldCheckpoints(hooks) {
|
|
|
1516
1052
|
};
|
|
1517
1053
|
}
|
|
1518
1054
|
|
|
1519
|
-
// src/persistence/durability/manager-state.ts
|
|
1520
|
-
function closePartitionWriter(partition) {
|
|
1521
|
-
return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
|
|
1522
|
-
}
|
|
1523
|
-
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
1524
|
-
indexState.unloading = true;
|
|
1525
|
-
try {
|
|
1526
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
1527
|
-
await pendingMetadataWrite;
|
|
1528
|
-
for (const partition of indexState.partitions.values()) {
|
|
1529
|
-
await closeWalWriter(closePartitionWriter(partition));
|
|
1530
|
-
}
|
|
1531
|
-
} catch (error) {
|
|
1532
|
-
indexState.unloading = false;
|
|
1533
|
-
throw error;
|
|
1534
|
-
}
|
|
1535
|
-
}
|
|
1536
|
-
|
|
1537
1055
|
// src/persistence/durability/restore-journal.ts
|
|
1538
1056
|
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
1539
1057
|
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
@@ -2174,6 +1692,8 @@ function createFilePins(remove) {
|
|
|
2174
1692
|
}
|
|
2175
1693
|
};
|
|
2176
1694
|
}
|
|
1695
|
+
|
|
1696
|
+
// src/persistence/durability/sync/published.ts
|
|
2177
1697
|
var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
|
|
2178
1698
|
var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
|
|
2179
1699
|
function hexOf(bytes) {
|
|
@@ -2842,13 +2362,13 @@ function createWalWriter(directory, config) {
|
|
|
2842
2362
|
for (const frame of frames) {
|
|
2843
2363
|
totalBytes += frame.length;
|
|
2844
2364
|
}
|
|
2845
|
-
const
|
|
2365
|
+
const joined2 = new Uint8Array(totalBytes);
|
|
2846
2366
|
let offset = 0;
|
|
2847
2367
|
for (const frame of frames) {
|
|
2848
|
-
|
|
2368
|
+
joined2.set(frame, offset);
|
|
2849
2369
|
offset += frame.length;
|
|
2850
2370
|
}
|
|
2851
|
-
return
|
|
2371
|
+
return joined2;
|
|
2852
2372
|
}
|
|
2853
2373
|
function framesThatFit(frames, start) {
|
|
2854
2374
|
const room = segmentMaxBytes - activeBytes;
|
|
@@ -2869,16 +2389,16 @@ function createWalWriter(directory, config) {
|
|
|
2869
2389
|
{ indexName: config.indexName, partitionId: config.partitionId }
|
|
2870
2390
|
);
|
|
2871
2391
|
}
|
|
2872
|
-
const
|
|
2873
|
-
if (activeBytes +
|
|
2392
|
+
const joined2 = joinFrames([...frames]);
|
|
2393
|
+
if (activeBytes + joined2.length > MAX_FILE_BYTES) {
|
|
2874
2394
|
throw new NarsilError(
|
|
2875
2395
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2876
|
-
`A log record of ${
|
|
2877
|
-
{ indexName: config.indexName, partitionId: config.partitionId, bytes:
|
|
2396
|
+
`A log record of ${joined2.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
|
|
2397
|
+
{ indexName: config.indexName, partitionId: config.partitionId, bytes: joined2.length }
|
|
2878
2398
|
);
|
|
2879
2399
|
}
|
|
2880
|
-
await activeHandle.append(
|
|
2881
|
-
activeBytes +=
|
|
2400
|
+
await activeHandle.append(joined2);
|
|
2401
|
+
activeBytes += joined2.length;
|
|
2882
2402
|
const lastSeqNo = entries[entries.length - 1].seqNo;
|
|
2883
2403
|
if (lastSeqNo > highestAppendedSeqNo) {
|
|
2884
2404
|
highestAppendedSeqNo = lastSeqNo;
|
|
@@ -2999,6 +2519,109 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
|
|
|
2999
2519
|
}
|
|
3000
2520
|
};
|
|
3001
2521
|
}
|
|
2522
|
+
|
|
2523
|
+
// src/serialization/payload-v2.ts
|
|
2524
|
+
function validateHnswMetric(value) {
|
|
2525
|
+
if (value === void 0 || value === null) return void 0;
|
|
2526
|
+
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
2527
|
+
return value;
|
|
2528
|
+
}
|
|
2529
|
+
return void 0;
|
|
2530
|
+
}
|
|
2531
|
+
function storedDocumentsOf(raw) {
|
|
2532
|
+
requireValidFieldTypes(raw.schema ?? {});
|
|
2533
|
+
const documents = {};
|
|
2534
|
+
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
2535
|
+
documents[docId] = {
|
|
2536
|
+
fields: doc.fields,
|
|
2537
|
+
fieldLengths: doc.field_lengths ?? {}
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
2540
|
+
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
2541
|
+
return documents;
|
|
2542
|
+
}
|
|
2543
|
+
function documentsOfPayloadV2(data) {
|
|
2544
|
+
const raw = decode(data);
|
|
2545
|
+
return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
|
|
2546
|
+
}
|
|
2547
|
+
function deserializePayloadV2(data) {
|
|
2548
|
+
const raw = decode(data);
|
|
2549
|
+
const documents = storedDocumentsOf(raw);
|
|
2550
|
+
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
2551
|
+
const invertedIndex = {};
|
|
2552
|
+
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
2553
|
+
const count = list.ids.length;
|
|
2554
|
+
const postings = new Array(count);
|
|
2555
|
+
for (let i = 0; i < count; i++) {
|
|
2556
|
+
postings[i] = {
|
|
2557
|
+
docId: list.ids[i],
|
|
2558
|
+
termFrequency: list.tf[i],
|
|
2559
|
+
field: fieldNames[list.fi[i]] ?? "",
|
|
2560
|
+
positions: list.pos ? list.pos[i] : []
|
|
2561
|
+
};
|
|
2562
|
+
}
|
|
2563
|
+
invertedIndex[token] = {
|
|
2564
|
+
docFrequency: list.df,
|
|
2565
|
+
postings
|
|
2566
|
+
};
|
|
2567
|
+
}
|
|
2568
|
+
const numeric = {};
|
|
2569
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
2570
|
+
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
2571
|
+
}
|
|
2572
|
+
const boolean = {};
|
|
2573
|
+
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
2574
|
+
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
2575
|
+
}
|
|
2576
|
+
const geopoint = {};
|
|
2577
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
2578
|
+
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
2579
|
+
}
|
|
2580
|
+
const vectorData = {};
|
|
2581
|
+
for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
|
|
2582
|
+
vectorData[field] = {
|
|
2583
|
+
dimension: data2.dimension,
|
|
2584
|
+
vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
2585
|
+
hnswGraph: data2.hnsw_graph ? {
|
|
2586
|
+
entryPoint: data2.hnsw_graph.entry_point,
|
|
2587
|
+
maxLayer: data2.hnsw_graph.max_layer,
|
|
2588
|
+
m: data2.hnsw_graph.m,
|
|
2589
|
+
efConstruction: data2.hnsw_graph.ef_construction,
|
|
2590
|
+
metric: validateHnswMetric(data2.hnsw_graph.metric),
|
|
2591
|
+
nodes: data2.hnsw_graph.nodes
|
|
2592
|
+
} : null,
|
|
2593
|
+
codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
|
|
2594
|
+
};
|
|
2595
|
+
}
|
|
2596
|
+
return {
|
|
2597
|
+
indexName: raw.index_name,
|
|
2598
|
+
partitionId: raw.partition_id,
|
|
2599
|
+
totalPartitions: raw.total_partitions,
|
|
2600
|
+
language: raw.language ?? "english",
|
|
2601
|
+
schema: raw.schema ?? {},
|
|
2602
|
+
docCount: raw.doc_count ?? 0,
|
|
2603
|
+
avgDocLength: raw.avg_doc_length ?? 0,
|
|
2604
|
+
documents,
|
|
2605
|
+
invertedIndex,
|
|
2606
|
+
fieldIndexes: {
|
|
2607
|
+
numeric,
|
|
2608
|
+
boolean,
|
|
2609
|
+
enum: raw.field_indexes?.enum ?? {},
|
|
2610
|
+
geopoint,
|
|
2611
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
2612
|
+
},
|
|
2613
|
+
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
2614
|
+
vectorData,
|
|
2615
|
+
statistics: {
|
|
2616
|
+
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
2617
|
+
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
2618
|
+
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
2619
|
+
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
2620
|
+
}
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
|
|
2624
|
+
// src/persistence/durability/earlier-layout/bundle.ts
|
|
3002
2625
|
var SNAPSHOT_BUNDLE_VERSION = 2;
|
|
3003
2626
|
function invalid(message) {
|
|
3004
2627
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Snapshot bundle ${message}`);
|
|
@@ -3058,7 +2681,7 @@ async function loadSnapshotBundle(bytes, deps) {
|
|
|
3058
2681
|
while (deps.manager.partitionCount < bundle.partitions.length) deps.manager.addPartition();
|
|
3059
2682
|
if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
|
|
3060
2683
|
for (let i = 0; i < bundle.partitions.length; i += 1) {
|
|
3061
|
-
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
|
|
2684
|
+
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]), deserializePartition);
|
|
3062
2685
|
}
|
|
3063
2686
|
for (const [fieldPath, parts] of Object.entries(bundle.vectorIndexes)) {
|
|
3064
2687
|
const rebuildsGraph = parts.some((part) => part.graphs.some((graph) => graph.m < MIN_M));
|
|
@@ -3366,6 +2989,8 @@ function validateMergedPartition(merged) {
|
|
|
3366
2989
|
}
|
|
3367
2990
|
}
|
|
3368
2991
|
}
|
|
2992
|
+
|
|
2993
|
+
// src/persistence/durability/earlier-layout/segment.ts
|
|
3369
2994
|
function normalizeTombstones(raw) {
|
|
3370
2995
|
if (raw === void 0) return [];
|
|
3371
2996
|
if (!Array.isArray(raw) || !raw.every((value) => typeof value === "string")) {
|
|
@@ -3404,7 +3029,6 @@ var decoder = new TextDecoder();
|
|
|
3404
3029
|
async function countPagedPartition(store, partition) {
|
|
3405
3030
|
const { directory } = store;
|
|
3406
3031
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
3407
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-NIYMEA5R.mjs');
|
|
3408
3032
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
3409
3033
|
const scans = [];
|
|
3410
3034
|
try {
|
|
@@ -3456,6 +3080,8 @@ async function countCheckpointDocuments(store, indexName) {
|
|
|
3456
3080
|
const bundle = await store.read(snapshotBundleKey(indexName));
|
|
3457
3081
|
return bundle === null ? 0 : countSnapshotBundleDocuments(await decodeSnapshotBundle(bundle));
|
|
3458
3082
|
}
|
|
3083
|
+
|
|
3084
|
+
// src/persistence/durability/earlier-layout/load.ts
|
|
3459
3085
|
function removeWhatTheSegmentReplaces(manager, segment) {
|
|
3460
3086
|
manager.beginBatchRemove();
|
|
3461
3087
|
try {
|
|
@@ -3471,20 +3097,21 @@ function removeWhatTheSegmentReplaces(manager, segment) {
|
|
|
3471
3097
|
}
|
|
3472
3098
|
function attachDocuments(manager, partitionId, segment) {
|
|
3473
3099
|
if (segment.documents.length === 0) return;
|
|
3474
|
-
const
|
|
3100
|
+
const outputs = buildSegmentPayloads(
|
|
3475
3101
|
segment.documents,
|
|
3476
3102
|
manager.config.schema,
|
|
3477
3103
|
manager.language,
|
|
3478
3104
|
resolvePartitionInsertOptions(manager.config, manager.analysis),
|
|
3479
3105
|
manager.config.trackPositions ?? true
|
|
3480
3106
|
);
|
|
3481
|
-
const
|
|
3482
|
-
|
|
3483
|
-
|
|
3484
|
-
|
|
3485
|
-
|
|
3107
|
+
const prepared = outputs.map(({ payload, documents: members }) => {
|
|
3108
|
+
const documents = members.map((entry) => entry.document);
|
|
3109
|
+
return { payload, documents, snapshot: freezeSegmentShared(payload, documents) };
|
|
3110
|
+
});
|
|
3111
|
+
for (const { payload, documents, snapshot } of prepared) {
|
|
3112
|
+
if (snapshot === null) manager.mergeSegment(partitionId, payload, documents);
|
|
3113
|
+
else manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
|
|
3486
3114
|
}
|
|
3487
|
-
manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
|
|
3488
3115
|
}
|
|
3489
3116
|
async function loadPartition(store, indexName, partition, deps) {
|
|
3490
3117
|
if (deps.storedTermsAreCurrent) {
|
|
@@ -3505,7 +3132,7 @@ async function loadPartition(store, indexName, partition, deps) {
|
|
|
3505
3132
|
totalPartitions: deps.manager.partitionCount,
|
|
3506
3133
|
language: deps.manager.language.name
|
|
3507
3134
|
});
|
|
3508
|
-
deps.manager.deserializePartition(partition.partitionId, merged);
|
|
3135
|
+
deps.manager.deserializePartition(partition.partitionId, merged, deserializePartition);
|
|
3509
3136
|
}
|
|
3510
3137
|
async function readGraph(store, ref) {
|
|
3511
3138
|
if (ref.graphKey === null) return null;
|
|
@@ -3704,13 +3331,68 @@ async function recoverCheckpoint(deps) {
|
|
|
3704
3331
|
return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
|
|
3705
3332
|
}
|
|
3706
3333
|
|
|
3707
|
-
// src/persistence/durability/
|
|
3708
|
-
function
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3334
|
+
// src/persistence/durability/layout-transition/recovery.ts
|
|
3335
|
+
async function settleStoredLayout(store, metadata) {
|
|
3336
|
+
const transition = await readTransition(store, metadata.indexName);
|
|
3337
|
+
const decoded = await readSegmentManifest(store, metadata.indexName);
|
|
3338
|
+
const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
|
|
3339
|
+
if (transition !== null && partitionCount !== transition.oldPartitionCount && partitionCount !== transition.newPartitionCount) {
|
|
3340
|
+
invalidTransition("matches neither the manifest nor the metadata");
|
|
3341
|
+
}
|
|
3342
|
+
if (transition !== null && decoded === null && partitionCount !== transition.oldPartitionCount) {
|
|
3343
|
+
invalidTransition("has no old checkpoint or matching metadata");
|
|
3344
|
+
}
|
|
3345
|
+
const changedMetadata = partitionCount !== metadata.partitionCount;
|
|
3346
|
+
metadata.partitionCount = partitionCount;
|
|
3347
|
+
return {
|
|
3348
|
+
transition,
|
|
3349
|
+
changedMetadata,
|
|
3350
|
+
replaysOldLayout: transition !== null && partitionCount === transition.oldPartitionCount
|
|
3351
|
+
};
|
|
3352
|
+
}
|
|
3353
|
+
async function checkRetiredHead(directory, indexName, head) {
|
|
3354
|
+
const wal = await readWalSegments(directory, indexName, head.partitionId);
|
|
3355
|
+
if (wal === null) return;
|
|
3356
|
+
if (wal.highestReadFromWal > head.lastSeqNo || wal.marker.state.highestDurableSeqNo > head.lastSeqNo || wal.activeTail?.entries.some((entry) => entry.seqNo > head.lastSeqNo)) {
|
|
3357
|
+
invalidTransition("has records above a retired partition head");
|
|
3358
|
+
}
|
|
3359
|
+
}
|
|
3360
|
+
async function replayAcrossLayout(directory, indexName, transition, checkpoint, deps) {
|
|
3361
|
+
const terms = [];
|
|
3362
|
+
for (const head of transition.oldHeads) {
|
|
3363
|
+
const from = checkpointLastSeqNo(checkpoint, head.partitionId);
|
|
3364
|
+
if (from > head.lastSeqNo) invalidTransition("has an old head below its checkpoint");
|
|
3365
|
+
const replayed2 = await replayWalUpTo(directory, indexName, head.partitionId, from, head.lastSeqNo, deps);
|
|
3366
|
+
if (replayed2.highestSeqNo < head.lastSeqNo) invalidTransition("is missing an admitted old record");
|
|
3367
|
+
terms[head.partitionId] = Math.max(
|
|
3368
|
+
head.primaryTerm,
|
|
3369
|
+
replayed2.highestPrimaryTerm,
|
|
3370
|
+
checkpointPrimaryTerm(checkpoint, head.partitionId)
|
|
3371
|
+
);
|
|
3372
|
+
if (head.partitionId >= transition.newPartitionCount) await checkRetiredHead(directory, indexName, head);
|
|
3373
|
+
}
|
|
3374
|
+
const spill = await rewriteSpillOf({ directory, writeRecords: deps.writeRecords ?? null }, deps.manager);
|
|
3375
|
+
await createRebalancer().rebalance(
|
|
3376
|
+
deps.manager,
|
|
3377
|
+
transition.newPartitionCount,
|
|
3378
|
+
createPartitionRouter(),
|
|
3379
|
+
void 0,
|
|
3380
|
+
spill
|
|
3381
|
+
);
|
|
3382
|
+
const replayed = [];
|
|
3383
|
+
for (let partitionId = 0; partitionId < transition.newPartitionCount; partitionId++) {
|
|
3384
|
+
const head = transition.oldHeads[partitionId];
|
|
3385
|
+
const next = await replayWal(directory, indexName, partitionId, head?.lastSeqNo ?? 0, deps);
|
|
3386
|
+
replayed.push({
|
|
3387
|
+
partitionId,
|
|
3388
|
+
lastSeqNo: next.highestSeqNo,
|
|
3389
|
+
primaryTerm: Math.max(terms[partitionId] ?? SINGLE_NODE_PRIMARY_TERM, next.highestPrimaryTerm)
|
|
3390
|
+
});
|
|
3391
|
+
}
|
|
3392
|
+
return { checkpoint: replayed, removeUnread: async () => spill?.removeUnread(deps.manager) };
|
|
3713
3393
|
}
|
|
3394
|
+
|
|
3395
|
+
// src/persistence/durability/stored-analysis.ts
|
|
3714
3396
|
function promotePendingAnalysis(metadata) {
|
|
3715
3397
|
const pending = metadata.pendingAnalysis;
|
|
3716
3398
|
if (pending === void 0) return;
|
|
@@ -3724,9 +3406,13 @@ function promotePendingAnalysis(metadata) {
|
|
|
3724
3406
|
}
|
|
3725
3407
|
async function settleStoredAnalysis(store, metadata, metadataOnly) {
|
|
3726
3408
|
if (metadataOnly && metadata.pendingAnalysis === void 0) return false;
|
|
3727
|
-
const
|
|
3409
|
+
const decoded = await readSegmentManifest(store, metadata.indexName);
|
|
3410
|
+
const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
|
|
3411
|
+
const changedLayout = partitionCount !== metadata.partitionCount;
|
|
3412
|
+
metadata.partitionCount = partitionCount;
|
|
3413
|
+
const stored = decoded === null ? null : decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
|
|
3728
3414
|
const current = metadata.analysisGeneration ?? 0;
|
|
3729
|
-
if (stored === null || stored === current) return
|
|
3415
|
+
if (stored === null || stored === current) return changedLayout;
|
|
3730
3416
|
if (stored === metadata.pendingAnalysis?.generation) {
|
|
3731
3417
|
promotePendingAnalysis(metadata);
|
|
3732
3418
|
return true;
|
|
@@ -3750,15 +3436,16 @@ async function writeCountedMetadata(deps, indexName, documentCount) {
|
|
|
3750
3436
|
if (metadata === void 0) return;
|
|
3751
3437
|
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
3752
3438
|
await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
3753
|
-
})
|
|
3439
|
+
});
|
|
3754
3440
|
}
|
|
3755
3441
|
async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
3756
3442
|
const { directory, hooks } = deps;
|
|
3757
3443
|
const store = directoryCheckpointStore(directory);
|
|
3758
3444
|
const metadata = await loadMetadata(directory, indexName);
|
|
3759
3445
|
if (metadata === null) {
|
|
3760
|
-
return;
|
|
3446
|
+
return null;
|
|
3761
3447
|
}
|
|
3448
|
+
const layout = metadataOnly ? { transition: null, changedMetadata: false, replaysOldLayout: false } : await settleStoredLayout(store, metadata);
|
|
3762
3449
|
let derivedCount;
|
|
3763
3450
|
if (metadataOnly && metadata.documentCount === void 0) {
|
|
3764
3451
|
try {
|
|
@@ -3771,15 +3458,15 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3771
3458
|
const promoted = await settleStoredAnalysis(store, metadata, metadataOnly);
|
|
3772
3459
|
await hooks.createIndexFromMetadata(metadata, !metadataOnly);
|
|
3773
3460
|
if (metadataOnly) {
|
|
3774
|
-
if (derivedCount !== void 0 || promoted) {
|
|
3461
|
+
if (derivedCount !== void 0 || promoted || layout.changedMetadata) {
|
|
3775
3462
|
await writeCountedMetadata(deps, indexName, derivedCount ?? metadata.documentCount ?? 0);
|
|
3776
3463
|
}
|
|
3777
|
-
return;
|
|
3464
|
+
return null;
|
|
3778
3465
|
}
|
|
3779
3466
|
const manager = hooks.getManager(indexName);
|
|
3780
3467
|
const current = hooks.buildMetadata(indexName);
|
|
3781
3468
|
if (manager === void 0 || current === void 0) {
|
|
3782
|
-
return;
|
|
3469
|
+
return null;
|
|
3783
3470
|
}
|
|
3784
3471
|
const writeRecords = hooks.writeRecords ?? null;
|
|
3785
3472
|
writeRecords?.dropIndex(indexName);
|
|
@@ -3790,7 +3477,13 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3790
3477
|
writeRecords,
|
|
3791
3478
|
defersGraphBuilds: true
|
|
3792
3479
|
};
|
|
3480
|
+
let removeUnread = async () => void 0;
|
|
3793
3481
|
const replay = async (checkpoint) => {
|
|
3482
|
+
if (layout.replaysOldLayout && layout.transition !== null) {
|
|
3483
|
+
const replayed2 = await replayAcrossLayout(directory, indexName, layout.transition, checkpoint, replayDeps);
|
|
3484
|
+
removeUnread = replayed2.removeUnread;
|
|
3485
|
+
return replayed2.checkpoint;
|
|
3486
|
+
}
|
|
3794
3487
|
const replayed = [];
|
|
3795
3488
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
3796
3489
|
const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
|
|
@@ -3834,9 +3527,19 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3834
3527
|
for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
|
|
3835
3528
|
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
3836
3529
|
}
|
|
3837
|
-
if (recovered.upgradedDocumentCount !== null || promoted) {
|
|
3530
|
+
if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
|
|
3838
3531
|
await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
|
|
3839
3532
|
}
|
|
3533
|
+
return layout.transition === null ? null : removeUnread;
|
|
3534
|
+
}
|
|
3535
|
+
async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkpoint) {
|
|
3536
|
+
const removeUnread = await recoverPersistedIndex(deps, indexName, metadataOnly);
|
|
3537
|
+
if (removeUnread === null) return;
|
|
3538
|
+
try {
|
|
3539
|
+
await checkpoint();
|
|
3540
|
+
} finally {
|
|
3541
|
+
await removeUnread();
|
|
3542
|
+
}
|
|
3840
3543
|
}
|
|
3841
3544
|
|
|
3842
3545
|
// src/persistence/durability/manager.ts
|
|
@@ -3913,21 +3616,25 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3913
3616
|
}
|
|
3914
3617
|
}
|
|
3915
3618
|
}
|
|
3916
|
-
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false) {
|
|
3619
|
+
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false, requested = false) {
|
|
3917
3620
|
const indexState = indexes.get(indexName);
|
|
3918
3621
|
if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
|
|
3919
|
-
return
|
|
3920
|
-
|
|
3921
|
-
|
|
3922
|
-
|
|
3923
|
-
|
|
3924
|
-
|
|
3622
|
+
return holdScheduledCheckpoint(
|
|
3623
|
+
checkpointInTurn(indexState, {
|
|
3624
|
+
joinsInFlight: !afterInFlight && !pinsRetiredVectors,
|
|
3625
|
+
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
3626
|
+
perform: () => performCheckpoint(indexName, indexState, fromMemory),
|
|
3627
|
+
afterSettling: () => releaseStalledWrites(indexState)
|
|
3628
|
+
}),
|
|
3629
|
+
requested
|
|
3630
|
+
);
|
|
3925
3631
|
}
|
|
3926
3632
|
async function requestedCheckpoint(indexName, fromMemory = false, pinsVectors = false) {
|
|
3927
3633
|
heldCheckpoints.refuse(indexName, indexes.get(indexName));
|
|
3928
|
-
|
|
3634
|
+
await requireCheckpointLayout(hooks.getManager(indexName));
|
|
3635
|
+
return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors, true);
|
|
3929
3636
|
}
|
|
3930
|
-
async function performCheckpoint(indexName, indexState, fromMemory = false
|
|
3637
|
+
async function performCheckpoint(indexName, indexState, fromMemory = false) {
|
|
3931
3638
|
await directoryWatch.verify();
|
|
3932
3639
|
if (fatalError !== null) throw fatalError;
|
|
3933
3640
|
await runDurableCheckpoint({
|
|
@@ -3938,7 +3645,6 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3938
3645
|
compactionThreshold,
|
|
3939
3646
|
canOffload: canOffloadCheckpoint,
|
|
3940
3647
|
fromMemory,
|
|
3941
|
-
pinsRetiredVectors,
|
|
3942
3648
|
mutationThreshold: checkpointMutationThreshold,
|
|
3943
3649
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
3944
3650
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
@@ -3970,9 +3676,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3970
3676
|
const recordedByIndex = /* @__PURE__ */ new Map();
|
|
3971
3677
|
for (let i = 0; i < records.length; i++) {
|
|
3972
3678
|
if (!outcomes[i].ok) continue;
|
|
3973
|
-
const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0 };
|
|
3679
|
+
const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0, deferred: false };
|
|
3974
3680
|
recorded.records += 1;
|
|
3975
3681
|
recorded.documentBytes += records[i].document?.byteLength ?? 0;
|
|
3682
|
+
recorded.deferred ||= hasDeferredPublication(records[i]);
|
|
3976
3683
|
recordedByIndex.set(records[i].indexName, recorded);
|
|
3977
3684
|
}
|
|
3978
3685
|
for (const [indexName, recorded] of recordedByIndex) {
|
|
@@ -3984,7 +3691,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3984
3691
|
markFatal(toError(err));
|
|
3985
3692
|
});
|
|
3986
3693
|
}
|
|
3987
|
-
await stallWhileCheckpointsFallBehind(indexState);
|
|
3694
|
+
if (!recorded.deferred) await stallWhileCheckpointsFallBehind(indexState);
|
|
3988
3695
|
}
|
|
3989
3696
|
if (recordedByIndex.size === 0) return outcomes;
|
|
3990
3697
|
if (mode === "sync") await directoryWatch.verify();
|
|
@@ -4019,7 +3726,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4019
3726
|
syncStorage
|
|
4020
3727
|
};
|
|
4021
3728
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
4022
|
-
|
|
3729
|
+
const recovered = recoverWithLayoutCheckpoint(
|
|
3730
|
+
recoveryDeps,
|
|
3731
|
+
indexName,
|
|
3732
|
+
metadataOnly,
|
|
3733
|
+
() => performCheckpoint(indexName, getOrCreateIndexState(indexName), true)
|
|
3734
|
+
);
|
|
3735
|
+
return loads.track(indexName, recovered, startsGraphRepairs);
|
|
4023
3736
|
}
|
|
4024
3737
|
return {
|
|
4025
3738
|
isActive: () => true,
|
|
@@ -4101,7 +3814,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4101
3814
|
},
|
|
4102
3815
|
async checkpointAll() {
|
|
4103
3816
|
for (const indexName of [...indexes.keys()]) {
|
|
4104
|
-
await
|
|
3817
|
+
await requestedCheckpoint(indexName);
|
|
4105
3818
|
}
|
|
4106
3819
|
},
|
|
4107
3820
|
...unloading,
|
|
@@ -4127,31 +3840,47 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
4127
3840
|
}
|
|
4128
3841
|
|
|
4129
3842
|
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
4130
|
-
function capture(input, prior) {
|
|
3843
|
+
function capture(input, prior, cutoffs) {
|
|
4131
3844
|
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
4132
3845
|
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
4133
3846
|
const targets = [];
|
|
4134
3847
|
const contents = [];
|
|
3848
|
+
const positions = /* @__PURE__ */ new Map();
|
|
4135
3849
|
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
4136
|
-
const lastSeqNo =
|
|
3850
|
+
const lastSeqNo = Math.min(
|
|
3851
|
+
input.appliedSeqNoByPartition.get(partitionId) ?? 0,
|
|
3852
|
+
cutoffs.get(partitionId) ?? Infinity
|
|
3853
|
+
);
|
|
3854
|
+
positions.set(partitionId, lastSeqNo);
|
|
4137
3855
|
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
4138
3856
|
if (unchanged) continue;
|
|
4139
3857
|
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
4140
3858
|
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
4141
3859
|
}
|
|
4142
3860
|
const plans = /* @__PURE__ */ new Map();
|
|
4143
|
-
|
|
4144
|
-
const
|
|
4145
|
-
|
|
3861
|
+
try {
|
|
3862
|
+
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
3863
|
+
const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
|
|
3864
|
+
plans.set(
|
|
3865
|
+
fieldPath,
|
|
3866
|
+
vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), false, true)
|
|
3867
|
+
);
|
|
3868
|
+
}
|
|
3869
|
+
return {
|
|
3870
|
+
targets,
|
|
3871
|
+
plans,
|
|
3872
|
+
documentCount: input.manager.countDocuments(),
|
|
3873
|
+
contents,
|
|
3874
|
+
positions,
|
|
3875
|
+
partitionCount: input.manager.partitionCount
|
|
3876
|
+
};
|
|
3877
|
+
} catch (error) {
|
|
3878
|
+
for (const plan of plans.values()) plan.release();
|
|
3879
|
+
throw error;
|
|
4146
3880
|
}
|
|
4147
|
-
return { targets, plans, documentCount: input.manager.countDocuments(), contents };
|
|
4148
3881
|
}
|
|
4149
3882
|
async function writeCaptured(input, store, prior, captured) {
|
|
4150
|
-
const
|
|
4151
|
-
const priorWithinCount = prior === null ? null : {
|
|
4152
|
-
checkpoint: prior.checkpoint.filter((entry) => entry.partitionId < partitionCount),
|
|
4153
|
-
partitions: prior.partitions.filter((partition) => partition.partitionId < partitionCount)
|
|
4154
|
-
};
|
|
3883
|
+
const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, captured.partitionCount);
|
|
4155
3884
|
const written = await writeCheckpointSegments({
|
|
4156
3885
|
store,
|
|
4157
3886
|
metadata: input.metadata,
|
|
@@ -4177,9 +3906,17 @@ async function writeSnapshotOnlyCheckpoint(input) {
|
|
|
4177
3906
|
const store = adapterCheckpointStore(input.adapter);
|
|
4178
3907
|
for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
4179
3908
|
const prior = await readCurrentManifest(store, input.indexName);
|
|
4180
|
-
const captured = capture(input, prior);
|
|
4181
|
-
|
|
4182
|
-
|
|
3909
|
+
const captured = await captureAppliedPrefix(input.manager, (cutoffs) => capture(input, prior, cutoffs));
|
|
3910
|
+
try {
|
|
3911
|
+
await writeCaptured(input, store, prior, captured);
|
|
3912
|
+
return {
|
|
3913
|
+
documentCount: captured.documentCount,
|
|
3914
|
+
positions: captured.positions,
|
|
3915
|
+
partitionCount: captured.partitionCount
|
|
3916
|
+
};
|
|
3917
|
+
} finally {
|
|
3918
|
+
for (const plan of captured.plans.values()) plan.release();
|
|
3919
|
+
}
|
|
4183
3920
|
}
|
|
4184
3921
|
|
|
4185
3922
|
// src/persistence/durability/snapshot-only/index-state.ts
|
|
@@ -4192,6 +3929,20 @@ function createSnapshotIndexState() {
|
|
|
4192
3929
|
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
4193
3930
|
};
|
|
4194
3931
|
}
|
|
3932
|
+
function metadataKey(indexName) {
|
|
3933
|
+
return `${indexName}/meta`;
|
|
3934
|
+
}
|
|
3935
|
+
function noteSnapshotCommitted(state, positions, partitionCount) {
|
|
3936
|
+
let recordsLeft = 0;
|
|
3937
|
+
for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
|
|
3938
|
+
if (partitionId >= partitionCount) {
|
|
3939
|
+
state.appliedSeqNoByPartition.delete(partitionId);
|
|
3940
|
+
continue;
|
|
3941
|
+
}
|
|
3942
|
+
recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
|
|
3943
|
+
}
|
|
3944
|
+
state.mutationsSinceCheckpoint = recordsLeft;
|
|
3945
|
+
}
|
|
4195
3946
|
|
|
4196
3947
|
// src/persistence/durability/snapshot-only/restores.ts
|
|
4197
3948
|
function toError5(error) {
|
|
@@ -4231,9 +3982,6 @@ function createPendingRestores(onFatalError) {
|
|
|
4231
3982
|
}
|
|
4232
3983
|
|
|
4233
3984
|
// src/persistence/durability/snapshot-only/index.ts
|
|
4234
|
-
function metadataKey(indexName) {
|
|
4235
|
-
return `${indexName}/meta`;
|
|
4236
|
-
}
|
|
4237
3985
|
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
4238
3986
|
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
4239
3987
|
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
@@ -4279,10 +4027,8 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4279
4027
|
await settleRestore(indexName);
|
|
4280
4028
|
const manager = hooks.getManager(indexName);
|
|
4281
4029
|
const metadata = hooks.buildTermsMetadata(indexName);
|
|
4282
|
-
if (manager === void 0 || metadata === void 0)
|
|
4283
|
-
|
|
4284
|
-
}
|
|
4285
|
-
const documentCount = await writeSnapshotOnlyCheckpoint({
|
|
4030
|
+
if (manager === void 0 || metadata === void 0) return;
|
|
4031
|
+
const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
|
|
4286
4032
|
adapter,
|
|
4287
4033
|
indexName,
|
|
4288
4034
|
metadata,
|
|
@@ -4295,11 +4041,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4295
4041
|
await queueMetadataWrite(indexName, async () => {
|
|
4296
4042
|
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
|
|
4297
4043
|
if (checkpointMetadata === void 0) return;
|
|
4044
|
+
checkpointMetadata.partitionCount = partitionCount;
|
|
4298
4045
|
const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
4299
4046
|
await adapter.save(metadataKey(indexName), metadataBytes);
|
|
4300
|
-
hooks.recordCheckpoint?.(indexName, documentCount,
|
|
4047
|
+
hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
|
|
4301
4048
|
});
|
|
4302
|
-
indexState
|
|
4049
|
+
noteSnapshotCommitted(indexState, positions, partitionCount);
|
|
4303
4050
|
if (publisher !== void 0) {
|
|
4304
4051
|
const partitionIds = [];
|
|
4305
4052
|
for (let i = 0; i < manager.partitionCount; i += 1) {
|
|
@@ -4308,15 +4055,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4308
4055
|
await publisher.publishPartitions(indexName, partitionIds);
|
|
4309
4056
|
}
|
|
4310
4057
|
}
|
|
4311
|
-
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
|
|
4058
|
+
async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, requested = false) {
|
|
4312
4059
|
const indexState = indexes.get(indexName);
|
|
4313
4060
|
if (indexState === void 0 || indexState.unloading) return;
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4061
|
+
if (requested) await requireCheckpointLayout(hooks.getManager(indexName));
|
|
4062
|
+
return holdScheduledCheckpoint(
|
|
4063
|
+
checkpointInTurn(indexState, {
|
|
4064
|
+
joinsInFlight: !afterInFlight,
|
|
4065
|
+
mayStart: () => !indexState.unloading && !shuttingDown,
|
|
4066
|
+
perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
|
|
4067
|
+
afterSettling: () => void 0
|
|
4068
|
+
}),
|
|
4069
|
+
requested
|
|
4070
|
+
);
|
|
4320
4071
|
}
|
|
4321
4072
|
function queueMetadataWrite(indexName, write) {
|
|
4322
4073
|
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
@@ -4346,7 +4097,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4346
4097
|
let derivedCount;
|
|
4347
4098
|
if (metadataOnly && metadata.documentCount === void 0) {
|
|
4348
4099
|
try {
|
|
4349
|
-
derivedCount = await
|
|
4100
|
+
derivedCount = await countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
|
|
4350
4101
|
metadata.documentCount = derivedCount;
|
|
4351
4102
|
} catch {
|
|
4352
4103
|
derivedCount = void 0;
|
|
@@ -4390,9 +4141,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4390
4141
|
await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
|
|
4391
4142
|
}
|
|
4392
4143
|
}
|
|
4393
|
-
function countStoredBundleDocuments(indexName) {
|
|
4394
|
-
return countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
|
|
4395
|
-
}
|
|
4396
4144
|
async function recordMutation(record) {
|
|
4397
4145
|
const indexState = getOrCreateIndexState(record.indexName);
|
|
4398
4146
|
const buffered = indexState.applyChain.then(async () => {
|
|
@@ -4472,10 +4220,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4472
4220
|
});
|
|
4473
4221
|
},
|
|
4474
4222
|
checkpoint(indexName) {
|
|
4475
|
-
return checkpointIndex(indexName);
|
|
4223
|
+
return checkpointIndex(indexName, false, false, true);
|
|
4476
4224
|
},
|
|
4477
4225
|
checkpointFromMemory(indexName) {
|
|
4478
|
-
return checkpointIndex(indexName, true);
|
|
4226
|
+
return checkpointIndex(indexName, true, true, true);
|
|
4479
4227
|
},
|
|
4480
4228
|
async checkpointAfterSwitch(indexName, swap) {
|
|
4481
4229
|
const indexState = getOrCreateIndexState(indexName);
|
|
@@ -4494,7 +4242,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4494
4242
|
},
|
|
4495
4243
|
async checkpointAll() {
|
|
4496
4244
|
for (const indexName of [...indexes.keys()]) {
|
|
4497
|
-
await checkpointIndex(indexName);
|
|
4245
|
+
await checkpointIndex(indexName, false, false, true);
|
|
4498
4246
|
}
|
|
4499
4247
|
},
|
|
4500
4248
|
stageRestore(indexName) {
|
|
@@ -4554,7 +4302,13 @@ function buildMetadata(indexName, hooks, documentCount) {
|
|
|
4554
4302
|
if (config === void 0) {
|
|
4555
4303
|
return void 0;
|
|
4556
4304
|
}
|
|
4557
|
-
|
|
4305
|
+
const manager = hooks.getManager(indexName);
|
|
4306
|
+
return metadataFromConfig(
|
|
4307
|
+
indexName,
|
|
4308
|
+
config,
|
|
4309
|
+
manager === void 0 ? void 0 : metadataPartitionCount(manager),
|
|
4310
|
+
documentCount
|
|
4311
|
+
);
|
|
4558
4312
|
}
|
|
4559
4313
|
function createDurabilityIntegration(tier, hooks) {
|
|
4560
4314
|
const managerHooks = {
|
|
@@ -4579,22 +4333,30 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4579
4333
|
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
4580
4334
|
};
|
|
4581
4335
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
4582
|
-
function partitionFor(indexName, docId) {
|
|
4583
|
-
const partitionManager = hooks.getManager(indexName);
|
|
4584
|
-
const partitionCount = partitionManager?.partitionCount ?? 1;
|
|
4585
|
-
return fnv1a(docId) % Math.max(1, partitionCount);
|
|
4586
|
-
}
|
|
4587
4336
|
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4337
|
+
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
4338
|
+
const partitionId = fnv1a(docId) % partitionCount;
|
|
4339
|
+
let queued;
|
|
4340
|
+
try {
|
|
4341
|
+
const record = {
|
|
4342
|
+
indexName,
|
|
4343
|
+
partitionId,
|
|
4344
|
+
operation,
|
|
4345
|
+
documentId: docId,
|
|
4346
|
+
document,
|
|
4347
|
+
async apply() {
|
|
4348
|
+
queued = await apply() ?? void 0;
|
|
4349
|
+
if (queued !== void 0) deferMutationPublication(record);
|
|
4350
|
+
}
|
|
4351
|
+
};
|
|
4352
|
+
const seqNo = await manager.recordMutation(record);
|
|
4353
|
+
queued?.commit(partitionId, partitionCount, seqNo);
|
|
4354
|
+
return { indexName, partitionId, seqNo };
|
|
4355
|
+
} catch (error) {
|
|
4356
|
+
queued?.fail(error);
|
|
4357
|
+
throw error;
|
|
4358
|
+
}
|
|
4596
4359
|
});
|
|
4597
|
-
return { indexName, partitionId, seqNo };
|
|
4598
4360
|
}
|
|
4599
4361
|
return {
|
|
4600
4362
|
manager,
|
|
@@ -4605,29 +4367,48 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4605
4367
|
const records = [];
|
|
4606
4368
|
const outcomes = new Array(mutations.length);
|
|
4607
4369
|
const recordPositions = [];
|
|
4370
|
+
const queued = [];
|
|
4608
4371
|
for (let i = 0; i < mutations.length; i++) {
|
|
4609
4372
|
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
4610
4373
|
const mutation = mutations[i];
|
|
4611
4374
|
try {
|
|
4612
|
-
|
|
4375
|
+
const record = {
|
|
4613
4376
|
indexName,
|
|
4614
|
-
partitionId:
|
|
4377
|
+
partitionId: 0,
|
|
4615
4378
|
operation: mutation.document === null ? "DELETE" : "INDEX",
|
|
4616
4379
|
documentId: mutation.docId,
|
|
4617
4380
|
document: mutation.document === null ? null : encodeDocument(mutation.document),
|
|
4618
|
-
apply
|
|
4619
|
-
|
|
4381
|
+
async apply() {
|
|
4382
|
+
queued[i] = await mutation.apply() ?? void 0;
|
|
4383
|
+
if (queued[i] !== void 0) deferMutationPublication(record);
|
|
4384
|
+
}
|
|
4385
|
+
};
|
|
4386
|
+
records.push(record);
|
|
4620
4387
|
recordPositions.push(i);
|
|
4621
4388
|
} catch (error) {
|
|
4622
4389
|
outcomes[i] = { ok: false, error };
|
|
4623
4390
|
}
|
|
4624
4391
|
}
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4392
|
+
return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
|
|
4393
|
+
let recorded;
|
|
4394
|
+
try {
|
|
4395
|
+
for (const record of records) {
|
|
4396
|
+
record.partitionId = fnv1a(record.documentId) % partitionCount;
|
|
4397
|
+
}
|
|
4398
|
+
recorded = await manager.recordMutations(records);
|
|
4399
|
+
} catch (error) {
|
|
4400
|
+
for (const entry of queued) entry?.fail(error);
|
|
4401
|
+
throw error;
|
|
4402
|
+
}
|
|
4403
|
+
for (let i = 0; i < recorded.length; i++) {
|
|
4404
|
+
const outcome = recorded[i];
|
|
4405
|
+
const entry = queued[recordPositions[i]];
|
|
4406
|
+
if (outcome.ok) entry?.commit(records[i].partitionId, partitionCount, outcome.seqNo);
|
|
4407
|
+
else entry?.fail(outcome.error);
|
|
4408
|
+
outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
|
|
4409
|
+
}
|
|
4410
|
+
return outcomes;
|
|
4411
|
+
});
|
|
4631
4412
|
},
|
|
4632
4413
|
recordRemove(indexName, docId, apply) {
|
|
4633
4414
|
return recordMutation(indexName, docId, "DELETE", null, apply);
|
|
@@ -4635,6 +4416,172 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4635
4416
|
};
|
|
4636
4417
|
}
|
|
4637
4418
|
|
|
4419
|
+
// src/engine/invalidation.ts
|
|
4420
|
+
function createInvalidationFromConfig(config, tierKind, deps, indexRegistry) {
|
|
4421
|
+
if (config?.invalidation === void 0) {
|
|
4422
|
+
return null;
|
|
4423
|
+
}
|
|
4424
|
+
if (tierKind !== "snapshot") {
|
|
4425
|
+
throw new NarsilError(
|
|
4426
|
+
ErrorCodes.CONFIG_INVALID,
|
|
4427
|
+
"Invalidation requires a shared persistence adapter. The write-ahead log tier owns its directory exclusively, so configure a non-filesystem persistence adapter to share data between instances"
|
|
4428
|
+
);
|
|
4429
|
+
}
|
|
4430
|
+
return createInvalidationIntegration({
|
|
4431
|
+
adapter: config.invalidation,
|
|
4432
|
+
instanceId: generateId(),
|
|
4433
|
+
...deps,
|
|
4434
|
+
listBroadcastIndexNames() {
|
|
4435
|
+
const indexes = [];
|
|
4436
|
+
for (const [name, entry] of indexRegistry) {
|
|
4437
|
+
if (entry.config.defaultScoring === "broadcast") indexes.push(name);
|
|
4438
|
+
}
|
|
4439
|
+
return indexes;
|
|
4440
|
+
}
|
|
4441
|
+
});
|
|
4442
|
+
}
|
|
4443
|
+
function createInvalidationIntegration(deps) {
|
|
4444
|
+
const { adapter, instanceId, reloadIndex, getManager, listBroadcastIndexNames, onError } = deps;
|
|
4445
|
+
const foreignStats = /* @__PURE__ */ new Map();
|
|
4446
|
+
const reloadChains = /* @__PURE__ */ new Map();
|
|
4447
|
+
let broadcastTimer = null;
|
|
4448
|
+
let stopped = false;
|
|
4449
|
+
function toError6(err) {
|
|
4450
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
4451
|
+
}
|
|
4452
|
+
function pruneStaleForeignStats(entries) {
|
|
4453
|
+
const cutoff = Date.now() - FOREIGN_STATS_TTL_MS;
|
|
4454
|
+
for (const [sourceInstanceId, entry] of entries) {
|
|
4455
|
+
if (entry.receivedAt < cutoff) {
|
|
4456
|
+
entries.delete(sourceInstanceId);
|
|
4457
|
+
}
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
function scheduleReload(indexName) {
|
|
4461
|
+
const chain = reloadChains.get(indexName) ?? Promise.resolve();
|
|
4462
|
+
const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(toError6(err)));
|
|
4463
|
+
reloadChains.set(indexName, next);
|
|
4464
|
+
}
|
|
4465
|
+
function storeForeignStats(indexName, sourceInstanceId, stats) {
|
|
4466
|
+
let entries = foreignStats.get(indexName);
|
|
4467
|
+
if (entries === void 0) {
|
|
4468
|
+
entries = /* @__PURE__ */ new Map();
|
|
4469
|
+
foreignStats.set(indexName, entries);
|
|
4470
|
+
}
|
|
4471
|
+
entries.set(sourceInstanceId, { stats, receivedAt: Date.now() });
|
|
4472
|
+
pruneStaleForeignStats(entries);
|
|
4473
|
+
}
|
|
4474
|
+
function handleEvent(event) {
|
|
4475
|
+
try {
|
|
4476
|
+
if (event.type === "partition") {
|
|
4477
|
+
if (event.sourceInstanceId === instanceId) {
|
|
4478
|
+
return;
|
|
4479
|
+
}
|
|
4480
|
+
scheduleReload(event.indexName);
|
|
4481
|
+
return;
|
|
4482
|
+
}
|
|
4483
|
+
if (event.instanceId === instanceId) {
|
|
4484
|
+
return;
|
|
4485
|
+
}
|
|
4486
|
+
storeForeignStats(event.indexName, event.instanceId, event.stats);
|
|
4487
|
+
} catch (err) {
|
|
4488
|
+
onError(toError6(err));
|
|
4489
|
+
}
|
|
4490
|
+
}
|
|
4491
|
+
async function publishOwnStatistics() {
|
|
4492
|
+
for (const indexName of listBroadcastIndexNames()) {
|
|
4493
|
+
const manager = getManager(indexName);
|
|
4494
|
+
if (manager === void 0) {
|
|
4495
|
+
continue;
|
|
4496
|
+
}
|
|
4497
|
+
const aggregate = manager.statisticsMessage();
|
|
4498
|
+
try {
|
|
4499
|
+
await adapter.publish({
|
|
4500
|
+
type: "statistics",
|
|
4501
|
+
indexName,
|
|
4502
|
+
instanceId,
|
|
4503
|
+
stats: {
|
|
4504
|
+
totalDocs: aggregate.totalDocuments,
|
|
4505
|
+
docFrequencies: aggregate.docFrequencies,
|
|
4506
|
+
totalFieldLengths: aggregate.totalFieldLengths
|
|
4507
|
+
}
|
|
4508
|
+
});
|
|
4509
|
+
} catch (err) {
|
|
4510
|
+
onError(toError6(err));
|
|
4511
|
+
}
|
|
4512
|
+
}
|
|
4513
|
+
}
|
|
4514
|
+
return {
|
|
4515
|
+
instanceId,
|
|
4516
|
+
async publishPartitions(indexName, partitions) {
|
|
4517
|
+
if (stopped || partitions.length === 0) {
|
|
4518
|
+
return;
|
|
4519
|
+
}
|
|
4520
|
+
try {
|
|
4521
|
+
await adapter.publish({
|
|
4522
|
+
type: "partition",
|
|
4523
|
+
indexName,
|
|
4524
|
+
partitions,
|
|
4525
|
+
timestamp: Date.now(),
|
|
4526
|
+
sourceInstanceId: instanceId
|
|
4527
|
+
});
|
|
4528
|
+
} catch (err) {
|
|
4529
|
+
onError(toError6(err));
|
|
4530
|
+
}
|
|
4531
|
+
},
|
|
4532
|
+
broadcastStats(indexName, tokens) {
|
|
4533
|
+
const manager = getManager(indexName);
|
|
4534
|
+
if (manager === void 0) {
|
|
4535
|
+
return void 0;
|
|
4536
|
+
}
|
|
4537
|
+
const own = manager.getAggregateStats(tokens);
|
|
4538
|
+
const entries = foreignStats.get(indexName);
|
|
4539
|
+
if (entries === void 0 || entries.size === 0) {
|
|
4540
|
+
return mergePartitionStats([own]);
|
|
4541
|
+
}
|
|
4542
|
+
pruneStaleForeignStats(entries);
|
|
4543
|
+
const combined = [own];
|
|
4544
|
+
for (const [, entry] of entries) {
|
|
4545
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
4546
|
+
for (const token of tokens) {
|
|
4547
|
+
if (Object.hasOwn(entry.stats.docFrequencies, token))
|
|
4548
|
+
docFrequencies[token] = entry.stats.docFrequencies[token];
|
|
4549
|
+
}
|
|
4550
|
+
combined.push({
|
|
4551
|
+
totalDocuments: entry.stats.totalDocs,
|
|
4552
|
+
docFrequencies,
|
|
4553
|
+
totalFieldLengths: entry.stats.totalFieldLengths
|
|
4554
|
+
});
|
|
4555
|
+
}
|
|
4556
|
+
return mergePartitionStats(combined);
|
|
4557
|
+
},
|
|
4558
|
+
async start() {
|
|
4559
|
+
await adapter.subscribe(handleEvent);
|
|
4560
|
+
if (broadcastTimer !== null || stopped) {
|
|
4561
|
+
return;
|
|
4562
|
+
}
|
|
4563
|
+
broadcastTimer = setInterval(() => {
|
|
4564
|
+
void publishOwnStatistics();
|
|
4565
|
+
}, STATISTICS_BROADCAST_INTERVAL_MS);
|
|
4566
|
+
if (typeof broadcastTimer.unref === "function") {
|
|
4567
|
+
broadcastTimer.unref();
|
|
4568
|
+
}
|
|
4569
|
+
},
|
|
4570
|
+
async shutdown() {
|
|
4571
|
+
stopped = true;
|
|
4572
|
+
if (broadcastTimer !== null) {
|
|
4573
|
+
clearInterval(broadcastTimer);
|
|
4574
|
+
broadcastTimer = null;
|
|
4575
|
+
}
|
|
4576
|
+
try {
|
|
4577
|
+
await adapter.shutdown();
|
|
4578
|
+
} catch (err) {
|
|
4579
|
+
onError(toError6(err));
|
|
4580
|
+
}
|
|
4581
|
+
}
|
|
4582
|
+
};
|
|
4583
|
+
}
|
|
4584
|
+
|
|
4638
4585
|
// src/engine/durability-wiring.ts
|
|
4639
4586
|
function createDurabilityFromTier(tier, wiring) {
|
|
4640
4587
|
return createDurabilityIntegration(tier, {
|
|
@@ -4665,9 +4612,1508 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4665
4612
|
keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
|
|
4666
4613
|
onHistoryMoved(indexName) {
|
|
4667
4614
|
if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
|
|
4668
|
-
|
|
4615
|
+
try {
|
|
4616
|
+
scheduleSyncRetention(wiring.engineCore(), indexName);
|
|
4617
|
+
} catch (error) {
|
|
4618
|
+
wiring.emitFatalError(error instanceof Error ? error : new Error(String(error)));
|
|
4619
|
+
}
|
|
4620
|
+
}
|
|
4621
|
+
});
|
|
4622
|
+
}
|
|
4623
|
+
async function reloadIndexAndCopies(orchestrator, durability, indexName) {
|
|
4624
|
+
const manager = durability?.manager;
|
|
4625
|
+
if (manager?.reloadIndex === void 0) return;
|
|
4626
|
+
const wasScaledOut = orchestrator.desyncIndex(indexName);
|
|
4627
|
+
try {
|
|
4628
|
+
await manager.reloadIndex(indexName);
|
|
4629
|
+
} finally {
|
|
4630
|
+
await orchestrator.resyncIndex(indexName, wasScaledOut);
|
|
4631
|
+
}
|
|
4632
|
+
}
|
|
4633
|
+
|
|
4634
|
+
// src/engine/export/lines.ts
|
|
4635
|
+
var NEWLINE = 10;
|
|
4636
|
+
function joined(parts, total) {
|
|
4637
|
+
if (parts.length === 1) return parts[0];
|
|
4638
|
+
const bytes = new Uint8Array(total);
|
|
4639
|
+
let offset = 0;
|
|
4640
|
+
for (const part of parts) {
|
|
4641
|
+
bytes.set(part, offset);
|
|
4642
|
+
offset += part.byteLength;
|
|
4643
|
+
}
|
|
4644
|
+
return bytes;
|
|
4645
|
+
}
|
|
4646
|
+
async function* exportLines(source, maxLineBytes) {
|
|
4647
|
+
const decoder3 = new TextDecoder("utf-8", { fatal: true });
|
|
4648
|
+
let parts = [];
|
|
4649
|
+
let pending = 0;
|
|
4650
|
+
let overflowing = false;
|
|
4651
|
+
let line = 0;
|
|
4652
|
+
const finishLine = () => {
|
|
4653
|
+
line += 1;
|
|
4654
|
+
if (overflowing) {
|
|
4655
|
+
overflowing = false;
|
|
4656
|
+
parts = [];
|
|
4657
|
+
pending = 0;
|
|
4658
|
+
return { line, refused: "too-long" };
|
|
4659
|
+
}
|
|
4660
|
+
const bytes = joined(parts, pending);
|
|
4661
|
+
parts = [];
|
|
4662
|
+
pending = 0;
|
|
4663
|
+
try {
|
|
4664
|
+
return { line, text: decoder3.decode(bytes) };
|
|
4665
|
+
} catch {
|
|
4666
|
+
return { line, refused: "not-utf8" };
|
|
4667
|
+
}
|
|
4668
|
+
};
|
|
4669
|
+
for await (const chunk of chunksOf(source)) {
|
|
4670
|
+
let start = 0;
|
|
4671
|
+
for (; ; ) {
|
|
4672
|
+
const newline = chunk.indexOf(NEWLINE, start);
|
|
4673
|
+
const end = newline < 0 ? chunk.byteLength : newline;
|
|
4674
|
+
const piece = chunk.subarray(start, end);
|
|
4675
|
+
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
4676
|
+
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
4677
|
+
pending += piece.byteLength;
|
|
4678
|
+
} else {
|
|
4679
|
+
overflowing = true;
|
|
4680
|
+
parts = [];
|
|
4681
|
+
}
|
|
4682
|
+
if (newline < 0) break;
|
|
4683
|
+
yield finishLine();
|
|
4684
|
+
start = newline + 1;
|
|
4669
4685
|
}
|
|
4686
|
+
}
|
|
4687
|
+
if (pending > 0 || overflowing) yield finishLine();
|
|
4688
|
+
}
|
|
4689
|
+
|
|
4690
|
+
// src/serialization/base64.ts
|
|
4691
|
+
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
4692
|
+
var PAD = 61;
|
|
4693
|
+
var BYTES_PER_GROUP = 3;
|
|
4694
|
+
var CHARACTERS_PER_GROUP = 4;
|
|
4695
|
+
var ASCII_LIMIT = 128;
|
|
4696
|
+
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
4697
|
+
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
4698
|
+
for (let at = 0; at < ALPHABET.length; at++) {
|
|
4699
|
+
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
4700
|
+
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
4701
|
+
}
|
|
4702
|
+
var asciiDecoder = new TextDecoder();
|
|
4703
|
+
function encodeBase64(bytes) {
|
|
4704
|
+
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
4705
|
+
let write = 0;
|
|
4706
|
+
let read = 0;
|
|
4707
|
+
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
4708
|
+
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
4709
|
+
out[write++] = ENCODE[group >>> 18];
|
|
4710
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
4711
|
+
out[write++] = ENCODE[group >>> 6 & 63];
|
|
4712
|
+
out[write++] = ENCODE[group & 63];
|
|
4713
|
+
}
|
|
4714
|
+
const left = bytes.byteLength - read;
|
|
4715
|
+
if (left > 0) {
|
|
4716
|
+
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
4717
|
+
out[write++] = ENCODE[group >>> 18];
|
|
4718
|
+
out[write++] = ENCODE[group >>> 12 & 63];
|
|
4719
|
+
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
4720
|
+
out[write++] = PAD;
|
|
4721
|
+
}
|
|
4722
|
+
return asciiDecoder.decode(out);
|
|
4723
|
+
}
|
|
4724
|
+
function sextetAt(text, at) {
|
|
4725
|
+
const code = text.charCodeAt(at);
|
|
4726
|
+
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
4727
|
+
}
|
|
4728
|
+
function decodeExactBase64(text) {
|
|
4729
|
+
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
4730
|
+
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
4731
|
+
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
4732
|
+
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
4733
|
+
let write = 0;
|
|
4734
|
+
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
4735
|
+
const a2 = sextetAt(text, at);
|
|
4736
|
+
const b2 = sextetAt(text, at + 1);
|
|
4737
|
+
const c2 = sextetAt(text, at + 2);
|
|
4738
|
+
const d = sextetAt(text, at + 3);
|
|
4739
|
+
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
4740
|
+
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
4741
|
+
out[write++] = group >>> 16;
|
|
4742
|
+
out[write++] = group >>> 8 & 255;
|
|
4743
|
+
out[write++] = group & 255;
|
|
4744
|
+
}
|
|
4745
|
+
if (padding === 0) return out;
|
|
4746
|
+
const a = sextetAt(text, fullEnd);
|
|
4747
|
+
const b = sextetAt(text, fullEnd + 1);
|
|
4748
|
+
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
4749
|
+
if ((a | b | c) < 0) return null;
|
|
4750
|
+
if (padding === 2 && (b & 15) !== 0) return null;
|
|
4751
|
+
if (padding === 1 && (c & 3) !== 0) return null;
|
|
4752
|
+
out[write++] = a << 2 | b >>> 4;
|
|
4753
|
+
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
4754
|
+
return out;
|
|
4755
|
+
}
|
|
4756
|
+
|
|
4757
|
+
// src/engine/export/tags.ts
|
|
4758
|
+
var TAG_PREFIX = "$";
|
|
4759
|
+
var DATE_TAG = "$date";
|
|
4760
|
+
var BYTES_TAG = "$bytes";
|
|
4761
|
+
var NUMBER_TAG = "$number";
|
|
4762
|
+
var OBJECT_TAG = "$object";
|
|
4763
|
+
var MILLISECONDS_PER_SECOND = 1e3;
|
|
4764
|
+
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
4765
|
+
var MAX_NANOSECONDS = 999999999;
|
|
4766
|
+
var MAX_DATE_MILLISECONDS = 864e13;
|
|
4767
|
+
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
4768
|
+
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
4769
|
+
["NaN", Number.NaN],
|
|
4770
|
+
["Infinity", Number.POSITIVE_INFINITY],
|
|
4771
|
+
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
4772
|
+
]);
|
|
4773
|
+
function refuse(path, reason) {
|
|
4774
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
4775
|
+
field: path
|
|
4670
4776
|
});
|
|
4671
4777
|
}
|
|
4778
|
+
function isRecord2(value) {
|
|
4779
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4780
|
+
}
|
|
4781
|
+
function startsWithTagPrefix(key) {
|
|
4782
|
+
return key.startsWith(TAG_PREFIX);
|
|
4783
|
+
}
|
|
4784
|
+
function soleTagKey(record) {
|
|
4785
|
+
let sole = null;
|
|
4786
|
+
for (const key of Object.keys(record)) {
|
|
4787
|
+
if (sole !== null) return null;
|
|
4788
|
+
sole = key;
|
|
4789
|
+
}
|
|
4790
|
+
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
4791
|
+
}
|
|
4792
|
+
function dateOf(value, path) {
|
|
4793
|
+
if (!Array.isArray(value) || value.length !== 2)
|
|
4794
|
+
refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
4795
|
+
const seconds = value[0];
|
|
4796
|
+
const nanoseconds = value[1];
|
|
4797
|
+
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
4798
|
+
refuse(
|
|
4799
|
+
path,
|
|
4800
|
+
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
4801
|
+
);
|
|
4802
|
+
}
|
|
4803
|
+
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
4804
|
+
refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
4805
|
+
}
|
|
4806
|
+
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
4807
|
+
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
4808
|
+
refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
4809
|
+
return new Date(milliseconds);
|
|
4810
|
+
}
|
|
4811
|
+
function bytesOf(value, path) {
|
|
4812
|
+
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
4813
|
+
if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
4814
|
+
return bytes;
|
|
4815
|
+
}
|
|
4816
|
+
function numberOf(value, path) {
|
|
4817
|
+
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
4818
|
+
if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
4819
|
+
return number;
|
|
4820
|
+
}
|
|
4821
|
+
function requireDepth(path, depth) {
|
|
4822
|
+
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
4823
|
+
}
|
|
4824
|
+
function untagMap(record, path, depth) {
|
|
4825
|
+
requireDepth(path, depth);
|
|
4826
|
+
for (const key of Object.keys(record)) {
|
|
4827
|
+
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
4828
|
+
}
|
|
4829
|
+
return record;
|
|
4830
|
+
}
|
|
4831
|
+
function untagList(list, path, depth) {
|
|
4832
|
+
requireDepth(path, depth);
|
|
4833
|
+
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
4834
|
+
return list;
|
|
4835
|
+
}
|
|
4836
|
+
function untagTag(tag, value, path, depth) {
|
|
4837
|
+
switch (tag) {
|
|
4838
|
+
case DATE_TAG:
|
|
4839
|
+
return dateOf(value, path);
|
|
4840
|
+
case BYTES_TAG:
|
|
4841
|
+
return bytesOf(value, path);
|
|
4842
|
+
case NUMBER_TAG:
|
|
4843
|
+
return numberOf(value, path);
|
|
4844
|
+
case OBJECT_TAG:
|
|
4845
|
+
if (!isRecord2(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
4846
|
+
return untagMap(value, path, depth);
|
|
4847
|
+
default:
|
|
4848
|
+
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
4849
|
+
}
|
|
4850
|
+
}
|
|
4851
|
+
function untag(value, path, depth) {
|
|
4852
|
+
if (typeof value !== "object" || value === null) return value;
|
|
4853
|
+
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
4854
|
+
const record = value;
|
|
4855
|
+
const tag = soleTagKey(record);
|
|
4856
|
+
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
4857
|
+
}
|
|
4858
|
+
function untagDocumentLine(text, parsed) {
|
|
4859
|
+
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
4860
|
+
return untagMap(parsed, "", 0);
|
|
4861
|
+
}
|
|
4862
|
+
|
|
4863
|
+
// src/engine/export/import.ts
|
|
4864
|
+
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
4865
|
+
var DOCUMENT_EXPORT_VERSION = 1;
|
|
4866
|
+
function invalidHeader(message) {
|
|
4867
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
4868
|
+
}
|
|
4869
|
+
function isRecord3(value) {
|
|
4870
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4871
|
+
}
|
|
4872
|
+
function configOfHeader(line) {
|
|
4873
|
+
if (line === void 0) invalidHeader("is missing");
|
|
4874
|
+
if (!("text" in line)) {
|
|
4875
|
+
invalidHeader(
|
|
4876
|
+
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
4877
|
+
);
|
|
4878
|
+
}
|
|
4879
|
+
let raw;
|
|
4880
|
+
try {
|
|
4881
|
+
raw = JSON.parse(line.text);
|
|
4882
|
+
} catch {
|
|
4883
|
+
invalidHeader("is not JSON");
|
|
4884
|
+
}
|
|
4885
|
+
if (!isRecord3(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
4886
|
+
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
4887
|
+
}
|
|
4888
|
+
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
4889
|
+
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
4890
|
+
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
4891
|
+
return {
|
|
4892
|
+
schema: configuration.schema,
|
|
4893
|
+
language: configuration.language,
|
|
4894
|
+
...fields,
|
|
4895
|
+
...embedding === void 0 ? {} : {
|
|
4896
|
+
embedding: {
|
|
4897
|
+
fields: embedding.fields,
|
|
4898
|
+
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
4899
|
+
}
|
|
4900
|
+
}
|
|
4901
|
+
};
|
|
4902
|
+
}
|
|
4903
|
+
async function prepareIndex(engine, indexName, config) {
|
|
4904
|
+
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
4905
|
+
await engine.createIndex(indexName, config);
|
|
4906
|
+
return;
|
|
4907
|
+
}
|
|
4908
|
+
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
4909
|
+
const exported = flattenSchema(config.schema);
|
|
4910
|
+
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
4911
|
+
for (const field of fields) {
|
|
4912
|
+
if (existing[field] === exported[field]) continue;
|
|
4913
|
+
throw new NarsilError(
|
|
4914
|
+
ErrorCodes.CONFIG_INVALID,
|
|
4915
|
+
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
4916
|
+
{ indexName, field }
|
|
4917
|
+
);
|
|
4918
|
+
}
|
|
4919
|
+
}
|
|
4920
|
+
function highestReportedAt(refused) {
|
|
4921
|
+
let highest = 0;
|
|
4922
|
+
for (let at = 1; at < refused.length; at++) {
|
|
4923
|
+
if (refused[at].line > refused[highest].line) highest = at;
|
|
4924
|
+
}
|
|
4925
|
+
return highest;
|
|
4926
|
+
}
|
|
4927
|
+
function refuse2(state, line, code, message) {
|
|
4928
|
+
state.result.refusedCount += 1;
|
|
4929
|
+
const refused = state.result.refused;
|
|
4930
|
+
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
4931
|
+
refused.push({ line, code, message });
|
|
4932
|
+
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
4933
|
+
return;
|
|
4934
|
+
}
|
|
4935
|
+
if (line >= refused[state.highestReported].line) return;
|
|
4936
|
+
refused[state.highestReported] = { line, code, message };
|
|
4937
|
+
state.highestReported = highestReportedAt(refused);
|
|
4938
|
+
}
|
|
4939
|
+
async function flush(state) {
|
|
4940
|
+
if (state.batch.length === 0) return;
|
|
4941
|
+
const batch = state.batch;
|
|
4942
|
+
state.batch = [];
|
|
4943
|
+
state.batchIds = /* @__PURE__ */ new Set();
|
|
4944
|
+
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
4945
|
+
const inserts = [];
|
|
4946
|
+
const updates = [];
|
|
4947
|
+
for (const entry of batch) {
|
|
4948
|
+
if (await state.engine.has(state.indexName, entry.docId))
|
|
4949
|
+
updates.push({ docId: entry.docId, document: entry.document });
|
|
4950
|
+
else inserts.push(entry.document);
|
|
4951
|
+
}
|
|
4952
|
+
for (const outcome of [
|
|
4953
|
+
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
4954
|
+
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
4955
|
+
]) {
|
|
4956
|
+
if (outcome === null) continue;
|
|
4957
|
+
state.result.imported += outcome.succeeded.length;
|
|
4958
|
+
for (const failure of outcome.failed) {
|
|
4959
|
+
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
4960
|
+
refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
4961
|
+
}
|
|
4962
|
+
}
|
|
4963
|
+
}
|
|
4964
|
+
async function takeLine(state, entry) {
|
|
4965
|
+
if (!("text" in entry)) {
|
|
4966
|
+
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
4967
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
4968
|
+
return;
|
|
4969
|
+
}
|
|
4970
|
+
if (entry.text.length === 0) return;
|
|
4971
|
+
let document;
|
|
4972
|
+
try {
|
|
4973
|
+
document = JSON.parse(entry.text);
|
|
4974
|
+
} catch {
|
|
4975
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
4976
|
+
return;
|
|
4977
|
+
}
|
|
4978
|
+
if (!isRecord3(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
4979
|
+
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
4980
|
+
return;
|
|
4981
|
+
}
|
|
4982
|
+
try {
|
|
4983
|
+
untagDocumentLine(entry.text, document);
|
|
4984
|
+
} catch (error) {
|
|
4985
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
4986
|
+
refuse2(state, entry.line, error.code, error.message);
|
|
4987
|
+
return;
|
|
4988
|
+
}
|
|
4989
|
+
if (state.batchIds.has(document.id)) await flush(state);
|
|
4990
|
+
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
4991
|
+
state.batchIds.add(document.id);
|
|
4992
|
+
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
4993
|
+
}
|
|
4994
|
+
async function readDocumentExport(engine, indexName, source) {
|
|
4995
|
+
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
4996
|
+
const header = await lines.next();
|
|
4997
|
+
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
4998
|
+
const state = {
|
|
4999
|
+
engine,
|
|
5000
|
+
indexName,
|
|
5001
|
+
batch: [],
|
|
5002
|
+
batchIds: /* @__PURE__ */ new Set(),
|
|
5003
|
+
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
5004
|
+
highestReported: 0
|
|
5005
|
+
};
|
|
5006
|
+
for await (const entry of lines) {
|
|
5007
|
+
await takeLine(state, entry);
|
|
5008
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
5009
|
+
}
|
|
5010
|
+
await flush(state);
|
|
5011
|
+
state.result.refused.sort((a, b) => a.line - b.line);
|
|
5012
|
+
return state.result;
|
|
5013
|
+
}
|
|
5014
|
+
|
|
5015
|
+
// src/engine/export/json-line.ts
|
|
5016
|
+
function refuse3(path, kind, docId) {
|
|
5017
|
+
throw new NarsilError(
|
|
5018
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5019
|
+
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
5020
|
+
{ docId, field: path }
|
|
5021
|
+
);
|
|
5022
|
+
}
|
|
5023
|
+
function encodeNumber(value, path, target) {
|
|
5024
|
+
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
5025
|
+
if (!target.tagged) refuse3(path, String(value), target.docId);
|
|
5026
|
+
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
5027
|
+
}
|
|
5028
|
+
function encodeDate(value, path, target) {
|
|
5029
|
+
const milliseconds = value.getTime();
|
|
5030
|
+
if (!target.tagged || Number.isNaN(milliseconds)) refuse3(path, "a date", target.docId);
|
|
5031
|
+
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
5032
|
+
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
5033
|
+
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
5034
|
+
}
|
|
5035
|
+
function encodeBytes(value, path, target) {
|
|
5036
|
+
if (!target.tagged) refuse3(path, "bytes", target.docId);
|
|
5037
|
+
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
5038
|
+
}
|
|
5039
|
+
function encodeList(value, path, target) {
|
|
5040
|
+
target.parts.push("[");
|
|
5041
|
+
for (let at = 0; at < value.length; at++) {
|
|
5042
|
+
if (at > 0) target.parts.push(",");
|
|
5043
|
+
encodeValue(value[at], `${path}[${at}]`, target);
|
|
5044
|
+
}
|
|
5045
|
+
target.parts.push("]");
|
|
5046
|
+
}
|
|
5047
|
+
function encodeMap(value, path, target) {
|
|
5048
|
+
const proto = Object.getPrototypeOf(value);
|
|
5049
|
+
if (proto !== Object.prototype && proto !== null) refuse3(path, "a value other than a plain object", target.docId);
|
|
5050
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
5051
|
+
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
5052
|
+
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
5053
|
+
for (let at = 0; at < entries.length; at++) {
|
|
5054
|
+
const [key, entry] = entries[at];
|
|
5055
|
+
if (at > 0) target.parts.push(",");
|
|
5056
|
+
target.parts.push(JSON.stringify(key), ":");
|
|
5057
|
+
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
5058
|
+
}
|
|
5059
|
+
target.parts.push(wrapped ? "}}" : "}");
|
|
5060
|
+
}
|
|
5061
|
+
function encodeValue(value, path, target) {
|
|
5062
|
+
if (value === null) {
|
|
5063
|
+
target.parts.push("null");
|
|
5064
|
+
return;
|
|
5065
|
+
}
|
|
5066
|
+
switch (typeof value) {
|
|
5067
|
+
case "string":
|
|
5068
|
+
target.parts.push(JSON.stringify(value));
|
|
5069
|
+
return;
|
|
5070
|
+
case "boolean":
|
|
5071
|
+
target.parts.push(value ? "true" : "false");
|
|
5072
|
+
return;
|
|
5073
|
+
case "number":
|
|
5074
|
+
target.parts.push(encodeNumber(value, path, target));
|
|
5075
|
+
return;
|
|
5076
|
+
case "object":
|
|
5077
|
+
break;
|
|
5078
|
+
default:
|
|
5079
|
+
refuse3(path, `a ${typeof value}`, target.docId);
|
|
5080
|
+
}
|
|
5081
|
+
if (Array.isArray(value)) encodeList(value, path, target);
|
|
5082
|
+
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
5083
|
+
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
5084
|
+
else encodeMap(value, path, target);
|
|
5085
|
+
}
|
|
5086
|
+
function encodeExactJson(value, docId) {
|
|
5087
|
+
const target = { docId, tagged: false, parts: [] };
|
|
5088
|
+
encodeValue(value, "", target);
|
|
5089
|
+
return target.parts.join("");
|
|
5090
|
+
}
|
|
5091
|
+
function encodeDocumentJson(document, docId) {
|
|
5092
|
+
const target = { docId, tagged: true, parts: [] };
|
|
5093
|
+
encodeValue(document, "", target);
|
|
5094
|
+
return target.parts.join("");
|
|
5095
|
+
}
|
|
5096
|
+
|
|
5097
|
+
// src/engine/export/paged-lines.ts
|
|
5098
|
+
async function coldFieldOf(checkpoint, field) {
|
|
5099
|
+
const { directory } = checkpoint;
|
|
5100
|
+
if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
|
|
5101
|
+
if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
|
|
5102
|
+
const fieldType = checkpoint.manifest.schema[field.fieldPath];
|
|
5103
|
+
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
5104
|
+
if (dimension === null) return null;
|
|
5105
|
+
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
5106
|
+
return openColdVectorField({
|
|
5107
|
+
fieldPath: field.fieldPath,
|
|
5108
|
+
dimension,
|
|
5109
|
+
metric: "cosine",
|
|
5110
|
+
codeBits: null,
|
|
5111
|
+
files: await Promise.all(field.files.map(withPath)),
|
|
5112
|
+
graphParts: await Promise.all(field.graphParts.map(withPath)),
|
|
5113
|
+
resolvePartition: () => void 0
|
|
5114
|
+
});
|
|
5115
|
+
}
|
|
5116
|
+
async function openFileVectorReads(checkpoint, wholeFileReads) {
|
|
5117
|
+
const fields = /* @__PURE__ */ new Map();
|
|
5118
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
5119
|
+
const opened = await coldFieldOf(checkpoint, field);
|
|
5120
|
+
if (opened !== null) fields.set(field.fieldPath, opened);
|
|
5121
|
+
}
|
|
5122
|
+
const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
|
|
5123
|
+
return {
|
|
5124
|
+
async vectorOf(fieldPath, docId) {
|
|
5125
|
+
const field = fields.get(fieldPath);
|
|
5126
|
+
if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
|
|
5127
|
+
const number = field.numberOf(docId);
|
|
5128
|
+
return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
|
|
5129
|
+
},
|
|
5130
|
+
close() {
|
|
5131
|
+
for (const field of fields.values()) field.close();
|
|
5132
|
+
fallback?.close();
|
|
5133
|
+
}
|
|
5134
|
+
};
|
|
5135
|
+
}
|
|
5136
|
+
async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
|
|
5137
|
+
const { directory } = checkpoint;
|
|
5138
|
+
if (directory === void 0) return;
|
|
5139
|
+
const scans = [];
|
|
5140
|
+
try {
|
|
5141
|
+
for (const ref of partition.segments) {
|
|
5142
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
5143
|
+
scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
|
|
5144
|
+
}
|
|
5145
|
+
for (let at = 0; at < scans.length; at++) {
|
|
5146
|
+
const scan = scans[at];
|
|
5147
|
+
for (let ordinal = 0; ordinal < scan.count; ordinal++) {
|
|
5148
|
+
const docId = scan.idAt(ordinal);
|
|
5149
|
+
if (!countsAsLatestHolder(scans, at, docId)) continue;
|
|
5150
|
+
const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
|
|
5151
|
+
document.id = docId;
|
|
5152
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
5153
|
+
const vector = await vectors.vectorOf(field.fieldPath, docId);
|
|
5154
|
+
if (vector !== null) setNestedValue(document, field.fieldPath, vector);
|
|
5155
|
+
}
|
|
5156
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
5157
|
+
`;
|
|
5158
|
+
}
|
|
5159
|
+
}
|
|
5160
|
+
} finally {
|
|
5161
|
+
for (const scan of scans) scan.close();
|
|
5162
|
+
}
|
|
5163
|
+
}
|
|
5164
|
+
|
|
5165
|
+
// src/engine/export/write.ts
|
|
5166
|
+
var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
|
|
5167
|
+
var FLOAT32_BYTES = 4;
|
|
5168
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
5169
|
+
async function locateVectors(checkpoint) {
|
|
5170
|
+
const located = /* @__PURE__ */ new Map();
|
|
5171
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
5172
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
5173
|
+
for (const ref of field.files) {
|
|
5174
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
5175
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
|
|
5176
|
+
what: `vector file "${ref.key}"`,
|
|
5177
|
+
allowsCompression: false
|
|
5178
|
+
});
|
|
5179
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
5180
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
5181
|
+
if (isDead(ref.dead, position)) continue;
|
|
5182
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
5183
|
+
}
|
|
5184
|
+
}
|
|
5185
|
+
located.set(field.fieldPath, byDocId);
|
|
5186
|
+
}
|
|
5187
|
+
return located;
|
|
5188
|
+
}
|
|
5189
|
+
async function readVector(checkpoint, location) {
|
|
5190
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
5191
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
5192
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
5193
|
+
}
|
|
5194
|
+
async function readSegment(checkpoint, ref) {
|
|
5195
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
5196
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
5197
|
+
what: `segment "${ref.key}"`,
|
|
5198
|
+
allowsCompression: true
|
|
5199
|
+
});
|
|
5200
|
+
return decodeKeywordSegment(payload, false);
|
|
5201
|
+
}
|
|
5202
|
+
var decoder2 = new TextDecoder();
|
|
5203
|
+
function idAt(contents, ordinal) {
|
|
5204
|
+
const ids = contents.arrays.idTable;
|
|
5205
|
+
return decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
5206
|
+
}
|
|
5207
|
+
async function winnersOf(checkpoint, partition) {
|
|
5208
|
+
const winners = /* @__PURE__ */ new Map();
|
|
5209
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
5210
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
5211
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
5212
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
5213
|
+
}
|
|
5214
|
+
return winners;
|
|
5215
|
+
}
|
|
5216
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
5217
|
+
const contents = await readSegment(checkpoint, ref);
|
|
5218
|
+
const table = contents.arrays.documentTable;
|
|
5219
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5220
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
5221
|
+
return {
|
|
5222
|
+
count: contents.arrays.documentCount,
|
|
5223
|
+
schema: contents.schema,
|
|
5224
|
+
tombstones: new Set(contents.tombstones),
|
|
5225
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
5226
|
+
holds: (docId) => ids.has(docId),
|
|
5227
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
5228
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
5229
|
+
table.discardedFields
|
|
5230
|
+
),
|
|
5231
|
+
close: () => void 0
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
5235
|
+
const located = await locateVectors(checkpoint);
|
|
5236
|
+
return {
|
|
5237
|
+
async vectorOf(fieldPath, docId) {
|
|
5238
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
5239
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
5240
|
+
},
|
|
5241
|
+
close: () => located.clear()
|
|
5242
|
+
};
|
|
5243
|
+
}
|
|
5244
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
5245
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
5246
|
+
try {
|
|
5247
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
5248
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
5249
|
+
}
|
|
5250
|
+
} finally {
|
|
5251
|
+
vectors.close();
|
|
5252
|
+
}
|
|
5253
|
+
}
|
|
5254
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
5255
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
5256
|
+
for (const ref of partition.segments) {
|
|
5257
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
5258
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
return false;
|
|
5262
|
+
}
|
|
5263
|
+
async function* documentLines(checkpoint) {
|
|
5264
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
5265
|
+
yield* pagedDocumentLines(checkpoint);
|
|
5266
|
+
return;
|
|
5267
|
+
}
|
|
5268
|
+
const vectors = await locateVectors(checkpoint);
|
|
5269
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
5270
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
5271
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
5272
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
5273
|
+
const table = contents.arrays.documentTable;
|
|
5274
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
5275
|
+
const docId = idAt(contents, ordinal);
|
|
5276
|
+
if (winners.get(docId) !== at) continue;
|
|
5277
|
+
const stored = decodeStoredDocument(
|
|
5278
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
5279
|
+
table.discardedFields
|
|
5280
|
+
);
|
|
5281
|
+
const document = { id: docId, ...stored };
|
|
5282
|
+
document.id = docId;
|
|
5283
|
+
for (const [fieldPath, located] of vectors) {
|
|
5284
|
+
const location = located.get(docId);
|
|
5285
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
5286
|
+
}
|
|
5287
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
5288
|
+
`;
|
|
5289
|
+
}
|
|
5290
|
+
}
|
|
5291
|
+
}
|
|
5292
|
+
}
|
|
5293
|
+
async function* exportLines2(header, checkpoint) {
|
|
5294
|
+
yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
|
|
5295
|
+
`;
|
|
5296
|
+
yield* documentLines(checkpoint);
|
|
5297
|
+
}
|
|
5298
|
+
function documentExportStream(header, checkpoint) {
|
|
5299
|
+
const lines = exportLines2(header, checkpoint);
|
|
5300
|
+
const encoder = new TextEncoder();
|
|
5301
|
+
let released = false;
|
|
5302
|
+
const release = () => {
|
|
5303
|
+
if (released) return;
|
|
5304
|
+
released = true;
|
|
5305
|
+
checkpoint.release();
|
|
5306
|
+
};
|
|
5307
|
+
return new ReadableStream({
|
|
5308
|
+
async pull(controller) {
|
|
5309
|
+
try {
|
|
5310
|
+
let text = "";
|
|
5311
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
5312
|
+
const step = await lines.next();
|
|
5313
|
+
if (step.done) {
|
|
5314
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
5315
|
+
release();
|
|
5316
|
+
controller.close();
|
|
5317
|
+
return;
|
|
5318
|
+
}
|
|
5319
|
+
text += step.value;
|
|
5320
|
+
}
|
|
5321
|
+
controller.enqueue(encoder.encode(text));
|
|
5322
|
+
} catch (error) {
|
|
5323
|
+
release();
|
|
5324
|
+
controller.error(error);
|
|
5325
|
+
}
|
|
5326
|
+
},
|
|
5327
|
+
async cancel() {
|
|
5328
|
+
release();
|
|
5329
|
+
await lines.return(void 0);
|
|
5330
|
+
}
|
|
5331
|
+
});
|
|
5332
|
+
}
|
|
5333
|
+
|
|
5334
|
+
// src/query-suggestions/archive-history.ts
|
|
5335
|
+
function inDayOrder(left, right) {
|
|
5336
|
+
if (left.day !== right.day) return left.day < right.day ? -1 : 1;
|
|
5337
|
+
return left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
|
|
5338
|
+
}
|
|
5339
|
+
async function* historyArchiveEntries(core, indexName) {
|
|
5340
|
+
const runtime = attachedRuntimeOf(core);
|
|
5341
|
+
const files = runtime?.host.files ?? historyFilesOf(core.durability?.manager.historyStorage?.() ?? null);
|
|
5342
|
+
await runtime?.saveDay(indexName);
|
|
5343
|
+
const names = [];
|
|
5344
|
+
for (const key of await listHistoryFolder(files, indexName)) {
|
|
5345
|
+
const name = historyFileOf(indexName, key);
|
|
5346
|
+
if (name !== null) names.push(name);
|
|
5347
|
+
}
|
|
5348
|
+
names.sort(inDayOrder);
|
|
5349
|
+
let position = 0;
|
|
5350
|
+
for (const name of names) {
|
|
5351
|
+
const bytes = await files.read(name.key);
|
|
5352
|
+
if (bytes === null || bytes === "too-long") continue;
|
|
5353
|
+
yield { name: `${QUERY_HISTORY_ARCHIVE_PREFIX}${position}`, bytes };
|
|
5354
|
+
position += 1;
|
|
5355
|
+
}
|
|
5356
|
+
}
|
|
5357
|
+
|
|
5358
|
+
// src/persistence/archive/validate.ts
|
|
5359
|
+
function requireDimension(key, found, expected) {
|
|
5360
|
+
if (expected === found) return;
|
|
5361
|
+
throw new NarsilError(
|
|
5362
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5363
|
+
`The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
|
|
5364
|
+
{ key, found, expected }
|
|
5365
|
+
);
|
|
5366
|
+
}
|
|
5367
|
+
function requireSchemaOfManifest(key, schema, manifest) {
|
|
5368
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
5369
|
+
const listed = manifest.schema[field];
|
|
5370
|
+
if (listed === void 0 || listed === type) continue;
|
|
5371
|
+
throw new NarsilError(
|
|
5372
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5373
|
+
`The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
|
|
5374
|
+
{ key, field }
|
|
5375
|
+
);
|
|
5376
|
+
}
|
|
5377
|
+
}
|
|
5378
|
+
async function validateStagedCheckpoint(store, manifest) {
|
|
5379
|
+
for (const partition of manifest.partitions) {
|
|
5380
|
+
for (const ref of partition.segments) {
|
|
5381
|
+
const { directory } = store;
|
|
5382
|
+
if (directory !== void 0) {
|
|
5383
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
5384
|
+
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
5385
|
+
continue;
|
|
5386
|
+
}
|
|
5387
|
+
}
|
|
5388
|
+
const contents = await readSegmentFile(store, ref);
|
|
5389
|
+
requireSchemaOfManifest(ref.key, contents.schema, manifest);
|
|
5390
|
+
decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
5391
|
+
}
|
|
5392
|
+
}
|
|
5393
|
+
for (const field of manifest.vectors) {
|
|
5394
|
+
const fieldType = manifest.schema[field.fieldPath];
|
|
5395
|
+
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
5396
|
+
for (const file of field.files) {
|
|
5397
|
+
const { payload } = await readVectorFile(store, file, field.fieldPath);
|
|
5398
|
+
requireDimension(file.key, payload.dimension, dimension);
|
|
5399
|
+
}
|
|
5400
|
+
await readGraphParts(store, field);
|
|
5401
|
+
}
|
|
5402
|
+
}
|
|
5403
|
+
|
|
5404
|
+
// src/engine/snapshot/restore-setup.ts
|
|
5405
|
+
function prepareRestoredIndex(configuration, deps, reject) {
|
|
5406
|
+
const configFields = restoredConfigFields(configuration, reject);
|
|
5407
|
+
const embedding = restoredEmbedding(configuration.embedding, reject);
|
|
5408
|
+
const languageName = configuration.language ?? "";
|
|
5409
|
+
const language = getLanguage(languageName);
|
|
5410
|
+
const schema = validateSchema(configuration.schema);
|
|
5411
|
+
validatePatternValueLimit(configFields.patternValueLimit);
|
|
5412
|
+
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
5413
|
+
validateVectorPromotion(configFields.vectorPromotion);
|
|
5414
|
+
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
5415
|
+
validateTextStorage(configFields.textStorage, deps.filesystemDurability);
|
|
5416
|
+
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
5417
|
+
validateRequiredFieldsInSchema(configFields.required, schema);
|
|
5418
|
+
}
|
|
5419
|
+
const config = {
|
|
5420
|
+
schema,
|
|
5421
|
+
language: languageName,
|
|
5422
|
+
...configFields,
|
|
5423
|
+
...embedding !== void 0 ? {
|
|
5424
|
+
embedding: {
|
|
5425
|
+
fields: embedding.fields,
|
|
5426
|
+
...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
|
|
5427
|
+
}
|
|
5428
|
+
} : {}
|
|
5429
|
+
};
|
|
5430
|
+
resolveIndexAnalysis(config);
|
|
5431
|
+
const adapterName = embedding?.adapter ?? null;
|
|
5432
|
+
let embeddingAdapter = null;
|
|
5433
|
+
if (embedding !== void 0) {
|
|
5434
|
+
const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
|
|
5435
|
+
if (candidate) {
|
|
5436
|
+
validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
|
|
5437
|
+
embeddingAdapter = candidate;
|
|
5438
|
+
}
|
|
5439
|
+
}
|
|
5440
|
+
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
5441
|
+
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
5442
|
+
const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
|
|
5443
|
+
return {
|
|
5444
|
+
config,
|
|
5445
|
+
language,
|
|
5446
|
+
embeddingAdapter,
|
|
5447
|
+
adapterName,
|
|
5448
|
+
storedRevision,
|
|
5449
|
+
analysisGeneration,
|
|
5450
|
+
storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
|
|
5451
|
+
synonyms,
|
|
5452
|
+
querySuggestions: storedStateOf(configuration)
|
|
5453
|
+
};
|
|
5454
|
+
}
|
|
5455
|
+
function restoredSynonyms(raw, config, language, reject) {
|
|
5456
|
+
if (raw === void 0 || raw === null) return null;
|
|
5457
|
+
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
5458
|
+
}
|
|
5459
|
+
async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
5460
|
+
const loadingSchemaCode = loadSchemaCodeFor(prepared.config.schema);
|
|
5461
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
5462
|
+
const detached = createDetachedIndex(
|
|
5463
|
+
indexName,
|
|
5464
|
+
prepared.config,
|
|
5465
|
+
prepared.language,
|
|
5466
|
+
prepared.synonyms?.lookup ?? null,
|
|
5467
|
+
deps.executor
|
|
5468
|
+
);
|
|
5469
|
+
while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
|
|
5470
|
+
return detached;
|
|
5471
|
+
}
|
|
5472
|
+
function restoredRegistryEntry(prepared, manager, deps) {
|
|
5473
|
+
return {
|
|
5474
|
+
config: prepared.config,
|
|
5475
|
+
language: prepared.language,
|
|
5476
|
+
embeddingAdapter: prepared.embeddingAdapter,
|
|
5477
|
+
embeddingAdapterName: prepared.adapterName,
|
|
5478
|
+
vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
|
|
5479
|
+
indexUuid: null,
|
|
5480
|
+
heldPartitions: null,
|
|
5481
|
+
documentCount: manager.countDocuments(),
|
|
5482
|
+
partitionCount: manager.partitionCount,
|
|
5483
|
+
analysisRevision: prepared.storedRevision,
|
|
5484
|
+
arrayPositionGap: prepared.storedArrayPositionGap,
|
|
5485
|
+
synonyms: prepared.synonyms?.encoded ?? null,
|
|
5486
|
+
analysisGeneration: prepared.analysisGeneration,
|
|
5487
|
+
pendingAnalysis: null,
|
|
5488
|
+
durableAnalysis: null,
|
|
5489
|
+
sync: null,
|
|
5490
|
+
querySuggestions: prepared.querySuggestions
|
|
5491
|
+
};
|
|
5492
|
+
}
|
|
5493
|
+
|
|
5494
|
+
// src/engine/snapshot/replace.ts
|
|
5495
|
+
function releaseTimedOut(indexName, timeoutMs) {
|
|
5496
|
+
return new NarsilError(
|
|
5497
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
5498
|
+
`The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
|
|
5499
|
+
{ indexName, timeoutMs }
|
|
5500
|
+
);
|
|
5501
|
+
}
|
|
5502
|
+
async function withinLimit(indexName, release, work, deps) {
|
|
5503
|
+
release.settled = work.catch(() => {
|
|
5504
|
+
release.reloads = true;
|
|
5505
|
+
});
|
|
5506
|
+
let timer;
|
|
5507
|
+
const limit = new Promise((_, reject) => {
|
|
5508
|
+
timer = setTimeout(() => {
|
|
5509
|
+
release.timedOut = true;
|
|
5510
|
+
reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
|
|
5511
|
+
}, deps.releaseTimeoutMs);
|
|
5512
|
+
});
|
|
5513
|
+
try {
|
|
5514
|
+
await Promise.race([work, limit]);
|
|
5515
|
+
} finally {
|
|
5516
|
+
clearTimeout(timer);
|
|
5517
|
+
}
|
|
5518
|
+
}
|
|
5519
|
+
function vectorIndexesOf2(indexName, deps) {
|
|
5520
|
+
const manager = deps.executor.getManager(indexName);
|
|
5521
|
+
return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
|
|
5522
|
+
}
|
|
5523
|
+
async function holdVectorCopies(vectorIndexes, release) {
|
|
5524
|
+
for (const vectorIndex of vectorIndexes) {
|
|
5525
|
+
if (release.held.includes(vectorIndex)) continue;
|
|
5526
|
+
release.held.push(vectorIndex);
|
|
5527
|
+
await vectorIndex.holdWorkerCopies();
|
|
5528
|
+
}
|
|
5529
|
+
}
|
|
5530
|
+
async function releaseCopies(indexName, release, deps) {
|
|
5531
|
+
release.reloads = await deps.copies.releaseIndexCopies(indexName);
|
|
5532
|
+
await holdVectorCopies(vectorIndexesOf2(indexName, deps), release);
|
|
5533
|
+
}
|
|
5534
|
+
async function endRelease(indexName, release, deps) {
|
|
5535
|
+
await release.settled;
|
|
5536
|
+
for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
|
|
5537
|
+
await deps.copies.endCopyRelease(indexName, release.reloads);
|
|
5538
|
+
}
|
|
5539
|
+
function persistedCheckpoint(indexName, partitionCount, deps) {
|
|
5540
|
+
const durable = deps.durability?.manager;
|
|
5541
|
+
if (durable === void 0) return [];
|
|
5542
|
+
return Array.from({ length: partitionCount }, (_, partitionId) => ({
|
|
5543
|
+
partitionId,
|
|
5544
|
+
lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
|
|
5545
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
5546
|
+
}));
|
|
5547
|
+
}
|
|
5548
|
+
async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
5549
|
+
const durable = deps.durability?.manager;
|
|
5550
|
+
const staging = restored.staging;
|
|
5551
|
+
if (durable === void 0 || staging === null) return;
|
|
5552
|
+
const previous = persistedCheckpoint(indexName, previousPartitions, deps);
|
|
5553
|
+
if (previousPartitions > 0) await durable.unloadIndex(indexName);
|
|
5554
|
+
const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
|
|
5555
|
+
try {
|
|
5556
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
5557
|
+
await staging.commit();
|
|
5558
|
+
} catch (error) {
|
|
5559
|
+
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
|
|
5560
|
+
throw error;
|
|
5561
|
+
}
|
|
5562
|
+
await staging.finish();
|
|
5563
|
+
const relocate = await staging.relocation().catch(() => null);
|
|
5564
|
+
if (relocate === null) return;
|
|
5565
|
+
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
5566
|
+
const { manager } = restored.detached;
|
|
5567
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
5568
|
+
const partition = manager.getPartition(partitionId);
|
|
5569
|
+
if (!isCompositePartition(partition)) continue;
|
|
5570
|
+
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
5571
|
+
}
|
|
5572
|
+
}
|
|
5573
|
+
function activate(indexName, restored, entry, deps) {
|
|
5574
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
5575
|
+
deps.indexRegistry.set(indexName, entry);
|
|
5576
|
+
restored.detached.attach();
|
|
5577
|
+
deps.clearAnalysisStale(indexName);
|
|
5578
|
+
const { prepared } = restored;
|
|
5579
|
+
if (prepared.storedRevision !== prepared.language.revision || arrayPositionsAreStale(prepared.config, prepared.storedArrayPositionGap)) {
|
|
5580
|
+
deps.markAnalysisStale({
|
|
5581
|
+
indexName,
|
|
5582
|
+
language: prepared.language.name,
|
|
5583
|
+
storedRevision: prepared.storedRevision,
|
|
5584
|
+
currentRevision: prepared.language.revision
|
|
5585
|
+
});
|
|
5586
|
+
}
|
|
5587
|
+
if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
|
|
5588
|
+
deps.forgetWatermark(indexName);
|
|
5589
|
+
}
|
|
5590
|
+
async function replaceExisting(indexName, restored, entry, deps) {
|
|
5591
|
+
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
5592
|
+
try {
|
|
5593
|
+
await deps.abandonAnalysisChange(indexName);
|
|
5594
|
+
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
5595
|
+
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
5596
|
+
await deps.abandonAnalysisRebuild(indexName);
|
|
5597
|
+
const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
|
|
5598
|
+
const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf2(indexName, deps);
|
|
5599
|
+
await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
|
|
5600
|
+
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
5601
|
+
await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
|
|
5602
|
+
if (previous === void 0) activate(indexName, restored, entry, deps);
|
|
5603
|
+
else await previous.publication.write(() => activate(indexName, restored, entry, deps));
|
|
5604
|
+
await deps.querySuggestions?.afterActivate(entry);
|
|
5605
|
+
});
|
|
5606
|
+
} finally {
|
|
5607
|
+
const ending = endRelease(indexName, release, deps).catch(() => void 0);
|
|
5608
|
+
if (!release.timedOut) await ending;
|
|
5609
|
+
}
|
|
5610
|
+
}
|
|
5611
|
+
async function createFresh(indexName, restored, entry, deps) {
|
|
5612
|
+
deps.copies.desyncIndex(indexName);
|
|
5613
|
+
deps.executor.installDetachedIndex(restored.detached);
|
|
5614
|
+
deps.indexRegistry.set(indexName, entry);
|
|
5615
|
+
try {
|
|
5616
|
+
await deps.indexState.registerReplacing(indexName, async () => {
|
|
5617
|
+
await commitToStorage(indexName, restored, 0, deps);
|
|
5618
|
+
activate(indexName, restored, entry, deps);
|
|
5619
|
+
await deps.querySuggestions?.afterActivate(entry);
|
|
5620
|
+
});
|
|
5621
|
+
} catch (error) {
|
|
5622
|
+
if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
|
|
5623
|
+
throw error;
|
|
5624
|
+
} finally {
|
|
5625
|
+
await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
|
|
5626
|
+
}
|
|
5627
|
+
}
|
|
5628
|
+
async function stageMetadata(indexName, staging, entry) {
|
|
5629
|
+
const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
|
|
5630
|
+
await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
|
|
5631
|
+
}
|
|
5632
|
+
async function installRestoredIndex(indexName, restored, deps, history = null) {
|
|
5633
|
+
const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
|
|
5634
|
+
const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
|
|
5635
|
+
if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
|
|
5636
|
+
let replaced;
|
|
5637
|
+
try {
|
|
5638
|
+
if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
|
|
5639
|
+
await deps.querySuggestions?.beforeInstall(entry, deps.indexRegistry.get(indexName));
|
|
5640
|
+
if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
|
|
5641
|
+
replaced = deps.indexRegistry.get(indexName);
|
|
5642
|
+
if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
|
|
5643
|
+
else await createFresh(indexName, restored, entry, deps);
|
|
5644
|
+
} catch (error) {
|
|
5645
|
+
if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
|
|
5646
|
+
else deps.executor.discardDetachedIndex(restored.detached);
|
|
5647
|
+
throw error;
|
|
5648
|
+
}
|
|
5649
|
+
await restored.staging?.removeRetired().catch(() => void 0);
|
|
5650
|
+
const durable = deps.durability?.manager;
|
|
5651
|
+
if (restored.staging === null && durable !== void 0) {
|
|
5652
|
+
await durable.persistMetadata(indexName);
|
|
5653
|
+
await durable.checkpoint(indexName);
|
|
5654
|
+
}
|
|
5655
|
+
if (previousSync !== null) {
|
|
5656
|
+
const storage = durable?.syncStorage?.();
|
|
5657
|
+
if (storage !== void 0)
|
|
5658
|
+
await oneSyncChangeAtATime(deps.indexRegistry, indexName, () => storage.release(indexName));
|
|
5659
|
+
emitSyncChange(deps.indexRegistry, { kind: "epoch", indexName });
|
|
5660
|
+
}
|
|
5661
|
+
await deps.querySuggestions?.afterInstall(entry, history);
|
|
5662
|
+
if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
|
|
5663
|
+
await deps.announceIndexCreated(indexName, restored.prepared.config);
|
|
5664
|
+
for (const vectorIndex of restored.detached.manager.getVectorIndexes().values()) {
|
|
5665
|
+
vectorIndex.scheduleBuild();
|
|
5666
|
+
vectorIndex.startShuffledRebuildWhenDue();
|
|
5667
|
+
}
|
|
5668
|
+
}
|
|
5669
|
+
|
|
5670
|
+
// src/engine/snapshot/archive-restore.ts
|
|
5671
|
+
function rejectConfiguration(message) {
|
|
5672
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
|
|
5673
|
+
}
|
|
5674
|
+
function renamed(manifest, sourceName, indexName) {
|
|
5675
|
+
const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
|
|
5676
|
+
return {
|
|
5677
|
+
...manifest,
|
|
5678
|
+
partitions: manifest.partitions.map((partition) => ({
|
|
5679
|
+
...partition,
|
|
5680
|
+
segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
|
|
5681
|
+
})),
|
|
5682
|
+
vectors: manifest.vectors.map((field) => ({
|
|
5683
|
+
...field,
|
|
5684
|
+
files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
|
|
5685
|
+
graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
|
|
5686
|
+
}))
|
|
5687
|
+
};
|
|
5688
|
+
}
|
|
5689
|
+
function requireSameSchema(prepared, manifest) {
|
|
5690
|
+
const configured = flattenSchema(prepared.config.schema);
|
|
5691
|
+
const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
|
|
5692
|
+
for (const field of keys) {
|
|
5693
|
+
if (configured[field] === manifest.schema[field]) continue;
|
|
5694
|
+
throw new NarsilError(
|
|
5695
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5696
|
+
`The archive's configuration gives field "${field}" another type than its manifest does`,
|
|
5697
|
+
{ field }
|
|
5698
|
+
);
|
|
5699
|
+
}
|
|
5700
|
+
}
|
|
5701
|
+
function partitionCountOf(manifest) {
|
|
5702
|
+
let count = 1;
|
|
5703
|
+
for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
|
|
5704
|
+
return count;
|
|
5705
|
+
}
|
|
5706
|
+
function missingPart(what) {
|
|
5707
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
|
|
5708
|
+
}
|
|
5709
|
+
function keepsTextOnDisk(prepared, staging) {
|
|
5710
|
+
return prepared.config.textStorage === "disk" && staging.directory !== void 0;
|
|
5711
|
+
}
|
|
5712
|
+
async function stageArchive(indexName, reader, staging, deps, staged) {
|
|
5713
|
+
await readSnapshotArchive(reader, {
|
|
5714
|
+
history: async (chunks) => {
|
|
5715
|
+
if (staged.history === void 0) {
|
|
5716
|
+
staged.history = await deps.querySuggestions?.historySink(staged.prepared?.querySuggestions ?? null) ?? null;
|
|
5717
|
+
}
|
|
5718
|
+
await staged.history?.add(chunks);
|
|
5719
|
+
},
|
|
5720
|
+
config: async (bytes) => {
|
|
5721
|
+
staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
|
|
5722
|
+
},
|
|
5723
|
+
manifest: async (manifest2, sourceName) => {
|
|
5724
|
+
if (staged.prepared === null) missingPart("configuration");
|
|
5725
|
+
requireSameSchema(staged.prepared, manifest2);
|
|
5726
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
5727
|
+
const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
|
|
5728
|
+
requireRestoreMemory(
|
|
5729
|
+
indexName,
|
|
5730
|
+
archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
|
|
5731
|
+
replacesIndex
|
|
5732
|
+
);
|
|
5733
|
+
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
5734
|
+
},
|
|
5735
|
+
file: async (entryName, _ref, chunks) => {
|
|
5736
|
+
if (staged.manifest === null) missingPart("manifest");
|
|
5737
|
+
await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
|
|
5738
|
+
}
|
|
5739
|
+
});
|
|
5740
|
+
const { prepared, manifest } = staged;
|
|
5741
|
+
if (prepared === null) missingPart("configuration");
|
|
5742
|
+
if (manifest === null) missingPart("manifest");
|
|
5743
|
+
const { directory } = staging;
|
|
5744
|
+
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
5745
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
5746
|
+
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
5747
|
+
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
5748
|
+
}
|
|
5749
|
+
await validateStagedCheckpoint(staging, manifest);
|
|
5750
|
+
return { prepared, manifest };
|
|
5751
|
+
}
|
|
5752
|
+
async function loadStagedIndex(indexName, staged, staging, store, deps) {
|
|
5753
|
+
const { prepared, manifest } = staged;
|
|
5754
|
+
await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
|
|
5755
|
+
const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
|
|
5756
|
+
try {
|
|
5757
|
+
await loadSegmentedCheckpoint(
|
|
5758
|
+
store,
|
|
5759
|
+
indexName,
|
|
5760
|
+
manifest,
|
|
5761
|
+
{ manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
|
|
5762
|
+
true
|
|
5763
|
+
);
|
|
5764
|
+
} catch (error) {
|
|
5765
|
+
deps.executor.discardDetachedIndex(detached);
|
|
5766
|
+
throw error;
|
|
5767
|
+
}
|
|
5768
|
+
return { prepared, detached, staging, checkpoint: manifest.checkpoint };
|
|
5769
|
+
}
|
|
5770
|
+
async function restoreArchive(indexName, reader, deps) {
|
|
5771
|
+
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
5772
|
+
const store = staging?.store ?? memoryCheckpointStore();
|
|
5773
|
+
const archive = { prepared: null, manifest: null, history: void 0 };
|
|
5774
|
+
try {
|
|
5775
|
+
const staged = await stageArchive(indexName, reader, store, deps, archive);
|
|
5776
|
+
const restored = await loadStagedIndex(indexName, staged, staging, store, deps);
|
|
5777
|
+
await installRestoredIndex(indexName, restored, deps, archive.history ?? null);
|
|
5778
|
+
} catch (error) {
|
|
5779
|
+
await archive.history?.abandon().catch(() => void 0);
|
|
5780
|
+
throw error;
|
|
5781
|
+
} finally {
|
|
5782
|
+
await staging?.discard().catch(() => void 0);
|
|
5783
|
+
await reader.close();
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
|
|
5787
|
+
// src/engine/snapshot/earlier-restore.ts
|
|
5788
|
+
function invalidSnapshot(message, details) {
|
|
5789
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
|
|
5790
|
+
}
|
|
5791
|
+
async function decodeEarlierSnapshot(data) {
|
|
5792
|
+
const payloadBytes = await unpackIndexSnapshotEnvelope(data);
|
|
5793
|
+
let decoded;
|
|
5794
|
+
try {
|
|
5795
|
+
decoded = decode(payloadBytes);
|
|
5796
|
+
} catch (err) {
|
|
5797
|
+
invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
|
|
5798
|
+
bytes: data.length
|
|
5799
|
+
});
|
|
5800
|
+
}
|
|
5801
|
+
if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
|
|
5802
|
+
invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
|
|
5803
|
+
}
|
|
5804
|
+
const envelope = decoded;
|
|
5805
|
+
if (envelope.version !== 1 && envelope.version !== 2) {
|
|
5806
|
+
invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
|
|
5807
|
+
version: envelope.version
|
|
5808
|
+
});
|
|
5809
|
+
}
|
|
5810
|
+
if (!envelope.schema || typeof envelope.schema !== "object")
|
|
5811
|
+
invalidSnapshot("Invalid snapshot: missing or invalid schema");
|
|
5812
|
+
if (!envelope.language || typeof envelope.language !== "string") {
|
|
5813
|
+
invalidSnapshot("Invalid snapshot: missing or invalid language");
|
|
5814
|
+
}
|
|
5815
|
+
if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
|
|
5816
|
+
return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
|
|
5817
|
+
}
|
|
5818
|
+
function vectorsOfSnapshot(envelope) {
|
|
5819
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
5820
|
+
for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
|
|
5821
|
+
for (const part of decodeVectorIndexParts(rawParts)) {
|
|
5822
|
+
const values = bytesToVectors(part.vectors);
|
|
5823
|
+
for (let i = 0; i < part.docIds.length; i++) {
|
|
5824
|
+
let fields = byDocument.get(part.docIds[i]);
|
|
5825
|
+
if (fields === void 0) {
|
|
5826
|
+
fields = /* @__PURE__ */ new Map();
|
|
5827
|
+
byDocument.set(part.docIds[i], fields);
|
|
5828
|
+
}
|
|
5829
|
+
fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
|
|
5830
|
+
}
|
|
5831
|
+
}
|
|
5832
|
+
}
|
|
5833
|
+
return byDocument;
|
|
5834
|
+
}
|
|
5835
|
+
function footprintOf(envelope, prepared, replacesIndex) {
|
|
5836
|
+
let partitionBytes = 0;
|
|
5837
|
+
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
5838
|
+
const vectorCounts = /* @__PURE__ */ new Map();
|
|
5839
|
+
for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
|
|
5840
|
+
let count = 0;
|
|
5841
|
+
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
5842
|
+
vectorCounts.set(fieldPath, count);
|
|
5843
|
+
}
|
|
5844
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
|
|
5845
|
+
}
|
|
5846
|
+
function documentsOfPartition(envelope, payload) {
|
|
5847
|
+
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
5848
|
+
}
|
|
5849
|
+
function insertRestoredDocument(manager, docId, fields, vectors) {
|
|
5850
|
+
manager.insert(docId, fields, { skipClone: true });
|
|
5851
|
+
if (vectors === void 0) return;
|
|
5852
|
+
try {
|
|
5853
|
+
insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
5854
|
+
} catch (err) {
|
|
5855
|
+
manager.remove(docId);
|
|
5856
|
+
throw err;
|
|
5857
|
+
}
|
|
5858
|
+
}
|
|
5859
|
+
function insertSnapshotDocuments(envelope, manager) {
|
|
5860
|
+
const vectors = vectorsOfSnapshot(envelope);
|
|
5861
|
+
for (const payload of envelope.partitions ?? []) {
|
|
5862
|
+
for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
|
|
5863
|
+
insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
|
|
5864
|
+
}
|
|
5865
|
+
}
|
|
5866
|
+
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
|
|
5867
|
+
}
|
|
5868
|
+
async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
5869
|
+
const entry = restoredRegistryEntry(prepared, built.manager, deps);
|
|
5870
|
+
const archive = await buildArchiveFromMemory({
|
|
5871
|
+
indexName,
|
|
5872
|
+
metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
|
|
5873
|
+
manager: built.manager,
|
|
5874
|
+
vectorIndexes: built.manager.getVectorIndexes(),
|
|
5875
|
+
appliedSeqNo: () => 0,
|
|
5876
|
+
partitionId: null
|
|
5877
|
+
});
|
|
5878
|
+
try {
|
|
5879
|
+
for (const ref of manifestFileRefs(archive.manifest))
|
|
5880
|
+
await staging.store.write(ref.key, [await archive.readFile(ref)]);
|
|
5881
|
+
} finally {
|
|
5882
|
+
archive.release();
|
|
5883
|
+
}
|
|
5884
|
+
await validateStagedCheckpoint(staging.store, archive.manifest);
|
|
5885
|
+
return archive.manifest;
|
|
5886
|
+
}
|
|
5887
|
+
async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
|
|
5888
|
+
const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
|
|
5889
|
+
let manifest;
|
|
5890
|
+
try {
|
|
5891
|
+
insertSnapshotDocuments(envelope, built.manager);
|
|
5892
|
+
if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
|
|
5893
|
+
manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
|
|
5894
|
+
} catch (err) {
|
|
5895
|
+
deps.executor.discardDetachedIndex(built);
|
|
5896
|
+
throw err;
|
|
5897
|
+
}
|
|
5898
|
+
deps.executor.discardDetachedIndex(built);
|
|
5899
|
+
return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
|
|
5900
|
+
}
|
|
5901
|
+
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
5902
|
+
const envelope = await decodeEarlierSnapshot(data);
|
|
5903
|
+
const prepared = prepareRestoredIndex(envelope, deps);
|
|
5904
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
5905
|
+
requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
|
|
5906
|
+
const analysedNow = {
|
|
5907
|
+
...prepared,
|
|
5908
|
+
storedRevision: prepared.language.revision,
|
|
5909
|
+
storedArrayPositionGap: ARRAY_POSITION_GAP
|
|
5910
|
+
};
|
|
5911
|
+
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
5912
|
+
try {
|
|
5913
|
+
await installRestoredIndex(
|
|
5914
|
+
indexName,
|
|
5915
|
+
await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
|
|
5916
|
+
deps
|
|
5917
|
+
);
|
|
5918
|
+
} finally {
|
|
5919
|
+
await staging?.discard().catch(() => void 0);
|
|
5920
|
+
}
|
|
5921
|
+
}
|
|
5922
|
+
|
|
5923
|
+
// src/engine/snapshot/index.ts
|
|
5924
|
+
var NRSL_MAGIC_BYTES = 4;
|
|
5925
|
+
async function checkpointForArchive(deps) {
|
|
5926
|
+
const durable = deps.durability?.manager;
|
|
5927
|
+
if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
|
|
5928
|
+
return buildArchiveFromMemory({
|
|
5929
|
+
indexName: deps.indexName,
|
|
5930
|
+
metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
|
|
5931
|
+
manager: deps.manager,
|
|
5932
|
+
vectorIndexes: deps.manager.getVectorIndexes(),
|
|
5933
|
+
appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
|
|
5934
|
+
partitionId: null
|
|
5935
|
+
});
|
|
5936
|
+
}
|
|
5937
|
+
async function* queryHistoryEntries(core, indexName) {
|
|
5938
|
+
yield* historyArchiveEntries(core, indexName);
|
|
5939
|
+
}
|
|
5940
|
+
async function createSnapshot(deps, coreWithAddedHistory) {
|
|
5941
|
+
const trailingEntries = coreWithAddedHistory === null ? void 0 : () => queryHistoryEntries(coreWithAddedHistory, deps.indexName);
|
|
5942
|
+
for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
|
|
5943
|
+
const checkpoint = await checkpointForArchive(deps);
|
|
5944
|
+
try {
|
|
5945
|
+
if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
|
|
5946
|
+
const config = await encodeIndexConfiguration(deps.entry);
|
|
5947
|
+
return archiveStream({
|
|
5948
|
+
...checkpoint,
|
|
5949
|
+
config,
|
|
5950
|
+
sourceIndexName: deps.indexName,
|
|
5951
|
+
...trailingEntries !== void 0 ? { trailingEntries } : {}
|
|
5952
|
+
});
|
|
5953
|
+
}
|
|
5954
|
+
} catch (error) {
|
|
5955
|
+
checkpoint.release();
|
|
5956
|
+
throw error;
|
|
5957
|
+
}
|
|
5958
|
+
checkpoint.release();
|
|
5959
|
+
}
|
|
5960
|
+
throw new NarsilError(
|
|
5961
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
5962
|
+
`The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
|
|
5963
|
+
{ indexName: deps.indexName }
|
|
5964
|
+
);
|
|
5965
|
+
}
|
|
5966
|
+
function requireSnapshotSource(data) {
|
|
5967
|
+
if (data instanceof Uint8Array) return data;
|
|
5968
|
+
if (data !== null && typeof data === "object") {
|
|
5969
|
+
if (typeof data.getReader === "function") return data;
|
|
5970
|
+
if (typeof data[Symbol.asyncIterator] === "function") {
|
|
5971
|
+
return data;
|
|
5972
|
+
}
|
|
5973
|
+
}
|
|
5974
|
+
throw new NarsilError(
|
|
5975
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
5976
|
+
"Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
|
|
5977
|
+
);
|
|
5978
|
+
}
|
|
5979
|
+
async function readEarlierSnapshotFile(reader) {
|
|
5980
|
+
const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
|
|
5981
|
+
if (length > MAX_FILE_BYTES) {
|
|
5982
|
+
throw new NarsilError(
|
|
5983
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5984
|
+
`The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
|
|
5985
|
+
{ bytes: length }
|
|
5986
|
+
);
|
|
5987
|
+
}
|
|
5988
|
+
const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
|
|
5989
|
+
if (bytes.byteLength < length) {
|
|
5990
|
+
throw new NarsilError(
|
|
5991
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5992
|
+
"The snapshot ends before the payload its header declares"
|
|
5993
|
+
);
|
|
5994
|
+
}
|
|
5995
|
+
const trailing = await reader.readExactly(1);
|
|
5996
|
+
if (trailing !== null && trailing.byteLength > 0) {
|
|
5997
|
+
throw new NarsilError(
|
|
5998
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5999
|
+
"The snapshot holds bytes after the payload its header declares"
|
|
6000
|
+
);
|
|
6001
|
+
}
|
|
6002
|
+
return bytes;
|
|
6003
|
+
}
|
|
6004
|
+
async function restoreFromSnapshot(indexName, data, deps) {
|
|
6005
|
+
const reader = createByteReader(requireSnapshotSource(data));
|
|
6006
|
+
const head = await reader.peek(NRSL_MAGIC_BYTES);
|
|
6007
|
+
if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
|
|
6008
|
+
let bytes;
|
|
6009
|
+
try {
|
|
6010
|
+
bytes = await readEarlierSnapshotFile(reader);
|
|
6011
|
+
} finally {
|
|
6012
|
+
await reader.close();
|
|
6013
|
+
}
|
|
6014
|
+
await restoreEarlierSnapshot(indexName, bytes, deps);
|
|
6015
|
+
return;
|
|
6016
|
+
}
|
|
6017
|
+
if (head.byteLength === 0) {
|
|
6018
|
+
await reader.close();
|
|
6019
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
|
|
6020
|
+
}
|
|
6021
|
+
await restoreArchive(indexName, reader, deps);
|
|
6022
|
+
}
|
|
6023
|
+
|
|
6024
|
+
// src/engine/sync/status.ts
|
|
6025
|
+
function syncStatusOf(sync) {
|
|
6026
|
+
if (sync === null) return { enabled: false, active: false, epoch: null };
|
|
6027
|
+
return {
|
|
6028
|
+
enabled: true,
|
|
6029
|
+
...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
|
|
6030
|
+
...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {},
|
|
6031
|
+
active: sync.active,
|
|
6032
|
+
epoch: sync.epoch
|
|
6033
|
+
};
|
|
6034
|
+
}
|
|
6035
|
+
|
|
6036
|
+
// src/engine/sync/methods.ts
|
|
6037
|
+
async function updateSyncOf(core, indexName, setting) {
|
|
6038
|
+
core.guardShutdown();
|
|
6039
|
+
core.requireIndex(indexName);
|
|
6040
|
+
await updateIndexSync(core, indexName, setting);
|
|
6041
|
+
}
|
|
6042
|
+
function syncOf(core, indexName) {
|
|
6043
|
+
core.guardShutdown();
|
|
6044
|
+
return syncStatusOf(core.requireIndex(indexName).sync);
|
|
6045
|
+
}
|
|
6046
|
+
|
|
6047
|
+
// src/persistence/paged/rewrite-spill.ts
|
|
6048
|
+
function openRewriteSpill(storage, manager, target) {
|
|
6049
|
+
const config = target?.config ?? manager.config;
|
|
6050
|
+
return createRewriteSpill(storage, manager.indexName, {
|
|
6051
|
+
schema: flattenSchema(config.schema),
|
|
6052
|
+
trackPositions: config.trackPositions ?? true,
|
|
6053
|
+
partialWordAnalysis: target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis,
|
|
6054
|
+
textAnalysis: {
|
|
6055
|
+
language: target?.language ?? manager.language,
|
|
6056
|
+
options: resolvePartitionInsertOptions(config, target?.analysis ?? manager.analysis)
|
|
6057
|
+
}
|
|
6058
|
+
});
|
|
6059
|
+
}
|
|
6060
|
+
function createRewriteSpill(storage, indexName, analysis) {
|
|
6061
|
+
const { directory, writeRecords } = storage;
|
|
6062
|
+
const prefix = `${indexName}/rebuilt/${generateId()}/`;
|
|
6063
|
+
const opened = [];
|
|
6064
|
+
let nextFile = 0;
|
|
6065
|
+
async function removeFiles() {
|
|
6066
|
+
for (const key of await directory.list(prefix)) await directory.remove(key);
|
|
6067
|
+
}
|
|
6068
|
+
function freezeLive(partition) {
|
|
6069
|
+
if (!isCompositePartition(partition)) return null;
|
|
6070
|
+
return partition.freezeLiveTail((payload, documents) => {
|
|
6071
|
+
const snapshot = freezeSegmentShared(payload, documents, void 0, analysis.partialWordAnalysis);
|
|
6072
|
+
return snapshot === null ? createFrozenSegment(payload, documents) : createSharedFrozenSegment(snapshot);
|
|
6073
|
+
});
|
|
6074
|
+
}
|
|
6075
|
+
return {
|
|
6076
|
+
newPartition(partitionId) {
|
|
6077
|
+
const live = createPartitionIndex(partitionId, analysis.trackPositions);
|
|
6078
|
+
return createCompositePartition(partitionId, analysis.trackPositions, live, analysis.textAnalysis);
|
|
6079
|
+
},
|
|
6080
|
+
async spillIfFull(partition, partitionId) {
|
|
6081
|
+
if (!isCompositePartition(partition)) return;
|
|
6082
|
+
if (partition.live.estimateMemoryBytes() < REWRITE_SPILL_BYTES) return;
|
|
6083
|
+
const frozen = freezeLive(partition);
|
|
6084
|
+
if (frozen === null) return;
|
|
6085
|
+
const key = `${prefix}${nextFile}`;
|
|
6086
|
+
nextFile += 1;
|
|
6087
|
+
const written = await writePagedSegment({
|
|
6088
|
+
directory,
|
|
6089
|
+
key,
|
|
6090
|
+
schema: analysis.schema,
|
|
6091
|
+
tombstones: [],
|
|
6092
|
+
selections: [{ segment: frozen, keep: (ordinal) => !frozen.isTombstoned(ordinal) }],
|
|
6093
|
+
partialWordAnalysis: analysis.partialWordAnalysis,
|
|
6094
|
+
textAnalysis: analysis.textAnalysis,
|
|
6095
|
+
...writeRecords === null ? {} : { writeRecords: writeRecords.captureSegments(indexName, partitionId, [frozen]) }
|
|
6096
|
+
});
|
|
6097
|
+
const listing = await pagedListingOf(directory, { key, bytes: written.bytes, checksum: written.checksum });
|
|
6098
|
+
const segment = createPagedFrozenSegment(openPagedSnapshot(listing));
|
|
6099
|
+
opened.push(segment);
|
|
6100
|
+
partition.swapFrozenSegments([frozen.segmentId], segment);
|
|
6101
|
+
},
|
|
6102
|
+
async abandon() {
|
|
6103
|
+
for (const segment of opened.splice(0)) segment.paged?.close();
|
|
6104
|
+
await removeFiles();
|
|
6105
|
+
},
|
|
6106
|
+
async removeUnread(manager) {
|
|
6107
|
+
for (const partition of manager.getAllPartitions()) {
|
|
6108
|
+
if (!isCompositePartition(partition)) continue;
|
|
6109
|
+
for (const segment of partition.frozenSegments()) {
|
|
6110
|
+
if (segment.paged?.key.startsWith(prefix)) return;
|
|
6111
|
+
}
|
|
6112
|
+
}
|
|
6113
|
+
for (const segment of opened.splice(0)) segment.paged?.close();
|
|
6114
|
+
await removeFiles();
|
|
6115
|
+
}
|
|
6116
|
+
};
|
|
6117
|
+
}
|
|
4672
6118
|
|
|
4673
|
-
export { createDurabilityFromTier };
|
|
6119
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, actionPartitionsOf, adoptSegmentsOnWorker, checkpointForArchive, closeUnusedSegments, createDurabilityFromTier, createInvalidationFromConfig, createSnapshot, documentExportStream, keysThatSegmentsStillRead, openFileVectorReads, openRewriteSpill, pagedPartitionLines, prepareDiskAdoption, readDocumentExport, reloadIndexAndCopies, restoreFromSnapshot, swapPreparedPartitions, syncOf, updateSyncOf };
|