@delali/narsil 0.3.1-next.157 → 0.3.1-next.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
- package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +66 -18
- package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1199 -1092
- package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -16
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -227
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
- package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +95 -52
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +33 -13
- package/dist/server.mjs +81 -100
- package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +88 -72
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +89 -59
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +66 -50
- package/package.json +233 -8
- package/dist/analysis-change-CKEMM2MC.mjs +0 -55
- package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
- package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-25MTVDYD.mjs +0 -432
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-AI5UI46D.mjs +0 -12
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
- package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
- package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
- package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MY2XKH22.mjs +0 -40
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
- package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
- package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
- package/dist/browser/chunk-V43N6TZO.mjs +0 -857
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
- package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-5ESVMFDJ.mjs +0 -20
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NDSQLPMQ.mjs +0 -25
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-HYXPVJVS.mjs +0 -32
- package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
- package/dist/browser/retention-TY3L3BFN.mjs +0 -211
- package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
- package/dist/browser/start-2UZTOBG5.mjs +0 -26
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-ULXAN5B4.mjs +0 -196
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-2ROUFMQX.mjs +0 -543
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-4B7JIGQD.mjs +0 -106
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BNL5OPRV.mjs +0 -634
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-C7XQYG6U.mjs +0 -50
- package/dist/chunk-CQZXZ75V.mjs +0 -562
- package/dist/chunk-CWS55TEX.mjs +0 -242
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FEZVDOWX.mjs +0 -6996
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GPWMUPCA.mjs +0 -1126
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JSV6OQ4C.mjs +0 -74
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KCQYFO4K.mjs +0 -62
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LDNCLRZU.mjs +0 -218
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MW6BR4QP.mjs +0 -316
- package/dist/chunk-MWD5VPN7.mjs +0 -21
- package/dist/chunk-ODYKO5TO.mjs +0 -101
- package/dist/chunk-OG7UGJ6F.mjs +0 -63
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-OYABFKTW.mjs +0 -1974
- package/dist/chunk-P5LANSH6.mjs +0 -121
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PIH5SJ4N.mjs +0 -7481
- package/dist/chunk-PSZI4O24.mjs +0 -253
- package/dist/chunk-PV2CG7EJ.mjs +0 -886
- package/dist/chunk-PWGYQXML.mjs +0 -470
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QLDABJZM.mjs +0 -20
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RCX4WNLC.mjs +0 -151
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RIHIVFIQ.mjs +0 -240
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RWURUDSP.mjs +0 -2639
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SOI6LTHM.mjs +0 -216
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UD5HIS4U.mjs +0 -382
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-UJDJB3NE.mjs +0 -913
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VSLSGASN.mjs +0 -30
- package/dist/chunk-VZ2ONHEE.mjs +0 -3715
- package/dist/chunk-W7OSSKH5.mjs +0 -1427
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-V5DRIECI.mjs +0 -34
- package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
- package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-GPKLTSKA.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-JYHFTVPS.mjs +0 -63
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
- package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
- package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
- package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
- package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
- package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
- package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
- package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
- package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
- package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
- package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
- package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
- package/dist/release-SMR4WJD4.mjs +0 -32
- package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
- package/dist/restore-queue-LZH7LMLR.mjs +0 -55
- package/dist/retention-3K7ARGAG.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-O7ICP7NJ.mjs +0 -783
- package/dist/start-S4PFP6RS.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-YLJ3ZPQ3.mjs +0 -50
- package/dist/write-2OBJGLFS.mjs +0 -233
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
|
@@ -0,0 +1,3549 @@
|
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-TSLV5P2J.mjs';
|
|
2
|
+
import { RESTORE_ESTIMATE_MARGIN, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-2JRERUNZ.mjs';
|
|
3
|
+
import { decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-OOO54OVC.mjs';
|
|
4
|
+
import { openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, requirePagedPayloadChecksum, checkPagedContents, isPagedSegmentFile, bytesOfResidentSections, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, residentSectionsOf, partitionContentsOf, SEGMENT_HEADER_SIZE } from './chunk-W34VJXY7.mjs';
|
|
5
|
+
import { createReadCache, MERGE_READ_CACHE_BYTES, loadFailed, decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-U6B7DUQW.mjs';
|
|
6
|
+
import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-7MWYORQC.mjs';
|
|
7
|
+
import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-QDZJM7NI.mjs';
|
|
8
|
+
import { encode } from './chunk-77AUVDQ5.mjs';
|
|
9
|
+
import { restoreVectorFields } from './chunk-QBVTYY4J.mjs';
|
|
10
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-IB4BYLE3.mjs';
|
|
11
|
+
import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-LAWIJRMJ.mjs';
|
|
12
|
+
import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-6EZW2GQL.mjs';
|
|
13
|
+
import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, decodeStoredDocument, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readResidentMemory, prepareConfiguredSynonyms, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, ARRAY_POSITION_GAP } from './chunk-TINW4BSH.mjs';
|
|
14
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
15
|
+
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
16
|
+
import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-R4CE4IVU.mjs';
|
|
17
|
+
import { collectAndReturnPages, ANALYSIS_CHANGE_MEMBERS, turnIsSpent, yieldTurn, countWrite, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-3B775F3C.mjs';
|
|
18
|
+
import { VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
|
|
19
|
+
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
20
|
+
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
21
|
+
|
|
22
|
+
// src/engine/analysis-change/pending-target.ts
|
|
23
|
+
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
|
24
|
+
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
|
|
25
|
+
return {
|
|
26
|
+
...rest,
|
|
27
|
+
language,
|
|
28
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
29
|
+
...stopWords !== void 0 ? { stopWords } : {}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function configOfNamedSettings(base, settings) {
|
|
33
|
+
const { language, tokenizer, stopWords, stopWordList } = settings;
|
|
34
|
+
const listed = stopWordList === void 0 ? void 0 : new Set(stopWordList);
|
|
35
|
+
return configUnderAnalysis(base, language, tokenizer, stopWords ?? listed);
|
|
36
|
+
}
|
|
37
|
+
function targetOfPending(entry, pending) {
|
|
38
|
+
const config = configOfNamedSettings(entry.config, pending);
|
|
39
|
+
return { config, language: getLanguage(pending.language), analysis: resolveIndexAnalysis(config) };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// src/partitioning/checkpoint-hold.ts
|
|
43
|
+
var heldCheckpointErrors = /* @__PURE__ */ new WeakSet();
|
|
44
|
+
function rebalanceCheckpointHeld() {
|
|
45
|
+
const error = new NarsilError(
|
|
46
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
47
|
+
"Queued writes from the previous partition layout must be replayed before a checkpoint"
|
|
48
|
+
);
|
|
49
|
+
heldCheckpointErrors.add(error);
|
|
50
|
+
return error;
|
|
51
|
+
}
|
|
52
|
+
async function holdScheduledCheckpoint(checkpoint, requested) {
|
|
53
|
+
try {
|
|
54
|
+
await checkpoint;
|
|
55
|
+
} catch (error) {
|
|
56
|
+
if (requested || typeof error !== "object" || error === null || !heldCheckpointErrors.has(error)) throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// src/partitioning/manager/checkpoint-prefix.ts
|
|
61
|
+
var queues = /* @__PURE__ */ new WeakMap();
|
|
62
|
+
var priorLayouts = /* @__PURE__ */ new WeakMap();
|
|
63
|
+
var heldLayouts = /* @__PURE__ */ new WeakSet();
|
|
64
|
+
function registerCheckpointQueue(manager, queue, holdsLayout = false) {
|
|
65
|
+
queues.set(manager, queue);
|
|
66
|
+
priorLayouts.set(manager, manager.partitionCount);
|
|
67
|
+
if (holdsLayout) heldLayouts.add(manager);
|
|
68
|
+
}
|
|
69
|
+
function releaseCheckpointQueue(manager, queue) {
|
|
70
|
+
if (queues.get(manager) !== queue) return;
|
|
71
|
+
queues.delete(manager);
|
|
72
|
+
priorLayouts.delete(manager);
|
|
73
|
+
heldLayouts.delete(manager);
|
|
74
|
+
}
|
|
75
|
+
function releaseLayoutCheckpoint(manager) {
|
|
76
|
+
heldLayouts.delete(manager);
|
|
77
|
+
}
|
|
78
|
+
function metadataPartitionCount(manager) {
|
|
79
|
+
return priorLayouts.get(manager) ?? manager.partitionCount;
|
|
80
|
+
}
|
|
81
|
+
function requiresLayoutCheckpoint(manager) {
|
|
82
|
+
const prior = priorLayouts.get(manager);
|
|
83
|
+
return prior !== void 0 && prior !== manager.partitionCount;
|
|
84
|
+
}
|
|
85
|
+
async function captureAppliedPrefix(manager, capture2) {
|
|
86
|
+
for (; ; ) {
|
|
87
|
+
const queue = queues.get(manager);
|
|
88
|
+
await queue?.settleAdmissions();
|
|
89
|
+
const captured = await manager.publication.read(async () => {
|
|
90
|
+
if (queue !== queues.get(manager) || (queue?.provisional ?? 0) > 0) return void 0;
|
|
91
|
+
queue?.requireSuccess();
|
|
92
|
+
if (heldLayouts.has(manager)) throw rebalanceCheckpointHeld();
|
|
93
|
+
return { value: await capture2(queue?.checkpointCutoffs(manager.partitionCount) ?? /* @__PURE__ */ new Map()) };
|
|
94
|
+
});
|
|
95
|
+
if (captured !== void 0) return captured.value;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async function requireCheckpointLayout(manager) {
|
|
99
|
+
if (manager !== void 0) await captureAppliedPrefix(manager, () => void 0);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// src/partitioning/wal-routing.ts
|
|
103
|
+
var routings = /* @__PURE__ */ new WeakMap();
|
|
104
|
+
function routingOf(manager) {
|
|
105
|
+
let routing = routings.get(manager);
|
|
106
|
+
if (routing === void 0) {
|
|
107
|
+
routing = { gate: createPublicationGate(), partitionCount: null, failure: null };
|
|
108
|
+
routings.set(manager, routing);
|
|
109
|
+
}
|
|
110
|
+
return routing;
|
|
111
|
+
}
|
|
112
|
+
function beginDurableRebalance(manager) {
|
|
113
|
+
routingOf(manager).partitionCount = manager.partitionCount;
|
|
114
|
+
}
|
|
115
|
+
function finishDurableRebalance(manager) {
|
|
116
|
+
routingOf(manager).partitionCount = null;
|
|
117
|
+
}
|
|
118
|
+
function admitDurableWrite(manager, write) {
|
|
119
|
+
if (manager === void 0) return write(1);
|
|
120
|
+
const routing = routingOf(manager);
|
|
121
|
+
return routing.gate.read(() => {
|
|
122
|
+
if (routing.failure !== null) throw routing.failure;
|
|
123
|
+
return write(routing.partitionCount ?? manager.partitionCount);
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
function switchDurableRouting(manager, commit) {
|
|
127
|
+
const routing = routingOf(manager);
|
|
128
|
+
return routing.gate.write(async () => {
|
|
129
|
+
try {
|
|
130
|
+
await commit(routing.partitionCount ?? manager.partitionCount);
|
|
131
|
+
routing.partitionCount = manager.partitionCount;
|
|
132
|
+
} catch (error) {
|
|
133
|
+
routing.failure = error instanceof Error ? error : new Error(String(error));
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// src/persistence/durability/checkpoint-turn.ts
|
|
140
|
+
function createIndexLoads(vectorIndexesOf) {
|
|
141
|
+
const loads = /* @__PURE__ */ new Map();
|
|
142
|
+
const startGraphRepairs = (indexName) => {
|
|
143
|
+
for (const vectorIndex of vectorIndexesOf(indexName).values()) {
|
|
144
|
+
vectorIndex.scheduleBuild();
|
|
145
|
+
vectorIndex.startShuffledRebuildWhenDue();
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
return {
|
|
149
|
+
track(indexName, load, startsGraphRepairs = true) {
|
|
150
|
+
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
151
|
+
loads.set(
|
|
152
|
+
indexName,
|
|
153
|
+
loaded.catch(() => void 0)
|
|
154
|
+
);
|
|
155
|
+
return loaded;
|
|
156
|
+
},
|
|
157
|
+
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
158
|
+
startGraphRepairs
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
async function switchInTurn(turns, turn) {
|
|
162
|
+
let swapped = false;
|
|
163
|
+
await checkpointInTurn(turns, {
|
|
164
|
+
joinsInFlight: false,
|
|
165
|
+
mayStart: turn.mayStart,
|
|
166
|
+
perform: async () => {
|
|
167
|
+
swapped = await turn.swap();
|
|
168
|
+
if (swapped) await turn.checkpoint();
|
|
169
|
+
},
|
|
170
|
+
afterSettling: turn.afterSettling
|
|
171
|
+
});
|
|
172
|
+
return swapped;
|
|
173
|
+
}
|
|
174
|
+
async function checkpointInTurn(turns, turn) {
|
|
175
|
+
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
176
|
+
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
177
|
+
if (!turn.mayStart()) return;
|
|
178
|
+
const run = turn.perform().finally(() => {
|
|
179
|
+
turns.checkpointInFlight = null;
|
|
180
|
+
turn.afterSettling();
|
|
181
|
+
});
|
|
182
|
+
turns.checkpointInFlight = run;
|
|
183
|
+
return run;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// src/persistence/durability/seq-owner.ts
|
|
187
|
+
var SINGLE_NODE_PRIMARY_TERM = 1;
|
|
188
|
+
function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
|
|
189
|
+
let current = startSeqNo;
|
|
190
|
+
let term = primaryTerm;
|
|
191
|
+
return {
|
|
192
|
+
next() {
|
|
193
|
+
current += 1;
|
|
194
|
+
return current;
|
|
195
|
+
},
|
|
196
|
+
advanceTo(seqNo, nextPrimaryTerm) {
|
|
197
|
+
current = seqNo;
|
|
198
|
+
term = nextPrimaryTerm;
|
|
199
|
+
},
|
|
200
|
+
get current() {
|
|
201
|
+
return current;
|
|
202
|
+
},
|
|
203
|
+
get primaryTerm() {
|
|
204
|
+
return term;
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// src/persistence/durability/layout-transition/codec.ts
|
|
210
|
+
var LAYOUT_TRANSITION_VERSION = 1;
|
|
211
|
+
var LAYOUT_TRANSITION_ENVELOPE_VERSION = 1;
|
|
212
|
+
function transitionKey(indexName) {
|
|
213
|
+
return `${indexName}/layout-transition`;
|
|
214
|
+
}
|
|
215
|
+
function invalidTransition(reason) {
|
|
216
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Partition layout transition ${reason}`);
|
|
217
|
+
}
|
|
218
|
+
function readPayload(raw) {
|
|
219
|
+
if (!isRecord(raw) || raw.version !== LAYOUT_TRANSITION_VERSION) invalidTransition("has an invalid version");
|
|
220
|
+
const { oldPartitionCount, newPartitionCount, oldHeads } = raw;
|
|
221
|
+
if (!isUint32(oldPartitionCount) || oldPartitionCount === 0 || !isUint32(newPartitionCount) || newPartitionCount === 0 || newPartitionCount === oldPartitionCount)
|
|
222
|
+
invalidTransition("has invalid partition counts");
|
|
223
|
+
if (!Array.isArray(oldHeads) || oldHeads.length !== oldPartitionCount)
|
|
224
|
+
invalidTransition("has an incomplete old head list");
|
|
225
|
+
const heads = oldHeads.map((head, partitionId) => {
|
|
226
|
+
if (!isRecord(head) || head.partitionId !== partitionId || !isSequenceNumber(head.lastSeqNo) || !isSequenceNumber(head.primaryTerm))
|
|
227
|
+
invalidTransition("has an invalid old head");
|
|
228
|
+
return { partitionId, lastSeqNo: head.lastSeqNo, primaryTerm: head.primaryTerm };
|
|
229
|
+
});
|
|
230
|
+
return { version: LAYOUT_TRANSITION_VERSION, oldPartitionCount, newPartitionCount, oldHeads: heads };
|
|
231
|
+
}
|
|
232
|
+
async function readTransition(store, indexName) {
|
|
233
|
+
const bytes = await store.read(transitionKey(indexName));
|
|
234
|
+
if (bytes === null) return null;
|
|
235
|
+
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
236
|
+
envelopeFormatVersion: LAYOUT_TRANSITION_ENVELOPE_VERSION,
|
|
237
|
+
what: "partition layout transition",
|
|
238
|
+
allowsCompression: false
|
|
239
|
+
});
|
|
240
|
+
let raw;
|
|
241
|
+
try {
|
|
242
|
+
raw = decode(payload);
|
|
243
|
+
} catch {
|
|
244
|
+
invalidTransition("has an invalid MessagePack payload");
|
|
245
|
+
}
|
|
246
|
+
return readPayload(raw);
|
|
247
|
+
}
|
|
248
|
+
async function writeTransition(store, indexName, transition) {
|
|
249
|
+
const packed = await packChecksummedEnvelope(
|
|
250
|
+
[encode(transition)],
|
|
251
|
+
LAYOUT_TRANSITION_ENVELOPE_VERSION,
|
|
252
|
+
"partition layout transition"
|
|
253
|
+
);
|
|
254
|
+
try {
|
|
255
|
+
await store.write(transitionKey(indexName), packed.chunks);
|
|
256
|
+
} catch (error) {
|
|
257
|
+
if (error instanceof NarsilError) throw error;
|
|
258
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Failed to persist the partition layout transition", {
|
|
259
|
+
indexName,
|
|
260
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// src/persistence/durability/layout-transition/index.ts
|
|
266
|
+
async function transitionWalLayout(registry, indexName, manager, replay) {
|
|
267
|
+
const state = registry.indexState(indexName);
|
|
268
|
+
await switchDurableRouting(
|
|
269
|
+
manager,
|
|
270
|
+
(oldPartitionCount) => checkpointInTurn(state, {
|
|
271
|
+
joinsInFlight: false,
|
|
272
|
+
mayStart: () => {
|
|
273
|
+
if (state.unloading) {
|
|
274
|
+
throw new NarsilError(
|
|
275
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
276
|
+
"The index is unloading during rebalance"
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
return true;
|
|
280
|
+
},
|
|
281
|
+
afterSettling: () => void 0,
|
|
282
|
+
perform: async () => {
|
|
283
|
+
await replay();
|
|
284
|
+
const oldHeads = [];
|
|
285
|
+
for (let partitionId = 0; partitionId < oldPartitionCount; partitionId++) {
|
|
286
|
+
const partition = state.partitions.get(partitionId);
|
|
287
|
+
if (partition !== void 0) {
|
|
288
|
+
await partition.appendChain;
|
|
289
|
+
if (partition.failed !== null) throw partition.failed;
|
|
290
|
+
await partition.walWriter.commit();
|
|
291
|
+
}
|
|
292
|
+
oldHeads.push({
|
|
293
|
+
partitionId,
|
|
294
|
+
lastSeqNo: partition?.appliedSeqNo ?? 0,
|
|
295
|
+
primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
await writeTransition(directoryCheckpointStore(registry.directory), indexName, {
|
|
299
|
+
version: 1,
|
|
300
|
+
oldPartitionCount,
|
|
301
|
+
newPartitionCount: manager.partitionCount,
|
|
302
|
+
oldHeads
|
|
303
|
+
});
|
|
304
|
+
releaseLayoutCheckpoint(manager);
|
|
305
|
+
}
|
|
306
|
+
})
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// src/partitioning/rebalancer.ts
|
|
311
|
+
function createRebalancer() {
|
|
312
|
+
const activeManagers = /* @__PURE__ */ new WeakSet();
|
|
313
|
+
let activeCount = 0;
|
|
314
|
+
async function rebalance(manager, newPartitionCount, router, onProgress, spill = null) {
|
|
315
|
+
if (activeManagers.has(manager)) {
|
|
316
|
+
throw new NarsilError(
|
|
317
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
318
|
+
"A rebalance operation is already in progress"
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
if (newPartitionCount <= 0) {
|
|
322
|
+
throw new NarsilError(
|
|
323
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
324
|
+
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
325
|
+
{ newPartitionCount }
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
if (newPartitionCount === manager.partitionCount) {
|
|
329
|
+
throw new NarsilError(
|
|
330
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
331
|
+
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
332
|
+
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
activeManagers.add(manager);
|
|
336
|
+
activeCount++;
|
|
337
|
+
try {
|
|
338
|
+
const currentPartitions = manager.getAllPartitions();
|
|
339
|
+
let documentsTotal = 0;
|
|
340
|
+
for (const partition of currentPartitions) {
|
|
341
|
+
documentsTotal += partition.count();
|
|
342
|
+
}
|
|
343
|
+
onProgress?.({
|
|
344
|
+
phase: "scanning",
|
|
345
|
+
documentsProcessed: 0,
|
|
346
|
+
documentsTotal
|
|
347
|
+
});
|
|
348
|
+
const newPartitions = [];
|
|
349
|
+
for (let i = 0; i < newPartitionCount; i++) {
|
|
350
|
+
newPartitions.push(spill?.newPartition(i) ?? createPartitionIndex(i, manager.config.trackPositions ?? true));
|
|
351
|
+
}
|
|
352
|
+
let documentsProcessed = 0;
|
|
353
|
+
let chunkFill = 0;
|
|
354
|
+
const insertOptions = {
|
|
355
|
+
validate: false,
|
|
356
|
+
skipClone: true,
|
|
357
|
+
stopWordOverride: manager.analysis.stopWords,
|
|
358
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
359
|
+
collectSurfaces: manager.config.surfaceForms !== false
|
|
360
|
+
};
|
|
361
|
+
for (const partition of currentPartitions) {
|
|
362
|
+
const docIds = [...partition.docIds()];
|
|
363
|
+
for (const docId of docIds) {
|
|
364
|
+
const document = partition.getRef(docId);
|
|
365
|
+
if (!document) continue;
|
|
366
|
+
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
367
|
+
const targetPartition = newPartitions[targetPartitionId];
|
|
368
|
+
if (!targetPartition.has(docId)) {
|
|
369
|
+
newPartitions[targetPartitionId] = writePartitionDocument(
|
|
370
|
+
targetPartition,
|
|
371
|
+
docId,
|
|
372
|
+
document,
|
|
373
|
+
manager.schema,
|
|
374
|
+
manager.language,
|
|
375
|
+
insertOptions
|
|
376
|
+
).partition;
|
|
377
|
+
documentsProcessed++;
|
|
378
|
+
}
|
|
379
|
+
chunkFill++;
|
|
380
|
+
if (chunkFill >= REBALANCE_CHUNK_SIZE) {
|
|
381
|
+
chunkFill = 0;
|
|
382
|
+
onProgress?.({
|
|
383
|
+
phase: "moving",
|
|
384
|
+
documentsProcessed,
|
|
385
|
+
documentsTotal
|
|
386
|
+
});
|
|
387
|
+
for (let at = 0; at < newPartitions.length; at++) await spill?.spillIfFull(newPartitions[at], at);
|
|
388
|
+
await yieldToEventLoop();
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
if (chunkFill > 0) {
|
|
393
|
+
onProgress?.({
|
|
394
|
+
phase: "moving",
|
|
395
|
+
documentsProcessed,
|
|
396
|
+
documentsTotal
|
|
397
|
+
});
|
|
398
|
+
await yieldToEventLoop();
|
|
399
|
+
}
|
|
400
|
+
onProgress?.({
|
|
401
|
+
phase: "swapping",
|
|
402
|
+
documentsProcessed,
|
|
403
|
+
documentsTotal
|
|
404
|
+
});
|
|
405
|
+
await manager.publication.write(() => manager.setPartitions(newPartitions));
|
|
406
|
+
onProgress?.({
|
|
407
|
+
phase: "complete",
|
|
408
|
+
documentsProcessed,
|
|
409
|
+
documentsTotal
|
|
410
|
+
});
|
|
411
|
+
} finally {
|
|
412
|
+
activeManagers.delete(manager);
|
|
413
|
+
activeCount--;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
return {
|
|
417
|
+
rebalance,
|
|
418
|
+
isRebalancing() {
|
|
419
|
+
return activeCount > 0;
|
|
420
|
+
}
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// src/partitioning/write-ahead-queue.ts
|
|
425
|
+
function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
|
|
426
|
+
const entries = /* @__PURE__ */ new Map();
|
|
427
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
428
|
+
let nextSequence = 1;
|
|
429
|
+
let failed = false;
|
|
430
|
+
let failure;
|
|
431
|
+
let replayFailed = false;
|
|
432
|
+
let provisional = 0;
|
|
433
|
+
let rejectReplay = () => void 0;
|
|
434
|
+
const replayFailure = new Promise((_resolve, reject) => {
|
|
435
|
+
rejectReplay = reject;
|
|
436
|
+
});
|
|
437
|
+
void replayFailure.catch(() => void 0);
|
|
438
|
+
let resolveReplayed = () => void 0;
|
|
439
|
+
let rejectReplayed = () => void 0;
|
|
440
|
+
const replayed = new Promise((resolve, reject) => {
|
|
441
|
+
resolveReplayed = resolve;
|
|
442
|
+
rejectReplayed = reject;
|
|
443
|
+
});
|
|
444
|
+
void replayed.catch(() => void 0);
|
|
445
|
+
function assertWritable() {
|
|
446
|
+
if (failed) throw failure;
|
|
447
|
+
if (entries.size >= maxSize) {
|
|
448
|
+
throw new NarsilError(
|
|
449
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
450
|
+
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
451
|
+
{ maxSize, currentSize: entries.size }
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
function refuseWrites(error) {
|
|
456
|
+
if (failed) return;
|
|
457
|
+
failed = true;
|
|
458
|
+
failure = error;
|
|
459
|
+
rejectReplayed(error);
|
|
460
|
+
}
|
|
461
|
+
function requireReplayable() {
|
|
462
|
+
if (replayFailed) throw failure;
|
|
463
|
+
}
|
|
464
|
+
function discard(entry) {
|
|
465
|
+
entries.delete(entry.sequenceNumber);
|
|
466
|
+
if (byDocument.get(entry.docId) !== entry) return;
|
|
467
|
+
byDocument.delete(entry.docId);
|
|
468
|
+
for (const remaining of entries.values()) {
|
|
469
|
+
if (remaining.docId === entry.docId) byDocument.set(entry.docId, remaining);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
return {
|
|
473
|
+
assertWritable,
|
|
474
|
+
push(entry, awaitsDurability) {
|
|
475
|
+
assertWritable();
|
|
476
|
+
const sequenceNumber = nextSequence++;
|
|
477
|
+
let settle = () => void 0;
|
|
478
|
+
let settled = !awaitsDurability;
|
|
479
|
+
if (awaitsDurability) provisional++;
|
|
480
|
+
const ready = awaitsDurability ? new Promise((resolve) => {
|
|
481
|
+
settle = resolve;
|
|
482
|
+
}) : Promise.resolve(true);
|
|
483
|
+
const prepared = { ...entry, sequenceNumber, ready };
|
|
484
|
+
entries.set(sequenceNumber, prepared);
|
|
485
|
+
byDocument.set(entry.docId, prepared);
|
|
486
|
+
return {
|
|
487
|
+
commit(partitionId, partitionCount, seqNo) {
|
|
488
|
+
if (settled) return;
|
|
489
|
+
settled = true;
|
|
490
|
+
provisional--;
|
|
491
|
+
prepared.durability = { partitionId, partitionCount, seqNo };
|
|
492
|
+
settle(true);
|
|
493
|
+
},
|
|
494
|
+
fail(error) {
|
|
495
|
+
if (settled) return;
|
|
496
|
+
settled = true;
|
|
497
|
+
provisional--;
|
|
498
|
+
discard(prepared);
|
|
499
|
+
refuseWrites(error);
|
|
500
|
+
settle(false);
|
|
501
|
+
}
|
|
502
|
+
};
|
|
503
|
+
},
|
|
504
|
+
first: () => entries.values().next().value,
|
|
505
|
+
complete(entry) {
|
|
506
|
+
requireReplayable();
|
|
507
|
+
if (entries.values().next().value !== entry) {
|
|
508
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "Queued writes must complete in sequence order");
|
|
509
|
+
}
|
|
510
|
+
entries.delete(entry.sequenceNumber);
|
|
511
|
+
if (byDocument.get(entry.docId) === entry) byDocument.delete(entry.docId);
|
|
512
|
+
},
|
|
513
|
+
latest: (docId) => byDocument.get(docId),
|
|
514
|
+
awaitReady: (entry) => Promise.race([entry.ready, replayFailure]),
|
|
515
|
+
requireReplayable,
|
|
516
|
+
requireSuccess() {
|
|
517
|
+
if (failed) throw failure;
|
|
518
|
+
},
|
|
519
|
+
async settleAdmissions() {
|
|
520
|
+
if (failed) throw failure;
|
|
521
|
+
await Promise.race([Promise.all([...entries.values()].map((entry) => entry.ready)), replayed]);
|
|
522
|
+
if (failed) throw failure;
|
|
523
|
+
},
|
|
524
|
+
checkpointCutoffs(partitionCount) {
|
|
525
|
+
if (failed) throw failure;
|
|
526
|
+
const cutoffs = /* @__PURE__ */ new Map();
|
|
527
|
+
for (const entry of entries.values()) {
|
|
528
|
+
const position = entry.durability;
|
|
529
|
+
if (position === void 0) continue;
|
|
530
|
+
if (position.partitionCount !== partitionCount) {
|
|
531
|
+
throw rebalanceCheckpointHeld();
|
|
532
|
+
}
|
|
533
|
+
const previous = cutoffs.get(position.partitionId);
|
|
534
|
+
cutoffs.set(position.partitionId, Math.min(previous ?? position.seqNo - 1, position.seqNo - 1));
|
|
535
|
+
}
|
|
536
|
+
return cutoffs;
|
|
537
|
+
},
|
|
538
|
+
get provisional() {
|
|
539
|
+
return provisional;
|
|
540
|
+
},
|
|
541
|
+
bufferedDocState(docId) {
|
|
542
|
+
const entry = byDocument.get(docId);
|
|
543
|
+
return entry === void 0 ? void 0 : entry.action === "remove" ? "absent" : "present";
|
|
544
|
+
},
|
|
545
|
+
whenReplayed() {
|
|
546
|
+
return replayed;
|
|
547
|
+
},
|
|
548
|
+
markReplayed() {
|
|
549
|
+
if (failed) throw failure;
|
|
550
|
+
if (entries.size > 0) {
|
|
551
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "Queued writes remain unapplied");
|
|
552
|
+
}
|
|
553
|
+
resolveReplayed();
|
|
554
|
+
},
|
|
555
|
+
fail(error) {
|
|
556
|
+
refuseWrites(error);
|
|
557
|
+
replayFailed = true;
|
|
558
|
+
rejectReplay(error);
|
|
559
|
+
},
|
|
560
|
+
get failed() {
|
|
561
|
+
return failed;
|
|
562
|
+
},
|
|
563
|
+
get size() {
|
|
564
|
+
return entries.size;
|
|
565
|
+
},
|
|
566
|
+
get isFull() {
|
|
567
|
+
return entries.size >= maxSize;
|
|
568
|
+
}
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
// src/engine/queued-mutation.ts
|
|
573
|
+
function bufferPreparedMutation(queue, manager, entry, vectorFieldPaths, awaitsDurability) {
|
|
574
|
+
queue.assertWritable();
|
|
575
|
+
const previous = queue.latest(entry.docId);
|
|
576
|
+
const previousDocument = previous === void 0 ? manager.get(entry.docId) : previous.document;
|
|
577
|
+
if (entry.action === "insert" && previousDocument !== void 0) {
|
|
578
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${entry.docId}" already exists`);
|
|
579
|
+
}
|
|
580
|
+
if (entry.action === "remove" && previousDocument === void 0) {
|
|
581
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${entry.docId}" not found`);
|
|
582
|
+
}
|
|
583
|
+
const document = entry.document === void 0 ? void 0 : structuredClone(entry.document);
|
|
584
|
+
const { options, language } = manager.textAnalysis;
|
|
585
|
+
if (document !== void 0) {
|
|
586
|
+
validateDocument(document, manager.schema, options?.patternValueLimit);
|
|
587
|
+
if (options?.strict) validateDocumentStrict(document, manager.schema);
|
|
588
|
+
}
|
|
589
|
+
const fields = schemaFieldsOf(manager.schema);
|
|
590
|
+
const surfaceFields = fields.filter((field) => field.base === "string" || field.base === "string[]").map((field) => field.path);
|
|
591
|
+
const prepareText = (value) => prepareRemovalText(prepareDocumentText(value, fields, language, options), value, surfaceFields, language, options);
|
|
592
|
+
const before = previousDocument === void 0 ? void 0 : previous?.text.after ?? prepareText(previousDocument);
|
|
593
|
+
const after = document === void 0 ? void 0 : prepareText(document);
|
|
594
|
+
const partitionDoc = document === void 0 ? void 0 : prepareDocumentVectors(document, vectorFieldPaths);
|
|
595
|
+
const vectors = /* @__PURE__ */ new Map();
|
|
596
|
+
for (const field of vectorFieldPaths) vectors.set(field, partitionDoc?.extractedVectors.get(field) ?? null);
|
|
597
|
+
return queue.push(
|
|
598
|
+
structuredClone({
|
|
599
|
+
...entry,
|
|
600
|
+
document,
|
|
601
|
+
partitionDoc: partitionDoc?.partitionDoc,
|
|
602
|
+
previousDocument,
|
|
603
|
+
previousPartitionDoc: previousDocument === void 0 ? void 0 : previous?.partitionDoc ?? prepareDocumentVectors(previousDocument, vectorFieldPaths).partitionDoc,
|
|
604
|
+
text: { before, after },
|
|
605
|
+
vectors,
|
|
606
|
+
removedSpellings: before === void 0 ? removedSurfaceSpellings(void 0) : previous?.afterSpellings ?? removedSurfaceSpellings(before),
|
|
607
|
+
afterSpellings: removedSurfaceSpellings(after)
|
|
608
|
+
}),
|
|
609
|
+
awaitsDurability
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// src/engine/rebalance-replay.ts
|
|
614
|
+
function applyEntry(manager, entry, ctx) {
|
|
615
|
+
const { docId, indexName, partitionDoc, previousPartitionDoc, text, removedSpellings } = entry;
|
|
616
|
+
const options = { preparedText: text, validate: false };
|
|
617
|
+
if (entry.action === "remove") {
|
|
618
|
+
manager.remove(docId, removedSpellings, options);
|
|
619
|
+
} else if (partitionDoc === void 0) {
|
|
620
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "A queued write has no prepared document");
|
|
621
|
+
} else if (previousPartitionDoc === void 0) {
|
|
622
|
+
manager.insert(docId, partitionDoc, options);
|
|
623
|
+
} else {
|
|
624
|
+
manager.update(docId, partitionDoc, options, removedSpellings);
|
|
625
|
+
}
|
|
626
|
+
try {
|
|
627
|
+
updateDocumentVectors(docId, entry.vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
628
|
+
} catch (error) {
|
|
629
|
+
const rollback = {
|
|
630
|
+
preparedText: { before: text.after, after: text.before },
|
|
631
|
+
validate: false
|
|
632
|
+
};
|
|
633
|
+
if (previousPartitionDoc === void 0) manager.remove(docId, entry.afterSpellings, rollback);
|
|
634
|
+
else if (entry.action === "remove") manager.insert(docId, previousPartitionDoc, rollback);
|
|
635
|
+
else manager.update(docId, previousPartitionDoc, rollback, entry.afterSpellings);
|
|
636
|
+
throw error;
|
|
637
|
+
}
|
|
638
|
+
if (entry.action === "remove") {
|
|
639
|
+
return ctx.orchestrator.publishToWorkers({
|
|
640
|
+
type: "remove",
|
|
641
|
+
indexName,
|
|
642
|
+
docId,
|
|
643
|
+
removedSpellings,
|
|
644
|
+
requestId: `replicate-remove-${docId}`
|
|
645
|
+
});
|
|
646
|
+
} else if (partitionDoc !== void 0) {
|
|
647
|
+
return ctx.orchestrator.publishToWorkers({
|
|
648
|
+
type: previousPartitionDoc === void 0 ? "insert" : "update",
|
|
649
|
+
indexName,
|
|
650
|
+
docId,
|
|
651
|
+
document: partitionDoc,
|
|
652
|
+
removedSpellings,
|
|
653
|
+
requestId: `replicate-${entry.action}-${docId}`
|
|
654
|
+
});
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
async function afterEntry(manager, entry, ctx) {
|
|
658
|
+
for (const field of entry.vectors.keys()) manager.getVectorIndexes().get(field)?.scheduleBuild();
|
|
659
|
+
try {
|
|
660
|
+
if (entry.action === "remove") {
|
|
661
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName: entry.indexName, docId: entry.docId });
|
|
662
|
+
} else if (entry.document !== void 0) {
|
|
663
|
+
if (entry.action === "insert") {
|
|
664
|
+
await ctx.pluginRegistry.runHook("afterInsert", {
|
|
665
|
+
indexName: entry.indexName,
|
|
666
|
+
docId: entry.docId,
|
|
667
|
+
document: structuredClone(entry.document)
|
|
668
|
+
});
|
|
669
|
+
} else {
|
|
670
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
671
|
+
indexName: entry.indexName,
|
|
672
|
+
docId: entry.docId,
|
|
673
|
+
oldDocument: structuredClone(entry.previousDocument ?? {}),
|
|
674
|
+
newDocument: structuredClone(entry.document)
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
} catch (error) {
|
|
679
|
+
console.warn("Queued write plugin hook error:", error instanceof Error ? error.message : String(error));
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
async function replayQueued(manager, queue, ctx, deferredHooks) {
|
|
683
|
+
for (; ; ) {
|
|
684
|
+
const entry = queue.first();
|
|
685
|
+
if (entry === void 0) {
|
|
686
|
+
queue.requireSuccess();
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
try {
|
|
690
|
+
if (!await queue.awaitReady(entry)) continue;
|
|
691
|
+
let publication;
|
|
692
|
+
await countWrite(
|
|
693
|
+
() => manager.publication.write(() => {
|
|
694
|
+
try {
|
|
695
|
+
queue.requireReplayable();
|
|
696
|
+
publication = applyEntry(manager, entry, ctx);
|
|
697
|
+
queue.complete(entry);
|
|
698
|
+
} catch (error) {
|
|
699
|
+
queue.fail(error);
|
|
700
|
+
throw error;
|
|
701
|
+
}
|
|
702
|
+
})
|
|
703
|
+
);
|
|
704
|
+
await publication;
|
|
705
|
+
} catch (error) {
|
|
706
|
+
queue.fail(error);
|
|
707
|
+
throw error;
|
|
708
|
+
}
|
|
709
|
+
if (deferredHooks === void 0) await afterEntry(manager, entry, ctx);
|
|
710
|
+
else deferredHooks.push(() => afterEntry(manager, entry, ctx));
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// src/engine/rebalance-executor.ts
|
|
715
|
+
async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
716
|
+
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
717
|
+
throw new NarsilError(
|
|
718
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
719
|
+
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
720
|
+
{ targetPartitionCount }
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
const entry = ctx.requireIndex(indexName);
|
|
724
|
+
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
725
|
+
if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
|
|
726
|
+
throw new NarsilError(
|
|
727
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
728
|
+
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
729
|
+
{ targetPartitionCount, maxPartitions }
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
if (ctx.rebalancingIndexes.has(indexName)) {
|
|
733
|
+
throw new NarsilError(
|
|
734
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
735
|
+
`Index "${indexName}" is already being rebalanced`
|
|
736
|
+
);
|
|
737
|
+
}
|
|
738
|
+
const registered = ctx.requireIndex(indexName);
|
|
739
|
+
if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
|
|
740
|
+
throw new NarsilError(
|
|
741
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
742
|
+
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
743
|
+
{ indexName }
|
|
744
|
+
);
|
|
745
|
+
}
|
|
746
|
+
if (targetPartitionCount === manager.partitionCount) {
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
ctx.prepareQueuedMutation = bufferPreparedMutation;
|
|
750
|
+
const waq = createWriteAheadQueue();
|
|
751
|
+
const registry = ctx.durabilityManager?.partitionRegistry;
|
|
752
|
+
registerCheckpointQueue(manager, waq, registry !== void 0);
|
|
753
|
+
if (registry !== void 0) beginDurableRebalance(manager);
|
|
754
|
+
ctx.waqMap.set(indexName, waq);
|
|
755
|
+
ctx.rebalancingIndexes.add(indexName);
|
|
756
|
+
ctx.rebalanceTargets.set(indexName, targetPartitionCount);
|
|
757
|
+
const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
|
|
758
|
+
let spill = null;
|
|
759
|
+
const oldCount = manager.partitionCount;
|
|
760
|
+
try {
|
|
761
|
+
spill = await rewriteSpillOf(ctx.durabilityManager?.rewriteStorage?.(), manager);
|
|
762
|
+
await createRebalancer().rebalance(
|
|
763
|
+
manager,
|
|
764
|
+
targetPartitionCount,
|
|
765
|
+
ctx.router,
|
|
766
|
+
(progress) => {
|
|
767
|
+
if (progress.phase === "complete") {
|
|
768
|
+
emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
|
|
769
|
+
indexName,
|
|
770
|
+
oldCount,
|
|
771
|
+
newCount: targetPartitionCount
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
},
|
|
775
|
+
spill
|
|
776
|
+
);
|
|
777
|
+
try {
|
|
778
|
+
await ctx.pluginRegistry.runHook("onPartitionSplit", {
|
|
779
|
+
indexName,
|
|
780
|
+
oldPartitionCount: oldCount,
|
|
781
|
+
newPartitionCount: targetPartitionCount
|
|
782
|
+
});
|
|
783
|
+
} catch (hookErr) {
|
|
784
|
+
console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
|
|
785
|
+
}
|
|
786
|
+
if (registry === void 0) await replayQueued(manager, waq, ctx);
|
|
787
|
+
else {
|
|
788
|
+
const hooks = [];
|
|
789
|
+
await transitionWalLayout(registry, indexName, manager, () => replayQueued(manager, waq, ctx, hooks));
|
|
790
|
+
for (const hook of hooks) await hook();
|
|
791
|
+
}
|
|
792
|
+
if (ctx.durabilityManager) {
|
|
793
|
+
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
794
|
+
await spill?.removeUnread(manager);
|
|
795
|
+
await replayQueued(manager, waq, ctx);
|
|
796
|
+
}
|
|
797
|
+
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
798
|
+
do {
|
|
799
|
+
await replayQueued(manager, waq, ctx);
|
|
800
|
+
} while (waq.size > 0);
|
|
801
|
+
} catch (err) {
|
|
802
|
+
waq.fail(err);
|
|
803
|
+
await spill?.removeUnread(manager).catch(() => void 0);
|
|
804
|
+
throw err;
|
|
805
|
+
}
|
|
806
|
+
ctx.rebalancingIndexes.delete(indexName);
|
|
807
|
+
ctx.rebalanceTargets.delete(indexName);
|
|
808
|
+
ctx.waqMap.delete(indexName);
|
|
809
|
+
waq.markReplayed();
|
|
810
|
+
releaseCheckpointQueue(manager, waq);
|
|
811
|
+
finishDurableRebalance(manager);
|
|
812
|
+
ctx.checkWatermark(indexName);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
// src/engine/analysis-change/adopt-saved.ts
|
|
816
|
+
function adoptSavedAnalysis(manager, entry, metadata) {
|
|
817
|
+
const saved = reconstructSchemaFromMetadata(metadata);
|
|
818
|
+
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = entry.config;
|
|
819
|
+
const config = {
|
|
820
|
+
...rest,
|
|
821
|
+
language: saved.language,
|
|
822
|
+
...saved.tokenizer !== void 0 ? { tokenizer: saved.tokenizer } : {},
|
|
823
|
+
...saved.stopWords !== void 0 ? { stopWords: saved.stopWords } : {}
|
|
824
|
+
};
|
|
825
|
+
const language = getLanguage(config.language ?? "english");
|
|
826
|
+
const analysis = resolveIndexAnalysis(config);
|
|
827
|
+
const synonyms = prepareStoredSynonyms(rulesOfStoredSynonyms(entry.synonyms), config, language);
|
|
828
|
+
entry.config = config;
|
|
829
|
+
entry.language = language;
|
|
830
|
+
entry.analysisGeneration = metadata.analysisGeneration ?? 0;
|
|
831
|
+
entry.analysisRevision = metadata.analysisRevision ?? null;
|
|
832
|
+
entry.arrayPositionGap = metadata.arrayPositionGap ?? null;
|
|
833
|
+
manager?.adoptAnalysis({
|
|
834
|
+
config,
|
|
835
|
+
language,
|
|
836
|
+
analysis,
|
|
837
|
+
partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis),
|
|
838
|
+
synonyms: synonyms?.lookup ?? null
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
// src/engine/snapshot/memory-budget.ts
|
|
843
|
+
var MEGABYTE_BYTES = 1048576;
|
|
844
|
+
var ARCHIVE_RATES = {
|
|
845
|
+
worst: {
|
|
846
|
+
heap: {
|
|
847
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
848
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
849
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
850
|
+
},
|
|
851
|
+
external: {
|
|
852
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
853
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
854
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
855
|
+
}
|
|
856
|
+
},
|
|
857
|
+
floor: {
|
|
858
|
+
heap: {
|
|
859
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
860
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
861
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
862
|
+
},
|
|
863
|
+
external: {
|
|
864
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
865
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
866
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
var EARLIER_SNAPSHOT_RATES = {
|
|
871
|
+
worst: {
|
|
872
|
+
heap: {
|
|
873
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
874
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
875
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
876
|
+
},
|
|
877
|
+
external: {
|
|
878
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
879
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
880
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
881
|
+
}
|
|
882
|
+
},
|
|
883
|
+
floor: {
|
|
884
|
+
heap: {
|
|
885
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
886
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
887
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
888
|
+
},
|
|
889
|
+
external: {
|
|
890
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
891
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
892
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
};
|
|
896
|
+
function bytesAt(rate, storedBytes, vectors) {
|
|
897
|
+
let bytes = storedBytes * rate.perStoredByte;
|
|
898
|
+
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
899
|
+
return bytes;
|
|
900
|
+
}
|
|
901
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
902
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
903
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
904
|
+
return {
|
|
905
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
906
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
function deadCountOf(dead) {
|
|
910
|
+
if (dead === null) return 0;
|
|
911
|
+
let count = 0;
|
|
912
|
+
for (const byte of dead) {
|
|
913
|
+
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
914
|
+
}
|
|
915
|
+
return count;
|
|
916
|
+
}
|
|
917
|
+
function storedBytesOfSegments(manifest) {
|
|
918
|
+
let bytes = 0;
|
|
919
|
+
for (const partition of manifest.partitions) {
|
|
920
|
+
for (const segment of partition.segments) bytes += segment.bytes;
|
|
921
|
+
}
|
|
922
|
+
return bytes;
|
|
923
|
+
}
|
|
924
|
+
function archiveFootprint(prepared, manifest, replacesIndex, segmentBytes) {
|
|
925
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
926
|
+
const vectors = [];
|
|
927
|
+
for (const field of manifest.vectors) {
|
|
928
|
+
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
929
|
+
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
930
|
+
}
|
|
931
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
932
|
+
}
|
|
933
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
934
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
935
|
+
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
936
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
937
|
+
}
|
|
938
|
+
function inMegabytes(bytes) {
|
|
939
|
+
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
940
|
+
}
|
|
941
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
942
|
+
const heap = readHeapStatistics();
|
|
943
|
+
if (heap !== null) {
|
|
944
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
945
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
946
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
947
|
+
}
|
|
948
|
+
const resident = readResidentMemory();
|
|
949
|
+
if (resident !== null) {
|
|
950
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
951
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
952
|
+
}
|
|
953
|
+
return null;
|
|
954
|
+
}
|
|
955
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
956
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
957
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
958
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
959
|
+
throw new NarsilError(
|
|
960
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
961
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
962
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
963
|
+
);
|
|
964
|
+
}
|
|
965
|
+
function requireAnalysisRebuildMemory(indexName, termBytes) {
|
|
966
|
+
if (shortfallOf(termBytes, termBytes, true) === null) return;
|
|
967
|
+
const shortfall = shortfallOf(termBytes, termBytes, collectAndReturnPages());
|
|
968
|
+
if (shortfall === null) return;
|
|
969
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
970
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
971
|
+
throw new NarsilError(
|
|
972
|
+
ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
|
|
973
|
+
`The analysis change of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where} for a second copy of its terms, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its current settings. Give the process more memory, then change the settings again`,
|
|
974
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
978
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
979
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
980
|
+
const collected = collectAndReturnPages();
|
|
981
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
982
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
// src/engine/analysis-change/run.ts
|
|
986
|
+
function createHold(generation) {
|
|
987
|
+
let markReady = () => void 0;
|
|
988
|
+
let refuse2 = () => void 0;
|
|
989
|
+
let wake = () => void 0;
|
|
990
|
+
const ready = new Promise((resolve, reject) => {
|
|
991
|
+
markReady = resolve;
|
|
992
|
+
refuse2 = reject;
|
|
993
|
+
});
|
|
994
|
+
ready.catch(() => void 0);
|
|
995
|
+
return {
|
|
996
|
+
generation,
|
|
997
|
+
built: false,
|
|
998
|
+
released: false,
|
|
999
|
+
ready,
|
|
1000
|
+
markReady,
|
|
1001
|
+
refuse: refuse2,
|
|
1002
|
+
wake: () => wake(),
|
|
1003
|
+
pause: () => new Promise((resolve) => {
|
|
1004
|
+
const timer = setTimeout(resolve, ANALYSIS_HOLD_APPLY_INTERVAL_MS);
|
|
1005
|
+
wake = () => {
|
|
1006
|
+
clearTimeout(timer);
|
|
1007
|
+
resolve();
|
|
1008
|
+
};
|
|
1009
|
+
})
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
var runsByCore = /* @__PURE__ */ new WeakMap();
|
|
1013
|
+
function runsOf(core) {
|
|
1014
|
+
let runs = runsByCore.get(core);
|
|
1015
|
+
if (runs === void 0) {
|
|
1016
|
+
runs = /* @__PURE__ */ new Map();
|
|
1017
|
+
runsByCore.set(core, runs);
|
|
1018
|
+
}
|
|
1019
|
+
return runs;
|
|
1020
|
+
}
|
|
1021
|
+
function createRun(target, hold = null) {
|
|
1022
|
+
let finish = () => void 0;
|
|
1023
|
+
let rejectFinished = () => void 0;
|
|
1024
|
+
let settle = () => void 0;
|
|
1025
|
+
const finished = new Promise((resolve, reject) => {
|
|
1026
|
+
finish = resolve;
|
|
1027
|
+
rejectFinished = reject;
|
|
1028
|
+
});
|
|
1029
|
+
finished.catch(() => void 0);
|
|
1030
|
+
const settled = new Promise((resolve) => {
|
|
1031
|
+
settle = resolve;
|
|
1032
|
+
});
|
|
1033
|
+
const fail = (error) => {
|
|
1034
|
+
rejectFinished(error);
|
|
1035
|
+
hold?.refuse(error);
|
|
1036
|
+
hold?.wake();
|
|
1037
|
+
};
|
|
1038
|
+
return { target, stopReason: null, switched: false, settled, finish, fail, finished, settle, hold };
|
|
1039
|
+
}
|
|
1040
|
+
async function waitForRelease(run, hold, keepCurrent) {
|
|
1041
|
+
hold.built = true;
|
|
1042
|
+
hold.markReady();
|
|
1043
|
+
while (!hold.released) {
|
|
1044
|
+
if (run.stopReason !== null || !keepCurrent()) {
|
|
1045
|
+
hold.built = false;
|
|
1046
|
+
return false;
|
|
1047
|
+
}
|
|
1048
|
+
await hold.pause();
|
|
1049
|
+
}
|
|
1050
|
+
return true;
|
|
1051
|
+
}
|
|
1052
|
+
function stopRun(run, indexName) {
|
|
1053
|
+
run.stopReason = new NarsilError(
|
|
1054
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
1055
|
+
`A later change or a restore replaced this analysis change of index "${indexName}"`,
|
|
1056
|
+
{ indexName }
|
|
1057
|
+
);
|
|
1058
|
+
run.fail(run.stopReason);
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// src/engine/snapshot/index-configuration-limit.ts
|
|
1062
|
+
function requireConfigurationWithinSizeLimit(config, language) {
|
|
1063
|
+
const payload = oversizedConfigurationPayloadOf(config, language);
|
|
1064
|
+
if (payload !== null) requireEncodedConfigurationWithinSizeLimit(encode(payload).byteLength);
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// src/engine/analysis-change/settings.ts
|
|
1068
|
+
function refuseTokenizer(indexName) {
|
|
1069
|
+
throw new NarsilError(
|
|
1070
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1071
|
+
`The tokenizer of index "${indexName}" must change to a name registered with registerTokenizer, because an instance survives neither a worker boundary nor a restart`,
|
|
1072
|
+
{ indexName }
|
|
1073
|
+
);
|
|
1074
|
+
}
|
|
1075
|
+
function refuseStopWords(indexName) {
|
|
1076
|
+
throw new NarsilError(
|
|
1077
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1078
|
+
`The stop words of index "${indexName}" must change to a name registered with registerStopWords or to a list of words, because a function survives neither a worker boundary nor a restart`,
|
|
1079
|
+
{ indexName }
|
|
1080
|
+
);
|
|
1081
|
+
}
|
|
1082
|
+
function stopWordsOfChange(indexName, value) {
|
|
1083
|
+
if (value === void 0 || value === null || typeof value === "string") return value;
|
|
1084
|
+
const words = value instanceof Set ? [...value] : value;
|
|
1085
|
+
if (!Array.isArray(words) || words.some((word) => typeof word !== "string")) refuseStopWords(indexName);
|
|
1086
|
+
return new Set(words);
|
|
1087
|
+
}
|
|
1088
|
+
function tokenizerOfChange(indexName, value) {
|
|
1089
|
+
if (value === void 0 || value === null || typeof value === "string") return value;
|
|
1090
|
+
refuseTokenizer(indexName);
|
|
1091
|
+
}
|
|
1092
|
+
function requireRegisteredLanguage(name) {
|
|
1093
|
+
if (typeof name !== "string") {
|
|
1094
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, "The language of an analysis change must be a name");
|
|
1095
|
+
}
|
|
1096
|
+
return getLanguage(name);
|
|
1097
|
+
}
|
|
1098
|
+
function resolveAnalysisTarget(indexName, base, change) {
|
|
1099
|
+
if (change === null || typeof change !== "object" || Array.isArray(change)) {
|
|
1100
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "An analysis change must be an object", { indexName });
|
|
1101
|
+
}
|
|
1102
|
+
const unknownMember = Object.keys(change).find((member) => !ANALYSIS_CHANGE_MEMBERS.includes(member));
|
|
1103
|
+
if (unknownMember !== void 0) {
|
|
1104
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `"${unknownMember}" is not an analysis setting`, { indexName });
|
|
1105
|
+
}
|
|
1106
|
+
const language = change.language === void 0 ? getLanguage(base.language ?? "english") : requireRegisteredLanguage(change.language);
|
|
1107
|
+
const tokenizer = tokenizerOfChange(indexName, change.tokenizer);
|
|
1108
|
+
const stopWords = stopWordsOfChange(indexName, change.stopWords);
|
|
1109
|
+
const config = configUnderAnalysis(
|
|
1110
|
+
base,
|
|
1111
|
+
language.name,
|
|
1112
|
+
tokenizer === void 0 ? base.tokenizer : tokenizer ?? void 0,
|
|
1113
|
+
stopWords === void 0 ? base.stopWords : stopWords ?? void 0
|
|
1114
|
+
);
|
|
1115
|
+
requireConfigurationWithinSizeLimit(config, language);
|
|
1116
|
+
return { config, language, analysis: resolveIndexAnalysis(config) };
|
|
1117
|
+
}
|
|
1118
|
+
function currentConfigUnderTarget(current, target) {
|
|
1119
|
+
return configUnderAnalysis(current, target.language.name, target.config.tokenizer, target.config.stopWords);
|
|
1120
|
+
}
|
|
1121
|
+
function sameStopWords(left, right) {
|
|
1122
|
+
if (left === right) return true;
|
|
1123
|
+
if (!(left instanceof Set) || !(right instanceof Set) || left.size !== right.size) return false;
|
|
1124
|
+
for (const word of left) if (!right.has(word)) return false;
|
|
1125
|
+
return true;
|
|
1126
|
+
}
|
|
1127
|
+
function holdsSettings(config, target) {
|
|
1128
|
+
return (config.language ?? "english") === target.language.name && config.tokenizer === target.config.tokenizer && sameStopWords(config.stopWords, target.config.stopWords);
|
|
1129
|
+
}
|
|
1130
|
+
function namedSettingsOf(config, language) {
|
|
1131
|
+
return {
|
|
1132
|
+
language,
|
|
1133
|
+
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
1134
|
+
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
1135
|
+
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {}
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
1138
|
+
function durableSettingsOf(entry) {
|
|
1139
|
+
return {
|
|
1140
|
+
...namedSettingsOf(entry.config, entry.language.name),
|
|
1141
|
+
generation: entry.analysisGeneration,
|
|
1142
|
+
analysisRevision: entry.analysisRevision,
|
|
1143
|
+
arrayPositionGap: entry.arrayPositionGap
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1146
|
+
function requireSynonymsUnderTarget(entry, target) {
|
|
1147
|
+
if (entry.synonyms === null) return;
|
|
1148
|
+
prepareConfiguredSynonyms({ ...target.config, synonyms: rulesOfStoredSynonyms(entry.synonyms) }, target.language);
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
// src/engine/analysis-change/index.ts
|
|
1152
|
+
function termBytesOf(core, indexName) {
|
|
1153
|
+
let bytes = 0;
|
|
1154
|
+
for (const partition of core.requireManager(indexName).getAllPartitions()) {
|
|
1155
|
+
bytes += partition.estimateMemoryBytes();
|
|
1156
|
+
for (const state of partition.readStates()) bytes -= state.docStore.contentBytes();
|
|
1157
|
+
}
|
|
1158
|
+
return Math.max(0, bytes);
|
|
1159
|
+
}
|
|
1160
|
+
function toError(value) {
|
|
1161
|
+
return value instanceof Error ? value : new Error(String(value));
|
|
1162
|
+
}
|
|
1163
|
+
async function requireNoUndurableSwitch(core, indexName, entry) {
|
|
1164
|
+
if (entry.durableAnalysis === null) return;
|
|
1165
|
+
const durable = core.durability?.manager;
|
|
1166
|
+
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
1167
|
+
if (entry.durableAnalysis === null) return;
|
|
1168
|
+
throw new NarsilError(
|
|
1169
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
1170
|
+
`The previous analysis change of index "${indexName}" has no durable checkpoint yet, so try this change again once a checkpoint succeeds`,
|
|
1171
|
+
{ indexName }
|
|
1172
|
+
);
|
|
1173
|
+
}
|
|
1174
|
+
function switchIndex({ core, indexName, entry, run, generation }) {
|
|
1175
|
+
const { language, analysis } = run.target;
|
|
1176
|
+
const config = currentConfigUnderTarget(entry.config, run.target);
|
|
1177
|
+
const synonyms = prepareStoredSynonyms(rulesOfStoredSynonyms(entry.synonyms), config, language);
|
|
1178
|
+
return () => {
|
|
1179
|
+
run.switched = true;
|
|
1180
|
+
entry.durableAnalysis = core.durability === null ? null : durableSettingsOf(entry);
|
|
1181
|
+
entry.config = config;
|
|
1182
|
+
entry.language = language;
|
|
1183
|
+
entry.analysisGeneration = generation;
|
|
1184
|
+
entry.analysisRevision = language.revision;
|
|
1185
|
+
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
1186
|
+
entry.pendingAnalysis = null;
|
|
1187
|
+
entry.synonyms = synonyms?.encoded ?? null;
|
|
1188
|
+
core.requireManager(indexName).adoptAnalysis({
|
|
1189
|
+
...textRebuildTargetOf(config, language, analysis),
|
|
1190
|
+
synonyms: synonyms?.lookup ?? null
|
|
1191
|
+
});
|
|
1192
|
+
core.analysisRebuild.clearStale(indexName);
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
async function buildAndSwitch(scope) {
|
|
1196
|
+
const { core, indexName, entry, run } = scope;
|
|
1197
|
+
const { hold } = run;
|
|
1198
|
+
const manager = core.requireManager(indexName);
|
|
1199
|
+
const emit = (status, partitionsRebuilt2, error) => {
|
|
1200
|
+
emitEngineEvent(core.eventHandlers, "analysisRebuild", {
|
|
1201
|
+
indexName,
|
|
1202
|
+
status,
|
|
1203
|
+
partitionsRebuilt: partitionsRebuilt2,
|
|
1204
|
+
partitionCount: manager.partitionCount,
|
|
1205
|
+
...error !== void 0 ? { error } : {}
|
|
1206
|
+
});
|
|
1207
|
+
};
|
|
1208
|
+
let partitionsRebuilt = 0;
|
|
1209
|
+
const refusal = { byRestore: null };
|
|
1210
|
+
const isCurrent = () => run.stopReason === null && refusal.byRestore === null && !core.shutdownState.isShutdown && core.indexRegistry.get(indexName) === entry && core.executor.getManager(indexName) === manager;
|
|
1211
|
+
emit("started", 0);
|
|
1212
|
+
const wasPromoted = core.orchestrator.desyncIndex(indexName);
|
|
1213
|
+
try {
|
|
1214
|
+
const durable = core.durability?.manager;
|
|
1215
|
+
let spill = null;
|
|
1216
|
+
const rebuilt = await rebuildTextIndexes(
|
|
1217
|
+
manager,
|
|
1218
|
+
{
|
|
1219
|
+
isCurrent,
|
|
1220
|
+
rewriteStorage: () => durable?.rewriteStorage?.(),
|
|
1221
|
+
onRewriteSpill(kept) {
|
|
1222
|
+
spill = kept;
|
|
1223
|
+
},
|
|
1224
|
+
onPartitionBuilt(built) {
|
|
1225
|
+
partitionsRebuilt = built;
|
|
1226
|
+
},
|
|
1227
|
+
onSwitch: () => switchIndex(scope)(),
|
|
1228
|
+
...hold === null ? {} : { holdBeforeSwitch: (keepCurrent) => waitForRelease(run, hold, keepCurrent) },
|
|
1229
|
+
async runSwitch(swap) {
|
|
1230
|
+
const swapWithSynonyms = () => {
|
|
1231
|
+
if (!isCurrent()) return false;
|
|
1232
|
+
if (core.restoresInFlight.has(indexName)) {
|
|
1233
|
+
refusal.byRestore = restoreInFlightError(indexName);
|
|
1234
|
+
return false;
|
|
1235
|
+
}
|
|
1236
|
+
return swap();
|
|
1237
|
+
};
|
|
1238
|
+
const releaseSyncEpoch = await holdSyncEpochChange(core, indexName);
|
|
1239
|
+
try {
|
|
1240
|
+
return await (durable?.checkpointAfterSwitch?.(indexName, swapWithSynonyms) ?? swapWithSynonyms());
|
|
1241
|
+
} finally {
|
|
1242
|
+
releaseSyncEpoch();
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
},
|
|
1246
|
+
textRebuildTargetOf(run.target.config, run.target.language, run.target.analysis)
|
|
1247
|
+
);
|
|
1248
|
+
if (!rebuilt) throw run.stopReason ?? refusal.byRestore ?? stoppedError(core, indexName);
|
|
1249
|
+
await spill?.removeUnread(manager);
|
|
1250
|
+
if (entry.durableAnalysis !== null) {
|
|
1251
|
+
throw new NarsilError(
|
|
1252
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1253
|
+
`The new terms of index "${indexName}" answer queries, but they have no durable checkpoint yet`,
|
|
1254
|
+
{ indexName }
|
|
1255
|
+
);
|
|
1256
|
+
}
|
|
1257
|
+
emit("completed", partitionsRebuilt);
|
|
1258
|
+
} catch (error) {
|
|
1259
|
+
emit("failed", partitionsRebuilt, toError(error));
|
|
1260
|
+
throw error;
|
|
1261
|
+
} finally {
|
|
1262
|
+
if (core.executor.getManager(indexName) === manager) {
|
|
1263
|
+
await core.orchestrator.resyncIndex(indexName, wasPromoted).catch(() => void 0);
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
function stoppedError(core, indexName) {
|
|
1268
|
+
if (core.shutdownState.isShutdown) {
|
|
1269
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
1270
|
+
}
|
|
1271
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" closed during its analysis change`, {
|
|
1272
|
+
indexName
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
async function clearPending(core, indexName, entry) {
|
|
1276
|
+
entry.pendingAnalysis = null;
|
|
1277
|
+
await core.durability?.manager.persistMetadata(indexName);
|
|
1278
|
+
}
|
|
1279
|
+
function restoreInFlightError(indexName) {
|
|
1280
|
+
return new NarsilError(
|
|
1281
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
1282
|
+
`A restore of index "${indexName}" is in flight and replaces this analysis change, so try the change again once the restore ends`,
|
|
1283
|
+
{ indexName }
|
|
1284
|
+
);
|
|
1285
|
+
}
|
|
1286
|
+
function requireNoRestoreInFlight(core, indexName) {
|
|
1287
|
+
if (core.restoresInFlight.has(indexName)) throw restoreInFlightError(indexName);
|
|
1288
|
+
}
|
|
1289
|
+
function requireNotRebalancing(core, indexName) {
|
|
1290
|
+
if (!core.rebalancingIndexes.has(indexName)) return;
|
|
1291
|
+
throw new NarsilError(
|
|
1292
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1293
|
+
`Index "${indexName}" is being rebalanced, so try its analysis change again once the rebalance ends`,
|
|
1294
|
+
{ indexName }
|
|
1295
|
+
);
|
|
1296
|
+
}
|
|
1297
|
+
async function driveRun(core, indexName, entry, run, previous, resumedGeneration) {
|
|
1298
|
+
const runs = runsOf(core);
|
|
1299
|
+
try {
|
|
1300
|
+
if (previous !== void 0) await previous.settled;
|
|
1301
|
+
if (runs.get(indexName) !== run) return;
|
|
1302
|
+
await requireNoUndurableSwitch(core, indexName, entry);
|
|
1303
|
+
if (run.stopReason !== null) return;
|
|
1304
|
+
if (core.indexRegistry.get(indexName) !== entry) throw stoppedError(core, indexName);
|
|
1305
|
+
requireNotRebalancing(core, indexName);
|
|
1306
|
+
requireNoRestoreInFlight(core, indexName);
|
|
1307
|
+
if (resumedGeneration === null && holdsSettings(entry.config, run.target)) {
|
|
1308
|
+
if (entry.pendingAnalysis !== null) await clearPending(core, indexName, entry);
|
|
1309
|
+
run.finish();
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
const generation = resumedGeneration ?? Math.max(entry.analysisGeneration, entry.pendingAnalysis?.generation ?? 0) + 1;
|
|
1313
|
+
if (entry.pendingAnalysis?.generation !== generation || run.hold !== null) {
|
|
1314
|
+
entry.pendingAnalysis = { generation, ...namedSettingsOf(run.target.config, run.target.language.name) };
|
|
1315
|
+
await core.durability?.manager.persistMetadata(indexName);
|
|
1316
|
+
}
|
|
1317
|
+
if (run.stopReason !== null) return;
|
|
1318
|
+
await core.analysisRebuild.abandon(indexName);
|
|
1319
|
+
await core.analysisRebuild.runExclusive(
|
|
1320
|
+
indexName,
|
|
1321
|
+
() => buildAndSwitch({ core, indexName, entry, run, generation }),
|
|
1322
|
+
run.hold?.ready
|
|
1323
|
+
);
|
|
1324
|
+
run.finish();
|
|
1325
|
+
} catch (error) {
|
|
1326
|
+
run.fail(error);
|
|
1327
|
+
const refusedByRestore = run.stopReason === null && error instanceof NarsilError && error.code === ErrorCodes.ANALYSIS_CHANGE_REPLACED;
|
|
1328
|
+
const keepsPending = !refusedByRestore && (resumedGeneration !== null || core.shutdownState.isShutdown);
|
|
1329
|
+
const stillOwnsPending = runs.get(indexName) === run && run.stopReason === null;
|
|
1330
|
+
if (!keepsPending && stillOwnsPending && core.indexRegistry.get(indexName) === entry) {
|
|
1331
|
+
if (entry.pendingAnalysis !== null) await clearPending(core, indexName, entry).catch(() => void 0);
|
|
1332
|
+
}
|
|
1333
|
+
} finally {
|
|
1334
|
+
if (runs.get(indexName) === run) runs.delete(indexName);
|
|
1335
|
+
run.settle();
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
async function changeIndexAnalysis(core, indexName, change) {
|
|
1339
|
+
const entry = core.requireIndex(indexName);
|
|
1340
|
+
requireNotRebalancing(core, indexName);
|
|
1341
|
+
requireNoRestoreInFlight(core, indexName);
|
|
1342
|
+
const target = resolveAnalysisTarget(indexName, entry.config, change);
|
|
1343
|
+
const runs = runsOf(core);
|
|
1344
|
+
const previous = runs.get(indexName);
|
|
1345
|
+
if (previous !== void 0 && holdsSettings(previous.target.config, target)) return previous.finished;
|
|
1346
|
+
const heldAlready = holdsSettings(entry.config, target);
|
|
1347
|
+
if (previous === void 0 && entry.durableAnalysis === null && heldAlready) return;
|
|
1348
|
+
if (!heldAlready) {
|
|
1349
|
+
requireSynonymsUnderTarget(entry, target);
|
|
1350
|
+
requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
|
|
1351
|
+
}
|
|
1352
|
+
const run = createRun(target);
|
|
1353
|
+
if (previous !== void 0 && !previous.switched) stopRun(previous, indexName);
|
|
1354
|
+
runs.set(indexName, run);
|
|
1355
|
+
void driveRun(core, indexName, entry, run, previous, null);
|
|
1356
|
+
return run.finished;
|
|
1357
|
+
}
|
|
1358
|
+
function resumeAnalysisChange(core, indexName) {
|
|
1359
|
+
const entry = core.requireIndex(indexName);
|
|
1360
|
+
const runs = runsOf(core);
|
|
1361
|
+
const running = runs.get(indexName);
|
|
1362
|
+
if (running !== void 0) return running.finished;
|
|
1363
|
+
const pending = entry.pendingAnalysis;
|
|
1364
|
+
if (pending === null) return Promise.resolve();
|
|
1365
|
+
let target;
|
|
1366
|
+
try {
|
|
1367
|
+
target = targetOfPending(entry, pending);
|
|
1368
|
+
requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
|
|
1369
|
+
} catch (error) {
|
|
1370
|
+
emitEngineEvent(core.eventHandlers, "analysisRebuild", {
|
|
1371
|
+
indexName,
|
|
1372
|
+
status: "failed",
|
|
1373
|
+
partitionsRebuilt: 0,
|
|
1374
|
+
partitionCount: entry.partitionCount,
|
|
1375
|
+
error: toError(error)
|
|
1376
|
+
});
|
|
1377
|
+
return Promise.reject(error);
|
|
1378
|
+
}
|
|
1379
|
+
const run = createRun(target);
|
|
1380
|
+
runs.set(indexName, run);
|
|
1381
|
+
void driveRun(core, indexName, entry, run, void 0, pending.generation);
|
|
1382
|
+
return run.finished;
|
|
1383
|
+
}
|
|
1384
|
+
async function abandonAnalysisChange(core, indexName) {
|
|
1385
|
+
const run = runsOf(core).get(indexName);
|
|
1386
|
+
if (run === void 0) return;
|
|
1387
|
+
stopRun(run, indexName);
|
|
1388
|
+
await run.settled;
|
|
1389
|
+
}
|
|
1390
|
+
function holdServes(run, entry, pending) {
|
|
1391
|
+
return run.hold?.generation === pending.generation && holdsSettings(run.target.config, targetOfPending(entry, pending));
|
|
1392
|
+
}
|
|
1393
|
+
function startHeldRun(core, indexName, pending) {
|
|
1394
|
+
const entry = core.requireIndex(indexName);
|
|
1395
|
+
if (entry.analysisGeneration >= pending.generation) return null;
|
|
1396
|
+
const runs = runsOf(core);
|
|
1397
|
+
const previous = runs.get(indexName);
|
|
1398
|
+
if (previous !== void 0 && holdServes(previous, entry, pending)) return previous;
|
|
1399
|
+
const target = targetOfPending(entry, pending);
|
|
1400
|
+
requireSynonymsUnderTarget(entry, target);
|
|
1401
|
+
requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
|
|
1402
|
+
const run = createRun(target, createHold(pending.generation));
|
|
1403
|
+
if (previous !== void 0 && !previous.switched) stopRun(previous, indexName);
|
|
1404
|
+
runs.set(indexName, run);
|
|
1405
|
+
void driveRun(core, indexName, entry, run, previous, pending.generation);
|
|
1406
|
+
return run;
|
|
1407
|
+
}
|
|
1408
|
+
async function buildHeldAnalysisChange(core, indexName, pending) {
|
|
1409
|
+
const running = runsOf(core).get(indexName);
|
|
1410
|
+
const entry = core.indexRegistry.get(indexName);
|
|
1411
|
+
if (running?.hold != null && entry !== void 0 && holdServes(running, entry, pending)) return running.hold.ready;
|
|
1412
|
+
const release = await core.indexState.acquire(indexName, false);
|
|
1413
|
+
let run;
|
|
1414
|
+
try {
|
|
1415
|
+
run = startHeldRun(core, indexName, pending);
|
|
1416
|
+
} catch (error) {
|
|
1417
|
+
release();
|
|
1418
|
+
throw error;
|
|
1419
|
+
}
|
|
1420
|
+
if (run?.hold === void 0 || run.hold === null) {
|
|
1421
|
+
release();
|
|
1422
|
+
return;
|
|
1423
|
+
}
|
|
1424
|
+
void run.settled.finally(release);
|
|
1425
|
+
return run.hold.ready;
|
|
1426
|
+
}
|
|
1427
|
+
function heldAnalysisIsBuilt(core, indexName, generation) {
|
|
1428
|
+
const hold = runsOf(core).get(indexName)?.hold;
|
|
1429
|
+
return hold !== void 0 && hold !== null && hold.generation === generation && hold.built;
|
|
1430
|
+
}
|
|
1431
|
+
async function releaseHeldAnalysisChange(core, indexName, generation) {
|
|
1432
|
+
const run = runsOf(core).get(indexName);
|
|
1433
|
+
if (run?.hold?.generation !== generation) return;
|
|
1434
|
+
run.hold.released = true;
|
|
1435
|
+
run.hold.wake();
|
|
1436
|
+
await run.finished;
|
|
1437
|
+
}
|
|
1438
|
+
async function dropHeldAnalysisChange(core, indexName) {
|
|
1439
|
+
await abandonAnalysisChange(core, indexName);
|
|
1440
|
+
const entry = core.indexRegistry.get(indexName);
|
|
1441
|
+
if (entry !== void 0 && entry.pendingAnalysis !== null) await clearPending(core, indexName, entry);
|
|
1442
|
+
}
|
|
1443
|
+
function holdsNamedSettings(entry, settings) {
|
|
1444
|
+
return holdsSettings(entry.config, targetOfPending(entry, { ...settings }));
|
|
1445
|
+
}
|
|
1446
|
+
async function stampAnalysisGeneration(core, indexName, generation) {
|
|
1447
|
+
const entry = core.requireIndex(indexName);
|
|
1448
|
+
if (entry.analysisGeneration >= generation) return;
|
|
1449
|
+
await requireNoUndurableSwitch(core, indexName, entry);
|
|
1450
|
+
const durable = core.durability?.manager;
|
|
1451
|
+
const stamp = () => {
|
|
1452
|
+
entry.durableAnalysis = core.durability === null ? null : durableSettingsOf(entry);
|
|
1453
|
+
entry.analysisGeneration = generation;
|
|
1454
|
+
entry.pendingAnalysis = null;
|
|
1455
|
+
return true;
|
|
1456
|
+
};
|
|
1457
|
+
await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
// src/persistence/paged/read-ahead.ts
|
|
1461
|
+
var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
1462
|
+
function rangeOf(path, section) {
|
|
1463
|
+
return { path, offset: HEADER_SIZE + section.offset, length: section.length };
|
|
1464
|
+
}
|
|
1465
|
+
async function readRangesAhead(ranges) {
|
|
1466
|
+
let rangeAt = 0;
|
|
1467
|
+
let doneInRange = 0;
|
|
1468
|
+
function nextChunk() {
|
|
1469
|
+
while (rangeAt < ranges.length && doneInRange >= ranges[rangeAt].length) {
|
|
1470
|
+
rangeAt += 1;
|
|
1471
|
+
doneInRange = 0;
|
|
1472
|
+
}
|
|
1473
|
+
if (rangeAt >= ranges.length) return null;
|
|
1474
|
+
const { path, offset, length } = ranges[rangeAt];
|
|
1475
|
+
const chunk = { path, offset: offset + doneInRange, length: Math.min(READ_AHEAD_CHUNK_BYTES, length - doneInRange) };
|
|
1476
|
+
doneInRange += chunk.length;
|
|
1477
|
+
return chunk;
|
|
1478
|
+
}
|
|
1479
|
+
const scratch = new Uint8Array(READ_AHEAD_CHUNK_BYTES);
|
|
1480
|
+
async function readChunks() {
|
|
1481
|
+
for (let chunk = nextChunk(); chunk !== null; chunk = nextChunk()) {
|
|
1482
|
+
const target = scratch.subarray(0, chunk.length);
|
|
1483
|
+
if (!await readFileRangeLater(chunk.path, chunk.offset, target)) return;
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
await Promise.all(Array.from({ length: READS_AHEAD_AT_ONCE }, readChunks));
|
|
1487
|
+
}
|
|
1488
|
+
async function headOf(path, ref) {
|
|
1489
|
+
const { headOffset, headLength } = ref;
|
|
1490
|
+
if (headOffset === void 0 || headLength === void 0) return null;
|
|
1491
|
+
const head = new Uint8Array(headLength);
|
|
1492
|
+
if (!await readFileRangeLater(path, HEADER_SIZE + headOffset, head)) return null;
|
|
1493
|
+
return decode(head);
|
|
1494
|
+
}
|
|
1495
|
+
async function levelsOfPart(directory, ref) {
|
|
1496
|
+
const path = await directory.pathOf(ref.key);
|
|
1497
|
+
const head = await headOf(path, ref);
|
|
1498
|
+
const graphs = head?.graphs;
|
|
1499
|
+
if (!Array.isArray(graphs) || graphs.length !== 1) return [];
|
|
1500
|
+
const { levels } = graphs[0];
|
|
1501
|
+
return decodeSections([levels], ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES).map(
|
|
1502
|
+
(section) => rangeOf(path, section)
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1505
|
+
async function residentSectionsOfSegment(directory, ref) {
|
|
1506
|
+
if (!await isPagedSegmentFile(directory, ref.key)) return [];
|
|
1507
|
+
const listing = await pagedListingOf(directory, ref);
|
|
1508
|
+
return residentSectionsOf(listing).map((section) => rangeOf(listing.path, section));
|
|
1509
|
+
}
|
|
1510
|
+
async function startReadingOpeningSectionsAhead(directory, manifest) {
|
|
1511
|
+
try {
|
|
1512
|
+
const groups = await Promise.all([
|
|
1513
|
+
...manifest.partitions.flatMap(
|
|
1514
|
+
(partition) => partition.segments.map((ref) => residentSectionsOfSegment(directory, ref))
|
|
1515
|
+
),
|
|
1516
|
+
...manifest.vectors.flatMap((field) => field.graphParts.map((ref) => levelsOfPart(directory, ref)))
|
|
1517
|
+
]);
|
|
1518
|
+
void readRangesAhead(groups.flat());
|
|
1519
|
+
} catch {
|
|
1520
|
+
return;
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
// src/persistence/paged/segment-scan.ts
|
|
1525
|
+
async function openPagedSegmentScan(directory, ref) {
|
|
1526
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1527
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1528
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
1529
|
+
const { paged } = segment;
|
|
1530
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
1531
|
+
const ids = segment.arrays.idTable;
|
|
1532
|
+
const tombstones = new Set(snapshot.tombstones);
|
|
1533
|
+
return {
|
|
1534
|
+
count: snapshot.documentCount,
|
|
1535
|
+
schema: snapshot.schema,
|
|
1536
|
+
tombstones,
|
|
1537
|
+
idAt: (ordinal) => ids.idAt(ordinal),
|
|
1538
|
+
holds: (docId) => ids.ordinalOf(docId) >= 0,
|
|
1539
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields),
|
|
1540
|
+
close: () => {
|
|
1541
|
+
paged.close();
|
|
1542
|
+
cache.clear();
|
|
1543
|
+
}
|
|
1544
|
+
};
|
|
1545
|
+
}
|
|
1546
|
+
async function checkPagedSegmentFully(directory, ref) {
|
|
1547
|
+
const listing = await pagedListingOf(directory, ref);
|
|
1548
|
+
await requirePagedPayloadChecksum(directory, listing);
|
|
1549
|
+
const snapshot = openPagedSnapshot(listing);
|
|
1550
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1551
|
+
const segment = createPagedFrozenSegment(snapshot, cache);
|
|
1552
|
+
const { paged } = segment;
|
|
1553
|
+
if (paged === null) loadFailed(`The segment "${ref.key}" opened without its file`);
|
|
1554
|
+
try {
|
|
1555
|
+
await checkPagedContents(segment, cache);
|
|
1556
|
+
return snapshot.schema;
|
|
1557
|
+
} finally {
|
|
1558
|
+
paged.close();
|
|
1559
|
+
cache.clear();
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
async function bytesThatSegmentsHoldInMemory(directory, manifest) {
|
|
1563
|
+
let bytes = 0;
|
|
1564
|
+
for (const partition of manifest.partitions) {
|
|
1565
|
+
for (const ref of partition.segments) {
|
|
1566
|
+
const paged = await isPagedSegmentFile(directory, ref.key);
|
|
1567
|
+
bytes += paged ? bytesOfResidentSections(await pagedListingOf(directory, ref)) : ref.bytes;
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
return bytes;
|
|
1571
|
+
}
|
|
1572
|
+
function countsAsLatestHolder(scans, at, docId) {
|
|
1573
|
+
for (let later = at + 1; later < scans.length; later++) {
|
|
1574
|
+
if (scans[later].tombstones.has(docId) || scans[later].holds(docId)) return false;
|
|
1575
|
+
}
|
|
1576
|
+
return true;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
// src/persistence/archive/byte-reader.ts
|
|
1580
|
+
async function* chunksOf(source) {
|
|
1581
|
+
if (source instanceof Uint8Array) {
|
|
1582
|
+
yield source;
|
|
1583
|
+
return;
|
|
1584
|
+
}
|
|
1585
|
+
if (typeof source.getReader === "function") {
|
|
1586
|
+
const reader = source.getReader();
|
|
1587
|
+
try {
|
|
1588
|
+
for (; ; ) {
|
|
1589
|
+
const step = await reader.read();
|
|
1590
|
+
if (step.done) return;
|
|
1591
|
+
yield step.value;
|
|
1592
|
+
}
|
|
1593
|
+
} finally {
|
|
1594
|
+
reader.releaseLock();
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
yield* source;
|
|
1598
|
+
}
|
|
1599
|
+
function createByteReader(source) {
|
|
1600
|
+
const iterator = chunksOf(source);
|
|
1601
|
+
const pending = [];
|
|
1602
|
+
let pendingBytes = 0;
|
|
1603
|
+
let ended = false;
|
|
1604
|
+
async function fill(length) {
|
|
1605
|
+
while (pendingBytes < length && !ended) {
|
|
1606
|
+
const step = await iterator.next();
|
|
1607
|
+
if (step.done) {
|
|
1608
|
+
ended = true;
|
|
1609
|
+
break;
|
|
1610
|
+
}
|
|
1611
|
+
if (step.value.byteLength === 0) continue;
|
|
1612
|
+
pending.push(step.value);
|
|
1613
|
+
pendingBytes += step.value.byteLength;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
function take(length) {
|
|
1617
|
+
const out = new Uint8Array(length);
|
|
1618
|
+
let filled = 0;
|
|
1619
|
+
while (filled < length) {
|
|
1620
|
+
const head = pending[0];
|
|
1621
|
+
const needed = length - filled;
|
|
1622
|
+
if (head.byteLength <= needed) {
|
|
1623
|
+
out.set(head, filled);
|
|
1624
|
+
filled += head.byteLength;
|
|
1625
|
+
pending.shift();
|
|
1626
|
+
} else {
|
|
1627
|
+
out.set(head.subarray(0, needed), filled);
|
|
1628
|
+
pending[0] = head.subarray(needed);
|
|
1629
|
+
filled += needed;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
pendingBytes -= length;
|
|
1633
|
+
return out;
|
|
1634
|
+
}
|
|
1635
|
+
return {
|
|
1636
|
+
async readExactly(length) {
|
|
1637
|
+
await fill(length);
|
|
1638
|
+
if (pendingBytes === 0 && length > 0) return null;
|
|
1639
|
+
if (pendingBytes < length) return take(pendingBytes);
|
|
1640
|
+
return take(length);
|
|
1641
|
+
},
|
|
1642
|
+
async peek(length) {
|
|
1643
|
+
await fill(length);
|
|
1644
|
+
const available = Math.min(length, pendingBytes);
|
|
1645
|
+
const copy = take(available);
|
|
1646
|
+
pending.unshift(copy);
|
|
1647
|
+
pendingBytes += copy.byteLength;
|
|
1648
|
+
return copy;
|
|
1649
|
+
},
|
|
1650
|
+
async close() {
|
|
1651
|
+
await iterator.return(void 0);
|
|
1652
|
+
}
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
// src/engine/durability-metadata.ts
|
|
1657
|
+
var VECTOR_TYPE = /^vector\[(\d+)\]$/;
|
|
1658
|
+
function vectorFieldsOf(schema, promotion) {
|
|
1659
|
+
const fields = {};
|
|
1660
|
+
for (const [fieldPath, type] of Object.entries(schema)) {
|
|
1661
|
+
const match = VECTOR_TYPE.exec(type);
|
|
1662
|
+
if (match === null) continue;
|
|
1663
|
+
const dimension = Number(match[1]);
|
|
1664
|
+
fields[fieldPath] = {
|
|
1665
|
+
dimension,
|
|
1666
|
+
metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
|
|
1667
|
+
quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
|
|
1668
|
+
};
|
|
1669
|
+
}
|
|
1670
|
+
return Object.keys(fields).length === 0 ? null : fields;
|
|
1671
|
+
}
|
|
1672
|
+
function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
1673
|
+
const metadata = {
|
|
1674
|
+
indexName,
|
|
1675
|
+
schema: config.schema,
|
|
1676
|
+
language: config.language,
|
|
1677
|
+
partitionCount: partitionCount ?? config.partitionCount,
|
|
1678
|
+
bm25Params: {
|
|
1679
|
+
k1: config.k1,
|
|
1680
|
+
b: config.b,
|
|
1681
|
+
...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
|
|
1682
|
+
},
|
|
1683
|
+
createdAt: Date.now(),
|
|
1684
|
+
engineVersion: VERSION
|
|
1685
|
+
};
|
|
1686
|
+
metadata.documentCount = documentCount ?? config.documentCount;
|
|
1687
|
+
const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
|
|
1688
|
+
if (vectorFields !== null) {
|
|
1689
|
+
metadata.vectorFields = vectorFields;
|
|
1690
|
+
}
|
|
1691
|
+
if (config.embedding) {
|
|
1692
|
+
metadata.embedding = config.embedding;
|
|
1693
|
+
}
|
|
1694
|
+
if (config.surfaceForms !== void 0) {
|
|
1695
|
+
metadata.surfaceForms = config.surfaceForms;
|
|
1696
|
+
}
|
|
1697
|
+
if (config.analysisRevision !== void 0) {
|
|
1698
|
+
metadata.analysisRevision = config.analysisRevision;
|
|
1699
|
+
}
|
|
1700
|
+
if (config.arrayPositionGap !== void 0) {
|
|
1701
|
+
metadata.arrayPositionGap = config.arrayPositionGap;
|
|
1702
|
+
}
|
|
1703
|
+
if (config.tokenizer !== void 0) {
|
|
1704
|
+
metadata.tokenizer = config.tokenizer;
|
|
1705
|
+
}
|
|
1706
|
+
if (config.stopWords !== void 0) {
|
|
1707
|
+
metadata.stopWords = config.stopWords;
|
|
1708
|
+
}
|
|
1709
|
+
if (config.stopWordList !== void 0) {
|
|
1710
|
+
metadata.stopWordList = config.stopWordList;
|
|
1711
|
+
}
|
|
1712
|
+
if (config.analysisGeneration !== void 0) {
|
|
1713
|
+
metadata.analysisGeneration = config.analysisGeneration;
|
|
1714
|
+
}
|
|
1715
|
+
if (config.pendingAnalysis !== void 0) {
|
|
1716
|
+
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
1717
|
+
}
|
|
1718
|
+
if (config.partitionLimits !== void 0) {
|
|
1719
|
+
metadata.partitionLimits = config.partitionLimits;
|
|
1720
|
+
}
|
|
1721
|
+
if (config.defaultScoring !== void 0) {
|
|
1722
|
+
metadata.defaultScoring = config.defaultScoring;
|
|
1723
|
+
}
|
|
1724
|
+
if (config.trackPositions !== void 0) {
|
|
1725
|
+
metadata.trackPositions = config.trackPositions;
|
|
1726
|
+
}
|
|
1727
|
+
if (config.textStorage !== void 0) {
|
|
1728
|
+
metadata.textStorage = config.textStorage;
|
|
1729
|
+
}
|
|
1730
|
+
if (config.strict !== void 0) {
|
|
1731
|
+
metadata.strict = config.strict;
|
|
1732
|
+
}
|
|
1733
|
+
if (config.required !== void 0) {
|
|
1734
|
+
metadata.required = config.required;
|
|
1735
|
+
}
|
|
1736
|
+
if (config.vectorPromotion !== void 0) {
|
|
1737
|
+
metadata.vectorPromotion = config.vectorPromotion;
|
|
1738
|
+
}
|
|
1739
|
+
if (config.patternValueLimit !== void 0) {
|
|
1740
|
+
metadata.patternValueLimit = config.patternValueLimit;
|
|
1741
|
+
}
|
|
1742
|
+
if (config.indexUuid !== void 0) {
|
|
1743
|
+
metadata.indexUuid = config.indexUuid;
|
|
1744
|
+
}
|
|
1745
|
+
if (config.synonyms !== void 0) {
|
|
1746
|
+
metadata.synonyms = config.synonyms;
|
|
1747
|
+
}
|
|
1748
|
+
if (config.heldPartitions !== void 0) {
|
|
1749
|
+
metadata.heldPartitions = config.heldPartitions;
|
|
1750
|
+
}
|
|
1751
|
+
if (config.sync !== void 0) metadata.sync = config.sync;
|
|
1752
|
+
if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
|
|
1753
|
+
if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
|
|
1754
|
+
if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
|
|
1755
|
+
return metadata;
|
|
1756
|
+
}
|
|
1757
|
+
function metadataConfigOf(entry) {
|
|
1758
|
+
const embedding = entry.config.embedding ? {
|
|
1759
|
+
fields: entry.config.embedding.fields,
|
|
1760
|
+
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
1761
|
+
} : void 0;
|
|
1762
|
+
return {
|
|
1763
|
+
...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
|
|
1764
|
+
...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
|
|
1765
|
+
documentCount: entry.documentCount,
|
|
1766
|
+
partitionCount: entry.partitionCount,
|
|
1767
|
+
schema: flattenSchema(entry.config.schema),
|
|
1768
|
+
language: entry.language.name,
|
|
1769
|
+
k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
|
|
1770
|
+
b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
|
|
1771
|
+
...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
|
|
1772
|
+
...embedding !== void 0 ? { embedding } : {},
|
|
1773
|
+
surfaceForms: entry.config.surfaceForms !== false,
|
|
1774
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
1775
|
+
...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
|
|
1776
|
+
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
1777
|
+
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
1778
|
+
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
1779
|
+
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
1780
|
+
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
1781
|
+
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
1782
|
+
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
1783
|
+
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
1784
|
+
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
1785
|
+
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
1786
|
+
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
1787
|
+
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
1788
|
+
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
|
|
1789
|
+
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
|
|
1790
|
+
...syncMetadataOf(entry),
|
|
1791
|
+
...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
function syncMetadataOf(entry) {
|
|
1795
|
+
const sync = entry.sync;
|
|
1796
|
+
if (sync === null) return {};
|
|
1797
|
+
return {
|
|
1798
|
+
sync: {
|
|
1799
|
+
enabled: true,
|
|
1800
|
+
...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
|
|
1801
|
+
...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
|
|
1802
|
+
},
|
|
1803
|
+
syncActive: sync.active,
|
|
1804
|
+
syncEpoch: sync.epoch
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
function durableMetadataConfigOf(entry) {
|
|
1808
|
+
const terms = metadataConfigOf(entry);
|
|
1809
|
+
const durable = entry.durableAnalysis;
|
|
1810
|
+
if (durable === null) return terms;
|
|
1811
|
+
const {
|
|
1812
|
+
language,
|
|
1813
|
+
tokenizer,
|
|
1814
|
+
stopWords,
|
|
1815
|
+
stopWordList,
|
|
1816
|
+
analysisRevision: _analysisRevision,
|
|
1817
|
+
arrayPositionGap: _arrayPositionGap,
|
|
1818
|
+
analysisGeneration: _analysisGeneration,
|
|
1819
|
+
...rest
|
|
1820
|
+
} = terms;
|
|
1821
|
+
return {
|
|
1822
|
+
...rest,
|
|
1823
|
+
language: durable.language,
|
|
1824
|
+
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
1825
|
+
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
1826
|
+
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
1827
|
+
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
1828
|
+
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
1829
|
+
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
1830
|
+
pendingAnalysis: {
|
|
1831
|
+
generation: entry.analysisGeneration,
|
|
1832
|
+
language,
|
|
1833
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
1834
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
1835
|
+
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
1836
|
+
}
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
1840
|
+
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
// src/serialization/crc32-combine.ts
|
|
1844
|
+
var IEEE_POLYNOMIAL = 3988292384;
|
|
1845
|
+
var CRC_BITS = 32;
|
|
1846
|
+
var zeroByteOperators = [];
|
|
1847
|
+
function timesVector(operator, vector) {
|
|
1848
|
+
let sum = 0;
|
|
1849
|
+
let rest = vector >>> 0;
|
|
1850
|
+
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
1851
|
+
if (rest & 1) sum ^= operator[row];
|
|
1852
|
+
}
|
|
1853
|
+
return sum >>> 0;
|
|
1854
|
+
}
|
|
1855
|
+
function squared(operator) {
|
|
1856
|
+
const result = new Uint32Array(CRC_BITS);
|
|
1857
|
+
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
1858
|
+
return result;
|
|
1859
|
+
}
|
|
1860
|
+
function firstOperator() {
|
|
1861
|
+
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
1862
|
+
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
1863
|
+
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
1864
|
+
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
1865
|
+
return oneZeroBit;
|
|
1866
|
+
}
|
|
1867
|
+
function operatorForZeroBytesPower(power) {
|
|
1868
|
+
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
1869
|
+
while (zeroByteOperators.length <= power) {
|
|
1870
|
+
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
1871
|
+
}
|
|
1872
|
+
return zeroByteOperators[power];
|
|
1873
|
+
}
|
|
1874
|
+
function crc32Combine(first, second, secondByteLength) {
|
|
1875
|
+
let crc = first >>> 0;
|
|
1876
|
+
let rest = secondByteLength;
|
|
1877
|
+
for (let power = 0; rest > 0; power++) {
|
|
1878
|
+
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
1879
|
+
rest = Math.floor(rest / 2);
|
|
1880
|
+
}
|
|
1881
|
+
return (crc ^ second) >>> 0;
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
// src/persistence/paged/vector-bytes.ts
|
|
1885
|
+
var BIN8 = 196;
|
|
1886
|
+
var BIN16 = 197;
|
|
1887
|
+
var BIN32 = 198;
|
|
1888
|
+
var ARRAY16 = 220;
|
|
1889
|
+
var ARRAY32 = 221;
|
|
1890
|
+
var FIXMAP = 128;
|
|
1891
|
+
var FIXARRAY = 144;
|
|
1892
|
+
var FIXED_LIST_MAX = 15;
|
|
1893
|
+
var FIXSTR = 160;
|
|
1894
|
+
var FIXSTR_MAX_LENGTH = 31;
|
|
1895
|
+
var ASCII_MAX = 127;
|
|
1896
|
+
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
1897
|
+
var STRING_HEADER_MAX_BYTES = 5;
|
|
1898
|
+
var BIN8_MAX_BYTES = 255;
|
|
1899
|
+
var UINT16_MAX = 65535;
|
|
1900
|
+
function binHeader(byteLength) {
|
|
1901
|
+
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
1902
|
+
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1903
|
+
const header = new Uint8Array(5);
|
|
1904
|
+
header[0] = BIN32;
|
|
1905
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1906
|
+
return header;
|
|
1907
|
+
}
|
|
1908
|
+
function arrayHeader(count) {
|
|
1909
|
+
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
1910
|
+
if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
|
|
1911
|
+
const header = new Uint8Array(5);
|
|
1912
|
+
header[0] = ARRAY32;
|
|
1913
|
+
new DataView(header.buffer).setUint32(1, count, false);
|
|
1914
|
+
return header;
|
|
1915
|
+
}
|
|
1916
|
+
function createPayloadBuilder() {
|
|
1917
|
+
const chunks = [];
|
|
1918
|
+
let offset = 0;
|
|
1919
|
+
function push(chunk) {
|
|
1920
|
+
chunks.push(chunk);
|
|
1921
|
+
offset += chunk.byteLength;
|
|
1922
|
+
}
|
|
1923
|
+
return {
|
|
1924
|
+
chunks,
|
|
1925
|
+
get offset() {
|
|
1926
|
+
return offset;
|
|
1927
|
+
},
|
|
1928
|
+
push,
|
|
1929
|
+
entry(key, value) {
|
|
1930
|
+
push(encode(key));
|
|
1931
|
+
push(encode(value));
|
|
1932
|
+
},
|
|
1933
|
+
rawEntry(key, encoded) {
|
|
1934
|
+
push(encode(key));
|
|
1935
|
+
const at = offset;
|
|
1936
|
+
push(encoded);
|
|
1937
|
+
return at;
|
|
1938
|
+
},
|
|
1939
|
+
bytesEntry(key, bytes) {
|
|
1940
|
+
push(encode(key));
|
|
1941
|
+
push(binHeader(bytes.byteLength));
|
|
1942
|
+
const at = offset;
|
|
1943
|
+
push(bytes);
|
|
1944
|
+
return at;
|
|
1945
|
+
}
|
|
1946
|
+
};
|
|
1947
|
+
}
|
|
1948
|
+
function pageChecksums(bytes, pageBytes) {
|
|
1949
|
+
const pages = Math.ceil(bytes.byteLength / pageBytes);
|
|
1950
|
+
const checksums = new Uint8Array(pages * 4);
|
|
1951
|
+
const view = new DataView(checksums.buffer);
|
|
1952
|
+
for (let page = 0; page < pages; page++) {
|
|
1953
|
+
const from = page * pageBytes;
|
|
1954
|
+
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
|
|
1955
|
+
}
|
|
1956
|
+
return checksums;
|
|
1957
|
+
}
|
|
1958
|
+
function sectionOf(name, offset, bytes, pageBytes) {
|
|
1959
|
+
return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
|
|
1960
|
+
}
|
|
1961
|
+
function uint32Bytes(values) {
|
|
1962
|
+
const bytes = new Uint8Array(values.length * 4);
|
|
1963
|
+
const view = new DataView(bytes.buffer);
|
|
1964
|
+
for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
|
|
1965
|
+
return bytes;
|
|
1966
|
+
}
|
|
1967
|
+
function writtenAsShortAscii(docId, list, at) {
|
|
1968
|
+
if (docId.length > FIXSTR_MAX_LENGTH) return false;
|
|
1969
|
+
for (let unit = 0; unit < docId.length; unit++) {
|
|
1970
|
+
const code = docId.charCodeAt(unit);
|
|
1971
|
+
if (code > ASCII_MAX) return false;
|
|
1972
|
+
list[at + 1 + unit] = code;
|
|
1973
|
+
}
|
|
1974
|
+
list[at] = FIXSTR | docId.length;
|
|
1975
|
+
return true;
|
|
1976
|
+
}
|
|
1977
|
+
function encodedIdList(docIds) {
|
|
1978
|
+
const header = arrayHeader(docIds.length);
|
|
1979
|
+
let capacity = header.byteLength;
|
|
1980
|
+
for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
|
|
1981
|
+
const list = new Uint8Array(capacity);
|
|
1982
|
+
list.set(header);
|
|
1983
|
+
const offsets = new Uint32Array(docIds.length + 1);
|
|
1984
|
+
let cursor = header.byteLength;
|
|
1985
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
1986
|
+
offsets[i] = cursor;
|
|
1987
|
+
if (writtenAsShortAscii(docIds[i], list, cursor)) {
|
|
1988
|
+
cursor += docIds[i].length + 1;
|
|
1989
|
+
continue;
|
|
1990
|
+
}
|
|
1991
|
+
const encoded = encode(docIds[i]);
|
|
1992
|
+
list.set(encoded, cursor);
|
|
1993
|
+
cursor += encoded.byteLength;
|
|
1994
|
+
}
|
|
1995
|
+
offsets[docIds.length] = cursor;
|
|
1996
|
+
return { list: list.subarray(0, cursor), offsets };
|
|
1997
|
+
}
|
|
1998
|
+
function positionsInIdOrder(docIds) {
|
|
1999
|
+
const positions = Array.from(docIds.keys());
|
|
2000
|
+
positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
|
|
2001
|
+
return Uint32Array.from(positions);
|
|
2002
|
+
}
|
|
2003
|
+
function magnitudeBytes(magnitudes) {
|
|
2004
|
+
const bytes = new Uint8Array(magnitudes.length * 8);
|
|
2005
|
+
const view = new DataView(bytes.buffer);
|
|
2006
|
+
for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
|
|
2007
|
+
return bytes;
|
|
2008
|
+
}
|
|
2009
|
+
function checksumAcrossPages(checksums, byteLength, pageBytes) {
|
|
2010
|
+
const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
|
|
2011
|
+
let whole = 0;
|
|
2012
|
+
for (let page = 0; page * 4 < checksums.byteLength; page++) {
|
|
2013
|
+
const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
|
|
2014
|
+
whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
|
|
2015
|
+
}
|
|
2016
|
+
return whole;
|
|
2017
|
+
}
|
|
2018
|
+
function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
|
|
2019
|
+
let headLength = 0;
|
|
2020
|
+
let head = new Uint8Array(0);
|
|
2021
|
+
let vectorsSection = {
|
|
2022
|
+
name: "vectors",
|
|
2023
|
+
offset: 0,
|
|
2024
|
+
length: vectors.byteLength,
|
|
2025
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
2026
|
+
checksums
|
|
2027
|
+
};
|
|
2028
|
+
for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
|
|
2029
|
+
vectorsSection = {
|
|
2030
|
+
...vectorsSection,
|
|
2031
|
+
offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
|
|
2032
|
+
};
|
|
2033
|
+
head = encode({ sections: [...sections, vectorsSection] });
|
|
2034
|
+
if (head.byteLength === headLength) break;
|
|
2035
|
+
headLength = head.byteLength;
|
|
2036
|
+
}
|
|
2037
|
+
return { head, vectorsSection };
|
|
2038
|
+
}
|
|
2039
|
+
function pagedVectorFileChunks(payload, heldMagnitudes) {
|
|
2040
|
+
const builder = createPayloadBuilder();
|
|
2041
|
+
const { codes } = payload;
|
|
2042
|
+
builder.push(Uint8Array.of(FIXMAP | 10));
|
|
2043
|
+
builder.entry("v", payload.v);
|
|
2044
|
+
builder.entry("fieldName", payload.fieldName);
|
|
2045
|
+
builder.entry("dimension", payload.dimension);
|
|
2046
|
+
const { list, offsets } = encodedIdList(payload.docIds);
|
|
2047
|
+
const docIdsAt = builder.rawEntry("docIds", list);
|
|
2048
|
+
const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
|
|
2049
|
+
builder.push(encode("codes"));
|
|
2050
|
+
if (codes === null) {
|
|
2051
|
+
builder.push(encode(null));
|
|
2052
|
+
} else {
|
|
2053
|
+
builder.push(Uint8Array.of(FIXMAP | 3));
|
|
2054
|
+
builder.entry("bits", codes.bits);
|
|
2055
|
+
builder.entry("centroid", codes.centroid);
|
|
2056
|
+
const recordsAt = builder.bytesEntry("records", codes.records);
|
|
2057
|
+
sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
|
|
2058
|
+
}
|
|
2059
|
+
const magnitudes = magnitudeBytes(heldMagnitudes);
|
|
2060
|
+
sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
|
|
2061
|
+
const idOffsets = uint32Bytes(offsets);
|
|
2062
|
+
sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
|
|
2063
|
+
const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
|
|
2064
|
+
sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
|
|
2065
|
+
const headKey = encode("head");
|
|
2066
|
+
const vectorsKey = encode("vectors");
|
|
2067
|
+
const vectorsHeader = binHeader(payload.vectors.byteLength);
|
|
2068
|
+
const beforeHead = builder.offset + headKey.byteLength;
|
|
2069
|
+
const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
|
|
2070
|
+
const { head } = encodedHeadAndVectorsPlace(
|
|
2071
|
+
sections,
|
|
2072
|
+
(headLength) => beforeHead + binHeader(headLength).byteLength,
|
|
2073
|
+
payload.vectors,
|
|
2074
|
+
vectorChecksums,
|
|
2075
|
+
vectorsKey.byteLength + vectorsHeader.byteLength
|
|
2076
|
+
);
|
|
2077
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
2078
|
+
builder.push(vectorsKey);
|
|
2079
|
+
builder.push(vectorsHeader);
|
|
2080
|
+
const beforeVectors = [...builder.chunks];
|
|
2081
|
+
builder.push(payload.vectors);
|
|
2082
|
+
return {
|
|
2083
|
+
chunks: builder.chunks,
|
|
2084
|
+
headOffset,
|
|
2085
|
+
headLength: head.byteLength,
|
|
2086
|
+
headChecksum: crc32(head),
|
|
2087
|
+
beforeVectors,
|
|
2088
|
+
vectors: {
|
|
2089
|
+
byteLength: payload.vectors.byteLength,
|
|
2090
|
+
checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
|
|
2091
|
+
}
|
|
2092
|
+
};
|
|
2093
|
+
}
|
|
2094
|
+
function nodeOffsetsOf(graph) {
|
|
2095
|
+
const offsets = new Uint32Array(graph.levels.length + 1);
|
|
2096
|
+
const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
|
|
2097
|
+
let cursor = 0;
|
|
2098
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
2099
|
+
offsets[at] = cursor;
|
|
2100
|
+
for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
|
|
2101
|
+
}
|
|
2102
|
+
offsets[graph.levels.length] = cursor;
|
|
2103
|
+
return uint32Bytes(offsets);
|
|
2104
|
+
}
|
|
2105
|
+
function pagedGraphPartChunks(part) {
|
|
2106
|
+
const builder = createPayloadBuilder();
|
|
2107
|
+
builder.push(Uint8Array.of(FIXMAP | 4));
|
|
2108
|
+
builder.entry("v", part.v);
|
|
2109
|
+
builder.entry("fieldName", part.fieldName);
|
|
2110
|
+
builder.push(encode("graphs"));
|
|
2111
|
+
builder.push(arrayHeader(part.graphs.length));
|
|
2112
|
+
const layouts = [];
|
|
2113
|
+
for (const graph of part.graphs) {
|
|
2114
|
+
const { levels, neighbours, ...header } = graph;
|
|
2115
|
+
const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
|
|
2116
|
+
builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
|
|
2117
|
+
for (const [key, value] of headerEntries) builder.entry(key, value);
|
|
2118
|
+
const levelsAt = builder.bytesEntry("levels", levels);
|
|
2119
|
+
const neighboursAt = builder.bytesEntry("neighbours", neighbours);
|
|
2120
|
+
layouts.push({
|
|
2121
|
+
...header,
|
|
2122
|
+
levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
|
|
2123
|
+
nodeOffsets: nodeOffsetsOf(graph),
|
|
2124
|
+
neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
|
|
2125
|
+
});
|
|
2126
|
+
}
|
|
2127
|
+
const head = encode({ graphs: layouts });
|
|
2128
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
2129
|
+
return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
// src/persistence/durability/segment/vector-part-bytes.ts
|
|
2133
|
+
var EMPTY_BIN = new Uint8Array(0);
|
|
2134
|
+
var EMPTY_BIN_BYTES = 2;
|
|
2135
|
+
var BIN82 = 196;
|
|
2136
|
+
var BIN162 = 197;
|
|
2137
|
+
var BIN322 = 198;
|
|
2138
|
+
var BIN8_MAX_BYTES2 = 255;
|
|
2139
|
+
var BIN16_MAX_BYTES = 65535;
|
|
2140
|
+
function binHeader2(byteLength) {
|
|
2141
|
+
if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
|
|
2142
|
+
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
2143
|
+
const header = new Uint8Array(5);
|
|
2144
|
+
header[0] = BIN322;
|
|
2145
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
2146
|
+
return header;
|
|
2147
|
+
}
|
|
2148
|
+
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
2149
|
+
const head = encode(withEmptyBytesInTheirPlace);
|
|
2150
|
+
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
|
|
2151
|
+
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
2152
|
+
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
|
|
2153
|
+
}
|
|
2154
|
+
function vectorPartChunks(part) {
|
|
2155
|
+
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
// src/persistence/durability/segment/vector-write.ts
|
|
2159
|
+
function keptFileRef(input, prior, kept) {
|
|
2160
|
+
const listed = prior?.files.find((file) => file.key === kept.key) ?? input.priorFileRefs?.get(kept.key);
|
|
2161
|
+
if (listed === void 0) {
|
|
2162
|
+
throw new NarsilError(
|
|
2163
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2164
|
+
`The checkpoint keeps the vector file "${kept.key}", which the previous manifest does not list`,
|
|
2165
|
+
{ key: kept.key }
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
|
|
2169
|
+
}
|
|
2170
|
+
function headFieldsOf(ref) {
|
|
2171
|
+
return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
|
|
2172
|
+
}
|
|
2173
|
+
function headRefOf(paged) {
|
|
2174
|
+
return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
|
|
2175
|
+
}
|
|
2176
|
+
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
2177
|
+
const parts = plan.graphParts ?? [];
|
|
2178
|
+
const refs = [];
|
|
2179
|
+
for (let part = 0; part < parts.length; part++) {
|
|
2180
|
+
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
2181
|
+
const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
|
|
2182
|
+
const packed = await packChecksummedEnvelope(
|
|
2183
|
+
paged === null ? [encode(parts[part])] : paged.chunks,
|
|
2184
|
+
GRAPH_PART_ENVELOPE_VERSION,
|
|
2185
|
+
`graph part "${key}"`
|
|
2186
|
+
);
|
|
2187
|
+
await input.store.write(key, packed.chunks);
|
|
2188
|
+
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
|
|
2189
|
+
}
|
|
2190
|
+
return refs;
|
|
2191
|
+
}
|
|
2192
|
+
async function writeField(input, fieldPath, plan, outcome) {
|
|
2193
|
+
const prior = input.priorVectors.find((ref) => ref.fieldPath === fieldPath);
|
|
2194
|
+
if (plan.unchanged && prior !== void 0) {
|
|
2195
|
+
outcome.refs.push(prior);
|
|
2196
|
+
return;
|
|
2197
|
+
}
|
|
2198
|
+
let nextFileId = prior?.nextFileId ?? 0;
|
|
2199
|
+
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
2200
|
+
const written = [];
|
|
2201
|
+
for (let index = 0; index < plan.newFiles; index++) {
|
|
2202
|
+
const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
|
|
2203
|
+
const id = nextFileId;
|
|
2204
|
+
nextFileId += 1;
|
|
2205
|
+
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
2206
|
+
const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
|
|
2207
|
+
const what = `vector file "${key}"`;
|
|
2208
|
+
const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
|
|
2209
|
+
paged.chunks,
|
|
2210
|
+
VECTOR_FILE_ENVELOPE_VERSION,
|
|
2211
|
+
what,
|
|
2212
|
+
crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
|
|
2213
|
+
);
|
|
2214
|
+
await input.store.write(key, packed.chunks);
|
|
2215
|
+
files.push({
|
|
2216
|
+
id,
|
|
2217
|
+
key,
|
|
2218
|
+
count: payload.docIds.length,
|
|
2219
|
+
dead,
|
|
2220
|
+
bytes: packed.byteLength,
|
|
2221
|
+
checksum: packed.checksum,
|
|
2222
|
+
...headRefOf(paged)
|
|
2223
|
+
});
|
|
2224
|
+
written.push({ id, key, ordinals });
|
|
2225
|
+
outcome.layouts.push({
|
|
2226
|
+
fieldPath,
|
|
2227
|
+
key,
|
|
2228
|
+
docIds: payload.docIds,
|
|
2229
|
+
dead,
|
|
2230
|
+
vectorsOffset: packed.byteLength - payload.vectors.byteLength
|
|
2231
|
+
});
|
|
2232
|
+
}
|
|
2233
|
+
const priorGeneration = prior?.graphGeneration ?? 0;
|
|
2234
|
+
const holdsGraph = plan.graphParts !== null && plan.graphParts.length > 0;
|
|
2235
|
+
const graphGeneration = holdsGraph ? priorGeneration + 1 : priorGeneration;
|
|
2236
|
+
const graphParts = holdsGraph ? await writeGraphParts(input, fieldPath, plan, graphGeneration) : [];
|
|
2237
|
+
outcome.refs.push({ fieldPath, nextFileId, files, graphGeneration, graphParts });
|
|
2238
|
+
outcome.written.push({ fieldPath, plan, files: written });
|
|
2239
|
+
}
|
|
2240
|
+
async function writeLiveVectors(input) {
|
|
2241
|
+
const outcome = { refs: [], layouts: [], written: [] };
|
|
2242
|
+
for (const [fieldPath, plan] of input.plans) await writeField(input, fieldPath, plan, outcome);
|
|
2243
|
+
return outcome;
|
|
2244
|
+
}
|
|
2245
|
+
|
|
2246
|
+
// src/persistence/archive/memory-store.ts
|
|
2247
|
+
function joined(chunks) {
|
|
2248
|
+
let total = 0;
|
|
2249
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
2250
|
+
const bytes = new Uint8Array(total);
|
|
2251
|
+
let offset = 0;
|
|
2252
|
+
for (const chunk of chunks) {
|
|
2253
|
+
bytes.set(chunk, offset);
|
|
2254
|
+
offset += chunk.byteLength;
|
|
2255
|
+
}
|
|
2256
|
+
return bytes;
|
|
2257
|
+
}
|
|
2258
|
+
function memoryCheckpointStore() {
|
|
2259
|
+
const files = /* @__PURE__ */ new Map();
|
|
2260
|
+
return {
|
|
2261
|
+
files,
|
|
2262
|
+
read: async (key) => files.get(key) ?? null,
|
|
2263
|
+
write: async (key, chunks) => {
|
|
2264
|
+
files.set(key, joined(chunks));
|
|
2265
|
+
},
|
|
2266
|
+
remove: async (key) => {
|
|
2267
|
+
files.delete(key);
|
|
2268
|
+
},
|
|
2269
|
+
list: async (prefix) => [...files.keys()].filter((key) => key.startsWith(prefix)).sort(),
|
|
2270
|
+
byteLength: async (key) => files.get(key)?.byteLength ?? null
|
|
2271
|
+
};
|
|
2272
|
+
}
|
|
2273
|
+
|
|
2274
|
+
// src/persistence/archive/from-memory.ts
|
|
2275
|
+
function frozenPlan(plan) {
|
|
2276
|
+
const files = [];
|
|
2277
|
+
for (let index = 0; index < plan.newFiles; index++) files.push(plan.readNewFile(index));
|
|
2278
|
+
return { ...plan, readNewFile: (index) => files[index] };
|
|
2279
|
+
}
|
|
2280
|
+
function partitionVectorPlan(vectorIndex, docIds) {
|
|
2281
|
+
const held = [];
|
|
2282
|
+
const vectors = [];
|
|
2283
|
+
for (const docId of docIds) {
|
|
2284
|
+
const vector = vectorIndex.getVector(docId);
|
|
2285
|
+
if (vector === null) continue;
|
|
2286
|
+
held.push(docId);
|
|
2287
|
+
vectors.push(Float32Array.from(vector));
|
|
2288
|
+
}
|
|
2289
|
+
const capacity = vectorsPerFile(vectorIndex.dimension, null);
|
|
2290
|
+
const files = [];
|
|
2291
|
+
for (let start = 0; start < held.length; start += capacity) {
|
|
2292
|
+
const slice = held.slice(start, start + capacity);
|
|
2293
|
+
const packed = new Float32Array(slice.length * vectorIndex.dimension);
|
|
2294
|
+
for (let i = 0; i < slice.length; i++) packed.set(vectors[start + i], i * vectorIndex.dimension);
|
|
2295
|
+
const payload = {
|
|
2296
|
+
v: VECTOR_FILE_PAYLOAD_VERSION,
|
|
2297
|
+
fieldName: vectorIndex.fieldName,
|
|
2298
|
+
dimension: vectorIndex.dimension,
|
|
2299
|
+
docIds: slice,
|
|
2300
|
+
codes: null,
|
|
2301
|
+
vectors: vectorsToBytes(packed)
|
|
2302
|
+
};
|
|
2303
|
+
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
|
|
2304
|
+
}
|
|
2305
|
+
return {
|
|
2306
|
+
unchanged: false,
|
|
2307
|
+
storesHeads: false,
|
|
2308
|
+
kept: [],
|
|
2309
|
+
keptFiles: [],
|
|
2310
|
+
newFiles: files.length,
|
|
2311
|
+
graphParts: null,
|
|
2312
|
+
signature: { revision: 0, graphSize: -1, calibration: null },
|
|
2313
|
+
readNewFile: (index) => files[index],
|
|
2314
|
+
release: () => void 0
|
|
2315
|
+
};
|
|
2316
|
+
}
|
|
2317
|
+
function capture(input) {
|
|
2318
|
+
const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
|
|
2319
|
+
const { indexName } = input;
|
|
2320
|
+
const writeRecords = input.writeRecords ?? null;
|
|
2321
|
+
const targets = partitionIds.map((partitionId) => {
|
|
2322
|
+
const target = {
|
|
2323
|
+
partitionId,
|
|
2324
|
+
lastSeqNo: input.appliedSeqNo(partitionId),
|
|
2325
|
+
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
2326
|
+
};
|
|
2327
|
+
if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
|
|
2328
|
+
return target;
|
|
2329
|
+
});
|
|
2330
|
+
const contents = partitionIds.map(
|
|
2331
|
+
(partitionId) => partitionContentsOf(
|
|
2332
|
+
input.manager,
|
|
2333
|
+
partitionId,
|
|
2334
|
+
writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
|
|
2335
|
+
)
|
|
2336
|
+
);
|
|
2337
|
+
const plans = /* @__PURE__ */ new Map();
|
|
2338
|
+
const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
|
|
2339
|
+
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
2340
|
+
plans.set(
|
|
2341
|
+
fieldPath,
|
|
2342
|
+
partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
return { targets, contents, plans };
|
|
2346
|
+
}
|
|
2347
|
+
async function buildArchiveFromMemory(input) {
|
|
2348
|
+
if (input.partitionId === null) {
|
|
2349
|
+
for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
2350
|
+
}
|
|
2351
|
+
const captured = capture(input);
|
|
2352
|
+
const store = memoryCheckpointStore();
|
|
2353
|
+
const written = await writeCheckpointSegments({
|
|
2354
|
+
store,
|
|
2355
|
+
metadata: input.metadata,
|
|
2356
|
+
targets: captured.targets,
|
|
2357
|
+
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
2358
|
+
prior: null,
|
|
2359
|
+
wholePartitions: captured.contents,
|
|
2360
|
+
keepsWriteRecords: input.writeRecords != null
|
|
2361
|
+
});
|
|
2362
|
+
const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
|
|
2363
|
+
const refs = vectors.refs;
|
|
2364
|
+
await commitCheckpointManifest(store, input.metadata, written, refs);
|
|
2365
|
+
const manifestBytes = store.files.get(manifestKey(input.indexName));
|
|
2366
|
+
const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName, input.partitionId);
|
|
2367
|
+
if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
|
|
2368
|
+
throw new NarsilError(
|
|
2369
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2370
|
+
`The archive of index "${input.indexName}" holds no manifest`
|
|
2371
|
+
);
|
|
2372
|
+
}
|
|
2373
|
+
return {
|
|
2374
|
+
manifestBytes,
|
|
2375
|
+
manifest: decoded.manifest,
|
|
2376
|
+
readFile: async (ref) => {
|
|
2377
|
+
const bytes = store.files.get(ref.key);
|
|
2378
|
+
if (bytes === void 0) {
|
|
2379
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The archive file "${ref.key}" is missing`, {
|
|
2380
|
+
key: ref.key
|
|
2381
|
+
});
|
|
2382
|
+
}
|
|
2383
|
+
return bytes;
|
|
2384
|
+
},
|
|
2385
|
+
readRange: async (ref, offset, length) => {
|
|
2386
|
+
const bytes = store.files.get(ref.key);
|
|
2387
|
+
if (bytes === void 0 || offset + length > bytes.byteLength) {
|
|
2388
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive file "${ref.key}" is missing`, {
|
|
2389
|
+
key: ref.key
|
|
2390
|
+
});
|
|
2391
|
+
}
|
|
2392
|
+
return bytes.subarray(offset, offset + length);
|
|
2393
|
+
},
|
|
2394
|
+
release: () => store.files.clear()
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
|
|
2398
|
+
// src/persistence/archive/write.ts
|
|
2399
|
+
function entryChunks(name, bytes) {
|
|
2400
|
+
return [...tarEntryHeaders(name, bytes.byteLength), bytes, tarPadding(bytes.byteLength)].filter(
|
|
2401
|
+
(chunk) => chunk.byteLength > 0
|
|
2402
|
+
);
|
|
2403
|
+
}
|
|
2404
|
+
async function* fileEntryChunks(source, name, ref) {
|
|
2405
|
+
yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
|
|
2406
|
+
for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
|
|
2407
|
+
yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
|
|
2408
|
+
}
|
|
2409
|
+
const padding = tarPadding(ref.bytes);
|
|
2410
|
+
if (padding.byteLength > 0) yield padding;
|
|
2411
|
+
}
|
|
2412
|
+
async function* archiveChunks(source) {
|
|
2413
|
+
yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
|
|
2414
|
+
yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
|
|
2415
|
+
const prefix = `${source.sourceIndexName}/`;
|
|
2416
|
+
for (const ref of manifestFileRefs(source.manifest)) {
|
|
2417
|
+
yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
|
|
2418
|
+
}
|
|
2419
|
+
if (source.trailingEntries !== void 0) {
|
|
2420
|
+
for await (const entry of source.trailingEntries()) yield* entryChunks(entry.name, entry.bytes);
|
|
2421
|
+
}
|
|
2422
|
+
yield tarEnd();
|
|
2423
|
+
}
|
|
2424
|
+
function archiveStream(source) {
|
|
2425
|
+
const chunks = archiveChunks(source);
|
|
2426
|
+
let released = false;
|
|
2427
|
+
const release = () => {
|
|
2428
|
+
if (released) return;
|
|
2429
|
+
released = true;
|
|
2430
|
+
source.release();
|
|
2431
|
+
};
|
|
2432
|
+
return new ReadableStream({
|
|
2433
|
+
async pull(controller) {
|
|
2434
|
+
try {
|
|
2435
|
+
const step = await chunks.next();
|
|
2436
|
+
if (step.done) {
|
|
2437
|
+
release();
|
|
2438
|
+
controller.close();
|
|
2439
|
+
return;
|
|
2440
|
+
}
|
|
2441
|
+
controller.enqueue(step.value);
|
|
2442
|
+
} catch (error) {
|
|
2443
|
+
release();
|
|
2444
|
+
controller.error(error);
|
|
2445
|
+
}
|
|
2446
|
+
},
|
|
2447
|
+
async cancel() {
|
|
2448
|
+
release();
|
|
2449
|
+
await chunks.return(void 0);
|
|
2450
|
+
}
|
|
2451
|
+
});
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
// src/serialization/stored-vector-values.ts
|
|
2455
|
+
function dropStoredVectorValues(documents, schema) {
|
|
2456
|
+
const vectorFieldPaths = [];
|
|
2457
|
+
for (const [fieldPath, fieldType] of Object.entries(schema)) {
|
|
2458
|
+
if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
|
|
2459
|
+
vectorFieldPaths.push(fieldPath);
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
if (vectorFieldPaths.length === 0) {
|
|
2463
|
+
return;
|
|
2464
|
+
}
|
|
2465
|
+
for (const document of Object.values(documents)) {
|
|
2466
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
2467
|
+
removeFieldAtPath(document.fields, fieldPath);
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
}
|
|
2471
|
+
function removeFieldAtPath(fields, fieldPath) {
|
|
2472
|
+
if (!fieldPath.includes(".")) {
|
|
2473
|
+
delete fields[fieldPath];
|
|
2474
|
+
return;
|
|
2475
|
+
}
|
|
2476
|
+
const segments = fieldPath.split(".");
|
|
2477
|
+
let current = fields;
|
|
2478
|
+
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
2479
|
+
const next = current[segments[i]];
|
|
2480
|
+
if (next === null || typeof next !== "object") {
|
|
2481
|
+
return;
|
|
2482
|
+
}
|
|
2483
|
+
current = next;
|
|
2484
|
+
}
|
|
2485
|
+
delete current[segments[segments.length - 1]];
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
// src/serialization/payload-v1/partition.ts
|
|
2489
|
+
function sanitizeSurfaceForms(raw) {
|
|
2490
|
+
if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
2491
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
2492
|
+
let count = 0;
|
|
2493
|
+
for (const [surface, value] of Object.entries(raw)) {
|
|
2494
|
+
if (typeof value === "number") {
|
|
2495
|
+
result[surface] = value;
|
|
2496
|
+
count++;
|
|
2497
|
+
} else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
|
|
2498
|
+
result[surface] = [value[0], value[1]];
|
|
2499
|
+
count++;
|
|
2500
|
+
}
|
|
2501
|
+
}
|
|
2502
|
+
return count > 0 ? result : void 0;
|
|
2503
|
+
}
|
|
2504
|
+
function isWireRecord(value) {
|
|
2505
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2506
|
+
}
|
|
2507
|
+
function corruptPatternEntry(field) {
|
|
2508
|
+
return new NarsilError(
|
|
2509
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2510
|
+
`The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
|
|
2511
|
+
{ field }
|
|
2512
|
+
);
|
|
2513
|
+
}
|
|
2514
|
+
function patternIndexesFromWire(raw) {
|
|
2515
|
+
if (raw === void 0 || raw === null) return {};
|
|
2516
|
+
if (!isWireRecord(raw)) {
|
|
2517
|
+
throw new NarsilError(
|
|
2518
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2519
|
+
"The pattern entry of this partition must be a map of fields"
|
|
2520
|
+
);
|
|
2521
|
+
}
|
|
2522
|
+
const pattern = /* @__PURE__ */ Object.create(null);
|
|
2523
|
+
for (const [field, index] of Object.entries(raw)) {
|
|
2524
|
+
if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
|
|
2525
|
+
throw corruptPatternEntry(field);
|
|
2526
|
+
}
|
|
2527
|
+
const docIds = [];
|
|
2528
|
+
for (const docId of index.doc_ids) {
|
|
2529
|
+
if (typeof docId !== "string") throw corruptPatternEntry(field);
|
|
2530
|
+
docIds.push(docId);
|
|
2531
|
+
}
|
|
2532
|
+
pattern[field] = { docIds, runs: index.runs };
|
|
2533
|
+
}
|
|
2534
|
+
return { pattern };
|
|
2535
|
+
}
|
|
2536
|
+
function validateHnswMetric(value) {
|
|
2537
|
+
if (value === void 0 || value === null) return void 0;
|
|
2538
|
+
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
2539
|
+
return value;
|
|
2540
|
+
}
|
|
2541
|
+
return void 0;
|
|
2542
|
+
}
|
|
2543
|
+
function wireToPartition(raw) {
|
|
2544
|
+
const documents = {};
|
|
2545
|
+
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
2546
|
+
documents[docId] = {
|
|
2547
|
+
fields: doc.fields,
|
|
2548
|
+
fieldLengths: doc.field_lengths ?? {}
|
|
2549
|
+
};
|
|
2550
|
+
}
|
|
2551
|
+
requireValidFieldTypes(raw.schema ?? {});
|
|
2552
|
+
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
2553
|
+
const invertedIndex = {};
|
|
2554
|
+
for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
|
|
2555
|
+
invertedIndex[token] = {
|
|
2556
|
+
docFrequency: list.doc_freq,
|
|
2557
|
+
postings: (list.postings ?? []).map((p) => ({
|
|
2558
|
+
docId: p.doc_id,
|
|
2559
|
+
termFrequency: p.term_freq,
|
|
2560
|
+
field: p.field,
|
|
2561
|
+
positions: p.positions ?? []
|
|
2562
|
+
}))
|
|
2563
|
+
};
|
|
2564
|
+
}
|
|
2565
|
+
const numeric = {};
|
|
2566
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
2567
|
+
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
2568
|
+
}
|
|
2569
|
+
const boolean = {};
|
|
2570
|
+
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
2571
|
+
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
2572
|
+
}
|
|
2573
|
+
const geopoint = {};
|
|
2574
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
2575
|
+
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
2576
|
+
}
|
|
2577
|
+
const vectorData = {};
|
|
2578
|
+
for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
|
|
2579
|
+
vectorData[field] = {
|
|
2580
|
+
dimension: data.dimension,
|
|
2581
|
+
vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
2582
|
+
hnswGraph: data.hnsw_graph ? {
|
|
2583
|
+
entryPoint: data.hnsw_graph.entry_point,
|
|
2584
|
+
maxLayer: data.hnsw_graph.max_layer,
|
|
2585
|
+
m: data.hnsw_graph.m,
|
|
2586
|
+
efConstruction: data.hnsw_graph.ef_construction,
|
|
2587
|
+
metric: validateHnswMetric(data.hnsw_graph.metric),
|
|
2588
|
+
nodes: data.hnsw_graph.nodes
|
|
2589
|
+
} : null,
|
|
2590
|
+
codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
|
|
2591
|
+
};
|
|
2592
|
+
}
|
|
2593
|
+
return {
|
|
2594
|
+
indexName: raw.index_name,
|
|
2595
|
+
partitionId: raw.partition_id,
|
|
2596
|
+
totalPartitions: raw.total_partitions,
|
|
2597
|
+
language: raw.language ?? "english",
|
|
2598
|
+
schema: raw.schema ?? {},
|
|
2599
|
+
docCount: raw.doc_count ?? 0,
|
|
2600
|
+
avgDocLength: raw.avg_doc_length ?? 0,
|
|
2601
|
+
documents,
|
|
2602
|
+
invertedIndex,
|
|
2603
|
+
fieldIndexes: {
|
|
2604
|
+
numeric,
|
|
2605
|
+
boolean,
|
|
2606
|
+
enum: raw.field_indexes?.enum ?? {},
|
|
2607
|
+
geopoint,
|
|
2608
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
2609
|
+
},
|
|
2610
|
+
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
2611
|
+
vectorData,
|
|
2612
|
+
statistics: {
|
|
2613
|
+
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
2614
|
+
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
2615
|
+
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
2616
|
+
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
2617
|
+
}
|
|
2618
|
+
};
|
|
2619
|
+
}
|
|
2620
|
+
function deserializePayloadV1(data) {
|
|
2621
|
+
const raw = decode(data);
|
|
2622
|
+
return wireToPartition(raw);
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
// src/serialization/payload-v1/sync-metadata.ts
|
|
2626
|
+
function invalid(message, details) {
|
|
2627
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The device sync state in the index metadata ${message}`, details);
|
|
2628
|
+
}
|
|
2629
|
+
function isPositiveSafeInteger(value) {
|
|
2630
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
|
|
2631
|
+
}
|
|
2632
|
+
function syncMetadataToWire(meta) {
|
|
2633
|
+
const wire = {};
|
|
2634
|
+
if (meta.sync !== void 0) {
|
|
2635
|
+
wire.sync = {
|
|
2636
|
+
enabled: meta.sync.enabled,
|
|
2637
|
+
...meta.sync.historyMaxAgeMs !== void 0 ? { history_max_age_ms: meta.sync.historyMaxAgeMs } : {},
|
|
2638
|
+
...meta.sync.historyMaxBytes !== void 0 ? { history_max_bytes: meta.sync.historyMaxBytes } : {}
|
|
2639
|
+
};
|
|
2640
|
+
}
|
|
2641
|
+
if (meta.syncActive !== void 0) wire.sync_active = meta.syncActive;
|
|
2642
|
+
if (meta.syncEpoch !== void 0) wire.sync_epoch = meta.syncEpoch;
|
|
2643
|
+
return wire;
|
|
2644
|
+
}
|
|
2645
|
+
function storedSyncSetting(raw) {
|
|
2646
|
+
if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
|
|
2647
|
+
const { enabled, history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
|
|
2648
|
+
if (typeof enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
|
|
2649
|
+
if (maxAge !== void 0 && !isPositiveSafeInteger(maxAge)) {
|
|
2650
|
+
invalid("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
|
|
2651
|
+
}
|
|
2652
|
+
if (maxBytes !== void 0 && !isPositiveSafeInteger(maxBytes)) {
|
|
2653
|
+
invalid("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
|
|
2654
|
+
}
|
|
2655
|
+
return {
|
|
2656
|
+
enabled,
|
|
2657
|
+
...maxAge !== void 0 ? { historyMaxAgeMs: maxAge } : {},
|
|
2658
|
+
...maxBytes !== void 0 ? { historyMaxBytes: maxBytes } : {}
|
|
2659
|
+
};
|
|
2660
|
+
}
|
|
2661
|
+
function syncMetadataFromWire(raw, meta) {
|
|
2662
|
+
if (raw.sync !== void 0 && raw.sync !== null) meta.sync = storedSyncSetting(raw.sync);
|
|
2663
|
+
if (raw.sync_active !== void 0 && raw.sync_active !== null) {
|
|
2664
|
+
if (typeof raw.sync_active !== "boolean") invalid("holds a sync_active that is not a boolean", {});
|
|
2665
|
+
meta.syncActive = raw.sync_active;
|
|
2666
|
+
}
|
|
2667
|
+
if (raw.sync_epoch !== void 0 && raw.sync_epoch !== null) {
|
|
2668
|
+
if (!isSyncEpoch(raw.sync_epoch)) invalid("holds an epoch that is not 32 lowercase hexadecimal digits", {});
|
|
2669
|
+
meta.syncEpoch = raw.sync_epoch;
|
|
2670
|
+
}
|
|
2671
|
+
}
|
|
2672
|
+
|
|
2673
|
+
// src/serialization/payload-v1/metadata.ts
|
|
2674
|
+
function metadataToWire(meta) {
|
|
2675
|
+
const wire = {
|
|
2676
|
+
index_name: meta.indexName,
|
|
2677
|
+
schema: meta.schema,
|
|
2678
|
+
language: meta.language,
|
|
2679
|
+
partition_count: meta.partitionCount,
|
|
2680
|
+
bm25_params: meta.bm25Params,
|
|
2681
|
+
created_at: meta.createdAt,
|
|
2682
|
+
engine_version: meta.engineVersion
|
|
2683
|
+
};
|
|
2684
|
+
if (meta.vectorFields) {
|
|
2685
|
+
wire.vector_fields = meta.vectorFields;
|
|
2686
|
+
}
|
|
2687
|
+
if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
|
|
2688
|
+
wire.document_count = meta.documentCount;
|
|
2689
|
+
}
|
|
2690
|
+
if (meta.embedding) {
|
|
2691
|
+
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
2692
|
+
}
|
|
2693
|
+
if (meta.surfaceForms !== void 0) {
|
|
2694
|
+
wire.surface_forms_enabled = meta.surfaceForms;
|
|
2695
|
+
}
|
|
2696
|
+
if (meta.analysisRevision !== void 0) {
|
|
2697
|
+
wire.analysis_revision = meta.analysisRevision;
|
|
2698
|
+
}
|
|
2699
|
+
if (meta.arrayPositionGap !== void 0) {
|
|
2700
|
+
wire.array_position_gap = meta.arrayPositionGap;
|
|
2701
|
+
}
|
|
2702
|
+
if (meta.tokenizer !== void 0) {
|
|
2703
|
+
wire.tokenizer = meta.tokenizer;
|
|
2704
|
+
}
|
|
2705
|
+
if (meta.stopWords !== void 0) {
|
|
2706
|
+
wire.stop_words = meta.stopWords;
|
|
2707
|
+
}
|
|
2708
|
+
if (meta.stopWordList !== void 0) {
|
|
2709
|
+
wire.stop_word_list = meta.stopWordList;
|
|
2710
|
+
}
|
|
2711
|
+
if (meta.analysisGeneration !== void 0) {
|
|
2712
|
+
wire.analysis_generation = meta.analysisGeneration;
|
|
2713
|
+
}
|
|
2714
|
+
if (meta.pendingAnalysis !== void 0) {
|
|
2715
|
+
const pending = meta.pendingAnalysis;
|
|
2716
|
+
wire.pending_analysis = {
|
|
2717
|
+
generation: pending.generation,
|
|
2718
|
+
language: pending.language,
|
|
2719
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
2720
|
+
...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
|
|
2721
|
+
...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
|
|
2722
|
+
};
|
|
2723
|
+
}
|
|
2724
|
+
if (meta.partitionLimits !== void 0) {
|
|
2725
|
+
wire.partition_limits = {
|
|
2726
|
+
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
2727
|
+
...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
|
|
2728
|
+
...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
|
|
2729
|
+
};
|
|
2730
|
+
}
|
|
2731
|
+
if (meta.defaultScoring !== void 0) {
|
|
2732
|
+
wire.default_scoring = meta.defaultScoring;
|
|
2733
|
+
}
|
|
2734
|
+
if (meta.trackPositions !== void 0) {
|
|
2735
|
+
wire.track_positions = meta.trackPositions;
|
|
2736
|
+
}
|
|
2737
|
+
if (meta.textStorage !== void 0) {
|
|
2738
|
+
wire.text_storage = meta.textStorage;
|
|
2739
|
+
}
|
|
2740
|
+
if (meta.strict !== void 0) {
|
|
2741
|
+
wire.strict = meta.strict;
|
|
2742
|
+
}
|
|
2743
|
+
if (meta.required !== void 0) {
|
|
2744
|
+
wire.required = meta.required;
|
|
2745
|
+
}
|
|
2746
|
+
if (meta.vectorPromotion !== void 0) {
|
|
2747
|
+
const promotion = meta.vectorPromotion;
|
|
2748
|
+
wire.vector_promotion = {
|
|
2749
|
+
...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
|
|
2750
|
+
...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
|
|
2751
|
+
...promotion.hnswConfig !== void 0 ? {
|
|
2752
|
+
hnsw_config: {
|
|
2753
|
+
...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
|
|
2754
|
+
...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
|
|
2755
|
+
...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
|
|
2756
|
+
}
|
|
2757
|
+
} : {},
|
|
2758
|
+
...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
|
|
2759
|
+
...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
|
|
2760
|
+
};
|
|
2761
|
+
}
|
|
2762
|
+
if (meta.patternValueLimit !== void 0) {
|
|
2763
|
+
wire.pattern_value_limit = meta.patternValueLimit;
|
|
2764
|
+
}
|
|
2765
|
+
if (meta.indexUuid !== void 0) {
|
|
2766
|
+
wire.index_uuid = meta.indexUuid;
|
|
2767
|
+
}
|
|
2768
|
+
if (meta.heldPartitions !== void 0) {
|
|
2769
|
+
wire.held_partitions = meta.heldPartitions;
|
|
2770
|
+
}
|
|
2771
|
+
if (Array.isArray(meta.synonyms) && meta.synonyms.length > 0) {
|
|
2772
|
+
wire.synonyms = meta.synonyms;
|
|
2773
|
+
}
|
|
2774
|
+
return {
|
|
2775
|
+
...wire,
|
|
2776
|
+
...syncMetadataToWire(meta),
|
|
2777
|
+
...meta.querySuggestions !== void 0 ? storedStateToWire(meta.querySuggestions) : {}
|
|
2778
|
+
};
|
|
2779
|
+
}
|
|
2780
|
+
function wireToMetadata(raw) {
|
|
2781
|
+
const meta = {
|
|
2782
|
+
indexName: raw.index_name,
|
|
2783
|
+
schema: raw.schema ?? {},
|
|
2784
|
+
language: raw.language ?? "english",
|
|
2785
|
+
partitionCount: raw.partition_count ?? 1,
|
|
2786
|
+
bm25Params: raw.bm25_params ?? { k1: DEFAULT_BM25_K1, b: DEFAULT_BM25_B },
|
|
2787
|
+
createdAt: raw.created_at ?? 0,
|
|
2788
|
+
engineVersion: raw.engine_version ?? "0.0.0"
|
|
2789
|
+
};
|
|
2790
|
+
if (raw.vector_fields) {
|
|
2791
|
+
for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
|
|
2792
|
+
if (!isQuantizationMode(field.quantization)) {
|
|
2793
|
+
throw new NarsilError(
|
|
2794
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2795
|
+
`Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
|
|
2796
|
+
{ fieldPath, quantization: field.quantization }
|
|
2797
|
+
);
|
|
2798
|
+
}
|
|
2799
|
+
}
|
|
2800
|
+
meta.vectorFields = raw.vector_fields;
|
|
2801
|
+
}
|
|
2802
|
+
if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
|
|
2803
|
+
meta.documentCount = raw.document_count;
|
|
2804
|
+
}
|
|
2805
|
+
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
2806
|
+
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
2807
|
+
}
|
|
2808
|
+
if (typeof raw.surface_forms_enabled === "boolean") {
|
|
2809
|
+
meta.surfaceForms = raw.surface_forms_enabled;
|
|
2810
|
+
}
|
|
2811
|
+
if (typeof raw.analysis_revision === "string") {
|
|
2812
|
+
meta.analysisRevision = raw.analysis_revision;
|
|
2813
|
+
}
|
|
2814
|
+
if (typeof raw.array_position_gap === "number") {
|
|
2815
|
+
meta.arrayPositionGap = raw.array_position_gap;
|
|
2816
|
+
}
|
|
2817
|
+
if (typeof raw.tokenizer === "string") {
|
|
2818
|
+
meta.tokenizer = raw.tokenizer;
|
|
2819
|
+
}
|
|
2820
|
+
if (typeof raw.stop_words === "string") {
|
|
2821
|
+
meta.stopWords = raw.stop_words;
|
|
2822
|
+
}
|
|
2823
|
+
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
2824
|
+
meta.stopWordList = raw.stop_word_list;
|
|
2825
|
+
}
|
|
2826
|
+
if (isPositiveInteger(raw.analysis_generation)) {
|
|
2827
|
+
meta.analysisGeneration = raw.analysis_generation;
|
|
2828
|
+
}
|
|
2829
|
+
const pending = raw.pending_analysis;
|
|
2830
|
+
if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
|
|
2831
|
+
meta.pendingAnalysis = {
|
|
2832
|
+
generation: pending.generation,
|
|
2833
|
+
language: pending.language,
|
|
2834
|
+
...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
|
|
2835
|
+
...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
|
|
2836
|
+
...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
|
|
2837
|
+
};
|
|
2838
|
+
}
|
|
2839
|
+
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
2840
|
+
const limits = {
|
|
2841
|
+
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
2842
|
+
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
|
|
2843
|
+
...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
|
|
2844
|
+
};
|
|
2845
|
+
if (Object.keys(limits).length > 0) {
|
|
2846
|
+
meta.partitionLimits = limits;
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
|
|
2850
|
+
meta.defaultScoring = raw.default_scoring;
|
|
2851
|
+
}
|
|
2852
|
+
if (typeof raw.track_positions === "boolean") {
|
|
2853
|
+
meta.trackPositions = raw.track_positions;
|
|
2854
|
+
}
|
|
2855
|
+
if (raw.text_storage !== void 0) {
|
|
2856
|
+
if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
|
|
2857
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
|
|
2858
|
+
textStorage: raw.text_storage
|
|
2859
|
+
});
|
|
2860
|
+
}
|
|
2861
|
+
meta.textStorage = raw.text_storage;
|
|
2862
|
+
}
|
|
2863
|
+
if (typeof raw.strict === "boolean") {
|
|
2864
|
+
meta.strict = raw.strict;
|
|
2865
|
+
}
|
|
2866
|
+
if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
|
|
2867
|
+
meta.required = raw.required;
|
|
2868
|
+
}
|
|
2869
|
+
if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
|
|
2870
|
+
const promotion = raw.vector_promotion;
|
|
2871
|
+
const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
|
|
2872
|
+
const metric = hnsw?.metric;
|
|
2873
|
+
const restored = {};
|
|
2874
|
+
const hnswConfig = {};
|
|
2875
|
+
if (isPositiveInteger(hnsw?.m)) {
|
|
2876
|
+
hnswConfig.m = hnsw?.m;
|
|
2877
|
+
}
|
|
2878
|
+
if (isPositiveInteger(hnsw?.ef_construction)) {
|
|
2879
|
+
hnswConfig.efConstruction = hnsw?.ef_construction;
|
|
2880
|
+
}
|
|
2881
|
+
if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
|
|
2882
|
+
hnswConfig.metric = metric;
|
|
2883
|
+
}
|
|
2884
|
+
if (isPositiveInteger(promotion.threshold)) {
|
|
2885
|
+
restored.threshold = promotion.threshold;
|
|
2886
|
+
}
|
|
2887
|
+
if (isSelectivityRatio(promotion.filter_threshold)) {
|
|
2888
|
+
restored.filterThreshold = promotion.filter_threshold;
|
|
2889
|
+
}
|
|
2890
|
+
if (Object.keys(hnswConfig).length > 0) {
|
|
2891
|
+
restored.hnswConfig = hnswConfig;
|
|
2892
|
+
}
|
|
2893
|
+
if (promotion.quantization !== void 0) {
|
|
2894
|
+
if (!isQuantizationMode(promotion.quantization)) {
|
|
2895
|
+
throw new NarsilError(
|
|
2896
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2897
|
+
`Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
|
|
2898
|
+
{ quantization: promotion.quantization }
|
|
2899
|
+
);
|
|
2900
|
+
}
|
|
2901
|
+
restored.quantization = promotion.quantization;
|
|
2902
|
+
}
|
|
2903
|
+
if (promotion.storage !== void 0) {
|
|
2904
|
+
if (!isStorageMode(promotion.storage)) {
|
|
2905
|
+
throw new NarsilError(
|
|
2906
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2907
|
+
`Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
|
|
2908
|
+
{ storage: promotion.storage }
|
|
2909
|
+
);
|
|
2910
|
+
}
|
|
2911
|
+
restored.storage = promotion.storage;
|
|
2912
|
+
}
|
|
2913
|
+
if (Object.keys(restored).length > 0) {
|
|
2914
|
+
meta.vectorPromotion = restored;
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
if (raw.pattern_value_limit !== void 0) {
|
|
2918
|
+
if (!isPositiveInteger(raw.pattern_value_limit)) {
|
|
2919
|
+
throw new NarsilError(
|
|
2920
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2921
|
+
`The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
|
|
2922
|
+
{ patternValueLimit: String(raw.pattern_value_limit) }
|
|
2923
|
+
);
|
|
2924
|
+
}
|
|
2925
|
+
meta.patternValueLimit = raw.pattern_value_limit;
|
|
2926
|
+
}
|
|
2927
|
+
if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
|
|
2928
|
+
meta.indexUuid = raw.index_uuid;
|
|
2929
|
+
}
|
|
2930
|
+
if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
|
|
2931
|
+
meta.heldPartitions = raw.held_partitions;
|
|
2932
|
+
}
|
|
2933
|
+
if (raw.synonyms !== void 0 && raw.synonyms !== null) meta.synonyms = raw.synonyms;
|
|
2934
|
+
syncMetadataFromWire(raw, meta);
|
|
2935
|
+
const querySuggestions = storedStateFromWire(raw);
|
|
2936
|
+
if (querySuggestions !== null) meta.querySuggestions = querySuggestions;
|
|
2937
|
+
return meta;
|
|
2938
|
+
}
|
|
2939
|
+
function isNonNegativeInteger(value) {
|
|
2940
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
2941
|
+
}
|
|
2942
|
+
function isStringList(value) {
|
|
2943
|
+
return Array.isArray(value) && value.every((word) => typeof word === "string");
|
|
2944
|
+
}
|
|
2945
|
+
function isPositiveInteger(value) {
|
|
2946
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
2947
|
+
}
|
|
2948
|
+
function isSelectivityRatio(value) {
|
|
2949
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
2950
|
+
}
|
|
2951
|
+
function isWatermarkFraction(value) {
|
|
2952
|
+
return typeof value === "number" && value > 0 && value <= 1;
|
|
2953
|
+
}
|
|
2954
|
+
function serializeMetadata(meta) {
|
|
2955
|
+
const wire = metadataToWire(meta);
|
|
2956
|
+
return encode(wire);
|
|
2957
|
+
}
|
|
2958
|
+
function deserializeMetadata(data) {
|
|
2959
|
+
const raw = decode(data);
|
|
2960
|
+
return wireToMetadata(raw);
|
|
2961
|
+
}
|
|
2962
|
+
|
|
2963
|
+
// src/serialization/envelope.ts
|
|
2964
|
+
var CURRENT_ENVELOPE_VERSION = 1;
|
|
2965
|
+
var SNAPSHOT_ENVELOPE_VERSION = 2;
|
|
2966
|
+
var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
|
|
2967
|
+
function buildFlags(options) {
|
|
2968
|
+
const useCompression = options.compression === "gzip";
|
|
2969
|
+
return {
|
|
2970
|
+
compressionEnabled: useCompression,
|
|
2971
|
+
compressionAlgorithm: useCompression ? "gzip" : "none",
|
|
2972
|
+
checksumPresent: options.checksum ?? false,
|
|
2973
|
+
encryptionEnabled: false
|
|
2974
|
+
};
|
|
2975
|
+
}
|
|
2976
|
+
async function packEnvelope(payloadBytes, options) {
|
|
2977
|
+
const flags = buildFlags(options);
|
|
2978
|
+
let finalPayload = payloadBytes;
|
|
2979
|
+
if (flags.compressionEnabled) {
|
|
2980
|
+
const { gzip } = await loadGzip();
|
|
2981
|
+
finalPayload = await gzip(payloadBytes);
|
|
2982
|
+
}
|
|
2983
|
+
const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
|
|
2984
|
+
const [major, minor, patch] = engineVersionParts();
|
|
2985
|
+
const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
|
|
2986
|
+
const header = {
|
|
2987
|
+
magic: "NRSL",
|
|
2988
|
+
envelopeFormatVersion,
|
|
2989
|
+
engineVersionMajor: major,
|
|
2990
|
+
engineVersionMinor: minor,
|
|
2991
|
+
engineVersionPatch: patch,
|
|
2992
|
+
payloadLength: finalPayload.length,
|
|
2993
|
+
flags,
|
|
2994
|
+
checksum,
|
|
2995
|
+
reserved: new Uint8Array(14)
|
|
2996
|
+
};
|
|
2997
|
+
const headerBytes = writeHeader(header);
|
|
2998
|
+
const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
|
|
2999
|
+
envelope.set(headerBytes, 0);
|
|
3000
|
+
envelope.set(finalPayload, HEADER_SIZE);
|
|
3001
|
+
return envelope;
|
|
3002
|
+
}
|
|
3003
|
+
async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
|
|
3004
|
+
const header = readHeader(data);
|
|
3005
|
+
if (header.envelopeFormatVersion > maxAcceptedVersion) {
|
|
3006
|
+
throw new NarsilError(
|
|
3007
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
3008
|
+
`This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
|
|
3009
|
+
);
|
|
3010
|
+
}
|
|
3011
|
+
if (header.flags.encryptionEnabled) {
|
|
3012
|
+
throw new NarsilError(
|
|
3013
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
3014
|
+
"Encrypted envelopes are not supported in this version of Narsil"
|
|
3015
|
+
);
|
|
3016
|
+
}
|
|
3017
|
+
const compAlgo = header.flags.compressionAlgorithm;
|
|
3018
|
+
if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
|
|
3019
|
+
throw new NarsilError(
|
|
3020
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
3021
|
+
`Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
|
|
3022
|
+
);
|
|
3023
|
+
}
|
|
3024
|
+
const availablePayload = data.length - HEADER_SIZE;
|
|
3025
|
+
if (availablePayload < header.payloadLength) {
|
|
3026
|
+
throw new NarsilError(
|
|
3027
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3028
|
+
`Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
|
|
3032
|
+
if (header.flags.checksumPresent) {
|
|
3033
|
+
const computed = crc32(payloadBytes);
|
|
3034
|
+
if (computed !== header.checksum) {
|
|
3035
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
|
|
3036
|
+
expected: header.checksum,
|
|
3037
|
+
computed
|
|
3038
|
+
});
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
if (header.flags.compressionEnabled) {
|
|
3042
|
+
const { gunzip } = await loadGzip();
|
|
3043
|
+
payloadBytes = await gunzip(payloadBytes);
|
|
3044
|
+
}
|
|
3045
|
+
return { header, payloadBytes };
|
|
3046
|
+
}
|
|
3047
|
+
async function unpackEnvelopeBytes(data) {
|
|
3048
|
+
return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
|
|
3049
|
+
}
|
|
3050
|
+
async function unpackIndexSnapshotEnvelope(data) {
|
|
3051
|
+
const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
|
|
3052
|
+
if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
|
|
3053
|
+
throw new NarsilError(
|
|
3054
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
3055
|
+
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
|
|
3056
|
+
);
|
|
3057
|
+
}
|
|
3058
|
+
return payloadBytes;
|
|
3059
|
+
}
|
|
3060
|
+
async function writeMetadataEnvelope(meta, options = {}) {
|
|
3061
|
+
const payloadBytes = serializeMetadata(meta);
|
|
3062
|
+
return packEnvelope(payloadBytes, options);
|
|
3063
|
+
}
|
|
3064
|
+
async function readMetadataEnvelope(data) {
|
|
3065
|
+
const { header, payloadBytes } = await unpackEnvelope(data);
|
|
3066
|
+
const metadata = deserializeMetadata(payloadBytes);
|
|
3067
|
+
return { header, metadata };
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
// src/distribution/replication/replica.ts
|
|
3071
|
+
function validateReplicationEntry(entry, localPrimaryTerm, log) {
|
|
3072
|
+
if (!log.verifyChecksum(entry)) {
|
|
3073
|
+
return { valid: false, error: ErrorCodes.REPLICATION_ENTRY_CORRUPT };
|
|
3074
|
+
}
|
|
3075
|
+
if (entry.primaryTerm < localPrimaryTerm) {
|
|
3076
|
+
return { valid: false, error: ErrorCodes.REPLICATION_TERM_MISMATCH };
|
|
3077
|
+
}
|
|
3078
|
+
return { valid: true };
|
|
3079
|
+
}
|
|
3080
|
+
function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes, schedulesGraphBuild = true) {
|
|
3081
|
+
if (entry.document === null) {
|
|
3082
|
+
return;
|
|
3083
|
+
}
|
|
3084
|
+
const document = decode(entry.document);
|
|
3085
|
+
restoreVectorFields(document, vectorFieldPaths);
|
|
3086
|
+
const documentId = entry.documentId;
|
|
3087
|
+
if (manager.has(documentId)) {
|
|
3088
|
+
removeDocumentVectors(documentId, vecIndexes);
|
|
3089
|
+
manager.remove(documentId);
|
|
3090
|
+
}
|
|
3091
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths);
|
|
3092
|
+
manager.insert(documentId, partitionDoc);
|
|
3093
|
+
try {
|
|
3094
|
+
insertDocumentVectors(documentId, extractedVectors, vecIndexes, manager.partitionIdOf(documentId));
|
|
3095
|
+
} catch (err) {
|
|
3096
|
+
manager.remove(documentId);
|
|
3097
|
+
throw err;
|
|
3098
|
+
}
|
|
3099
|
+
if (!schedulesGraphBuild) return;
|
|
3100
|
+
for (const fieldPath of extractedVectors.keys()) {
|
|
3101
|
+
const vecIndex = vecIndexes.get(fieldPath);
|
|
3102
|
+
if (vecIndex) {
|
|
3103
|
+
vecIndex.scheduleBuild();
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
|
|
3108
|
+
removeDocumentVectors(entry.documentId, vecIndexes);
|
|
3109
|
+
if (!manager.has(entry.documentId)) {
|
|
3110
|
+
return;
|
|
3111
|
+
}
|
|
3112
|
+
manager.remove(entry.documentId, removedSpellings);
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
// src/persistence/durability/segment/load-partition.ts
|
|
3116
|
+
function requireAgreedSchema(agreed, schema, key) {
|
|
3117
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
3118
|
+
const existing = agreed[field];
|
|
3119
|
+
if (existing !== void 0 && existing !== type) {
|
|
3120
|
+
throw new NarsilError(
|
|
3121
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3122
|
+
`The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
|
|
3123
|
+
{ key, field, existing, found: type }
|
|
3124
|
+
);
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
}
|
|
3128
|
+
function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
3129
|
+
manager.beginBatchRemove();
|
|
3130
|
+
try {
|
|
3131
|
+
for (const docId of contents.tombstones) {
|
|
3132
|
+
if (manager.has(docId)) manager.remove(docId);
|
|
3133
|
+
}
|
|
3134
|
+
const ids = segment.arrays.idTable;
|
|
3135
|
+
for (let ordinal = 0; ordinal < ids.count; ordinal++) {
|
|
3136
|
+
const docId = ids.idAt(ordinal);
|
|
3137
|
+
if (manager.has(docId)) manager.remove(docId);
|
|
3138
|
+
}
|
|
3139
|
+
} finally {
|
|
3140
|
+
manager.endBatchRemove();
|
|
3141
|
+
}
|
|
3142
|
+
}
|
|
3143
|
+
function yieldToEventLoop2() {
|
|
3144
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
3145
|
+
}
|
|
3146
|
+
function removeFromPartition(manager, partitionId, docIds) {
|
|
3147
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
3148
|
+
composite.beginBatch();
|
|
3149
|
+
try {
|
|
3150
|
+
for (const docId of docIds) {
|
|
3151
|
+
if (!composite.has(docId)) continue;
|
|
3152
|
+
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
3153
|
+
docPartitionMap.delete(docId);
|
|
3154
|
+
}
|
|
3155
|
+
} finally {
|
|
3156
|
+
composite.endBatch();
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
3160
|
+
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
3161
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
3162
|
+
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
3163
|
+
const segment = createPagedFrozenSegment(snapshot);
|
|
3164
|
+
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
3165
|
+
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
3166
|
+
if (snapshot.documentCount > 0) {
|
|
3167
|
+
manager.attachFrozenSegment(partitionId, segment);
|
|
3168
|
+
return segment;
|
|
3169
|
+
}
|
|
3170
|
+
segment.paged?.close();
|
|
3171
|
+
return null;
|
|
3172
|
+
}
|
|
3173
|
+
function documentsHeldBefore(manager, partitionId) {
|
|
3174
|
+
const held = manager.partitionAt(partitionId);
|
|
3175
|
+
return held === void 0 ? [] : [...held.docIds()];
|
|
3176
|
+
}
|
|
3177
|
+
function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
|
|
3178
|
+
const absent = heldBefore.filter(
|
|
3179
|
+
(docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
|
|
3180
|
+
);
|
|
3181
|
+
if (absent.length === 0) return;
|
|
3182
|
+
manager.beginBatchRemove();
|
|
3183
|
+
try {
|
|
3184
|
+
for (const docId of absent) manager.remove(docId);
|
|
3185
|
+
} finally {
|
|
3186
|
+
manager.endBatchRemove();
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3189
|
+
async function loadPartitionSegments(store, partition, manager, options) {
|
|
3190
|
+
const heldBefore = documentsHeldBefore(manager, partition.partitionId);
|
|
3191
|
+
const loaded = await attachPartitionSegments(store, partition, manager, options);
|
|
3192
|
+
removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
|
|
3193
|
+
}
|
|
3194
|
+
async function attachPartitionSegments(store, partition, manager, options) {
|
|
3195
|
+
const loaded = [];
|
|
3196
|
+
const { directory } = store;
|
|
3197
|
+
for (const ref of partition.segments) {
|
|
3198
|
+
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
3199
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
3200
|
+
const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
3201
|
+
if (paged !== null) loaded.push(paged);
|
|
3202
|
+
await yieldToEventLoop2();
|
|
3203
|
+
continue;
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
const contents = await readSegmentFile(store, ref);
|
|
3207
|
+
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
3208
|
+
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
3209
|
+
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
|
|
3210
|
+
removeWhatTheSegmentReplaces(manager, segment, contents);
|
|
3211
|
+
options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
|
|
3212
|
+
if (contents.arrays.documentCount > 0) {
|
|
3213
|
+
manager.attachFrozenSegment(partition.partitionId, segment);
|
|
3214
|
+
loaded.push(segment);
|
|
3215
|
+
}
|
|
3216
|
+
await yieldToEventLoop2();
|
|
3217
|
+
}
|
|
3218
|
+
return loaded;
|
|
3219
|
+
}
|
|
3220
|
+
|
|
3221
|
+
// src/persistence/durability/segment/load.ts
|
|
3222
|
+
async function readSegmentManifest(store, indexName) {
|
|
3223
|
+
const bytes = await store.read(manifestKey(indexName));
|
|
3224
|
+
if (bytes === null) return null;
|
|
3225
|
+
return decodeSegmentManifest(bytes, indexName);
|
|
3226
|
+
}
|
|
3227
|
+
async function readCurrentManifest(store, indexName) {
|
|
3228
|
+
const decoded = await readSegmentManifest(store, indexName);
|
|
3229
|
+
if (decoded === null) return null;
|
|
3230
|
+
if (decoded.kind === "current") return decoded.manifest;
|
|
3231
|
+
throw new NarsilError(
|
|
3232
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3233
|
+
`Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
|
|
3234
|
+
{ indexName }
|
|
3235
|
+
);
|
|
3236
|
+
}
|
|
3237
|
+
function growPartitionsTo(manager, highestPartitionId) {
|
|
3238
|
+
while (manager.partitionCount <= highestPartitionId) {
|
|
3239
|
+
const before = manager.partitionCount;
|
|
3240
|
+
try {
|
|
3241
|
+
manager.addPartition();
|
|
3242
|
+
} catch {
|
|
3243
|
+
break;
|
|
3244
|
+
}
|
|
3245
|
+
if (manager.partitionCount === before) break;
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
3248
|
+
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
3249
|
+
let highestPartitionId = manifest.partitions.length - 1;
|
|
3250
|
+
for (const partition of manifest.partitions) {
|
|
3251
|
+
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
3252
|
+
}
|
|
3253
|
+
growPartitionsTo(deps.manager, highestPartitionId);
|
|
3254
|
+
const { directory } = store;
|
|
3255
|
+
const opensVectorsOnDemand = manifest.vectors.some(
|
|
3256
|
+
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
3257
|
+
);
|
|
3258
|
+
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
3259
|
+
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
3260
|
+
}
|
|
3261
|
+
for (const partition of manifest.partitions) {
|
|
3262
|
+
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
3263
|
+
throw new NarsilError(
|
|
3264
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3265
|
+
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
3266
|
+
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
3267
|
+
);
|
|
3268
|
+
}
|
|
3269
|
+
await loadPartitionSegments(store, partition, deps.manager, {
|
|
3270
|
+
schema: manifest.schema,
|
|
3271
|
+
decodesEveryDocument,
|
|
3272
|
+
writeRecords: deps.writeRecords ?? null
|
|
3273
|
+
});
|
|
3274
|
+
}
|
|
3275
|
+
for (const vector of manifest.vectors) {
|
|
3276
|
+
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
3277
|
+
if (vectorIndex === void 0) continue;
|
|
3278
|
+
await loadVectorField(
|
|
3279
|
+
store,
|
|
3280
|
+
vector,
|
|
3281
|
+
vectorIndex,
|
|
3282
|
+
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
3283
|
+
);
|
|
3284
|
+
}
|
|
3285
|
+
return manifest.checkpoint;
|
|
3286
|
+
}
|
|
3287
|
+
async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
3288
|
+
for (const key of await store.list(segmentsPrefix(indexName))) {
|
|
3289
|
+
if (!referenced.has(key)) await store.remove(key);
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
|
|
3293
|
+
// src/persistence/archive/file-entry.ts
|
|
3294
|
+
async function writeArchiveFile(store, key, chunks) {
|
|
3295
|
+
const { directory } = store;
|
|
3296
|
+
if (directory === void 0) {
|
|
3297
|
+
const held = [];
|
|
3298
|
+
for await (const chunk of chunks) held.push(chunk);
|
|
3299
|
+
await store.write(key, held);
|
|
3300
|
+
return;
|
|
3301
|
+
}
|
|
3302
|
+
const receiving = `${key}.receiving`;
|
|
3303
|
+
await directory.remove(receiving);
|
|
3304
|
+
const handle = await directory.appendHandle(receiving);
|
|
3305
|
+
try {
|
|
3306
|
+
for await (const chunk of chunks) await handle.append(chunk);
|
|
3307
|
+
await handle.sync();
|
|
3308
|
+
} finally {
|
|
3309
|
+
await handle.close();
|
|
3310
|
+
}
|
|
3311
|
+
await directory.move(receiving, key);
|
|
3312
|
+
}
|
|
3313
|
+
|
|
3314
|
+
// src/persistence/durability/marker-writer.ts
|
|
3315
|
+
function markerKey(indexName, partitionId) {
|
|
3316
|
+
return `${indexName}/wal/${partitionId}/commit`;
|
|
3317
|
+
}
|
|
3318
|
+
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
3319
|
+
const key = markerKey(indexName, partitionId);
|
|
3320
|
+
const handle = await directory.markerHandle(key);
|
|
3321
|
+
const existing = await handle.read();
|
|
3322
|
+
const current = readCommitMarker(existing);
|
|
3323
|
+
const createdFresh = existing.length === 0;
|
|
3324
|
+
let lastSlot = current?.slotIndex ?? null;
|
|
3325
|
+
let writeSeq = current?.state.writeSeq ?? 0;
|
|
3326
|
+
return {
|
|
3327
|
+
created: createdFresh,
|
|
3328
|
+
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
3329
|
+
async commit(state) {
|
|
3330
|
+
const slotIndex = nextMarkerSlot(lastSlot);
|
|
3331
|
+
writeSeq += 1;
|
|
3332
|
+
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
3333
|
+
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
3334
|
+
lastSlot = slotIndex;
|
|
3335
|
+
},
|
|
3336
|
+
async close() {
|
|
3337
|
+
await handle.close();
|
|
3338
|
+
}
|
|
3339
|
+
};
|
|
3340
|
+
}
|
|
3341
|
+
|
|
3342
|
+
// src/persistence/durability/recovery.ts
|
|
3343
|
+
async function removeLeftoverRewriteFiles(directory, names) {
|
|
3344
|
+
for (const indexName of names) {
|
|
3345
|
+
for (const place of ["rebuilt", "received"]) {
|
|
3346
|
+
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
3347
|
+
}
|
|
3348
|
+
}
|
|
3349
|
+
}
|
|
3350
|
+
async function listPersistedIndexes(directory) {
|
|
3351
|
+
const metaKeys = await directory.list("");
|
|
3352
|
+
const names = [];
|
|
3353
|
+
for (const key of metaKeys) {
|
|
3354
|
+
if (key.endsWith("/meta")) {
|
|
3355
|
+
names.push(key.slice(0, -"/meta".length));
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
return names;
|
|
3359
|
+
}
|
|
3360
|
+
async function loadMetadata(directory, indexName) {
|
|
3361
|
+
const bytes = await directory.read(`${indexName}/meta`);
|
|
3362
|
+
if (bytes === null) {
|
|
3363
|
+
return null;
|
|
3364
|
+
}
|
|
3365
|
+
const { metadata } = await readMetadataEnvelope(bytes);
|
|
3366
|
+
return metadata;
|
|
3367
|
+
}
|
|
3368
|
+
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
3369
|
+
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
3370
|
+
const read = await readWalSegments(directory, indexName, partitionId);
|
|
3371
|
+
if (read === null) {
|
|
3372
|
+
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
3373
|
+
}
|
|
3374
|
+
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
3375
|
+
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
3376
|
+
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
3377
|
+
}
|
|
3378
|
+
let highestSeqNo = fromSeqNoExclusive;
|
|
3379
|
+
let highestPrimaryTerm = 0;
|
|
3380
|
+
const replay = async (entries) => {
|
|
3381
|
+
for (const entry of entries) {
|
|
3382
|
+
if (entry.seqNo <= highestSeqNo) {
|
|
3383
|
+
continue;
|
|
3384
|
+
}
|
|
3385
|
+
if (turnIsSpent()) await yieldTurn();
|
|
3386
|
+
applyEntry2(entry, deps);
|
|
3387
|
+
highestSeqNo = entry.seqNo;
|
|
3388
|
+
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
3389
|
+
highestPrimaryTerm = entry.primaryTerm;
|
|
3390
|
+
}
|
|
3391
|
+
}
|
|
3392
|
+
};
|
|
3393
|
+
await replay(durableEntries);
|
|
3394
|
+
if (activeTail !== null) {
|
|
3395
|
+
await replay(activeTail.entries);
|
|
3396
|
+
}
|
|
3397
|
+
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
3398
|
+
if (activeTail !== null) {
|
|
3399
|
+
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
3400
|
+
}
|
|
3401
|
+
return { highestSeqNo, highestPrimaryTerm };
|
|
3402
|
+
}
|
|
3403
|
+
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
3404
|
+
const lastReplayed = tail.entries.at(-1);
|
|
3405
|
+
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
3406
|
+
const handle = await directory.appendHandle(tail.key);
|
|
3407
|
+
try {
|
|
3408
|
+
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
3409
|
+
else await handle.sync();
|
|
3410
|
+
} finally {
|
|
3411
|
+
await handle.close();
|
|
3412
|
+
}
|
|
3413
|
+
if (lastReplayed === void 0) return;
|
|
3414
|
+
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
3415
|
+
try {
|
|
3416
|
+
await writer.commit({
|
|
3417
|
+
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
3418
|
+
durableByteLength: tail.cleanEnd,
|
|
3419
|
+
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
3420
|
+
});
|
|
3421
|
+
} finally {
|
|
3422
|
+
await writer.close();
|
|
3423
|
+
}
|
|
3424
|
+
}
|
|
3425
|
+
async function replayWalUpTo(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps) {
|
|
3426
|
+
const read = await readWalSegments(directory, indexName, partitionId);
|
|
3427
|
+
if (read === null) {
|
|
3428
|
+
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
3429
|
+
}
|
|
3430
|
+
return replayDurableEntries(read, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps);
|
|
3431
|
+
}
|
|
3432
|
+
async function replaySyncedWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
3433
|
+
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
3434
|
+
const read = await readWalSegments(directory, indexName, partitionId);
|
|
3435
|
+
if (read === null) {
|
|
3436
|
+
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
3437
|
+
}
|
|
3438
|
+
const replayed = replayDurableEntries(
|
|
3439
|
+
read,
|
|
3440
|
+
indexName,
|
|
3441
|
+
partitionId,
|
|
3442
|
+
fromSeqNoExclusive,
|
|
3443
|
+
Number.POSITIVE_INFINITY,
|
|
3444
|
+
deps
|
|
3445
|
+
);
|
|
3446
|
+
await deleteOrphanSegments(directory, read.segments, read.marker.state.activeSegmentSeqNo);
|
|
3447
|
+
if (read.activeTail !== null) {
|
|
3448
|
+
await truncateSegmentTail(directory, read.activeTail.key, read.marker.state.durableByteLength);
|
|
3449
|
+
}
|
|
3450
|
+
return replayed;
|
|
3451
|
+
}
|
|
3452
|
+
function replayDurableEntries(read, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps) {
|
|
3453
|
+
if (Math.max(fromSeqNoExclusive, read.highestReadFromWal) < read.marker.state.highestDurableSeqNo) {
|
|
3454
|
+
throw durableRecordMissing(indexName, partitionId, read.highestReadFromWal, read.marker);
|
|
3455
|
+
}
|
|
3456
|
+
let highestSeqNo = fromSeqNoExclusive;
|
|
3457
|
+
let highestPrimaryTerm = 0;
|
|
3458
|
+
for (const entry of read.durableEntries) {
|
|
3459
|
+
if (entry.seqNo <= fromSeqNoExclusive || entry.seqNo > upToSeqNoInclusive) {
|
|
3460
|
+
continue;
|
|
3461
|
+
}
|
|
3462
|
+
applyEntry2(entry, deps);
|
|
3463
|
+
if (entry.seqNo > highestSeqNo) {
|
|
3464
|
+
highestSeqNo = entry.seqNo;
|
|
3465
|
+
}
|
|
3466
|
+
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
3467
|
+
highestPrimaryTerm = entry.primaryTerm;
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
return { highestSeqNo, highestPrimaryTerm };
|
|
3471
|
+
}
|
|
3472
|
+
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
3473
|
+
for (const { key, startSeqNo } of segments) {
|
|
3474
|
+
if (startSeqNo > activeSegmentSeqNo) {
|
|
3475
|
+
await directory.remove(key);
|
|
3476
|
+
}
|
|
3477
|
+
}
|
|
3478
|
+
}
|
|
3479
|
+
async function truncateSegmentTail(directory, key, durableByteLength) {
|
|
3480
|
+
const cleanLength = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
|
|
3481
|
+
const handle = await directory.appendHandle(key);
|
|
3482
|
+
try {
|
|
3483
|
+
const currentSize = await handle.size();
|
|
3484
|
+
if (cleanLength < currentSize) {
|
|
3485
|
+
await handle.truncate(cleanLength);
|
|
3486
|
+
}
|
|
3487
|
+
} finally {
|
|
3488
|
+
await handle.close();
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
function applyEntry2(entry, deps) {
|
|
3492
|
+
deps.writeRecords?.replayed(entry);
|
|
3493
|
+
if (entry.operation === "DELETE") {
|
|
3494
|
+
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
3495
|
+
return;
|
|
3496
|
+
}
|
|
3497
|
+
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
3498
|
+
}
|
|
3499
|
+
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
3500
|
+
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
3501
|
+
}
|
|
3502
|
+
|
|
3503
|
+
// src/persistence/durability/manager-state.ts
|
|
3504
|
+
function closePartitionWriter(partition) {
|
|
3505
|
+
return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
|
|
3506
|
+
}
|
|
3507
|
+
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
3508
|
+
indexState.unloading = true;
|
|
3509
|
+
try {
|
|
3510
|
+
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
3511
|
+
await pendingMetadataWrite;
|
|
3512
|
+
for (const partition of indexState.partitions.values()) {
|
|
3513
|
+
await closeWalWriter(closePartitionWriter(partition));
|
|
3514
|
+
}
|
|
3515
|
+
} catch (error) {
|
|
3516
|
+
indexState.unloading = false;
|
|
3517
|
+
throw error;
|
|
3518
|
+
}
|
|
3519
|
+
}
|
|
3520
|
+
|
|
3521
|
+
// src/workers/detached-index.ts
|
|
3522
|
+
function attachablePolicy(shared) {
|
|
3523
|
+
let attached = false;
|
|
3524
|
+
const policy = {
|
|
3525
|
+
get enabled() {
|
|
3526
|
+
return attached && shared.enabled;
|
|
3527
|
+
},
|
|
3528
|
+
get count() {
|
|
3529
|
+
return shared.count;
|
|
3530
|
+
},
|
|
3531
|
+
get host() {
|
|
3532
|
+
return shared.host;
|
|
3533
|
+
}
|
|
3534
|
+
};
|
|
3535
|
+
return {
|
|
3536
|
+
policy,
|
|
3537
|
+
attach() {
|
|
3538
|
+
attached = true;
|
|
3539
|
+
}
|
|
3540
|
+
};
|
|
3541
|
+
}
|
|
3542
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor) {
|
|
3543
|
+
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
3544
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy);
|
|
3545
|
+
entry.manager.synonyms = synonyms;
|
|
3546
|
+
return { indexName, manager: entry.manager, entry, attach };
|
|
3547
|
+
}
|
|
3548
|
+
|
|
3549
|
+
export { SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, captureAppliedPrefix, changeIndexAnalysis, checkPagedSegmentFully, checkpointInTurn, chunksOf, closePartitionWriter, configOfNamedSettings, countsAsLatestHolder, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, growPartitionsTo, heldAnalysisIsBuilt, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, openPagedSegmentScan, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, validateReplicationEntry, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };
|