@delali/narsil 0.3.1-next.157 → 0.3.1-next.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
- package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +66 -18
- package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1199 -1092
- package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -16
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -227
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
- package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +95 -52
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +33 -13
- package/dist/server.mjs +81 -100
- package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +88 -72
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +89 -59
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +66 -50
- package/package.json +233 -8
- package/dist/analysis-change-CKEMM2MC.mjs +0 -55
- package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
- package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-25MTVDYD.mjs +0 -432
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-AI5UI46D.mjs +0 -12
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
- package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
- package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
- package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MY2XKH22.mjs +0 -40
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
- package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
- package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
- package/dist/browser/chunk-V43N6TZO.mjs +0 -857
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
- package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-5ESVMFDJ.mjs +0 -20
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NDSQLPMQ.mjs +0 -25
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-HYXPVJVS.mjs +0 -32
- package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
- package/dist/browser/retention-TY3L3BFN.mjs +0 -211
- package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
- package/dist/browser/start-2UZTOBG5.mjs +0 -26
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-ULXAN5B4.mjs +0 -196
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-2ROUFMQX.mjs +0 -543
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-4B7JIGQD.mjs +0 -106
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BNL5OPRV.mjs +0 -634
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-C7XQYG6U.mjs +0 -50
- package/dist/chunk-CQZXZ75V.mjs +0 -562
- package/dist/chunk-CWS55TEX.mjs +0 -242
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FEZVDOWX.mjs +0 -6996
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GPWMUPCA.mjs +0 -1126
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JSV6OQ4C.mjs +0 -74
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KCQYFO4K.mjs +0 -62
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LDNCLRZU.mjs +0 -218
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MW6BR4QP.mjs +0 -316
- package/dist/chunk-MWD5VPN7.mjs +0 -21
- package/dist/chunk-ODYKO5TO.mjs +0 -101
- package/dist/chunk-OG7UGJ6F.mjs +0 -63
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-OYABFKTW.mjs +0 -1974
- package/dist/chunk-P5LANSH6.mjs +0 -121
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PIH5SJ4N.mjs +0 -7481
- package/dist/chunk-PSZI4O24.mjs +0 -253
- package/dist/chunk-PV2CG7EJ.mjs +0 -886
- package/dist/chunk-PWGYQXML.mjs +0 -470
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QLDABJZM.mjs +0 -20
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RCX4WNLC.mjs +0 -151
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RIHIVFIQ.mjs +0 -240
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RWURUDSP.mjs +0 -2639
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SOI6LTHM.mjs +0 -216
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UD5HIS4U.mjs +0 -382
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-UJDJB3NE.mjs +0 -913
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VSLSGASN.mjs +0 -30
- package/dist/chunk-VZ2ONHEE.mjs +0 -3715
- package/dist/chunk-W7OSSKH5.mjs +0 -1427
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-V5DRIECI.mjs +0 -34
- package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
- package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-GPKLTSKA.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-JYHFTVPS.mjs +0 -63
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
- package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
- package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
- package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
- package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
- package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
- package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
- package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
- package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
- package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
- package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
- package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
- package/dist/release-SMR4WJD4.mjs +0 -32
- package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
- package/dist/restore-queue-LZH7LMLR.mjs +0 -55
- package/dist/retention-3K7ARGAG.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-O7ICP7NJ.mjs +0 -783
- package/dist/start-S4PFP6RS.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-YLJ3ZPQ3.mjs +0 -50
- package/dist/write-2OBJGLFS.mjs +0 -233
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
|
@@ -0,0 +1,4723 @@
|
|
|
1
|
+
import { readWholeSectionChecked, loadFailed, sectionNamed, openPagedFile, closePagedFile, readCacheOfThisThread, PAGED_SEGMENT_ENVELOPE_VERSION, FILE_ASSEMBLY_CHUNK_BYTES, createReadCache, MERGE_READ_CACHE_BYTES, createSectionReader, checkEverySectionPage, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, readListedFile, requireListedChecksum, encodeSegmentManifest, readFileRange, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES, ID_SAMPLE_INTERVAL, relocatePagedFile, cachedFileBytes, requireNativePage, LIST_VIEWS_HELD_PER_SEGMENT, WRITER_YIELD_INTERVAL_TOKENS, DOCUMENT_BLOCK_MIN_RAW_BYTES, CACHED_POSITIONS, POSITION_WIDTH_BYTES, CACHED_LIST, POSTING_WIDTH_BYTES, CACHED_DOCUMENT_BLOCK, failNativePage, readFileRangeLater, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, POSTING_WRITE_ROWS, POSTING_WRITE_POSITIONS, SECTION_WRITE_BUFFER_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES } from './chunk-U6B7DUQW.mjs';
|
|
2
|
+
import { ReplicationMessageTypes } from './chunk-5FLRXF73.mjs';
|
|
3
|
+
import { WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
4
|
+
import { SEGMENT_MANIFEST_VERSION, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION, segmentKey } from './chunk-7MWYORQC.mjs';
|
|
5
|
+
import { DEFAULT_COMPACTION_THRESHOLD, SEGMENT_FILE_FRAMING_SLACK_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-QDZJM7NI.mjs';
|
|
6
|
+
import { encode } from './chunk-77AUVDQ5.mjs';
|
|
7
|
+
import { restoreVectorFields } from './chunk-QBVTYY4J.mjs';
|
|
8
|
+
import { readHeader, HEADER_SIZE, crc32Init, crc32Update, crc32Final, requireWithinFileLimit, engineVersionParts, writeHeader, crc32, openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-6EZW2GQL.mjs';
|
|
9
|
+
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, buildPartialWordTableFrom, mergeFieldIndexes, isCompositePartition, reconstructSchemaFromMetadata, decodeSurfaceTable, mergeDocFrequencies, TermLookup, createFrozenRemovals, selectionKeeps, fieldIndexRemapOf, mergeKeywordSources, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, createSharedFrozenSegment, shareSegmentArrays, getLanguage, emptyPendingRegions, mergePostings2, scanPostingBlocks, holdsADiscardedField, SegmentRefusal, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayloads, resolvePartitionInsertOptions, endOfValue, segmentArraysOf, mergeSegmentContents, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-TINW4BSH.mjs';
|
|
10
|
+
import { encodeDocument } from './chunk-KLVMK3R5.mjs';
|
|
11
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
12
|
+
import { extractVectorFieldsFromSchema, VECTOR_PATTERN, requireValidFieldTypes, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-R4CE4IVU.mjs';
|
|
13
|
+
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
15
|
+
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
16
|
+
import { deflateRawSync, inflateRawSync } from 'zlib';
|
|
17
|
+
|
|
18
|
+
// src/serialization/keyword-segment/little-endian.ts
|
|
19
|
+
var LITTLE_ENDIAN_PLATFORM = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
20
|
+
function swappedCopy(values) {
|
|
21
|
+
const width = values.BYTES_PER_ELEMENT;
|
|
22
|
+
const source = new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
23
|
+
const copy = new Uint8Array(values.byteLength);
|
|
24
|
+
for (let at = 0; at < copy.length; at += width) {
|
|
25
|
+
for (let k = 0; k < width; k++) copy[at + k] = source[at + width - 1 - k];
|
|
26
|
+
}
|
|
27
|
+
return copy;
|
|
28
|
+
}
|
|
29
|
+
function littleEndianBytes(values) {
|
|
30
|
+
if (!LITTLE_ENDIAN_PLATFORM) return swappedCopy(values);
|
|
31
|
+
return new Uint8Array(values.buffer, values.byteOffset, values.byteLength);
|
|
32
|
+
}
|
|
33
|
+
function allocate(byteLength, shared) {
|
|
34
|
+
return shared ? new SharedArrayBuffer(byteLength) : new ArrayBuffer(byteLength);
|
|
35
|
+
}
|
|
36
|
+
function copyInto(bytes, width, shared) {
|
|
37
|
+
const buffer = allocate(bytes.byteLength, shared);
|
|
38
|
+
const target = new Uint8Array(buffer);
|
|
39
|
+
if (LITTLE_ENDIAN_PLATFORM) {
|
|
40
|
+
target.set(bytes);
|
|
41
|
+
return buffer;
|
|
42
|
+
}
|
|
43
|
+
for (let at = 0; at < bytes.length; at += width) {
|
|
44
|
+
for (let k = 0; k < width; k++) target[at + k] = bytes[at + width - 1 - k];
|
|
45
|
+
}
|
|
46
|
+
return buffer;
|
|
47
|
+
}
|
|
48
|
+
function uint32sFrom(bytes, shared) {
|
|
49
|
+
return new Uint32Array(copyInto(bytes, 4, shared));
|
|
50
|
+
}
|
|
51
|
+
function uint16sFrom(bytes, shared) {
|
|
52
|
+
return new Uint16Array(copyInto(bytes, 2, shared));
|
|
53
|
+
}
|
|
54
|
+
function float64sFrom(bytes, shared) {
|
|
55
|
+
return new Float64Array(copyInto(bytes, 8, shared));
|
|
56
|
+
}
|
|
57
|
+
function bytesFrom(bytes, shared) {
|
|
58
|
+
return new Uint8Array(copyInto(bytes, 1, shared));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// src/serialization/keyword-segment/raw-fields.ts
|
|
62
|
+
function corrupt(message, details) {
|
|
63
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Invalid keyword segment: ${message}`, details);
|
|
64
|
+
}
|
|
65
|
+
function isRecord(value) {
|
|
66
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && !(value instanceof Uint8Array);
|
|
67
|
+
}
|
|
68
|
+
function requireRecord(value, name) {
|
|
69
|
+
if (!isRecord(value)) corrupt(`${name} must be a map`);
|
|
70
|
+
return value;
|
|
71
|
+
}
|
|
72
|
+
function requireList(value, name) {
|
|
73
|
+
if (!Array.isArray(value)) corrupt(`${name} must be a list`);
|
|
74
|
+
return value;
|
|
75
|
+
}
|
|
76
|
+
function requireString(value, name) {
|
|
77
|
+
if (typeof value !== "string") corrupt(`${name} must be a string`);
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
function requireStrings(value, name) {
|
|
81
|
+
const list = requireList(value, name);
|
|
82
|
+
for (const item of list) {
|
|
83
|
+
if (typeof item !== "string") corrupt(`${name} must hold strings alone`);
|
|
84
|
+
}
|
|
85
|
+
return list;
|
|
86
|
+
}
|
|
87
|
+
function requireBytes(value, name, width) {
|
|
88
|
+
if (!(value instanceof Uint8Array)) corrupt(`${name} must be bytes`);
|
|
89
|
+
if (value.byteLength % width !== 0) {
|
|
90
|
+
corrupt(`${name} holds ${value.byteLength} bytes, which is not a whole number of ${width}-byte values`, {
|
|
91
|
+
field: name
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
return value;
|
|
95
|
+
}
|
|
96
|
+
function requireBytesOrNil(value, name, width) {
|
|
97
|
+
if (value === null || value === void 0) return null;
|
|
98
|
+
return requireBytes(value, name, width);
|
|
99
|
+
}
|
|
100
|
+
function requireUint32(value, name) {
|
|
101
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > 4294967295) {
|
|
102
|
+
corrupt(`${name} must be a uint32`);
|
|
103
|
+
}
|
|
104
|
+
return value;
|
|
105
|
+
}
|
|
106
|
+
function requireCount(found, expected, name) {
|
|
107
|
+
if (found === expected) return;
|
|
108
|
+
corrupt(`${name} holds ${found} values where the segment needs ${expected}`, { field: name, found, expected });
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// src/serialization/keyword-segment/utf8.ts
|
|
112
|
+
var CONTINUATION_MASK = 192;
|
|
113
|
+
var CONTINUATION_BYTE = 128;
|
|
114
|
+
function continues(bytes, at, count) {
|
|
115
|
+
if (at + count > bytes.length) return false;
|
|
116
|
+
for (let k = 1; k <= count; k++) {
|
|
117
|
+
if ((bytes[at + k] & CONTINUATION_MASK) !== CONTINUATION_BYTE) return false;
|
|
118
|
+
}
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
function sequenceLengthAt(bytes, at) {
|
|
122
|
+
const lead = bytes[at];
|
|
123
|
+
if (lead < 128) return 1;
|
|
124
|
+
if (lead >= 194 && lead <= 223) return continues(bytes, at, 1) ? 2 : 0;
|
|
125
|
+
if (lead >= 224 && lead <= 239) {
|
|
126
|
+
if (!continues(bytes, at, 2)) return 0;
|
|
127
|
+
const second = bytes[at + 1];
|
|
128
|
+
if (lead === 224 && second < 160) return 0;
|
|
129
|
+
if (lead === 237 && second > 159) return 0;
|
|
130
|
+
return 3;
|
|
131
|
+
}
|
|
132
|
+
if (lead >= 240 && lead <= 244) {
|
|
133
|
+
if (!continues(bytes, at, 3)) return 0;
|
|
134
|
+
const second = bytes[at + 1];
|
|
135
|
+
if (lead === 240 && second < 144) return 0;
|
|
136
|
+
if (lead === 244 && second > 143) return 0;
|
|
137
|
+
return 4;
|
|
138
|
+
}
|
|
139
|
+
return 0;
|
|
140
|
+
}
|
|
141
|
+
function isValidUtf8(bytes) {
|
|
142
|
+
let at = 0;
|
|
143
|
+
while (at < bytes.length) {
|
|
144
|
+
const length = sequenceLengthAt(bytes, at);
|
|
145
|
+
if (length === 0) return false;
|
|
146
|
+
at += length;
|
|
147
|
+
}
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// src/serialization/keyword-segment/validate.ts
|
|
152
|
+
var CONTINUATION_MASK2 = 192;
|
|
153
|
+
var CONTINUATION_BYTE2 = 128;
|
|
154
|
+
function requireOffsets(offsets, listSize, name) {
|
|
155
|
+
if (offsets.length === 0) corrupt(`${name} must hold at least one value`, { field: name });
|
|
156
|
+
if (offsets[0] !== 0) corrupt(`${name} must start at 0`, { field: name });
|
|
157
|
+
for (let at = 1; at < offsets.length; at++) {
|
|
158
|
+
if (offsets[at] < offsets[at - 1]) corrupt(`${name} decreases at entry ${at}`, { field: name, at });
|
|
159
|
+
}
|
|
160
|
+
if (offsets[offsets.length - 1] !== listSize) {
|
|
161
|
+
corrupt(`${name} ends at ${offsets[offsets.length - 1]} where the list it divides holds ${listSize}`, {
|
|
162
|
+
field: name
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
function requireUtf8Entries(blob, offsets, name) {
|
|
167
|
+
if (!isValidUtf8(blob)) corrupt(`${name} is not valid UTF-8`, { field: name });
|
|
168
|
+
for (let at = 0; at < offsets.length; at++) {
|
|
169
|
+
const offset = offsets[at];
|
|
170
|
+
if (offset < blob.length && (blob[offset] & CONTINUATION_MASK2) === CONTINUATION_BYTE2) {
|
|
171
|
+
corrupt(`${name} splits a character at entry ${at}`, { field: name, at });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function requireOrdinals(values, documentCount, name) {
|
|
176
|
+
for (let at = 0; at < values.length; at++) {
|
|
177
|
+
if (values[at] >= documentCount) {
|
|
178
|
+
corrupt(`${name} names document ${values[at]} of a segment of ${documentCount}`, { field: name, at });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
function requireStrictlyAscending(values, start, end, name) {
|
|
183
|
+
for (let at = start + 1; at < end; at++) {
|
|
184
|
+
if (values[at] <= values[at - 1]) corrupt(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function compareEntries(blob, offsets, left, right) {
|
|
188
|
+
const leftStart = offsets[left];
|
|
189
|
+
const leftLength = offsets[left + 1] - leftStart;
|
|
190
|
+
const rightStart = offsets[right];
|
|
191
|
+
const rightLength = offsets[right + 1] - rightStart;
|
|
192
|
+
const shared = leftLength < rightLength ? leftLength : rightLength;
|
|
193
|
+
for (let k = 0; k < shared; k++) {
|
|
194
|
+
const diff = blob[leftStart + k] - blob[rightStart + k];
|
|
195
|
+
if (diff !== 0) return diff;
|
|
196
|
+
}
|
|
197
|
+
return leftLength - rightLength;
|
|
198
|
+
}
|
|
199
|
+
function requireEntriesAscend(blob, offsets, order, name) {
|
|
200
|
+
const count = offsets.length - 1;
|
|
201
|
+
for (let at = 1; at < count; at++) {
|
|
202
|
+
const left = order === null ? at - 1 : order[at - 1];
|
|
203
|
+
const right = order === null ? at : order[at];
|
|
204
|
+
if (compareEntries(blob, offsets, left, right) >= 0) {
|
|
205
|
+
corrupt(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
function requireSpellingEntriesAscend(blob, offsets, tokens, name) {
|
|
210
|
+
for (let at = 1; at < tokens.length; at++) {
|
|
211
|
+
const order = compareEntries(blob, offsets, at - 1, at) || tokens[at - 1] - tokens[at];
|
|
212
|
+
if (order >= 0) corrupt(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function requireBelow(values, limit, name) {
|
|
216
|
+
for (let at = 0; at < values.length; at++) {
|
|
217
|
+
if (values[at] >= limit)
|
|
218
|
+
corrupt(`${name} holds ${values[at]} at entry ${at}, at or above ${limit}`, { field: name, at });
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
function requireNonZero(values, name) {
|
|
222
|
+
for (let at = 0; at < values.length; at++) {
|
|
223
|
+
if (values[at] === 0) corrupt(`${name} holds 0 at entry ${at}`, { field: name, at });
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function requirePermutation(order, name) {
|
|
227
|
+
const seen = new Uint8Array(order.length);
|
|
228
|
+
for (let at = 0; at < order.length; at++) {
|
|
229
|
+
const ordinal = order[at];
|
|
230
|
+
if (ordinal >= order.length || seen[ordinal] === 1) {
|
|
231
|
+
corrupt(`${name} repeats or overruns ordinal ${ordinal}`, { field: name, at });
|
|
232
|
+
}
|
|
233
|
+
seen[ordinal] = 1;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
function requireValuesAscendWithoutNaN(values, name) {
|
|
237
|
+
for (let at = 0; at < values.length; at++) {
|
|
238
|
+
if (Number.isNaN(values[at])) corrupt(`${name} holds NaN at entry ${at}`, { field: name, at });
|
|
239
|
+
if (at > 0 && values[at] < values[at - 1]) corrupt(`${name} descends at entry ${at}`, { field: name, at });
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
function requireWithin(values, lowest, highest, name) {
|
|
243
|
+
for (let at = 0; at < values.length; at++) {
|
|
244
|
+
const value = values[at];
|
|
245
|
+
if (!(value >= lowest && value <= highest)) {
|
|
246
|
+
corrupt(`${name} holds ${value} at entry ${at}, outside ${lowest} to ${highest}`, { field: name, at });
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
function requireRunsAscend(runs, name) {
|
|
251
|
+
for (let run = 1; run * 3 < runs.length; run++) {
|
|
252
|
+
const at = run * 3;
|
|
253
|
+
const before = at - 3;
|
|
254
|
+
let diff = runs[at] - runs[before];
|
|
255
|
+
if (diff === 0) diff = runs[at + 1] - runs[before + 1];
|
|
256
|
+
if (diff === 0) diff = runs[at + 2] - runs[before + 2];
|
|
257
|
+
if (diff <= 0) corrupt(`${name} fails to ascend strictly at run ${run}`, { field: name, run });
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// src/serialization/keyword-segment/decode-columns.ts
|
|
262
|
+
var LATITUDE_LIMIT = 90;
|
|
263
|
+
var LONGITUDE_LIMIT = 180;
|
|
264
|
+
var CODE_POINTS_PER_RUN = 3;
|
|
265
|
+
function uint32Column(raw, key, name, context) {
|
|
266
|
+
return uint32sFrom(requireBytes(raw[key], `${name}.${key}`, 4), context.shared);
|
|
267
|
+
}
|
|
268
|
+
function ordinalColumn(raw, key, name, context) {
|
|
269
|
+
const values = uint32Column(raw, key, name, context);
|
|
270
|
+
requireOrdinals(values, context.documentCount, `${name}.${key}`);
|
|
271
|
+
return values;
|
|
272
|
+
}
|
|
273
|
+
function eachColumn(raw, name) {
|
|
274
|
+
return requireList(raw, name).map((entry, at) => {
|
|
275
|
+
const column = requireRecord(entry, `${name}[${at}]`);
|
|
276
|
+
return { column, label: `${name}[${at}]` };
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
function decodeFieldLengths(raw, context) {
|
|
280
|
+
const fieldLengthNames = [];
|
|
281
|
+
const fieldLengthColumns = [];
|
|
282
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
283
|
+
for (const { column, label } of eachColumn(raw, "fieldLengths")) {
|
|
284
|
+
const field = requireString(column.field, `${label}.field`);
|
|
285
|
+
const lengths = uint32Column(column, "lengths", label, context);
|
|
286
|
+
requireCount(lengths.length, context.documentCount, `${label}.lengths`);
|
|
287
|
+
let total = 0;
|
|
288
|
+
for (let at = 0; at < lengths.length; at++) total += lengths[at];
|
|
289
|
+
fieldLengthNames.push(field);
|
|
290
|
+
fieldLengthColumns.push(lengths);
|
|
291
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + total;
|
|
292
|
+
}
|
|
293
|
+
return { fieldLengthNames, fieldLengthColumns, totalFieldLengths };
|
|
294
|
+
}
|
|
295
|
+
function decodeValueColumns(raw, name, context) {
|
|
296
|
+
return eachColumn(raw, name).map(({ column, label }) => {
|
|
297
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
298
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
299
|
+
const values = float64sFrom(requireBytes(column.values, `${label}.values`, 8), context.shared);
|
|
300
|
+
requireCount(values.length, docIds.length, `${label}.values`);
|
|
301
|
+
requireValuesAscendWithoutNaN(values, `${label}.values`);
|
|
302
|
+
return { fieldPath, docIds, values };
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
function decodeBooleanColumns(raw, context) {
|
|
306
|
+
return eachColumn(raw, "boolean").map(({ column, label }) => ({
|
|
307
|
+
fieldPath: requireString(column.field, `${label}.field`),
|
|
308
|
+
trueDocs: ordinalColumn(column, "trueDocuments", label, context),
|
|
309
|
+
falseDocs: ordinalColumn(column, "falseDocuments", label, context)
|
|
310
|
+
}));
|
|
311
|
+
}
|
|
312
|
+
function decodeEnumColumns(raw, context) {
|
|
313
|
+
return eachColumn(raw, "enum").map(({ column, label }) => {
|
|
314
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
315
|
+
const values = requireStrings(column.values, `${label}.values`);
|
|
316
|
+
if (new Set(values).size !== values.length) corrupt(`${label}.values names a value twice`, { field: label });
|
|
317
|
+
const offsets = uint32Column(column, "offsets", label, context);
|
|
318
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
319
|
+
requireCount(offsets.length, values.length + 1, `${label}.offsets`);
|
|
320
|
+
requireOffsets(offsets, docIds.length, `${label}.offsets`);
|
|
321
|
+
return { fieldPath, values, offsets, docIds };
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
function decodeGeopointColumns(raw, context) {
|
|
325
|
+
return eachColumn(raw, "geopoint").map(({ column, label }) => {
|
|
326
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
327
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
328
|
+
const latitudes = float64sFrom(requireBytes(column.latitudes, `${label}.latitudes`, 8), context.shared);
|
|
329
|
+
const longitudes = float64sFrom(requireBytes(column.longitudes, `${label}.longitudes`, 8), context.shared);
|
|
330
|
+
requireCount(latitudes.length, docIds.length, `${label}.latitudes`);
|
|
331
|
+
requireCount(longitudes.length, docIds.length, `${label}.longitudes`);
|
|
332
|
+
requireWithin(latitudes, -LATITUDE_LIMIT, LATITUDE_LIMIT, `${label}.latitudes`);
|
|
333
|
+
requireWithin(longitudes, -LONGITUDE_LIMIT, LONGITUDE_LIMIT, `${label}.longitudes`);
|
|
334
|
+
return { fieldPath, docIds, latitudes, longitudes };
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
function decodePatternColumns(raw, context) {
|
|
338
|
+
return eachColumn(raw, "pattern").map(({ column, label }) => {
|
|
339
|
+
const fieldPath = requireString(column.field, `${label}.field`);
|
|
340
|
+
const docIds = ordinalColumn(column, "documents", label, context);
|
|
341
|
+
requireStrictlyAscending(docIds, 0, docIds.length, `${label}.documents`);
|
|
342
|
+
const runs = uint32sFrom(requireBytes(column.runs, `${label}.runs`, 4 * CODE_POINTS_PER_RUN), context.shared);
|
|
343
|
+
requireRunsAscend(runs, `${label}.runs`);
|
|
344
|
+
const offsets = uint32Column(column, "offsets", label, context);
|
|
345
|
+
const postings = ordinalColumn(column, "postings", label, context);
|
|
346
|
+
requireCount(offsets.length, runs.length / CODE_POINTS_PER_RUN + 1, `${label}.offsets`);
|
|
347
|
+
requireOffsets(offsets, postings.length, `${label}.offsets`);
|
|
348
|
+
for (let run = 0; run + 1 < offsets.length; run++) {
|
|
349
|
+
requireStrictlyAscending(postings, offsets[run], offsets[run + 1], `${label}.postings`);
|
|
350
|
+
}
|
|
351
|
+
return { fieldPath, docIds, runs, offsets, postings };
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
function decodeSurfaceForms(raw, context) {
|
|
355
|
+
if (raw === null || raw === void 0) return null;
|
|
356
|
+
const column = requireRecord(raw, "surfaceForms");
|
|
357
|
+
const blob = bytesFrom(requireBytes(column.text, "surfaceForms.text", 1), context.shared);
|
|
358
|
+
const offsets = uint32Column(column, "offsets", "surfaceForms", context);
|
|
359
|
+
const counts = uint32Column(column, "counts", "surfaceForms", context);
|
|
360
|
+
requireCount(offsets.length, counts.length * 2 + 1, "surfaceForms.offsets");
|
|
361
|
+
requireOffsets(offsets, blob.length, "surfaceForms.offsets");
|
|
362
|
+
requireUtf8Entries(blob, offsets, "surfaceForms.text");
|
|
363
|
+
return counts.length === 0 ? null : { blob, offsets, counts };
|
|
364
|
+
}
|
|
365
|
+
function decodePartialWords(raw, tokenCount, context) {
|
|
366
|
+
if (raw === null || raw === void 0) return null;
|
|
367
|
+
const column = requireRecord(raw, "partialWords");
|
|
368
|
+
const fields = requireStrings(column.fields, "partialWords.fields");
|
|
369
|
+
const spellings = bytesFrom(requireBytes(column.text, "partialWords.text", 1), context.shared);
|
|
370
|
+
const spellingOffsets = uint32Column(column, "offsets", "partialWords", context);
|
|
371
|
+
const spellingTokens = uint32Column(column, "tokens", "partialWords", context);
|
|
372
|
+
const spellingCounts = uint32Column(column, "counts", "partialWords", context);
|
|
373
|
+
const entries = spellingTokens.length;
|
|
374
|
+
requireCount(spellingOffsets.length, entries + 1, "partialWords.offsets");
|
|
375
|
+
requireCount(spellingCounts.length, entries, "partialWords.counts");
|
|
376
|
+
requireOffsets(spellingOffsets, spellings.length, "partialWords.offsets");
|
|
377
|
+
requireUtf8Entries(spellings, spellingOffsets, "partialWords.text");
|
|
378
|
+
requireBelow(spellingTokens, tokenCount, "partialWords.tokens");
|
|
379
|
+
requireNonZero(spellingCounts, "partialWords.counts");
|
|
380
|
+
requireSpellingEntriesAscend(spellings, spellingOffsets, spellingTokens, "partialWords");
|
|
381
|
+
const runs = uint32sFrom(requireBytes(column.runs, "partialWords.runs", 4 * CODE_POINTS_PER_RUN), context.shared);
|
|
382
|
+
requireRunsAscend(runs, "partialWords.runs");
|
|
383
|
+
const runOffsets = uint32Column(column, "runOffsets", "partialWords", context);
|
|
384
|
+
const runEntries = uint32Column(column, "runEntries", "partialWords", context);
|
|
385
|
+
requireCount(runOffsets.length, runs.length / CODE_POINTS_PER_RUN + 1, "partialWords.runOffsets");
|
|
386
|
+
requireOffsets(runOffsets, runEntries.length, "partialWords.runOffsets");
|
|
387
|
+
requireBelow(runEntries, entries, "partialWords.runEntries");
|
|
388
|
+
for (let run = 0; run + 1 < runOffsets.length; run++) {
|
|
389
|
+
requireStrictlyAscending(runEntries, runOffsets[run], runOffsets[run + 1], "partialWords.runEntries");
|
|
390
|
+
}
|
|
391
|
+
return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// src/persistence/paged/column-layout.ts
|
|
395
|
+
function requirePagedColumnLengths(sections, documents, tokens, postings, key) {
|
|
396
|
+
const required = (name) => {
|
|
397
|
+
const section = sectionNamed(sections, name);
|
|
398
|
+
if (section === void 0) loadFailed(`The segment "${key}" holds no section "${name}"`);
|
|
399
|
+
return section;
|
|
400
|
+
};
|
|
401
|
+
const exact = (name, length) => {
|
|
402
|
+
if (required(name).length !== length) loadFailed(`Section "${name}" of "${key}" has the wrong length`);
|
|
403
|
+
};
|
|
404
|
+
exact("idOffsets", (documents + 1) * 4);
|
|
405
|
+
exact("idOrder", documents * 4);
|
|
406
|
+
exact("removalOffsets", (documents + 1) * 4);
|
|
407
|
+
exact("tokenTotals", tokens * 8);
|
|
408
|
+
if (required("removals").length % 8 !== 0) loadFailed(`The removal pairs of "${key}" are cut short`);
|
|
409
|
+
if (documents === 0) {
|
|
410
|
+
exact("ids", 0);
|
|
411
|
+
exact("removals", 0);
|
|
412
|
+
}
|
|
413
|
+
const positions = sectionNamed(sections, "positions");
|
|
414
|
+
if (positions !== void 0) {
|
|
415
|
+
exact("positionOffsets", (postings + 1) * 4);
|
|
416
|
+
if (positions.length % 4 !== 0) loadFailed(`The positions of "${key}" are cut short`);
|
|
417
|
+
if (postings === 0) exact("positions", 0);
|
|
418
|
+
}
|
|
419
|
+
const sequences = sectionNamed(sections, "seqNos");
|
|
420
|
+
const terms = sectionNamed(sections, "primaryTerms");
|
|
421
|
+
if (sequences === void 0 !== (terms === void 0)) loadFailed(`The write records of "${key}" are incomplete`);
|
|
422
|
+
if (sequences !== void 0) {
|
|
423
|
+
exact("seqNos", documents * 8);
|
|
424
|
+
exact("primaryTerms", documents * 8);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
function requireBlockSummaryList(summaries, first, end, postings, key) {
|
|
428
|
+
const view = new DataView(summaries.buffer, summaries.byteOffset, summaries.byteLength);
|
|
429
|
+
let previous = 0;
|
|
430
|
+
for (let block = first; block < end; block++) {
|
|
431
|
+
const at = block * BLOCK_SUMMARY_BYTES;
|
|
432
|
+
const next = view.getUint32(at, true);
|
|
433
|
+
const minimum = view.getFloat64(at + 6, true);
|
|
434
|
+
const entries = view.getUint32(at + 14, true);
|
|
435
|
+
const documents = view.getUint32(at + 18, true);
|
|
436
|
+
if (next <= previous || next > postings || !Number.isFinite(minimum) || minimum < 0 || entries === 0 || entries > next - previous || documents === 0 || documents > next - previous)
|
|
437
|
+
loadFailed(`Block ${block} of "${key}" holds invalid bounds`, { block });
|
|
438
|
+
previous = next;
|
|
439
|
+
}
|
|
440
|
+
if (first === end || previous !== postings) loadFailed(`The block summaries of "${key}" do not cover their list`);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// src/persistence/paged/resident.ts
|
|
444
|
+
var RESIDENT_SECTION_NAMES = /* @__PURE__ */ new Set([
|
|
445
|
+
"tokens",
|
|
446
|
+
"tokenOffsets",
|
|
447
|
+
"documentFrequencies",
|
|
448
|
+
"postingOffsets",
|
|
449
|
+
"blockLists",
|
|
450
|
+
"blockListOffsets",
|
|
451
|
+
"blockSummaries",
|
|
452
|
+
"documentBlocks",
|
|
453
|
+
"idSample",
|
|
454
|
+
"fieldLengths",
|
|
455
|
+
"numeric",
|
|
456
|
+
"sizes",
|
|
457
|
+
"boolean",
|
|
458
|
+
"enum",
|
|
459
|
+
"geopoint",
|
|
460
|
+
"pattern",
|
|
461
|
+
"partialWords"
|
|
462
|
+
]);
|
|
463
|
+
function sharedMemoryAvailable() {
|
|
464
|
+
return typeof SharedArrayBuffer === "function";
|
|
465
|
+
}
|
|
466
|
+
function identitySlots(count, shared) {
|
|
467
|
+
const slots = shared ? new Uint32Array(new SharedArrayBuffer(count * 4)) : new Uint32Array(count);
|
|
468
|
+
for (let at = 0; at < count; at++) slots[at] = at;
|
|
469
|
+
return slots;
|
|
470
|
+
}
|
|
471
|
+
function readPagedHead(listing) {
|
|
472
|
+
const { path, key, bytes, checksum } = listing;
|
|
473
|
+
const header2 = new Uint8Array(HEADER_SIZE + PAYLOAD_PREFIX_BYTES);
|
|
474
|
+
if (bytes < header2.byteLength) loadFailed(`The segment "${key}" is too short to hold a head`, { key, bytes });
|
|
475
|
+
readFileRange(path, 0, header2);
|
|
476
|
+
const opened = readHeader(header2);
|
|
477
|
+
if (opened.envelopeFormatVersion !== PAGED_SEGMENT_ENVELOPE_VERSION) {
|
|
478
|
+
throw new NarsilError(
|
|
479
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
480
|
+
`The segment "${key}" holds envelope format v${opened.envelopeFormatVersion}`,
|
|
481
|
+
{ key, version: opened.envelopeFormatVersion }
|
|
482
|
+
);
|
|
483
|
+
}
|
|
484
|
+
if (opened.flags.compressionEnabled) loadFailed(`The segment "${key}" is compressed, while its format stores it raw`);
|
|
485
|
+
if (HEADER_SIZE + opened.payloadLength !== bytes) {
|
|
486
|
+
loadFailed(
|
|
487
|
+
`The segment "${key}" holds ${bytes} bytes where its header gives ${HEADER_SIZE + opened.payloadLength}`,
|
|
488
|
+
{
|
|
489
|
+
key
|
|
490
|
+
}
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
if (opened.checksum !== checksum) {
|
|
494
|
+
loadFailed(`The segment "${key}" carries checksum ${opened.checksum} where the manifest lists ${checksum}`, { key });
|
|
495
|
+
}
|
|
496
|
+
const prefix = new DataView(header2.buffer, HEADER_SIZE, PAYLOAD_PREFIX_BYTES);
|
|
497
|
+
const headLength = prefix.getUint32(0, true);
|
|
498
|
+
const headChecksum = prefix.getUint32(4, true);
|
|
499
|
+
if (PAYLOAD_PREFIX_BYTES + headLength > opened.payloadLength) {
|
|
500
|
+
loadFailed(`The head of the segment "${key}" ends outside its payload`, { key, headLength });
|
|
501
|
+
}
|
|
502
|
+
const head = new Uint8Array(headLength);
|
|
503
|
+
readFileRange(path, HEADER_SIZE + PAYLOAD_PREFIX_BYTES, head);
|
|
504
|
+
if (crc32(head) !== headChecksum) {
|
|
505
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, `The head of the segment "${key}" fails its checksum`, {
|
|
506
|
+
key
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
const decoded = decodePagedSegmentHead(head, opened.payloadLength, `the segment "${key}"`);
|
|
510
|
+
for (const section of decoded.sections) {
|
|
511
|
+
if (section.offset < PAYLOAD_PREFIX_BYTES + headLength || section.offset % SECTION_ALIGNMENT_BYTES !== 0)
|
|
512
|
+
loadFailed(`Section "${section.name}" of "${key}" starts inside its head or between aligned offsets`);
|
|
513
|
+
}
|
|
514
|
+
return {
|
|
515
|
+
head: decoded,
|
|
516
|
+
payloadLength: opened.payloadLength
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
function requireSection(sections, name, key) {
|
|
520
|
+
const section = sectionNamed(sections, name);
|
|
521
|
+
if (section === void 0) loadFailed(`The segment "${key}" holds no section "${name}"`, { key, section: name });
|
|
522
|
+
return section;
|
|
523
|
+
}
|
|
524
|
+
function decodeResidentList(path, section) {
|
|
525
|
+
const bytes = readWholeSectionChecked(path, section, false);
|
|
526
|
+
try {
|
|
527
|
+
return decode(bytes);
|
|
528
|
+
} catch (error) {
|
|
529
|
+
return loadFailed(
|
|
530
|
+
`Section "${section.name}" of "${path}" is not MessagePack: ${error instanceof Error ? error.message : String(error)}`
|
|
531
|
+
);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
function checkedUint32s(path, section, shared) {
|
|
535
|
+
if (section.length % 4 !== 0) loadFailed(`Section "${section.name}" of "${path}" is not a whole number of values`);
|
|
536
|
+
return uint32sFrom(readWholeSectionChecked(path, section, false), shared);
|
|
537
|
+
}
|
|
538
|
+
function readIdSample(path, section, shared, count) {
|
|
539
|
+
const raw = decodeResidentList(path, section);
|
|
540
|
+
if (!Array.isArray(raw)) loadFailed(`Section "idSample" of "${path}" is not a list`);
|
|
541
|
+
if (raw.length !== Math.ceil(count / ID_SAMPLE_INTERVAL))
|
|
542
|
+
loadFailed(`The ID sample of "${path}" has the wrong length`);
|
|
543
|
+
const ids = new Array(raw.length);
|
|
544
|
+
const positions = shared ? new Uint32Array(new SharedArrayBuffer(raw.length * 4)) : new Uint32Array(raw.length);
|
|
545
|
+
for (let at = 0; at < raw.length; at++) {
|
|
546
|
+
const entry = raw[at];
|
|
547
|
+
if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string" || entry[1] !== at * ID_SAMPLE_INTERVAL) {
|
|
548
|
+
loadFailed(`Section "idSample" of "${path}" holds an invalid entry at ${at}`, { at });
|
|
549
|
+
}
|
|
550
|
+
ids[at] = entry[0];
|
|
551
|
+
positions[at] = entry[1];
|
|
552
|
+
if (at > 0 && compareCodePoints(ids[at - 1], ids[at]) >= 0) {
|
|
553
|
+
loadFailed(`Section "idSample" of "${path}" fails to ascend at entry ${at}`, { at });
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
return { ids, positions };
|
|
557
|
+
}
|
|
558
|
+
function readIdSampleOf(path, head) {
|
|
559
|
+
return readIdSample(path, requireSection(head.sections, "idSample", path), false, head.documentCount);
|
|
560
|
+
}
|
|
561
|
+
function requireBlockLayout(lists, offsets, summaries, postingOffsets, tokenCount, key) {
|
|
562
|
+
if (summaries.byteLength % BLOCK_SUMMARY_BYTES !== 0) loadFailed(`The block summaries of "${key}" are cut short`);
|
|
563
|
+
if (offsets.length !== lists.length + 1) loadFailed(`The block list offsets of "${key}" have the wrong length`);
|
|
564
|
+
requireOffsets(offsets, summaries.byteLength / BLOCK_SUMMARY_BYTES, "blockListOffsets");
|
|
565
|
+
for (let at = 0; at < lists.length; at++) {
|
|
566
|
+
const token = lists[at];
|
|
567
|
+
if (at > 0 && token <= lists[at - 1]) loadFailed(`blockLists of "${key}" fails to ascend at entry ${at}`, { at });
|
|
568
|
+
if (token >= tokenCount) loadFailed(`blockLists of "${key}" holds token ${token} of ${tokenCount}`, { at });
|
|
569
|
+
if (postingOffsets[token + 1] - postingOffsets[token] <= SMALLEST_BLOCK_POSTINGS) {
|
|
570
|
+
loadFailed(`blockLists of "${key}" lists token ${token}, which holds too few postings`, { token });
|
|
571
|
+
}
|
|
572
|
+
requireBlockSummaryList(
|
|
573
|
+
summaries,
|
|
574
|
+
offsets[at],
|
|
575
|
+
offsets[at + 1],
|
|
576
|
+
postingOffsets[token + 1] - postingOffsets[token],
|
|
577
|
+
key
|
|
578
|
+
);
|
|
579
|
+
}
|
|
580
|
+
let listed = 0;
|
|
581
|
+
for (let token = 0; token < tokenCount; token++) {
|
|
582
|
+
if (postingOffsets[token + 1] - postingOffsets[token] <= SMALLEST_BLOCK_POSTINGS) continue;
|
|
583
|
+
if (lists[listed++] !== token) loadFailed(`The block lists of "${key}" omit token ${token}`, { token });
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
function requireDocumentBlockLayout(blocks, documentCount, sectionLength, key) {
|
|
587
|
+
if (blocks.byteLength % DOCUMENT_BLOCK_BYTES !== 0) loadFailed(`The document blocks of "${key}" are cut short`);
|
|
588
|
+
const view = new DataView(blocks.buffer, blocks.byteOffset, blocks.byteLength);
|
|
589
|
+
const count = blocks.byteLength / DOCUMENT_BLOCK_BYTES;
|
|
590
|
+
let previousFirst = -1;
|
|
591
|
+
let previousEnd = 0;
|
|
592
|
+
for (let at = 0; at < count; at++) {
|
|
593
|
+
const place = at * DOCUMENT_BLOCK_BYTES;
|
|
594
|
+
const first = view.getUint32(place, true);
|
|
595
|
+
const offset = Number(view.getBigUint64(place + 4, true));
|
|
596
|
+
const length = view.getUint32(place + 12, true);
|
|
597
|
+
const rawLength = view.getUint32(place + 16, true);
|
|
598
|
+
if (rawLength > MAX_DOCUMENT_BLOCK_RAW_BYTES)
|
|
599
|
+
loadFailed(`Document block ${at} of "${key}" exceeds its raw byte limit`);
|
|
600
|
+
if (first <= previousFirst || first >= documentCount) loadFailed(`Document block ${at} of "${key}" starts wrongly`);
|
|
601
|
+
if (offset !== previousEnd || offset + length > sectionLength) {
|
|
602
|
+
loadFailed(`Document block ${at} of "${key}" lies outside its section`, { at });
|
|
603
|
+
}
|
|
604
|
+
previousFirst = first;
|
|
605
|
+
previousEnd = offset + length;
|
|
606
|
+
}
|
|
607
|
+
if (documentCount > 0 && (count === 0 || view.getUint32(0, true) !== 0)) {
|
|
608
|
+
loadFailed(`The document blocks of "${key}" fail to cover its documents`);
|
|
609
|
+
}
|
|
610
|
+
if (previousEnd !== sectionLength) loadFailed(`The document blocks of "${key}" leave bytes outside their blocks`);
|
|
611
|
+
}
|
|
612
|
+
function vectorFieldsOfSchema(schema) {
|
|
613
|
+
return Object.keys(schema).filter((fieldPath) => VECTOR_PATTERN.test(schema[fieldPath]));
|
|
614
|
+
}
|
|
615
|
+
function residentSectionsOf(listing) {
|
|
616
|
+
return readPagedHead(listing).head.sections.filter((section) => RESIDENT_SECTION_NAMES.has(section.name));
|
|
617
|
+
}
|
|
618
|
+
function bytesOfResidentSections(listing) {
|
|
619
|
+
let bytes = 0;
|
|
620
|
+
for (const section of residentSectionsOf(listing)) bytes += section.length;
|
|
621
|
+
return bytes;
|
|
622
|
+
}
|
|
623
|
+
function openPagedSnapshot(listing) {
|
|
624
|
+
const { head } = readPagedHead(listing);
|
|
625
|
+
const { path, key } = listing;
|
|
626
|
+
const { sections, documentCount } = head;
|
|
627
|
+
const shared = sharedMemoryAvailable();
|
|
628
|
+
const context = { documentCount, shared };
|
|
629
|
+
const take = (name) => requireSection(sections, name, key);
|
|
630
|
+
const tokens = readWholeSectionChecked(path, take("tokens"), shared);
|
|
631
|
+
const tokenOffsets = checkedUint32s(path, take("tokenOffsets"), shared);
|
|
632
|
+
requireOffsets(tokenOffsets, tokens.length, "tokenOffsets");
|
|
633
|
+
requireUtf8Entries(tokens, tokenOffsets, "tokens");
|
|
634
|
+
requireEntriesAscend(tokens, tokenOffsets, null, "tokens");
|
|
635
|
+
const tokenCount = tokenOffsets.length - 1;
|
|
636
|
+
const documentFrequencies = checkedUint32s(path, take("documentFrequencies"), shared);
|
|
637
|
+
const postingOffsets = checkedUint32s(path, take("postingOffsets"), shared);
|
|
638
|
+
if (documentFrequencies.length !== tokenCount || postingOffsets.length !== tokenCount + 1) {
|
|
639
|
+
loadFailed(`The token columns of "${key}" disagree on the token count`, { key });
|
|
640
|
+
}
|
|
641
|
+
const postingBytes = requireSection(sections, "postings", key).length;
|
|
642
|
+
requireOffsets(postingOffsets, postingBytes / 8, "postingOffsets");
|
|
643
|
+
requirePagedColumnLengths(sections, documentCount, tokenCount, postingBytes / 8, key);
|
|
644
|
+
for (let token = 0; token < tokenCount; token++) {
|
|
645
|
+
const postings = postingOffsets[token + 1] - postingOffsets[token];
|
|
646
|
+
if (documentFrequencies[token] === 0 || documentFrequencies[token] > postings) {
|
|
647
|
+
loadFailed(`documentFrequencies of "${key}" gives token ${token} a count of ${documentFrequencies[token]}`, {
|
|
648
|
+
token
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
const blockLists = checkedUint32s(path, take("blockLists"), shared);
|
|
653
|
+
const blockListOffsets = checkedUint32s(path, take("blockListOffsets"), shared);
|
|
654
|
+
const blockSummaries = readWholeSectionChecked(path, take("blockSummaries"), shared);
|
|
655
|
+
requireBlockLayout(blockLists, blockListOffsets, blockSummaries, postingOffsets, tokenCount, key);
|
|
656
|
+
const documentBlocks = readWholeSectionChecked(path, take("documentBlocks"), shared);
|
|
657
|
+
requireDocumentBlockLayout(documentBlocks, documentCount, take("documents").length, key);
|
|
658
|
+
const idSample = readIdSample(path, take("idSample"), shared, documentCount);
|
|
659
|
+
const lengths = decodeFieldLengths(decodeResidentList(path, take("fieldLengths")), context);
|
|
660
|
+
const positionOffsetsSection = sectionNamed(sections, "positionOffsets");
|
|
661
|
+
const positionsSection = sectionNamed(sections, "positions");
|
|
662
|
+
if (positionOffsetsSection === void 0 !== (positionsSection === void 0)) {
|
|
663
|
+
loadFailed(`The segment "${key}" holds one of positionOffsets and positions`, { key });
|
|
664
|
+
}
|
|
665
|
+
return {
|
|
666
|
+
segmentId: generateId(),
|
|
667
|
+
key,
|
|
668
|
+
path,
|
|
669
|
+
tracksPositions: positionOffsetsSection !== void 0,
|
|
670
|
+
schema: head.schema,
|
|
671
|
+
discardedFields: vectorFieldsOfSchema(head.schema),
|
|
672
|
+
sections,
|
|
673
|
+
documentCount,
|
|
674
|
+
fieldNames: head.fieldNames,
|
|
675
|
+
...lengths,
|
|
676
|
+
tokenTable: {
|
|
677
|
+
blob: tokens,
|
|
678
|
+
offsets: tokenOffsets,
|
|
679
|
+
payloadSlots: identitySlots(tokenCount, shared),
|
|
680
|
+
documentFrequencies
|
|
681
|
+
},
|
|
682
|
+
postingOffsets,
|
|
683
|
+
blockLists,
|
|
684
|
+
blockListOffsets,
|
|
685
|
+
blockSummaries,
|
|
686
|
+
documentBlocks,
|
|
687
|
+
idSampleIds: idSample.ids,
|
|
688
|
+
idSamplePositions: idSample.positions,
|
|
689
|
+
numeric: decodeValueColumns(decodeResidentList(path, take("numeric")), "numeric", context),
|
|
690
|
+
sizes: decodeValueColumns(decodeResidentList(path, take("sizes")), "sizes", context),
|
|
691
|
+
boolean: decodeBooleanColumns(decodeResidentList(path, take("boolean")), context),
|
|
692
|
+
enums: decodeEnumColumns(decodeResidentList(path, take("enum")), context),
|
|
693
|
+
geo: decodeGeopointColumns(decodeResidentList(path, take("geopoint")), context),
|
|
694
|
+
pattern: decodePatternColumns(decodeResidentList(path, take("pattern")), context),
|
|
695
|
+
partialWords: decodePartialWords(decodeResidentList(path, take("partialWords")), tokenCount, context),
|
|
696
|
+
tombstones: head.tombstones
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
function residentBytesOf(snapshot) {
|
|
700
|
+
const table = snapshot.tokenTable;
|
|
701
|
+
let bytes = table.blob.byteLength + table.offsets.byteLength;
|
|
702
|
+
bytes += table.payloadSlots.byteLength + table.documentFrequencies.byteLength;
|
|
703
|
+
bytes += snapshot.postingOffsets.byteLength + snapshot.blockLists.byteLength;
|
|
704
|
+
bytes += snapshot.blockListOffsets.byteLength + snapshot.blockSummaries.byteLength;
|
|
705
|
+
bytes += snapshot.documentBlocks.byteLength + snapshot.idSamplePositions.byteLength;
|
|
706
|
+
for (const id of snapshot.idSampleIds) bytes += id.length;
|
|
707
|
+
for (const id of snapshot.tombstones) bytes += id.length;
|
|
708
|
+
for (const column of snapshot.fieldLengthColumns) bytes += column.byteLength;
|
|
709
|
+
for (const entry of [...snapshot.numeric, ...snapshot.sizes])
|
|
710
|
+
bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
711
|
+
for (const entry of snapshot.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
|
|
712
|
+
for (const entry of snapshot.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
|
|
713
|
+
for (const entry of snapshot.geo) {
|
|
714
|
+
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
715
|
+
}
|
|
716
|
+
for (const entry of snapshot.pattern) {
|
|
717
|
+
bytes += entry.docIds.byteLength + entry.runs.byteLength + entry.offsets.byteLength + entry.postings.byteLength;
|
|
718
|
+
}
|
|
719
|
+
const partial = snapshot.partialWords;
|
|
720
|
+
if (partial !== null) {
|
|
721
|
+
bytes += partial.spellings.byteLength + partial.spellingOffsets.byteLength + partial.spellingTokens.byteLength;
|
|
722
|
+
bytes += partial.spellingCounts.byteLength + partial.runs.byteLength;
|
|
723
|
+
bytes += partial.runOffsets.byteLength + partial.runEntries.byteLength;
|
|
724
|
+
}
|
|
725
|
+
return bytes;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
// src/persistence/paged/totals.ts
|
|
729
|
+
var VALUE_BYTES = Float64Array.BYTES_PER_ELEMENT;
|
|
730
|
+
function readPagedTotals(file, totals, first, output, fault) {
|
|
731
|
+
const section = totals.native;
|
|
732
|
+
if (section !== null && file.native !== null) {
|
|
733
|
+
const status = file.native.core.readPagedTotals(
|
|
734
|
+
section,
|
|
735
|
+
file.cache.pageCache(),
|
|
736
|
+
first,
|
|
737
|
+
output.length,
|
|
738
|
+
output,
|
|
739
|
+
fault
|
|
740
|
+
);
|
|
741
|
+
requireNativePage(status, file.path, "tokenTotals", fault[2]);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
const bytes = totals.bytesAt(first * VALUE_BYTES, output.byteLength);
|
|
745
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
746
|
+
for (let at = 0; at < output.length; at++) {
|
|
747
|
+
const value = view.getFloat64(at * VALUE_BYTES, true);
|
|
748
|
+
if (!Number.isFinite(value) || value < 0) loadFailed(`The term total at ${first + at} of "${file.path}" is invalid`);
|
|
749
|
+
}
|
|
750
|
+
for (let at = 0; at < output.length; at++) output[at] = view.getFloat64(at * VALUE_BYTES, true);
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// src/persistence/paged/expansion-source.ts
|
|
754
|
+
var NUMBER_BYTES = 8;
|
|
755
|
+
function createPagedExpansionSource(file, snapshot, table, tombstones, removedDocuments, removedOccurrences, totals) {
|
|
756
|
+
const words = new Uint32Array(0);
|
|
757
|
+
const shorts = new Uint16Array(0);
|
|
758
|
+
const partialFields = new Uint8Array(snapshot.fieldNames.length);
|
|
759
|
+
let occurrences = null;
|
|
760
|
+
const frozen = {
|
|
761
|
+
terms: { text: table.data.blob, offsets: table.data.offsets, counts: table.data.documentFrequencies },
|
|
762
|
+
payloadSlots: table.data.payloadSlots,
|
|
763
|
+
postingOffsets: snapshot.postingOffsets,
|
|
764
|
+
postingDocuments: words,
|
|
765
|
+
postingFrequencies: shorts,
|
|
766
|
+
postingFields: shorts,
|
|
767
|
+
partialFields,
|
|
768
|
+
documentCount: snapshot.documentCount,
|
|
769
|
+
surfaces: null,
|
|
770
|
+
spellings: null,
|
|
771
|
+
spellingTokens: words,
|
|
772
|
+
removalIndex: null
|
|
773
|
+
};
|
|
774
|
+
const state = {
|
|
775
|
+
tombstones: tombstones.bits,
|
|
776
|
+
tombstoneCount: tombstones.count,
|
|
777
|
+
removedDocuments,
|
|
778
|
+
removedOccurrences,
|
|
779
|
+
removedSurfaces: words,
|
|
780
|
+
removedSpellings: words,
|
|
781
|
+
...emptyPendingRegions()
|
|
782
|
+
};
|
|
783
|
+
return {
|
|
784
|
+
frozen,
|
|
785
|
+
state,
|
|
786
|
+
read() {
|
|
787
|
+
if (occurrences === null) {
|
|
788
|
+
if (totals.length !== table.size * NUMBER_BYTES)
|
|
789
|
+
loadFailed(`The term totals of "${snapshot.key}" have the wrong length`);
|
|
790
|
+
const values = new Float64Array(table.size);
|
|
791
|
+
readPagedTotals(file, totals, 0, values, new BigUint64Array(3));
|
|
792
|
+
occurrences = values;
|
|
793
|
+
}
|
|
794
|
+
return {
|
|
795
|
+
growing: null,
|
|
796
|
+
frozen,
|
|
797
|
+
state,
|
|
798
|
+
occurrences
|
|
799
|
+
};
|
|
800
|
+
},
|
|
801
|
+
bytes: () => partialFields.byteLength + (occurrences?.byteLength ?? 0)
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// src/persistence/paged/removals.ts
|
|
806
|
+
var VALUE_BYTES2 = 4;
|
|
807
|
+
function createRemovalReader(file, removalOffsets, removals, tokenCount) {
|
|
808
|
+
const fault = new BigUint64Array(3);
|
|
809
|
+
return (ordinal) => {
|
|
810
|
+
const bounds = removalOffsets.bytesAt(ordinal * VALUE_BYTES2, VALUE_BYTES2 * 2);
|
|
811
|
+
const boundsView = new DataView(bounds.buffer, bounds.byteOffset, bounds.byteLength);
|
|
812
|
+
const from = boundsView.getUint32(0, true);
|
|
813
|
+
const to = boundsView.getUint32(VALUE_BYTES2, true);
|
|
814
|
+
if (to < from || to * VALUE_BYTES2 > removals.length || from % 2 !== 0 || to % 2 !== 0 || ordinal === 0 && from !== 0 || (ordinal + 2) * VALUE_BYTES2 === removalOffsets.length && to * VALUE_BYTES2 !== removals.length) {
|
|
815
|
+
loadFailed(`The removal entry of document ${ordinal} of "${file.path}" leaves its list`, { ordinal });
|
|
816
|
+
}
|
|
817
|
+
const values = new Uint32Array(to - from);
|
|
818
|
+
const offsetSection = removalOffsets.native;
|
|
819
|
+
const valueSection = removals.native;
|
|
820
|
+
if (offsetSection !== null && valueSection !== null && file.native !== null) {
|
|
821
|
+
const status = file.native.core.readPagedRemovals(
|
|
822
|
+
offsetSection,
|
|
823
|
+
valueSection,
|
|
824
|
+
file.cache.pageCache(),
|
|
825
|
+
ordinal,
|
|
826
|
+
tokenCount,
|
|
827
|
+
values,
|
|
828
|
+
fault
|
|
829
|
+
);
|
|
830
|
+
if (status < 0)
|
|
831
|
+
requireNativePage(
|
|
832
|
+
status,
|
|
833
|
+
file.path,
|
|
834
|
+
fault[1] === offsetSection.offset ? "removalOffsets" : "removals",
|
|
835
|
+
fault[2]
|
|
836
|
+
);
|
|
837
|
+
} else {
|
|
838
|
+
const entry = removals.bytesAt(from * VALUE_BYTES2, values.byteLength);
|
|
839
|
+
const view = new DataView(entry.buffer, entry.byteOffset, entry.byteLength);
|
|
840
|
+
let previous = -1;
|
|
841
|
+
for (let at = 0; at < values.length; at += 2) {
|
|
842
|
+
const token = view.getUint32(at * VALUE_BYTES2, true);
|
|
843
|
+
if (token >= tokenCount || token <= previous) {
|
|
844
|
+
loadFailed(`The removal entry of document ${ordinal} of "${file.path}" holds invalid token ${token}`, {
|
|
845
|
+
ordinal
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
previous = token;
|
|
849
|
+
values[at] = token;
|
|
850
|
+
values[at + 1] = view.getUint32((at + 1) * VALUE_BYTES2, true);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
return values;
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
// src/persistence/paged/counts.ts
|
|
858
|
+
function createPagedLiveCounts(file, snapshot, table, tombstones, removalOffsets, removals, totals) {
|
|
859
|
+
const removedDocuments = new Uint32Array(table.size);
|
|
860
|
+
const removedOccurrences = new Float64Array(table.size);
|
|
861
|
+
const dictionary = table.dictionary(removedDocuments);
|
|
862
|
+
const lookups = new TermLookup();
|
|
863
|
+
const fault = new BigUint64Array(3);
|
|
864
|
+
const paged = {
|
|
865
|
+
native() {
|
|
866
|
+
const offsets = removalOffsets.native;
|
|
867
|
+
const values = removals.native;
|
|
868
|
+
if (offsets === null || values === null || file.native === null) return null;
|
|
869
|
+
return { offsets, values, cache: file.cache.pageCache(), fault };
|
|
870
|
+
},
|
|
871
|
+
read: createRemovalReader(file, removalOffsets, removals, table.size),
|
|
872
|
+
fail(status) {
|
|
873
|
+
const section = fault[1] === removalOffsets.native?.offset ? "removalOffsets" : "removals";
|
|
874
|
+
return failNativePage(status, file.path, section, fault[2]);
|
|
875
|
+
}
|
|
876
|
+
};
|
|
877
|
+
let lengths = null;
|
|
878
|
+
const expansion = createPagedExpansionSource(
|
|
879
|
+
file,
|
|
880
|
+
snapshot,
|
|
881
|
+
table,
|
|
882
|
+
tombstones,
|
|
883
|
+
removedDocuments,
|
|
884
|
+
removedOccurrences,
|
|
885
|
+
totals
|
|
886
|
+
);
|
|
887
|
+
const pending = createFrozenRemovals(expansion.frozen, expansion.state, paged);
|
|
888
|
+
function documentFrequency(sortedIndex) {
|
|
889
|
+
pending.settle();
|
|
890
|
+
return lookups.frequency(dictionary, sortedIndex);
|
|
891
|
+
}
|
|
892
|
+
return {
|
|
893
|
+
expansionSource() {
|
|
894
|
+
pending.settle();
|
|
895
|
+
return expansion.read();
|
|
896
|
+
},
|
|
897
|
+
pendingRemovals: pending.pending,
|
|
898
|
+
settleRemovals: pending.settle,
|
|
899
|
+
dictionary() {
|
|
900
|
+
pending.settle();
|
|
901
|
+
return dictionary;
|
|
902
|
+
},
|
|
903
|
+
totalDocuments() {
|
|
904
|
+
return snapshot.documentCount - tombstones.size;
|
|
905
|
+
},
|
|
906
|
+
totalFieldLengths() {
|
|
907
|
+
return lengths ?? snapshot.totalFieldLengths;
|
|
908
|
+
},
|
|
909
|
+
documentFrequency,
|
|
910
|
+
removedOccurrences(sortedIndex) {
|
|
911
|
+
pending.settle();
|
|
912
|
+
return removedOccurrences[sortedIndex];
|
|
913
|
+
},
|
|
914
|
+
docFrequencies() {
|
|
915
|
+
pending.settle();
|
|
916
|
+
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
917
|
+
for (let at = 0; at < table.size; at++) {
|
|
918
|
+
const documents = documentFrequency(at);
|
|
919
|
+
if (documents > 0) frequencies[table.tokenAt(at)] = documents;
|
|
920
|
+
}
|
|
921
|
+
return frequencies;
|
|
922
|
+
},
|
|
923
|
+
prepareRemoval(ordinal, strings) {
|
|
924
|
+
const subtractTerms = pending.prepare(ordinal, strings?.fields ?? [], strings);
|
|
925
|
+
const nextLengths = lengths ?? { ...snapshot.totalFieldLengths };
|
|
926
|
+
return () => {
|
|
927
|
+
lengths ??= nextLengths;
|
|
928
|
+
for (let field = 0; field < snapshot.fieldLengthNames.length; field++) {
|
|
929
|
+
lengths[snapshot.fieldLengthNames[field]] -= snapshot.fieldLengthColumns[field][ordinal];
|
|
930
|
+
}
|
|
931
|
+
subtractTerms();
|
|
932
|
+
};
|
|
933
|
+
},
|
|
934
|
+
bytes() {
|
|
935
|
+
return removedDocuments.byteLength + removedOccurrences.byteLength + expansion.bytes() + pending.bytes();
|
|
936
|
+
}
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
function deflateBlock(raw) {
|
|
940
|
+
return deflateRawSync(raw);
|
|
941
|
+
}
|
|
942
|
+
function inflateBlock(compressed, rawLength) {
|
|
943
|
+
return inflateRawSync(compressed, { maxOutputLength: rawLength });
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// src/persistence/paged/documents.ts
|
|
947
|
+
function createDocumentBlockReader(snapshot, documents, cache, fileNumber) {
|
|
948
|
+
const { documentBlocks, documentCount, key } = snapshot;
|
|
949
|
+
const view = new DataView(documentBlocks.buffer, documentBlocks.byteOffset, documentBlocks.byteLength);
|
|
950
|
+
const blockCount = documentBlocks.byteLength / DOCUMENT_BLOCK_BYTES;
|
|
951
|
+
function firstOrdinalOf(block) {
|
|
952
|
+
return view.getUint32(block * DOCUMENT_BLOCK_BYTES, true);
|
|
953
|
+
}
|
|
954
|
+
function blockOf(ordinal) {
|
|
955
|
+
let low = 0;
|
|
956
|
+
let high = blockCount;
|
|
957
|
+
while (high - low > 1) {
|
|
958
|
+
const middle = low + high >>> 1;
|
|
959
|
+
if (firstOrdinalOf(middle) <= ordinal) low = middle;
|
|
960
|
+
else high = middle;
|
|
961
|
+
}
|
|
962
|
+
return low;
|
|
963
|
+
}
|
|
964
|
+
function inflate(block) {
|
|
965
|
+
const place = block * DOCUMENT_BLOCK_BYTES;
|
|
966
|
+
const offset = Number(view.getBigUint64(place + 4, true));
|
|
967
|
+
const length = view.getUint32(place + 12, true);
|
|
968
|
+
const rawLength = view.getUint32(place + 16, true);
|
|
969
|
+
if (rawLength > MAX_DOCUMENT_BLOCK_RAW_BYTES) {
|
|
970
|
+
loadFailed(`Document block ${block} of "${key}" claims ${rawLength} bytes`, { block, rawLength });
|
|
971
|
+
}
|
|
972
|
+
const compressed = documents.bytesAt(offset, length);
|
|
973
|
+
let raw;
|
|
974
|
+
try {
|
|
975
|
+
raw = inflateBlock(compressed, rawLength);
|
|
976
|
+
} catch (error) {
|
|
977
|
+
return loadFailed(
|
|
978
|
+
`Document block ${block} of "${key}" fails to inflate: ${error instanceof Error ? error.message : String(error)}`,
|
|
979
|
+
{ block }
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
if (raw.byteLength !== rawLength) {
|
|
983
|
+
loadFailed(`Document block ${block} of "${key}" inflates to ${raw.byteLength} bytes of ${rawLength}`, { block });
|
|
984
|
+
}
|
|
985
|
+
return raw;
|
|
986
|
+
}
|
|
987
|
+
function startsOf(block, raw) {
|
|
988
|
+
const first = firstOrdinalOf(block);
|
|
989
|
+
const end = block + 1 < blockCount ? firstOrdinalOf(block + 1) : documentCount;
|
|
990
|
+
const starts = new Uint32Array(end - first + 1);
|
|
991
|
+
const bytes = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
992
|
+
let cursor = 0;
|
|
993
|
+
for (let at = 0; at < end - first; at++) {
|
|
994
|
+
starts[at] = cursor;
|
|
995
|
+
const next = cursor < raw.byteLength ? endOfValue(bytes, cursor) : null;
|
|
996
|
+
if (next === null)
|
|
997
|
+
loadFailed(`Document block ${block} of "${key}" holds a broken document at ${at}`, { block, at });
|
|
998
|
+
cursor = next;
|
|
999
|
+
}
|
|
1000
|
+
starts[end - first] = cursor;
|
|
1001
|
+
if (cursor !== raw.byteLength)
|
|
1002
|
+
loadFailed(`Document block ${block} of "${key}" holds bytes past its documents`, { block });
|
|
1003
|
+
return starts;
|
|
1004
|
+
}
|
|
1005
|
+
function loadBlock(block) {
|
|
1006
|
+
const held = cache.get(fileNumber, CACHED_DOCUMENT_BLOCK, block);
|
|
1007
|
+
if (held !== void 0) return held;
|
|
1008
|
+
const bytes = inflate(block);
|
|
1009
|
+
const loaded = { bytes, starts: startsOf(block, bytes) };
|
|
1010
|
+
cache.put(fileNumber, CACHED_DOCUMENT_BLOCK, block, loaded, bytes.byteLength + loaded.starts.byteLength);
|
|
1011
|
+
return loaded;
|
|
1012
|
+
}
|
|
1013
|
+
return {
|
|
1014
|
+
encodedDocumentAt(ordinal) {
|
|
1015
|
+
if (ordinal < 0 || ordinal >= documentCount) {
|
|
1016
|
+
loadFailed(`Document ${ordinal} lies outside "${key}", which holds ${documentCount}`, { ordinal });
|
|
1017
|
+
}
|
|
1018
|
+
const block = blockOf(ordinal);
|
|
1019
|
+
const { bytes, starts } = loadBlock(block);
|
|
1020
|
+
const row = ordinal - firstOrdinalOf(block);
|
|
1021
|
+
return bytes.subarray(starts[row], starts[row + 1]);
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
// src/persistence/paged/ids.ts
|
|
1027
|
+
var VALUE_BYTES3 = 4;
|
|
1028
|
+
var decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
1029
|
+
function uint32At(reader, index) {
|
|
1030
|
+
const bytes = reader.bytesAt(index * VALUE_BYTES3, VALUE_BYTES3);
|
|
1031
|
+
return new DataView(bytes.buffer, bytes.byteOffset, VALUE_BYTES3).getUint32(0, true);
|
|
1032
|
+
}
|
|
1033
|
+
function createPagedIdTable(snapshot, ids, idOffsets, idOrder) {
|
|
1034
|
+
const count = snapshot.documentCount;
|
|
1035
|
+
const { idSampleIds, idSamplePositions, key } = snapshot;
|
|
1036
|
+
let sampleBytes = idSamplePositions.byteLength;
|
|
1037
|
+
for (const id of idSampleIds) sampleBytes += id.length;
|
|
1038
|
+
function idAt(ordinal) {
|
|
1039
|
+
const from = uint32At(idOffsets, ordinal);
|
|
1040
|
+
const to = uint32At(idOffsets, ordinal + 1);
|
|
1041
|
+
if (to < from || to > ids.length || ordinal === 0 && from !== 0 || ordinal + 1 === count && to !== ids.length)
|
|
1042
|
+
loadFailed(`The ID of document ${ordinal} of "${key}" leaves its list`, { ordinal });
|
|
1043
|
+
try {
|
|
1044
|
+
return decoder.decode(ids.bytesAt(from, to - from));
|
|
1045
|
+
} catch {
|
|
1046
|
+
return loadFailed(`The ID of document ${ordinal} of "${key}" is not valid UTF-8`, { ordinal });
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
function ordinalAtPosition(position) {
|
|
1050
|
+
const ordinal = uint32At(idOrder, position);
|
|
1051
|
+
if (ordinal >= count) loadFailed(`idOrder of "${key}" names document ${ordinal} of ${count}`, { position });
|
|
1052
|
+
return ordinal;
|
|
1053
|
+
}
|
|
1054
|
+
function windowOf(docId) {
|
|
1055
|
+
let low = 0;
|
|
1056
|
+
let high = idSampleIds.length;
|
|
1057
|
+
while (low < high) {
|
|
1058
|
+
const middle = low + high >>> 1;
|
|
1059
|
+
if (compareCodePoints(idSampleIds[middle], docId) <= 0) low = middle + 1;
|
|
1060
|
+
else high = middle;
|
|
1061
|
+
}
|
|
1062
|
+
const from = low === 0 ? 0 : idSamplePositions[low - 1];
|
|
1063
|
+
const to = low < idSamplePositions.length ? idSamplePositions[low] + 1 : count;
|
|
1064
|
+
return { from, to: Math.min(to, count) };
|
|
1065
|
+
}
|
|
1066
|
+
return {
|
|
1067
|
+
count,
|
|
1068
|
+
bytes: sampleBytes,
|
|
1069
|
+
idAt,
|
|
1070
|
+
ordinalOf(docId) {
|
|
1071
|
+
if (count === 0) return -1;
|
|
1072
|
+
const window = windowOf(docId);
|
|
1073
|
+
let low = window.from;
|
|
1074
|
+
let high = window.to;
|
|
1075
|
+
while (low < high) {
|
|
1076
|
+
const middle = low + high >>> 1;
|
|
1077
|
+
const ordinal = ordinalAtPosition(middle);
|
|
1078
|
+
const order = compareCodePoints(idAt(ordinal), docId);
|
|
1079
|
+
if (order === 0) return ordinal;
|
|
1080
|
+
if (order < 0) low = middle + 1;
|
|
1081
|
+
else high = middle;
|
|
1082
|
+
}
|
|
1083
|
+
return -1;
|
|
1084
|
+
},
|
|
1085
|
+
sortedOrdinalAt: ordinalAtPosition,
|
|
1086
|
+
collectSortedIds(excluded) {
|
|
1087
|
+
const kept = [];
|
|
1088
|
+
for (let position = 0; position < count; position++) {
|
|
1089
|
+
const ordinal = ordinalAtPosition(position);
|
|
1090
|
+
if (!excluded(ordinal)) kept.push(idAt(ordinal));
|
|
1091
|
+
}
|
|
1092
|
+
return kept;
|
|
1093
|
+
}
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
// src/persistence/paged/lists.ts
|
|
1098
|
+
var LITTLE_ENDIAN_PLATFORM2 = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
1099
|
+
function swappedInPlace(bytes, width) {
|
|
1100
|
+
for (let at = 0; at + width <= bytes.length; at += width) {
|
|
1101
|
+
for (let low = 0, high = width - 1; low < high; low++, high--) {
|
|
1102
|
+
const held = bytes[at + low];
|
|
1103
|
+
bytes[at + low] = bytes[at + high];
|
|
1104
|
+
bytes[at + high] = held;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
function fromLittleEndian(bytes, width) {
|
|
1109
|
+
if (!LITTLE_ENDIAN_PLATFORM2) swappedInPlace(bytes, width);
|
|
1110
|
+
}
|
|
1111
|
+
function requireListRules(list, snapshot, token) {
|
|
1112
|
+
const { docIds, fieldNameIndices } = list;
|
|
1113
|
+
const documents = snapshot.documentCount;
|
|
1114
|
+
const fields = snapshot.fieldNames.length;
|
|
1115
|
+
for (let row = 0; row < docIds.length; row++) {
|
|
1116
|
+
if (docIds[row] >= documents) {
|
|
1117
|
+
loadFailed(`Token ${token} of "${snapshot.key}" names document ${docIds[row]} of ${documents}`, { token, row });
|
|
1118
|
+
}
|
|
1119
|
+
if (fieldNameIndices[row] >= fields) {
|
|
1120
|
+
loadFailed(`Token ${token} of "${snapshot.key}" names field ${fieldNameIndices[row]} of ${fields}`, {
|
|
1121
|
+
token,
|
|
1122
|
+
row
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
if (row === 0) continue;
|
|
1126
|
+
const before = docIds[row - 1];
|
|
1127
|
+
if (docIds[row] < before || docIds[row] === before && fieldNameIndices[row] <= fieldNameIndices[row - 1]) {
|
|
1128
|
+
loadFailed(`The list of token ${token} of "${snapshot.key}" descends at posting ${row}`, { token, row });
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
function createListReader(file, snapshot, sectionReader) {
|
|
1133
|
+
const { postingOffsets } = snapshot;
|
|
1134
|
+
const postings = sectionReader("postings");
|
|
1135
|
+
const positionOffsets = sectionReader("positionOffsets");
|
|
1136
|
+
const positions = sectionReader("positions");
|
|
1137
|
+
const totals = sectionReader("tokenTotals");
|
|
1138
|
+
const { cache, fileNumber } = file;
|
|
1139
|
+
const fault = new BigUint64Array(3);
|
|
1140
|
+
const total = new Float64Array(1);
|
|
1141
|
+
function listAt(token) {
|
|
1142
|
+
const held = cache.get(fileNumber, CACHED_LIST, token);
|
|
1143
|
+
if (held !== void 0) return held;
|
|
1144
|
+
if (postings === null) return loadFailed(`The segment "${snapshot.key}" holds no postings`);
|
|
1145
|
+
const start = postingOffsets[token];
|
|
1146
|
+
const count = postingOffsets[token + 1] - start;
|
|
1147
|
+
const buffer = new ArrayBuffer(count * POSTING_WIDTH_BYTES);
|
|
1148
|
+
const bytes = new Uint8Array(buffer);
|
|
1149
|
+
const descriptor = postings.native;
|
|
1150
|
+
if (descriptor !== null && file.native !== null) {
|
|
1151
|
+
const status = file.native.core.readPagedPostings(
|
|
1152
|
+
descriptor,
|
|
1153
|
+
cache.pageCache(),
|
|
1154
|
+
start,
|
|
1155
|
+
count,
|
|
1156
|
+
snapshot.documentCount,
|
|
1157
|
+
snapshot.fieldNames.length,
|
|
1158
|
+
bytes,
|
|
1159
|
+
fault
|
|
1160
|
+
);
|
|
1161
|
+
requireNativePage(status, file.path, "postings", fault[2]);
|
|
1162
|
+
} else {
|
|
1163
|
+
postings.read(start * POSTING_WIDTH_BYTES, bytes);
|
|
1164
|
+
}
|
|
1165
|
+
if (!LITTLE_ENDIAN_PLATFORM2) {
|
|
1166
|
+
swappedInPlace(bytes.subarray(0, count * 4), 4);
|
|
1167
|
+
swappedInPlace(bytes.subarray(count * 4), 2);
|
|
1168
|
+
}
|
|
1169
|
+
const list = {
|
|
1170
|
+
docIds: new Uint32Array(buffer, 0, count),
|
|
1171
|
+
termFrequencies: new Uint16Array(buffer, count * 4, count),
|
|
1172
|
+
fieldNameIndices: new Uint16Array(buffer, count * 6, count)
|
|
1173
|
+
};
|
|
1174
|
+
if (descriptor === null || file.native === null) requireListRules(list, snapshot, token);
|
|
1175
|
+
cache.put(fileNumber, CACHED_LIST, token, list, buffer.byteLength);
|
|
1176
|
+
return list;
|
|
1177
|
+
}
|
|
1178
|
+
function positionsAt(token) {
|
|
1179
|
+
if (positionOffsets === null || positions === null) return null;
|
|
1180
|
+
const held = cache.get(fileNumber, CACHED_POSITIONS, token);
|
|
1181
|
+
if (held !== void 0) return held;
|
|
1182
|
+
const start = postingOffsets[token];
|
|
1183
|
+
const count = postingOffsets[token + 1] - start;
|
|
1184
|
+
const offsetBytes = new Uint8Array((count + 1) * POSITION_WIDTH_BYTES);
|
|
1185
|
+
positionOffsets.read(start * POSITION_WIDTH_BYTES, offsetBytes);
|
|
1186
|
+
fromLittleEndian(offsetBytes, POSITION_WIDTH_BYTES);
|
|
1187
|
+
const absolute = new Uint32Array(offsetBytes.buffer);
|
|
1188
|
+
const first = absolute[0];
|
|
1189
|
+
const end = absolute[count];
|
|
1190
|
+
if (end < first || end * POSITION_WIDTH_BYTES > positions.length || start === 0 && first !== 0 || (start + count + 1) * POSITION_WIDTH_BYTES === positionOffsets.length && end * POSITION_WIDTH_BYTES !== positions.length)
|
|
1191
|
+
loadFailed(`The positions of token ${token} of "${snapshot.key}" leave their section`, { token });
|
|
1192
|
+
const offsets = new Uint32Array(count + 1);
|
|
1193
|
+
const valueBytes = new Uint8Array((end - first) * POSITION_WIDTH_BYTES);
|
|
1194
|
+
const loaded = { offsets, values: new Uint32Array(valueBytes.buffer) };
|
|
1195
|
+
const offsetSection = positionOffsets.native;
|
|
1196
|
+
const valueSection = positions.native;
|
|
1197
|
+
if (offsetSection !== null && valueSection !== null && file.native !== null) {
|
|
1198
|
+
const status = file.native.core.readPagedPositions(
|
|
1199
|
+
offsetSection,
|
|
1200
|
+
valueSection,
|
|
1201
|
+
cache.pageCache(),
|
|
1202
|
+
start,
|
|
1203
|
+
count,
|
|
1204
|
+
offsets,
|
|
1205
|
+
loaded.values,
|
|
1206
|
+
fault
|
|
1207
|
+
);
|
|
1208
|
+
requireNativePage(
|
|
1209
|
+
status,
|
|
1210
|
+
file.path,
|
|
1211
|
+
fault[1] === offsetSection.offset ? "positionOffsets" : "positions",
|
|
1212
|
+
fault[2]
|
|
1213
|
+
);
|
|
1214
|
+
} else {
|
|
1215
|
+
for (let row = 0; row <= count; row++) {
|
|
1216
|
+
if (absolute[row] < first || absolute[row] > end || row > 0 && absolute[row] < absolute[row - 1]) {
|
|
1217
|
+
loadFailed(`The position offsets of token ${token} of "${snapshot.key}" leave their list`, { token, row });
|
|
1218
|
+
}
|
|
1219
|
+
offsets[row] = absolute[row] - first;
|
|
1220
|
+
}
|
|
1221
|
+
positions.read(first * POSITION_WIDTH_BYTES, valueBytes);
|
|
1222
|
+
fromLittleEndian(valueBytes, POSITION_WIDTH_BYTES);
|
|
1223
|
+
for (let row = 0; row < count; row++) {
|
|
1224
|
+
for (let at = offsets[row] + 1; at < offsets[row + 1]; at++) {
|
|
1225
|
+
if (loaded.values[at] < loaded.values[at - 1])
|
|
1226
|
+
loadFailed(`The positions of token ${token} of "${snapshot.key}" descend`, { token, row });
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
cache.put(fileNumber, CACHED_POSITIONS, token, loaded, offsetBytes.byteLength + valueBytes.byteLength);
|
|
1231
|
+
return loaded;
|
|
1232
|
+
}
|
|
1233
|
+
function totalAt(token) {
|
|
1234
|
+
if (totals === null) return loadFailed(`The segment "${snapshot.key}" holds no token totals`);
|
|
1235
|
+
readPagedTotals(file, totals, token, total, fault);
|
|
1236
|
+
return total[0];
|
|
1237
|
+
}
|
|
1238
|
+
function listedIndexOf(token) {
|
|
1239
|
+
const { blockLists } = snapshot;
|
|
1240
|
+
let low = 0;
|
|
1241
|
+
let high = blockLists.length;
|
|
1242
|
+
while (low < high) {
|
|
1243
|
+
const middle = low + high >>> 1;
|
|
1244
|
+
if (blockLists[middle] < token) low = middle + 1;
|
|
1245
|
+
else high = middle;
|
|
1246
|
+
}
|
|
1247
|
+
return low < blockLists.length && blockLists[low] === token ? low : -1;
|
|
1248
|
+
}
|
|
1249
|
+
function storedBoundsAt(token) {
|
|
1250
|
+
const listed = listedIndexOf(token);
|
|
1251
|
+
if (listed < 0) return null;
|
|
1252
|
+
const first = snapshot.blockListOffsets[listed];
|
|
1253
|
+
const blockCount = snapshot.blockListOffsets[listed + 1] - first;
|
|
1254
|
+
const { blockSummaries } = snapshot;
|
|
1255
|
+
const view = new DataView(blockSummaries.buffer, blockSummaries.byteOffset, blockSummaries.byteLength);
|
|
1256
|
+
const bounds = {
|
|
1257
|
+
structureRevision: 0,
|
|
1258
|
+
blockCount,
|
|
1259
|
+
entryEnd: new Int32Array(blockCount),
|
|
1260
|
+
maxTermFrequency: new Int32Array(blockCount),
|
|
1261
|
+
minFieldLength: new Float64Array(blockCount),
|
|
1262
|
+
maxEntriesPerDocument: new Int32Array(blockCount),
|
|
1263
|
+
documentCount: new Int32Array(blockCount)
|
|
1264
|
+
};
|
|
1265
|
+
const length = postingOffsets[token + 1] - postingOffsets[token];
|
|
1266
|
+
for (let block = 0; block < blockCount; block++) {
|
|
1267
|
+
const place = (first + block) * BLOCK_SUMMARY_BYTES;
|
|
1268
|
+
bounds.entryEnd[block] = view.getUint32(place, true);
|
|
1269
|
+
bounds.maxTermFrequency[block] = view.getUint16(place + 4, true);
|
|
1270
|
+
bounds.minFieldLength[block] = view.getFloat64(place + 6, true);
|
|
1271
|
+
bounds.maxEntriesPerDocument[block] = view.getUint32(place + 14, true);
|
|
1272
|
+
bounds.documentCount[block] = view.getUint32(place + 18, true);
|
|
1273
|
+
const before = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
1274
|
+
if (bounds.entryEnd[block] <= before || bounds.entryEnd[block] > length) {
|
|
1275
|
+
loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" ends outside its list`, { token, block });
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
if (blockCount === 0 || bounds.entryEnd[blockCount - 1] !== length) {
|
|
1279
|
+
loadFailed(`The blocks of token ${token} of "${snapshot.key}" fail to cover its list`, { token });
|
|
1280
|
+
}
|
|
1281
|
+
return bounds;
|
|
1282
|
+
}
|
|
1283
|
+
return { listAt, positionsAt, totalAt, storedBoundsAt };
|
|
1284
|
+
}
|
|
1285
|
+
function createPagedPostingViews(snapshot, lists, tombstones, counts) {
|
|
1286
|
+
const { postingOffsets, tracksPositions } = snapshot;
|
|
1287
|
+
const views = /* @__PURE__ */ new Map();
|
|
1288
|
+
function buildView(token) {
|
|
1289
|
+
const length = postingOffsets[token + 1] - postingOffsets[token];
|
|
1290
|
+
return {
|
|
1291
|
+
length,
|
|
1292
|
+
get docIds() {
|
|
1293
|
+
return lists.listAt(token).docIds;
|
|
1294
|
+
},
|
|
1295
|
+
get termFrequencies() {
|
|
1296
|
+
return lists.listAt(token).termFrequencies;
|
|
1297
|
+
},
|
|
1298
|
+
get fieldNameIndices() {
|
|
1299
|
+
return lists.listAt(token).fieldNameIndices;
|
|
1300
|
+
},
|
|
1301
|
+
get positions() {
|
|
1302
|
+
const loaded = lists.positionsAt(token);
|
|
1303
|
+
if (loaded === null) return null;
|
|
1304
|
+
const rows = new Array(length);
|
|
1305
|
+
for (let row = 0; row < length; row++) {
|
|
1306
|
+
rows[row] = [...loaded.values.subarray(loaded.offsets[row], loaded.offsets[row + 1])];
|
|
1307
|
+
}
|
|
1308
|
+
return rows;
|
|
1309
|
+
},
|
|
1310
|
+
holdsPositions: tracksPositions,
|
|
1311
|
+
positionCountAt(row) {
|
|
1312
|
+
const loaded = lists.positionsAt(token);
|
|
1313
|
+
return loaded === null ? 0 : loaded.offsets[row + 1] - loaded.offsets[row];
|
|
1314
|
+
},
|
|
1315
|
+
get positionStore() {
|
|
1316
|
+
return lists.positionsAt(token)?.values ?? null;
|
|
1317
|
+
},
|
|
1318
|
+
positionStartAt(row) {
|
|
1319
|
+
return lists.positionsAt(token)?.offsets[row] ?? 0;
|
|
1320
|
+
},
|
|
1321
|
+
docIdSet: {
|
|
1322
|
+
get size() {
|
|
1323
|
+
return counts.documentFrequency(token);
|
|
1324
|
+
}
|
|
1325
|
+
},
|
|
1326
|
+
deletedDocs: tombstones,
|
|
1327
|
+
get totalTermFrequency() {
|
|
1328
|
+
return lists.totalAt(token) - counts.removedOccurrences(token);
|
|
1329
|
+
},
|
|
1330
|
+
get structureRevision() {
|
|
1331
|
+
return tombstones.revision;
|
|
1332
|
+
},
|
|
1333
|
+
ordered: true,
|
|
1334
|
+
storedBlockBounds() {
|
|
1335
|
+
return tombstones.size === 0 ? lists.storedBoundsAt(token) : null;
|
|
1336
|
+
}
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
return {
|
|
1340
|
+
viewAt(payloadSlot) {
|
|
1341
|
+
let view = views.get(payloadSlot);
|
|
1342
|
+
if (view === void 0) {
|
|
1343
|
+
if (views.size >= LIST_VIEWS_HELD_PER_SEGMENT) views.clear();
|
|
1344
|
+
view = buildView(payloadSlot);
|
|
1345
|
+
views.set(payloadSlot, view);
|
|
1346
|
+
}
|
|
1347
|
+
return view;
|
|
1348
|
+
},
|
|
1349
|
+
viewReadOnce(payloadSlot) {
|
|
1350
|
+
return buildView(payloadSlot);
|
|
1351
|
+
}
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
// src/persistence/paged/surface-candidates.ts
|
|
1356
|
+
var surfaceDecoder = new TextDecoder("utf-8", { ignoreBOM: true });
|
|
1357
|
+
var surfaceEncoder = new TextEncoder();
|
|
1358
|
+
function surfaceCandidatesInTable(table, prefix) {
|
|
1359
|
+
if (prefix.length === 0) return [];
|
|
1360
|
+
const wanted = surfaceEncoder.encode(prefix);
|
|
1361
|
+
const { blob, offsets, counts } = table;
|
|
1362
|
+
const found = [];
|
|
1363
|
+
for (let entry = 0; entry < counts.length; entry++) {
|
|
1364
|
+
const start = offsets[2 * entry];
|
|
1365
|
+
const tokenStart = offsets[2 * entry + 1];
|
|
1366
|
+
if (tokenStart - start < wanted.length) continue;
|
|
1367
|
+
let at = 0;
|
|
1368
|
+
while (at < wanted.length && blob[start + at] === wanted[at]) at += 1;
|
|
1369
|
+
if (at < wanted.length || counts[entry] === 0) continue;
|
|
1370
|
+
found.push({
|
|
1371
|
+
surface: surfaceDecoder.decode(blob.subarray(start, tokenStart)),
|
|
1372
|
+
token: surfaceDecoder.decode(blob.subarray(tokenStart, offsets[2 * entry + 2])),
|
|
1373
|
+
occurrences: counts[entry]
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
return found;
|
|
1377
|
+
}
|
|
1378
|
+
function createSurfaceTableReader(read) {
|
|
1379
|
+
let table;
|
|
1380
|
+
return {
|
|
1381
|
+
candidatesForPrefix(prefix) {
|
|
1382
|
+
if (table === void 0) table = read();
|
|
1383
|
+
return table === null ? [] : surfaceCandidatesInTable(table, prefix);
|
|
1384
|
+
},
|
|
1385
|
+
heldBytes() {
|
|
1386
|
+
if (table === void 0 || table === null) return 0;
|
|
1387
|
+
return table.blob.byteLength + table.offsets.byteLength + table.counts.byteLength;
|
|
1388
|
+
}
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
// src/persistence/paged/term-frequencies.ts
|
|
1393
|
+
function termFrequenciesAcross(states) {
|
|
1394
|
+
function countOf(token) {
|
|
1395
|
+
if (typeof token !== "string") return 0;
|
|
1396
|
+
let count = 0;
|
|
1397
|
+
for (const state of states()) count += state.invertedIdx.documentFrequency(token);
|
|
1398
|
+
return count;
|
|
1399
|
+
}
|
|
1400
|
+
return new Proxy(/* @__PURE__ */ Object.create(null), {
|
|
1401
|
+
get: (_target, token) => countOf(token) || void 0,
|
|
1402
|
+
has: (_target, token) => countOf(token) > 0,
|
|
1403
|
+
getOwnPropertyDescriptor(_target, token) {
|
|
1404
|
+
const value = countOf(token);
|
|
1405
|
+
return value > 0 ? { value, enumerable: true, configurable: true, writable: false } : void 0;
|
|
1406
|
+
},
|
|
1407
|
+
ownKeys: () => Object.keys(mergeDocFrequencies(states()))
|
|
1408
|
+
});
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
// src/persistence/paged/open-segment.ts
|
|
1412
|
+
function onlyMergesThroughTheFile(column) {
|
|
1413
|
+
throw new NarsilError(
|
|
1414
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
1415
|
+
`A paged segment holds "${column}" in its file, so a caller must read it through the segment's paged access`,
|
|
1416
|
+
{ column }
|
|
1417
|
+
);
|
|
1418
|
+
}
|
|
1419
|
+
function readUint64(section, ordinal) {
|
|
1420
|
+
const bytes = section.bytesAt(ordinal * 8, 8);
|
|
1421
|
+
const value = new DataView(bytes.buffer, bytes.byteOffset, 8).getBigUint64(0, true);
|
|
1422
|
+
if (value > BigInt(Number.MAX_SAFE_INTEGER)) loadFailed(`A write record of document ${ordinal} lies above 2^53 - 1`);
|
|
1423
|
+
return Number(value);
|
|
1424
|
+
}
|
|
1425
|
+
function createPagedFrozenSegment(snapshot, cache = readCacheOfThisThread()) {
|
|
1426
|
+
const file = openPagedFile(snapshot.path, cache);
|
|
1427
|
+
try {
|
|
1428
|
+
return assembleOpenedSegment(file, snapshot, cache);
|
|
1429
|
+
} catch (error) {
|
|
1430
|
+
closePagedFile(file);
|
|
1431
|
+
throw error;
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
function assembleOpenedSegment(file, snapshot, cache) {
|
|
1435
|
+
const sections = snapshot.sections;
|
|
1436
|
+
function reader(name) {
|
|
1437
|
+
const section = sectionNamed(sections, name);
|
|
1438
|
+
return section === void 0 ? null : createSectionReader(file, section);
|
|
1439
|
+
}
|
|
1440
|
+
function required(name) {
|
|
1441
|
+
const section = reader(name);
|
|
1442
|
+
if (section === null) loadFailed(`The segment "${snapshot.key}" holds no section "${name}"`, { section: name });
|
|
1443
|
+
return section;
|
|
1444
|
+
}
|
|
1445
|
+
const tokenTable = wrapFrozenTokenTable(snapshot.tokenTable);
|
|
1446
|
+
const idTable = createPagedIdTable(snapshot, required("ids"), required("idOffsets"), required("idOrder"));
|
|
1447
|
+
const lists = createListReader(file, snapshot, reader);
|
|
1448
|
+
const documents = createDocumentBlockReader(snapshot, required("documents"), cache, file.fileNumber);
|
|
1449
|
+
const removalOffsets = required("removalOffsets");
|
|
1450
|
+
const removals = required("removals");
|
|
1451
|
+
const totals = required("tokenTotals");
|
|
1452
|
+
const surfaceSection = reader("surfaceForms");
|
|
1453
|
+
const seqNos = reader("seqNos");
|
|
1454
|
+
const primaryTerms = reader("primaryTerms");
|
|
1455
|
+
if (seqNos === null !== (primaryTerms === null))
|
|
1456
|
+
loadFailed(`The segment "${snapshot.key}" holds an incomplete write record`);
|
|
1457
|
+
const writeRecord = new BigUint64Array(2);
|
|
1458
|
+
const fault = new BigUint64Array(3);
|
|
1459
|
+
const resident = residentBytesOf(snapshot);
|
|
1460
|
+
function readSurfaceTable() {
|
|
1461
|
+
if (surfaceSection === null) return null;
|
|
1462
|
+
let raw;
|
|
1463
|
+
try {
|
|
1464
|
+
raw = decode(surfaceSection.whole());
|
|
1465
|
+
} catch (error) {
|
|
1466
|
+
loadFailed(
|
|
1467
|
+
`Section "surfaceForms" of "${snapshot.key}" is not MessagePack: ${error instanceof Error ? error.message : String(error)}`
|
|
1468
|
+
);
|
|
1469
|
+
}
|
|
1470
|
+
return decodeSurfaceForms(raw, { documentCount: snapshot.documentCount, shared: false });
|
|
1471
|
+
}
|
|
1472
|
+
const access = {
|
|
1473
|
+
key: snapshot.key,
|
|
1474
|
+
snapshot,
|
|
1475
|
+
writeRecordsHeld: seqNos !== null && primaryTerms !== null,
|
|
1476
|
+
payloadChecked: false,
|
|
1477
|
+
listAt: lists.listAt,
|
|
1478
|
+
positionsAt: lists.positionsAt,
|
|
1479
|
+
encodedDocumentAt: documents.encodedDocumentAt,
|
|
1480
|
+
writeRecordAt(ordinal) {
|
|
1481
|
+
if (seqNos === null || primaryTerms === null) return { seqNo: 0, primaryTerm: 0 };
|
|
1482
|
+
const sequences = seqNos.native;
|
|
1483
|
+
const terms = primaryTerms.native;
|
|
1484
|
+
let seqNo;
|
|
1485
|
+
let primaryTerm;
|
|
1486
|
+
if (sequences !== null && terms !== null && file.native !== null) {
|
|
1487
|
+
const status = file.native.core.readPagedWriteRecord(
|
|
1488
|
+
sequences,
|
|
1489
|
+
terms,
|
|
1490
|
+
cache.pageCache(),
|
|
1491
|
+
ordinal,
|
|
1492
|
+
writeRecord,
|
|
1493
|
+
fault
|
|
1494
|
+
);
|
|
1495
|
+
requireNativePage(status, file.path, fault[1] === sequences.offset ? "seqNos" : "primaryTerms", fault[2]);
|
|
1496
|
+
if (writeRecord[0] > BigInt(Number.MAX_SAFE_INTEGER) || writeRecord[1] > BigInt(Number.MAX_SAFE_INTEGER))
|
|
1497
|
+
loadFailed(`A write record of document ${ordinal} lies above 2^53 - 1`);
|
|
1498
|
+
seqNo = Number(writeRecord[0]);
|
|
1499
|
+
primaryTerm = Number(writeRecord[1]);
|
|
1500
|
+
} else {
|
|
1501
|
+
seqNo = readUint64(seqNos, ordinal);
|
|
1502
|
+
primaryTerm = readUint64(primaryTerms, ordinal);
|
|
1503
|
+
}
|
|
1504
|
+
if (seqNo === 0 && primaryTerm !== 0)
|
|
1505
|
+
loadFailed(`Document ${ordinal} holds a primary term without a sequence number`);
|
|
1506
|
+
return { seqNo, primaryTerm };
|
|
1507
|
+
},
|
|
1508
|
+
heldBytes(segment) {
|
|
1509
|
+
return resident + cachedFileBytes(file) + segment.removalBytes() + segment.surfaces.heldBytes() + (segment.vectorOrdinalMaps?.bytes ?? 0);
|
|
1510
|
+
},
|
|
1511
|
+
termFrequenciesAcross,
|
|
1512
|
+
close() {
|
|
1513
|
+
closePagedFile(file);
|
|
1514
|
+
},
|
|
1515
|
+
relocate(rewrite) {
|
|
1516
|
+
relocatePagedFile(file, rewrite(file.path));
|
|
1517
|
+
snapshot.path = file.path;
|
|
1518
|
+
}
|
|
1519
|
+
};
|
|
1520
|
+
const source = {
|
|
1521
|
+
documentCount: snapshot.documentCount,
|
|
1522
|
+
fieldNames: snapshot.fieldNames,
|
|
1523
|
+
fieldLengthNames: snapshot.fieldLengthNames,
|
|
1524
|
+
fieldLengthColumns: snapshot.fieldLengthColumns,
|
|
1525
|
+
totalFieldLengths: snapshot.totalFieldLengths,
|
|
1526
|
+
numeric: snapshot.numeric,
|
|
1527
|
+
sizes: snapshot.sizes,
|
|
1528
|
+
boolean: snapshot.boolean,
|
|
1529
|
+
enums: snapshot.enums,
|
|
1530
|
+
geo: snapshot.geo,
|
|
1531
|
+
pattern: snapshot.pattern,
|
|
1532
|
+
tokenTable,
|
|
1533
|
+
idTable,
|
|
1534
|
+
documentTable: null,
|
|
1535
|
+
postingOffsets: snapshot.postingOffsets,
|
|
1536
|
+
get postingDocIds() {
|
|
1537
|
+
return onlyMergesThroughTheFile("postingDocIds");
|
|
1538
|
+
},
|
|
1539
|
+
get postingFrequencies() {
|
|
1540
|
+
return onlyMergesThroughTheFile("postingFrequencies");
|
|
1541
|
+
},
|
|
1542
|
+
get postingFieldIndices() {
|
|
1543
|
+
return onlyMergesThroughTheFile("postingFieldIndices");
|
|
1544
|
+
},
|
|
1545
|
+
get positionOffsets() {
|
|
1546
|
+
return snapshot.tracksPositions ? onlyMergesThroughTheFile("positionOffsets") : null;
|
|
1547
|
+
},
|
|
1548
|
+
get positionValues() {
|
|
1549
|
+
return snapshot.tracksPositions ? onlyMergesThroughTheFile("positionValues") : null;
|
|
1550
|
+
},
|
|
1551
|
+
readSurfaceForms() {
|
|
1552
|
+
const table = readSurfaceTable();
|
|
1553
|
+
return table === null ? null : decodeSurfaceTable(table);
|
|
1554
|
+
},
|
|
1555
|
+
surfaceTableOnDisk: createSurfaceTableReader(readSurfaceTable),
|
|
1556
|
+
docFrequencies: () => {
|
|
1557
|
+
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
1558
|
+
for (let at = 0; at < tokenTable.size; at++) {
|
|
1559
|
+
frequencies[tokenTable.tokenAt(at)] = tokenTable.documentFrequencyAt(at);
|
|
1560
|
+
}
|
|
1561
|
+
return frequencies;
|
|
1562
|
+
}
|
|
1563
|
+
};
|
|
1564
|
+
const documentSource = createEncodedDocumentSource(
|
|
1565
|
+
snapshot.documentCount,
|
|
1566
|
+
required("documents").length,
|
|
1567
|
+
documents.encodedDocumentAt,
|
|
1568
|
+
snapshot.discardedFields.length === 0 ? void 0 : snapshot.discardedFields
|
|
1569
|
+
);
|
|
1570
|
+
return assemblePagedFrozenSegment(snapshot.segmentId, source, documentSource, snapshot.partialWords, {
|
|
1571
|
+
access,
|
|
1572
|
+
createCounts: (tombstones) => createPagedLiveCounts(file, snapshot, tokenTable, tombstones, removalOffsets, removals, totals),
|
|
1573
|
+
createViews: (tombstones, counts) => createPagedPostingViews(snapshot, lists, tombstones, counts)
|
|
1574
|
+
});
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
// src/persistence/paged/verify.ts
|
|
1578
|
+
async function orLoadFailure(key, read) {
|
|
1579
|
+
try {
|
|
1580
|
+
return await read();
|
|
1581
|
+
} catch (error) {
|
|
1582
|
+
if (error instanceof NarsilError) throw error;
|
|
1583
|
+
return loadFailed(
|
|
1584
|
+
`The segment "${key}" cannot be read: ${error instanceof Error ? error.message : String(error)}`,
|
|
1585
|
+
{
|
|
1586
|
+
key
|
|
1587
|
+
}
|
|
1588
|
+
);
|
|
1589
|
+
}
|
|
1590
|
+
}
|
|
1591
|
+
async function envelopeVersionOf(directory, key) {
|
|
1592
|
+
const { readRange } = directory;
|
|
1593
|
+
if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
|
|
1594
|
+
const bytes = await orLoadFailure(key, () => readRange.call(directory, key, 4, 1));
|
|
1595
|
+
return bytes[0];
|
|
1596
|
+
}
|
|
1597
|
+
async function isPagedSegmentFile(directory, key) {
|
|
1598
|
+
return await envelopeVersionOf(directory, key) === PAGED_SEGMENT_ENVELOPE_VERSION;
|
|
1599
|
+
}
|
|
1600
|
+
async function requirePagedPayloadChecksum(directory, listing) {
|
|
1601
|
+
const { readRange } = directory;
|
|
1602
|
+
const { key, bytes } = listing;
|
|
1603
|
+
if (readRange === void 0) return loadFailed(`The directory holding "${key}" reads no ranges`, { key });
|
|
1604
|
+
const header2 = readHeader(await readRange.call(directory, key, 0, HEADER_SIZE));
|
|
1605
|
+
if (HEADER_SIZE + header2.payloadLength !== bytes) {
|
|
1606
|
+
loadFailed(`The segment "${key}" holds ${bytes} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`);
|
|
1607
|
+
}
|
|
1608
|
+
await requirePayloadChecksum(key, header2, (offset, length) => readRange.call(directory, key, offset, length));
|
|
1609
|
+
}
|
|
1610
|
+
async function requirePagedPathChecksum(path) {
|
|
1611
|
+
const read = async (offset, length) => {
|
|
1612
|
+
const bytes = new Uint8Array(length);
|
|
1613
|
+
if (!await readFileRangeLater(path, offset, bytes))
|
|
1614
|
+
loadFailed(`The segment "${path}" cannot be read at ${offset}`);
|
|
1615
|
+
return bytes;
|
|
1616
|
+
};
|
|
1617
|
+
await requirePayloadChecksum(path, readHeader(await read(0, HEADER_SIZE)), read);
|
|
1618
|
+
}
|
|
1619
|
+
async function requirePayloadChecksum(key, header2, read) {
|
|
1620
|
+
const bytes = HEADER_SIZE + header2.payloadLength;
|
|
1621
|
+
let state = crc32Init();
|
|
1622
|
+
for (let at = HEADER_SIZE; at < bytes; at += FILE_ASSEMBLY_CHUNK_BYTES) {
|
|
1623
|
+
const size = Math.min(FILE_ASSEMBLY_CHUNK_BYTES, bytes - at);
|
|
1624
|
+
state = crc32Update(state, await read(at, size));
|
|
1625
|
+
}
|
|
1626
|
+
const computed = crc32Final(state);
|
|
1627
|
+
if (computed !== header2.checksum) {
|
|
1628
|
+
throw new NarsilError(
|
|
1629
|
+
ErrorCodes.PERSISTENCE_CRC_MISMATCH,
|
|
1630
|
+
`The checksum of the segment "${key}" does not match its payload`,
|
|
1631
|
+
{ key, expected: header2.checksum, computed }
|
|
1632
|
+
);
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
async function pagedListingOf(directory, ref) {
|
|
1636
|
+
const { byteLength } = directory;
|
|
1637
|
+
const length = byteLength === void 0 ? ref.bytes : await byteLength.call(directory, ref.key);
|
|
1638
|
+
if (length === null) loadFailed(`The segment "${ref.key}" that the manifest lists is missing`, { key: ref.key });
|
|
1639
|
+
if (length !== ref.bytes) {
|
|
1640
|
+
loadFailed(`The segment "${ref.key}" holds ${length} bytes where the manifest lists ${ref.bytes}`, {
|
|
1641
|
+
key: ref.key,
|
|
1642
|
+
bytes: length,
|
|
1643
|
+
listed: ref.bytes
|
|
1644
|
+
});
|
|
1645
|
+
}
|
|
1646
|
+
return { key: ref.key, path: await directory.pathOf(ref.key), bytes: ref.bytes, checksum: ref.checksum };
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
// src/persistence/paged/older-holders.ts
|
|
1650
|
+
function everyDocumentId(segments) {
|
|
1651
|
+
const ids = [];
|
|
1652
|
+
for (const segment of segments) {
|
|
1653
|
+
const table = segment.arrays.idTable;
|
|
1654
|
+
for (let ordinal = 0; ordinal < table.count; ordinal++) ids.push(table.idAt(ordinal));
|
|
1655
|
+
}
|
|
1656
|
+
return ids;
|
|
1657
|
+
}
|
|
1658
|
+
async function pagedTombstonesFor(directory, older, written, deleted) {
|
|
1659
|
+
const tombstones = new Set(deleted);
|
|
1660
|
+
if (older.length === 0) return [...tombstones];
|
|
1661
|
+
const ids = everyDocumentId(written);
|
|
1662
|
+
for (const ref of older) {
|
|
1663
|
+
if (!await isPagedSegmentFile(directory, ref.key)) return [.../* @__PURE__ */ new Set([...deleted, ...ids])];
|
|
1664
|
+
}
|
|
1665
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
1666
|
+
try {
|
|
1667
|
+
for (const ref of older) {
|
|
1668
|
+
const listing = await pagedListingOf(directory, ref);
|
|
1669
|
+
const { head } = readPagedHead(listing);
|
|
1670
|
+
const sample = readIdSampleOf(listing.path, head);
|
|
1671
|
+
const file = openPagedFile(listing.path, cache);
|
|
1672
|
+
try {
|
|
1673
|
+
const reader = (name) => {
|
|
1674
|
+
const section = sectionNamed(head.sections, name);
|
|
1675
|
+
if (section === void 0) loadFailed(`The segment "${ref.key}" holds no section "${name}"`);
|
|
1676
|
+
return createSectionReader(file, section);
|
|
1677
|
+
};
|
|
1678
|
+
const table = createPagedIdTable(
|
|
1679
|
+
{
|
|
1680
|
+
documentCount: head.documentCount,
|
|
1681
|
+
key: ref.key,
|
|
1682
|
+
idSampleIds: sample.ids,
|
|
1683
|
+
idSamplePositions: sample.positions
|
|
1684
|
+
},
|
|
1685
|
+
reader("ids"),
|
|
1686
|
+
reader("idOffsets"),
|
|
1687
|
+
reader("idOrder")
|
|
1688
|
+
);
|
|
1689
|
+
for (const docId of ids) {
|
|
1690
|
+
if (!tombstones.has(docId) && table.ordinalOf(docId) >= 0) tombstones.add(docId);
|
|
1691
|
+
}
|
|
1692
|
+
} finally {
|
|
1693
|
+
closePagedFile(file);
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
} finally {
|
|
1697
|
+
cache.clear();
|
|
1698
|
+
}
|
|
1699
|
+
return [...tombstones];
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
// src/persistence/paged/check-contents.ts
|
|
1703
|
+
function requireIds(segment, snapshot) {
|
|
1704
|
+
const ids = segment.arrays.idTable;
|
|
1705
|
+
let previous = null;
|
|
1706
|
+
for (let position = 0; position < snapshot.documentCount; position++) {
|
|
1707
|
+
const id = ids.idAt(ids.sortedOrdinalAt(position));
|
|
1708
|
+
if (previous !== null && compareCodePoints(previous, id) >= 0)
|
|
1709
|
+
loadFailed(`The ID order of "${snapshot.key}" does not ascend at position ${position}`, { position });
|
|
1710
|
+
if (position % ID_SAMPLE_INTERVAL === 0 && snapshot.idSampleIds[position / ID_SAMPLE_INTERVAL] !== id)
|
|
1711
|
+
loadFailed(`The ID sample of "${snapshot.key}" differs at position ${position}`, { position });
|
|
1712
|
+
previous = id;
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
function requirePostings(lists, snapshot) {
|
|
1716
|
+
const fields = snapshot.fieldNames.map((field) => {
|
|
1717
|
+
const at = snapshot.fieldLengthNames.indexOf(field);
|
|
1718
|
+
return at < 0 ? null : snapshot.fieldLengthColumns[at];
|
|
1719
|
+
});
|
|
1720
|
+
for (let token = 0; token < snapshot.tokenTable.documentFrequencies.length; token++) {
|
|
1721
|
+
const list = lists.listAt(token);
|
|
1722
|
+
lists.positionsAt(token);
|
|
1723
|
+
const bounds = lists.storedBoundsAt(token);
|
|
1724
|
+
if (list.docIds.length > SMALLEST_BLOCK_POSTINGS !== (bounds !== null))
|
|
1725
|
+
loadFailed(`The block summaries of token ${token} of "${snapshot.key}" do not cover its postings`, { token });
|
|
1726
|
+
let documents = 0;
|
|
1727
|
+
let occurrences = 0;
|
|
1728
|
+
let block = 0;
|
|
1729
|
+
let start = 0;
|
|
1730
|
+
let maxFrequency = 0;
|
|
1731
|
+
let minLength = Number.POSITIVE_INFINITY;
|
|
1732
|
+
let maxEntries = 0;
|
|
1733
|
+
let blockDocuments = 0;
|
|
1734
|
+
let row = 0;
|
|
1735
|
+
while (row < list.docIds.length) {
|
|
1736
|
+
const document = list.docIds[row];
|
|
1737
|
+
let entries = 0;
|
|
1738
|
+
do {
|
|
1739
|
+
const frequency = list.termFrequencies[row];
|
|
1740
|
+
occurrences += frequency;
|
|
1741
|
+
maxFrequency = Math.max(maxFrequency, frequency);
|
|
1742
|
+
minLength = Math.min(minLength, fields[list.fieldNameIndices[row]]?.[document] ?? 0);
|
|
1743
|
+
row++;
|
|
1744
|
+
entries++;
|
|
1745
|
+
} while (row < list.docIds.length && list.docIds[row] === document);
|
|
1746
|
+
documents++;
|
|
1747
|
+
blockDocuments++;
|
|
1748
|
+
maxEntries = Math.max(maxEntries, entries);
|
|
1749
|
+
if (bounds === null || row - start < SMALLEST_BLOCK_POSTINGS && row < list.docIds.length) continue;
|
|
1750
|
+
if (block >= bounds.blockCount || bounds.entryEnd[block] !== row || bounds.maxTermFrequency[block] !== maxFrequency || bounds.minFieldLength[block] !== minLength || bounds.maxEntriesPerDocument[block] !== maxEntries || bounds.documentCount[block] !== blockDocuments)
|
|
1751
|
+
loadFailed(`Block ${block} of token ${token} of "${snapshot.key}" differs from its postings`, { token, block });
|
|
1752
|
+
block++;
|
|
1753
|
+
start = row;
|
|
1754
|
+
maxFrequency = 0;
|
|
1755
|
+
minLength = Number.POSITIVE_INFINITY;
|
|
1756
|
+
maxEntries = 0;
|
|
1757
|
+
blockDocuments = 0;
|
|
1758
|
+
}
|
|
1759
|
+
if (documents !== snapshot.tokenTable.documentFrequencies[token] || occurrences !== lists.totalAt(token))
|
|
1760
|
+
loadFailed(`The counts of token ${token} of "${snapshot.key}" differ from its postings`, { token });
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
async function checkPagedContents(segment, cache) {
|
|
1764
|
+
const { paged } = segment;
|
|
1765
|
+
if (paged === null) return;
|
|
1766
|
+
const { snapshot } = paged;
|
|
1767
|
+
for (const section of snapshot.sections) await checkEverySectionPage(snapshot.path, section, () => true);
|
|
1768
|
+
const file = openPagedFile(snapshot.path, cache);
|
|
1769
|
+
try {
|
|
1770
|
+
const sectionReader = (name) => {
|
|
1771
|
+
const section = sectionNamed(snapshot.sections, name);
|
|
1772
|
+
return section === void 0 ? null : createSectionReader(file, section);
|
|
1773
|
+
};
|
|
1774
|
+
const required = (name) => {
|
|
1775
|
+
const section = sectionNamed(snapshot.sections, name);
|
|
1776
|
+
if (section === void 0) loadFailed(`The segment "${snapshot.key}" holds no section "${name}"`);
|
|
1777
|
+
return createSectionReader(file, section);
|
|
1778
|
+
};
|
|
1779
|
+
const removals = createRemovalReader(
|
|
1780
|
+
file,
|
|
1781
|
+
required("removalOffsets"),
|
|
1782
|
+
required("removals"),
|
|
1783
|
+
segment.arrays.tokenTable.size
|
|
1784
|
+
);
|
|
1785
|
+
for (let ordinal = 0; ordinal < snapshot.documentCount; ordinal++) {
|
|
1786
|
+
decodeStoredDocument(paged.encodedDocumentAt(ordinal), snapshot.discardedFields);
|
|
1787
|
+
paged.writeRecordAt(ordinal);
|
|
1788
|
+
removals(ordinal);
|
|
1789
|
+
}
|
|
1790
|
+
if (snapshot.documentCount === 0) {
|
|
1791
|
+
for (const name of ["idOffsets", "removalOffsets"]) {
|
|
1792
|
+
const offsets = required(name).bytesAt(0, 4);
|
|
1793
|
+
if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
|
|
1794
|
+
loadFailed(`Section "${name}" of "${snapshot.key}" does not start at zero`);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
if (snapshot.tracksPositions && snapshot.postingOffsets[snapshot.postingOffsets.length - 1] === 0) {
|
|
1798
|
+
const offsets = required("positionOffsets").bytesAt(0, 4);
|
|
1799
|
+
if (new DataView(offsets.buffer, offsets.byteOffset, 4).getUint32(0, true) !== 0)
|
|
1800
|
+
loadFailed(`The position offsets of "${snapshot.key}" do not start at zero`);
|
|
1801
|
+
}
|
|
1802
|
+
requireIds(segment, snapshot);
|
|
1803
|
+
requirePostings(createListReader(file, snapshot, sectionReader), snapshot);
|
|
1804
|
+
segment.arrays.readSurfaceForms();
|
|
1805
|
+
} finally {
|
|
1806
|
+
closePagedFile(file);
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
async function checkPagedSelections(selections) {
|
|
1810
|
+
for (const { segment } of selections) {
|
|
1811
|
+
const { paged } = segment;
|
|
1812
|
+
if (paged === null || paged.payloadChecked) continue;
|
|
1813
|
+
await requirePagedPathChecksum(paged.snapshot.path);
|
|
1814
|
+
paged.payloadChecked = true;
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
// src/core/segment-merges/metadata.ts
|
|
1819
|
+
function emptySource(input) {
|
|
1820
|
+
return {
|
|
1821
|
+
terms: { text: new Uint8Array(), offsets: new Uint32Array(1), counts: new Uint32Array() },
|
|
1822
|
+
payloadSlots: new Uint32Array(),
|
|
1823
|
+
postingOffsets: new Uint32Array(1),
|
|
1824
|
+
postingDocuments: new Uint32Array(),
|
|
1825
|
+
postingFrequencies: new Uint16Array(),
|
|
1826
|
+
postingFields: new Uint16Array(),
|
|
1827
|
+
positionOffsets: null,
|
|
1828
|
+
positionValues: null,
|
|
1829
|
+
mapping: new Uint32Array(input.remap.buffer, input.remap.byteOffset, input.remap.length),
|
|
1830
|
+
fieldMapping: new Uint16Array(),
|
|
1831
|
+
fieldLengths: [],
|
|
1832
|
+
documentCount: input.remap.length,
|
|
1833
|
+
surfaces: null,
|
|
1834
|
+
spellings: null
|
|
1835
|
+
};
|
|
1836
|
+
}
|
|
1837
|
+
function mergePagedLengths(inputs, documentCount) {
|
|
1838
|
+
const fieldNames = [];
|
|
1839
|
+
const sources = inputs.map((input) => {
|
|
1840
|
+
const arrays = input.segment.arrays;
|
|
1841
|
+
const fieldMapping = Uint16Array.from(
|
|
1842
|
+
arrays.fieldLengthNames.map((name) => {
|
|
1843
|
+
let at = fieldNames.indexOf(name);
|
|
1844
|
+
if (at < 0) {
|
|
1845
|
+
at = fieldNames.length;
|
|
1846
|
+
fieldNames.push(name);
|
|
1847
|
+
}
|
|
1848
|
+
return at;
|
|
1849
|
+
})
|
|
1850
|
+
);
|
|
1851
|
+
return { ...emptySource(input), fieldMapping, fieldLengths: arrays.fieldLengthColumns };
|
|
1852
|
+
});
|
|
1853
|
+
const merged = mergeKeywordSources(sources, documentCount, fieldNames.length);
|
|
1854
|
+
const columns = [];
|
|
1855
|
+
const totals = /* @__PURE__ */ Object.create(null);
|
|
1856
|
+
for (let at = 0; at < fieldNames.length; at++) {
|
|
1857
|
+
columns.push(merged.fieldLengths.subarray(at * documentCount, (at + 1) * documentCount));
|
|
1858
|
+
totals[fieldNames[at]] = merged.totalFieldLengths[at];
|
|
1859
|
+
}
|
|
1860
|
+
return { fieldLengthNames: fieldNames, fieldLengthColumns: columns, totalFieldLengths: totals };
|
|
1861
|
+
}
|
|
1862
|
+
function mergePagedStrings(inputs, documentCount, terms, analysis, textAnalysis) {
|
|
1863
|
+
const sources = inputs.map((input) => ({
|
|
1864
|
+
...emptySource(input),
|
|
1865
|
+
surfaces: mergeSourceSurfaces(input, textAnalysis),
|
|
1866
|
+
spellings: mergeSourceSpellings(input, analysis)
|
|
1867
|
+
}));
|
|
1868
|
+
const retained = { text: terms.blob, offsets: terms.offsets, counts: terms.documentFrequencies };
|
|
1869
|
+
const output = mergeKeywordSources(sources, documentCount, 0, retained);
|
|
1870
|
+
return {
|
|
1871
|
+
surfaceTable: output.surfaces.counts.length === 0 ? null : {
|
|
1872
|
+
blob: output.surfaces.text,
|
|
1873
|
+
offsets: output.surfaces.offsets,
|
|
1874
|
+
counts: output.surfaces.counts
|
|
1875
|
+
},
|
|
1876
|
+
partialSpellings: mergedPartialSpellings(output.spellings, analysis)
|
|
1877
|
+
};
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
// src/serialization/keyword-segment/msgpack-chunks.ts
|
|
1881
|
+
var FIXMAP = 128;
|
|
1882
|
+
var MAP16 = 222;
|
|
1883
|
+
var MAP32 = 223;
|
|
1884
|
+
var FIXARRAY = 144;
|
|
1885
|
+
var ARRAY16 = 220;
|
|
1886
|
+
var ARRAY32 = 221;
|
|
1887
|
+
var BIN8 = 196;
|
|
1888
|
+
var BIN16 = 197;
|
|
1889
|
+
var BIN32 = 198;
|
|
1890
|
+
var FIX_COUNT_LIMIT = 16;
|
|
1891
|
+
var UINT8_LIMIT = 256;
|
|
1892
|
+
var UINT16_LIMIT = 65536;
|
|
1893
|
+
function header(fixed, sixteen, thirtyTwo, count) {
|
|
1894
|
+
if (count < FIX_COUNT_LIMIT) return Uint8Array.of(fixed | count);
|
|
1895
|
+
if (count < UINT16_LIMIT) return Uint8Array.of(sixteen, count >> 8, count & 255);
|
|
1896
|
+
const bytes = new Uint8Array(5);
|
|
1897
|
+
bytes[0] = thirtyTwo;
|
|
1898
|
+
new DataView(bytes.buffer).setUint32(1, count, false);
|
|
1899
|
+
return bytes;
|
|
1900
|
+
}
|
|
1901
|
+
function binHeader(byteLength) {
|
|
1902
|
+
if (byteLength < UINT8_LIMIT) return Uint8Array.of(BIN8, byteLength);
|
|
1903
|
+
if (byteLength < UINT16_LIMIT) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1904
|
+
const bytes = new Uint8Array(5);
|
|
1905
|
+
bytes[0] = BIN32;
|
|
1906
|
+
new DataView(bytes.buffer).setUint32(1, byteLength, false);
|
|
1907
|
+
return bytes;
|
|
1908
|
+
}
|
|
1909
|
+
var MessagePackChunks = class {
|
|
1910
|
+
chunks = [];
|
|
1911
|
+
byteLength = 0;
|
|
1912
|
+
push(chunk) {
|
|
1913
|
+
if (chunk.byteLength === 0) return;
|
|
1914
|
+
this.chunks.push(chunk);
|
|
1915
|
+
this.byteLength += chunk.byteLength;
|
|
1916
|
+
}
|
|
1917
|
+
map(entries) {
|
|
1918
|
+
this.push(header(FIXMAP, MAP16, MAP32, entries));
|
|
1919
|
+
return this;
|
|
1920
|
+
}
|
|
1921
|
+
array(items) {
|
|
1922
|
+
this.push(header(FIXARRAY, ARRAY16, ARRAY32, items));
|
|
1923
|
+
return this;
|
|
1924
|
+
}
|
|
1925
|
+
value(value) {
|
|
1926
|
+
this.push(encode(value));
|
|
1927
|
+
return this;
|
|
1928
|
+
}
|
|
1929
|
+
key(name) {
|
|
1930
|
+
return this.value(name);
|
|
1931
|
+
}
|
|
1932
|
+
binary(bytes) {
|
|
1933
|
+
this.push(binHeader(bytes.byteLength));
|
|
1934
|
+
this.push(bytes);
|
|
1935
|
+
return this;
|
|
1936
|
+
}
|
|
1937
|
+
binaryOrNil(bytes) {
|
|
1938
|
+
return bytes === null ? this.value(null) : this.binary(bytes);
|
|
1939
|
+
}
|
|
1940
|
+
};
|
|
1941
|
+
|
|
1942
|
+
// src/serialization/keyword-segment/token-order.ts
|
|
1943
|
+
function slotsAreInTokenOrder(payloadSlots) {
|
|
1944
|
+
for (let at = 0; at < payloadSlots.length; at++) {
|
|
1945
|
+
if (payloadSlots[at] !== at) return false;
|
|
1946
|
+
}
|
|
1947
|
+
return true;
|
|
1948
|
+
}
|
|
1949
|
+
function reorderedPositions(arrays, postingOffsets) {
|
|
1950
|
+
const { positionOffsets, positionValues, tokenTable } = arrays;
|
|
1951
|
+
if (positionOffsets === null || positionValues === null) return { positionOffsets: null, positionValues: null };
|
|
1952
|
+
const postings = postingOffsets[postingOffsets.length - 1];
|
|
1953
|
+
const offsets = new Uint32Array(postings + 1);
|
|
1954
|
+
const values = new Uint32Array(positionValues.length);
|
|
1955
|
+
let cursor = 0;
|
|
1956
|
+
for (let token = 0; token < tokenTable.payloadSlots.length; token++) {
|
|
1957
|
+
const slot = tokenTable.payloadSlots[token];
|
|
1958
|
+
for (let from = arrays.postingOffsets[slot]; from < arrays.postingOffsets[slot + 1]; from++) {
|
|
1959
|
+
const to = postingOffsets[token] + (from - arrays.postingOffsets[slot]);
|
|
1960
|
+
offsets[to] = cursor;
|
|
1961
|
+
const start = positionOffsets[from];
|
|
1962
|
+
const end = positionOffsets[from + 1];
|
|
1963
|
+
values.set(positionValues.subarray(start, end), cursor);
|
|
1964
|
+
cursor += end - start;
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
offsets[postings] = cursor;
|
|
1968
|
+
return { positionOffsets: offsets, positionValues: values };
|
|
1969
|
+
}
|
|
1970
|
+
function postingsInTokenOrder(arrays) {
|
|
1971
|
+
const { tokenTable } = arrays;
|
|
1972
|
+
if (slotsAreInTokenOrder(tokenTable.payloadSlots)) return arrays;
|
|
1973
|
+
const tokens = tokenTable.payloadSlots.length;
|
|
1974
|
+
const postingOffsets = new Uint32Array(tokens + 1);
|
|
1975
|
+
for (let token = 0; token < tokens; token++) {
|
|
1976
|
+
const slot = tokenTable.payloadSlots[token];
|
|
1977
|
+
postingOffsets[token + 1] = postingOffsets[token] + arrays.postingOffsets[slot + 1] - arrays.postingOffsets[slot];
|
|
1978
|
+
}
|
|
1979
|
+
const postings = postingOffsets[tokens];
|
|
1980
|
+
const postingDocIds = new Uint32Array(postings);
|
|
1981
|
+
const postingFrequencies = new Uint16Array(postings);
|
|
1982
|
+
const postingFieldIndices = new Uint16Array(postings);
|
|
1983
|
+
for (let token = 0; token < tokens; token++) {
|
|
1984
|
+
const slot = tokenTable.payloadSlots[token];
|
|
1985
|
+
const start = arrays.postingOffsets[slot];
|
|
1986
|
+
const end = arrays.postingOffsets[slot + 1];
|
|
1987
|
+
postingDocIds.set(arrays.postingDocIds.subarray(start, end), postingOffsets[token]);
|
|
1988
|
+
postingFrequencies.set(arrays.postingFrequencies.subarray(start, end), postingOffsets[token]);
|
|
1989
|
+
postingFieldIndices.set(arrays.postingFieldIndices.subarray(start, end), postingOffsets[token]);
|
|
1990
|
+
}
|
|
1991
|
+
return {
|
|
1992
|
+
postingOffsets,
|
|
1993
|
+
postingDocIds,
|
|
1994
|
+
postingFrequencies,
|
|
1995
|
+
postingFieldIndices,
|
|
1996
|
+
...reorderedPositions(arrays, postingOffsets)
|
|
1997
|
+
};
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// src/serialization/keyword-segment/types.ts
|
|
2001
|
+
var KEYWORD_SEGMENT_PAYLOAD_VERSION = 1;
|
|
2002
|
+
var KEYWORD_SEGMENT_ENVELOPE_VERSION = 4;
|
|
2003
|
+
var MAX_SEGMENT_FIELD_NAMES = 65536;
|
|
2004
|
+
|
|
2005
|
+
// src/serialization/keyword-segment/write-records.ts
|
|
2006
|
+
var UINT64_BYTES = 8;
|
|
2007
|
+
function uint64Bytes(values) {
|
|
2008
|
+
const bytes = new Uint8Array(values.length * UINT64_BYTES);
|
|
2009
|
+
const view = new DataView(bytes.buffer);
|
|
2010
|
+
for (let at = 0; at < values.length; at++) view.setBigUint64(at * UINT64_BYTES, BigInt(values[at]), true);
|
|
2011
|
+
return bytes;
|
|
2012
|
+
}
|
|
2013
|
+
function uint64sOf(bytes, name) {
|
|
2014
|
+
const values = new Float64Array(bytes.byteLength / UINT64_BYTES);
|
|
2015
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2016
|
+
for (let at = 0; at < values.length; at++) {
|
|
2017
|
+
const value = view.getBigUint64(at * UINT64_BYTES, true);
|
|
2018
|
+
if (value > BigInt(Number.MAX_SAFE_INTEGER)) corrupt(`${name} holds a value above 2^53 - 1`, { at });
|
|
2019
|
+
values[at] = Number(value);
|
|
2020
|
+
}
|
|
2021
|
+
return values;
|
|
2022
|
+
}
|
|
2023
|
+
function decodeWriteRecords(raw, documentCount) {
|
|
2024
|
+
const rawSeqNos = requireBytesOrNil(raw.seqNos, "seqNos", UINT64_BYTES);
|
|
2025
|
+
const rawTerms = requireBytesOrNil(raw.primaryTerms, "primaryTerms", UINT64_BYTES);
|
|
2026
|
+
if (rawSeqNos === null !== (rawTerms === null))
|
|
2027
|
+
corrupt("seqNos and primaryTerms must both be present or both absent");
|
|
2028
|
+
if (rawSeqNos === null || rawTerms === null) return null;
|
|
2029
|
+
const seqNos = uint64sOf(rawSeqNos, "seqNos");
|
|
2030
|
+
const primaryTerms = uint64sOf(rawTerms, "primaryTerms");
|
|
2031
|
+
requireCount(seqNos.length, documentCount, "seqNos");
|
|
2032
|
+
requireCount(primaryTerms.length, documentCount, "primaryTerms");
|
|
2033
|
+
for (let at = 0; at < documentCount; at++) {
|
|
2034
|
+
if (seqNos[at] === 0 && primaryTerms[at] !== 0) {
|
|
2035
|
+
corrupt(`primaryTerms gives document ${at} a term without a seqNo`, { at });
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
return { seqNos, primaryTerms };
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
// src/serialization/keyword-segment/encode.ts
|
|
2042
|
+
var TOP_LEVEL_ENTRIES = 28;
|
|
2043
|
+
function writeFieldLengths(out, arrays) {
|
|
2044
|
+
out.array(arrays.fieldLengthNames.length);
|
|
2045
|
+
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
2046
|
+
out.map(2).key("field").value(arrays.fieldLengthNames[f]);
|
|
2047
|
+
out.key("lengths").binary(littleEndianBytes(arrays.fieldLengthColumns[f]));
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
function writeValueColumns(out, columns) {
|
|
2051
|
+
out.array(columns.length);
|
|
2052
|
+
for (const column of columns) {
|
|
2053
|
+
out.map(3).key("field").value(column.fieldPath);
|
|
2054
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2055
|
+
out.key("values").binary(littleEndianBytes(column.values));
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
function writeBooleanColumns(out, columns) {
|
|
2059
|
+
out.array(columns.length);
|
|
2060
|
+
for (const column of columns) {
|
|
2061
|
+
out.map(3).key("field").value(column.fieldPath);
|
|
2062
|
+
out.key("trueDocuments").binary(littleEndianBytes(column.trueDocs));
|
|
2063
|
+
out.key("falseDocuments").binary(littleEndianBytes(column.falseDocs));
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
function writeEnumColumns(out, columns) {
|
|
2067
|
+
out.array(columns.length);
|
|
2068
|
+
for (const column of columns) {
|
|
2069
|
+
out.map(4).key("field").value(column.fieldPath);
|
|
2070
|
+
out.key("values").value(column.values);
|
|
2071
|
+
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
2072
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
function writeGeopointColumns(out, columns) {
|
|
2076
|
+
out.array(columns.length);
|
|
2077
|
+
for (const column of columns) {
|
|
2078
|
+
out.map(4).key("field").value(column.fieldPath);
|
|
2079
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2080
|
+
out.key("latitudes").binary(littleEndianBytes(column.latitudes));
|
|
2081
|
+
out.key("longitudes").binary(littleEndianBytes(column.longitudes));
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
function writePatternColumns(out, columns) {
|
|
2085
|
+
out.array(columns.length);
|
|
2086
|
+
for (const column of columns) {
|
|
2087
|
+
out.map(5).key("field").value(column.fieldPath);
|
|
2088
|
+
out.key("documents").binary(littleEndianBytes(column.docIds));
|
|
2089
|
+
out.key("runs").binary(littleEndianBytes(column.runs));
|
|
2090
|
+
out.key("offsets").binary(littleEndianBytes(column.offsets));
|
|
2091
|
+
out.key("postings").binary(littleEndianBytes(column.postings));
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
function writeSurfaceForms(out, arrays) {
|
|
2095
|
+
const table = arrays.surfaceTable;
|
|
2096
|
+
if (table === null || table.counts.length === 0) {
|
|
2097
|
+
out.value(null);
|
|
2098
|
+
return;
|
|
2099
|
+
}
|
|
2100
|
+
out.map(3).key("text").binary(table.blob);
|
|
2101
|
+
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
2102
|
+
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
2103
|
+
}
|
|
2104
|
+
function writePartialWords(out, arrays) {
|
|
2105
|
+
const table = arrays.partialWords;
|
|
2106
|
+
if (table === null) {
|
|
2107
|
+
out.value(null);
|
|
2108
|
+
return;
|
|
2109
|
+
}
|
|
2110
|
+
out.map(8).key("fields").value(table.fields);
|
|
2111
|
+
out.key("text").binary(table.spellings);
|
|
2112
|
+
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
2113
|
+
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
2114
|
+
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
2115
|
+
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
2116
|
+
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
2117
|
+
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
2118
|
+
}
|
|
2119
|
+
function encodeKeywordSegment(contents) {
|
|
2120
|
+
const { arrays } = contents;
|
|
2121
|
+
const postings = postingsInTokenOrder(arrays);
|
|
2122
|
+
const out = new MessagePackChunks();
|
|
2123
|
+
const { writeRecords } = contents;
|
|
2124
|
+
out.map(writeRecords === void 0 ? TOP_LEVEL_ENTRIES : TOP_LEVEL_ENTRIES + 2);
|
|
2125
|
+
out.key("v").value(KEYWORD_SEGMENT_PAYLOAD_VERSION);
|
|
2126
|
+
out.key("schema").value(contents.schema);
|
|
2127
|
+
out.key("tombstones").value(contents.tombstones);
|
|
2128
|
+
out.key("documentCount").value(arrays.documentCount);
|
|
2129
|
+
out.key("ids").binary(arrays.idTable.blob);
|
|
2130
|
+
out.key("idOffsets").binary(littleEndianBytes(arrays.idTable.offsets));
|
|
2131
|
+
out.key("idOrder").binary(littleEndianBytes(arrays.idTable.sortedOrdinals));
|
|
2132
|
+
out.key("documents").binary(arrays.documentTable.blob);
|
|
2133
|
+
out.key("documentOffsets").binary(littleEndianBytes(arrays.documentTable.offsets));
|
|
2134
|
+
out.key("fieldNames").value(arrays.fieldNames);
|
|
2135
|
+
out.key("tokens").binary(arrays.tokenTable.blob);
|
|
2136
|
+
out.key("tokenOffsets").binary(littleEndianBytes(arrays.tokenTable.offsets));
|
|
2137
|
+
out.key("documentFrequencies").binary(littleEndianBytes(arrays.tokenTable.documentFrequencies));
|
|
2138
|
+
out.key("postingOffsets").binary(littleEndianBytes(postings.postingOffsets));
|
|
2139
|
+
out.key("postingDocuments").binary(littleEndianBytes(postings.postingDocIds));
|
|
2140
|
+
out.key("postingFrequencies").binary(littleEndianBytes(postings.postingFrequencies));
|
|
2141
|
+
out.key("postingFields").binary(littleEndianBytes(postings.postingFieldIndices));
|
|
2142
|
+
out.key("positionOffsets").binaryOrNil(postings.positionOffsets === null ? null : littleEndianBytes(postings.positionOffsets));
|
|
2143
|
+
out.key("positions").binaryOrNil(postings.positionValues === null ? null : littleEndianBytes(postings.positionValues));
|
|
2144
|
+
out.key("fieldLengths");
|
|
2145
|
+
writeFieldLengths(out, arrays);
|
|
2146
|
+
out.key("numeric");
|
|
2147
|
+
writeValueColumns(out, arrays.numeric);
|
|
2148
|
+
out.key("sizes");
|
|
2149
|
+
writeValueColumns(out, arrays.sizes);
|
|
2150
|
+
out.key("boolean");
|
|
2151
|
+
writeBooleanColumns(out, arrays.boolean);
|
|
2152
|
+
out.key("enum");
|
|
2153
|
+
writeEnumColumns(out, arrays.enums);
|
|
2154
|
+
out.key("geopoint");
|
|
2155
|
+
writeGeopointColumns(out, arrays.geo);
|
|
2156
|
+
out.key("pattern");
|
|
2157
|
+
writePatternColumns(out, arrays.pattern);
|
|
2158
|
+
out.key("surfaceForms");
|
|
2159
|
+
writeSurfaceForms(out, arrays);
|
|
2160
|
+
out.key("partialWords");
|
|
2161
|
+
writePartialWords(out, arrays);
|
|
2162
|
+
if (writeRecords !== void 0) {
|
|
2163
|
+
out.key("seqNos").binary(uint64Bytes(writeRecords.seqNos));
|
|
2164
|
+
out.key("primaryTerms").binary(uint64Bytes(writeRecords.primaryTerms));
|
|
2165
|
+
}
|
|
2166
|
+
return out;
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
// src/persistence/paged/removal-sort.ts
|
|
2170
|
+
var TRIPLE_VALUES = 3;
|
|
2171
|
+
var VALUE_BYTES4 = 4;
|
|
2172
|
+
var TRIPLE_BYTES = TRIPLE_VALUES * VALUE_BYTES4;
|
|
2173
|
+
function bucketCountFor(upperBoundTriples) {
|
|
2174
|
+
const wanted = Math.ceil(upperBoundTriples * TRIPLE_BYTES / REMOVAL_BUCKET_TARGET_BYTES);
|
|
2175
|
+
return Math.max(1, Math.min(LARGEST_REMOVAL_BUCKET_COUNT, wanted));
|
|
2176
|
+
}
|
|
2177
|
+
async function openRemovalWriter(directory, keyPrefix, documentCount, upperBoundTriples) {
|
|
2178
|
+
const bucketCount = bucketCountFor(upperBoundTriples);
|
|
2179
|
+
const ordinalsPerBucket = Math.max(1, Math.ceil(documentCount / bucketCount));
|
|
2180
|
+
const heldValues = Math.floor(REMOVAL_BUCKET_WRITE_BUFFER_BYTES / TRIPLE_BYTES) * TRIPLE_VALUES;
|
|
2181
|
+
const buckets = [];
|
|
2182
|
+
try {
|
|
2183
|
+
for (let at = 0; at < bucketCount; at++) {
|
|
2184
|
+
const key = `${keyPrefix}.part-removals-${at}`;
|
|
2185
|
+
const held = new Uint32Array(heldValues);
|
|
2186
|
+
await directory.remove(key);
|
|
2187
|
+
buckets.push({ key, handle: await directory.appendHandle(key), held, heldValues: 0, triples: 0 });
|
|
2188
|
+
}
|
|
2189
|
+
} catch (error) {
|
|
2190
|
+
for (const bucket of buckets) await bucket.handle.close().catch(() => void 0);
|
|
2191
|
+
for (let at = 0; at <= buckets.length; at++)
|
|
2192
|
+
await directory.remove(`${keyPrefix}.part-removals-${at}`).catch(() => void 0);
|
|
2193
|
+
throw error;
|
|
2194
|
+
}
|
|
2195
|
+
async function flush(bucket) {
|
|
2196
|
+
if (bucket.heldValues === 0) return;
|
|
2197
|
+
await bucket.handle.append(new Uint8Array(bucket.held.buffer, 0, bucket.heldValues * VALUE_BYTES4));
|
|
2198
|
+
bucket.heldValues = 0;
|
|
2199
|
+
}
|
|
2200
|
+
async function drainBucket(bucket, firstOrdinal, endOrdinal, firstValue, emit) {
|
|
2201
|
+
await flush(bucket);
|
|
2202
|
+
await bucket.handle.close();
|
|
2203
|
+
const stored = await directory.read(bucket.key);
|
|
2204
|
+
if (stored === null || stored.byteLength !== bucket.triples * TRIPLE_BYTES) {
|
|
2205
|
+
loadFailed(`The removal bucket "${bucket.key}" holds the wrong number of bytes`, { key: bucket.key });
|
|
2206
|
+
}
|
|
2207
|
+
const triples = new Uint32Array(bucket.triples * TRIPLE_VALUES);
|
|
2208
|
+
new Uint8Array(triples.buffer).set(stored);
|
|
2209
|
+
const span = endOrdinal - firstOrdinal;
|
|
2210
|
+
const counts = new Uint32Array(span + 1);
|
|
2211
|
+
for (let at = 0; at < triples.length; at += TRIPLE_VALUES) counts[triples[at] - firstOrdinal + 1] += 2;
|
|
2212
|
+
for (let at = 0; at < span; at++) counts[at + 1] += counts[at];
|
|
2213
|
+
const values = new Uint32Array(bucket.triples * 2);
|
|
2214
|
+
const cursors = counts.slice(0, span);
|
|
2215
|
+
for (let at = 0; at < triples.length; at += TRIPLE_VALUES) {
|
|
2216
|
+
const row = triples[at] - firstOrdinal;
|
|
2217
|
+
values[cursors[row]] = triples[at + 1];
|
|
2218
|
+
values[cursors[row] + 1] = triples[at + 2];
|
|
2219
|
+
cursors[row] += 2;
|
|
2220
|
+
}
|
|
2221
|
+
const offsets = new Uint32Array(span);
|
|
2222
|
+
for (let at = 0; at < span; at++) offsets[at] = firstValue + counts[at];
|
|
2223
|
+
await emit(offsets, values);
|
|
2224
|
+
await directory.remove(bucket.key);
|
|
2225
|
+
return firstValue + values.length;
|
|
2226
|
+
}
|
|
2227
|
+
return {
|
|
2228
|
+
async add(ordinal, token, occurrences) {
|
|
2229
|
+
const bucket = buckets[Math.floor(ordinal / ordinalsPerBucket)];
|
|
2230
|
+
bucket.held[bucket.heldValues] = ordinal;
|
|
2231
|
+
bucket.held[bucket.heldValues + 1] = token;
|
|
2232
|
+
bucket.held[bucket.heldValues + 2] = occurrences;
|
|
2233
|
+
bucket.heldValues += TRIPLE_VALUES;
|
|
2234
|
+
bucket.triples += 1;
|
|
2235
|
+
if (bucket.heldValues === bucket.held.length) await flush(bucket);
|
|
2236
|
+
},
|
|
2237
|
+
async drain(emit) {
|
|
2238
|
+
let nextValue = 0;
|
|
2239
|
+
for (let at = 0; at < buckets.length; at++) {
|
|
2240
|
+
const first = at * ordinalsPerBucket;
|
|
2241
|
+
const end = Math.min(documentCount, first + ordinalsPerBucket);
|
|
2242
|
+
nextValue = await drainBucket(buckets[at], first, Math.max(first, end), nextValue, emit);
|
|
2243
|
+
}
|
|
2244
|
+
await emit(Uint32Array.of(nextValue), new Uint32Array(0));
|
|
2245
|
+
},
|
|
2246
|
+
async abandon() {
|
|
2247
|
+
for (const bucket of buckets) {
|
|
2248
|
+
await bucket.handle.close().catch(() => void 0);
|
|
2249
|
+
await directory.remove(bucket.key).catch(() => void 0);
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
};
|
|
2253
|
+
}
|
|
2254
|
+
|
|
2255
|
+
// src/persistence/paged/sink.ts
|
|
2256
|
+
async function openSectionSink(directory, temporaryKey) {
|
|
2257
|
+
const buffer = new Uint8Array(SECTION_WRITE_BUFFER_BYTES);
|
|
2258
|
+
const page = new Uint8Array(SEGMENT_PAGE_BYTES);
|
|
2259
|
+
const checksums = [];
|
|
2260
|
+
await directory.remove(temporaryKey);
|
|
2261
|
+
const handle = await directory.appendHandle(temporaryKey);
|
|
2262
|
+
let buffered = 0;
|
|
2263
|
+
let pageFilled = 0;
|
|
2264
|
+
let length = 0;
|
|
2265
|
+
let closed = false;
|
|
2266
|
+
function checksumPage() {
|
|
2267
|
+
checksums.push(crc32(page.subarray(0, pageFilled)));
|
|
2268
|
+
pageFilled = 0;
|
|
2269
|
+
}
|
|
2270
|
+
function addToPage(bytes) {
|
|
2271
|
+
let at = 0;
|
|
2272
|
+
while (at < bytes.byteLength) {
|
|
2273
|
+
const take = Math.min(SEGMENT_PAGE_BYTES - pageFilled, bytes.byteLength - at);
|
|
2274
|
+
page.set(bytes.subarray(at, at + take), pageFilled);
|
|
2275
|
+
pageFilled += take;
|
|
2276
|
+
at += take;
|
|
2277
|
+
if (pageFilled === SEGMENT_PAGE_BYTES) checksumPage();
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
async function flush() {
|
|
2281
|
+
if (buffered === 0) return;
|
|
2282
|
+
await handle.append(buffer.subarray(0, buffered));
|
|
2283
|
+
buffered = 0;
|
|
2284
|
+
}
|
|
2285
|
+
async function close() {
|
|
2286
|
+
if (closed) return;
|
|
2287
|
+
closed = true;
|
|
2288
|
+
await handle.close();
|
|
2289
|
+
}
|
|
2290
|
+
return {
|
|
2291
|
+
get length() {
|
|
2292
|
+
return length;
|
|
2293
|
+
},
|
|
2294
|
+
async write(bytes) {
|
|
2295
|
+
addToPage(bytes);
|
|
2296
|
+
length += bytes.byteLength;
|
|
2297
|
+
let at = 0;
|
|
2298
|
+
while (at < bytes.byteLength) {
|
|
2299
|
+
const take = Math.min(buffer.byteLength - buffered, bytes.byteLength - at);
|
|
2300
|
+
buffer.set(bytes.subarray(at, at + take), buffered);
|
|
2301
|
+
buffered += take;
|
|
2302
|
+
at += take;
|
|
2303
|
+
if (buffered === buffer.byteLength) await flush();
|
|
2304
|
+
}
|
|
2305
|
+
},
|
|
2306
|
+
async finish() {
|
|
2307
|
+
await flush();
|
|
2308
|
+
await close();
|
|
2309
|
+
if (pageFilled > 0) checksumPage();
|
|
2310
|
+
const bytes = new Uint8Array(checksums.length * 4);
|
|
2311
|
+
const view = new DataView(bytes.buffer);
|
|
2312
|
+
for (let at = 0; at < checksums.length; at++) view.setUint32(at * 4, checksums[at], true);
|
|
2313
|
+
return { temporaryKey, length, checksums: bytes };
|
|
2314
|
+
},
|
|
2315
|
+
async abandon() {
|
|
2316
|
+
await close();
|
|
2317
|
+
await directory.remove(temporaryKey);
|
|
2318
|
+
}
|
|
2319
|
+
};
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
// src/persistence/paged/write-documents.ts
|
|
2323
|
+
var encoder = new TextEncoder();
|
|
2324
|
+
function yieldToEventLoop() {
|
|
2325
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
2326
|
+
}
|
|
2327
|
+
function encodedDocumentOf(segment, ordinal) {
|
|
2328
|
+
if (segment.paged !== null) return segment.paged.encodedDocumentAt(ordinal);
|
|
2329
|
+
const table = segment.arrays.documentTable;
|
|
2330
|
+
if (table === null) return encodeDocument(segment.documentSource.docAt(ordinal));
|
|
2331
|
+
const stored = table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]);
|
|
2332
|
+
const { discardedFields } = table;
|
|
2333
|
+
if (discardedFields === void 0 || !holdsADiscardedField(stored, discardedFields)) return stored;
|
|
2334
|
+
return encodeDocument(segment.documentSource.docAt(ordinal));
|
|
2335
|
+
}
|
|
2336
|
+
async function writeDocumentRows(inputs, sinks, writeRecords) {
|
|
2337
|
+
const blocks = [];
|
|
2338
|
+
let pending = [];
|
|
2339
|
+
let pendingBytes = 0;
|
|
2340
|
+
let firstOrdinal = 0;
|
|
2341
|
+
let compressedAt = 0;
|
|
2342
|
+
let idBytes = 0;
|
|
2343
|
+
const single = new Uint32Array(1);
|
|
2344
|
+
const seqNo = new Float64Array(1);
|
|
2345
|
+
const primaryTerm = new Float64Array(1);
|
|
2346
|
+
async function closeBlock() {
|
|
2347
|
+
if (pendingBytes === 0) return;
|
|
2348
|
+
const raw = new Uint8Array(pendingBytes);
|
|
2349
|
+
let at = 0;
|
|
2350
|
+
for (const chunk of pending) {
|
|
2351
|
+
raw.set(chunk, at);
|
|
2352
|
+
at += chunk.byteLength;
|
|
2353
|
+
}
|
|
2354
|
+
const compressed = deflateBlock(raw);
|
|
2355
|
+
const entry = new Uint8Array(DOCUMENT_BLOCK_BYTES);
|
|
2356
|
+
const view = new DataView(entry.buffer);
|
|
2357
|
+
view.setUint32(0, firstOrdinal, true);
|
|
2358
|
+
view.setBigUint64(4, BigInt(compressedAt), true);
|
|
2359
|
+
view.setUint32(12, compressed.byteLength, true);
|
|
2360
|
+
view.setUint32(16, raw.byteLength, true);
|
|
2361
|
+
blocks.push(entry);
|
|
2362
|
+
await sinks.documents.write(compressed);
|
|
2363
|
+
compressedAt += compressed.byteLength;
|
|
2364
|
+
pending = [];
|
|
2365
|
+
pendingBytes = 0;
|
|
2366
|
+
await yieldToEventLoop();
|
|
2367
|
+
}
|
|
2368
|
+
single[0] = 0;
|
|
2369
|
+
await sinks.idOffsets.write(littleEndianBytes(single));
|
|
2370
|
+
for (const input of inputs) {
|
|
2371
|
+
const { segment, remap } = input;
|
|
2372
|
+
const records = writeRecords?.get(segment);
|
|
2373
|
+
for (let ordinal = 0; ordinal < remap.length; ordinal++) {
|
|
2374
|
+
const target = remap[ordinal];
|
|
2375
|
+
if (target < 0) continue;
|
|
2376
|
+
if (pendingBytes === 0) firstOrdinal = target;
|
|
2377
|
+
const encoded = encodedDocumentOf(segment, ordinal);
|
|
2378
|
+
pending.push(encoded.slice());
|
|
2379
|
+
pendingBytes += encoded.byteLength;
|
|
2380
|
+
if (pendingBytes >= DOCUMENT_BLOCK_MIN_RAW_BYTES) await closeBlock();
|
|
2381
|
+
const id = encoder.encode(segment.arrays.idTable.idAt(ordinal));
|
|
2382
|
+
await sinks.ids.write(id);
|
|
2383
|
+
idBytes += id.byteLength;
|
|
2384
|
+
single[0] = idBytes;
|
|
2385
|
+
await sinks.idOffsets.write(littleEndianBytes(single));
|
|
2386
|
+
if (sinks.seqNos !== null && sinks.primaryTerms !== null) {
|
|
2387
|
+
if (segment.paged !== null) {
|
|
2388
|
+
const held = segment.paged.writeRecordAt(ordinal);
|
|
2389
|
+
seqNo[0] = held.seqNo;
|
|
2390
|
+
primaryTerm[0] = held.primaryTerm;
|
|
2391
|
+
} else {
|
|
2392
|
+
seqNo[0] = records?.seqNos[ordinal] ?? 0;
|
|
2393
|
+
primaryTerm[0] = records?.primaryTerms[ordinal] ?? 0;
|
|
2394
|
+
}
|
|
2395
|
+
await sinks.seqNos.write(uint64Bytes(seqNo));
|
|
2396
|
+
await sinks.primaryTerms.write(uint64Bytes(primaryTerm));
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
await closeBlock();
|
|
2401
|
+
const blockBytes = new Uint8Array(blocks.length * DOCUMENT_BLOCK_BYTES);
|
|
2402
|
+
for (let at = 0; at < blocks.length; at++) blockBytes.set(blocks[at], at * DOCUMENT_BLOCK_BYTES);
|
|
2403
|
+
return blockBytes;
|
|
2404
|
+
}
|
|
2405
|
+
function advance(cursor) {
|
|
2406
|
+
const table = cursor.input.segment.arrays.idTable;
|
|
2407
|
+
for (; ; ) {
|
|
2408
|
+
cursor.position += 1;
|
|
2409
|
+
if (cursor.position >= table.count) return false;
|
|
2410
|
+
const ordinal = table.sortedOrdinalAt(cursor.position);
|
|
2411
|
+
if (cursor.input.remap[ordinal] < 0) continue;
|
|
2412
|
+
cursor.ordinal = ordinal;
|
|
2413
|
+
cursor.id = table.idAt(ordinal);
|
|
2414
|
+
return true;
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
async function writeIdOrder(inputs, idOrder) {
|
|
2418
|
+
const cursors = [];
|
|
2419
|
+
for (const input of inputs) {
|
|
2420
|
+
const cursor = { input, position: -1, ordinal: 0, id: "" };
|
|
2421
|
+
if (advance(cursor)) cursors.push(cursor);
|
|
2422
|
+
}
|
|
2423
|
+
const sample = [];
|
|
2424
|
+
const single = new Uint32Array(1);
|
|
2425
|
+
let position = 0;
|
|
2426
|
+
while (cursors.length > 0) {
|
|
2427
|
+
let least = 0;
|
|
2428
|
+
for (let at = 1; at < cursors.length; at++) {
|
|
2429
|
+
if (compareCodePoints(cursors[at].id, cursors[least].id) < 0) least = at;
|
|
2430
|
+
}
|
|
2431
|
+
const cursor = cursors[least];
|
|
2432
|
+
single[0] = cursor.input.remap[cursor.ordinal];
|
|
2433
|
+
await idOrder.write(littleEndianBytes(single));
|
|
2434
|
+
if (position % ID_SAMPLE_INTERVAL === 0) sample.push([cursor.id, position]);
|
|
2435
|
+
position += 1;
|
|
2436
|
+
if (!advance(cursor)) cursors.splice(least, 1);
|
|
2437
|
+
if (position % DOCUMENT_BLOCK_MIN_RAW_BYTES === 0) await yieldToEventLoop();
|
|
2438
|
+
}
|
|
2439
|
+
return encode(sample);
|
|
2440
|
+
}
|
|
2441
|
+
|
|
2442
|
+
// src/persistence/paged/write-rows.ts
|
|
2443
|
+
function sourceRows(input, sortedIndex) {
|
|
2444
|
+
const { arrays, paged } = input.segment;
|
|
2445
|
+
const slot = arrays.tokenTable.payloadSlot(sortedIndex);
|
|
2446
|
+
if (paged !== null) {
|
|
2447
|
+
const list = paged.listAt(slot);
|
|
2448
|
+
const positions = paged.positionsAt(slot);
|
|
2449
|
+
return {
|
|
2450
|
+
postingDocuments: list.docIds,
|
|
2451
|
+
postingFrequencies: list.termFrequencies,
|
|
2452
|
+
postingFields: list.fieldNameIndices,
|
|
2453
|
+
positionOffsets: positions?.offsets ?? null,
|
|
2454
|
+
positionValues: positions?.values ?? null
|
|
2455
|
+
};
|
|
2456
|
+
}
|
|
2457
|
+
const start = arrays.postingOffsets[slot];
|
|
2458
|
+
const end = arrays.postingOffsets[slot + 1];
|
|
2459
|
+
return {
|
|
2460
|
+
postingDocuments: arrays.postingDocIds.subarray(start, end),
|
|
2461
|
+
postingFrequencies: arrays.postingFrequencies.subarray(start, end),
|
|
2462
|
+
postingFields: arrays.postingFieldIndices.subarray(start, end),
|
|
2463
|
+
positionOffsets: arrays.positionOffsets?.subarray(start, end + 1) ?? null,
|
|
2464
|
+
positionValues: arrays.positionValues
|
|
2465
|
+
};
|
|
2466
|
+
}
|
|
2467
|
+
var TokenRows = class {
|
|
2468
|
+
constructor(documentCount, fieldCount) {
|
|
2469
|
+
this.documentCount = documentCount;
|
|
2470
|
+
this.fieldCount = fieldCount;
|
|
2471
|
+
}
|
|
2472
|
+
docIds = new Uint32Array(POSTING_WRITE_ROWS);
|
|
2473
|
+
frequencies = new Uint16Array(POSTING_WRITE_ROWS);
|
|
2474
|
+
fields = new Uint16Array(POSTING_WRITE_ROWS);
|
|
2475
|
+
positionStarts = new Uint32Array(POSTING_WRITE_ROWS + 1);
|
|
2476
|
+
positions = new Uint32Array(POSTING_WRITE_POSITIONS);
|
|
2477
|
+
count = 0;
|
|
2478
|
+
positionCount = 0;
|
|
2479
|
+
totals = new Float64Array(2);
|
|
2480
|
+
inputs = [];
|
|
2481
|
+
sources = /* @__PURE__ */ new Map();
|
|
2482
|
+
rowCapacity = 0;
|
|
2483
|
+
positionCapacity = 0;
|
|
2484
|
+
output = this.outputColumns();
|
|
2485
|
+
clear() {
|
|
2486
|
+
this.inputs.length = 0;
|
|
2487
|
+
this.rowCapacity = 0;
|
|
2488
|
+
this.positionCapacity = 0;
|
|
2489
|
+
}
|
|
2490
|
+
appendFrom(input, sortedIndex) {
|
|
2491
|
+
let source = this.sources.get(input);
|
|
2492
|
+
const rows = sourceRows(input, sortedIndex);
|
|
2493
|
+
if (source === void 0) {
|
|
2494
|
+
source = {
|
|
2495
|
+
...rows,
|
|
2496
|
+
terms: { text: new Uint8Array(), offsets: new Uint32Array(1), counts: new Uint32Array() },
|
|
2497
|
+
payloadSlots: new Uint32Array(),
|
|
2498
|
+
postingOffsets: new Uint32Array(1),
|
|
2499
|
+
mapping: new Uint32Array(input.remap.buffer, input.remap.byteOffset, input.remap.length),
|
|
2500
|
+
fieldMapping: input.fieldIndexRemap,
|
|
2501
|
+
fieldLengths: input.segment.arrays.fieldNames.map(() => null),
|
|
2502
|
+
documentCount: input.segment.arrays.documentCount,
|
|
2503
|
+
surfaces: null,
|
|
2504
|
+
spellings: null
|
|
2505
|
+
};
|
|
2506
|
+
this.sources.set(input, source);
|
|
2507
|
+
} else Object.assign(source, rows);
|
|
2508
|
+
this.inputs.push(source);
|
|
2509
|
+
this.rowCapacity += rows.postingDocuments.length;
|
|
2510
|
+
if (rows.positionOffsets !== null) {
|
|
2511
|
+
this.positionCapacity += rows.positionOffsets[rows.postingDocuments.length] - rows.positionOffsets[0];
|
|
2512
|
+
}
|
|
2513
|
+
}
|
|
2514
|
+
outputColumns() {
|
|
2515
|
+
return {
|
|
2516
|
+
documents: this.docIds,
|
|
2517
|
+
frequencies: this.frequencies,
|
|
2518
|
+
fields: this.fields,
|
|
2519
|
+
positionOffsets: this.positionStarts,
|
|
2520
|
+
positions: this.positions,
|
|
2521
|
+
totals: this.totals
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
finish() {
|
|
2525
|
+
let grown = false;
|
|
2526
|
+
if (this.rowCapacity > this.docIds.length) {
|
|
2527
|
+
const capacity = Math.max(this.rowCapacity, this.docIds.length * 2);
|
|
2528
|
+
this.docIds = new Uint32Array(capacity);
|
|
2529
|
+
this.frequencies = new Uint16Array(capacity);
|
|
2530
|
+
this.fields = new Uint16Array(capacity);
|
|
2531
|
+
this.positionStarts = new Uint32Array(capacity + 1);
|
|
2532
|
+
grown = true;
|
|
2533
|
+
}
|
|
2534
|
+
if (this.positionCapacity > this.positions.length) {
|
|
2535
|
+
this.positions = new Uint32Array(Math.max(this.positionCapacity, this.positions.length * 2));
|
|
2536
|
+
grown = true;
|
|
2537
|
+
}
|
|
2538
|
+
if (grown) this.output = this.outputColumns();
|
|
2539
|
+
const count = mergePostings2(this.inputs, this.documentCount, this.fieldCount, this.output);
|
|
2540
|
+
if (count < 0) {
|
|
2541
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "The paged posting list refused a merge", {
|
|
2542
|
+
status: -count
|
|
2543
|
+
});
|
|
2544
|
+
}
|
|
2545
|
+
this.count = count;
|
|
2546
|
+
this.positionCount = this.positionStarts[count];
|
|
2547
|
+
}
|
|
2548
|
+
distinctDocuments() {
|
|
2549
|
+
return this.totals[0];
|
|
2550
|
+
}
|
|
2551
|
+
occurrences() {
|
|
2552
|
+
return this.totals[1];
|
|
2553
|
+
}
|
|
2554
|
+
};
|
|
2555
|
+
|
|
2556
|
+
// src/persistence/paged/write-lists.ts
|
|
2557
|
+
var NO_DELETED_DOCUMENTS = { size: 0, has: () => false };
|
|
2558
|
+
function compareBytes(left, right) {
|
|
2559
|
+
const shared = Math.min(left.byteLength, right.byteLength);
|
|
2560
|
+
for (let at = 0; at < shared; at++) {
|
|
2561
|
+
if (left[at] !== right[at]) return left[at] - right[at];
|
|
2562
|
+
}
|
|
2563
|
+
return left.byteLength - right.byteLength;
|
|
2564
|
+
}
|
|
2565
|
+
var GrowingBytes = class {
|
|
2566
|
+
bytes = new Uint8Array(65536);
|
|
2567
|
+
length = 0;
|
|
2568
|
+
add(chunk) {
|
|
2569
|
+
if (this.length + chunk.byteLength > this.bytes.byteLength) {
|
|
2570
|
+
const next = new Uint8Array(Math.max(this.bytes.byteLength * 2, this.length + chunk.byteLength));
|
|
2571
|
+
next.set(this.bytes.subarray(0, this.length));
|
|
2572
|
+
this.bytes = next;
|
|
2573
|
+
}
|
|
2574
|
+
this.bytes.set(chunk, this.length);
|
|
2575
|
+
this.length += chunk.byteLength;
|
|
2576
|
+
}
|
|
2577
|
+
finished(shared) {
|
|
2578
|
+
const target = new Uint8Array(shared ? new SharedArrayBuffer(this.length) : new ArrayBuffer(this.length));
|
|
2579
|
+
target.set(this.bytes.subarray(0, this.length));
|
|
2580
|
+
return target;
|
|
2581
|
+
}
|
|
2582
|
+
};
|
|
2583
|
+
var GrowingUint32s = class {
|
|
2584
|
+
values = new Uint32Array(4096);
|
|
2585
|
+
length = 0;
|
|
2586
|
+
add(value) {
|
|
2587
|
+
if (this.length === this.values.length) {
|
|
2588
|
+
const next = new Uint32Array(this.values.length * 2);
|
|
2589
|
+
next.set(this.values);
|
|
2590
|
+
this.values = next;
|
|
2591
|
+
}
|
|
2592
|
+
this.values[this.length] = value;
|
|
2593
|
+
this.length += 1;
|
|
2594
|
+
}
|
|
2595
|
+
finished(shared) {
|
|
2596
|
+
const target = new Uint32Array(shared ? new SharedArrayBuffer(this.length * 4) : new ArrayBuffer(this.length * 4));
|
|
2597
|
+
target.set(this.values.subarray(0, this.length));
|
|
2598
|
+
return target;
|
|
2599
|
+
}
|
|
2600
|
+
};
|
|
2601
|
+
function openCursors(inputs) {
|
|
2602
|
+
const cursors = [];
|
|
2603
|
+
for (const input of inputs) {
|
|
2604
|
+
const table = input.segment.arrays.tokenTable;
|
|
2605
|
+
if (table.size > 0) cursors.push({ input, at: 0, bytes: table.tokenBytesAt(0) });
|
|
2606
|
+
}
|
|
2607
|
+
return cursors;
|
|
2608
|
+
}
|
|
2609
|
+
function blockSummaryBytesOf(rows, columns) {
|
|
2610
|
+
const list = {
|
|
2611
|
+
length: rows.count,
|
|
2612
|
+
docIds: rows.docIds,
|
|
2613
|
+
termFrequencies: rows.frequencies,
|
|
2614
|
+
fieldNameIndices: rows.fields,
|
|
2615
|
+
positions: null,
|
|
2616
|
+
docIdSet: { size: 0 },
|
|
2617
|
+
deletedDocs: NO_DELETED_DOCUMENTS,
|
|
2618
|
+
totalTermFrequency: 0,
|
|
2619
|
+
structureRevision: 0,
|
|
2620
|
+
ordered: true
|
|
2621
|
+
};
|
|
2622
|
+
const bounds = scanPostingBlocks(list, columns, null, 0);
|
|
2623
|
+
const bytes = new Uint8Array(bounds.blockCount * BLOCK_SUMMARY_BYTES);
|
|
2624
|
+
const view = new DataView(bytes.buffer);
|
|
2625
|
+
for (let block = 0; block < bounds.blockCount; block++) {
|
|
2626
|
+
const place = block * BLOCK_SUMMARY_BYTES;
|
|
2627
|
+
view.setUint32(place, bounds.entryEnd[block], true);
|
|
2628
|
+
view.setUint16(place + 4, bounds.maxTermFrequency[block], true);
|
|
2629
|
+
view.setFloat64(place + 6, bounds.minFieldLength[block], true);
|
|
2630
|
+
view.setUint32(place + 14, bounds.maxEntriesPerDocument[block], true);
|
|
2631
|
+
view.setUint32(place + 18, bounds.documentCount[block], true);
|
|
2632
|
+
}
|
|
2633
|
+
return bytes;
|
|
2634
|
+
}
|
|
2635
|
+
async function addRemovalsOf(rows, token, removals) {
|
|
2636
|
+
let at = 0;
|
|
2637
|
+
while (at < rows.count) {
|
|
2638
|
+
const ordinal = rows.docIds[at];
|
|
2639
|
+
let occurrences = 0;
|
|
2640
|
+
while (at < rows.count && rows.docIds[at] === ordinal) {
|
|
2641
|
+
occurrences += rows.frequencies[at];
|
|
2642
|
+
at += 1;
|
|
2643
|
+
}
|
|
2644
|
+
await removals.add(ordinal, token, occurrences);
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2647
|
+
function yieldToEventLoop2() {
|
|
2648
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
2649
|
+
}
|
|
2650
|
+
async function writeMergedLists(inputs, documentCount, lengthColumnsByField, sinks, removals, shared) {
|
|
2651
|
+
const tokens = new GrowingBytes();
|
|
2652
|
+
const tokenOffsets = new GrowingUint32s();
|
|
2653
|
+
const frequencies = new GrowingUint32s();
|
|
2654
|
+
const postingOffsets = new GrowingUint32s();
|
|
2655
|
+
const blockLists = new GrowingUint32s();
|
|
2656
|
+
const blockListOffsets = new GrowingUint32s();
|
|
2657
|
+
const blockSummaries = new GrowingBytes();
|
|
2658
|
+
tokenOffsets.add(0);
|
|
2659
|
+
postingOffsets.add(0);
|
|
2660
|
+
blockListOffsets.add(0);
|
|
2661
|
+
const rows = new TokenRows(documentCount, lengthColumnsByField.length);
|
|
2662
|
+
const total = new Float64Array(1);
|
|
2663
|
+
const offsetScratch = new Uint32Array(1);
|
|
2664
|
+
const cursors = openCursors(inputs);
|
|
2665
|
+
let postingsWritten = 0;
|
|
2666
|
+
let positionsWritten = 0;
|
|
2667
|
+
let tokensWalked = 0;
|
|
2668
|
+
while (cursors.length > 0) {
|
|
2669
|
+
let smallest = cursors[0].bytes;
|
|
2670
|
+
for (let at = 1; at < cursors.length; at++) {
|
|
2671
|
+
if (compareBytes(cursors[at].bytes, smallest) < 0) smallest = cursors[at].bytes;
|
|
2672
|
+
}
|
|
2673
|
+
rows.clear();
|
|
2674
|
+
for (let at = 0; at < cursors.length; at++) {
|
|
2675
|
+
const cursor = cursors[at];
|
|
2676
|
+
if (compareBytes(cursor.bytes, smallest) !== 0) continue;
|
|
2677
|
+
rows.appendFrom(cursor.input, cursor.at);
|
|
2678
|
+
cursor.at += 1;
|
|
2679
|
+
}
|
|
2680
|
+
const held = smallest;
|
|
2681
|
+
for (let at = cursors.length - 1; at >= 0; at--) {
|
|
2682
|
+
const cursor = cursors[at];
|
|
2683
|
+
if (cursor.bytes !== held && compareBytes(cursor.bytes, held) !== 0) continue;
|
|
2684
|
+
const table = cursor.input.segment.arrays.tokenTable;
|
|
2685
|
+
if (cursor.at >= table.size) cursors.splice(at, 1);
|
|
2686
|
+
else cursor.bytes = table.tokenBytesAt(cursor.at);
|
|
2687
|
+
}
|
|
2688
|
+
tokensWalked += 1;
|
|
2689
|
+
if (tokensWalked % WRITER_YIELD_INTERVAL_TOKENS === 0) await yieldToEventLoop2();
|
|
2690
|
+
rows.finish();
|
|
2691
|
+
if (rows.count === 0) continue;
|
|
2692
|
+
const token = frequencies.length;
|
|
2693
|
+
tokens.add(held);
|
|
2694
|
+
tokenOffsets.add(tokens.length);
|
|
2695
|
+
frequencies.add(rows.distinctDocuments());
|
|
2696
|
+
await sinks.postings.write(littleEndianBytes(rows.docIds.subarray(0, rows.count)));
|
|
2697
|
+
await sinks.postings.write(littleEndianBytes(rows.frequencies.subarray(0, rows.count)));
|
|
2698
|
+
await sinks.postings.write(littleEndianBytes(rows.fields.subarray(0, rows.count)));
|
|
2699
|
+
total[0] = rows.occurrences();
|
|
2700
|
+
await sinks.tokenTotals.write(littleEndianBytes(total));
|
|
2701
|
+
if (sinks.positionOffsets !== null && sinks.positions !== null) {
|
|
2702
|
+
for (let at = 0; at < rows.count; at++) {
|
|
2703
|
+
offsetScratch[0] = positionsWritten + rows.positionStarts[at];
|
|
2704
|
+
await sinks.positionOffsets.write(littleEndianBytes(offsetScratch));
|
|
2705
|
+
}
|
|
2706
|
+
await sinks.positions.write(littleEndianBytes(rows.positions.subarray(0, rows.positionCount)));
|
|
2707
|
+
positionsWritten += rows.positionCount;
|
|
2708
|
+
}
|
|
2709
|
+
if (rows.count > SMALLEST_BLOCK_POSTINGS) {
|
|
2710
|
+
blockLists.add(token);
|
|
2711
|
+
const summaries = blockSummaryBytesOf(rows, lengthColumnsByField);
|
|
2712
|
+
blockSummaries.add(summaries);
|
|
2713
|
+
blockListOffsets.add(blockSummaries.length / BLOCK_SUMMARY_BYTES);
|
|
2714
|
+
}
|
|
2715
|
+
await addRemovalsOf(rows, token, removals);
|
|
2716
|
+
postingsWritten += rows.count;
|
|
2717
|
+
postingOffsets.add(postingsWritten);
|
|
2718
|
+
}
|
|
2719
|
+
if (sinks.positionOffsets !== null) {
|
|
2720
|
+
offsetScratch[0] = positionsWritten;
|
|
2721
|
+
await sinks.positionOffsets.write(littleEndianBytes(offsetScratch));
|
|
2722
|
+
}
|
|
2723
|
+
const tokenCount = frequencies.length;
|
|
2724
|
+
const slots = new Uint32Array(new ArrayBuffer(tokenCount * 4));
|
|
2725
|
+
for (let at = 0; at < tokenCount; at++) slots[at] = at;
|
|
2726
|
+
return {
|
|
2727
|
+
tokenTable: {
|
|
2728
|
+
blob: tokens.finished(shared),
|
|
2729
|
+
offsets: tokenOffsets.finished(shared),
|
|
2730
|
+
payloadSlots: slots,
|
|
2731
|
+
documentFrequencies: frequencies.finished(shared)
|
|
2732
|
+
},
|
|
2733
|
+
postingOffsets: postingOffsets.finished(shared),
|
|
2734
|
+
blockLists: blockLists.finished(shared),
|
|
2735
|
+
blockListOffsets: blockListOffsets.finished(shared),
|
|
2736
|
+
blockSummaries: blockSummaries.finished(shared)
|
|
2737
|
+
};
|
|
2738
|
+
}
|
|
2739
|
+
|
|
2740
|
+
// src/persistence/paged/write-segment.ts
|
|
2741
|
+
var RESERVED_BYTES = 14;
|
|
2742
|
+
var RESIDENT_SECTION_ORDER = [
|
|
2743
|
+
"tokens",
|
|
2744
|
+
"tokenOffsets",
|
|
2745
|
+
"documentFrequencies",
|
|
2746
|
+
"postingOffsets",
|
|
2747
|
+
"blockLists",
|
|
2748
|
+
"blockListOffsets",
|
|
2749
|
+
"blockSummaries",
|
|
2750
|
+
"fieldLengths",
|
|
2751
|
+
"numeric",
|
|
2752
|
+
"sizes",
|
|
2753
|
+
"boolean",
|
|
2754
|
+
"enum",
|
|
2755
|
+
"geopoint",
|
|
2756
|
+
"pattern",
|
|
2757
|
+
"partialWords",
|
|
2758
|
+
"documentBlocks",
|
|
2759
|
+
"idSample"
|
|
2760
|
+
];
|
|
2761
|
+
function buildInputs(selections, fieldNames) {
|
|
2762
|
+
let documentCount = 0;
|
|
2763
|
+
const inputs = selections.map((selection) => {
|
|
2764
|
+
const count = selection.segment.arrays.documentCount;
|
|
2765
|
+
const remap = new Int32Array(count).fill(-1);
|
|
2766
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
2767
|
+
if (selectionKeeps(selection, ordinal)) {
|
|
2768
|
+
remap[ordinal] = documentCount;
|
|
2769
|
+
documentCount += 1;
|
|
2770
|
+
}
|
|
2771
|
+
}
|
|
2772
|
+
return { segment: selection.segment, remap, fieldIndexRemap: fieldIndexRemapOf(selection.segment, fieldNames) };
|
|
2773
|
+
});
|
|
2774
|
+
return { inputs, documentCount };
|
|
2775
|
+
}
|
|
2776
|
+
function postingUpperBound(inputs) {
|
|
2777
|
+
let total = 0;
|
|
2778
|
+
for (const { segment } of inputs) {
|
|
2779
|
+
const { postingOffsets, tokenTable } = segment.arrays;
|
|
2780
|
+
if (tokenTable.size > 0) {
|
|
2781
|
+
let largest = 0;
|
|
2782
|
+
for (let at = 0; at < postingOffsets.length; at++) {
|
|
2783
|
+
if (postingOffsets[at] > largest) largest = postingOffsets[at];
|
|
2784
|
+
}
|
|
2785
|
+
total += largest;
|
|
2786
|
+
}
|
|
2787
|
+
}
|
|
2788
|
+
return total;
|
|
2789
|
+
}
|
|
2790
|
+
function onlyBytes(out) {
|
|
2791
|
+
const joined = new Uint8Array(out.byteLength);
|
|
2792
|
+
let at = 0;
|
|
2793
|
+
for (const chunk of out.chunks) {
|
|
2794
|
+
joined.set(chunk, at);
|
|
2795
|
+
at += chunk.byteLength;
|
|
2796
|
+
}
|
|
2797
|
+
return joined;
|
|
2798
|
+
}
|
|
2799
|
+
function keepsRecordsOf(request, inputs) {
|
|
2800
|
+
if (request.keepsWriteRecords === true || request.writeRecords !== void 0) return true;
|
|
2801
|
+
return inputs.length > 0 && inputs.every((input) => input.segment.paged?.writeRecordsHeld === true);
|
|
2802
|
+
}
|
|
2803
|
+
async function writePagedSegment(request) {
|
|
2804
|
+
const { directory, key } = request;
|
|
2805
|
+
await checkPagedSelections(request.selections);
|
|
2806
|
+
const fieldNames = [];
|
|
2807
|
+
const { inputs, documentCount } = buildInputs(request.selections, fieldNames);
|
|
2808
|
+
const lengths = mergePagedLengths(inputs, documentCount);
|
|
2809
|
+
const keepsRecords = keepsRecordsOf(request, inputs);
|
|
2810
|
+
const sinks = [];
|
|
2811
|
+
const finished = /* @__PURE__ */ new Map();
|
|
2812
|
+
const residentBytes = /* @__PURE__ */ new Map();
|
|
2813
|
+
const temporaries = [];
|
|
2814
|
+
async function sink(name) {
|
|
2815
|
+
const temporaryKey = `${key}.part-${name}`;
|
|
2816
|
+
temporaries.push(temporaryKey);
|
|
2817
|
+
const opened = await openSectionSink(directory, temporaryKey);
|
|
2818
|
+
sinks.push(opened);
|
|
2819
|
+
return opened;
|
|
2820
|
+
}
|
|
2821
|
+
let removals = null;
|
|
2822
|
+
const temporaryAssembly = `${key}.part-assembled`;
|
|
2823
|
+
try {
|
|
2824
|
+
const postings = await sink("postings");
|
|
2825
|
+
const tokenTotals = await sink("tokenTotals");
|
|
2826
|
+
const tracksPositions = inputs.some(
|
|
2827
|
+
(input) => input.segment.paged?.snapshot.tracksPositions ?? input.segment.arrays.positionOffsets !== null
|
|
2828
|
+
);
|
|
2829
|
+
const positionOffsets = tracksPositions ? await sink("positionOffsets") : null;
|
|
2830
|
+
const positions = tracksPositions ? await sink("positions") : null;
|
|
2831
|
+
const documents = await sink("documents");
|
|
2832
|
+
const ids = await sink("ids");
|
|
2833
|
+
const idOffsets = await sink("idOffsets");
|
|
2834
|
+
const idOrder = await sink("idOrder");
|
|
2835
|
+
const removalOffsets = await sink("removalOffsets");
|
|
2836
|
+
const removalValues = await sink("removals");
|
|
2837
|
+
const seqNos = keepsRecords ? await sink("seqNos") : null;
|
|
2838
|
+
const primaryTerms = keepsRecords ? await sink("primaryTerms") : null;
|
|
2839
|
+
removals = await openRemovalWriter(directory, key, documentCount, postingUpperBound(inputs));
|
|
2840
|
+
const lengthByName = /* @__PURE__ */ new Map();
|
|
2841
|
+
for (let at = 0; at < lengths.fieldLengthNames.length; at++) {
|
|
2842
|
+
lengthByName.set(lengths.fieldLengthNames[at], lengths.fieldLengthColumns[at]);
|
|
2843
|
+
}
|
|
2844
|
+
const columnsByField = fieldNames.map((name) => lengthByName.get(name) ?? null);
|
|
2845
|
+
const columns = await writeMergedLists(
|
|
2846
|
+
inputs,
|
|
2847
|
+
documentCount,
|
|
2848
|
+
columnsByField,
|
|
2849
|
+
{ postings, tokenTotals, positionOffsets, positions },
|
|
2850
|
+
removals,
|
|
2851
|
+
false
|
|
2852
|
+
);
|
|
2853
|
+
await removals.drain(async (offsets, values) => {
|
|
2854
|
+
await removalOffsets.write(littleEndianBytes(offsets));
|
|
2855
|
+
await removalValues.write(littleEndianBytes(values));
|
|
2856
|
+
});
|
|
2857
|
+
const documentBlocks = await writeDocumentRows(
|
|
2858
|
+
inputs,
|
|
2859
|
+
{ documents, ids, idOffsets, seqNos, primaryTerms },
|
|
2860
|
+
request.writeRecords
|
|
2861
|
+
);
|
|
2862
|
+
const idSample = await writeIdOrder(inputs, idOrder);
|
|
2863
|
+
const { surfaceTable, partialSpellings } = mergePagedStrings(
|
|
2864
|
+
inputs,
|
|
2865
|
+
documentCount,
|
|
2866
|
+
columns.tokenTable,
|
|
2867
|
+
request.partialWordAnalysis,
|
|
2868
|
+
request.textAnalysis
|
|
2869
|
+
);
|
|
2870
|
+
if (surfaceTable !== null && surfaceTable.counts.length > 0) {
|
|
2871
|
+
const surfaceChunks = new MessagePackChunks();
|
|
2872
|
+
writeSurfaceForms(surfaceChunks, { surfaceTable });
|
|
2873
|
+
const surfaceForms = await sink("surfaceForms");
|
|
2874
|
+
await surfaceForms.write(onlyBytes(surfaceChunks));
|
|
2875
|
+
}
|
|
2876
|
+
const partialTable = partialSpellings === null ? null : buildPartialWordTableFrom(partialSpellings, wrapFrozenTokenTable(columns.tokenTable));
|
|
2877
|
+
const fields = mergeFieldIndexes(inputs, documentCount);
|
|
2878
|
+
const resident = (name, write) => {
|
|
2879
|
+
const out = new MessagePackChunks();
|
|
2880
|
+
write(out);
|
|
2881
|
+
residentBytes.set(name, onlyBytes(out));
|
|
2882
|
+
};
|
|
2883
|
+
residentBytes.set("tokens", columns.tokenTable.blob);
|
|
2884
|
+
residentBytes.set("tokenOffsets", littleEndianBytes(columns.tokenTable.offsets));
|
|
2885
|
+
residentBytes.set("documentFrequencies", littleEndianBytes(columns.tokenTable.documentFrequencies));
|
|
2886
|
+
residentBytes.set("postingOffsets", littleEndianBytes(columns.postingOffsets));
|
|
2887
|
+
residentBytes.set("blockLists", littleEndianBytes(columns.blockLists));
|
|
2888
|
+
residentBytes.set("blockListOffsets", littleEndianBytes(columns.blockListOffsets));
|
|
2889
|
+
residentBytes.set("blockSummaries", columns.blockSummaries);
|
|
2890
|
+
resident("fieldLengths", (out) => writeFieldLengths(out, lengths));
|
|
2891
|
+
resident("numeric", (out) => writeValueColumns(out, fields.numeric));
|
|
2892
|
+
resident("sizes", (out) => writeValueColumns(out, fields.sizes));
|
|
2893
|
+
resident("boolean", (out) => writeBooleanColumns(out, fields.boolean));
|
|
2894
|
+
resident("enum", (out) => writeEnumColumns(out, fields.enums));
|
|
2895
|
+
resident("geopoint", (out) => writeGeopointColumns(out, fields.geo));
|
|
2896
|
+
resident("pattern", (out) => writePatternColumns(out, fields.pattern));
|
|
2897
|
+
resident("partialWords", (out) => writePartialWords(out, { partialWords: partialTable }));
|
|
2898
|
+
residentBytes.set("documentBlocks", documentBlocks);
|
|
2899
|
+
residentBytes.set("idSample", idSample);
|
|
2900
|
+
for (const opened of sinks) {
|
|
2901
|
+
const done = await opened.finish();
|
|
2902
|
+
const name = done.temporaryKey.slice(`${key}.part-`.length);
|
|
2903
|
+
finished.set(name, done);
|
|
2904
|
+
}
|
|
2905
|
+
const planned = [];
|
|
2906
|
+
for (const name of RESIDENT_SECTION_ORDER) {
|
|
2907
|
+
const bytes = residentBytes.get(name);
|
|
2908
|
+
if (bytes === void 0) continue;
|
|
2909
|
+
planned.push({ name, length: bytes.byteLength, checksums: checksumsOfPages(bytes) });
|
|
2910
|
+
}
|
|
2911
|
+
for (const [name, done] of finished) planned.push({ name, length: done.length, checksums: done.checksums });
|
|
2912
|
+
const encoded = encodePagedSegmentHead(
|
|
2913
|
+
{ schema: request.schema, tombstones: [...request.tombstones], documentCount, fieldNames },
|
|
2914
|
+
planned
|
|
2915
|
+
);
|
|
2916
|
+
requireWithinFileLimit(HEADER_SIZE + encoded.payloadLength, `segment "${key}"`);
|
|
2917
|
+
await directory.remove(temporaryAssembly);
|
|
2918
|
+
temporaries.push(temporaryAssembly);
|
|
2919
|
+
const assembly = await directory.appendHandle(temporaryAssembly);
|
|
2920
|
+
let state = crc32Init();
|
|
2921
|
+
let cursor = 0;
|
|
2922
|
+
const appendPayload = async (bytes) => {
|
|
2923
|
+
state = crc32Update(state, bytes);
|
|
2924
|
+
await assembly.append(bytes);
|
|
2925
|
+
cursor += bytes.byteLength;
|
|
2926
|
+
};
|
|
2927
|
+
const alignTo = async (offset) => {
|
|
2928
|
+
if (offset < cursor || offset - cursor >= SECTION_ALIGNMENT_BYTES) {
|
|
2929
|
+
throw new NarsilError(
|
|
2930
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2931
|
+
`Section offset ${offset} of "${key}" does not follow byte ${cursor}`,
|
|
2932
|
+
{ key, offset, cursor }
|
|
2933
|
+
);
|
|
2934
|
+
}
|
|
2935
|
+
if (offset > cursor) await appendPayload(new Uint8Array(offset - cursor));
|
|
2936
|
+
};
|
|
2937
|
+
try {
|
|
2938
|
+
await assembly.append(new Uint8Array(HEADER_SIZE));
|
|
2939
|
+
await appendPayload(encoded.prefix);
|
|
2940
|
+
await appendPayload(encoded.head);
|
|
2941
|
+
for (const section of encoded.sections) {
|
|
2942
|
+
await alignTo(section.offset);
|
|
2943
|
+
const name = section.name;
|
|
2944
|
+
const heldBytes = residentBytes.get(name);
|
|
2945
|
+
if (heldBytes !== void 0) {
|
|
2946
|
+
await appendPayload(heldBytes);
|
|
2947
|
+
continue;
|
|
2948
|
+
}
|
|
2949
|
+
const done = finished.get(name);
|
|
2950
|
+
if (done === void 0) {
|
|
2951
|
+
throw new NarsilError(
|
|
2952
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2953
|
+
`Section "${name}" of "${key}" holds no written bytes`,
|
|
2954
|
+
{ key, section: name }
|
|
2955
|
+
);
|
|
2956
|
+
}
|
|
2957
|
+
await copySection(directory, done, appendPayload);
|
|
2958
|
+
}
|
|
2959
|
+
await assembly.sync();
|
|
2960
|
+
} finally {
|
|
2961
|
+
await assembly.close();
|
|
2962
|
+
}
|
|
2963
|
+
const checksum = crc32Final(state);
|
|
2964
|
+
const [major, minor, patch] = engineVersionParts();
|
|
2965
|
+
const header2 = writeHeader({
|
|
2966
|
+
magic: "NRSL",
|
|
2967
|
+
envelopeFormatVersion: PAGED_SEGMENT_ENVELOPE_VERSION,
|
|
2968
|
+
engineVersionMajor: major,
|
|
2969
|
+
engineVersionMinor: minor,
|
|
2970
|
+
engineVersionPatch: patch,
|
|
2971
|
+
payloadLength: encoded.payloadLength,
|
|
2972
|
+
flags: {
|
|
2973
|
+
compressionEnabled: false,
|
|
2974
|
+
compressionAlgorithm: "none",
|
|
2975
|
+
checksumPresent: true,
|
|
2976
|
+
encryptionEnabled: false
|
|
2977
|
+
},
|
|
2978
|
+
checksum,
|
|
2979
|
+
reserved: new Uint8Array(RESERVED_BYTES)
|
|
2980
|
+
});
|
|
2981
|
+
const patched = await directory.markerHandle(temporaryAssembly);
|
|
2982
|
+
try {
|
|
2983
|
+
await patched.writeSlot(0, header2);
|
|
2984
|
+
} finally {
|
|
2985
|
+
await patched.close();
|
|
2986
|
+
}
|
|
2987
|
+
await directory.move(temporaryAssembly, key);
|
|
2988
|
+
await directory.syncDirectoryOf(key);
|
|
2989
|
+
return {
|
|
2990
|
+
bytes: HEADER_SIZE + encoded.payloadLength,
|
|
2991
|
+
checksum,
|
|
2992
|
+
documentCount,
|
|
2993
|
+
tombstoneCount: request.tombstones.length
|
|
2994
|
+
};
|
|
2995
|
+
} catch (error) {
|
|
2996
|
+
await removals?.abandon();
|
|
2997
|
+
throw error;
|
|
2998
|
+
} finally {
|
|
2999
|
+
for (const opened of sinks) await opened.abandon().catch(() => void 0);
|
|
3000
|
+
for (const temporaryKey of temporaries) await directory.remove(temporaryKey).catch(() => void 0);
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
function checksumsOfPages(bytes) {
|
|
3004
|
+
const pages = Math.ceil(bytes.byteLength / SEGMENT_PAGE_BYTES);
|
|
3005
|
+
const checksums = new Uint8Array(pages * 4);
|
|
3006
|
+
const view = new DataView(checksums.buffer);
|
|
3007
|
+
for (let page = 0; page < pages; page++) {
|
|
3008
|
+
const from = page * SEGMENT_PAGE_BYTES;
|
|
3009
|
+
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + SEGMENT_PAGE_BYTES, bytes.byteLength))), true);
|
|
3010
|
+
}
|
|
3011
|
+
return checksums;
|
|
3012
|
+
}
|
|
3013
|
+
async function copySection(directory, section, append) {
|
|
3014
|
+
const { readRange } = directory;
|
|
3015
|
+
if (readRange === void 0) {
|
|
3016
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "A paged segment requires a directory that reads ranges");
|
|
3017
|
+
}
|
|
3018
|
+
for (let at = 0; at < section.length; at += FILE_ASSEMBLY_CHUNK_BYTES) {
|
|
3019
|
+
const size = Math.min(FILE_ASSEMBLY_CHUNK_BYTES, section.length - at);
|
|
3020
|
+
await append(await readRange.call(directory, section.temporaryKey, at, size));
|
|
3021
|
+
}
|
|
3022
|
+
}
|
|
3023
|
+
|
|
3024
|
+
// src/serialization/keyword-segment/decode.ts
|
|
3025
|
+
function decodeSchema(raw) {
|
|
3026
|
+
const schema = requireRecord(raw, "schema");
|
|
3027
|
+
requireValidFieldTypes(schema);
|
|
3028
|
+
return schema;
|
|
3029
|
+
}
|
|
3030
|
+
function decodeIds(raw, context) {
|
|
3031
|
+
const blob = bytesFrom(requireBytes(raw.ids, "ids", 1), context.shared);
|
|
3032
|
+
const offsets = uint32sFrom(requireBytes(raw.idOffsets, "idOffsets", 4), context.shared);
|
|
3033
|
+
const sortedOrdinals = uint32sFrom(requireBytes(raw.idOrder, "idOrder", 4), context.shared);
|
|
3034
|
+
requireCount(offsets.length, context.documentCount + 1, "idOffsets");
|
|
3035
|
+
requireCount(sortedOrdinals.length, context.documentCount, "idOrder");
|
|
3036
|
+
requireOffsets(offsets, blob.length, "idOffsets");
|
|
3037
|
+
requireUtf8Entries(blob, offsets, "ids");
|
|
3038
|
+
requirePermutation(sortedOrdinals, "idOrder");
|
|
3039
|
+
requireEntriesAscend(blob, offsets, sortedOrdinals, "ids");
|
|
3040
|
+
return { blob, offsets, sortedOrdinals };
|
|
3041
|
+
}
|
|
3042
|
+
function vectorFieldPathsOf(schema) {
|
|
3043
|
+
return Object.keys(schema).filter((fieldPath) => VECTOR_PATTERN.test(schema[fieldPath]));
|
|
3044
|
+
}
|
|
3045
|
+
function decodeDocuments(raw, schema, context) {
|
|
3046
|
+
const blob = bytesFrom(requireBytes(raw.documents, "documents", 1), context.shared);
|
|
3047
|
+
const offsets = uint32sFrom(requireBytes(raw.documentOffsets, "documentOffsets", 4), context.shared);
|
|
3048
|
+
requireCount(offsets.length, context.documentCount + 1, "documentOffsets");
|
|
3049
|
+
requireOffsets(offsets, blob.length, "documentOffsets");
|
|
3050
|
+
const discardedFields = vectorFieldPathsOf(schema);
|
|
3051
|
+
return discardedFields.length === 0 ? { blob, offsets } : { blob, offsets, discardedFields };
|
|
3052
|
+
}
|
|
3053
|
+
function identitySlots2(count, shared) {
|
|
3054
|
+
const slots = uint32sFrom(new Uint8Array(count * 4), shared);
|
|
3055
|
+
for (let at = 0; at < count; at++) slots[at] = at;
|
|
3056
|
+
return slots;
|
|
3057
|
+
}
|
|
3058
|
+
function requireDocumentFrequencies(frequencies, postingOffsets) {
|
|
3059
|
+
for (let token = 0; token < frequencies.length; token++) {
|
|
3060
|
+
const postings = postingOffsets[token + 1] - postingOffsets[token];
|
|
3061
|
+
if (frequencies[token] === 0 || frequencies[token] > postings) {
|
|
3062
|
+
corrupt(`documentFrequencies gives token ${token} a count of ${frequencies[token]} over ${postings} postings`, {
|
|
3063
|
+
token
|
|
3064
|
+
});
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
function requireFieldNumbers(fields, fieldNames) {
|
|
3069
|
+
for (let at = 0; at < fields.length; at++) {
|
|
3070
|
+
if (fields[at] >= fieldNames) corrupt(`postingFields names field ${fields[at]} of ${fieldNames}`, { at });
|
|
3071
|
+
}
|
|
3072
|
+
}
|
|
3073
|
+
function decodePostings(raw, fieldNames, context) {
|
|
3074
|
+
const blob = bytesFrom(requireBytes(raw.tokens, "tokens", 1), context.shared);
|
|
3075
|
+
const offsets = uint32sFrom(requireBytes(raw.tokenOffsets, "tokenOffsets", 4), context.shared);
|
|
3076
|
+
requireOffsets(offsets, blob.length, "tokenOffsets");
|
|
3077
|
+
requireUtf8Entries(blob, offsets, "tokens");
|
|
3078
|
+
requireEntriesAscend(blob, offsets, null, "tokens");
|
|
3079
|
+
const tokens = offsets.length - 1;
|
|
3080
|
+
const documentFrequencies = uint32sFrom(
|
|
3081
|
+
requireBytes(raw.documentFrequencies, "documentFrequencies", 4),
|
|
3082
|
+
context.shared
|
|
3083
|
+
);
|
|
3084
|
+
const postingOffsets = uint32sFrom(requireBytes(raw.postingOffsets, "postingOffsets", 4), context.shared);
|
|
3085
|
+
const postingDocIds = uint32sFrom(requireBytes(raw.postingDocuments, "postingDocuments", 4), context.shared);
|
|
3086
|
+
const postingFrequencies = uint16sFrom(requireBytes(raw.postingFrequencies, "postingFrequencies", 2), context.shared);
|
|
3087
|
+
const postingFieldIndices = uint16sFrom(requireBytes(raw.postingFields, "postingFields", 2), context.shared);
|
|
3088
|
+
requireCount(documentFrequencies.length, tokens, "documentFrequencies");
|
|
3089
|
+
requireCount(postingOffsets.length, tokens + 1, "postingOffsets");
|
|
3090
|
+
requireOffsets(postingOffsets, postingDocIds.length, "postingOffsets");
|
|
3091
|
+
requireCount(postingFrequencies.length, postingDocIds.length, "postingFrequencies");
|
|
3092
|
+
requireCount(postingFieldIndices.length, postingDocIds.length, "postingFields");
|
|
3093
|
+
requireOrdinals(postingDocIds, context.documentCount, "postingDocuments");
|
|
3094
|
+
requireFieldNumbers(postingFieldIndices, fieldNames.length);
|
|
3095
|
+
requireDocumentFrequencies(documentFrequencies, postingOffsets);
|
|
3096
|
+
const rawPositionOffsets = requireBytesOrNil(raw.positionOffsets, "positionOffsets", 4);
|
|
3097
|
+
const rawPositions = requireBytesOrNil(raw.positions, "positions", 4);
|
|
3098
|
+
if (rawPositionOffsets === null !== (rawPositions === null)) {
|
|
3099
|
+
corrupt("positionOffsets and positions must both be nil or both hold bytes");
|
|
3100
|
+
}
|
|
3101
|
+
let positionOffsets = null;
|
|
3102
|
+
let positionValues = null;
|
|
3103
|
+
if (rawPositionOffsets !== null && rawPositions !== null) {
|
|
3104
|
+
positionOffsets = uint32sFrom(rawPositionOffsets, context.shared);
|
|
3105
|
+
positionValues = uint32sFrom(rawPositions, context.shared);
|
|
3106
|
+
requireCount(positionOffsets.length, postingDocIds.length + 1, "positionOffsets");
|
|
3107
|
+
requireOffsets(positionOffsets, positionValues.length, "positionOffsets");
|
|
3108
|
+
}
|
|
3109
|
+
return {
|
|
3110
|
+
tokenTable: { blob, offsets, payloadSlots: identitySlots2(tokens, context.shared), documentFrequencies },
|
|
3111
|
+
postingOffsets,
|
|
3112
|
+
postingDocIds,
|
|
3113
|
+
postingFrequencies,
|
|
3114
|
+
postingFieldIndices,
|
|
3115
|
+
positionOffsets,
|
|
3116
|
+
positionValues
|
|
3117
|
+
};
|
|
3118
|
+
}
|
|
3119
|
+
function decodeKeywordSegment(payload, shared) {
|
|
3120
|
+
let decoded;
|
|
3121
|
+
try {
|
|
3122
|
+
decoded = decode(payload);
|
|
3123
|
+
} catch (error) {
|
|
3124
|
+
corrupt(`payload is not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
|
|
3125
|
+
}
|
|
3126
|
+
const raw = requireRecord(decoded, "payload");
|
|
3127
|
+
if (raw.v !== KEYWORD_SEGMENT_PAYLOAD_VERSION) {
|
|
3128
|
+
throw new NarsilError(
|
|
3129
|
+
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
3130
|
+
`This keyword segment payload was written in layout ${String(raw.v)}, and this build reads layout ${KEYWORD_SEGMENT_PAYLOAD_VERSION} alone`,
|
|
3131
|
+
{ version: raw.v }
|
|
3132
|
+
);
|
|
3133
|
+
}
|
|
3134
|
+
const schema = decodeSchema(raw.schema);
|
|
3135
|
+
const tombstones = requireStrings(raw.tombstones, "tombstones");
|
|
3136
|
+
const documentCount = requireUint32(raw.documentCount, "documentCount");
|
|
3137
|
+
const fieldNames = requireStrings(raw.fieldNames, "fieldNames");
|
|
3138
|
+
if (fieldNames.length > MAX_SEGMENT_FIELD_NAMES) {
|
|
3139
|
+
corrupt(`fieldNames holds ${fieldNames.length} names, above the limit of ${MAX_SEGMENT_FIELD_NAMES}`);
|
|
3140
|
+
}
|
|
3141
|
+
const context = { documentCount, shared };
|
|
3142
|
+
const postings = decodePostings(raw, fieldNames, context);
|
|
3143
|
+
const arrays = {
|
|
3144
|
+
documentCount,
|
|
3145
|
+
fieldNames,
|
|
3146
|
+
idTable: decodeIds(raw, context),
|
|
3147
|
+
documentTable: decodeDocuments(raw, schema, context),
|
|
3148
|
+
...postings,
|
|
3149
|
+
...decodeFieldLengths(raw.fieldLengths, context),
|
|
3150
|
+
numeric: decodeValueColumns(raw.numeric, "numeric", context),
|
|
3151
|
+
sizes: decodeValueColumns(raw.sizes, "sizes", context),
|
|
3152
|
+
boolean: decodeBooleanColumns(raw.boolean, context),
|
|
3153
|
+
enums: decodeEnumColumns(raw.enum, context),
|
|
3154
|
+
geo: decodeGeopointColumns(raw.geopoint, context),
|
|
3155
|
+
pattern: decodePatternColumns(raw.pattern, context),
|
|
3156
|
+
surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
|
|
3157
|
+
partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
|
|
3158
|
+
};
|
|
3159
|
+
const writeRecords = decodeWriteRecords(raw, documentCount);
|
|
3160
|
+
return writeRecords === null ? { schema, tombstones, arrays } : { schema, tombstones, arrays, writeRecords };
|
|
3161
|
+
}
|
|
3162
|
+
|
|
3163
|
+
// src/persistence/durability/segment/segment-file.ts
|
|
3164
|
+
function sharesMemoryWithWorkers() {
|
|
3165
|
+
return typeof SharedArrayBuffer === "function";
|
|
3166
|
+
}
|
|
3167
|
+
async function readSegmentFile(store, ref) {
|
|
3168
|
+
const bytes = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "segment" });
|
|
3169
|
+
const { header: header2, payload } = await openChecksummedEnvelope(bytes, {
|
|
3170
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
3171
|
+
what: `segment "${ref.key}"`,
|
|
3172
|
+
allowsCompression: true
|
|
3173
|
+
});
|
|
3174
|
+
requireListedChecksum("segment", ref.key, header2.checksum, ref.checksum);
|
|
3175
|
+
return decodeKeywordSegment(payload, sharesMemoryWithWorkers());
|
|
3176
|
+
}
|
|
3177
|
+
|
|
3178
|
+
// src/persistence/durability/segment/segment-split.ts
|
|
3179
|
+
var ORDINAL_BYTES = 4;
|
|
3180
|
+
var POSTING_BYTES = 8;
|
|
3181
|
+
var VALUE_ENTRY_BYTES = 12;
|
|
3182
|
+
var GEOPOINT_ENTRY_BYTES = 20;
|
|
3183
|
+
var TOKEN_ENTRY_BYTES = 12;
|
|
3184
|
+
var STRING_HEADER_BYTES = 5;
|
|
3185
|
+
var encoder2 = new TextEncoder();
|
|
3186
|
+
function documentBytes(segment, ordinal) {
|
|
3187
|
+
const table = segment.arrays.documentTable;
|
|
3188
|
+
if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
|
|
3189
|
+
return messagePackUpperBoundBytes(segment.documentSource.docAt(ordinal));
|
|
3190
|
+
}
|
|
3191
|
+
function addPerEntry(costs, docIds, bytes) {
|
|
3192
|
+
for (let at = 0; at < docIds.length; at++) costs[docIds[at]] += bytes;
|
|
3193
|
+
}
|
|
3194
|
+
function chargePostings(segment, costs, chargesTokens) {
|
|
3195
|
+
const arrays = segment.arrays;
|
|
3196
|
+
const positions = arrays.positionOffsets;
|
|
3197
|
+
for (let p = 0; p < arrays.postingDocIds.length; p++) {
|
|
3198
|
+
let bytes = POSTING_BYTES;
|
|
3199
|
+
if (positions !== null) bytes += ORDINAL_BYTES * (1 + positions[p + 1] - positions[p]);
|
|
3200
|
+
costs[arrays.postingDocIds[p]] += bytes;
|
|
3201
|
+
}
|
|
3202
|
+
if (!chargesTokens) return;
|
|
3203
|
+
const table = arrays.tokenTable;
|
|
3204
|
+
for (let at = 0; at < table.size; at++) {
|
|
3205
|
+
const slot = table.payloadSlot(at);
|
|
3206
|
+
const bytes = table.byteLengthAt(at) + TOKEN_ENTRY_BYTES;
|
|
3207
|
+
for (let p = arrays.postingOffsets[slot]; p < arrays.postingOffsets[slot + 1]; p++) {
|
|
3208
|
+
costs[arrays.postingDocIds[p]] += bytes;
|
|
3209
|
+
}
|
|
3210
|
+
}
|
|
3211
|
+
}
|
|
3212
|
+
function chargeColumns(segment, costs) {
|
|
3213
|
+
const arrays = segment.arrays;
|
|
3214
|
+
for (const column of [...arrays.numeric, ...arrays.sizes]) addPerEntry(costs, column.docIds, VALUE_ENTRY_BYTES);
|
|
3215
|
+
for (const column of arrays.boolean) {
|
|
3216
|
+
addPerEntry(costs, column.trueDocs, ORDINAL_BYTES);
|
|
3217
|
+
addPerEntry(costs, column.falseDocs, ORDINAL_BYTES);
|
|
3218
|
+
}
|
|
3219
|
+
for (const column of arrays.enums) addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
3220
|
+
for (const column of arrays.geo) addPerEntry(costs, column.docIds, GEOPOINT_ENTRY_BYTES);
|
|
3221
|
+
for (const column of arrays.pattern) {
|
|
3222
|
+
addPerEntry(costs, column.docIds, ORDINAL_BYTES);
|
|
3223
|
+
addPerEntry(costs, column.postings, ORDINAL_BYTES);
|
|
3224
|
+
}
|
|
3225
|
+
}
|
|
3226
|
+
function stringListBytes(values) {
|
|
3227
|
+
let bytes = STRING_HEADER_BYTES;
|
|
3228
|
+
for (const value of values) bytes += STRING_HEADER_BYTES + encoder2.encode(value).byteLength;
|
|
3229
|
+
return bytes;
|
|
3230
|
+
}
|
|
3231
|
+
function fixedBytesOf(segment, chargesTokens) {
|
|
3232
|
+
const arrays = segment.arrays;
|
|
3233
|
+
let bytes = stringListBytes(arrays.fieldNames);
|
|
3234
|
+
if (!chargesTokens) bytes += arrays.tokenTable.bytes;
|
|
3235
|
+
for (const column of arrays.enums) bytes += stringListBytes(column.values) + column.offsets.byteLength;
|
|
3236
|
+
for (const column of arrays.pattern) bytes += column.runs.byteLength + column.offsets.byteLength;
|
|
3237
|
+
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
3238
|
+
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
3239
|
+
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
3240
|
+
return bytes + segment.partialWords.bytes();
|
|
3241
|
+
}
|
|
3242
|
+
function keptDocumentsOf(selection, chargesTokens) {
|
|
3243
|
+
const { segment } = selection;
|
|
3244
|
+
const count = segment.arrays.documentCount;
|
|
3245
|
+
const costs = new Float64Array(count);
|
|
3246
|
+
const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
|
|
3247
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
3248
|
+
costs[ordinal] = perDocument + encoder2.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
|
|
3249
|
+
}
|
|
3250
|
+
chargePostings(segment, costs, chargesTokens);
|
|
3251
|
+
chargeColumns(segment, costs);
|
|
3252
|
+
const ordinals = [];
|
|
3253
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
3254
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
3255
|
+
costs[ordinal] += documentBytes(segment, ordinal);
|
|
3256
|
+
ordinals.push(ordinal);
|
|
3257
|
+
}
|
|
3258
|
+
return { segment, ordinals, costs };
|
|
3259
|
+
}
|
|
3260
|
+
function selectionOf(segment, ordinals) {
|
|
3261
|
+
const marked = new Uint8Array(segment.arrays.documentCount);
|
|
3262
|
+
for (const ordinal of ordinals) marked[ordinal] = 1;
|
|
3263
|
+
return { segment, keep: (ordinal) => marked[ordinal] === 1 };
|
|
3264
|
+
}
|
|
3265
|
+
function tooLargeForOneFile(cost, budget) {
|
|
3266
|
+
throw new NarsilError(
|
|
3267
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3268
|
+
`A document needs up to ${Math.ceil(cost)} bytes of a segment, more than the ${Math.max(0, Math.floor(budget))} that one file leaves it`,
|
|
3269
|
+
{ bytes: Math.ceil(cost), budget: Math.floor(budget), maximum: MAX_FILE_BYTES }
|
|
3270
|
+
);
|
|
3271
|
+
}
|
|
3272
|
+
function splitIntoSegmentFiles(selections, schema, tombstones) {
|
|
3273
|
+
let tokenBytes = 0;
|
|
3274
|
+
for (const { segment } of selections) tokenBytes += segment.arrays.tokenTable.bytes;
|
|
3275
|
+
const chargesTokens = tokenBytes > MAX_FILE_BYTES * SEGMENT_TOKEN_TABLE_SHARE_OF_FILE;
|
|
3276
|
+
let fixed = HEADER_SIZE + SEGMENT_FILE_FRAMING_SLACK_BYTES + encode(schema).byteLength + encode(tombstones).byteLength;
|
|
3277
|
+
for (const { segment } of selections) fixed += fixedBytesOf(segment, chargesTokens);
|
|
3278
|
+
const budget = MAX_FILE_BYTES - fixed;
|
|
3279
|
+
const groups = [];
|
|
3280
|
+
let current = [];
|
|
3281
|
+
let currentBytes = 0;
|
|
3282
|
+
let currentCount = 0;
|
|
3283
|
+
const flush = () => {
|
|
3284
|
+
if (currentCount === 0) return;
|
|
3285
|
+
groups.push({
|
|
3286
|
+
selections: current.map((entry) => selectionOf(entry.segment, entry.ordinals)),
|
|
3287
|
+
documentCount: currentCount
|
|
3288
|
+
});
|
|
3289
|
+
current = [];
|
|
3290
|
+
currentBytes = 0;
|
|
3291
|
+
currentCount = 0;
|
|
3292
|
+
};
|
|
3293
|
+
for (const selection of selections) {
|
|
3294
|
+
const kept = keptDocumentsOf(selection, chargesTokens);
|
|
3295
|
+
let entry = null;
|
|
3296
|
+
for (const ordinal of kept.ordinals) {
|
|
3297
|
+
const cost = kept.costs[ordinal];
|
|
3298
|
+
if (cost > budget) tooLargeForOneFile(cost, budget);
|
|
3299
|
+
if (currentBytes + cost > budget) {
|
|
3300
|
+
flush();
|
|
3301
|
+
entry = null;
|
|
3302
|
+
}
|
|
3303
|
+
if (entry === null) {
|
|
3304
|
+
entry = { segment: kept.segment, ordinals: [] };
|
|
3305
|
+
current.push(entry);
|
|
3306
|
+
}
|
|
3307
|
+
entry.ordinals.push(ordinal);
|
|
3308
|
+
currentBytes += cost;
|
|
3309
|
+
currentCount += 1;
|
|
3310
|
+
}
|
|
3311
|
+
}
|
|
3312
|
+
flush();
|
|
3313
|
+
return groups;
|
|
3314
|
+
}
|
|
3315
|
+
function halvesOf(group) {
|
|
3316
|
+
const flat = [];
|
|
3317
|
+
for (const selection of group.selections) {
|
|
3318
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
3319
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
3320
|
+
flat.push({ segment: selection.segment, ordinal });
|
|
3321
|
+
}
|
|
3322
|
+
}
|
|
3323
|
+
const middle = Math.ceil(flat.length / 2);
|
|
3324
|
+
const groupOf = (entries) => {
|
|
3325
|
+
const bySegment = /* @__PURE__ */ new Map();
|
|
3326
|
+
for (const { segment, ordinal } of entries) {
|
|
3327
|
+
let ordinals = bySegment.get(segment);
|
|
3328
|
+
if (ordinals === void 0) {
|
|
3329
|
+
ordinals = [];
|
|
3330
|
+
bySegment.set(segment, ordinals);
|
|
3331
|
+
}
|
|
3332
|
+
ordinals.push(ordinal);
|
|
3333
|
+
}
|
|
3334
|
+
return {
|
|
3335
|
+
selections: [...bySegment].map(([segment, ordinals]) => selectionOf(segment, ordinals)),
|
|
3336
|
+
documentCount: entries.length
|
|
3337
|
+
};
|
|
3338
|
+
};
|
|
3339
|
+
return [groupOf(flat.slice(0, middle)), groupOf(flat.slice(middle))];
|
|
3340
|
+
}
|
|
3341
|
+
|
|
3342
|
+
// src/persistence/durability/segment/segment-write.ts
|
|
3343
|
+
function emptyPayload() {
|
|
3344
|
+
return {
|
|
3345
|
+
documentCount: 0,
|
|
3346
|
+
docIds: [],
|
|
3347
|
+
fieldNames: [],
|
|
3348
|
+
tokenTable: {
|
|
3349
|
+
blob: new Uint8Array(0),
|
|
3350
|
+
offsets: new Uint32Array(1),
|
|
3351
|
+
payloadSlots: new Uint32Array(0),
|
|
3352
|
+
documentFrequencies: new Uint32Array(0)
|
|
3353
|
+
},
|
|
3354
|
+
postingOffsets: new Uint32Array(1),
|
|
3355
|
+
postingDocIds: new Uint32Array(0),
|
|
3356
|
+
postingFrequencies: new Uint16Array(0),
|
|
3357
|
+
postingFieldIndices: new Uint16Array(0),
|
|
3358
|
+
positionOffsets: null,
|
|
3359
|
+
positionValues: null,
|
|
3360
|
+
fieldLengthNames: [],
|
|
3361
|
+
fieldLengthColumns: [],
|
|
3362
|
+
totalFieldLengths: {},
|
|
3363
|
+
surfaceTable: null,
|
|
3364
|
+
partialSpellings: null,
|
|
3365
|
+
numeric: [],
|
|
3366
|
+
boolean: [],
|
|
3367
|
+
enums: [],
|
|
3368
|
+
geo: [],
|
|
3369
|
+
pattern: [],
|
|
3370
|
+
sizes: []
|
|
3371
|
+
};
|
|
3372
|
+
}
|
|
3373
|
+
function noSpellingsOf(analysis) {
|
|
3374
|
+
if (analysis === null || analysis.fields.length === 0) return null;
|
|
3375
|
+
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
3376
|
+
}
|
|
3377
|
+
function writeRecordsOfGroup(group, records, count) {
|
|
3378
|
+
const seqNos = new Float64Array(count);
|
|
3379
|
+
const primaryTerms = new Float64Array(count);
|
|
3380
|
+
let at = 0;
|
|
3381
|
+
for (const selection of group?.selections ?? []) {
|
|
3382
|
+
const source = records.get(selection.segment);
|
|
3383
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
3384
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
3385
|
+
if (source !== void 0) {
|
|
3386
|
+
seqNos[at] = source.seqNos[ordinal];
|
|
3387
|
+
primaryTerms[at] = source.primaryTerms[ordinal];
|
|
3388
|
+
}
|
|
3389
|
+
at += 1;
|
|
3390
|
+
}
|
|
3391
|
+
}
|
|
3392
|
+
return { seqNos, primaryTerms };
|
|
3393
|
+
}
|
|
3394
|
+
function withWriteRecords(contents, group, request) {
|
|
3395
|
+
if (request.writeRecords === void 0) return contents;
|
|
3396
|
+
const writeRecords = writeRecordsOfGroup(group, request.writeRecords, contents.arrays.documentCount);
|
|
3397
|
+
return { ...contents, writeRecords };
|
|
3398
|
+
}
|
|
3399
|
+
function contentsOfGroup(group, request, tombstones) {
|
|
3400
|
+
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
3401
|
+
if (group === null) {
|
|
3402
|
+
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
3403
|
+
return {
|
|
3404
|
+
schema,
|
|
3405
|
+
tombstones,
|
|
3406
|
+
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
3407
|
+
};
|
|
3408
|
+
}
|
|
3409
|
+
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
3410
|
+
return {
|
|
3411
|
+
schema,
|
|
3412
|
+
tombstones,
|
|
3413
|
+
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
3414
|
+
};
|
|
3415
|
+
}
|
|
3416
|
+
function requireRoomForAnotherSegment(state) {
|
|
3417
|
+
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
3418
|
+
throw new NarsilError(
|
|
3419
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3420
|
+
`Partition ${state.request.partitionId} would hold more than ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
3421
|
+
{ partitionId: state.request.partitionId, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
3422
|
+
);
|
|
3423
|
+
}
|
|
3424
|
+
async function writePagedGroup(state, group) {
|
|
3425
|
+
const { request } = state;
|
|
3426
|
+
const { directory } = request.store;
|
|
3427
|
+
if (directory === void 0) {
|
|
3428
|
+
throw new NarsilError(
|
|
3429
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
3430
|
+
`Index "${request.indexName}" stores its text on disk, which needs a durable directory`,
|
|
3431
|
+
{ indexName: request.indexName }
|
|
3432
|
+
);
|
|
3433
|
+
}
|
|
3434
|
+
requireRoomForAnotherSegment(state);
|
|
3435
|
+
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
3436
|
+
const id = state.nextSegmentId;
|
|
3437
|
+
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
3438
|
+
const written = await writePagedSegment({
|
|
3439
|
+
directory,
|
|
3440
|
+
key,
|
|
3441
|
+
schema: request.schema,
|
|
3442
|
+
tombstones,
|
|
3443
|
+
selections: group?.selections ?? [],
|
|
3444
|
+
partialWordAnalysis: request.partialWordAnalysis,
|
|
3445
|
+
textAnalysis: request.textAnalysis,
|
|
3446
|
+
...request.writeRecords === void 0 ? {} : { writeRecords: request.writeRecords },
|
|
3447
|
+
...request.keepsWriteRecords === void 0 ? {} : { keepsWriteRecords: request.keepsWriteRecords }
|
|
3448
|
+
});
|
|
3449
|
+
state.segments.push({
|
|
3450
|
+
id,
|
|
3451
|
+
key,
|
|
3452
|
+
bytes: written.bytes,
|
|
3453
|
+
checksum: written.checksum,
|
|
3454
|
+
docCount: written.documentCount,
|
|
3455
|
+
tombstoneCount: written.tombstoneCount
|
|
3456
|
+
});
|
|
3457
|
+
state.nextSegmentId = id + 1;
|
|
3458
|
+
state.tombstonesWritten = true;
|
|
3459
|
+
}
|
|
3460
|
+
function pagedGroupsOf(request) {
|
|
3461
|
+
const held = request.selections.filter((selection) => selection.segment.paged !== null);
|
|
3462
|
+
if (request.mergesIntoOneFile === true) {
|
|
3463
|
+
return [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }];
|
|
3464
|
+
}
|
|
3465
|
+
const inMemory = request.selections.filter((selection) => selection.segment.paged === null);
|
|
3466
|
+
return [
|
|
3467
|
+
...held.map((selection) => ({ selections: [selection], documentCount: keptDocumentCount([selection]) })),
|
|
3468
|
+
...splitIntoSegmentFiles(inMemory, request.schema, request.tombstones)
|
|
3469
|
+
];
|
|
3470
|
+
}
|
|
3471
|
+
async function writeGroup(state, group) {
|
|
3472
|
+
const { request } = state;
|
|
3473
|
+
if (request.paged === true) {
|
|
3474
|
+
try {
|
|
3475
|
+
await writePagedGroup(state, group);
|
|
3476
|
+
} catch (error) {
|
|
3477
|
+
if (!(error instanceof SegmentRefusal) || error.status !== 8 || group === null || group.documentCount < 2)
|
|
3478
|
+
throw error;
|
|
3479
|
+
const [first, second] = halvesOf(group);
|
|
3480
|
+
await writeGroup(state, first);
|
|
3481
|
+
await writeGroup(state, second);
|
|
3482
|
+
}
|
|
3483
|
+
return;
|
|
3484
|
+
}
|
|
3485
|
+
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
3486
|
+
let contents;
|
|
3487
|
+
try {
|
|
3488
|
+
contents = withWriteRecords(contentsOfGroup(group, request, tombstones), group, request);
|
|
3489
|
+
} catch (error) {
|
|
3490
|
+
if (!(error instanceof SegmentRefusal) || error.status !== 8 || group === null || group.documentCount < 2)
|
|
3491
|
+
throw error;
|
|
3492
|
+
const [first, second] = halvesOf(group);
|
|
3493
|
+
await writeGroup(state, first);
|
|
3494
|
+
await writeGroup(state, second);
|
|
3495
|
+
return;
|
|
3496
|
+
}
|
|
3497
|
+
const encoded = encodeKeywordSegment(contents);
|
|
3498
|
+
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
3499
|
+
const [first, second] = halvesOf(group);
|
|
3500
|
+
await writeGroup(state, first);
|
|
3501
|
+
await writeGroup(state, second);
|
|
3502
|
+
return;
|
|
3503
|
+
}
|
|
3504
|
+
requireRoomForAnotherSegment(state);
|
|
3505
|
+
const id = state.nextSegmentId;
|
|
3506
|
+
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
3507
|
+
const packed = await packChecksummedEnvelope(encoded.chunks, KEYWORD_SEGMENT_ENVELOPE_VERSION, `segment "${key}"`);
|
|
3508
|
+
await request.store.write(key, packed.chunks);
|
|
3509
|
+
state.segments.push({
|
|
3510
|
+
id,
|
|
3511
|
+
key,
|
|
3512
|
+
bytes: packed.byteLength,
|
|
3513
|
+
checksum: packed.checksum,
|
|
3514
|
+
docCount: contents.arrays.documentCount,
|
|
3515
|
+
tombstoneCount: tombstones.length
|
|
3516
|
+
});
|
|
3517
|
+
state.nextSegmentId = id + 1;
|
|
3518
|
+
state.tombstonesWritten = true;
|
|
3519
|
+
}
|
|
3520
|
+
function keptDocumentCount(selections) {
|
|
3521
|
+
let count = 0;
|
|
3522
|
+
for (const selection of selections) {
|
|
3523
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
3524
|
+
if (selectionKeeps(selection, ordinal)) count += 1;
|
|
3525
|
+
}
|
|
3526
|
+
}
|
|
3527
|
+
return count;
|
|
3528
|
+
}
|
|
3529
|
+
async function writeSegmentsOf(request) {
|
|
3530
|
+
const state = { request, segments: [], nextSegmentId: request.nextSegmentId, tombstonesWritten: false };
|
|
3531
|
+
const groups = request.paged === true ? pagedGroupsOf(request) : request.mergesIntoOneFile === true ? [{ selections: [...request.selections], documentCount: keptDocumentCount(request.selections) }] : splitIntoSegmentFiles(request.selections, request.schema, request.tombstones);
|
|
3532
|
+
for (const group of groups) {
|
|
3533
|
+
if (group.documentCount > 0) await writeGroup(state, group);
|
|
3534
|
+
else if (request.paged === true) {
|
|
3535
|
+
await checkPagedSelections(group.selections);
|
|
3536
|
+
}
|
|
3537
|
+
}
|
|
3538
|
+
if (!state.tombstonesWritten && request.tombstones.length > 0) await writeGroup(state, null);
|
|
3539
|
+
return { segments: state.segments, nextSegmentId: state.nextSegmentId };
|
|
3540
|
+
}
|
|
3541
|
+
|
|
3542
|
+
// src/persistence/paged/compact.ts
|
|
3543
|
+
async function openSource(input, ref, cache) {
|
|
3544
|
+
const { directory, store } = input;
|
|
3545
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
3546
|
+
const listing = await pagedListingOf(directory, ref);
|
|
3547
|
+
const snapshot = openPagedSnapshot(listing);
|
|
3548
|
+
const segment2 = createPagedFrozenSegment(snapshot, cache);
|
|
3549
|
+
const opened2 = {
|
|
3550
|
+
segment: segment2,
|
|
3551
|
+
replaces: snapshot.tombstones,
|
|
3552
|
+
close: () => segment2.paged?.close()
|
|
3553
|
+
};
|
|
3554
|
+
return opened2;
|
|
3555
|
+
}
|
|
3556
|
+
const contents = await readSegmentFile(store, ref);
|
|
3557
|
+
const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, null));
|
|
3558
|
+
const ids = segment.arrays.idTable;
|
|
3559
|
+
const replaces = [...contents.tombstones];
|
|
3560
|
+
for (let ordinal = 0; ordinal < ids.count; ordinal++) replaces.push(ids.idAt(ordinal));
|
|
3561
|
+
const opened = { segment, replaces, close: () => void 0 };
|
|
3562
|
+
if (contents.writeRecords !== void 0) opened.writeRecords = contents.writeRecords;
|
|
3563
|
+
return opened;
|
|
3564
|
+
}
|
|
3565
|
+
async function mergePagedWindow(input) {
|
|
3566
|
+
const cache = createReadCache(MERGE_READ_CACHE_BYTES);
|
|
3567
|
+
const sources = [];
|
|
3568
|
+
try {
|
|
3569
|
+
for (const ref of input.refs) sources.push(await openSource(input, ref, cache));
|
|
3570
|
+
for (let newer = 1; newer < sources.length; newer++) {
|
|
3571
|
+
for (const docId of sources[newer].replaces) {
|
|
3572
|
+
for (let older = 0; older < newer; older++) sources[older].segment.tombstoneDocument(docId);
|
|
3573
|
+
}
|
|
3574
|
+
}
|
|
3575
|
+
const tombstones = /* @__PURE__ */ new Set();
|
|
3576
|
+
if (!input.dropsEveryTombstone) {
|
|
3577
|
+
for (const source of sources) {
|
|
3578
|
+
for (const docId of source.replaces) tombstones.add(docId);
|
|
3579
|
+
}
|
|
3580
|
+
}
|
|
3581
|
+
const writeRecords = /* @__PURE__ */ new Map();
|
|
3582
|
+
for (const source of sources) {
|
|
3583
|
+
if (source.writeRecords !== void 0) writeRecords.set(source.segment, source.writeRecords);
|
|
3584
|
+
}
|
|
3585
|
+
return await writeSegmentsOf({
|
|
3586
|
+
store: input.store,
|
|
3587
|
+
indexName: input.indexName,
|
|
3588
|
+
partitionId: input.partitionId,
|
|
3589
|
+
nextSegmentId: input.nextSegmentId,
|
|
3590
|
+
schema: input.schema,
|
|
3591
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
3592
|
+
textAnalysis: input.textAnalysis,
|
|
3593
|
+
selections: sources.map((source) => ({ segment: source.segment })),
|
|
3594
|
+
tombstones: [...tombstones],
|
|
3595
|
+
mergesIntoOneFile: true,
|
|
3596
|
+
paged: true,
|
|
3597
|
+
...writeRecords.size === 0 ? {} : { writeRecords }
|
|
3598
|
+
});
|
|
3599
|
+
} finally {
|
|
3600
|
+
for (const source of sources) source.close();
|
|
3601
|
+
cache.clear();
|
|
3602
|
+
}
|
|
3603
|
+
}
|
|
3604
|
+
|
|
3605
|
+
// src/persistence/durability/partition-checkpoint.ts
|
|
3606
|
+
function checkpointLastSeqNo(checkpoint, partitionId) {
|
|
3607
|
+
for (const c of checkpoint) {
|
|
3608
|
+
if (c.partitionId === partitionId) {
|
|
3609
|
+
return c.lastSeqNo;
|
|
3610
|
+
}
|
|
3611
|
+
}
|
|
3612
|
+
return 0;
|
|
3613
|
+
}
|
|
3614
|
+
function checkpointPrimaryTerm(checkpoint, partitionId) {
|
|
3615
|
+
for (const c of checkpoint) {
|
|
3616
|
+
if (c.partitionId === partitionId) {
|
|
3617
|
+
return c.primaryTerm;
|
|
3618
|
+
}
|
|
3619
|
+
}
|
|
3620
|
+
return 0;
|
|
3621
|
+
}
|
|
3622
|
+
|
|
3623
|
+
// src/persistence/durability/commit-marker.ts
|
|
3624
|
+
var COMMIT_MARKER_SLOT_SIZE = 36;
|
|
3625
|
+
var COMMIT_MARKER_SLOT_COUNT = 2;
|
|
3626
|
+
var CRC_OFFSET = 32;
|
|
3627
|
+
function encodeMarkerSlot(state) {
|
|
3628
|
+
const slot = new Uint8Array(COMMIT_MARKER_SLOT_SIZE);
|
|
3629
|
+
const view = new DataView(slot.buffer);
|
|
3630
|
+
view.setBigUint64(0, BigInt(state.writeSeq), false);
|
|
3631
|
+
view.setBigUint64(8, BigInt(state.activeSegmentSeqNo), false);
|
|
3632
|
+
view.setBigUint64(16, BigInt(state.durableByteLength), false);
|
|
3633
|
+
view.setBigUint64(24, BigInt(state.highestDurableSeqNo), false);
|
|
3634
|
+
view.setUint32(CRC_OFFSET, crc32(slot.subarray(0, CRC_OFFSET)), false);
|
|
3635
|
+
return slot;
|
|
3636
|
+
}
|
|
3637
|
+
function decodeSlot(data, slotIndex) {
|
|
3638
|
+
const base = slotIndex * COMMIT_MARKER_SLOT_SIZE;
|
|
3639
|
+
if (base + COMMIT_MARKER_SLOT_SIZE > data.length) {
|
|
3640
|
+
return null;
|
|
3641
|
+
}
|
|
3642
|
+
const slot = data.subarray(base, base + COMMIT_MARKER_SLOT_SIZE);
|
|
3643
|
+
const view = new DataView(slot.buffer, slot.byteOffset, slot.byteLength);
|
|
3644
|
+
const storedCrc = view.getUint32(CRC_OFFSET, false);
|
|
3645
|
+
if (crc32(slot.subarray(0, CRC_OFFSET)) !== storedCrc) {
|
|
3646
|
+
return null;
|
|
3647
|
+
}
|
|
3648
|
+
const writeSeq = view.getBigUint64(0, false);
|
|
3649
|
+
const activeSegmentSeqNo = view.getBigUint64(8, false);
|
|
3650
|
+
const durableByteLength = view.getBigUint64(16, false);
|
|
3651
|
+
const highestDurableSeqNo = view.getBigUint64(24, false);
|
|
3652
|
+
if (writeSeq > BigInt(Number.MAX_SAFE_INTEGER) || activeSegmentSeqNo > BigInt(Number.MAX_SAFE_INTEGER) || durableByteLength > BigInt(Number.MAX_SAFE_INTEGER) || highestDurableSeqNo > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
3653
|
+
return null;
|
|
3654
|
+
}
|
|
3655
|
+
return {
|
|
3656
|
+
writeSeq: Number(writeSeq),
|
|
3657
|
+
activeSegmentSeqNo: Number(activeSegmentSeqNo),
|
|
3658
|
+
durableByteLength: Number(durableByteLength),
|
|
3659
|
+
highestDurableSeqNo: Number(highestDurableSeqNo)
|
|
3660
|
+
};
|
|
3661
|
+
}
|
|
3662
|
+
function readCommitMarker(data) {
|
|
3663
|
+
let best = null;
|
|
3664
|
+
for (let slotIndex = 0; slotIndex < COMMIT_MARKER_SLOT_COUNT; slotIndex += 1) {
|
|
3665
|
+
const state = decodeSlot(data, slotIndex);
|
|
3666
|
+
if (state === null) {
|
|
3667
|
+
continue;
|
|
3668
|
+
}
|
|
3669
|
+
if (best === null || state.writeSeq > best.state.writeSeq) {
|
|
3670
|
+
best = { state, slotIndex };
|
|
3671
|
+
}
|
|
3672
|
+
}
|
|
3673
|
+
return best;
|
|
3674
|
+
}
|
|
3675
|
+
function nextMarkerSlot(lastWrittenSlot) {
|
|
3676
|
+
if (lastWrittenSlot === null) {
|
|
3677
|
+
return 0;
|
|
3678
|
+
}
|
|
3679
|
+
return (lastWrittenSlot + 1) % COMMIT_MARKER_SLOT_COUNT;
|
|
3680
|
+
}
|
|
3681
|
+
function markerSlotOffset(slotIndex) {
|
|
3682
|
+
return slotIndex * COMMIT_MARKER_SLOT_SIZE;
|
|
3683
|
+
}
|
|
3684
|
+
|
|
3685
|
+
// src/distribution/replication/codec.ts
|
|
3686
|
+
function createEntryMessage(entry, sourceId, indexUuid, previousPrimaryTerm) {
|
|
3687
|
+
const payload = { entry };
|
|
3688
|
+
if (indexUuid !== void 0) payload.indexUuid = indexUuid;
|
|
3689
|
+
if (previousPrimaryTerm !== void 0) payload.previousPrimaryTerm = previousPrimaryTerm;
|
|
3690
|
+
return {
|
|
3691
|
+
type: ReplicationMessageTypes.ENTRY,
|
|
3692
|
+
sourceId,
|
|
3693
|
+
requestId: generateId(),
|
|
3694
|
+
payload: encode(payload)
|
|
3695
|
+
};
|
|
3696
|
+
}
|
|
3697
|
+
function createEntryBatchMessage(entries, sourceId, indexUuid, previousPrimaryTerm) {
|
|
3698
|
+
const payload = { entries };
|
|
3699
|
+
if (indexUuid !== void 0) payload.indexUuid = indexUuid;
|
|
3700
|
+
if (previousPrimaryTerm !== void 0) payload.previousPrimaryTerm = previousPrimaryTerm;
|
|
3701
|
+
return {
|
|
3702
|
+
type: ReplicationMessageTypes.ENTRY_BATCH,
|
|
3703
|
+
sourceId,
|
|
3704
|
+
requestId: generateId(),
|
|
3705
|
+
payload: encode(payload)
|
|
3706
|
+
};
|
|
3707
|
+
}
|
|
3708
|
+
function createAckMessage(seqNo, partitionId, indexName, sourceId, requestId) {
|
|
3709
|
+
const payload = { seqNo, partitionId, indexName };
|
|
3710
|
+
return {
|
|
3711
|
+
type: ReplicationMessageTypes.ACK,
|
|
3712
|
+
sourceId,
|
|
3713
|
+
requestId,
|
|
3714
|
+
payload: encode(payload)
|
|
3715
|
+
};
|
|
3716
|
+
}
|
|
3717
|
+
function createForwardMessage(payload, sourceId) {
|
|
3718
|
+
return {
|
|
3719
|
+
type: ReplicationMessageTypes.FORWARD,
|
|
3720
|
+
sourceId,
|
|
3721
|
+
requestId: generateId(),
|
|
3722
|
+
payload: encode(payload)
|
|
3723
|
+
};
|
|
3724
|
+
}
|
|
3725
|
+
var MAX_FORWARD_BATCH_OPERATIONS = WIRE_BATCH_BUDGET.maxCount;
|
|
3726
|
+
function createForwardBatchMessage(payload, sourceId) {
|
|
3727
|
+
return {
|
|
3728
|
+
type: ReplicationMessageTypes.FORWARD_BATCH,
|
|
3729
|
+
sourceId,
|
|
3730
|
+
requestId: generateId(),
|
|
3731
|
+
payload: encode(payload)
|
|
3732
|
+
};
|
|
3733
|
+
}
|
|
3734
|
+
function createForwardBatchResultMessage(payload, sourceId, requestId) {
|
|
3735
|
+
return {
|
|
3736
|
+
type: ReplicationMessageTypes.FORWARD_BATCH,
|
|
3737
|
+
sourceId,
|
|
3738
|
+
requestId,
|
|
3739
|
+
payload: encode(payload)
|
|
3740
|
+
};
|
|
3741
|
+
}
|
|
3742
|
+
function createInsyncAddMessage(payload, sourceId) {
|
|
3743
|
+
return {
|
|
3744
|
+
type: ReplicationMessageTypes.INSYNC_ADD,
|
|
3745
|
+
sourceId,
|
|
3746
|
+
requestId: generateId(),
|
|
3747
|
+
payload: encode(payload)
|
|
3748
|
+
};
|
|
3749
|
+
}
|
|
3750
|
+
function createInsyncRemoveMessage(payload, sourceId) {
|
|
3751
|
+
return {
|
|
3752
|
+
type: ReplicationMessageTypes.INSYNC_REMOVE,
|
|
3753
|
+
sourceId,
|
|
3754
|
+
requestId: generateId(),
|
|
3755
|
+
payload: encode(payload)
|
|
3756
|
+
};
|
|
3757
|
+
}
|
|
3758
|
+
function createPrimaryResignMessage(payload, sourceId) {
|
|
3759
|
+
return {
|
|
3760
|
+
type: ReplicationMessageTypes.PRIMARY_RESIGN,
|
|
3761
|
+
sourceId,
|
|
3762
|
+
requestId: generateId(),
|
|
3763
|
+
payload: encode(payload)
|
|
3764
|
+
};
|
|
3765
|
+
}
|
|
3766
|
+
function createInsyncConfirmMessage(payload, sourceId, requestId) {
|
|
3767
|
+
return {
|
|
3768
|
+
type: ReplicationMessageTypes.INSYNC_CONFIRM,
|
|
3769
|
+
sourceId,
|
|
3770
|
+
requestId,
|
|
3771
|
+
payload: encode(payload)
|
|
3772
|
+
};
|
|
3773
|
+
}
|
|
3774
|
+
function isRecord2(value) {
|
|
3775
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
3776
|
+
}
|
|
3777
|
+
function optionalIndexUuid(value, payloadName) {
|
|
3778
|
+
if (value === void 0 || value === null) {
|
|
3779
|
+
return void 0;
|
|
3780
|
+
}
|
|
3781
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
3782
|
+
throw new Error(`Invalid ${payloadName}: "indexUuid" must be a non-empty string or absent`);
|
|
3783
|
+
}
|
|
3784
|
+
return value;
|
|
3785
|
+
}
|
|
3786
|
+
function optionalPreviousPrimaryTerm(value, payloadName) {
|
|
3787
|
+
if (value === void 0 || value === null) {
|
|
3788
|
+
return void 0;
|
|
3789
|
+
}
|
|
3790
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
3791
|
+
throw new Error(`Invalid ${payloadName}: "previousPrimaryTerm" must be a non-negative integer or absent`);
|
|
3792
|
+
}
|
|
3793
|
+
return value;
|
|
3794
|
+
}
|
|
3795
|
+
function isPlainObject(value) {
|
|
3796
|
+
if (!isRecord2(value)) {
|
|
3797
|
+
return false;
|
|
3798
|
+
}
|
|
3799
|
+
const prototype = Object.getPrototypeOf(value);
|
|
3800
|
+
return prototype === Object.prototype || prototype === null;
|
|
3801
|
+
}
|
|
3802
|
+
function validateEntryPayload(decoded) {
|
|
3803
|
+
if (!isRecord2(decoded) || !isRecord2(decoded.entry)) {
|
|
3804
|
+
throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
|
|
3805
|
+
}
|
|
3806
|
+
const entry = decoded.entry;
|
|
3807
|
+
if (typeof entry.seqNo !== "number") {
|
|
3808
|
+
throw new Error('Invalid EntryPayload: "entry.seqNo" must be a number');
|
|
3809
|
+
}
|
|
3810
|
+
if (typeof entry.primaryTerm !== "number") {
|
|
3811
|
+
throw new Error('Invalid EntryPayload: "entry.primaryTerm" must be a number');
|
|
3812
|
+
}
|
|
3813
|
+
if (entry.operation !== "INDEX" && entry.operation !== "DELETE") {
|
|
3814
|
+
throw new Error('Invalid EntryPayload: "entry.operation" must be "INDEX" or "DELETE"');
|
|
3815
|
+
}
|
|
3816
|
+
if (typeof entry.partitionId !== "number") {
|
|
3817
|
+
throw new Error('Invalid EntryPayload: "entry.partitionId" must be a number');
|
|
3818
|
+
}
|
|
3819
|
+
if (typeof entry.indexName !== "string") {
|
|
3820
|
+
throw new Error('Invalid EntryPayload: "entry.indexName" must be a string');
|
|
3821
|
+
}
|
|
3822
|
+
if (typeof entry.documentId !== "string") {
|
|
3823
|
+
throw new Error('Invalid EntryPayload: "entry.documentId" must be a string');
|
|
3824
|
+
}
|
|
3825
|
+
if (entry.document !== null && !(entry.document instanceof Uint8Array)) {
|
|
3826
|
+
throw new Error('Invalid EntryPayload: "entry.document" must be Uint8Array or null');
|
|
3827
|
+
}
|
|
3828
|
+
if (typeof entry.checksum !== "number") {
|
|
3829
|
+
throw new Error('Invalid EntryPayload: "entry.checksum" must be a number');
|
|
3830
|
+
}
|
|
3831
|
+
optionalIndexUuid(decoded.indexUuid, "EntryPayload");
|
|
3832
|
+
optionalPreviousPrimaryTerm(decoded.previousPrimaryTerm, "EntryPayload");
|
|
3833
|
+
return decoded;
|
|
3834
|
+
}
|
|
3835
|
+
function validateEntryBatchPayload(decoded) {
|
|
3836
|
+
if (!isRecord2(decoded) || !Array.isArray(decoded.entries)) {
|
|
3837
|
+
throw new Error('Invalid EntryBatchPayload: missing or invalid "entries" field');
|
|
3838
|
+
}
|
|
3839
|
+
if (decoded.entries.length === 0) {
|
|
3840
|
+
throw new Error('Invalid EntryBatchPayload: "entries" must not be empty');
|
|
3841
|
+
}
|
|
3842
|
+
const validated = decoded.entries.map((candidate) => validateEntryPayload({ entry: candidate }).entry);
|
|
3843
|
+
const first = validated[0];
|
|
3844
|
+
for (let index = 1; index < validated.length; index++) {
|
|
3845
|
+
const entry = validated[index];
|
|
3846
|
+
if (entry.indexName !== first.indexName || entry.partitionId !== first.partitionId) {
|
|
3847
|
+
throw new Error("Invalid EntryBatchPayload: entries must belong to one partition of one index");
|
|
3848
|
+
}
|
|
3849
|
+
if (entry.primaryTerm !== first.primaryTerm) {
|
|
3850
|
+
throw new Error("Invalid EntryBatchPayload: entries must share one primary term");
|
|
3851
|
+
}
|
|
3852
|
+
if (entry.seqNo !== validated[index - 1].seqNo + 1) {
|
|
3853
|
+
throw new Error("Invalid EntryBatchPayload: entries must carry contiguous ascending sequence numbers");
|
|
3854
|
+
}
|
|
3855
|
+
}
|
|
3856
|
+
const payload = { entries: validated };
|
|
3857
|
+
const indexUuid = optionalIndexUuid(decoded.indexUuid, "EntryBatchPayload");
|
|
3858
|
+
if (indexUuid !== void 0) payload.indexUuid = indexUuid;
|
|
3859
|
+
const previousPrimaryTerm = optionalPreviousPrimaryTerm(decoded.previousPrimaryTerm, "EntryBatchPayload");
|
|
3860
|
+
if (previousPrimaryTerm !== void 0) payload.previousPrimaryTerm = previousPrimaryTerm;
|
|
3861
|
+
return payload;
|
|
3862
|
+
}
|
|
3863
|
+
function validateForwardBatchPayload(decoded) {
|
|
3864
|
+
if (!isRecord2(decoded)) {
|
|
3865
|
+
throw new Error("Invalid ForwardBatchPayload: expected an object");
|
|
3866
|
+
}
|
|
3867
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
3868
|
+
throw new Error('Invalid ForwardBatchPayload: "indexName" must be a non-empty string');
|
|
3869
|
+
}
|
|
3870
|
+
if (!Array.isArray(decoded.operations)) {
|
|
3871
|
+
throw new Error('Invalid ForwardBatchPayload: "operations" must be an array');
|
|
3872
|
+
}
|
|
3873
|
+
if (decoded.operations.length === 0) {
|
|
3874
|
+
throw new Error('Invalid ForwardBatchPayload: "operations" must not be empty');
|
|
3875
|
+
}
|
|
3876
|
+
if (decoded.operations.length > MAX_FORWARD_BATCH_OPERATIONS) {
|
|
3877
|
+
throw new Error(
|
|
3878
|
+
`Invalid ForwardBatchPayload: "operations" exceeds maximum length of ${MAX_FORWARD_BATCH_OPERATIONS}`
|
|
3879
|
+
);
|
|
3880
|
+
}
|
|
3881
|
+
for (let index = 0; index < decoded.operations.length; index++) {
|
|
3882
|
+
const operation = decoded.operations[index];
|
|
3883
|
+
if (!isRecord2(operation)) {
|
|
3884
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}]" must be an object`);
|
|
3885
|
+
}
|
|
3886
|
+
if (typeof operation.documentId !== "string" || operation.documentId.length === 0) {
|
|
3887
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}].documentId" must be a non-empty string`);
|
|
3888
|
+
}
|
|
3889
|
+
if (operation.operation !== "insert" && operation.operation !== "remove" && operation.operation !== "update") {
|
|
3890
|
+
throw new Error(
|
|
3891
|
+
`Invalid ForwardBatchPayload: "operations[${index}].operation" must be "insert", "remove", or "update"`
|
|
3892
|
+
);
|
|
3893
|
+
}
|
|
3894
|
+
if (operation.document !== null && !(operation.document instanceof Uint8Array)) {
|
|
3895
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}].document" must be Uint8Array or null`);
|
|
3896
|
+
}
|
|
3897
|
+
if (operation.operation === "insert" && operation.document === null) {
|
|
3898
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}]" requires a document`);
|
|
3899
|
+
}
|
|
3900
|
+
if (operation.operation === "update" && operation.document === null && !isRecord2(operation.updateFields)) {
|
|
3901
|
+
throw new Error(`Invalid ForwardBatchPayload: "operations[${index}]" requires a document or updateFields`);
|
|
3902
|
+
}
|
|
3903
|
+
}
|
|
3904
|
+
optionalIndexUuid(decoded.indexUuid, "ForwardBatchPayload");
|
|
3905
|
+
return decoded;
|
|
3906
|
+
}
|
|
3907
|
+
function validateForwardBatchResultPayload(decoded) {
|
|
3908
|
+
if (!isRecord2(decoded)) {
|
|
3909
|
+
throw new Error("Invalid ForwardBatchResultPayload: expected an object");
|
|
3910
|
+
}
|
|
3911
|
+
if (!Array.isArray(decoded.results)) {
|
|
3912
|
+
throw new Error('Invalid ForwardBatchResultPayload: "results" must be an array');
|
|
3913
|
+
}
|
|
3914
|
+
for (let index = 0; index < decoded.results.length; index++) {
|
|
3915
|
+
const result = decoded.results[index];
|
|
3916
|
+
if (!isRecord2(result)) {
|
|
3917
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}]" must be an object`);
|
|
3918
|
+
}
|
|
3919
|
+
if (typeof result.documentId !== "string") {
|
|
3920
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].documentId" must be a string`);
|
|
3921
|
+
}
|
|
3922
|
+
if (typeof result.success !== "boolean") {
|
|
3923
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].success" must be a boolean`);
|
|
3924
|
+
}
|
|
3925
|
+
if (result.errorCode !== null && typeof result.errorCode !== "string") {
|
|
3926
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorCode" must be a string or null`);
|
|
3927
|
+
}
|
|
3928
|
+
if (result.errorMessage !== null && typeof result.errorMessage !== "string") {
|
|
3929
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorMessage" must be a string or null`);
|
|
3930
|
+
}
|
|
3931
|
+
if (result.errorDetails !== void 0 && result.errorDetails !== null && !isPlainObject(result.errorDetails)) {
|
|
3932
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorDetails" must be an object or absent`);
|
|
3933
|
+
}
|
|
3934
|
+
}
|
|
3935
|
+
return decoded;
|
|
3936
|
+
}
|
|
3937
|
+
function validateInsyncConfirmPayload(decoded) {
|
|
3938
|
+
if (!isRecord2(decoded)) {
|
|
3939
|
+
throw new Error("Invalid InsyncConfirmPayload: expected an object");
|
|
3940
|
+
}
|
|
3941
|
+
if (typeof decoded.indexName !== "string") {
|
|
3942
|
+
throw new Error('Invalid InsyncConfirmPayload: "indexName" must be a string');
|
|
3943
|
+
}
|
|
3944
|
+
if (typeof decoded.partitionId !== "number") {
|
|
3945
|
+
throw new Error('Invalid InsyncConfirmPayload: "partitionId" must be a number');
|
|
3946
|
+
}
|
|
3947
|
+
if (typeof decoded.accepted !== "boolean") {
|
|
3948
|
+
throw new Error('Invalid InsyncConfirmPayload: "accepted" must be a boolean');
|
|
3949
|
+
}
|
|
3950
|
+
return decoded;
|
|
3951
|
+
}
|
|
3952
|
+
function validateAckPayload(decoded) {
|
|
3953
|
+
if (!isRecord2(decoded)) {
|
|
3954
|
+
throw new Error("Invalid AckPayload: expected an object");
|
|
3955
|
+
}
|
|
3956
|
+
if (typeof decoded.seqNo !== "number") {
|
|
3957
|
+
throw new Error('Invalid AckPayload: "seqNo" must be a number');
|
|
3958
|
+
}
|
|
3959
|
+
if (typeof decoded.partitionId !== "number") {
|
|
3960
|
+
throw new Error('Invalid AckPayload: "partitionId" must be a number');
|
|
3961
|
+
}
|
|
3962
|
+
if (typeof decoded.indexName !== "string") {
|
|
3963
|
+
throw new Error('Invalid AckPayload: "indexName" must be a string');
|
|
3964
|
+
}
|
|
3965
|
+
return decoded;
|
|
3966
|
+
}
|
|
3967
|
+
|
|
3968
|
+
// src/distribution/replication/entry-checksum.ts
|
|
3969
|
+
function computeEntryChecksum(entry) {
|
|
3970
|
+
const payload = encode([
|
|
3971
|
+
entry.seqNo,
|
|
3972
|
+
entry.primaryTerm,
|
|
3973
|
+
entry.operation,
|
|
3974
|
+
entry.partitionId,
|
|
3975
|
+
entry.indexName,
|
|
3976
|
+
entry.documentId,
|
|
3977
|
+
entry.document
|
|
3978
|
+
]);
|
|
3979
|
+
return crc32(payload);
|
|
3980
|
+
}
|
|
3981
|
+
function buildEntry(fields) {
|
|
3982
|
+
return { ...fields, checksum: computeEntryChecksum(fields) };
|
|
3983
|
+
}
|
|
3984
|
+
|
|
3985
|
+
// src/persistence/durability/wal-framing.ts
|
|
3986
|
+
var WAL_MAGIC = new Uint8Array([78, 82, 83, 87]);
|
|
3987
|
+
var WAL_FORMAT_VERSION = 1;
|
|
3988
|
+
var SEGMENT_HEADER_SIZE = 8;
|
|
3989
|
+
var RECORD_LENGTH_SIZE = 4;
|
|
3990
|
+
var FRAME_CRC_SIZE = 4;
|
|
3991
|
+
var MAX_WAL_RECORD_BYTES = 268435456;
|
|
3992
|
+
function writeSegmentHeader() {
|
|
3993
|
+
const header2 = new Uint8Array(SEGMENT_HEADER_SIZE);
|
|
3994
|
+
header2.set(WAL_MAGIC, 0);
|
|
3995
|
+
header2[4] = WAL_FORMAT_VERSION & 255;
|
|
3996
|
+
return header2;
|
|
3997
|
+
}
|
|
3998
|
+
function frameRecord(entry) {
|
|
3999
|
+
const payload = encode(entry);
|
|
4000
|
+
const frame = new Uint8Array(RECORD_LENGTH_SIZE + payload.length + FRAME_CRC_SIZE);
|
|
4001
|
+
const view = new DataView(frame.buffer);
|
|
4002
|
+
view.setUint32(0, payload.length, false);
|
|
4003
|
+
frame.set(payload, RECORD_LENGTH_SIZE);
|
|
4004
|
+
view.setUint32(RECORD_LENGTH_SIZE + payload.length, crc32(payload), false);
|
|
4005
|
+
return frame;
|
|
4006
|
+
}
|
|
4007
|
+
function checkSegmentHeader(data) {
|
|
4008
|
+
if (data.length < SEGMENT_HEADER_SIZE) {
|
|
4009
|
+
return { ok: false, reason: "segment shorter than the 8-byte header", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
4010
|
+
}
|
|
4011
|
+
for (let i = 0; i < WAL_MAGIC.length; i += 1) {
|
|
4012
|
+
if (data[i] !== WAL_MAGIC[i]) {
|
|
4013
|
+
return { ok: false, reason: "missing NRSW magic", code: ErrorCodes.PERSISTENCE_WAL_CORRUPT };
|
|
4014
|
+
}
|
|
4015
|
+
}
|
|
4016
|
+
const formatVersion = data[4];
|
|
4017
|
+
if (formatVersion > WAL_FORMAT_VERSION) {
|
|
4018
|
+
return {
|
|
4019
|
+
ok: false,
|
|
4020
|
+
reason: `WAL format version ${formatVersion} is newer than supported version ${WAL_FORMAT_VERSION}, so a newer release wrote this segment`,
|
|
4021
|
+
code: ErrorCodes.ENVELOPE_VERSION_MISMATCH
|
|
4022
|
+
};
|
|
4023
|
+
}
|
|
4024
|
+
return { ok: true };
|
|
4025
|
+
}
|
|
4026
|
+
function decodeWalEntry(payload) {
|
|
4027
|
+
const decoded = decode(payload);
|
|
4028
|
+
const { entry } = validateEntryPayload({ entry: decoded });
|
|
4029
|
+
if (computeEntryChecksum(entry) !== entry.checksum) {
|
|
4030
|
+
throw new Error("WAL record failed its logical entry checksum");
|
|
4031
|
+
}
|
|
4032
|
+
return entry;
|
|
4033
|
+
}
|
|
4034
|
+
function corrupt2(message, details) {
|
|
4035
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, message, details);
|
|
4036
|
+
}
|
|
4037
|
+
function readDurableRegion(data, durableByteLength, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
4038
|
+
const header2 = checkSegmentHeader(data);
|
|
4039
|
+
if (!header2.ok) {
|
|
4040
|
+
throw new NarsilError(header2.code, `WAL segment header invalid in durable region: ${header2.reason}`, {
|
|
4041
|
+
reason: header2.reason
|
|
4042
|
+
});
|
|
4043
|
+
}
|
|
4044
|
+
if (durableByteLength < SEGMENT_HEADER_SIZE || durableByteLength > data.length) {
|
|
4045
|
+
corrupt2("Commit marker durable byte length is outside the segment", {
|
|
4046
|
+
durableByteLength,
|
|
4047
|
+
segmentLength: data.length
|
|
4048
|
+
});
|
|
4049
|
+
}
|
|
4050
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
4051
|
+
const entries = [];
|
|
4052
|
+
let offset = SEGMENT_HEADER_SIZE;
|
|
4053
|
+
while (offset < durableByteLength) {
|
|
4054
|
+
if (offset + RECORD_LENGTH_SIZE > durableByteLength) {
|
|
4055
|
+
corrupt2("WAL record length field overruns the durable region", { offset, durableByteLength });
|
|
4056
|
+
}
|
|
4057
|
+
const recordLength = view.getUint32(offset, false);
|
|
4058
|
+
if (recordLength === 0 || recordLength > maxRecordBytes) {
|
|
4059
|
+
corrupt2("WAL record length is zero or exceeds the maximum record size", { offset, recordLength });
|
|
4060
|
+
}
|
|
4061
|
+
const payloadStart = offset + RECORD_LENGTH_SIZE;
|
|
4062
|
+
const crcStart = payloadStart + recordLength;
|
|
4063
|
+
const frameEnd = crcStart + FRAME_CRC_SIZE;
|
|
4064
|
+
if (frameEnd > durableByteLength) {
|
|
4065
|
+
corrupt2("WAL record overruns the durable region", { offset, frameEnd, durableByteLength });
|
|
4066
|
+
}
|
|
4067
|
+
const payload = data.subarray(payloadStart, crcStart);
|
|
4068
|
+
const storedCrc = view.getUint32(crcStart, false);
|
|
4069
|
+
if (crc32(payload) !== storedCrc) {
|
|
4070
|
+
corrupt2("WAL record failed its frame checksum inside the durable region", { offset });
|
|
4071
|
+
}
|
|
4072
|
+
let entry;
|
|
4073
|
+
try {
|
|
4074
|
+
entry = decodeWalEntry(payload);
|
|
4075
|
+
} catch (err) {
|
|
4076
|
+
corrupt2("WAL record payload failed to decode inside the durable region", {
|
|
4077
|
+
offset,
|
|
4078
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
4079
|
+
});
|
|
4080
|
+
}
|
|
4081
|
+
if (entries.length > 0 && entry.seqNo <= entries[entries.length - 1].seqNo) {
|
|
4082
|
+
corrupt2("WAL sequence numbers are not strictly increasing", {
|
|
4083
|
+
previousSeqNo: entries[entries.length - 1].seqNo,
|
|
4084
|
+
seqNo: entry.seqNo
|
|
4085
|
+
});
|
|
4086
|
+
}
|
|
4087
|
+
entries.push(entry);
|
|
4088
|
+
offset = frameEnd;
|
|
4089
|
+
}
|
|
4090
|
+
return entries;
|
|
4091
|
+
}
|
|
4092
|
+
function readTailBeyondFrontier(data, durableByteLength, lastDurableSeqNo, maxRecordBytes = MAX_WAL_RECORD_BYTES) {
|
|
4093
|
+
const start = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
|
|
4094
|
+
const entries = [];
|
|
4095
|
+
if (start >= data.length) {
|
|
4096
|
+
return { entries, cleanEnd: start };
|
|
4097
|
+
}
|
|
4098
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
4099
|
+
let offset = start;
|
|
4100
|
+
let cleanEnd = start;
|
|
4101
|
+
let previousSeqNo = lastDurableSeqNo;
|
|
4102
|
+
while (offset < data.length) {
|
|
4103
|
+
if (offset + RECORD_LENGTH_SIZE > data.length) {
|
|
4104
|
+
break;
|
|
4105
|
+
}
|
|
4106
|
+
const recordLength = view.getUint32(offset, false);
|
|
4107
|
+
if (recordLength === 0 || recordLength > maxRecordBytes) {
|
|
4108
|
+
break;
|
|
4109
|
+
}
|
|
4110
|
+
const payloadStart = offset + RECORD_LENGTH_SIZE;
|
|
4111
|
+
const crcStart = payloadStart + recordLength;
|
|
4112
|
+
const frameEnd = crcStart + FRAME_CRC_SIZE;
|
|
4113
|
+
if (frameEnd > data.length) {
|
|
4114
|
+
break;
|
|
4115
|
+
}
|
|
4116
|
+
const payload = data.subarray(payloadStart, crcStart);
|
|
4117
|
+
if (crc32(payload) !== view.getUint32(crcStart, false)) {
|
|
4118
|
+
break;
|
|
4119
|
+
}
|
|
4120
|
+
let entry;
|
|
4121
|
+
try {
|
|
4122
|
+
entry = decodeWalEntry(payload);
|
|
4123
|
+
} catch {
|
|
4124
|
+
break;
|
|
4125
|
+
}
|
|
4126
|
+
if (entry.seqNo <= previousSeqNo) {
|
|
4127
|
+
break;
|
|
4128
|
+
}
|
|
4129
|
+
entries.push(entry);
|
|
4130
|
+
previousSeqNo = entry.seqNo;
|
|
4131
|
+
offset = frameEnd;
|
|
4132
|
+
cleanEnd = frameEnd;
|
|
4133
|
+
}
|
|
4134
|
+
return { entries, cleanEnd };
|
|
4135
|
+
}
|
|
4136
|
+
|
|
4137
|
+
// src/persistence/durability/wal-segments.ts
|
|
4138
|
+
function walPrefix(indexName, partitionId) {
|
|
4139
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
4140
|
+
}
|
|
4141
|
+
function nextSegmentKey(indexName, partitionId) {
|
|
4142
|
+
return `${walPrefix(indexName, partitionId)}next`;
|
|
4143
|
+
}
|
|
4144
|
+
function segmentStartSeqNo(key, prefix) {
|
|
4145
|
+
if (!key.startsWith(prefix)) {
|
|
4146
|
+
return null;
|
|
4147
|
+
}
|
|
4148
|
+
const tail = key.slice(prefix.length);
|
|
4149
|
+
if (!/^\d{16}$/.test(tail)) {
|
|
4150
|
+
return null;
|
|
4151
|
+
}
|
|
4152
|
+
const value = Number.parseInt(tail, 10);
|
|
4153
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
4154
|
+
}
|
|
4155
|
+
async function collectSegments(directory, prefix) {
|
|
4156
|
+
const keys = await directory.list(prefix);
|
|
4157
|
+
const refs = [];
|
|
4158
|
+
for (const key of keys) {
|
|
4159
|
+
const startSeqNo = segmentStartSeqNo(key, prefix);
|
|
4160
|
+
if (startSeqNo !== null) {
|
|
4161
|
+
refs.push({ key, startSeqNo });
|
|
4162
|
+
}
|
|
4163
|
+
}
|
|
4164
|
+
refs.sort((a, b) => a.startSeqNo - b.startSeqNo);
|
|
4165
|
+
return refs;
|
|
4166
|
+
}
|
|
4167
|
+
async function openWal(directory, indexName, partitionId) {
|
|
4168
|
+
const prefix = walPrefix(indexName, partitionId);
|
|
4169
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
4170
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
4171
|
+
if (marker === null) {
|
|
4172
|
+
return null;
|
|
4173
|
+
}
|
|
4174
|
+
return { marker, segments: await collectSegments(directory, prefix) };
|
|
4175
|
+
}
|
|
4176
|
+
async function* readSegmentsInOrder(directory, wal) {
|
|
4177
|
+
const { marker, segments } = wal;
|
|
4178
|
+
let highestRead = 0;
|
|
4179
|
+
for (const { key, startSeqNo } of segments) {
|
|
4180
|
+
if (startSeqNo > marker.state.activeSegmentSeqNo) {
|
|
4181
|
+
continue;
|
|
4182
|
+
}
|
|
4183
|
+
const bytes = await directory.read(key);
|
|
4184
|
+
if (bytes === null) {
|
|
4185
|
+
continue;
|
|
4186
|
+
}
|
|
4187
|
+
if (startSeqNo < marker.state.activeSegmentSeqNo) {
|
|
4188
|
+
const header2 = checkSegmentHeader(bytes);
|
|
4189
|
+
if (!header2.ok) {
|
|
4190
|
+
throw new NarsilError(header2.code, `Sealed WAL segment header invalid: ${header2.reason}`, {
|
|
4191
|
+
key,
|
|
4192
|
+
reason: header2.reason
|
|
4193
|
+
});
|
|
4194
|
+
}
|
|
4195
|
+
const durableEntries2 = readDurableRegion(bytes, bytes.length);
|
|
4196
|
+
for (const entry of durableEntries2) highestRead = Math.max(highestRead, entry.seqNo);
|
|
4197
|
+
yield { durableEntries: durableEntries2, activeTail: null };
|
|
4198
|
+
continue;
|
|
4199
|
+
}
|
|
4200
|
+
const durableEntries = readDurableRegion(bytes, marker.state.durableByteLength);
|
|
4201
|
+
for (const entry of durableEntries) highestRead = Math.max(highestRead, entry.seqNo);
|
|
4202
|
+
const tail = readTailBeyondFrontier(
|
|
4203
|
+
bytes,
|
|
4204
|
+
marker.state.durableByteLength,
|
|
4205
|
+
Math.max(highestRead, marker.state.highestDurableSeqNo)
|
|
4206
|
+
);
|
|
4207
|
+
yield {
|
|
4208
|
+
durableEntries,
|
|
4209
|
+
activeTail: { key, entries: tail.entries, cleanEnd: tail.cleanEnd, segmentLength: bytes.length }
|
|
4210
|
+
};
|
|
4211
|
+
}
|
|
4212
|
+
}
|
|
4213
|
+
async function readWalSegments(directory, indexName, partitionId) {
|
|
4214
|
+
const wal = await openWal(directory, indexName, partitionId);
|
|
4215
|
+
if (wal === null) {
|
|
4216
|
+
return null;
|
|
4217
|
+
}
|
|
4218
|
+
const durableEntries = [];
|
|
4219
|
+
let highestReadFromWal = 0;
|
|
4220
|
+
let activeTail = null;
|
|
4221
|
+
for await (const segment of readSegmentsInOrder(directory, wal)) {
|
|
4222
|
+
for (const entry of segment.durableEntries) {
|
|
4223
|
+
durableEntries.push(entry);
|
|
4224
|
+
highestReadFromWal = Math.max(highestReadFromWal, entry.seqNo);
|
|
4225
|
+
}
|
|
4226
|
+
activeTail = segment.activeTail ?? activeTail;
|
|
4227
|
+
}
|
|
4228
|
+
return { marker: wal.marker, segments: wal.segments, durableEntries, activeTail, highestReadFromWal };
|
|
4229
|
+
}
|
|
4230
|
+
function durableRecordMissing(indexName, partitionId, highestRead, marker) {
|
|
4231
|
+
return new NarsilError(
|
|
4232
|
+
ErrorCodes.PERSISTENCE_WAL_CORRUPT,
|
|
4233
|
+
"A durable WAL record is missing: the highest recovered seqNo is below the commit marker",
|
|
4234
|
+
{ indexName, partitionId, highestRead, highestDurable: marker.state.highestDurableSeqNo }
|
|
4235
|
+
);
|
|
4236
|
+
}
|
|
4237
|
+
async function* walEntriesInRange(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive) {
|
|
4238
|
+
const wal = await openWal(directory, indexName, partitionId);
|
|
4239
|
+
if (wal === null) {
|
|
4240
|
+
return;
|
|
4241
|
+
}
|
|
4242
|
+
let highestRead = 0;
|
|
4243
|
+
for await (const segment of readSegmentsInOrder(directory, wal)) {
|
|
4244
|
+
for (const entry of segment.durableEntries) {
|
|
4245
|
+
highestRead = Math.max(highestRead, entry.seqNo);
|
|
4246
|
+
if (entry.seqNo > fromSeqNoExclusive && entry.seqNo <= upToSeqNoInclusive) {
|
|
4247
|
+
yield entry;
|
|
4248
|
+
}
|
|
4249
|
+
}
|
|
4250
|
+
}
|
|
4251
|
+
if (Math.max(fromSeqNoExclusive, highestRead) < wal.marker.state.highestDurableSeqNo) {
|
|
4252
|
+
throw durableRecordMissing(indexName, partitionId, highestRead, wal.marker);
|
|
4253
|
+
}
|
|
4254
|
+
}
|
|
4255
|
+
|
|
4256
|
+
// src/persistence/durability/segment/build-segment.ts
|
|
4257
|
+
function writeRecordsOf(documents, written) {
|
|
4258
|
+
const seqNos = new Float64Array(documents.length);
|
|
4259
|
+
const primaryTerms = new Float64Array(documents.length);
|
|
4260
|
+
for (let at = 0; at < documents.length; at++) {
|
|
4261
|
+
const entry = written.get(documents[at].docId);
|
|
4262
|
+
if (entry === void 0) continue;
|
|
4263
|
+
seqNos[at] = entry.seqNo;
|
|
4264
|
+
primaryTerms[at] = entry.primaryTerm;
|
|
4265
|
+
}
|
|
4266
|
+
return { seqNos, primaryTerms };
|
|
4267
|
+
}
|
|
4268
|
+
function everyVectorFieldIsTopLevel(vectorFieldPaths) {
|
|
4269
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
4270
|
+
if (fieldPath.includes(".")) return false;
|
|
4271
|
+
}
|
|
4272
|
+
return vectorFieldPaths.size > 0;
|
|
4273
|
+
}
|
|
4274
|
+
function documentWithoutVectors(encoded, vectorFieldPaths, leavesVectorsUndecoded) {
|
|
4275
|
+
const skipped = leavesVectorsUndecoded ? decodeMapWithoutFields(encoded, vectorFieldPaths) : null;
|
|
4276
|
+
if (skipped !== null) return skipped;
|
|
4277
|
+
const document = decode(encoded);
|
|
4278
|
+
restoreVectorFields(document, vectorFieldPaths);
|
|
4279
|
+
return prepareDocumentVectors(document, vectorFieldPaths).partitionDoc;
|
|
4280
|
+
}
|
|
4281
|
+
async function buildSegmentFromEntries(input) {
|
|
4282
|
+
const leavesVectorsUndecoded = everyVectorFieldIsTopLevel(input.vectorFieldPaths);
|
|
4283
|
+
const live = /* @__PURE__ */ new Map();
|
|
4284
|
+
const deleted = /* @__PURE__ */ new Set();
|
|
4285
|
+
const written = /* @__PURE__ */ new Map();
|
|
4286
|
+
for await (const entry of input.entries) {
|
|
4287
|
+
if (entry.operation === "DELETE") {
|
|
4288
|
+
deleted.add(entry.documentId);
|
|
4289
|
+
live.delete(entry.documentId);
|
|
4290
|
+
written.delete(entry.documentId);
|
|
4291
|
+
continue;
|
|
4292
|
+
}
|
|
4293
|
+
if (entry.document === null) continue;
|
|
4294
|
+
live.delete(entry.documentId);
|
|
4295
|
+
live.set(entry.documentId, documentWithoutVectors(entry.document, input.vectorFieldPaths, leavesVectorsUndecoded));
|
|
4296
|
+
if (input.keepsWriteRecords) written.set(entry.documentId, entry);
|
|
4297
|
+
}
|
|
4298
|
+
const tombstones = [];
|
|
4299
|
+
for (const docId of deleted) {
|
|
4300
|
+
if (!live.has(docId)) tombstones.push(docId);
|
|
4301
|
+
}
|
|
4302
|
+
if (live.size === 0 && tombstones.length === 0) {
|
|
4303
|
+
return null;
|
|
4304
|
+
}
|
|
4305
|
+
const documents = [];
|
|
4306
|
+
for (const [docId, document] of live) documents.push({ docId, document });
|
|
4307
|
+
const manager = createPartitionManager(input.indexName, input.config, input.language, createPartitionRouter(), 1);
|
|
4308
|
+
const outputs = buildSegmentPayloads(
|
|
4309
|
+
documents,
|
|
4310
|
+
input.config.schema,
|
|
4311
|
+
input.language,
|
|
4312
|
+
resolvePartitionInsertOptions(input.config, manager.analysis),
|
|
4313
|
+
input.config.trackPositions ?? true
|
|
4314
|
+
);
|
|
4315
|
+
const built = { selections: [], tombstones, docCount: live.size };
|
|
4316
|
+
if (input.keepsWriteRecords) built.writeRecords = /* @__PURE__ */ new Map();
|
|
4317
|
+
for (const { payload, documents: members } of outputs) {
|
|
4318
|
+
const segment = createFrozenSegment(
|
|
4319
|
+
payload,
|
|
4320
|
+
members.map((entry) => entry.document)
|
|
4321
|
+
);
|
|
4322
|
+
built.selections.push({ segment });
|
|
4323
|
+
built.writeRecords?.set(segment, writeRecordsOf(members, written));
|
|
4324
|
+
}
|
|
4325
|
+
return built;
|
|
4326
|
+
}
|
|
4327
|
+
|
|
4328
|
+
// src/persistence/durability/segment/captured-partition.ts
|
|
4329
|
+
function keepMaskOf(segment) {
|
|
4330
|
+
const keep = new Uint8Array(segment.arrays.documentCount);
|
|
4331
|
+
for (let ordinal = 0; ordinal < keep.length; ordinal++) keep[ordinal] = segment.isTombstoned(ordinal) ? 0 : 1;
|
|
4332
|
+
return keep;
|
|
4333
|
+
}
|
|
4334
|
+
function selectionOfMask(segment, keep) {
|
|
4335
|
+
return { segment, keep: (ordinal) => keep[ordinal] === 1 };
|
|
4336
|
+
}
|
|
4337
|
+
function frozenLiveTail(live, shared) {
|
|
4338
|
+
if (live.count() === 0) return null;
|
|
4339
|
+
const payload = live.encodeSegment();
|
|
4340
|
+
const documents = survivorDocumentsOf(partitionStateOf(live), payload);
|
|
4341
|
+
if (shared) {
|
|
4342
|
+
const snapshot = freezeSegmentShared(payload, documents);
|
|
4343
|
+
if (snapshot !== null) return createSharedFrozenSegment(snapshot);
|
|
4344
|
+
}
|
|
4345
|
+
return createFrozenSegment(payload, documents);
|
|
4346
|
+
}
|
|
4347
|
+
function frozenSegmentsOf(partition, shared) {
|
|
4348
|
+
const segments = [];
|
|
4349
|
+
const live = isCompositePartition(partition) ? partition.live : partition;
|
|
4350
|
+
if (isCompositePartition(partition)) {
|
|
4351
|
+
const ids = partition.frozenSegmentSizes().map((size) => size.segmentId);
|
|
4352
|
+
segments.push(...partition.frozenSegmentsById(ids));
|
|
4353
|
+
}
|
|
4354
|
+
const tail = frozenLiveTail(live, shared);
|
|
4355
|
+
if (tail !== null) segments.push(tail);
|
|
4356
|
+
return segments;
|
|
4357
|
+
}
|
|
4358
|
+
function partitionContentsOf(manager, partitionId, captureRecords) {
|
|
4359
|
+
const partition = manager.getPartition(partitionId);
|
|
4360
|
+
const segments = frozenSegmentsOf(partition, false);
|
|
4361
|
+
const selections = segments.map((segment) => selectionOfMask(segment, keepMaskOf(segment)));
|
|
4362
|
+
const contents = { partitionId, docCount: partition.count(), selections };
|
|
4363
|
+
if (captureRecords !== void 0) contents.writeRecords = captureRecords(segments);
|
|
4364
|
+
return contents;
|
|
4365
|
+
}
|
|
4366
|
+
function captureWholePartition(manager, partitionId, captureRecords) {
|
|
4367
|
+
if (typeof SharedArrayBuffer !== "function") return null;
|
|
4368
|
+
const partition = manager.getPartition(partitionId);
|
|
4369
|
+
if (!isCompositePartition(partition) || partition.frozenSegmentCount() === 0) return null;
|
|
4370
|
+
const frozen = frozenSegmentsOf(partition, true);
|
|
4371
|
+
const records = captureRecords?.(frozen);
|
|
4372
|
+
const segments = [];
|
|
4373
|
+
for (const segment of frozen) {
|
|
4374
|
+
if (segment.sharedSnapshot === null) return null;
|
|
4375
|
+
const captured = { snapshot: segment.sharedSnapshot, keep: keepMaskOf(segment) };
|
|
4376
|
+
const held = records?.get(segment);
|
|
4377
|
+
if (held !== void 0) captured.writeRecords = held;
|
|
4378
|
+
segments.push(captured);
|
|
4379
|
+
}
|
|
4380
|
+
return { partitionId, docCount: partition.count(), segments };
|
|
4381
|
+
}
|
|
4382
|
+
function contentsOfCaptured(captured) {
|
|
4383
|
+
const writeRecords = /* @__PURE__ */ new Map();
|
|
4384
|
+
const selections = captured.segments.map(({ snapshot, keep, writeRecords: held }) => {
|
|
4385
|
+
const segment = createSharedFrozenSegment(snapshot);
|
|
4386
|
+
if (held !== void 0) writeRecords.set(segment, held);
|
|
4387
|
+
return selectionOfMask(segment, keep);
|
|
4388
|
+
});
|
|
4389
|
+
const contents = { partitionId: captured.partitionId, docCount: captured.docCount, selections };
|
|
4390
|
+
if (writeRecords.size > 0) contents.writeRecords = writeRecords;
|
|
4391
|
+
return contents;
|
|
4392
|
+
}
|
|
4393
|
+
|
|
4394
|
+
// src/persistence/durability/segment/compaction.ts
|
|
4395
|
+
function windowFits(segments, start, windowLength, paged) {
|
|
4396
|
+
let documents = 0;
|
|
4397
|
+
let bytes = SEGMENT_FILE_FRAMING_SLACK_BYTES;
|
|
4398
|
+
for (let at = start; at < start + windowLength; at++) {
|
|
4399
|
+
documents += segments[at].docCount;
|
|
4400
|
+
bytes += segments[at].bytes;
|
|
4401
|
+
}
|
|
4402
|
+
if (paged) return bytes * MERGED_SEGMENT_GROWTH_ALLOWANCE <= MERGED_SEGMENT_TARGET_BYTES;
|
|
4403
|
+
return documents <= COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT && bytes <= MAX_FILE_BYTES;
|
|
4404
|
+
}
|
|
4405
|
+
function smallestFittingWindow(segments, windowLength, paged) {
|
|
4406
|
+
let best = null;
|
|
4407
|
+
let bestBytes = Number.POSITIVE_INFINITY;
|
|
4408
|
+
for (let start = 0; start + windowLength <= segments.length; start++) {
|
|
4409
|
+
if (!windowFits(segments, start, windowLength, paged)) continue;
|
|
4410
|
+
let bytes = 0;
|
|
4411
|
+
for (let at = start; at < start + windowLength; at++) bytes += segments[at].bytes;
|
|
4412
|
+
if (bytes < bestBytes) {
|
|
4413
|
+
bestBytes = bytes;
|
|
4414
|
+
best = { start, windowLength };
|
|
4415
|
+
}
|
|
4416
|
+
}
|
|
4417
|
+
return best;
|
|
4418
|
+
}
|
|
4419
|
+
function chooseWindow(segments, wantedLength, paged) {
|
|
4420
|
+
for (let windowLength = Math.min(wantedLength, segments.length); windowLength >= 2; windowLength--) {
|
|
4421
|
+
const window = smallestFittingWindow(segments, windowLength, paged);
|
|
4422
|
+
if (window !== null) return window;
|
|
4423
|
+
}
|
|
4424
|
+
return null;
|
|
4425
|
+
}
|
|
4426
|
+
function frozenOf(contents) {
|
|
4427
|
+
return createSharedFrozenSegment(shareSegmentArrays(contents.arrays, null));
|
|
4428
|
+
}
|
|
4429
|
+
function applyTimeOrder(frozen, contents) {
|
|
4430
|
+
for (let newer = 1; newer < frozen.length; newer++) {
|
|
4431
|
+
const replaced = new Set(contents[newer].tombstones);
|
|
4432
|
+
const ids = frozen[newer].arrays.idTable;
|
|
4433
|
+
for (let ordinal = 0; ordinal < ids.count; ordinal++) replaced.add(ids.idAt(ordinal));
|
|
4434
|
+
for (let older = 0; older < newer; older++) {
|
|
4435
|
+
for (const docId of replaced) frozen[older].tombstoneDocument(docId);
|
|
4436
|
+
}
|
|
4437
|
+
}
|
|
4438
|
+
}
|
|
4439
|
+
function writeRecordsOf2(frozen, contents) {
|
|
4440
|
+
if (contents.every((segment) => segment.writeRecords === void 0)) return void 0;
|
|
4441
|
+
const records = /* @__PURE__ */ new Map();
|
|
4442
|
+
for (let at = 0; at < frozen.length; at++) {
|
|
4443
|
+
const held = contents[at].writeRecords;
|
|
4444
|
+
if (held !== void 0) records.set(frozen[at], held);
|
|
4445
|
+
}
|
|
4446
|
+
return records;
|
|
4447
|
+
}
|
|
4448
|
+
function retainedTombstones(frozen, contents) {
|
|
4449
|
+
const retained = /* @__PURE__ */ new Set();
|
|
4450
|
+
for (const segment of contents) {
|
|
4451
|
+
for (const docId of segment.tombstones) {
|
|
4452
|
+
if (!frozen.some((candidate) => candidate.hasDocument(docId))) retained.add(docId);
|
|
4453
|
+
}
|
|
4454
|
+
}
|
|
4455
|
+
return [...retained];
|
|
4456
|
+
}
|
|
4457
|
+
async function compactPartitionSegments(input) {
|
|
4458
|
+
const { segments, compactionThreshold } = input;
|
|
4459
|
+
if (segments.length <= compactionThreshold) {
|
|
4460
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
4461
|
+
}
|
|
4462
|
+
const paged = input.paged === true;
|
|
4463
|
+
const window = chooseWindow(segments, segments.length - compactionThreshold + 1, paged);
|
|
4464
|
+
if (window === null) {
|
|
4465
|
+
return { segments, nextSegmentId: input.nextSegmentId };
|
|
4466
|
+
}
|
|
4467
|
+
const windowRefs = segments.slice(window.start, window.start + window.windowLength);
|
|
4468
|
+
const replacing = (written2) => ({
|
|
4469
|
+
segments: [
|
|
4470
|
+
...segments.slice(0, window.start),
|
|
4471
|
+
...written2.segments,
|
|
4472
|
+
...segments.slice(window.start + window.windowLength)
|
|
4473
|
+
],
|
|
4474
|
+
nextSegmentId: written2.nextSegmentId
|
|
4475
|
+
});
|
|
4476
|
+
if (paged) {
|
|
4477
|
+
const { directory } = input.store;
|
|
4478
|
+
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
4479
|
+
return replacing(
|
|
4480
|
+
await mergePagedWindow({
|
|
4481
|
+
store: input.store,
|
|
4482
|
+
directory,
|
|
4483
|
+
indexName: input.indexName,
|
|
4484
|
+
partitionId: input.partitionId,
|
|
4485
|
+
schema: input.schema,
|
|
4486
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
4487
|
+
textAnalysis: input.textAnalysis,
|
|
4488
|
+
nextSegmentId: input.nextSegmentId,
|
|
4489
|
+
refs: windowRefs,
|
|
4490
|
+
dropsEveryTombstone: window.start === 0
|
|
4491
|
+
})
|
|
4492
|
+
);
|
|
4493
|
+
}
|
|
4494
|
+
const contents = [];
|
|
4495
|
+
for (const ref of windowRefs) contents.push(await readSegmentFile(input.store, ref));
|
|
4496
|
+
const frozen = contents.map(frozenOf);
|
|
4497
|
+
applyTimeOrder(frozen, contents);
|
|
4498
|
+
const writeRecords = writeRecordsOf2(frozen, contents);
|
|
4499
|
+
const written = await writeSegmentsOf({
|
|
4500
|
+
store: input.store,
|
|
4501
|
+
indexName: input.indexName,
|
|
4502
|
+
partitionId: input.partitionId,
|
|
4503
|
+
nextSegmentId: input.nextSegmentId,
|
|
4504
|
+
schema: input.schema,
|
|
4505
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
4506
|
+
textAnalysis: input.textAnalysis,
|
|
4507
|
+
selections: frozen.map((segment) => ({ segment })),
|
|
4508
|
+
tombstones: retainedTombstones(frozen, contents),
|
|
4509
|
+
mergesIntoOneFile: true,
|
|
4510
|
+
...writeRecords === void 0 ? {} : { writeRecords }
|
|
4511
|
+
});
|
|
4512
|
+
return replacing(written);
|
|
4513
|
+
}
|
|
4514
|
+
|
|
4515
|
+
// src/persistence/durability/segment/write.ts
|
|
4516
|
+
function textAnalysisOf(config) {
|
|
4517
|
+
let resolved = null;
|
|
4518
|
+
const resolve = () => {
|
|
4519
|
+
resolved ??= {
|
|
4520
|
+
language: getLanguage(config.language ?? "english"),
|
|
4521
|
+
options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
|
|
4522
|
+
};
|
|
4523
|
+
return resolved;
|
|
4524
|
+
};
|
|
4525
|
+
return {
|
|
4526
|
+
get language() {
|
|
4527
|
+
return resolve().language;
|
|
4528
|
+
},
|
|
4529
|
+
get options() {
|
|
4530
|
+
return resolve().options;
|
|
4531
|
+
}
|
|
4532
|
+
};
|
|
4533
|
+
}
|
|
4534
|
+
function partialWordAnalysisOf(config, text) {
|
|
4535
|
+
const fields = partialFieldsOf(config.schema);
|
|
4536
|
+
if (fields.length === 0) return null;
|
|
4537
|
+
return {
|
|
4538
|
+
fields,
|
|
4539
|
+
get language() {
|
|
4540
|
+
return text.language;
|
|
4541
|
+
},
|
|
4542
|
+
get options() {
|
|
4543
|
+
return text.options;
|
|
4544
|
+
}
|
|
4545
|
+
};
|
|
4546
|
+
}
|
|
4547
|
+
function resolveCompactionThreshold(configured) {
|
|
4548
|
+
if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
|
|
4549
|
+
return DEFAULT_COMPACTION_THRESHOLD;
|
|
4550
|
+
}
|
|
4551
|
+
return configured;
|
|
4552
|
+
}
|
|
4553
|
+
function wholeContentsOf(input, partitionId) {
|
|
4554
|
+
const whole = input.wholePartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
4555
|
+
if (whole !== void 0) return whole;
|
|
4556
|
+
const captured = input.capturedPartitions?.find((candidate) => candidate.partitionId === partitionId);
|
|
4557
|
+
return captured === void 0 ? void 0 : contentsOfCaptured(captured);
|
|
4558
|
+
}
|
|
4559
|
+
function pagedSettingsOf(context) {
|
|
4560
|
+
if (context.input.metadata.textStorage !== "disk") return {};
|
|
4561
|
+
return { paged: true, keepsWriteRecords: context.input.keepsWriteRecords === true };
|
|
4562
|
+
}
|
|
4563
|
+
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
4564
|
+
const { directory } = context.input.store;
|
|
4565
|
+
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
4566
|
+
return pagedTombstonesFor(
|
|
4567
|
+
directory,
|
|
4568
|
+
older,
|
|
4569
|
+
selections.map((selection) => selection.segment),
|
|
4570
|
+
deleted
|
|
4571
|
+
);
|
|
4572
|
+
}
|
|
4573
|
+
async function writeWholePartition(context, partitionId, priorPartition, whole) {
|
|
4574
|
+
const written = await writeSegmentsOf({
|
|
4575
|
+
store: context.input.store,
|
|
4576
|
+
indexName: context.indexName,
|
|
4577
|
+
partitionId,
|
|
4578
|
+
nextSegmentId: priorPartition?.nextSegmentId ?? 0,
|
|
4579
|
+
schema: context.input.metadata.schema,
|
|
4580
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
4581
|
+
textAnalysis: context.textAnalysis,
|
|
4582
|
+
selections: whole.selections,
|
|
4583
|
+
tombstones: [],
|
|
4584
|
+
...whole.writeRecords === void 0 ? {} : { writeRecords: whole.writeRecords },
|
|
4585
|
+
...pagedSettingsOf(context)
|
|
4586
|
+
});
|
|
4587
|
+
return { partitionId, nextSegmentId: written.nextSegmentId, segments: written.segments };
|
|
4588
|
+
}
|
|
4589
|
+
async function writeLoggedChanges(context, partitionId, priorPartition, entries) {
|
|
4590
|
+
const built = await buildSegmentFromEntries({
|
|
4591
|
+
indexName: context.indexName,
|
|
4592
|
+
config: context.config,
|
|
4593
|
+
language: getLanguage(context.config.language ?? "english"),
|
|
4594
|
+
vectorFieldPaths: context.vectorFieldPaths,
|
|
4595
|
+
entries,
|
|
4596
|
+
keepsWriteRecords: context.input.keepsWriteRecords === true
|
|
4597
|
+
});
|
|
4598
|
+
let segments = priorPartition ? [...priorPartition.segments] : [];
|
|
4599
|
+
let nextSegmentId = priorPartition?.nextSegmentId ?? 0;
|
|
4600
|
+
if (built !== null) {
|
|
4601
|
+
const tombstones = await tombstonesOfDelta(context, segments, built.selections, built.tombstones);
|
|
4602
|
+
const written = await writeSegmentsOf({
|
|
4603
|
+
store: context.input.store,
|
|
4604
|
+
indexName: context.indexName,
|
|
4605
|
+
partitionId,
|
|
4606
|
+
nextSegmentId,
|
|
4607
|
+
schema: context.input.metadata.schema,
|
|
4608
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
4609
|
+
textAnalysis: context.textAnalysis,
|
|
4610
|
+
selections: built.selections,
|
|
4611
|
+
tombstones,
|
|
4612
|
+
...built.writeRecords === void 0 ? {} : { writeRecords: built.writeRecords },
|
|
4613
|
+
...pagedSettingsOf(context)
|
|
4614
|
+
});
|
|
4615
|
+
segments.push(...written.segments);
|
|
4616
|
+
nextSegmentId = written.nextSegmentId;
|
|
4617
|
+
}
|
|
4618
|
+
if (segments.length > MAX_SEGMENTS_PER_PARTITION) {
|
|
4619
|
+
throw new NarsilError(
|
|
4620
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
4621
|
+
`Partition ${partitionId} would exceed the maximum of ${MAX_SEGMENTS_PER_PARTITION} segments`,
|
|
4622
|
+
{ partitionId, segmentCount: segments.length, maximum: MAX_SEGMENTS_PER_PARTITION }
|
|
4623
|
+
);
|
|
4624
|
+
}
|
|
4625
|
+
const compacted = await compactPartitionSegments({
|
|
4626
|
+
store: context.input.store,
|
|
4627
|
+
indexName: context.indexName,
|
|
4628
|
+
partitionId,
|
|
4629
|
+
schema: context.input.metadata.schema,
|
|
4630
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
4631
|
+
textAnalysis: context.textAnalysis,
|
|
4632
|
+
segments,
|
|
4633
|
+
nextSegmentId,
|
|
4634
|
+
compactionThreshold: context.compactionThreshold,
|
|
4635
|
+
paged: context.input.metadata.textStorage === "disk"
|
|
4636
|
+
});
|
|
4637
|
+
segments = compacted.segments;
|
|
4638
|
+
nextSegmentId = compacted.nextSegmentId;
|
|
4639
|
+
return { partitionId, nextSegmentId, segments };
|
|
4640
|
+
}
|
|
4641
|
+
function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkpointByPartition) {
|
|
4642
|
+
if (prior === null) return;
|
|
4643
|
+
const covered = new Set(targets.map((target) => target.partitionId));
|
|
4644
|
+
for (const priorPartition of prior.partitions) {
|
|
4645
|
+
if (covered.has(priorPartition.partitionId)) continue;
|
|
4646
|
+
partitions.push(priorPartition);
|
|
4647
|
+
const priorCheckpoint = prior.checkpoint.find((c) => c.partitionId === priorPartition.partitionId);
|
|
4648
|
+
if (priorCheckpoint !== void 0 && !checkpointByPartition.has(priorPartition.partitionId)) {
|
|
4649
|
+
checkpointByPartition.set(priorPartition.partitionId, priorCheckpoint);
|
|
4650
|
+
}
|
|
4651
|
+
}
|
|
4652
|
+
}
|
|
4653
|
+
async function writeCheckpointSegments(input) {
|
|
4654
|
+
const config = reconstructSchemaFromMetadata(input.metadata);
|
|
4655
|
+
const textAnalysis = textAnalysisOf(config);
|
|
4656
|
+
const context = {
|
|
4657
|
+
input,
|
|
4658
|
+
indexName: input.metadata.indexName,
|
|
4659
|
+
config,
|
|
4660
|
+
partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
|
|
4661
|
+
textAnalysis,
|
|
4662
|
+
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
4663
|
+
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
4664
|
+
};
|
|
4665
|
+
const checkpointByPartition = /* @__PURE__ */ new Map();
|
|
4666
|
+
for (const target of input.targets) checkpointByPartition.set(target.partitionId, { ...target });
|
|
4667
|
+
const partitions = [];
|
|
4668
|
+
for (const target of input.targets) {
|
|
4669
|
+
const priorPartition = input.prior?.partitions.find((p) => p.partitionId === target.partitionId);
|
|
4670
|
+
const whole = wholeContentsOf(input, target.partitionId);
|
|
4671
|
+
if (whole !== void 0) {
|
|
4672
|
+
partitions.push(await writeWholePartition(context, target.partitionId, priorPartition, whole));
|
|
4673
|
+
continue;
|
|
4674
|
+
}
|
|
4675
|
+
if (input.directory === void 0) {
|
|
4676
|
+
throw new NarsilError(
|
|
4677
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
4678
|
+
`The checkpoint of partition ${target.partitionId} holds neither the partition's content nor a log to read it from`,
|
|
4679
|
+
{ partitionId: target.partitionId }
|
|
4680
|
+
);
|
|
4681
|
+
}
|
|
4682
|
+
const priorSeqNo = checkpointLastSeqNo(input.prior?.checkpoint ?? [], target.partitionId);
|
|
4683
|
+
const entries = walEntriesInRange(
|
|
4684
|
+
input.directory,
|
|
4685
|
+
context.indexName,
|
|
4686
|
+
target.partitionId,
|
|
4687
|
+
priorSeqNo,
|
|
4688
|
+
target.lastSeqNo
|
|
4689
|
+
);
|
|
4690
|
+
partitions.push(await writeLoggedChanges(context, target.partitionId, priorPartition, entries));
|
|
4691
|
+
}
|
|
4692
|
+
carryForwardUncheckpointedPartitions(input.prior, input.targets, partitions, checkpointByPartition);
|
|
4693
|
+
const checkpoint = [...checkpointByPartition.values()].sort((left, right) => left.partitionId - right.partitionId);
|
|
4694
|
+
partitions.sort((left, right) => left.partitionId - right.partitionId);
|
|
4695
|
+
return { checkpoint, partitions };
|
|
4696
|
+
}
|
|
4697
|
+
async function commitCheckpointManifest(store, metadata, written, vectors) {
|
|
4698
|
+
const indexName = metadata.indexName;
|
|
4699
|
+
const manifest = {
|
|
4700
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
4701
|
+
schema: metadata.schema,
|
|
4702
|
+
language: metadata.language,
|
|
4703
|
+
...metadata.analysisGeneration ? { analysisGeneration: metadata.analysisGeneration } : {},
|
|
4704
|
+
checkpoint: written.checkpoint,
|
|
4705
|
+
partitions: written.partitions,
|
|
4706
|
+
vectors
|
|
4707
|
+
};
|
|
4708
|
+
const packed = await encodeSegmentManifest(manifest);
|
|
4709
|
+
await store.write(manifestKey(indexName), packed.chunks);
|
|
4710
|
+
return { garbage: await unreferencedKeys(store, indexName, manifest) };
|
|
4711
|
+
}
|
|
4712
|
+
async function unreferencedKeys(store, indexName, manifest) {
|
|
4713
|
+
const referenced = manifestReferencedKeys(manifest);
|
|
4714
|
+
const stored = await store.list(segmentsPrefix(indexName));
|
|
4715
|
+
return [...stored.filter((key) => !referenced.has(key)), snapshotBundleKey(indexName)];
|
|
4716
|
+
}
|
|
4717
|
+
async function removeCheckpointGarbage(store, garbage) {
|
|
4718
|
+
for (const key of garbage) {
|
|
4719
|
+
await store.remove(key);
|
|
4720
|
+
}
|
|
4721
|
+
}
|
|
4722
|
+
|
|
4723
|
+
export { FRAME_CRC_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, MAX_WAL_RECORD_BYTES, RECORD_LENGTH_SIZE, SEGMENT_HEADER_SIZE, buildEntry, bytesOfResidentSections, captureWholePartition, checkPagedContents, checkPagedSelections, checkSegmentHeader, checkpointLastSeqNo, checkpointPrimaryTerm, commitCheckpointManifest, computeEntryChecksum, createAckMessage, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createPagedFrozenSegment, createPrimaryResignMessage, decodeKeywordSegment, decodeWalEntry, durableRecordMissing, encodeMarkerSlot, frameRecord, isPagedSegmentFile, markerSlotOffset, mergePagedWindow, nextMarkerSlot, nextSegmentKey, openPagedSnapshot, optionalIndexUuid, pagedListingOf, pagedTombstonesFor, partitionContentsOf, readCommitMarker, readSegmentFile, readWalSegments, removeCheckpointGarbage, requirePagedPayloadChecksum, residentSectionsOf, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateInsyncConfirmPayload, writeCheckpointSegments, writePagedSegment, writeSegmentHeader };
|