@delali/narsil 0.3.1-next.157 → 0.3.1-next.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
- package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +66 -18
- package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1199 -1092
- package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -16
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -227
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
- package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +95 -52
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +33 -13
- package/dist/server.mjs +81 -100
- package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +88 -72
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +89 -59
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +66 -50
- package/package.json +233 -8
- package/dist/analysis-change-CKEMM2MC.mjs +0 -55
- package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
- package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-25MTVDYD.mjs +0 -432
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-AI5UI46D.mjs +0 -12
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
- package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
- package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
- package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MY2XKH22.mjs +0 -40
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
- package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
- package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
- package/dist/browser/chunk-V43N6TZO.mjs +0 -857
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
- package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-5ESVMFDJ.mjs +0 -20
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NDSQLPMQ.mjs +0 -25
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-HYXPVJVS.mjs +0 -32
- package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
- package/dist/browser/retention-TY3L3BFN.mjs +0 -211
- package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
- package/dist/browser/start-2UZTOBG5.mjs +0 -26
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-ULXAN5B4.mjs +0 -196
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-2ROUFMQX.mjs +0 -543
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-4B7JIGQD.mjs +0 -106
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BNL5OPRV.mjs +0 -634
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-C7XQYG6U.mjs +0 -50
- package/dist/chunk-CQZXZ75V.mjs +0 -562
- package/dist/chunk-CWS55TEX.mjs +0 -242
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FEZVDOWX.mjs +0 -6996
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GPWMUPCA.mjs +0 -1126
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JSV6OQ4C.mjs +0 -74
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KCQYFO4K.mjs +0 -62
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LDNCLRZU.mjs +0 -218
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MW6BR4QP.mjs +0 -316
- package/dist/chunk-MWD5VPN7.mjs +0 -21
- package/dist/chunk-ODYKO5TO.mjs +0 -101
- package/dist/chunk-OG7UGJ6F.mjs +0 -63
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-OYABFKTW.mjs +0 -1974
- package/dist/chunk-P5LANSH6.mjs +0 -121
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PIH5SJ4N.mjs +0 -7481
- package/dist/chunk-PSZI4O24.mjs +0 -253
- package/dist/chunk-PV2CG7EJ.mjs +0 -886
- package/dist/chunk-PWGYQXML.mjs +0 -470
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QLDABJZM.mjs +0 -20
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RCX4WNLC.mjs +0 -151
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RIHIVFIQ.mjs +0 -240
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RWURUDSP.mjs +0 -2639
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SOI6LTHM.mjs +0 -216
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UD5HIS4U.mjs +0 -382
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-UJDJB3NE.mjs +0 -913
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VSLSGASN.mjs +0 -30
- package/dist/chunk-VZ2ONHEE.mjs +0 -3715
- package/dist/chunk-W7OSSKH5.mjs +0 -1427
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-V5DRIECI.mjs +0 -34
- package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
- package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-GPKLTSKA.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-JYHFTVPS.mjs +0 -63
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
- package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
- package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
- package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
- package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
- package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
- package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
- package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
- package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
- package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
- package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
- package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
- package/dist/release-SMR4WJD4.mjs +0 -32
- package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
- package/dist/restore-queue-LZH7LMLR.mjs +0 -55
- package/dist/retention-3K7ARGAG.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-O7ICP7NJ.mjs +0 -783
- package/dist/start-S4PFP6RS.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-YLJ3ZPQ3.mjs +0 -50
- package/dist/write-2OBJGLFS.mjs +0 -233
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
|
@@ -0,0 +1,3196 @@
|
|
|
1
|
+
// src/core/frozen-writes/kernel-host.browser.ts
|
|
2
|
+
var PENDING_DOCUMENTS;
|
|
3
|
+
var PENDING_TOKEN_WORDS;
|
|
4
|
+
var PENDING_TOKENS;
|
|
5
|
+
var PENDING_UNITS;
|
|
6
|
+
var SegmentRefusal;
|
|
7
|
+
var checkText;
|
|
8
|
+
var compareBytes;
|
|
9
|
+
var packedTokens;
|
|
10
|
+
var pendingSettled;
|
|
11
|
+
var readFrozenParts;
|
|
12
|
+
var requireRoom;
|
|
13
|
+
var requireSegment;
|
|
14
|
+
var resolveStopWords;
|
|
15
|
+
var storedText;
|
|
16
|
+
var textAt;
|
|
17
|
+
var transformToken;
|
|
18
|
+
var visitDocumentSurfaceSpellings;
|
|
19
|
+
function bindFrozenKernelHost(host) {
|
|
20
|
+
({
|
|
21
|
+
PENDING_DOCUMENTS,
|
|
22
|
+
PENDING_TOKEN_WORDS,
|
|
23
|
+
PENDING_TOKENS,
|
|
24
|
+
PENDING_UNITS,
|
|
25
|
+
SegmentRefusal,
|
|
26
|
+
checkText,
|
|
27
|
+
compareBytes,
|
|
28
|
+
packedTokens,
|
|
29
|
+
pendingSettled,
|
|
30
|
+
readFrozenParts,
|
|
31
|
+
requireRoom,
|
|
32
|
+
requireSegment,
|
|
33
|
+
resolveStopWords,
|
|
34
|
+
storedText,
|
|
35
|
+
textAt,
|
|
36
|
+
transformToken,
|
|
37
|
+
visitDocumentSurfaceSpellings
|
|
38
|
+
} = host);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// src/core/frozen-writes/constants.ts
|
|
42
|
+
var PENDING_REMOVAL_DOCUMENTS = 64;
|
|
43
|
+
var PENDING_REMOVAL_TOKENS = 1024;
|
|
44
|
+
var PENDING_REMOVAL_UNITS = 16384;
|
|
45
|
+
var PENDING_REMOVAL_GROWTH_FACTOR = 2;
|
|
46
|
+
var PENDING_REMOVAL_GROWTH_LIMIT_BYTES = 67108864;
|
|
47
|
+
var PENDING_REMOVAL_LOW_MEMORY_LIMIT_BYTES = 65536;
|
|
48
|
+
var SETTLE_KEY_SLOTS_PER_KEY = 2;
|
|
49
|
+
var SETTLE_ORDER_CHECKS_PER_BUFFER = 8;
|
|
50
|
+
var REMOVAL_SCRATCH_START_TOKEN_WORDS = 2048;
|
|
51
|
+
var REMOVAL_SCRATCH_START_UNITS = 8192;
|
|
52
|
+
var REMOVAL_SCRATCH_START_KEYS = 256;
|
|
53
|
+
var REMOVAL_SCRATCH_START_KEY_UNITS = 2048;
|
|
54
|
+
var REMOVAL_SCRATCH_SLOTS_PER_KEY = 2;
|
|
55
|
+
var REMOVAL_SCRATCH_GROWTH_FACTOR = 2;
|
|
56
|
+
|
|
57
|
+
// src/core/frozen-writes/pending-writes.ts
|
|
58
|
+
var NO_SURFACE = 4294967295;
|
|
59
|
+
var FIELD_TOKEN_WORDS = 8;
|
|
60
|
+
function pendingNeeds(fields) {
|
|
61
|
+
let tokens = 0;
|
|
62
|
+
let units = 0;
|
|
63
|
+
for (const field of fields) {
|
|
64
|
+
for (let at = 0; at < field.tokens.length; at += FIELD_TOKEN_WORDS) {
|
|
65
|
+
tokens++;
|
|
66
|
+
units += field.tokens[at + 2];
|
|
67
|
+
if (field.tokens[at + 3] !== NO_SURFACE) units += field.tokens[at + 4];
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return { documents: 1, tokens, units };
|
|
71
|
+
}
|
|
72
|
+
function pendingFits(state, needed, empty) {
|
|
73
|
+
const header = state.pendingHeader;
|
|
74
|
+
const held = (word) => empty ? 0 : header[word];
|
|
75
|
+
return held(PENDING_DOCUMENTS) + needed.documents <= state.pendingDocuments.length && held(PENDING_TOKENS) + needed.tokens <= state.pendingTokens.length / PENDING_TOKEN_WORDS && held(PENDING_UNITS) + needed.units <= state.pendingUnits.length;
|
|
76
|
+
}
|
|
77
|
+
function grownLength(held, required, smallest) {
|
|
78
|
+
return held >= required ? held : Math.max(smallest, required * PENDING_REMOVAL_GROWTH_FACTOR);
|
|
79
|
+
}
|
|
80
|
+
function grown(held, length, create) {
|
|
81
|
+
if (held.length >= length) return held;
|
|
82
|
+
const next = new create(length);
|
|
83
|
+
next.set(held);
|
|
84
|
+
return next;
|
|
85
|
+
}
|
|
86
|
+
function growTo(state, documents, tokens, units, processHasRoomFor) {
|
|
87
|
+
const documentLength = grownLength(state.pendingDocuments.length, documents, PENDING_REMOVAL_DOCUMENTS);
|
|
88
|
+
const tokenLength = grownLength(
|
|
89
|
+
state.pendingTokens.length,
|
|
90
|
+
tokens * PENDING_TOKEN_WORDS,
|
|
91
|
+
PENDING_REMOVAL_TOKENS * PENDING_TOKEN_WORDS
|
|
92
|
+
);
|
|
93
|
+
const unitLength = grownLength(state.pendingUnits.length, units, PENDING_REMOVAL_UNITS);
|
|
94
|
+
const bytes = documentLength * Uint32Array.BYTES_PER_ELEMENT + tokenLength * Uint32Array.BYTES_PER_ELEMENT + unitLength * Uint16Array.BYTES_PER_ELEMENT;
|
|
95
|
+
if (bytes > PENDING_REMOVAL_GROWTH_LIMIT_BYTES) return false;
|
|
96
|
+
const heldBytes = state.pendingDocuments.byteLength + state.pendingTokens.byteLength + state.pendingUnits.byteLength;
|
|
97
|
+
if (bytes > PENDING_REMOVAL_LOW_MEMORY_LIMIT_BYTES && !processHasRoomFor(bytes - heldBytes)) return false;
|
|
98
|
+
state.pendingDocuments = grown(state.pendingDocuments, documentLength, Uint32Array);
|
|
99
|
+
state.pendingTokens = grown(state.pendingTokens, tokenLength, Uint32Array);
|
|
100
|
+
state.pendingUnits = grown(state.pendingUnits, unitLength, Uint16Array);
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
var growFrozenRemovalRegions = (_segment, state, fields, processHasRoomFor) => {
|
|
104
|
+
const needed = pendingNeeds(fields);
|
|
105
|
+
if (pendingFits(state, needed, false)) return;
|
|
106
|
+
const header = state.pendingHeader;
|
|
107
|
+
const heldGrows = growTo(
|
|
108
|
+
state,
|
|
109
|
+
header[PENDING_DOCUMENTS] + needed.documents,
|
|
110
|
+
header[PENDING_TOKENS] + needed.tokens,
|
|
111
|
+
header[PENDING_UNITS] + needed.units,
|
|
112
|
+
processHasRoomFor
|
|
113
|
+
);
|
|
114
|
+
if (heldGrows || pendingFits(state, needed, true)) return;
|
|
115
|
+
state.pendingDocuments = grown(state.pendingDocuments, needed.documents, Uint32Array);
|
|
116
|
+
state.pendingTokens = grown(state.pendingTokens, needed.tokens * PENDING_TOKEN_WORDS, Uint32Array);
|
|
117
|
+
state.pendingUnits = grown(state.pendingUnits, needed.units, Uint16Array);
|
|
118
|
+
};
|
|
119
|
+
function appendPending(state, ordinal, fields) {
|
|
120
|
+
const header = state.pendingHeader;
|
|
121
|
+
state.pendingDocuments[header[PENDING_DOCUMENTS]] = ordinal;
|
|
122
|
+
let written = header[PENDING_TOKENS];
|
|
123
|
+
let units = header[PENDING_UNITS];
|
|
124
|
+
for (const field of fields) {
|
|
125
|
+
for (let at = 0; at < field.tokens.length; at += FIELD_TOKEN_WORDS) {
|
|
126
|
+
const word = written++ * PENDING_TOKEN_WORDS;
|
|
127
|
+
const surface = field.tokens[at + 3];
|
|
128
|
+
state.pendingTokens[word] = field.field;
|
|
129
|
+
state.pendingTokens[word + 1] = units;
|
|
130
|
+
state.pendingTokens[word + 2] = field.tokens[at + 2];
|
|
131
|
+
units = copyUnits(state.pendingUnits, units, field.units, field.tokens[at + 1], field.tokens[at + 2]);
|
|
132
|
+
state.pendingTokens[word + 3] = surface === NO_SURFACE ? NO_SURFACE : units;
|
|
133
|
+
state.pendingTokens[word + 4] = surface === NO_SURFACE ? 0 : field.tokens[at + 4];
|
|
134
|
+
if (surface !== NO_SURFACE)
|
|
135
|
+
units = copyUnits(state.pendingUnits, units, field.units, surface, field.tokens[at + 4]);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
function copyUnits(target, into, source, start, length) {
|
|
140
|
+
for (let at = 0; at < length; at++) target[into + at] = source[start + at];
|
|
141
|
+
return into + length;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// src/core/frozen-writes/counts.ts
|
|
145
|
+
var termChangesByCounts = /* @__PURE__ */ new WeakMap();
|
|
146
|
+
function termChangesOf(segment) {
|
|
147
|
+
const counts = segment.terms.counts;
|
|
148
|
+
const held = termChangesByCounts.get(counts);
|
|
149
|
+
if (held !== void 0) return held;
|
|
150
|
+
const terms = counts.length;
|
|
151
|
+
const created = {
|
|
152
|
+
documents: new Float64Array(terms),
|
|
153
|
+
amounts: new Float64Array(terms),
|
|
154
|
+
marks: new Float64Array(terms),
|
|
155
|
+
touched: [],
|
|
156
|
+
mark: 0,
|
|
157
|
+
seen: new Float64Array(segment.documentCount),
|
|
158
|
+
seenMark: 0
|
|
159
|
+
};
|
|
160
|
+
termChangesByCounts.set(counts, created);
|
|
161
|
+
return created;
|
|
162
|
+
}
|
|
163
|
+
function clearTermChanges(changes) {
|
|
164
|
+
for (const term of changes.touched) {
|
|
165
|
+
changes.documents[term] = 0;
|
|
166
|
+
changes.amounts[term] = 0;
|
|
167
|
+
}
|
|
168
|
+
changes.touched.length = 0;
|
|
169
|
+
}
|
|
170
|
+
function slotOfRow(offsets, row) {
|
|
171
|
+
let low = 0;
|
|
172
|
+
let high = offsets.length - 1;
|
|
173
|
+
while (low < high) {
|
|
174
|
+
const middle = low + (high - low >>> 1);
|
|
175
|
+
if (offsets[middle + 1] <= row) low = middle + 1;
|
|
176
|
+
else high = middle;
|
|
177
|
+
}
|
|
178
|
+
return low;
|
|
179
|
+
}
|
|
180
|
+
function addTerm(changes, term, documents, amount) {
|
|
181
|
+
if (changes.documents[term] === 0) changes.touched.push(term);
|
|
182
|
+
changes.documents[term] += documents;
|
|
183
|
+
changes.amounts[term] += amount;
|
|
184
|
+
}
|
|
185
|
+
function addOccurrences(changes, term, amount) {
|
|
186
|
+
if (changes.marks[term] === changes.mark) {
|
|
187
|
+
changes.amounts[term] += amount;
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
changes.marks[term] = changes.mark;
|
|
191
|
+
addTerm(changes, term, 1, amount);
|
|
192
|
+
}
|
|
193
|
+
function addIndexedDocument(changes, segment, ordinal) {
|
|
194
|
+
const { removalIndex, postingDocuments, postingFrequencies, postingFields, postingOffsets, payloadSlots, terms } = segment;
|
|
195
|
+
if (removalIndex === null) return;
|
|
196
|
+
const start = removalIndex.starts[ordinal];
|
|
197
|
+
const end = removalIndex.starts[ordinal + 1];
|
|
198
|
+
if (start > end || end > postingDocuments.length) return;
|
|
199
|
+
changes.mark++;
|
|
200
|
+
let previous = -1;
|
|
201
|
+
for (let at = start; at < end; at++) {
|
|
202
|
+
const row = removalIndex.rows[at];
|
|
203
|
+
if (row >= postingDocuments.length || row <= previous) continue;
|
|
204
|
+
previous = row;
|
|
205
|
+
if (postingDocuments[row] !== ordinal || postingFields[row] >= segment.partialFields.length) continue;
|
|
206
|
+
const slot = slotOfRow(postingOffsets, row);
|
|
207
|
+
if (slot >= terms.counts.length || postingOffsets[slot] > row || postingOffsets[slot + 1] > postingDocuments.length)
|
|
208
|
+
continue;
|
|
209
|
+
const term = removalIndex.termOfSlot[slot];
|
|
210
|
+
if (term >= terms.counts.length || payloadSlots[term] !== slot) continue;
|
|
211
|
+
addOccurrences(changes, term, postingFrequencies[row]);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
var UNCHECKED_INDEX = -2;
|
|
215
|
+
var MALFORMED_INDEX = -3;
|
|
216
|
+
var indexFieldLimits = /* @__PURE__ */ new WeakMap();
|
|
217
|
+
function rowsAreOrdered(segment, index) {
|
|
218
|
+
const { postingDocuments, postingOffsets, postingFields, payloadSlots, terms } = segment;
|
|
219
|
+
const { starts, rows, termOfSlot } = index;
|
|
220
|
+
const slots = terms.counts.length;
|
|
221
|
+
const postings = postingDocuments.length;
|
|
222
|
+
if (postingOffsets[slots] > postings) return MALFORMED_INDEX;
|
|
223
|
+
for (let slot = 0; slot < slots; slot++) {
|
|
224
|
+
const term = termOfSlot[slot];
|
|
225
|
+
if (postingOffsets[slot] > postingOffsets[slot + 1] || term >= slots || payloadSlots[term] !== slot)
|
|
226
|
+
return MALFORMED_INDEX;
|
|
227
|
+
}
|
|
228
|
+
let widestField = -1;
|
|
229
|
+
for (let ordinal = 0; ordinal < segment.documentCount; ordinal++) {
|
|
230
|
+
const end = starts[ordinal + 1];
|
|
231
|
+
if (starts[ordinal] > end || end > postings) return MALFORMED_INDEX;
|
|
232
|
+
let previous = -1;
|
|
233
|
+
for (let at = starts[ordinal]; at < end; at++) {
|
|
234
|
+
const row = rows[at];
|
|
235
|
+
if (row <= previous || row < postingOffsets[0] || row >= postingOffsets[slots]) return MALFORMED_INDEX;
|
|
236
|
+
if (postingDocuments[row] !== ordinal) return MALFORMED_INDEX;
|
|
237
|
+
if (postingFields[row] > widestField) widestField = postingFields[row];
|
|
238
|
+
previous = row;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return widestField;
|
|
242
|
+
}
|
|
243
|
+
var checkedOccurrences = /* @__PURE__ */ new WeakMap();
|
|
244
|
+
function occurrencesAreCounts(occurrences) {
|
|
245
|
+
const held = checkedOccurrences.get(occurrences);
|
|
246
|
+
if (held !== void 0) return held;
|
|
247
|
+
let counts = true;
|
|
248
|
+
for (let term = 0; term < occurrences.length && counts; term++)
|
|
249
|
+
counts = Number.isFinite(occurrences[term]) && occurrences[term] >= 0;
|
|
250
|
+
checkedOccurrences.set(occurrences, counts);
|
|
251
|
+
return counts;
|
|
252
|
+
}
|
|
253
|
+
function trustedIndexOf(segment, state) {
|
|
254
|
+
const index = segment.removalIndex;
|
|
255
|
+
if (index === null || !occurrencesAreCounts(state.removedOccurrences)) return null;
|
|
256
|
+
let widestField = indexFieldLimits.get(segment) ?? UNCHECKED_INDEX;
|
|
257
|
+
if (widestField === UNCHECKED_INDEX) {
|
|
258
|
+
widestField = rowsAreOrdered(segment, index);
|
|
259
|
+
indexFieldLimits.set(segment, widestField);
|
|
260
|
+
}
|
|
261
|
+
return widestField !== MALFORMED_INDEX && widestField < segment.partialFields.length ? index : null;
|
|
262
|
+
}
|
|
263
|
+
function subtractTrusted(segment, state, index) {
|
|
264
|
+
const { postingOffsets, postingFrequencies } = segment;
|
|
265
|
+
const stored = segment.terms.counts;
|
|
266
|
+
const { starts, rows, termOfSlot } = index;
|
|
267
|
+
const { removedDocuments, removedOccurrences, pendingDocuments } = state;
|
|
268
|
+
const pending = state.pendingHeader[PENDING_DOCUMENTS];
|
|
269
|
+
for (let document = 0; document < pending; document++) {
|
|
270
|
+
const ordinal = pendingDocuments[document];
|
|
271
|
+
let previousSlot = -1;
|
|
272
|
+
let term = 0;
|
|
273
|
+
for (let at = starts[ordinal]; at < starts[ordinal + 1]; at++) {
|
|
274
|
+
const row = rows[at];
|
|
275
|
+
const slot = slotOfRow(postingOffsets, row);
|
|
276
|
+
if (slot !== previousSlot) {
|
|
277
|
+
previousSlot = slot;
|
|
278
|
+
term = termOfSlot[slot];
|
|
279
|
+
removedDocuments[term] = Math.min(stored[term], removedDocuments[term] + 1);
|
|
280
|
+
}
|
|
281
|
+
removedOccurrences[term] += postingFrequencies[row];
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
function addPagedDocument(changes, segment, paged, ordinal) {
|
|
286
|
+
const pairs = paged.read(ordinal);
|
|
287
|
+
changes.mark++;
|
|
288
|
+
for (let at = 0; at + 1 < pairs.length; at += 2) {
|
|
289
|
+
requireSegment(pairs[at] < segment.terms.counts.length);
|
|
290
|
+
addOccurrences(changes, pairs[at], pairs[at + 1]);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
function scanPostings(changes, segment, marks) {
|
|
294
|
+
const { postingDocuments, postingFrequencies, postingFields, postingOffsets, payloadSlots, terms } = segment;
|
|
295
|
+
const stamps = new Uint32Array(marks.length);
|
|
296
|
+
for (let term = 0; term < terms.counts.length; term++) {
|
|
297
|
+
const slot = payloadSlots[term];
|
|
298
|
+
if (slot >= terms.counts.length) continue;
|
|
299
|
+
const start = postingOffsets[slot];
|
|
300
|
+
const end = postingOffsets[slot + 1];
|
|
301
|
+
if (start > end || end > postingDocuments.length) continue;
|
|
302
|
+
const stamp = term + 1;
|
|
303
|
+
let documents = 0;
|
|
304
|
+
let amount = 0;
|
|
305
|
+
for (let row = start; row < end; row++) {
|
|
306
|
+
const ordinal = postingDocuments[row];
|
|
307
|
+
if (ordinal >= marks.length || marks[ordinal] === 0 || postingFields[row] >= segment.partialFields.length)
|
|
308
|
+
continue;
|
|
309
|
+
amount += postingFrequencies[row];
|
|
310
|
+
if (stamps[ordinal] !== stamp) {
|
|
311
|
+
stamps[ordinal] = stamp;
|
|
312
|
+
documents++;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (documents > 0) addTerm(changes, term, documents, amount);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function requirePendingDocuments(changes, segment, state) {
|
|
319
|
+
const pending = state.pendingHeader[PENDING_DOCUMENTS];
|
|
320
|
+
if (changes.seen.length !== segment.documentCount) changes.seen = new Float64Array(segment.documentCount);
|
|
321
|
+
const { seen } = changes;
|
|
322
|
+
const stamp = ++changes.seenMark;
|
|
323
|
+
for (let at = 0; at < pending; at++) {
|
|
324
|
+
const ordinal = state.pendingDocuments[at];
|
|
325
|
+
requireSegment(
|
|
326
|
+
ordinal < segment.documentCount && (state.tombstones[ordinal >>> 5] & 1 << (ordinal & 31)) !== 0 && seen[ordinal] !== stamp
|
|
327
|
+
);
|
|
328
|
+
seen[ordinal] = stamp;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
function settleTermCounts(changes, segment, state, paged) {
|
|
332
|
+
const pending = state.pendingHeader[PENDING_DOCUMENTS];
|
|
333
|
+
if (paged === null && segment.removalIndex === null) {
|
|
334
|
+
const marks = new Uint8Array(segment.documentCount);
|
|
335
|
+
for (let at = 0; at < pending; at++) marks[state.pendingDocuments[at]] = 1;
|
|
336
|
+
scanPostings(changes, segment, marks);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
for (let at = 0; at < pending; at++) {
|
|
340
|
+
const ordinal = state.pendingDocuments[at];
|
|
341
|
+
if (paged === null) addIndexedDocument(changes, segment, ordinal);
|
|
342
|
+
else addPagedDocument(changes, segment, paged, ordinal);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// src/core/frozen-writes/removal-request.ts
|
|
347
|
+
function copyField(value) {
|
|
348
|
+
requireSegment(typeof value === "object" && value !== null);
|
|
349
|
+
return { ...value };
|
|
350
|
+
}
|
|
351
|
+
function readFrozenFields(input, output, ordinal, rawFields) {
|
|
352
|
+
const { segment, state, regions } = readFrozenParts(input, output);
|
|
353
|
+
requireSegment(Array.isArray(rawFields) && rawFields.length <= 65536);
|
|
354
|
+
const fields = rawFields.map(copyField);
|
|
355
|
+
requireSegment(ordinal === null || ordinal < segment.documentCount);
|
|
356
|
+
const fieldCount = segment.partialFields.length;
|
|
357
|
+
requireSegment(fields.length <= fieldCount);
|
|
358
|
+
let previous = -1;
|
|
359
|
+
for (const field of fields) {
|
|
360
|
+
requireSegment(Number.isInteger(field.field) && field.field > previous && field.field < fieldCount);
|
|
361
|
+
field.tokens = regions.array(field.tokens, "Uint32Array");
|
|
362
|
+
field.units = regions.array(field.units, "Uint16Array");
|
|
363
|
+
requireSegment(
|
|
364
|
+
field.tokens.length % FIELD_TOKEN_WORDS === 0 && field.tokens.length / FIELD_TOKEN_WORDS <= 4294967295 && field.units.length <= 4294967295
|
|
365
|
+
);
|
|
366
|
+
for (let at = 0; at < field.tokens.length; at += FIELD_TOKEN_WORDS) {
|
|
367
|
+
checkText(field.units, field.tokens[at + 1], field.tokens[at + 2]);
|
|
368
|
+
if (field.tokens[at + 3] !== NO_SURFACE) checkText(field.units, field.tokens[at + 3], field.tokens[at + 4]);
|
|
369
|
+
}
|
|
370
|
+
previous = field.field;
|
|
371
|
+
}
|
|
372
|
+
regions.check();
|
|
373
|
+
return {
|
|
374
|
+
segment,
|
|
375
|
+
state,
|
|
376
|
+
fields,
|
|
377
|
+
readOutput(rawEntries, rawValues) {
|
|
378
|
+
const entries = regions.array(rawEntries, "Uint32Array", void 0, true);
|
|
379
|
+
const values = regions.array(rawValues, "Float64Array", void 0, true);
|
|
380
|
+
requireSegment(entries.length % 2 === 0);
|
|
381
|
+
regions.check();
|
|
382
|
+
return { entries, values };
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
function readFrozenRequest(input, output, ordinal, fields) {
|
|
387
|
+
requireSegment(Number.isInteger(ordinal) && ordinal >= 0);
|
|
388
|
+
return readFrozenFields(input, output, ordinal, fields);
|
|
389
|
+
}
|
|
390
|
+
function readFrozenSettle(input, output) {
|
|
391
|
+
return readFrozenFields(input, output, null, []);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// src/core/frozen-writes/unit-order.ts
|
|
395
|
+
var UTF8_LEADS = [0, 0, 192, 224, 240];
|
|
396
|
+
function holdsLoneSurrogate(units, start, length) {
|
|
397
|
+
const end = start + length;
|
|
398
|
+
for (let at = start; at < end; at++) {
|
|
399
|
+
const unit = units[at];
|
|
400
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
401
|
+
if (unit >= 56320 || at + 1 >= end || units[at + 1] < 56320 || units[at + 1] >= 57344) return true;
|
|
402
|
+
at++;
|
|
403
|
+
}
|
|
404
|
+
return false;
|
|
405
|
+
}
|
|
406
|
+
function utf8Width(point) {
|
|
407
|
+
if (point < 128) return 1;
|
|
408
|
+
if (point < 2048) return 2;
|
|
409
|
+
return point < 65536 ? 3 : 4;
|
|
410
|
+
}
|
|
411
|
+
function utf8Byte(point, width, index) {
|
|
412
|
+
if (width === 1) return point;
|
|
413
|
+
const shift = 6 * (width - 1 - index);
|
|
414
|
+
return index === 0 ? UTF8_LEADS[width] | point >> shift : 128 | point >> shift & 63;
|
|
415
|
+
}
|
|
416
|
+
function compareTableToUnits(table, from, to, units, start, length) {
|
|
417
|
+
let byte = from;
|
|
418
|
+
const end = start + length;
|
|
419
|
+
for (let at = start; at < end; at++) {
|
|
420
|
+
let point = units[at];
|
|
421
|
+
if (point >= 55296 && point < 56320) point = 65536 + (point - 55296 << 10) + (units[++at] - 56320);
|
|
422
|
+
const width = utf8Width(point);
|
|
423
|
+
for (let index = 0; index < width; index++) {
|
|
424
|
+
if (byte >= to) return -1;
|
|
425
|
+
const order = table[byte++] - utf8Byte(point, width, index);
|
|
426
|
+
if (order !== 0) return order;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return byte < to ? 1 : 0;
|
|
430
|
+
}
|
|
431
|
+
function codePointRank(unit) {
|
|
432
|
+
if (unit < 55296) return unit;
|
|
433
|
+
return unit < 57344 ? unit + 8192 : unit - 2048;
|
|
434
|
+
}
|
|
435
|
+
function compareUnits(units, left, leftLength, right, rightLength) {
|
|
436
|
+
const shared = leftLength < rightLength ? leftLength : rightLength;
|
|
437
|
+
for (let at = 0; at < shared; at++) {
|
|
438
|
+
const order = codePointRank(units[left + at]) - codePointRank(units[right + at]);
|
|
439
|
+
if (order !== 0) return order;
|
|
440
|
+
}
|
|
441
|
+
return leftLength - rightLength;
|
|
442
|
+
}
|
|
443
|
+
function sameUnits(units, left, leftLength, right, rightLength) {
|
|
444
|
+
if (leftLength !== rightLength) return false;
|
|
445
|
+
for (let at = 0; at < leftLength; at++) if (units[left + at] !== units[right + at]) return false;
|
|
446
|
+
return true;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// src/core/frozen-writes/table-order.ts
|
|
450
|
+
var heldOrders = /* @__PURE__ */ new WeakMap();
|
|
451
|
+
function sameView(left, right) {
|
|
452
|
+
if (left === void 0 || right === void 0) return left === right;
|
|
453
|
+
return left.buffer === right.buffer && left.byteOffset === right.byteOffset && left.byteLength === right.byteLength;
|
|
454
|
+
}
|
|
455
|
+
function compareEntries(text, leftFrom, leftTo, rightFrom, rightTo) {
|
|
456
|
+
const leftLength = leftTo - leftFrom;
|
|
457
|
+
const rightLength = rightTo - rightFrom;
|
|
458
|
+
const shared = leftLength < rightLength ? leftLength : rightLength;
|
|
459
|
+
for (let at = 0; at < shared; at++) {
|
|
460
|
+
const order = text[leftFrom + at] - text[rightFrom + at];
|
|
461
|
+
if (order !== 0) return order;
|
|
462
|
+
}
|
|
463
|
+
return leftLength - rightLength;
|
|
464
|
+
}
|
|
465
|
+
function checkAscends(table, stride, tokens) {
|
|
466
|
+
const { offsets, text, counts } = table;
|
|
467
|
+
const size = counts.length;
|
|
468
|
+
if (size === 0) return true;
|
|
469
|
+
if ((size - 1) * stride + 1 >= offsets.length) return false;
|
|
470
|
+
if (tokens !== void 0 && tokens.length < size) return false;
|
|
471
|
+
for (let entry = 0; entry < size; entry++) {
|
|
472
|
+
const from = offsets[entry * stride];
|
|
473
|
+
const to = offsets[entry * stride + 1];
|
|
474
|
+
if (from > to || to > text.length) return false;
|
|
475
|
+
if (entry === 0) continue;
|
|
476
|
+
const before = (entry - 1) * stride;
|
|
477
|
+
const order = compareEntries(text, offsets[before], offsets[before + 1], from, to) || (tokens === void 0 ? 0 : tokens[entry - 1] - tokens[entry]);
|
|
478
|
+
if (order >= 0) return false;
|
|
479
|
+
}
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
function tableAscends(table, stride, tokens) {
|
|
483
|
+
const buffer = table.offsets.buffer;
|
|
484
|
+
const held = heldOrders.get(buffer);
|
|
485
|
+
if (held !== void 0) {
|
|
486
|
+
for (const order of held)
|
|
487
|
+
if (order.stride === stride && sameView(order.offsets, table.offsets) && sameView(order.text, table.text) && sameView(order.counts, table.counts) && sameView(order.tokens, tokens))
|
|
488
|
+
return order.ascends;
|
|
489
|
+
}
|
|
490
|
+
const ascends = checkAscends(table, stride, tokens);
|
|
491
|
+
const orders = held ?? [];
|
|
492
|
+
if (orders.length >= SETTLE_ORDER_CHECKS_PER_BUFFER) orders.shift();
|
|
493
|
+
orders.push({ stride, offsets: table.offsets, text: table.text, counts: table.counts, tokens, ascends });
|
|
494
|
+
if (held === void 0) heldOrders.set(buffer, orders);
|
|
495
|
+
return ascends;
|
|
496
|
+
}
|
|
497
|
+
function entryOrder(table, stride, tokens, entry, units, start, length, token) {
|
|
498
|
+
return compareTableToUnits(
|
|
499
|
+
table.text,
|
|
500
|
+
table.offsets[entry * stride],
|
|
501
|
+
table.offsets[entry * stride + 1],
|
|
502
|
+
units,
|
|
503
|
+
start,
|
|
504
|
+
length
|
|
505
|
+
) || (tokens === void 0 ? 0 : tokens[entry] - token);
|
|
506
|
+
}
|
|
507
|
+
function findUnits(table, units, start, length, stride, tokens, token = 0) {
|
|
508
|
+
if (holdsLoneSurrogate(units, start, length)) return -1;
|
|
509
|
+
let low = 0;
|
|
510
|
+
let high = table.counts.length;
|
|
511
|
+
while (low < high) {
|
|
512
|
+
const middle = low + (high - low >>> 1);
|
|
513
|
+
const from = table.offsets[middle * stride];
|
|
514
|
+
const to = table.offsets[middle * stride + 1];
|
|
515
|
+
if (from > to || to > table.text.length) break;
|
|
516
|
+
const order = compareTableToUnits(table.text, from, to, units, start, length) || (tokens === void 0 ? 0 : tokens[middle] - token);
|
|
517
|
+
if (order === 0) return middle;
|
|
518
|
+
if (order < 0) low = middle + 1;
|
|
519
|
+
else high = middle;
|
|
520
|
+
}
|
|
521
|
+
if (low === high) return -1;
|
|
522
|
+
for (let at = 0; at < table.counts.length; at++) {
|
|
523
|
+
const from = table.offsets[at * stride];
|
|
524
|
+
const to = table.offsets[at * stride + 1];
|
|
525
|
+
if ((tokens === void 0 || tokens[at] === token) && from <= to && to <= table.text.length) {
|
|
526
|
+
if (compareTableToUnits(table.text, from, to, units, start, length) === 0) return at;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
return -1;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// src/core/frozen-writes/distinct-units.ts
|
|
533
|
+
var EMPTY_SLOT = -1;
|
|
534
|
+
var MISSING_ENTRY = -1;
|
|
535
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
536
|
+
var FNV_PRIME = 16777619;
|
|
537
|
+
function slotCountFor(keys) {
|
|
538
|
+
let slots = 1;
|
|
539
|
+
while (slots < keys * SETTLE_KEY_SLOTS_PER_KEY) slots *= 2;
|
|
540
|
+
return slots;
|
|
541
|
+
}
|
|
542
|
+
var DistinctUnits = class {
|
|
543
|
+
count = 0;
|
|
544
|
+
hits = new Uint32Array(0);
|
|
545
|
+
entries = new Int32Array(0);
|
|
546
|
+
starts = new Uint32Array(0);
|
|
547
|
+
lengths = new Uint32Array(0);
|
|
548
|
+
tags = new Uint32Array(0);
|
|
549
|
+
order = new Uint32Array(0);
|
|
550
|
+
slots = new Int32Array(0);
|
|
551
|
+
mask = 0;
|
|
552
|
+
units = new Uint16Array(0);
|
|
553
|
+
byUnits = (left, right) => compareUnits(this.units, this.starts[left], this.lengths[left], this.starts[right], this.lengths[right]) || this.tags[left] - this.tags[right];
|
|
554
|
+
begin(units, capacity) {
|
|
555
|
+
this.units = units;
|
|
556
|
+
this.count = 0;
|
|
557
|
+
if (this.starts.length < capacity) {
|
|
558
|
+
this.hits = new Uint32Array(capacity);
|
|
559
|
+
this.entries = new Int32Array(capacity);
|
|
560
|
+
this.starts = new Uint32Array(capacity);
|
|
561
|
+
this.lengths = new Uint32Array(capacity);
|
|
562
|
+
this.tags = new Uint32Array(capacity);
|
|
563
|
+
this.order = new Uint32Array(capacity);
|
|
564
|
+
}
|
|
565
|
+
const slots = slotCountFor(capacity);
|
|
566
|
+
if (this.slots.length < slots) this.slots = new Int32Array(slots);
|
|
567
|
+
this.slots.fill(EMPTY_SLOT, 0, slots);
|
|
568
|
+
this.mask = slots - 1;
|
|
569
|
+
}
|
|
570
|
+
add(start, length, tag) {
|
|
571
|
+
const { units, slots, starts, lengths, tags } = this;
|
|
572
|
+
const mask = this.mask;
|
|
573
|
+
let hash = FNV_OFFSET_BASIS;
|
|
574
|
+
for (let at = start; at < start + length; at++) hash = Math.imul(hash ^ units[at], FNV_PRIME);
|
|
575
|
+
let slot = Math.imul(hash ^ tag, FNV_PRIME) & mask;
|
|
576
|
+
let key = slots[slot];
|
|
577
|
+
while (key !== EMPTY_SLOT && !(tags[key] === tag && sameUnits(units, starts[key], lengths[key], start, length))) {
|
|
578
|
+
slot = slot + 1 & mask;
|
|
579
|
+
key = slots[slot];
|
|
580
|
+
}
|
|
581
|
+
if (key === EMPTY_SLOT) {
|
|
582
|
+
key = this.count++;
|
|
583
|
+
slots[slot] = key;
|
|
584
|
+
starts[key] = start;
|
|
585
|
+
lengths[key] = length;
|
|
586
|
+
tags[key] = tag;
|
|
587
|
+
this.hits[key] = 0;
|
|
588
|
+
}
|
|
589
|
+
this.hits[key]++;
|
|
590
|
+
return key;
|
|
591
|
+
}
|
|
592
|
+
resolve(table, stride, tokens) {
|
|
593
|
+
const { count, starts, lengths, tags, entries, units } = this;
|
|
594
|
+
if (!tableAscends(table, stride, tokens)) {
|
|
595
|
+
for (let key = 0; key < count; key++)
|
|
596
|
+
entries[key] = findUnits(table, units, starts[key], lengths[key], stride, tokens, tags[key]);
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const order = this.order.subarray(0, count);
|
|
600
|
+
for (let key = 0; key < count; key++) order[key] = key;
|
|
601
|
+
order.sort(this.byUnits);
|
|
602
|
+
const size = table.counts.length;
|
|
603
|
+
let cursor = 0;
|
|
604
|
+
for (let rank = 0; rank < count; rank++) {
|
|
605
|
+
const key = order[rank];
|
|
606
|
+
const start = starts[key];
|
|
607
|
+
const length = lengths[key];
|
|
608
|
+
const tag = tags[key];
|
|
609
|
+
cursor = lowerBound(table, stride, tokens, units, start, length, tag, cursor, size);
|
|
610
|
+
if (cursor < size && entryOrder(table, stride, tokens, cursor, units, start, length, tag) === 0)
|
|
611
|
+
entries[key] = cursor++;
|
|
612
|
+
else entries[key] = MISSING_ENTRY;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
function lowerBound(table, stride, tokens, units, start, length, tag, cursor, size) {
|
|
617
|
+
if (cursor >= size || entryOrder(table, stride, tokens, cursor, units, start, length, tag) >= 0) return cursor;
|
|
618
|
+
let below = cursor;
|
|
619
|
+
let step = 1;
|
|
620
|
+
let above = cursor + 1;
|
|
621
|
+
while (above < size && entryOrder(table, stride, tokens, above, units, start, length, tag) < 0) {
|
|
622
|
+
below = above;
|
|
623
|
+
step *= 2;
|
|
624
|
+
above = below + step;
|
|
625
|
+
}
|
|
626
|
+
if (above > size) above = size;
|
|
627
|
+
while (above - below > 1) {
|
|
628
|
+
const middle = below + (above - below >>> 1);
|
|
629
|
+
if (entryOrder(table, stride, tokens, middle, units, start, length, tag) < 0) below = middle;
|
|
630
|
+
else above = middle;
|
|
631
|
+
}
|
|
632
|
+
return above;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// src/core/frozen-writes/string-counts.ts
|
|
636
|
+
var SURFACE_STRIDE = 2;
|
|
637
|
+
var STRING_STRIDE = 1;
|
|
638
|
+
var UNTAGGED = 0;
|
|
639
|
+
var NO_TERM_KEY = -1;
|
|
640
|
+
var surfaceKeys = new DistinctUnits();
|
|
641
|
+
var termKeys = new DistinctUnits();
|
|
642
|
+
var tokenTerms = new Int32Array(0);
|
|
643
|
+
function requireUnits(state, start, length) {
|
|
644
|
+
const units = state.pendingHeader[PENDING_UNITS];
|
|
645
|
+
requireSegment(start <= units && length <= units - start);
|
|
646
|
+
}
|
|
647
|
+
function add(counts, entry, amount) {
|
|
648
|
+
counts.set(entry, (counts.get(entry) ?? 0) + amount);
|
|
649
|
+
}
|
|
650
|
+
function addResolved(counts, keys) {
|
|
651
|
+
for (let key = 0; key < keys.count; key++) if (keys.entries[key] >= 0) add(counts, keys.entries[key], keys.hits[key]);
|
|
652
|
+
}
|
|
653
|
+
function countUnorderedSurfaces(changes, surfaces, written) {
|
|
654
|
+
for (let at = 0; at < surfaces.counts.length; at++) {
|
|
655
|
+
const held = written.get(textAt(surfaces, at * 2) ?? "");
|
|
656
|
+
if (held !== void 0) add(changes.surfaces, at, held);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
function countSpellings(changes, segment, state, spellings, count) {
|
|
660
|
+
const units = state.pendingUnits;
|
|
661
|
+
termKeys.resolve(segment.terms, STRING_STRIDE, void 0);
|
|
662
|
+
surfaceKeys.begin(units, count);
|
|
663
|
+
for (let token = 0; token < count; token++) {
|
|
664
|
+
const termKey = tokenTerms[token];
|
|
665
|
+
if (termKey === NO_TERM_KEY) continue;
|
|
666
|
+
const termEntry = termKeys.entries[termKey];
|
|
667
|
+
if (termEntry < 0) continue;
|
|
668
|
+
const at = token * PENDING_TOKEN_WORDS;
|
|
669
|
+
const start = state.pendingTokens[at + 3];
|
|
670
|
+
const length = start === NO_SURFACE ? 0 : state.pendingTokens[at + 4];
|
|
671
|
+
const spellingStart = length === 0 ? state.pendingTokens[at + 1] : start;
|
|
672
|
+
const spellingLength = length === 0 ? state.pendingTokens[at + 2] : length;
|
|
673
|
+
if (!holdsLoneSurrogate(units, spellingStart, spellingLength))
|
|
674
|
+
surfaceKeys.add(spellingStart, spellingLength, termEntry);
|
|
675
|
+
}
|
|
676
|
+
surfaceKeys.resolve(spellings, STRING_STRIDE, segment.spellingTokens);
|
|
677
|
+
addResolved(changes.spellings, surfaceKeys);
|
|
678
|
+
}
|
|
679
|
+
function settleStringCounts(changes, segment, state) {
|
|
680
|
+
const { surfaces, spellings, partialFields } = segment;
|
|
681
|
+
const ordered = surfaces === null || tableAscends(surfaces, SURFACE_STRIDE, void 0);
|
|
682
|
+
let unordered = null;
|
|
683
|
+
const units = state.pendingUnits;
|
|
684
|
+
const count = state.pendingHeader[PENDING_TOKENS];
|
|
685
|
+
surfaceKeys.begin(units, count);
|
|
686
|
+
termKeys.begin(units, count);
|
|
687
|
+
if (tokenTerms.length < count) tokenTerms = new Int32Array(count);
|
|
688
|
+
for (let token = 0; token < count; token++) {
|
|
689
|
+
const at = token * PENDING_TOKEN_WORDS;
|
|
690
|
+
const field = state.pendingTokens[at];
|
|
691
|
+
const termStart = state.pendingTokens[at + 1];
|
|
692
|
+
const termLength = state.pendingTokens[at + 2];
|
|
693
|
+
const start = state.pendingTokens[at + 3];
|
|
694
|
+
const length = start === NO_SURFACE ? 0 : state.pendingTokens[at + 4];
|
|
695
|
+
requireSegment(field < partialFields.length);
|
|
696
|
+
requireUnits(state, termStart, termLength);
|
|
697
|
+
if (start !== NO_SURFACE) requireUnits(state, start, length);
|
|
698
|
+
tokenTerms[token] = NO_TERM_KEY;
|
|
699
|
+
if (surfaces === null && spellings === null) continue;
|
|
700
|
+
if (surfaces !== null && length > 0 && !sameUnits(units, termStart, termLength, start, length)) {
|
|
701
|
+
if (!ordered) {
|
|
702
|
+
unordered ??= /* @__PURE__ */ new Map();
|
|
703
|
+
const text = storedText(units, start, length);
|
|
704
|
+
unordered.set(text, (unordered.get(text) ?? 0) + 1);
|
|
705
|
+
} else if (!holdsLoneSurrogate(units, start, length)) surfaceKeys.add(start, length, UNTAGGED);
|
|
706
|
+
}
|
|
707
|
+
if (spellings === null || partialFields[field] === 0) continue;
|
|
708
|
+
if (!holdsLoneSurrogate(units, termStart, termLength))
|
|
709
|
+
tokenTerms[token] = termKeys.add(termStart, termLength, UNTAGGED);
|
|
710
|
+
}
|
|
711
|
+
if (surfaces !== null) {
|
|
712
|
+
surfaceKeys.resolve(surfaces, SURFACE_STRIDE, void 0);
|
|
713
|
+
addResolved(changes.surfaces, surfaceKeys);
|
|
714
|
+
if (unordered !== null) countUnorderedSurfaces(changes, surfaces, unordered);
|
|
715
|
+
}
|
|
716
|
+
if (spellings !== null) countSpellings(changes, segment, state, spellings, count);
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// src/core/frozen-writes/portable.ts
|
|
720
|
+
var PREPARED_REPLACEMENTS = 5;
|
|
721
|
+
var TOMBSTONE_WORDS = 4;
|
|
722
|
+
var TOMBSTONE_COUNT = 5;
|
|
723
|
+
var PENDING_HEADER = 6;
|
|
724
|
+
function replace(entries, values, at, kind, entry, value) {
|
|
725
|
+
entries[at * 2] = kind;
|
|
726
|
+
entries[at * 2 + 1] = entry;
|
|
727
|
+
values[at] = value;
|
|
728
|
+
}
|
|
729
|
+
function clamped(removed, stored, amount) {
|
|
730
|
+
return Math.min(stored, removed + amount);
|
|
731
|
+
}
|
|
732
|
+
function publishTerms(terms, segment, state, paged) {
|
|
733
|
+
settleTermCounts(terms, segment, state, paged);
|
|
734
|
+
for (const term of terms.touched)
|
|
735
|
+
requireSegment(Number.isFinite(state.removedOccurrences[term]) && state.removedOccurrences[term] >= 0);
|
|
736
|
+
for (const term of terms.touched) {
|
|
737
|
+
state.removedDocuments[term] = clamped(
|
|
738
|
+
state.removedDocuments[term],
|
|
739
|
+
segment.terms.counts[term],
|
|
740
|
+
terms.documents[term]
|
|
741
|
+
);
|
|
742
|
+
state.removedOccurrences[term] += terms.amounts[term];
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
function settle(segment, state, paged) {
|
|
746
|
+
if (pendingSettled(state)) return;
|
|
747
|
+
const changes = { terms: termChangesOf(segment), surfaces: /* @__PURE__ */ new Map(), spellings: /* @__PURE__ */ new Map() };
|
|
748
|
+
settleStringCounts(changes, segment, state);
|
|
749
|
+
requirePendingDocuments(changes.terms, segment, state);
|
|
750
|
+
const trusted = paged === null ? trustedIndexOf(segment, state) : null;
|
|
751
|
+
if (trusted !== null) subtractTrusted(segment, state, trusted);
|
|
752
|
+
else
|
|
753
|
+
try {
|
|
754
|
+
publishTerms(changes.terms, segment, state, paged);
|
|
755
|
+
} finally {
|
|
756
|
+
clearTermChanges(changes.terms);
|
|
757
|
+
}
|
|
758
|
+
if (segment.surfaces !== null)
|
|
759
|
+
for (const [entry, amount] of changes.surfaces)
|
|
760
|
+
state.removedSurfaces[entry] = clamped(state.removedSurfaces[entry], segment.surfaces.counts[entry], amount);
|
|
761
|
+
if (segment.spellings !== null)
|
|
762
|
+
for (const [entry, amount] of changes.spellings)
|
|
763
|
+
state.removedSpellings[entry] = clamped(state.removedSpellings[entry], segment.spellings.counts[entry], amount);
|
|
764
|
+
state.pendingHeader[PENDING_DOCUMENTS] = 0;
|
|
765
|
+
state.pendingHeader[PENDING_TOKENS] = 0;
|
|
766
|
+
state.pendingHeader[PENDING_UNITS] = 0;
|
|
767
|
+
}
|
|
768
|
+
function tombstoned(state, ordinal) {
|
|
769
|
+
return (state.tombstones[ordinal >>> 5] & 1 << (ordinal & 31)) !== 0;
|
|
770
|
+
}
|
|
771
|
+
function makeRoom(request, paged) {
|
|
772
|
+
const { segment, state, fields } = request;
|
|
773
|
+
const needed = pendingNeeds(fields);
|
|
774
|
+
requireRoom(pendingFits(state, needed, true));
|
|
775
|
+
if (!pendingFits(state, needed, false)) settle(segment, state, paged);
|
|
776
|
+
return needed;
|
|
777
|
+
}
|
|
778
|
+
function refusal(error) {
|
|
779
|
+
if (error instanceof SegmentRefusal) return error.status;
|
|
780
|
+
throw error;
|
|
781
|
+
}
|
|
782
|
+
var removeFrozenDocument = (input, output, ordinal, tokens, paged) => {
|
|
783
|
+
try {
|
|
784
|
+
const request = readFrozenRequest(input, output, ordinal, tokens);
|
|
785
|
+
const { segment, state, fields } = request;
|
|
786
|
+
if (tombstoned(state, ordinal)) return 0;
|
|
787
|
+
requireSegment(state.tombstoneCount[0] < segment.documentCount);
|
|
788
|
+
const needed = makeRoom(request, paged);
|
|
789
|
+
appendPending(state, ordinal, fields);
|
|
790
|
+
state.pendingHeader[PENDING_DOCUMENTS] += needed.documents;
|
|
791
|
+
state.pendingHeader[PENDING_TOKENS] += needed.tokens;
|
|
792
|
+
state.pendingHeader[PENDING_UNITS] += needed.units;
|
|
793
|
+
state.tombstones[ordinal >>> 5] |= 1 << (ordinal & 31);
|
|
794
|
+
state.tombstoneCount[0]++;
|
|
795
|
+
return 0;
|
|
796
|
+
} catch (error) {
|
|
797
|
+
return refusal(error);
|
|
798
|
+
}
|
|
799
|
+
};
|
|
800
|
+
var prepareFrozenRemoval = (input, output, ordinal, tokens, rawEntries, rawValues, paged) => {
|
|
801
|
+
try {
|
|
802
|
+
const request = readFrozenRequest(input, output, ordinal, tokens);
|
|
803
|
+
const { entries, values } = request.readOutput(rawEntries, rawValues);
|
|
804
|
+
const { segment, state, fields } = request;
|
|
805
|
+
if (tombstoned(state, ordinal)) return 0;
|
|
806
|
+
requireSegment(state.tombstoneCount[0] < segment.documentCount);
|
|
807
|
+
requireRoom(entries.length / 2 >= PREPARED_REPLACEMENTS && values.length >= PREPARED_REPLACEMENTS);
|
|
808
|
+
const needed = makeRoom(request, paged);
|
|
809
|
+
appendPending(state, ordinal, fields);
|
|
810
|
+
const header = state.pendingHeader;
|
|
811
|
+
replace(
|
|
812
|
+
entries,
|
|
813
|
+
values,
|
|
814
|
+
0,
|
|
815
|
+
TOMBSTONE_WORDS,
|
|
816
|
+
ordinal >>> 5,
|
|
817
|
+
(state.tombstones[ordinal >>> 5] | 1 << (ordinal & 31)) >>> 0
|
|
818
|
+
);
|
|
819
|
+
replace(entries, values, 1, TOMBSTONE_COUNT, 0, state.tombstoneCount[0] + 1);
|
|
820
|
+
replace(entries, values, 2, PENDING_HEADER, PENDING_DOCUMENTS, header[PENDING_DOCUMENTS] + needed.documents);
|
|
821
|
+
replace(entries, values, 3, PENDING_HEADER, PENDING_TOKENS, header[PENDING_TOKENS] + needed.tokens);
|
|
822
|
+
replace(entries, values, 4, PENDING_HEADER, PENDING_UNITS, header[PENDING_UNITS] + needed.units);
|
|
823
|
+
return PREPARED_REPLACEMENTS;
|
|
824
|
+
} catch (error) {
|
|
825
|
+
return -refusal(error);
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
var settleFrozenState = (input, output, paged) => {
|
|
829
|
+
try {
|
|
830
|
+
const { segment, state } = readFrozenSettle(input, output);
|
|
831
|
+
settle(segment, state, paged);
|
|
832
|
+
return 0;
|
|
833
|
+
} catch (error) {
|
|
834
|
+
return refusal(error);
|
|
835
|
+
}
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
// src/core/frozen-writes/removal-scratch.ts
|
|
839
|
+
var NO_SURFACE2 = 4294967295;
|
|
840
|
+
var FIELD_TOKEN_WORDS2 = 8;
|
|
841
|
+
var NO_KEY = -1;
|
|
842
|
+
var ABSENT_TEXT = 4294967295;
|
|
843
|
+
var NO_ENTRY = 4294967295;
|
|
844
|
+
var NO_FIELD = -1;
|
|
845
|
+
var LISTED_TAG = 65535;
|
|
846
|
+
var FNV_OFFSET_BASIS2 = 2166136261;
|
|
847
|
+
var FNV_PRIME2 = 16777619;
|
|
848
|
+
function hashText(text, tag) {
|
|
849
|
+
let hash = FNV_OFFSET_BASIS2;
|
|
850
|
+
for (let at = 0; at < text.length; at++) hash = Math.imul(hash ^ text.charCodeAt(at), FNV_PRIME2);
|
|
851
|
+
return Math.imul(hash ^ tag, FNV_PRIME2);
|
|
852
|
+
}
|
|
853
|
+
function hashUnits(units, start, length, tag) {
|
|
854
|
+
let hash = FNV_OFFSET_BASIS2;
|
|
855
|
+
for (let at = start; at < start + length; at++) hash = Math.imul(hash ^ units[at], FNV_PRIME2);
|
|
856
|
+
return Math.imul(hash ^ tag, FNV_PRIME2);
|
|
857
|
+
}
|
|
858
|
+
function grownUint32(held, used, needed) {
|
|
859
|
+
if (needed <= held.length) return held;
|
|
860
|
+
const grown2 = new Uint32Array(Math.max(held.length * REMOVAL_SCRATCH_GROWTH_FACTOR, needed));
|
|
861
|
+
grown2.set(held.subarray(0, used));
|
|
862
|
+
return grown2;
|
|
863
|
+
}
|
|
864
|
+
function grownUint16(held, used, needed) {
|
|
865
|
+
if (needed <= held.length) return held;
|
|
866
|
+
const grown2 = new Uint16Array(Math.max(held.length * REMOVAL_SCRATCH_GROWTH_FACTOR, needed));
|
|
867
|
+
grown2.set(held.subarray(0, used));
|
|
868
|
+
return grown2;
|
|
869
|
+
}
|
|
870
|
+
var RemovalScratch = class {
|
|
871
|
+
tokens = new Uint32Array(REMOVAL_SCRATCH_START_TOKEN_WORDS);
|
|
872
|
+
units = new Uint16Array(REMOVAL_SCRATCH_START_UNITS);
|
|
873
|
+
tokenWords = 0;
|
|
874
|
+
unitCount = 0;
|
|
875
|
+
field = NO_FIELD;
|
|
876
|
+
fieldStart = 0;
|
|
877
|
+
keyCount = 0;
|
|
878
|
+
keyUnits = new Uint16Array(REMOVAL_SCRATCH_START_KEY_UNITS);
|
|
879
|
+
keyUnitCount = 0;
|
|
880
|
+
keyStarts = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
881
|
+
keyLengths = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
882
|
+
keyTags = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
883
|
+
keyFields = new Int32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
884
|
+
termAt = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
885
|
+
surfaceAt = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
886
|
+
listedAt = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
887
|
+
terms = [];
|
|
888
|
+
surfaces = [];
|
|
889
|
+
listed = [];
|
|
890
|
+
listedCount = 0;
|
|
891
|
+
listedCounts = [];
|
|
892
|
+
slots = new Int32Array(REMOVAL_SCRATCH_START_KEYS * REMOVAL_SCRATCH_SLOTS_PER_KEY).fill(NO_KEY);
|
|
893
|
+
reset() {
|
|
894
|
+
this.tokenWords = 0;
|
|
895
|
+
this.unitCount = 0;
|
|
896
|
+
this.field = NO_FIELD;
|
|
897
|
+
if (this.keyCount > 0) this.slots.fill(NO_KEY);
|
|
898
|
+
this.keyCount = 0;
|
|
899
|
+
this.keyUnitCount = 0;
|
|
900
|
+
this.listedCount = 0;
|
|
901
|
+
}
|
|
902
|
+
beginField() {
|
|
903
|
+
this.field++;
|
|
904
|
+
this.fieldStart = this.unitCount;
|
|
905
|
+
}
|
|
906
|
+
fieldUnits() {
|
|
907
|
+
return this.units.subarray(this.fieldStart, this.unitCount);
|
|
908
|
+
}
|
|
909
|
+
writeText(text) {
|
|
910
|
+
const start = this.unitCount;
|
|
911
|
+
this.units = grownUint16(this.units, start, start + text.length);
|
|
912
|
+
const units = this.units;
|
|
913
|
+
for (let at = 0; at < text.length; at++) units[start + at] = text.charCodeAt(at);
|
|
914
|
+
this.unitCount = start + text.length;
|
|
915
|
+
return start - this.fieldStart;
|
|
916
|
+
}
|
|
917
|
+
sameText(key, text, tag) {
|
|
918
|
+
if (this.keyTags[key] !== tag || this.keyLengths[key] !== text.length) return false;
|
|
919
|
+
const start = this.keyStarts[key];
|
|
920
|
+
const units = this.keyUnits;
|
|
921
|
+
for (let at = 0; at < text.length; at++) if (units[start + at] !== text.charCodeAt(at)) return false;
|
|
922
|
+
return true;
|
|
923
|
+
}
|
|
924
|
+
slotOf(text, tag) {
|
|
925
|
+
const slots = this.slots;
|
|
926
|
+
const mask = slots.length - 1;
|
|
927
|
+
let slot = hashText(text, tag) & mask;
|
|
928
|
+
let key = slots[slot];
|
|
929
|
+
while (key !== NO_KEY && !this.sameText(key, text, tag)) {
|
|
930
|
+
slot = slot + 1 & mask;
|
|
931
|
+
key = slots[slot];
|
|
932
|
+
}
|
|
933
|
+
return slot;
|
|
934
|
+
}
|
|
935
|
+
find(text, tag) {
|
|
936
|
+
return this.slots[this.slotOf(text, tag)];
|
|
937
|
+
}
|
|
938
|
+
rehash() {
|
|
939
|
+
const slots = new Int32Array(this.slots.length * REMOVAL_SCRATCH_GROWTH_FACTOR).fill(NO_KEY);
|
|
940
|
+
const mask = slots.length - 1;
|
|
941
|
+
for (let key = 0; key < this.keyCount; key++) {
|
|
942
|
+
let slot = hashUnits(this.keyUnits, this.keyStarts[key], this.keyLengths[key], this.keyTags[key]) & mask;
|
|
943
|
+
while (slots[slot] !== NO_KEY) slot = slot + 1 & mask;
|
|
944
|
+
slots[slot] = key;
|
|
945
|
+
}
|
|
946
|
+
this.slots = slots;
|
|
947
|
+
}
|
|
948
|
+
growKeys() {
|
|
949
|
+
const used = this.keyCount;
|
|
950
|
+
const needed = used + 1;
|
|
951
|
+
if (needed <= this.keyStarts.length) return;
|
|
952
|
+
this.keyStarts = grownUint32(this.keyStarts, used, needed);
|
|
953
|
+
this.keyLengths = grownUint32(this.keyLengths, used, needed);
|
|
954
|
+
this.keyTags = grownUint32(this.keyTags, used, needed);
|
|
955
|
+
this.termAt = grownUint32(this.termAt, used, needed);
|
|
956
|
+
this.surfaceAt = grownUint32(this.surfaceAt, used, needed);
|
|
957
|
+
this.listedAt = grownUint32(this.listedAt, used, needed);
|
|
958
|
+
const fields = new Int32Array(this.keyStarts.length);
|
|
959
|
+
fields.set(this.keyFields.subarray(0, used));
|
|
960
|
+
this.keyFields = fields;
|
|
961
|
+
}
|
|
962
|
+
add(text, tag, term, surface) {
|
|
963
|
+
if ((this.keyCount + 1) * REMOVAL_SCRATCH_SLOTS_PER_KEY > this.slots.length) this.rehash();
|
|
964
|
+
this.growKeys();
|
|
965
|
+
const key = this.keyCount++;
|
|
966
|
+
const start = this.keyUnitCount;
|
|
967
|
+
this.keyUnits = grownUint16(this.keyUnits, start, start + text.length);
|
|
968
|
+
for (let at = 0; at < text.length; at++) this.keyUnits[start + at] = text.charCodeAt(at);
|
|
969
|
+
this.keyUnitCount = start + text.length;
|
|
970
|
+
this.keyStarts[key] = start;
|
|
971
|
+
this.keyLengths[key] = text.length;
|
|
972
|
+
this.keyTags[key] = tag;
|
|
973
|
+
this.keyFields[key] = NO_FIELD;
|
|
974
|
+
this.listedAt[key] = NO_ENTRY;
|
|
975
|
+
this.terms[key] = term;
|
|
976
|
+
this.surfaces[key] = surface;
|
|
977
|
+
this.slots[this.slotOf(text, tag)] = key;
|
|
978
|
+
return key;
|
|
979
|
+
}
|
|
980
|
+
appendListed(spelling) {
|
|
981
|
+
const entry = this.listedCount++;
|
|
982
|
+
this.listedCounts[entry] = 0;
|
|
983
|
+
this.listed[entry] = spelling;
|
|
984
|
+
return entry;
|
|
985
|
+
}
|
|
986
|
+
listSpelling(key, spelling, shared) {
|
|
987
|
+
if (!shared) {
|
|
988
|
+
this.listedAt[key] = this.appendListed(spelling);
|
|
989
|
+
return;
|
|
990
|
+
}
|
|
991
|
+
let listing = this.find(spelling, LISTED_TAG);
|
|
992
|
+
if (listing === NO_KEY) {
|
|
993
|
+
listing = this.add(spelling, LISTED_TAG, null, null);
|
|
994
|
+
this.listedAt[listing] = this.appendListed(spelling);
|
|
995
|
+
}
|
|
996
|
+
this.listedAt[key] = this.listedAt[listing];
|
|
997
|
+
}
|
|
998
|
+
countListed(key) {
|
|
999
|
+
const entry = this.listedAt[key];
|
|
1000
|
+
if (entry !== NO_ENTRY) this.listedCounts[entry]++;
|
|
1001
|
+
}
|
|
1002
|
+
recordsTokens(key) {
|
|
1003
|
+
return this.terms[key] !== null;
|
|
1004
|
+
}
|
|
1005
|
+
writeRecord(position, key) {
|
|
1006
|
+
const term = this.terms[key];
|
|
1007
|
+
if (term === null) return;
|
|
1008
|
+
if (this.keyFields[key] !== this.field) {
|
|
1009
|
+
const surface = this.surfaces[key];
|
|
1010
|
+
this.termAt[key] = this.writeText(term);
|
|
1011
|
+
this.surfaceAt[key] = surface === null ? NO_SURFACE2 : this.writeText(surface);
|
|
1012
|
+
this.keyFields[key] = this.field;
|
|
1013
|
+
}
|
|
1014
|
+
const at = this.tokenWords;
|
|
1015
|
+
this.tokens = grownUint32(this.tokens, at, at + FIELD_TOKEN_WORDS2);
|
|
1016
|
+
const tokens = this.tokens;
|
|
1017
|
+
const surfaceAt = this.surfaceAt[key];
|
|
1018
|
+
tokens[at] = position;
|
|
1019
|
+
tokens[at + 1] = this.termAt[key];
|
|
1020
|
+
tokens[at + 2] = term.length;
|
|
1021
|
+
tokens[at + 3] = surfaceAt;
|
|
1022
|
+
tokens[at + 4] = surfaceAt === NO_SURFACE2 ? 0 : this.surfaces[key]?.length ?? 0;
|
|
1023
|
+
tokens[at + 5] = 0;
|
|
1024
|
+
tokens[at + 6] = 0;
|
|
1025
|
+
tokens[at + 7] = ABSENT_TEXT;
|
|
1026
|
+
this.tokenWords = at + FIELD_TOKEN_WORDS2;
|
|
1027
|
+
}
|
|
1028
|
+
copyRecord(source, at) {
|
|
1029
|
+
this.tokens = grownUint32(this.tokens, this.tokenWords, this.tokenWords + FIELD_TOKEN_WORDS2);
|
|
1030
|
+
this.tokens.set(source.subarray(at, at + FIELD_TOKEN_WORDS2), this.tokenWords);
|
|
1031
|
+
this.tokenWords += FIELD_TOKEN_WORDS2;
|
|
1032
|
+
}
|
|
1033
|
+
removedSpellings() {
|
|
1034
|
+
const size = this.listedCount;
|
|
1035
|
+
if (size === 0) return { spellings: [], counts: [] };
|
|
1036
|
+
return { spellings: this.listed.slice(0, size), counts: this.listedCounts.slice(0, size) };
|
|
1037
|
+
}
|
|
1038
|
+
};
|
|
1039
|
+
|
|
1040
|
+
// src/core/frozen-writes/removal-strings.ts
|
|
1041
|
+
var SURFACE_TAG = 0;
|
|
1042
|
+
var PARTIAL_TAG = 1;
|
|
1043
|
+
var NO_SPELLING_TOKENS = new Uint32Array(0);
|
|
1044
|
+
var NO_REMOVED_PARTIAL_SPELLINGS = new Uint32Array(0);
|
|
1045
|
+
var scratch = new RemovalScratch();
|
|
1046
|
+
var visitedFields = [""];
|
|
1047
|
+
var visitedTag = SURFACE_TAG;
|
|
1048
|
+
var visitedPosition = 0;
|
|
1049
|
+
var sharedListing = false;
|
|
1050
|
+
var visitedSurfaces = null;
|
|
1051
|
+
var visitedPartialWords = null;
|
|
1052
|
+
var normalizingLanguage = null;
|
|
1053
|
+
var normalizingStopWords = null;
|
|
1054
|
+
var flaggedFieldNames = null;
|
|
1055
|
+
var flaggedPartialFields;
|
|
1056
|
+
var heldPartialFlags = new Uint8Array(0);
|
|
1057
|
+
function normalizedOf(spelling) {
|
|
1058
|
+
if (normalizingLanguage === null || normalizingStopWords?.has(spelling) === true) return null;
|
|
1059
|
+
return transformToken(spelling, normalizingLanguage, false, false);
|
|
1060
|
+
}
|
|
1061
|
+
function resolveSpelling(written, surfaceToken) {
|
|
1062
|
+
const partial = visitedTag === PARTIAL_TAG;
|
|
1063
|
+
let spelling = written;
|
|
1064
|
+
let token = partial ? visitedPartialWords?.tokenForSpelling(written) : surfaceToken;
|
|
1065
|
+
if (token === void 0 && partial) {
|
|
1066
|
+
const normalized = normalizedOf(written);
|
|
1067
|
+
if (normalized !== null) {
|
|
1068
|
+
token = visitedPartialWords?.tokenForSpelling(normalized);
|
|
1069
|
+
if (token !== void 0) spelling = normalized;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
token ??= visitedSurfaces?.tokenOf(written);
|
|
1073
|
+
const differs = token !== void 0 && spelling !== token && spelling.length > 0;
|
|
1074
|
+
const records = token !== void 0 && (partial || differs);
|
|
1075
|
+
const key = scratch.add(
|
|
1076
|
+
written,
|
|
1077
|
+
visitedTag,
|
|
1078
|
+
records && token !== void 0 ? token : null,
|
|
1079
|
+
records && spelling !== token ? spelling : null
|
|
1080
|
+
);
|
|
1081
|
+
if (differs) scratch.listSpelling(key, spelling, sharedListing);
|
|
1082
|
+
return key;
|
|
1083
|
+
}
|
|
1084
|
+
function visitSpelling(spelling) {
|
|
1085
|
+
const surfaceToken = visitedTag === SURFACE_TAG ? visitedSurfaces?.tokenOf(spelling) : void 0;
|
|
1086
|
+
if (visitedTag === SURFACE_TAG && (surfaceToken === void 0 || surfaceToken === spelling)) return;
|
|
1087
|
+
let key = scratch.find(spelling, visitedTag);
|
|
1088
|
+
if (key === NO_KEY) key = resolveSpelling(spelling, surfaceToken);
|
|
1089
|
+
scratch.countListed(key);
|
|
1090
|
+
if (scratch.recordsTokens(key)) scratch.writeRecord(visitedPosition++, key);
|
|
1091
|
+
}
|
|
1092
|
+
function visitedRecords(field, document, analysis, isPartial) {
|
|
1093
|
+
const firstWord = scratch.tokenWords;
|
|
1094
|
+
scratch.beginField();
|
|
1095
|
+
visitedTag = isPartial ? PARTIAL_TAG : SURFACE_TAG;
|
|
1096
|
+
visitedPosition = 0;
|
|
1097
|
+
visitDocumentSurfaceSpellings(document, analysis, visitSpelling);
|
|
1098
|
+
if (scratch.tokenWords === firstWord) return null;
|
|
1099
|
+
return { field, tokens: scratch.tokens.subarray(firstWord, scratch.tokenWords), units: scratch.fieldUnits() };
|
|
1100
|
+
}
|
|
1101
|
+
function preparedRecords(field, isPartial, { tokens, units }) {
|
|
1102
|
+
const firstWord = scratch.tokenWords;
|
|
1103
|
+
for (let at = 0; at < tokens.length; at += FIELD_TOKEN_WORDS2) {
|
|
1104
|
+
const surface = tokens[at + 3] === NO_SURFACE2 || tokens[at + 4] === 0 ? "" : storedText(units, tokens[at + 3], tokens[at + 4]);
|
|
1105
|
+
const differs = surface.length > 0 && surface !== storedText(units, tokens[at + 1], tokens[at + 2]);
|
|
1106
|
+
if (differs) {
|
|
1107
|
+
let key = scratch.find(surface, SURFACE_TAG);
|
|
1108
|
+
if (key === NO_KEY) {
|
|
1109
|
+
key = scratch.add(surface, SURFACE_TAG, null, null);
|
|
1110
|
+
scratch.listSpelling(key, surface, true);
|
|
1111
|
+
}
|
|
1112
|
+
scratch.countListed(key);
|
|
1113
|
+
}
|
|
1114
|
+
if (isPartial || differs) scratch.copyRecord(tokens, at);
|
|
1115
|
+
}
|
|
1116
|
+
if (scratch.tokenWords === firstWord) return null;
|
|
1117
|
+
return { field, tokens: scratch.tokens.subarray(firstWord, scratch.tokenWords), units };
|
|
1118
|
+
}
|
|
1119
|
+
function beginVisit(surfaces, partialWords, analysis, partialFields) {
|
|
1120
|
+
scratch.reset();
|
|
1121
|
+
visitedSurfaces = surfaces;
|
|
1122
|
+
visitedPartialWords = partialWords;
|
|
1123
|
+
sharedListing = partialFields !== void 0;
|
|
1124
|
+
const { language, options } = analysis;
|
|
1125
|
+
const normalizes = partialFields !== void 0 && options?.customTokenizer === void 0 && language.normalizer !== void 0 && language.tokenizer?.normalizeDiacritics !== true;
|
|
1126
|
+
normalizingLanguage = normalizes ? language : null;
|
|
1127
|
+
normalizingStopWords = normalizes ? resolveStopWords(language, options?.stopWordOverride) : null;
|
|
1128
|
+
}
|
|
1129
|
+
function endVisit() {
|
|
1130
|
+
visitedSurfaces = null;
|
|
1131
|
+
visitedPartialWords = null;
|
|
1132
|
+
normalizingLanguage = null;
|
|
1133
|
+
normalizingStopWords = null;
|
|
1134
|
+
}
|
|
1135
|
+
function partialFlagsOf(fieldNames, partialFields) {
|
|
1136
|
+
if (fieldNames === flaggedFieldNames && partialFields === flaggedPartialFields) return heldPartialFlags;
|
|
1137
|
+
const flags = new Uint8Array(fieldNames.length);
|
|
1138
|
+
for (const path of partialFields ?? []) {
|
|
1139
|
+
const field = fieldNames.indexOf(path);
|
|
1140
|
+
if (field >= 0) flags[field] = 1;
|
|
1141
|
+
}
|
|
1142
|
+
flaggedFieldNames = fieldNames;
|
|
1143
|
+
flaggedPartialFields = partialFields;
|
|
1144
|
+
heldPartialFlags = flags;
|
|
1145
|
+
return flags;
|
|
1146
|
+
}
|
|
1147
|
+
function collectRecords(fieldNames, textFields, document, analysis, surfaces, partialFields) {
|
|
1148
|
+
const fields = [];
|
|
1149
|
+
const custom = analysis.options?.customTokenizer !== void 0;
|
|
1150
|
+
if (!custom && partialFields === void 0 && !surfaces.holdsAny()) return fields;
|
|
1151
|
+
const visiting = {
|
|
1152
|
+
language: analysis.language,
|
|
1153
|
+
options: analysis.options,
|
|
1154
|
+
fields: visitedFields
|
|
1155
|
+
};
|
|
1156
|
+
for (let field = 0; field < fieldNames.length; field++) {
|
|
1157
|
+
const name = fieldNames[field];
|
|
1158
|
+
const isPartial = partialFields?.includes(name) === true;
|
|
1159
|
+
let records = null;
|
|
1160
|
+
if (custom) {
|
|
1161
|
+
const prepared = analysis.text?.fields.get(name);
|
|
1162
|
+
if (prepared !== void 0) records = preparedRecords(field, isPartial, packedTokens(prepared.records));
|
|
1163
|
+
} else if (textFields.includes(name)) {
|
|
1164
|
+
visitedFields[0] = name;
|
|
1165
|
+
records = visitedRecords(field, document, visiting, isPartial);
|
|
1166
|
+
}
|
|
1167
|
+
if (records !== null) fields.push(records);
|
|
1168
|
+
}
|
|
1169
|
+
return fields;
|
|
1170
|
+
}
|
|
1171
|
+
function prepareFrozenStrings(fieldNames, textFields, ordinal, document, analysis, surfaces, partialWords) {
|
|
1172
|
+
const partial = partialWords.removalState();
|
|
1173
|
+
const partialFields = partial?.table.fields;
|
|
1174
|
+
beginVisit(surfaces, partialWords, analysis, partialFields);
|
|
1175
|
+
let fields;
|
|
1176
|
+
try {
|
|
1177
|
+
fields = collectRecords(fieldNames, textFields, document, analysis, surfaces, partialFields);
|
|
1178
|
+
} finally {
|
|
1179
|
+
endVisit();
|
|
1180
|
+
}
|
|
1181
|
+
const spellings = scratch.removedSpellings();
|
|
1182
|
+
const surface = surfaces.prepareCoreRemoval(ordinal, spellings);
|
|
1183
|
+
const partialFlags = partialFlagsOf(fieldNames, partialFields);
|
|
1184
|
+
return {
|
|
1185
|
+
spellings,
|
|
1186
|
+
core: {
|
|
1187
|
+
fields,
|
|
1188
|
+
partialFields: partialFlags,
|
|
1189
|
+
surfaces: surface.table,
|
|
1190
|
+
removedSurfaces: surface.removed,
|
|
1191
|
+
spellings: partial === null ? null : {
|
|
1192
|
+
text: partial.table.spellings,
|
|
1193
|
+
offsets: partial.table.spellingOffsets,
|
|
1194
|
+
counts: partial.table.spellingCounts
|
|
1195
|
+
},
|
|
1196
|
+
spellingTokens: partial?.table.spellingTokens ?? NO_SPELLING_TOKENS,
|
|
1197
|
+
removedSpellings: partial?.removed ?? NO_REMOVED_PARTIAL_SPELLINGS
|
|
1198
|
+
},
|
|
1199
|
+
publish: surface.publish
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
// src/core/segment-writes/removal-host.browser.ts
|
|
1204
|
+
var POSTING_LIST_COMPACTION_THRESHOLD;
|
|
1205
|
+
var SegmentRefusal2;
|
|
1206
|
+
var StringPlan;
|
|
1207
|
+
var acquireTokenPlan;
|
|
1208
|
+
var findString;
|
|
1209
|
+
var holdsUnpairedSurrogate;
|
|
1210
|
+
var prepareSpellingRemoval;
|
|
1211
|
+
var preparedSources;
|
|
1212
|
+
var readSegmentView;
|
|
1213
|
+
var releaseSlice;
|
|
1214
|
+
var releaseTokenPlan;
|
|
1215
|
+
var removeString;
|
|
1216
|
+
var requireSegment2;
|
|
1217
|
+
var rowRemoved;
|
|
1218
|
+
var sharedTexts;
|
|
1219
|
+
var stringHash;
|
|
1220
|
+
var trimClass;
|
|
1221
|
+
var trimSlice;
|
|
1222
|
+
function bindSegmentRemovalHost(host) {
|
|
1223
|
+
({
|
|
1224
|
+
POSTING_LIST_COMPACTION_THRESHOLD,
|
|
1225
|
+
SegmentRefusal: SegmentRefusal2,
|
|
1226
|
+
StringPlan,
|
|
1227
|
+
acquireTokenPlan,
|
|
1228
|
+
findString,
|
|
1229
|
+
holdsUnpairedSurrogate,
|
|
1230
|
+
prepareSpellingRemoval,
|
|
1231
|
+
preparedSources,
|
|
1232
|
+
readSegmentView,
|
|
1233
|
+
releaseSlice,
|
|
1234
|
+
releaseTokenPlan,
|
|
1235
|
+
removeString,
|
|
1236
|
+
requireSegment: requireSegment2,
|
|
1237
|
+
rowRemoved,
|
|
1238
|
+
sharedTexts,
|
|
1239
|
+
stringHash,
|
|
1240
|
+
trimClass,
|
|
1241
|
+
trimSlice
|
|
1242
|
+
} = host);
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
// src/core/segment-writes/document-removal.ts
|
|
1246
|
+
var PREPARED_ROW_WORDS = 3;
|
|
1247
|
+
var SURFACE_WORD = 1;
|
|
1248
|
+
var ABSENT_TEXT2 = 4294967295;
|
|
1249
|
+
var LAST_MARK = 4294967295;
|
|
1250
|
+
var ABSENT = -1;
|
|
1251
|
+
var DocumentRemoval = class {
|
|
1252
|
+
size = 0;
|
|
1253
|
+
ordinals = new Uint32Array(0);
|
|
1254
|
+
hashes = new Uint32Array(0);
|
|
1255
|
+
texts = [];
|
|
1256
|
+
surfaces = null;
|
|
1257
|
+
termMarks = new Uint32Array(0);
|
|
1258
|
+
termAt = new Int32Array(0);
|
|
1259
|
+
surfaceMarks = new Uint32Array(0);
|
|
1260
|
+
surfaceAt = new Int32Array(0);
|
|
1261
|
+
mark = 0;
|
|
1262
|
+
surfacePlan = null;
|
|
1263
|
+
collect(segment, texts, fields) {
|
|
1264
|
+
this.reserve(texts.length);
|
|
1265
|
+
this.size = 0;
|
|
1266
|
+
this.texts.length = 0;
|
|
1267
|
+
const mark = this.takeMark();
|
|
1268
|
+
const table = segment.surfaces;
|
|
1269
|
+
const surfaces = table === null ? null : this.surfacePlanFor(table.strings);
|
|
1270
|
+
this.surfaces = surfaces;
|
|
1271
|
+
const termMarks = this.termMarks;
|
|
1272
|
+
const termAt = this.termAt;
|
|
1273
|
+
for (const { text } of fields) {
|
|
1274
|
+
const rows = text.rows;
|
|
1275
|
+
const end = text.count * PREPARED_ROW_WORDS;
|
|
1276
|
+
for (let row = 0; row < end; row += PREPARED_ROW_WORDS) {
|
|
1277
|
+
const termId = rows[row];
|
|
1278
|
+
if (termMarks[termId] !== mark) {
|
|
1279
|
+
termMarks[termId] = mark;
|
|
1280
|
+
termAt[termId] = this.addTerm(segment.terms, texts[termId]);
|
|
1281
|
+
}
|
|
1282
|
+
if (surfaces === null || table === null || termAt[termId] === ABSENT) continue;
|
|
1283
|
+
const surfaceId = rows[row + SURFACE_WORD];
|
|
1284
|
+
if (surfaceId !== ABSENT_TEXT2 && surfaceId !== termId)
|
|
1285
|
+
this.countSurface(table, surfaces, texts, surfaceId, termId, mark);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
addTerm(terms, text) {
|
|
1290
|
+
const hash = stringHash(text);
|
|
1291
|
+
const found = findString(terms, text, hash);
|
|
1292
|
+
if (found < 0) return ABSENT;
|
|
1293
|
+
const at = this.size++;
|
|
1294
|
+
this.ordinals[at] = found;
|
|
1295
|
+
this.hashes[at] = hash;
|
|
1296
|
+
this.texts.push(text);
|
|
1297
|
+
return at;
|
|
1298
|
+
}
|
|
1299
|
+
countSurface(table, surfaces, texts, surfaceId, termId, mark) {
|
|
1300
|
+
const surface = texts[surfaceId];
|
|
1301
|
+
if (surface === "" || surface === texts[termId]) return;
|
|
1302
|
+
if (this.surfaceMarks[surfaceId] !== mark) {
|
|
1303
|
+
this.surfaceMarks[surfaceId] = mark;
|
|
1304
|
+
this.surfaceAt[surfaceId] = surfaces.prepareExisting(surface, false);
|
|
1305
|
+
}
|
|
1306
|
+
const at = this.surfaceAt[surfaceId];
|
|
1307
|
+
if (at < 0) return;
|
|
1308
|
+
if (surfaces.counts[at] === 0) {
|
|
1309
|
+
surfaces.tokens[at] = table.token[surfaces.ordinals[at]];
|
|
1310
|
+
surfaces.points[at] = 0;
|
|
1311
|
+
}
|
|
1312
|
+
surfaces.counts[at]++;
|
|
1313
|
+
}
|
|
1314
|
+
surfacePlanFor(strings) {
|
|
1315
|
+
const plan = this.surfacePlan === null ? new StringPlan(strings) : this.surfacePlan.reset(strings);
|
|
1316
|
+
this.surfacePlan = plan;
|
|
1317
|
+
return plan;
|
|
1318
|
+
}
|
|
1319
|
+
takeMark() {
|
|
1320
|
+
if (this.mark === LAST_MARK) {
|
|
1321
|
+
this.termMarks.fill(0);
|
|
1322
|
+
this.surfaceMarks.fill(0);
|
|
1323
|
+
this.mark = 0;
|
|
1324
|
+
}
|
|
1325
|
+
return ++this.mark;
|
|
1326
|
+
}
|
|
1327
|
+
reserve(texts) {
|
|
1328
|
+
if (texts <= this.termMarks.length) return;
|
|
1329
|
+
const capacity = Math.max(texts, this.termMarks.length * 2);
|
|
1330
|
+
this.ordinals = new Uint32Array(capacity);
|
|
1331
|
+
this.hashes = new Uint32Array(capacity);
|
|
1332
|
+
this.termMarks = new Uint32Array(capacity);
|
|
1333
|
+
this.termAt = new Int32Array(capacity);
|
|
1334
|
+
this.surfaceMarks = new Uint32Array(capacity);
|
|
1335
|
+
this.surfaceAt = new Int32Array(capacity);
|
|
1336
|
+
}
|
|
1337
|
+
};
|
|
1338
|
+
|
|
1339
|
+
// src/core/segment-writes/removal.ts
|
|
1340
|
+
var NO_SLICE_CLASS = 255;
|
|
1341
|
+
var ORDERED_FLAG = 2;
|
|
1342
|
+
var POSITIONS_FLAG = 4;
|
|
1343
|
+
function firstRow(docIds, start, length, ordered, document) {
|
|
1344
|
+
let low = start;
|
|
1345
|
+
if (!ordered) return low;
|
|
1346
|
+
let high = low + length;
|
|
1347
|
+
while (low < high) {
|
|
1348
|
+
const middle = low + (high - low >>> 1);
|
|
1349
|
+
if (docIds[middle] < document) low = middle + 1;
|
|
1350
|
+
else high = middle;
|
|
1351
|
+
}
|
|
1352
|
+
return low;
|
|
1353
|
+
}
|
|
1354
|
+
var RemovalPlan = class {
|
|
1355
|
+
found = new Uint8Array(0);
|
|
1356
|
+
compacts = new Uint8Array(0);
|
|
1357
|
+
ordered = new Uint8Array(0);
|
|
1358
|
+
sliceClasses = new Uint8Array(0);
|
|
1359
|
+
kept = new Uint32Array(0);
|
|
1360
|
+
removedFrequencies = new Float64Array(0);
|
|
1361
|
+
freedPositions = 0;
|
|
1362
|
+
freedRows = 0;
|
|
1363
|
+
prepare(segment, document, terms) {
|
|
1364
|
+
this.reserve(terms.size);
|
|
1365
|
+
const allocation = segment.postings.allocation;
|
|
1366
|
+
requireSegment2(allocation[0] <= segment.postings.docIds.length && allocation[1] <= allocation[0]);
|
|
1367
|
+
this.freedRows = allocation[1];
|
|
1368
|
+
this.freedPositions = 0;
|
|
1369
|
+
const positions = segment.positions;
|
|
1370
|
+
if (positions !== null)
|
|
1371
|
+
requireSegment2(
|
|
1372
|
+
positions.allocation[0] > 0 && positions.allocation[0] <= positions.values.length && positions.allocation[1] < positions.allocation[0]
|
|
1373
|
+
);
|
|
1374
|
+
for (let at = 0; at < terms.size; at++) this.prepareTerm(segment, document, at, terms.ordinals[at]);
|
|
1375
|
+
if (positions !== null) requireSegment2(this.freedPositions <= positions.allocation[0] - positions.allocation[1]);
|
|
1376
|
+
}
|
|
1377
|
+
prepareTerm(segment, document, at, token) {
|
|
1378
|
+
const { columns, postings } = segment;
|
|
1379
|
+
const start = columns.sliceStart[token];
|
|
1380
|
+
const length = columns.lengths[token];
|
|
1381
|
+
const kind = columns.sliceClass[token];
|
|
1382
|
+
const end = postings.allocation[0];
|
|
1383
|
+
this.found[at] = 0;
|
|
1384
|
+
this.compacts[at] = 0;
|
|
1385
|
+
this.removedFrequencies[at] = 0;
|
|
1386
|
+
const docFrequency = columns.docFrequency[token];
|
|
1387
|
+
const totalTermFrequency = columns.totalTermFrequency[token];
|
|
1388
|
+
const surfaceOccurrences = columns.surfaceOccurrences[token];
|
|
1389
|
+
requireSegment2(
|
|
1390
|
+
docFrequency >= columns.deadDocuments[token] && docFrequency <= length && Number.isFinite(totalTermFrequency) && totalTermFrequency >= 0 && Number.isFinite(surfaceOccurrences) && surfaceOccurrences >= 0 && start <= end
|
|
1391
|
+
);
|
|
1392
|
+
if (kind === NO_SLICE_CLASS && length === 0) return;
|
|
1393
|
+
requireSegment2(
|
|
1394
|
+
kind < postings.sliceCapacities.length && postings.sliceCapacities[kind] <= end - start && length <= postings.sliceCapacities[kind]
|
|
1395
|
+
);
|
|
1396
|
+
const ordered = (columns.flags[token] & ORDERED_FLAG) !== 0;
|
|
1397
|
+
const { docIds, fieldIndices, frequencies } = postings;
|
|
1398
|
+
const ordinalCapacity = segment.ordinalCapacity;
|
|
1399
|
+
const fieldCount = segment.partialFields.length;
|
|
1400
|
+
let removedFrequency = 0;
|
|
1401
|
+
let found = false;
|
|
1402
|
+
for (let row = firstRow(docIds, start, length, ordered, document); row < start + length; row++) {
|
|
1403
|
+
const doc = docIds[row];
|
|
1404
|
+
if (ordered && doc > document) break;
|
|
1405
|
+
requireSegment2(doc < ordinalCapacity && fieldIndices[row] < fieldCount);
|
|
1406
|
+
if (doc === document) {
|
|
1407
|
+
found = true;
|
|
1408
|
+
removedFrequency += frequencies[row];
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
if (!found) return;
|
|
1412
|
+
this.found[at] = 1;
|
|
1413
|
+
this.removedFrequencies[at] = removedFrequency;
|
|
1414
|
+
const dead = columns.deadDocuments[token];
|
|
1415
|
+
const live = docFrequency - dead;
|
|
1416
|
+
requireSegment2(live > 0);
|
|
1417
|
+
if (live !== 1 && (dead + 1) / length <= POSTING_LIST_COMPACTION_THRESHOLD) return;
|
|
1418
|
+
this.compacts[at] = 1;
|
|
1419
|
+
this.prepareCompaction(segment, document, at, token, live);
|
|
1420
|
+
}
|
|
1421
|
+
prepareCompaction(segment, document, at, token, live) {
|
|
1422
|
+
const { columns, postings } = segment;
|
|
1423
|
+
const start = columns.sliceStart[token];
|
|
1424
|
+
const length = columns.lengths[token];
|
|
1425
|
+
const kind = columns.sliceClass[token];
|
|
1426
|
+
let kept = 0;
|
|
1427
|
+
let last = 0;
|
|
1428
|
+
let ordered = 1;
|
|
1429
|
+
for (let row = start; row < start + length; row++) {
|
|
1430
|
+
const doc = postings.docIds[row];
|
|
1431
|
+
requireSegment2(doc < segment.ordinalCapacity && postings.fieldIndices[row] < segment.partialFields.length);
|
|
1432
|
+
if (doc === document || rowRemoved(segment, doc)) this.preparePositions(segment, token, row);
|
|
1433
|
+
else {
|
|
1434
|
+
if (kept > 0 && doc < last) ordered = 0;
|
|
1435
|
+
last = doc;
|
|
1436
|
+
kept++;
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
requireSegment2(live === 1 === (kept === 0) && live - 1 <= kept);
|
|
1440
|
+
const capacities = postings.sliceCapacities;
|
|
1441
|
+
const sliceClass = kept > 0 ? trimClass(capacities, kind, kept) : kind;
|
|
1442
|
+
const retained = kept === 0 ? 0 : capacities[sliceClass];
|
|
1443
|
+
const released = capacities[kind] - retained;
|
|
1444
|
+
requireSegment2(released <= postings.allocation[0] - this.freedRows);
|
|
1445
|
+
this.freedRows += released;
|
|
1446
|
+
this.kept[at] = kept;
|
|
1447
|
+
this.ordered[at] = ordered;
|
|
1448
|
+
this.sliceClasses[at] = sliceClass;
|
|
1449
|
+
}
|
|
1450
|
+
preparePositions(segment, token, row) {
|
|
1451
|
+
const { positions, postings, columns } = segment;
|
|
1452
|
+
if (positions === null || postings.positionStarts === null || (columns.flags[token] & POSITIONS_FLAG) === 0) return;
|
|
1453
|
+
const start = postings.positionStarts[row];
|
|
1454
|
+
if (start === 0) return;
|
|
1455
|
+
const end = positions.allocation[0];
|
|
1456
|
+
requireSegment2(start < end && positions.values[start] < end - start);
|
|
1457
|
+
const count = positions.values[start] + 1;
|
|
1458
|
+
requireSegment2(count <= 4294967295 - this.freedPositions);
|
|
1459
|
+
this.freedPositions += count;
|
|
1460
|
+
}
|
|
1461
|
+
publish(segment, document, terms) {
|
|
1462
|
+
const columns = segment.columns;
|
|
1463
|
+
for (let at = 0; at < terms.size; at++) {
|
|
1464
|
+
if (this.found[at] === 0) continue;
|
|
1465
|
+
const token = terms.ordinals[at];
|
|
1466
|
+
columns.revision[token]++;
|
|
1467
|
+
columns.deadDocuments[token]++;
|
|
1468
|
+
columns.totalTermFrequency[token] = Math.max(0, columns.totalTermFrequency[token] - this.removedFrequencies[at]);
|
|
1469
|
+
if (this.compacts[at] === 1) this.publishCompaction(segment, document, terms, at, token);
|
|
1470
|
+
}
|
|
1471
|
+
if (segment.positions !== null) segment.positions.allocation[1] += this.freedPositions;
|
|
1472
|
+
}
|
|
1473
|
+
publishCompaction(segment, document, terms, at, token) {
|
|
1474
|
+
const { columns, postings } = segment;
|
|
1475
|
+
this.compactRows(segment, document, at, token);
|
|
1476
|
+
const kept = this.kept[at];
|
|
1477
|
+
const sliceClass = this.sliceClasses[at];
|
|
1478
|
+
const kind = columns.sliceClass[token];
|
|
1479
|
+
if (kept > 0 && kind > sliceClass) {
|
|
1480
|
+
const pool2 = { ...postings, values: postings.docIds };
|
|
1481
|
+
trimSlice(
|
|
1482
|
+
pool2,
|
|
1483
|
+
{ start: columns.sliceStart[token], length: pool2.sliceCapacities[kind], sliceClass: kind },
|
|
1484
|
+
sliceClass
|
|
1485
|
+
);
|
|
1486
|
+
columns.sliceClass[token] = sliceClass;
|
|
1487
|
+
}
|
|
1488
|
+
if (kept > 0) return;
|
|
1489
|
+
const pool = { ...postings, values: postings.docIds };
|
|
1490
|
+
releaseSlice(pool, { start: columns.sliceStart[token], length: pool.sliceCapacities[kind], sliceClass: kind });
|
|
1491
|
+
removeString(segment.terms, terms.texts[at], token, terms.hashes[at]);
|
|
1492
|
+
columns.sliceClass[token] = NO_SLICE_CLASS;
|
|
1493
|
+
columns.flags[token] = 0;
|
|
1494
|
+
columns.totalTermFrequency[token] = 0;
|
|
1495
|
+
}
|
|
1496
|
+
compactRows(segment, document, at, token) {
|
|
1497
|
+
const { columns, postings } = segment;
|
|
1498
|
+
const start = columns.sliceStart[token];
|
|
1499
|
+
let into = start;
|
|
1500
|
+
for (let row = start; row < start + columns.lengths[token]; row++) {
|
|
1501
|
+
const doc = postings.docIds[row];
|
|
1502
|
+
if (doc === document || rowRemoved(segment, doc)) continue;
|
|
1503
|
+
postings.docIds[into] = doc;
|
|
1504
|
+
postings.frequencies[into] = postings.frequencies[row];
|
|
1505
|
+
postings.fieldIndices[into] = postings.fieldIndices[row];
|
|
1506
|
+
if (postings.positionStarts !== null) postings.positionStarts[into] = postings.positionStarts[row];
|
|
1507
|
+
into++;
|
|
1508
|
+
}
|
|
1509
|
+
columns.lengths[token] = this.kept[at];
|
|
1510
|
+
columns.docFrequency[token] -= columns.deadDocuments[token];
|
|
1511
|
+
columns.deadDocuments[token] = 0;
|
|
1512
|
+
columns.flags[token] = this.ordered[at] === 1 ? columns.flags[token] | ORDERED_FLAG : columns.flags[token] & ~ORDERED_FLAG;
|
|
1513
|
+
}
|
|
1514
|
+
reserve(size) {
|
|
1515
|
+
if (size <= this.found.length) return;
|
|
1516
|
+
const capacity = Math.max(size, this.found.length * 2);
|
|
1517
|
+
this.found = new Uint8Array(capacity);
|
|
1518
|
+
this.compacts = new Uint8Array(capacity);
|
|
1519
|
+
this.ordered = new Uint8Array(capacity);
|
|
1520
|
+
this.sliceClasses = new Uint8Array(capacity);
|
|
1521
|
+
this.kept = new Uint32Array(capacity);
|
|
1522
|
+
this.removedFrequencies = new Float64Array(capacity);
|
|
1523
|
+
}
|
|
1524
|
+
};
|
|
1525
|
+
|
|
1526
|
+
// src/core/segment-writes/remove.ts
|
|
1527
|
+
var removalPlan = new RemovalPlan();
|
|
1528
|
+
var documentRemoval = new DocumentRemoval();
|
|
1529
|
+
function nonnegative(value) {
|
|
1530
|
+
return value > 0 ? value : 0;
|
|
1531
|
+
}
|
|
1532
|
+
function requireOrdinal(segment, ordinal) {
|
|
1533
|
+
requireSegment2(Number.isInteger(ordinal) && ordinal >= 0 && ordinal < segment.ordinalCapacity);
|
|
1534
|
+
}
|
|
1535
|
+
function requireFields(segment, fields) {
|
|
1536
|
+
const fieldCount = segment.partialFields.length;
|
|
1537
|
+
requireSegment2(fields.length <= fieldCount);
|
|
1538
|
+
let previous = -1;
|
|
1539
|
+
for (const { field } of fields) {
|
|
1540
|
+
requireSegment2(Number.isInteger(field) && field > previous && field < fieldCount);
|
|
1541
|
+
previous = field;
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
function removeSurfaces(plan, table, segment) {
|
|
1545
|
+
for (let at = 0; at < plan.size; at++) {
|
|
1546
|
+
const ordinal = plan.ordinals[at];
|
|
1547
|
+
const token = plan.tokens[at];
|
|
1548
|
+
const removed = Math.min(plan.counts[at], table.count[ordinal]);
|
|
1549
|
+
table.count[ordinal] -= removed;
|
|
1550
|
+
if (table === segment.surfaces && token < segment.terms.header[0])
|
|
1551
|
+
segment.columns.surfaceOccurrences[token] = nonnegative(segment.columns.surfaceOccurrences[token] - removed);
|
|
1552
|
+
if (table.count[ordinal] === 0) removeString(table.strings, plan.texts[at], ordinal, plan.hashes[at]);
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
function removeSharedTexts(segment, ordinal, texts, fields) {
|
|
1556
|
+
for (const text of texts) requireSegment2(!holdsUnpairedSurrogate(text));
|
|
1557
|
+
documentRemoval.collect(segment, texts, fields);
|
|
1558
|
+
removalPlan.prepare(segment, ordinal, documentRemoval);
|
|
1559
|
+
removalPlan.publish(segment, ordinal, documentRemoval);
|
|
1560
|
+
const surfaces = documentRemoval.surfaces;
|
|
1561
|
+
if (surfaces !== null && segment.surfaces !== null) removeSurfaces(surfaces, segment.surfaces, segment);
|
|
1562
|
+
}
|
|
1563
|
+
function removeTokens(segment, ordinal, tokens) {
|
|
1564
|
+
removalPlan.prepare(segment, ordinal, tokens.terms);
|
|
1565
|
+
const spellings = tokens.spellings;
|
|
1566
|
+
if (spellings !== null && segment.spellings !== null) {
|
|
1567
|
+
for (let at = 0; at < spellings.size; at++)
|
|
1568
|
+
spellings.added[at] = spellings.counts[at] >= segment.spellings.count[spellings.ordinals[at]] ? 1 : 0;
|
|
1569
|
+
}
|
|
1570
|
+
const runs = spellings === null || segment.spellings === null ? null : prepareSpellingRemoval(segment.spellings.runs, spellings);
|
|
1571
|
+
removalPlan.publish(segment, ordinal, tokens.terms);
|
|
1572
|
+
if (tokens.surfaces !== null && segment.surfaces !== null) removeSurfaces(tokens.surfaces, segment.surfaces, segment);
|
|
1573
|
+
if (spellings !== null && segment.spellings !== null && runs !== null) {
|
|
1574
|
+
runs();
|
|
1575
|
+
removeSurfaces(spellings, segment.spellings, segment);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
function retirable(segment, ordinal) {
|
|
1579
|
+
if (rowRemoved(segment, ordinal)) return false;
|
|
1580
|
+
requireSegment2(segment.totalDocuments[0] > 0 && segment.totalDocuments[0] <= segment.ordinalCapacity);
|
|
1581
|
+
for (let field = 0; field < segment.totalFieldLengths.length; field++) {
|
|
1582
|
+
const total = segment.totalFieldLengths[field];
|
|
1583
|
+
requireSegment2(Number.isFinite(total) && total >= segment.fieldLengths[field * segment.ordinalCapacity + ordinal]);
|
|
1584
|
+
}
|
|
1585
|
+
return true;
|
|
1586
|
+
}
|
|
1587
|
+
function retire(segment, ordinal) {
|
|
1588
|
+
for (let field = 0; field < segment.partialFields.length; field++) {
|
|
1589
|
+
const at = field * segment.ordinalCapacity + ordinal;
|
|
1590
|
+
segment.totalFieldLengths[field] = nonnegative(segment.totalFieldLengths[field] - segment.fieldLengths[at]);
|
|
1591
|
+
segment.fieldLengths[at] = 0;
|
|
1592
|
+
}
|
|
1593
|
+
segment.retired[ordinal >>> 5] |= 1 << (ordinal & 31);
|
|
1594
|
+
segment.totalDocuments[0]--;
|
|
1595
|
+
}
|
|
1596
|
+
function holdsText(fields) {
|
|
1597
|
+
for (const { text } of fields) if (text.count > 0) return true;
|
|
1598
|
+
return false;
|
|
1599
|
+
}
|
|
1600
|
+
function removeDocument(input, generation, ordinal, fields) {
|
|
1601
|
+
try {
|
|
1602
|
+
const segment = readSegmentView(input, generation);
|
|
1603
|
+
requireOrdinal(segment, ordinal);
|
|
1604
|
+
requireFields(segment, fields);
|
|
1605
|
+
if (!retirable(segment, ordinal)) return 0;
|
|
1606
|
+
if (!holdsText(fields)) {
|
|
1607
|
+
retire(segment, ordinal);
|
|
1608
|
+
return 0;
|
|
1609
|
+
}
|
|
1610
|
+
const texts = sharedTexts(fields);
|
|
1611
|
+
if (texts !== null && segment.spellings === null) removeSharedTexts(segment, ordinal, texts, fields);
|
|
1612
|
+
else {
|
|
1613
|
+
const tokens = acquireTokenPlan(segment, preparedSources(fields), true);
|
|
1614
|
+
try {
|
|
1615
|
+
removeTokens(segment, ordinal, tokens);
|
|
1616
|
+
} finally {
|
|
1617
|
+
releaseTokenPlan(tokens);
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
retire(segment, ordinal);
|
|
1621
|
+
return 0;
|
|
1622
|
+
} catch (error) {
|
|
1623
|
+
if (error instanceof SegmentRefusal2) return error.status;
|
|
1624
|
+
throw error;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
// src/engine/optional-writes-host.browser.ts
|
|
1629
|
+
var BATCH_CHUNK_SIZE;
|
|
1630
|
+
var ErrorCodes;
|
|
1631
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS;
|
|
1632
|
+
var NarsilError;
|
|
1633
|
+
var admitBufferedUpsert;
|
|
1634
|
+
var admitInsert;
|
|
1635
|
+
var arrayPositionsAreStale;
|
|
1636
|
+
var asBatchInsertError;
|
|
1637
|
+
var awaitWriteVisibility;
|
|
1638
|
+
var changeIndexOutsideExternalWaits;
|
|
1639
|
+
var collectRequiredFieldFailures;
|
|
1640
|
+
var createFrozenSegment;
|
|
1641
|
+
var createPendingAdmissions;
|
|
1642
|
+
var deleteNestedValue;
|
|
1643
|
+
var embedChunkDocuments;
|
|
1644
|
+
var emitSyncChange;
|
|
1645
|
+
var extractVectorFromDoc;
|
|
1646
|
+
var fnv1a;
|
|
1647
|
+
var generateId;
|
|
1648
|
+
var getLanguage;
|
|
1649
|
+
var getVectorFieldPaths;
|
|
1650
|
+
var indexChangeOutsideExternalWaits;
|
|
1651
|
+
var insertDocument;
|
|
1652
|
+
var insertDocumentVectors;
|
|
1653
|
+
var isSyncEpoch;
|
|
1654
|
+
var loadEmbedCode;
|
|
1655
|
+
var loadSchemaCodeFor;
|
|
1656
|
+
var loadStorageCode;
|
|
1657
|
+
var lookupOfStoredSynonyms;
|
|
1658
|
+
var newSyncEpoch;
|
|
1659
|
+
var oneAtATime;
|
|
1660
|
+
var prepareDocumentVectors;
|
|
1661
|
+
var prepareStoredSynonyms;
|
|
1662
|
+
var providedDocId;
|
|
1663
|
+
var readHeapStatistics;
|
|
1664
|
+
var recordPendingAdmission;
|
|
1665
|
+
var removeDocumentVectors;
|
|
1666
|
+
var requireDocumentObject;
|
|
1667
|
+
var requireDocumentWithinSizeLimit;
|
|
1668
|
+
var requireMatchingDocId;
|
|
1669
|
+
var requireSurfaceFormsForPartialFields;
|
|
1670
|
+
var storedBM25Params;
|
|
1671
|
+
var turnIsSpent;
|
|
1672
|
+
var turnsAreBudgeted;
|
|
1673
|
+
var updateDocumentVectors;
|
|
1674
|
+
var validateBM25Params;
|
|
1675
|
+
var validateBatchList;
|
|
1676
|
+
var validateDocId;
|
|
1677
|
+
var validateDocument;
|
|
1678
|
+
var validateDocumentStrict;
|
|
1679
|
+
var validateEmbeddingConfig;
|
|
1680
|
+
var validatePartitionConfig;
|
|
1681
|
+
var validatePatternValueLimit;
|
|
1682
|
+
var validateRequiredFields;
|
|
1683
|
+
var validateSchema;
|
|
1684
|
+
var validateTextStorage;
|
|
1685
|
+
var validateVectorDimensions;
|
|
1686
|
+
var validateVectorStorage;
|
|
1687
|
+
var withOpenIndex;
|
|
1688
|
+
var yieldTurn;
|
|
1689
|
+
var createSharedFrozenSegment;
|
|
1690
|
+
var reconstructSchemaFromMetadata;
|
|
1691
|
+
function bindOptionalWritesHost(host, segmentCodec) {
|
|
1692
|
+
bindFrozenKernelHost(host);
|
|
1693
|
+
host.bindFrozenRemovalKernel({
|
|
1694
|
+
growFrozenRemovalRegions,
|
|
1695
|
+
prepareFrozenRemoval,
|
|
1696
|
+
prepareFrozenStrings,
|
|
1697
|
+
removeFrozenDocument,
|
|
1698
|
+
settleFrozenState
|
|
1699
|
+
});
|
|
1700
|
+
bindSegmentRemovalHost(host);
|
|
1701
|
+
host.bindSegmentRemoval(removeDocument);
|
|
1702
|
+
({
|
|
1703
|
+
BATCH_CHUNK_SIZE,
|
|
1704
|
+
ErrorCodes,
|
|
1705
|
+
MIN_DOCUMENTS_FOR_SEGMENTS,
|
|
1706
|
+
NarsilError,
|
|
1707
|
+
admitBufferedUpsert,
|
|
1708
|
+
admitInsert,
|
|
1709
|
+
arrayPositionsAreStale,
|
|
1710
|
+
asBatchInsertError,
|
|
1711
|
+
awaitWriteVisibility,
|
|
1712
|
+
changeIndexOutsideExternalWaits,
|
|
1713
|
+
collectRequiredFieldFailures,
|
|
1714
|
+
createFrozenSegment,
|
|
1715
|
+
createPendingAdmissions,
|
|
1716
|
+
deleteNestedValue,
|
|
1717
|
+
embedChunkDocuments,
|
|
1718
|
+
emitSyncChange,
|
|
1719
|
+
extractVectorFromDoc,
|
|
1720
|
+
fnv1a,
|
|
1721
|
+
generateId,
|
|
1722
|
+
getLanguage,
|
|
1723
|
+
getVectorFieldPaths,
|
|
1724
|
+
indexChangeOutsideExternalWaits,
|
|
1725
|
+
insertDocument,
|
|
1726
|
+
insertDocumentVectors,
|
|
1727
|
+
isSyncEpoch,
|
|
1728
|
+
loadEmbedCode,
|
|
1729
|
+
loadSchemaCodeFor,
|
|
1730
|
+
loadStorageCode,
|
|
1731
|
+
lookupOfStoredSynonyms,
|
|
1732
|
+
newSyncEpoch,
|
|
1733
|
+
oneAtATime,
|
|
1734
|
+
prepareDocumentVectors,
|
|
1735
|
+
prepareStoredSynonyms,
|
|
1736
|
+
providedDocId,
|
|
1737
|
+
readHeapStatistics,
|
|
1738
|
+
recordPendingAdmission,
|
|
1739
|
+
removeDocumentVectors,
|
|
1740
|
+
requireDocumentObject,
|
|
1741
|
+
requireDocumentWithinSizeLimit,
|
|
1742
|
+
requireMatchingDocId,
|
|
1743
|
+
requireSurfaceFormsForPartialFields,
|
|
1744
|
+
storedBM25Params,
|
|
1745
|
+
turnIsSpent,
|
|
1746
|
+
turnsAreBudgeted,
|
|
1747
|
+
updateDocumentVectors,
|
|
1748
|
+
validateBM25Params,
|
|
1749
|
+
validateBatchList,
|
|
1750
|
+
validateDocId,
|
|
1751
|
+
validateDocument,
|
|
1752
|
+
validateDocumentStrict,
|
|
1753
|
+
validateEmbeddingConfig,
|
|
1754
|
+
validatePartitionConfig,
|
|
1755
|
+
validatePatternValueLimit,
|
|
1756
|
+
validateRequiredFields,
|
|
1757
|
+
validateSchema,
|
|
1758
|
+
validateTextStorage,
|
|
1759
|
+
validateVectorDimensions,
|
|
1760
|
+
validateVectorStorage,
|
|
1761
|
+
withOpenIndex,
|
|
1762
|
+
yieldTurn
|
|
1763
|
+
} = host);
|
|
1764
|
+
({ createSharedFrozenSegment, reconstructSchemaFromMetadata } = segmentCodec);
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
// src/runtime/process-memory.ts
|
|
1768
|
+
function readProcessMemory() {
|
|
1769
|
+
const proc = globalThis.process;
|
|
1770
|
+
if (!proc || typeof proc.memoryUsage !== "function") {
|
|
1771
|
+
return null;
|
|
1772
|
+
}
|
|
1773
|
+
try {
|
|
1774
|
+
const usage = proc.memoryUsage();
|
|
1775
|
+
const heap = readHeapStatistics();
|
|
1776
|
+
if (!Number.isFinite(usage.heapUsed) || !Number.isFinite(usage.heapTotal) || !Number.isFinite(usage.external) || !Number.isFinite(usage.rss)) {
|
|
1777
|
+
return null;
|
|
1778
|
+
}
|
|
1779
|
+
return {
|
|
1780
|
+
heapUsed: usage.heapUsed,
|
|
1781
|
+
heapTotal: usage.heapTotal,
|
|
1782
|
+
heapLimit: heap === null ? null : heap.configuredLimitBytes ?? heap.limitBytes,
|
|
1783
|
+
external: usage.external,
|
|
1784
|
+
rss: usage.rss
|
|
1785
|
+
};
|
|
1786
|
+
} catch {
|
|
1787
|
+
return null;
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
// src/engine/bm25-update.ts
|
|
1792
|
+
async function replaceNow(core, indexName, bm25) {
|
|
1793
|
+
const { config } = core.requireIndex(indexName);
|
|
1794
|
+
const manager = core.requireManager(indexName);
|
|
1795
|
+
const previous = config.bm25;
|
|
1796
|
+
await manager.publication.write(() => {
|
|
1797
|
+
config.bm25 = bm25;
|
|
1798
|
+
core.orchestrator.publishToWorkers({
|
|
1799
|
+
type: "replaceBm25",
|
|
1800
|
+
indexName,
|
|
1801
|
+
requestId: `replace-bm25-${indexName}`,
|
|
1802
|
+
bm25
|
|
1803
|
+
});
|
|
1804
|
+
});
|
|
1805
|
+
try {
|
|
1806
|
+
if (core.durability) await core.durability.manager.persistMetadata(indexName);
|
|
1807
|
+
} catch (err) {
|
|
1808
|
+
await manager.publication.write(() => {
|
|
1809
|
+
config.bm25 = previous;
|
|
1810
|
+
core.orchestrator.publishToWorkers({
|
|
1811
|
+
type: "replaceBm25",
|
|
1812
|
+
indexName,
|
|
1813
|
+
requestId: `restore-bm25-${indexName}`,
|
|
1814
|
+
bm25: previous
|
|
1815
|
+
});
|
|
1816
|
+
});
|
|
1817
|
+
throw err;
|
|
1818
|
+
}
|
|
1819
|
+
await core.orchestrator.awaitWrites(indexName);
|
|
1820
|
+
if (core.requireIndex(indexName).sync !== null) emitSyncChange(core.indexRegistry, { kind: "settings", indexName });
|
|
1821
|
+
}
|
|
1822
|
+
function updateIndexBm25(core, indexName, value) {
|
|
1823
|
+
return withOpenIndex(
|
|
1824
|
+
core,
|
|
1825
|
+
indexName,
|
|
1826
|
+
() => oneAtATime(core, indexName, () => {
|
|
1827
|
+
const entry = core.requireIndex(indexName);
|
|
1828
|
+
validateBM25Params(value, entry.config.schema, ErrorCodes.CONFIG_INVALID);
|
|
1829
|
+
return replaceNow(core, indexName, storedBM25Params(value));
|
|
1830
|
+
})
|
|
1831
|
+
);
|
|
1832
|
+
}
|
|
1833
|
+
function applyStoredIndexBm25(core, indexName, load) {
|
|
1834
|
+
return withOpenIndex(
|
|
1835
|
+
core,
|
|
1836
|
+
indexName,
|
|
1837
|
+
() => oneAtATime(core, indexName, async () => {
|
|
1838
|
+
const loaded = load();
|
|
1839
|
+
if (loaded === null) return false;
|
|
1840
|
+
await replaceNow(core, indexName, loaded.bm25);
|
|
1841
|
+
return true;
|
|
1842
|
+
})
|
|
1843
|
+
);
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
// src/engine/sync-state.ts
|
|
1847
|
+
function syncStateOfMetadata(metadata) {
|
|
1848
|
+
if (metadata.sync?.enabled !== true) return null;
|
|
1849
|
+
const epoch = isSyncEpoch(metadata.syncEpoch) ? metadata.syncEpoch : null;
|
|
1850
|
+
return {
|
|
1851
|
+
...metadata.sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: metadata.sync.historyMaxAgeMs } : {},
|
|
1852
|
+
...metadata.sync.historyMaxBytes !== void 0 ? { historyMaxBytes: metadata.sync.historyMaxBytes } : {},
|
|
1853
|
+
active: epoch !== null && metadata.syncActive === true,
|
|
1854
|
+
epoch: epoch ?? newSyncEpoch()
|
|
1855
|
+
};
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
// src/engine/index-from-metadata.ts
|
|
1859
|
+
function isRecoveryConfigError(err) {
|
|
1860
|
+
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
1861
|
+
}
|
|
1862
|
+
function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
1863
|
+
const name = metadata.embedding?.adapter ?? null;
|
|
1864
|
+
if (!metadata.embedding) return { adapter: null, name };
|
|
1865
|
+
const candidate = name !== null ? deps.embeddingAdapters.get(name) ?? null : deps.config?.embedding ?? null;
|
|
1866
|
+
if (!candidate) return { adapter: null, name };
|
|
1867
|
+
try {
|
|
1868
|
+
validateEmbeddingConfig({ fields: metadata.embedding.fields, adapter: candidate }, schema, void 0);
|
|
1869
|
+
} catch (err) {
|
|
1870
|
+
if (err instanceof NarsilError) {
|
|
1871
|
+
throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
1872
|
+
indexName: metadata.indexName,
|
|
1873
|
+
adapter: name ?? void 0
|
|
1874
|
+
});
|
|
1875
|
+
}
|
|
1876
|
+
throw err;
|
|
1877
|
+
}
|
|
1878
|
+
return { adapter: candidate, name };
|
|
1879
|
+
}
|
|
1880
|
+
function recoveryFailure(metadata, err) {
|
|
1881
|
+
if (!isRecoveryConfigError(err)) return err;
|
|
1882
|
+
return new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
1883
|
+
...err.details,
|
|
1884
|
+
indexName: metadata.indexName,
|
|
1885
|
+
tokenizer: metadata.tokenizer,
|
|
1886
|
+
stopWords: metadata.stopWords
|
|
1887
|
+
});
|
|
1888
|
+
}
|
|
1889
|
+
async function restorePartitionCount(deps, metadata) {
|
|
1890
|
+
if (!Number.isInteger(metadata.partitionCount) || metadata.partitionCount < 1 || metadata.partitionCount > 4294967295) {
|
|
1891
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "The saved index has an invalid partition count");
|
|
1892
|
+
}
|
|
1893
|
+
const manager = deps.executor.getManager(metadata.indexName);
|
|
1894
|
+
if (manager === void 0) return;
|
|
1895
|
+
await manager.publication.write(() => {
|
|
1896
|
+
while (manager.partitionCount < metadata.partitionCount) manager.addPartition();
|
|
1897
|
+
if (manager.partitionCount > metadata.partitionCount) {
|
|
1898
|
+
manager.setPartitions(manager.getAllPartitions().slice(0, metadata.partitionCount));
|
|
1899
|
+
}
|
|
1900
|
+
});
|
|
1901
|
+
}
|
|
1902
|
+
async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
1903
|
+
const loadingSchemaCode = loadData ? loadSchemaCodeFor(metadata.schema) : null;
|
|
1904
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
1905
|
+
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
1906
|
+
if (existing !== void 0) {
|
|
1907
|
+
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
1908
|
+
const { adoptSavedAnalysis } = await loadStorageCode();
|
|
1909
|
+
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
1910
|
+
}
|
|
1911
|
+
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
1912
|
+
deps.executor.createIndex(
|
|
1913
|
+
metadata.indexName,
|
|
1914
|
+
existing.config,
|
|
1915
|
+
existing.language,
|
|
1916
|
+
false,
|
|
1917
|
+
lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language)
|
|
1918
|
+
);
|
|
1919
|
+
await restorePartitionCount(deps, metadata);
|
|
1920
|
+
}
|
|
1921
|
+
return;
|
|
1922
|
+
}
|
|
1923
|
+
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
1924
|
+
const language = getLanguage(indexConfig.language ?? "english");
|
|
1925
|
+
try {
|
|
1926
|
+
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
1927
|
+
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
1928
|
+
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
1929
|
+
} catch (err) {
|
|
1930
|
+
throw recoveryFailure(metadata, err);
|
|
1931
|
+
}
|
|
1932
|
+
const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
|
|
1933
|
+
const prepared = prepareStoredSynonyms(
|
|
1934
|
+
metadata.synonyms,
|
|
1935
|
+
indexConfig,
|
|
1936
|
+
language,
|
|
1937
|
+
"The synonym set in the index metadata is invalid: "
|
|
1938
|
+
);
|
|
1939
|
+
const synonyms = prepared?.encoded ?? null;
|
|
1940
|
+
try {
|
|
1941
|
+
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
1942
|
+
validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
|
|
1943
|
+
if (loadData) {
|
|
1944
|
+
deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
|
|
1945
|
+
await restorePartitionCount(deps, metadata);
|
|
1946
|
+
}
|
|
1947
|
+
} catch (err) {
|
|
1948
|
+
throw recoveryFailure(metadata, err);
|
|
1949
|
+
}
|
|
1950
|
+
deps.indexRegistry.set(metadata.indexName, {
|
|
1951
|
+
config: indexConfig,
|
|
1952
|
+
language,
|
|
1953
|
+
embeddingAdapter: embedding.adapter,
|
|
1954
|
+
embeddingAdapterName: embedding.name,
|
|
1955
|
+
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema),
|
|
1956
|
+
indexUuid: metadata.indexUuid ?? null,
|
|
1957
|
+
heldPartitions: metadata.heldPartitions ?? null,
|
|
1958
|
+
documentCount: metadata.documentCount ?? 0,
|
|
1959
|
+
partitionCount: metadata.partitionCount,
|
|
1960
|
+
analysisRevision: metadata.analysisRevision ?? null,
|
|
1961
|
+
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
1962
|
+
synonyms,
|
|
1963
|
+
analysisGeneration: metadata.analysisGeneration ?? 0,
|
|
1964
|
+
pendingAnalysis: metadata.pendingAnalysis ?? null,
|
|
1965
|
+
durableAnalysis: null,
|
|
1966
|
+
sync: metadata.sync?.enabled === true ? syncStateOfMetadata(metadata) : null,
|
|
1967
|
+
querySuggestions: metadata.querySuggestions ?? null
|
|
1968
|
+
});
|
|
1969
|
+
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
1970
|
+
else deps.indexState().registerClosed(metadata.indexName);
|
|
1971
|
+
if (metadata.analysisRevision !== language.revision || arrayPositionsAreStale(indexConfig, metadata.arrayPositionGap ?? null)) {
|
|
1972
|
+
deps.analysisRebuild().markStale({
|
|
1973
|
+
indexName: metadata.indexName,
|
|
1974
|
+
language: language.name,
|
|
1975
|
+
storedRevision: metadata.analysisRevision ?? null,
|
|
1976
|
+
currentRevision: language.revision
|
|
1977
|
+
});
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
// src/engine/mutations/durable-rollback.ts
|
|
1982
|
+
function rollbackFailed(docId, originalError, rollbackError) {
|
|
1983
|
+
throw new NarsilError(
|
|
1984
|
+
ErrorCodes.REPLICATION_ROLLBACK_FAILED,
|
|
1985
|
+
`Durable write failed for document "${docId}" and the in-memory rollback also failed; the partition may expose an unacknowledged mutation`,
|
|
1986
|
+
{
|
|
1987
|
+
docId,
|
|
1988
|
+
originalError: originalError instanceof Error ? originalError.message : String(originalError),
|
|
1989
|
+
rollbackError: rollbackError instanceof Error ? rollbackError.message : String(rollbackError)
|
|
1990
|
+
}
|
|
1991
|
+
);
|
|
1992
|
+
}
|
|
1993
|
+
async function rollbackInsertedDocument(ctx, indexName, docId, inserted, originalError, preparedText, appliedRef, removedSpellings) {
|
|
1994
|
+
if (!inserted) {
|
|
1995
|
+
return;
|
|
1996
|
+
}
|
|
1997
|
+
try {
|
|
1998
|
+
const manager = ctx.requireManager(indexName);
|
|
1999
|
+
await manager.publication.write(
|
|
2000
|
+
() => changeIndexOutsideExternalWaits(async () => {
|
|
2001
|
+
if (appliedRef !== void 0 && manager.getRef(docId) !== appliedRef) return;
|
|
2002
|
+
manager.remove(docId, removedSpellings, { preparedText: { before: preparedText?.after } });
|
|
2003
|
+
removeDocumentVectors(docId, manager.getVectorIndexes());
|
|
2004
|
+
ctx.orchestrator.publishToWorkers({
|
|
2005
|
+
type: "remove",
|
|
2006
|
+
indexName,
|
|
2007
|
+
docId,
|
|
2008
|
+
removedSpellings,
|
|
2009
|
+
requestId: `rollback-${docId}`
|
|
2010
|
+
});
|
|
2011
|
+
})
|
|
2012
|
+
);
|
|
2013
|
+
} catch (rollbackError) {
|
|
2014
|
+
rollbackFailed(docId, originalError, rollbackError);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, originalError, preparedText) {
|
|
2018
|
+
if (restoreDoc === void 0) {
|
|
2019
|
+
return;
|
|
2020
|
+
}
|
|
2021
|
+
const manager = ctx.requireManager(indexName);
|
|
2022
|
+
try {
|
|
2023
|
+
await manager.publication.write(
|
|
2024
|
+
() => changeIndexOutsideExternalWaits(async () => {
|
|
2025
|
+
if (manager.has(docId)) return;
|
|
2026
|
+
const previous = prepareDocumentVectors(restoreDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
2027
|
+
manager.insert(docId, previous.partitionDoc, {
|
|
2028
|
+
preparedText: { after: preparedText?.before }
|
|
2029
|
+
});
|
|
2030
|
+
insertDocumentVectors(
|
|
2031
|
+
docId,
|
|
2032
|
+
previous.extractedVectors,
|
|
2033
|
+
manager.getVectorIndexes(),
|
|
2034
|
+
manager.partitionIdOf(docId)
|
|
2035
|
+
);
|
|
2036
|
+
ctx.orchestrator.publishToWorkers({
|
|
2037
|
+
type: "insert",
|
|
2038
|
+
indexName,
|
|
2039
|
+
docId,
|
|
2040
|
+
document: previous.partitionDoc,
|
|
2041
|
+
requestId: `rollback-${docId}`
|
|
2042
|
+
});
|
|
2043
|
+
})
|
|
2044
|
+
);
|
|
2045
|
+
} catch (rollbackError) {
|
|
2046
|
+
rollbackFailed(docId, originalError, rollbackError);
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, originalError, preparedText, appliedRef, rollbackRef) {
|
|
2050
|
+
if (restoreDoc === void 0 || appliedRef === void 0) {
|
|
2051
|
+
return;
|
|
2052
|
+
}
|
|
2053
|
+
try {
|
|
2054
|
+
const manager = ctx.requireManager(indexName);
|
|
2055
|
+
await manager.publication.write(
|
|
2056
|
+
() => changeIndexOutsideExternalWaits(async () => {
|
|
2057
|
+
if (manager.getRef(docId) !== appliedRef) return;
|
|
2058
|
+
const fields = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
2059
|
+
const previous = prepareDocumentVectors(restoreDoc, fields);
|
|
2060
|
+
manager.update(docId, rollbackRef ?? previous.partitionDoc, {
|
|
2061
|
+
skipClone: rollbackRef !== void 0,
|
|
2062
|
+
preparedText: { before: preparedText?.after, after: preparedText?.before }
|
|
2063
|
+
});
|
|
2064
|
+
const vectors = new Map([...fields].map((field) => [field, previous.extractedVectors.get(field) ?? null]));
|
|
2065
|
+
updateDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
2066
|
+
ctx.orchestrator.publishToWorkers({
|
|
2067
|
+
type: "update",
|
|
2068
|
+
indexName,
|
|
2069
|
+
docId,
|
|
2070
|
+
document: previous.partitionDoc,
|
|
2071
|
+
requestId: `rollback-${docId}`
|
|
2072
|
+
});
|
|
2073
|
+
})
|
|
2074
|
+
);
|
|
2075
|
+
} catch (rollbackError) {
|
|
2076
|
+
rollbackFailed(docId, originalError, rollbackError);
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
// src/engine/mutations/record-batch.ts
|
|
2081
|
+
async function recordChunk(ctx, indexName, mutations, applies) {
|
|
2082
|
+
if (mutations.length === 0) return [];
|
|
2083
|
+
if (ctx.durability) {
|
|
2084
|
+
const outcomes = await ctx.durability.recordMutationBatch(
|
|
2085
|
+
indexName,
|
|
2086
|
+
mutations.map((mutation, i) => ({ docId: mutation.docId, document: mutation.document, apply: applies[i] }))
|
|
2087
|
+
);
|
|
2088
|
+
return outcomes.map((outcome) => outcome.ok ? null : { error: outcome.error });
|
|
2089
|
+
}
|
|
2090
|
+
const failures = [];
|
|
2091
|
+
for (const apply of applies) {
|
|
2092
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2093
|
+
try {
|
|
2094
|
+
await apply();
|
|
2095
|
+
failures.push(null);
|
|
2096
|
+
} catch (error) {
|
|
2097
|
+
failures.push({ error });
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
return failures;
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
// src/engine/mutations/insert-batch-apply.ts
|
|
2104
|
+
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
2105
|
+
const manager = ctx.requireManager(indexName);
|
|
2106
|
+
const insertIntoIndex = () => {
|
|
2107
|
+
if (ctx.abortController.signal.aborted) {
|
|
2108
|
+
progress.skipped = true;
|
|
2109
|
+
throw ctx.abortController.signal.reason;
|
|
2110
|
+
}
|
|
2111
|
+
admitInsert(ctx, indexName, manager, doc.docId);
|
|
2112
|
+
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
2113
|
+
action: "insert",
|
|
2114
|
+
docId: doc.docId,
|
|
2115
|
+
document: doc.document,
|
|
2116
|
+
indexName
|
|
2117
|
+
});
|
|
2118
|
+
if (queued !== void 0) {
|
|
2119
|
+
progress.buffered = true;
|
|
2120
|
+
return queued;
|
|
2121
|
+
}
|
|
2122
|
+
const stored = manager.insert(doc.docId, doc.partitionDoc, {
|
|
2123
|
+
skipClone: doc.extractedVectors.size > 0 ? true : options?.skipClone,
|
|
2124
|
+
onPreparedText: doc.extractedVectors.size > 0 || ctx.durability !== null ? (change) => {
|
|
2125
|
+
progress.preparedText = change;
|
|
2126
|
+
} : void 0
|
|
2127
|
+
});
|
|
2128
|
+
progress.inserted = true;
|
|
2129
|
+
try {
|
|
2130
|
+
insertDocumentVectors(
|
|
2131
|
+
doc.docId,
|
|
2132
|
+
doc.extractedVectors,
|
|
2133
|
+
manager.getVectorIndexes(),
|
|
2134
|
+
manager.partitionIdOf(doc.docId)
|
|
2135
|
+
);
|
|
2136
|
+
} catch (vecErr) {
|
|
2137
|
+
try {
|
|
2138
|
+
manager.remove(doc.docId, void 0, { preparedText: { before: progress.preparedText?.after } });
|
|
2139
|
+
progress.inserted = false;
|
|
2140
|
+
} catch (rollbackErr) {
|
|
2141
|
+
console.warn(
|
|
2142
|
+
`Rollback failed for doc "${doc.docId}" during batch insert atomicity:`,
|
|
2143
|
+
rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
|
|
2144
|
+
);
|
|
2145
|
+
}
|
|
2146
|
+
throw vecErr;
|
|
2147
|
+
}
|
|
2148
|
+
if (ctx.durability === null) progress.preparedText = void 0;
|
|
2149
|
+
else progress.appliedRef = stored;
|
|
2150
|
+
ctx.orchestrator.publishToWorkers({
|
|
2151
|
+
type: "insert",
|
|
2152
|
+
indexName,
|
|
2153
|
+
docId: doc.docId,
|
|
2154
|
+
document: doc.partitionDoc,
|
|
2155
|
+
requestId: `replicate-insert-${doc.docId}`,
|
|
2156
|
+
skipClone: options?.skipClone
|
|
2157
|
+
});
|
|
2158
|
+
};
|
|
2159
|
+
return () => manager.publication.write(indexChangeOutsideExternalWaits(insertIntoIndex));
|
|
2160
|
+
}
|
|
2161
|
+
async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
2162
|
+
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
2163
|
+
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
2164
|
+
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2165
|
+
await ctx.orchestrator.settlePublications(indexName);
|
|
2166
|
+
const applications = [];
|
|
2167
|
+
for (let i = 0; i < docs.length; i++) {
|
|
2168
|
+
if (progress[i].skipped) {
|
|
2169
|
+
applications.push({ status: "skipped" });
|
|
2170
|
+
continue;
|
|
2171
|
+
}
|
|
2172
|
+
const failure = failures[i];
|
|
2173
|
+
if (failure !== null) {
|
|
2174
|
+
let error = failure.error;
|
|
2175
|
+
if (ctx.durability) {
|
|
2176
|
+
try {
|
|
2177
|
+
await rollbackInsertedDocument(
|
|
2178
|
+
ctx,
|
|
2179
|
+
indexName,
|
|
2180
|
+
docs[i].docId,
|
|
2181
|
+
progress[i].inserted,
|
|
2182
|
+
error,
|
|
2183
|
+
progress[i].preparedText,
|
|
2184
|
+
progress[i].appliedRef
|
|
2185
|
+
);
|
|
2186
|
+
} catch (rollbackError) {
|
|
2187
|
+
error = rollbackError;
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
applications.push({ status: "failed", error });
|
|
2191
|
+
continue;
|
|
2192
|
+
}
|
|
2193
|
+
applications.push({ status: progress[i].buffered ? "buffered" : "inserted" });
|
|
2194
|
+
}
|
|
2195
|
+
return applications;
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
2199
|
+
async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
2200
|
+
const entry = ctx.requireIndex(indexName);
|
|
2201
|
+
const manager = ctx.requireManager(indexName);
|
|
2202
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2203
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2204
|
+
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
2205
|
+
const strict = entry.config.strict === true;
|
|
2206
|
+
const required = entry.config.required;
|
|
2207
|
+
const admitted = [];
|
|
2208
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
2209
|
+
const pending = createPendingAdmissions();
|
|
2210
|
+
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2211
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2212
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2213
|
+
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
2214
|
+
if (required && required.length > 0) {
|
|
2215
|
+
collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
2216
|
+
}
|
|
2217
|
+
await embedChunkDocuments(
|
|
2218
|
+
entry,
|
|
2219
|
+
documents,
|
|
2220
|
+
chunkStart,
|
|
2221
|
+
chunkEnd,
|
|
2222
|
+
ctx.abortController.signal,
|
|
2223
|
+
chunkFailedIndexes,
|
|
2224
|
+
failed
|
|
2225
|
+
);
|
|
2226
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2227
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2228
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
2229
|
+
const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
2230
|
+
try {
|
|
2231
|
+
const sizing = requireDocumentObject(documents[i]);
|
|
2232
|
+
if (sizing !== null) await sizing;
|
|
2233
|
+
validateDocId(docId);
|
|
2234
|
+
if (hasBeforeHook) {
|
|
2235
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
2236
|
+
}
|
|
2237
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2238
|
+
documents[i],
|
|
2239
|
+
vectorFieldPaths
|
|
2240
|
+
);
|
|
2241
|
+
if (extractedVectors.size > 0) {
|
|
2242
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2243
|
+
}
|
|
2244
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
2245
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
2246
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
2247
|
+
}
|
|
2248
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
2249
|
+
if (strict) {
|
|
2250
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
2251
|
+
}
|
|
2252
|
+
admittedIds.add(docId);
|
|
2253
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2254
|
+
admitted.push({ docId, document: documents[i], partitionDoc, extractedVectors });
|
|
2255
|
+
} catch (err) {
|
|
2256
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2257
|
+
}
|
|
2258
|
+
}
|
|
2259
|
+
if (chunkEnd < documents.length) {
|
|
2260
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
return admitted;
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
2267
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed) {
|
|
2268
|
+
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2269
|
+
const succeeded = [];
|
|
2270
|
+
const buffered = /* @__PURE__ */ new Set();
|
|
2271
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2272
|
+
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2273
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2274
|
+
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2275
|
+
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
2276
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
2277
|
+
const doc = chunk[i];
|
|
2278
|
+
const application = applications[i];
|
|
2279
|
+
if (application.status === "skipped") continue;
|
|
2280
|
+
if (application.status === "failed") {
|
|
2281
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2282
|
+
continue;
|
|
2283
|
+
}
|
|
2284
|
+
if (application.status === "buffered") {
|
|
2285
|
+
buffered.add(doc.docId);
|
|
2286
|
+
succeeded.push(doc.docId);
|
|
2287
|
+
continue;
|
|
2288
|
+
}
|
|
2289
|
+
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2290
|
+
touchedVectorFields.add(fieldPath);
|
|
2291
|
+
}
|
|
2292
|
+
if (hasAfterHook) {
|
|
2293
|
+
try {
|
|
2294
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2295
|
+
} catch (err) {
|
|
2296
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
succeeded.push(doc.docId);
|
|
2300
|
+
}
|
|
2301
|
+
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
2302
|
+
if (applied < admitted.length) {
|
|
2303
|
+
ctx.checkHeapPressure(indexName);
|
|
2304
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
2307
|
+
return { succeeded, buffered, touchedVectorFields };
|
|
2308
|
+
}
|
|
2309
|
+
|
|
2310
|
+
// src/engine/mutations/segment-publication.ts
|
|
2311
|
+
function applySegment(ctx, indexName, attachment, doc, options) {
|
|
2312
|
+
const manager = ctx.requireManager(indexName);
|
|
2313
|
+
const publish = async () => {
|
|
2314
|
+
if (attachment.failure !== void 0) throw attachment.failure;
|
|
2315
|
+
if (attachment.attached) return;
|
|
2316
|
+
if (ctx.abortController.signal.aborted) throw ctx.abortController.signal.reason;
|
|
2317
|
+
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
2318
|
+
attachment.individual = true;
|
|
2319
|
+
}
|
|
2320
|
+
if (attachment.individual) return;
|
|
2321
|
+
const inserted = [];
|
|
2322
|
+
const vectors = manager.getVectorIndexes();
|
|
2323
|
+
try {
|
|
2324
|
+
const pending = createPendingAdmissions();
|
|
2325
|
+
for (const doc2 of attachment.members) {
|
|
2326
|
+
if (manager.has(doc2.docId)) {
|
|
2327
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${doc2.docId}" already exists`);
|
|
2328
|
+
}
|
|
2329
|
+
admitInsert(ctx, indexName, manager, doc2.docId, pending);
|
|
2330
|
+
recordPendingAdmission(ctx, pending, manager, doc2.docId);
|
|
2331
|
+
}
|
|
2332
|
+
if (ctx.durability !== null) {
|
|
2333
|
+
attachment.segment.removals.bind(manager.textAnalysis);
|
|
2334
|
+
for (const doc2 of attachment.members) {
|
|
2335
|
+
attachment.rollback.set(doc2.docId, {
|
|
2336
|
+
inserted: true,
|
|
2337
|
+
buffered: false,
|
|
2338
|
+
skipped: false,
|
|
2339
|
+
removedSpellings: attachment.segment.removedSpellingsOf(doc2.docId)
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
for (const doc2 of attachment.members) {
|
|
2344
|
+
insertDocumentVectors(doc2.docId, doc2.extractedVectors, vectors, attachment.partitionId);
|
|
2345
|
+
inserted.push(doc2.docId);
|
|
2346
|
+
}
|
|
2347
|
+
manager.attachFrozenSegment(attachment.partitionId, attachment.segment);
|
|
2348
|
+
attachment.attached = true;
|
|
2349
|
+
for (const doc2 of attachment.members) attachment.applied.add(doc2.docId);
|
|
2350
|
+
ctx.orchestrator.publishToWorkers(attachment.action);
|
|
2351
|
+
} catch (error) {
|
|
2352
|
+
for (const docId of inserted) removeDocumentVectors(docId, vectors);
|
|
2353
|
+
attachment.failure = error;
|
|
2354
|
+
throw error;
|
|
2355
|
+
}
|
|
2356
|
+
};
|
|
2357
|
+
return async () => {
|
|
2358
|
+
await manager.publication.write(indexChangeOutsideExternalWaits(publish));
|
|
2359
|
+
if (!attachment.individual) return;
|
|
2360
|
+
const progress = { inserted: false, buffered: false, skipped: false };
|
|
2361
|
+
attachment.rollback.set(doc.docId, progress);
|
|
2362
|
+
try {
|
|
2363
|
+
return await applyInsertDocument(ctx, indexName, doc, options, progress)() ?? void 0;
|
|
2364
|
+
} finally {
|
|
2365
|
+
if (progress.inserted) attachment.applied.add(doc.docId);
|
|
2366
|
+
if (progress.buffered) attachment.buffered.add(doc.docId);
|
|
2367
|
+
}
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2370
|
+
|
|
2371
|
+
// src/engine/mutations/segment-replication.ts
|
|
2372
|
+
function groupByPartition(docIds, partitionCount) {
|
|
2373
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2374
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
2375
|
+
const partitionId = partitionCount <= 1 ? 0 : fnv1a(docIds[i]) % partitionCount;
|
|
2376
|
+
let group = groups.get(partitionId);
|
|
2377
|
+
if (group === void 0) {
|
|
2378
|
+
group = [];
|
|
2379
|
+
groups.set(partitionId, group);
|
|
2380
|
+
}
|
|
2381
|
+
group.push(i);
|
|
2382
|
+
}
|
|
2383
|
+
return groups;
|
|
2384
|
+
}
|
|
2385
|
+
function buildSegmentRequests(indexName, docIds, documents, partitionCount, skipClone) {
|
|
2386
|
+
const groups = groupByPartition(docIds, partitionCount);
|
|
2387
|
+
const requests = [];
|
|
2388
|
+
const memberIndexes = [];
|
|
2389
|
+
for (const [partitionId, indexes] of groups) {
|
|
2390
|
+
requests.push({
|
|
2391
|
+
partitionId,
|
|
2392
|
+
action: {
|
|
2393
|
+
type: "buildSegment",
|
|
2394
|
+
indexName,
|
|
2395
|
+
segmentId: generateId(),
|
|
2396
|
+
documents: indexes.map((i) => ({ docId: docIds[i], document: documents[i] })),
|
|
2397
|
+
options: skipClone === true ? { skipClone: true } : void 0,
|
|
2398
|
+
requestId: `build-segment-${indexName}-${partitionId}`
|
|
2399
|
+
},
|
|
2400
|
+
documents: indexes.map((i) => documents[i])
|
|
2401
|
+
});
|
|
2402
|
+
memberIndexes.push(indexes);
|
|
2403
|
+
}
|
|
2404
|
+
return { requests, memberIndexes };
|
|
2405
|
+
}
|
|
2406
|
+
|
|
2407
|
+
// src/engine/mutations/insert-batch-segments.ts
|
|
2408
|
+
function mainStoreDocument(doc, options) {
|
|
2409
|
+
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
2410
|
+
return structuredClone(doc.partitionDoc);
|
|
2411
|
+
}
|
|
2412
|
+
async function applyIndividually(ctx, indexName, admitted, options, failed) {
|
|
2413
|
+
const applied = await applyAdmittedDocuments(ctx, indexName, admitted, options, failed);
|
|
2414
|
+
return { succeeded: applied.succeeded, touchedVectorFields: applied.touchedVectorFields };
|
|
2415
|
+
}
|
|
2416
|
+
function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted) {
|
|
2417
|
+
const chunks = [];
|
|
2418
|
+
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
2419
|
+
for (let i = 0; i < built.length; i++) {
|
|
2420
|
+
const builtSegment = built[i];
|
|
2421
|
+
const requestId = `attach-${builtSegment.segmentId}`;
|
|
2422
|
+
const action = builtSegment.snapshot !== null ? {
|
|
2423
|
+
type: "attachSegments",
|
|
2424
|
+
indexName,
|
|
2425
|
+
requestId,
|
|
2426
|
+
segments: [{ partitionId: builtSegment.partitionId, snapshot: builtSegment.snapshot }]
|
|
2427
|
+
} : {
|
|
2428
|
+
type: "mergeSegments",
|
|
2429
|
+
indexName,
|
|
2430
|
+
requestId,
|
|
2431
|
+
segments: builtSegment.payload === null ? [] : [
|
|
2432
|
+
{
|
|
2433
|
+
partitionId: builtSegment.partitionId,
|
|
2434
|
+
payload: builtSegment.payload,
|
|
2435
|
+
documents: builtSegment.documents
|
|
2436
|
+
}
|
|
2437
|
+
]
|
|
2438
|
+
};
|
|
2439
|
+
const attachment = {
|
|
2440
|
+
partitionId: built[i].partitionId,
|
|
2441
|
+
partitionCount,
|
|
2442
|
+
segment: frozen[i],
|
|
2443
|
+
members: memberIndexes[i].map((member) => admitted[member]),
|
|
2444
|
+
applied: /* @__PURE__ */ new Set(),
|
|
2445
|
+
rollback: /* @__PURE__ */ new Map(),
|
|
2446
|
+
buffered: /* @__PURE__ */ new Set(),
|
|
2447
|
+
individual: false,
|
|
2448
|
+
attached: false,
|
|
2449
|
+
failure: void 0,
|
|
2450
|
+
action
|
|
2451
|
+
};
|
|
2452
|
+
for (const member of memberIndexes[i]) {
|
|
2453
|
+
const doc = admitted[member];
|
|
2454
|
+
chunk.docs.push(doc);
|
|
2455
|
+
chunk.attachmentOf.set(doc.docId, attachment);
|
|
2456
|
+
}
|
|
2457
|
+
if (chunk.docs.length >= BATCH_CHUNK_SIZE && i + 1 < built.length) {
|
|
2458
|
+
chunks.push(chunk);
|
|
2459
|
+
chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
2463
|
+
return chunks;
|
|
2464
|
+
}
|
|
2465
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed) {
|
|
2466
|
+
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2467
|
+
const succeededIds = /* @__PURE__ */ new Set();
|
|
2468
|
+
const errorOf = /* @__PURE__ */ new Map();
|
|
2469
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2470
|
+
const chunks = chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted);
|
|
2471
|
+
for (let index = 0; index < chunks.length; index++) {
|
|
2472
|
+
const { docs, attachmentOf } = chunks[index];
|
|
2473
|
+
const applies = docs.map((doc) => {
|
|
2474
|
+
const attachment = attachmentOf.get(doc.docId);
|
|
2475
|
+
if (attachment === void 0) return async () => void 0;
|
|
2476
|
+
return applySegment(ctx, indexName, attachment, doc, options);
|
|
2477
|
+
});
|
|
2478
|
+
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2479
|
+
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
2480
|
+
for (let i = 0; i < docs.length; i++) {
|
|
2481
|
+
const doc = docs[i];
|
|
2482
|
+
const failure = failures[i];
|
|
2483
|
+
if (failure !== null) {
|
|
2484
|
+
let error = failure.error;
|
|
2485
|
+
const attachment = attachmentOf.get(doc.docId);
|
|
2486
|
+
const rollback = attachment?.rollback.get(doc.docId);
|
|
2487
|
+
try {
|
|
2488
|
+
await rollbackInsertedDocument(
|
|
2489
|
+
ctx,
|
|
2490
|
+
indexName,
|
|
2491
|
+
doc.docId,
|
|
2492
|
+
attachment?.applied.has(doc.docId) === true,
|
|
2493
|
+
error,
|
|
2494
|
+
rollback?.preparedText,
|
|
2495
|
+
rollback?.appliedRef,
|
|
2496
|
+
rollback?.removedSpellings
|
|
2497
|
+
);
|
|
2498
|
+
} catch (rollbackError) {
|
|
2499
|
+
error = rollbackError;
|
|
2500
|
+
}
|
|
2501
|
+
errorOf.set(doc.docId, error);
|
|
2502
|
+
continue;
|
|
2503
|
+
}
|
|
2504
|
+
if (attachmentOf.get(doc.docId)?.buffered.has(doc.docId) === true) {
|
|
2505
|
+
succeededIds.add(doc.docId);
|
|
2506
|
+
continue;
|
|
2507
|
+
}
|
|
2508
|
+
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2509
|
+
touchedVectorFields.add(fieldPath);
|
|
2510
|
+
}
|
|
2511
|
+
if (hasAfterHook) {
|
|
2512
|
+
try {
|
|
2513
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2514
|
+
} catch (err) {
|
|
2515
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
succeededIds.add(doc.docId);
|
|
2519
|
+
}
|
|
2520
|
+
if (index + 1 < chunks.length) {
|
|
2521
|
+
ctx.checkHeapPressure(indexName);
|
|
2522
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
const succeeded = [];
|
|
2526
|
+
for (const doc of admitted) {
|
|
2527
|
+
if (succeededIds.has(doc.docId)) {
|
|
2528
|
+
succeeded.push(doc.docId);
|
|
2529
|
+
continue;
|
|
2530
|
+
}
|
|
2531
|
+
if (!errorOf.has(doc.docId)) continue;
|
|
2532
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
2533
|
+
}
|
|
2534
|
+
return { succeeded, touchedVectorFields };
|
|
2535
|
+
}
|
|
2536
|
+
function frozenSegmentOf(segment, members, options) {
|
|
2537
|
+
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
2538
|
+
if (segment.payload === null) return null;
|
|
2539
|
+
return createFrozenSegment(
|
|
2540
|
+
segment.payload,
|
|
2541
|
+
members.map((doc) => mainStoreDocument(doc, options)),
|
|
2542
|
+
segment.segmentId
|
|
2543
|
+
);
|
|
2544
|
+
}
|
|
2545
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, failed) {
|
|
2546
|
+
if (admitted.length === 0) {
|
|
2547
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set() };
|
|
2548
|
+
}
|
|
2549
|
+
const manager = ctx.requireManager(indexName);
|
|
2550
|
+
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
2551
|
+
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
2552
|
+
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2553
|
+
}
|
|
2554
|
+
const docIds = admitted.map((doc) => doc.docId);
|
|
2555
|
+
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
2556
|
+
const partitionCount = manager.partitionCount;
|
|
2557
|
+
const { requests } = buildSegmentRequests(indexName, docIds, partitionDocuments, partitionCount, options?.skipClone);
|
|
2558
|
+
let built;
|
|
2559
|
+
try {
|
|
2560
|
+
built = await ctx.orchestrator.buildSegments(requests);
|
|
2561
|
+
} catch {
|
|
2562
|
+
built = null;
|
|
2563
|
+
}
|
|
2564
|
+
if (built === null) {
|
|
2565
|
+
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2566
|
+
}
|
|
2567
|
+
const positions = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
2568
|
+
const used = /* @__PURE__ */ new Set();
|
|
2569
|
+
const memberIndexes = [];
|
|
2570
|
+
for (const segment of built) {
|
|
2571
|
+
const members = [];
|
|
2572
|
+
for (const docId of segment.docIds) {
|
|
2573
|
+
const position = positions.get(docId);
|
|
2574
|
+
if (position === void 0 || used.has(position))
|
|
2575
|
+
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2576
|
+
used.add(position);
|
|
2577
|
+
members.push(position);
|
|
2578
|
+
}
|
|
2579
|
+
memberIndexes.push(members);
|
|
2580
|
+
}
|
|
2581
|
+
if (used.size !== admitted.length) return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2582
|
+
if (ctx.isRebalancing(indexName)) {
|
|
2583
|
+
return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2584
|
+
}
|
|
2585
|
+
const frozen = [];
|
|
2586
|
+
for (let i = 0; i < built.length; i++) {
|
|
2587
|
+
const segment = frozenSegmentOf(
|
|
2588
|
+
built[i],
|
|
2589
|
+
memberIndexes[i].map((m) => admitted[m]),
|
|
2590
|
+
options
|
|
2591
|
+
);
|
|
2592
|
+
if (segment === null) return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2593
|
+
frozen.push(segment);
|
|
2594
|
+
}
|
|
2595
|
+
for (const doc of admitted) {
|
|
2596
|
+
if (manager.has(doc.docId)) return applyIndividually(ctx, indexName, admitted, options, failed);
|
|
2597
|
+
}
|
|
2598
|
+
return recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, failed);
|
|
2599
|
+
}
|
|
2600
|
+
async function insertBatchViaSegments(ctx, indexName, documents, options) {
|
|
2601
|
+
const failed = [];
|
|
2602
|
+
const admitted = await admitBatchDocuments(ctx, indexName, documents, failed);
|
|
2603
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, failed);
|
|
2604
|
+
const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
|
|
2605
|
+
for (const fieldPath of outcome.touchedVectorFields) {
|
|
2606
|
+
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
2607
|
+
}
|
|
2608
|
+
ctx.checkWatermark(indexName, outcome.succeeded);
|
|
2609
|
+
ctx.checkHeapPressure(indexName);
|
|
2610
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2611
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2612
|
+
return { succeeded: outcome.succeeded, failed };
|
|
2613
|
+
}
|
|
2614
|
+
|
|
2615
|
+
// src/engine/mutations/insert-batch.ts
|
|
2616
|
+
async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
2617
|
+
ctx.guardShutdown();
|
|
2618
|
+
const entry = ctx.requireIndex(indexName);
|
|
2619
|
+
validateBatchList(documents, "documents");
|
|
2620
|
+
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
|
|
2621
|
+
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
2622
|
+
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
2623
|
+
return insertBatchViaSegments(ctx, indexName, documents, options);
|
|
2624
|
+
}
|
|
2625
|
+
const succeeded = [];
|
|
2626
|
+
const failed = [];
|
|
2627
|
+
const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
|
|
2628
|
+
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
2629
|
+
const required = entry.config.required;
|
|
2630
|
+
const batchManager = ctx.requireManager(indexName);
|
|
2631
|
+
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
2632
|
+
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2633
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2634
|
+
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2635
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2636
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2637
|
+
const chunkFailedIndexes = /* @__PURE__ */ new Set();
|
|
2638
|
+
if (required && required.length > 0) {
|
|
2639
|
+
collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed);
|
|
2640
|
+
}
|
|
2641
|
+
await embedChunkDocuments(
|
|
2642
|
+
entry,
|
|
2643
|
+
documents,
|
|
2644
|
+
chunkStart,
|
|
2645
|
+
chunkEnd,
|
|
2646
|
+
ctx.abortController.signal,
|
|
2647
|
+
chunkFailedIndexes,
|
|
2648
|
+
failed
|
|
2649
|
+
);
|
|
2650
|
+
async function applyPrepared(prepared2) {
|
|
2651
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared2, options);
|
|
2652
|
+
for (let i = 0; i < prepared2.length; i++) {
|
|
2653
|
+
const doc = prepared2[i];
|
|
2654
|
+
const application = applications[i];
|
|
2655
|
+
if (application.status === "skipped") continue;
|
|
2656
|
+
if (application.status === "failed") {
|
|
2657
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2658
|
+
continue;
|
|
2659
|
+
}
|
|
2660
|
+
if (application.status === "buffered") {
|
|
2661
|
+
succeeded.push(doc.docId);
|
|
2662
|
+
continue;
|
|
2663
|
+
}
|
|
2664
|
+
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2665
|
+
touchedVectorFields.add(fieldPath);
|
|
2666
|
+
}
|
|
2667
|
+
if (hasAfterHook) {
|
|
2668
|
+
try {
|
|
2669
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2670
|
+
} catch (err) {
|
|
2671
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2672
|
+
}
|
|
2673
|
+
}
|
|
2674
|
+
succeeded.push(doc.docId);
|
|
2675
|
+
}
|
|
2676
|
+
}
|
|
2677
|
+
const prepared = [];
|
|
2678
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2679
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2680
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2681
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
2682
|
+
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
2683
|
+
try {
|
|
2684
|
+
const sizing = requireDocumentObject(documents[i]);
|
|
2685
|
+
if (sizing !== null) await sizing;
|
|
2686
|
+
validateDocId(batchDocId);
|
|
2687
|
+
if (hasBeforeHook) {
|
|
2688
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
|
|
2689
|
+
}
|
|
2690
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2691
|
+
documents[i],
|
|
2692
|
+
batchVectorFieldPaths
|
|
2693
|
+
);
|
|
2694
|
+
if (extractedVectors.size > 0) {
|
|
2695
|
+
validateVectorDimensions(extractedVectors, batchVecIndexes);
|
|
2696
|
+
}
|
|
2697
|
+
admitInsert(ctx, indexName, batchManager, batchDocId);
|
|
2698
|
+
prepared.push({
|
|
2699
|
+
docId: batchDocId,
|
|
2700
|
+
document: documents[i],
|
|
2701
|
+
partitionDoc,
|
|
2702
|
+
extractedVectors
|
|
2703
|
+
});
|
|
2704
|
+
} catch (err) {
|
|
2705
|
+
failed.push({ docId: batchDocId, error: asBatchInsertError(err) });
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
if (prepared.length > 0) await applyPrepared(prepared);
|
|
2709
|
+
if (chunkEnd < documents.length) {
|
|
2710
|
+
ctx.checkHeapPressure(indexName);
|
|
2711
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
for (const fieldPath of touchedVectorFields) {
|
|
2715
|
+
const vecIndex = batchVecIndexes.get(fieldPath);
|
|
2716
|
+
if (vecIndex) {
|
|
2717
|
+
vecIndex.scheduleBuild();
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
ctx.checkWatermark(indexName, succeeded);
|
|
2721
|
+
ctx.checkHeapPressure(indexName);
|
|
2722
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2723
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2724
|
+
return { succeeded, failed };
|
|
2725
|
+
}
|
|
2726
|
+
|
|
2727
|
+
// src/engine/mutations/remove.ts
|
|
2728
|
+
function asRemoveError(err) {
|
|
2729
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
2730
|
+
}
|
|
2731
|
+
async function prepareRemoval(ctx, indexName, docId) {
|
|
2732
|
+
ctx.guardShutdown();
|
|
2733
|
+
validateDocId(docId);
|
|
2734
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
2735
|
+
const rebalancing = ctx.isRebalancing(indexName);
|
|
2736
|
+
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
2737
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
2738
|
+
}
|
|
2739
|
+
return {
|
|
2740
|
+
docId,
|
|
2741
|
+
document: null,
|
|
2742
|
+
restoreDoc: void 0,
|
|
2743
|
+
removedSpellings: void 0,
|
|
2744
|
+
buffered: false
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
function applyOfRemoval(ctx, indexName, prepared) {
|
|
2748
|
+
const { docId } = prepared;
|
|
2749
|
+
const manager = ctx.requireManager(indexName);
|
|
2750
|
+
const removeFromIndex = () => {
|
|
2751
|
+
if (ctx.isRebalancing(indexName)) {
|
|
2752
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
2753
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : ctx.requireManager(indexName).has(docId);
|
|
2754
|
+
if (!exists) {
|
|
2755
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2758
|
+
const queued = ctx.bufferIfRebalancing(indexName, { action: "remove", docId, indexName });
|
|
2759
|
+
if (queued !== void 0) {
|
|
2760
|
+
prepared.buffered = true;
|
|
2761
|
+
return queued;
|
|
2762
|
+
}
|
|
2763
|
+
prepared.restoreDoc = ctx.durability ? manager.get(docId) : void 0;
|
|
2764
|
+
manager.remove(docId, void 0, {
|
|
2765
|
+
onRemovedSpellings: (spellings) => {
|
|
2766
|
+
prepared.removedSpellings = spellings;
|
|
2767
|
+
},
|
|
2768
|
+
onPreparedText: ctx.durability === null ? void 0 : (change) => {
|
|
2769
|
+
prepared.preparedText = change;
|
|
2770
|
+
}
|
|
2771
|
+
});
|
|
2772
|
+
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
2773
|
+
ctx.orchestrator.publishToWorkers({
|
|
2774
|
+
type: "remove",
|
|
2775
|
+
indexName,
|
|
2776
|
+
docId,
|
|
2777
|
+
requestId: `replicate-remove-${docId}`,
|
|
2778
|
+
removedSpellings: prepared.removedSpellings
|
|
2779
|
+
});
|
|
2780
|
+
};
|
|
2781
|
+
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
2782
|
+
}
|
|
2783
|
+
async function settleRemoval(ctx, indexName, prepared) {
|
|
2784
|
+
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
2785
|
+
if (settling !== void 0) await settling;
|
|
2786
|
+
const { docId } = prepared;
|
|
2787
|
+
try {
|
|
2788
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
2789
|
+
} catch (err) {
|
|
2790
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
async function removeDocument2(ctx, indexName, docId, options) {
|
|
2794
|
+
ctx.guardShutdown();
|
|
2795
|
+
ctx.requireIndex(indexName);
|
|
2796
|
+
const prepared = await prepareRemoval(ctx, indexName, docId);
|
|
2797
|
+
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
2798
|
+
if (ctx.durability) {
|
|
2799
|
+
try {
|
|
2800
|
+
await ctx.durability.recordRemove(indexName, docId, apply);
|
|
2801
|
+
} catch (err) {
|
|
2802
|
+
await rollbackRemovedDocument(ctx, indexName, docId, prepared.restoreDoc, err, prepared.preparedText);
|
|
2803
|
+
throw err;
|
|
2804
|
+
}
|
|
2805
|
+
} else {
|
|
2806
|
+
await apply();
|
|
2807
|
+
}
|
|
2808
|
+
if (!prepared.buffered) await settleRemoval(ctx, indexName, prepared);
|
|
2809
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2810
|
+
}
|
|
2811
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
2812
|
+
ctx.guardShutdown();
|
|
2813
|
+
ctx.requireIndex(indexName);
|
|
2814
|
+
validateBatchList(docIds, "document IDs");
|
|
2815
|
+
const succeeded = [];
|
|
2816
|
+
const failed = [];
|
|
2817
|
+
const manager = ctx.requireManager(indexName);
|
|
2818
|
+
await manager.publication.write(() => manager.beginBatchRemove());
|
|
2819
|
+
try {
|
|
2820
|
+
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2821
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
2822
|
+
const prepared = [];
|
|
2823
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
2824
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2825
|
+
try {
|
|
2826
|
+
prepared.push(await prepareRemoval(ctx, indexName, docIds[i]));
|
|
2827
|
+
} catch (err) {
|
|
2828
|
+
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
const applies = prepared.map((removal) => applyOfRemoval(ctx, indexName, removal));
|
|
2832
|
+
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
2833
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
2834
|
+
const removal = prepared[i];
|
|
2835
|
+
const failure = failures[i];
|
|
2836
|
+
if (failure !== null) {
|
|
2837
|
+
let error = failure.error;
|
|
2838
|
+
try {
|
|
2839
|
+
await rollbackRemovedDocument(
|
|
2840
|
+
ctx,
|
|
2841
|
+
indexName,
|
|
2842
|
+
removal.docId,
|
|
2843
|
+
removal.restoreDoc,
|
|
2844
|
+
error,
|
|
2845
|
+
removal.preparedText
|
|
2846
|
+
);
|
|
2847
|
+
} catch (rollbackError) {
|
|
2848
|
+
error = rollbackError;
|
|
2849
|
+
}
|
|
2850
|
+
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
2851
|
+
continue;
|
|
2852
|
+
}
|
|
2853
|
+
if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
|
|
2854
|
+
succeeded.push(removal.docId);
|
|
2855
|
+
}
|
|
2856
|
+
if (chunkEnd < docIds.length) {
|
|
2857
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2858
|
+
}
|
|
2859
|
+
}
|
|
2860
|
+
} finally {
|
|
2861
|
+
await manager.publication.write(() => manager.endBatchRemove());
|
|
2862
|
+
}
|
|
2863
|
+
ctx.checkHeapPressure(indexName);
|
|
2864
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2865
|
+
return { succeeded, failed };
|
|
2866
|
+
}
|
|
2867
|
+
|
|
2868
|
+
// src/engine/mutations/update.ts
|
|
2869
|
+
function extractVectorFromDocForUpdate(document, fieldPath) {
|
|
2870
|
+
return extractVectorFromDoc(document, fieldPath);
|
|
2871
|
+
}
|
|
2872
|
+
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
2873
|
+
if (extractedVectors.size === 0) {
|
|
2874
|
+
return { partitionDoc: document };
|
|
2875
|
+
}
|
|
2876
|
+
let source = document;
|
|
2877
|
+
const prototype = Object.getPrototypeOf(document);
|
|
2878
|
+
let topLevelVectors = prototype === Object.prototype || prototype === null;
|
|
2879
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
2880
|
+
if (fieldPath.includes(".") || vector === null) {
|
|
2881
|
+
topLevelVectors = false;
|
|
2882
|
+
break;
|
|
2883
|
+
}
|
|
2884
|
+
const value = document[fieldPath];
|
|
2885
|
+
if (Array.isArray(value) && Object.keys(value).length !== value.length) {
|
|
2886
|
+
topLevelVectors = false;
|
|
2887
|
+
break;
|
|
2888
|
+
}
|
|
2889
|
+
}
|
|
2890
|
+
if (topLevelVectors) {
|
|
2891
|
+
source = { ...document };
|
|
2892
|
+
for (const fieldPath of extractedVectors.keys()) delete source[fieldPath];
|
|
2893
|
+
}
|
|
2894
|
+
const partitionDoc = structuredClone(source);
|
|
2895
|
+
for (const fieldPath of extractedVectors.keys()) {
|
|
2896
|
+
deleteNestedValue(partitionDoc, fieldPath);
|
|
2897
|
+
}
|
|
2898
|
+
return { partitionDoc };
|
|
2899
|
+
}
|
|
2900
|
+
function asUpdateError(err) {
|
|
2901
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
2902
|
+
}
|
|
2903
|
+
async function settleUpdate(ctx, indexName, prepared) {
|
|
2904
|
+
await ctx.orchestrator.settlePublications(indexName);
|
|
2905
|
+
try {
|
|
2906
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
2907
|
+
indexName,
|
|
2908
|
+
docId: prepared.docId,
|
|
2909
|
+
oldDocument: prepared.oldDocument ?? {},
|
|
2910
|
+
newDocument: prepared.document
|
|
2911
|
+
});
|
|
2912
|
+
} catch (err) {
|
|
2913
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2914
|
+
}
|
|
2915
|
+
const vecIndexes = ctx.requireManager(indexName).getVectorIndexes();
|
|
2916
|
+
for (const [fieldPath, vec] of prepared.extractedVectors) {
|
|
2917
|
+
if (vec === null) continue;
|
|
2918
|
+
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2921
|
+
async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
2922
|
+
ctx.guardShutdown();
|
|
2923
|
+
validateDocId(docId);
|
|
2924
|
+
requireMatchingDocId(docId, document);
|
|
2925
|
+
const manager = ctx.requireManager(indexName);
|
|
2926
|
+
if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
|
|
2927
|
+
await ctx.pluginRegistry.runHook("beforeUpdate", {
|
|
2928
|
+
indexName,
|
|
2929
|
+
docId,
|
|
2930
|
+
oldDocument: manager.get(docId) ?? {},
|
|
2931
|
+
newDocument: document
|
|
2932
|
+
});
|
|
2933
|
+
}
|
|
2934
|
+
if (entry.config.required && entry.config.required.length > 0) {
|
|
2935
|
+
validateRequiredFields(document, entry.config.required);
|
|
2936
|
+
}
|
|
2937
|
+
if (entry.config.embedding) {
|
|
2938
|
+
const { assertDocumentCarriesMappedVectors, embedDocumentFields } = await loadEmbedCode();
|
|
2939
|
+
if (entry.embeddingAdapter) {
|
|
2940
|
+
await embedDocumentFields(
|
|
2941
|
+
document,
|
|
2942
|
+
entry.config.embedding,
|
|
2943
|
+
entry.embeddingAdapter,
|
|
2944
|
+
ctx.abortController.signal
|
|
2945
|
+
);
|
|
2946
|
+
} else {
|
|
2947
|
+
assertDocumentCarriesMappedVectors(
|
|
2948
|
+
document,
|
|
2949
|
+
entry.config.embedding,
|
|
2950
|
+
entry.embeddingAdapterName
|
|
2951
|
+
);
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
const sizing = requireDocumentWithinSizeLimit(document);
|
|
2955
|
+
if (sizing !== null) await sizing;
|
|
2956
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2957
|
+
const extractedVectors = /* @__PURE__ */ new Map();
|
|
2958
|
+
if (vecIndexes.size > 0) {
|
|
2959
|
+
const dimensionCheckVectors = /* @__PURE__ */ new Map();
|
|
2960
|
+
for (const fieldPath of entry.vectorFieldPaths) {
|
|
2961
|
+
const newVec = extractVectorFromDocForUpdate(document, fieldPath);
|
|
2962
|
+
extractedVectors.set(fieldPath, newVec);
|
|
2963
|
+
if (newVec) {
|
|
2964
|
+
dimensionCheckVectors.set(fieldPath, newVec);
|
|
2965
|
+
}
|
|
2966
|
+
}
|
|
2967
|
+
if (dimensionCheckVectors.size > 0) {
|
|
2968
|
+
validateVectorDimensions(dimensionCheckVectors, vecIndexes);
|
|
2969
|
+
}
|
|
2970
|
+
}
|
|
2971
|
+
const { partitionDoc } = prepareUpdatePartitionDoc(document, extractedVectors);
|
|
2972
|
+
return {
|
|
2973
|
+
docId,
|
|
2974
|
+
document,
|
|
2975
|
+
partitionDoc,
|
|
2976
|
+
oldDocument: void 0,
|
|
2977
|
+
rollbackDoc: void 0,
|
|
2978
|
+
extractedVectors,
|
|
2979
|
+
removedSpellings: void 0,
|
|
2980
|
+
buffered: false
|
|
2981
|
+
};
|
|
2982
|
+
}
|
|
2983
|
+
function applyOfUpdate(ctx, indexName, prepared) {
|
|
2984
|
+
const manager = ctx.requireManager(indexName);
|
|
2985
|
+
const { docId, document, partitionDoc, extractedVectors } = prepared;
|
|
2986
|
+
const updateInIndex = () => {
|
|
2987
|
+
if (ctx.isRebalancing(indexName)) {
|
|
2988
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
2989
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
2990
|
+
if (!exists) {
|
|
2991
|
+
admitBufferedUpsert(ctx, indexName, manager, docId);
|
|
2992
|
+
}
|
|
2993
|
+
}
|
|
2994
|
+
const queued = ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName });
|
|
2995
|
+
if (queued !== void 0) {
|
|
2996
|
+
prepared.buffered = true;
|
|
2997
|
+
return queued;
|
|
2998
|
+
}
|
|
2999
|
+
prepared.oldDocument = extractedVectors.size > 0 || ctx.durability !== null || ctx.pluginRegistry.hasHooks("afterUpdate") ? manager.get(docId) : void 0;
|
|
3000
|
+
prepared.rollbackDoc = prepared.oldDocument;
|
|
3001
|
+
prepared.rollbackRef = ctx.durability === null ? void 0 : manager.getRef(docId);
|
|
3002
|
+
const stored = manager.update(docId, partitionDoc, {
|
|
3003
|
+
onRemovedSpellings: (spellings) => {
|
|
3004
|
+
prepared.removedSpellings = spellings;
|
|
3005
|
+
},
|
|
3006
|
+
onPreparedText: extractedVectors.size > 0 || ctx.durability !== null ? (change) => {
|
|
3007
|
+
prepared.preparedText = change;
|
|
3008
|
+
} : void 0
|
|
3009
|
+
});
|
|
3010
|
+
try {
|
|
3011
|
+
updateDocumentVectors(docId, extractedVectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
3012
|
+
} catch (err) {
|
|
3013
|
+
if (prepared.rollbackDoc === void 0) {
|
|
3014
|
+
manager.remove(docId, void 0, { preparedText: { before: prepared.preparedText?.after } });
|
|
3015
|
+
} else {
|
|
3016
|
+
try {
|
|
3017
|
+
const previous = prepareDocumentVectors(prepared.rollbackDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
3018
|
+
manager.update(docId, previous.partitionDoc, {
|
|
3019
|
+
preparedText: { before: prepared.preparedText?.after, after: prepared.preparedText?.before }
|
|
3020
|
+
});
|
|
3021
|
+
} catch (rollbackErr) {
|
|
3022
|
+
console.warn(
|
|
3023
|
+
`Rollback failed for doc "${docId}" during update atomicity:`,
|
|
3024
|
+
rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
|
|
3025
|
+
);
|
|
3026
|
+
}
|
|
3027
|
+
}
|
|
3028
|
+
throw err;
|
|
3029
|
+
}
|
|
3030
|
+
if (ctx.durability === null) prepared.preparedText = void 0;
|
|
3031
|
+
else prepared.appliedRef = stored;
|
|
3032
|
+
ctx.orchestrator.publishToWorkers({
|
|
3033
|
+
type: "update",
|
|
3034
|
+
indexName,
|
|
3035
|
+
docId,
|
|
3036
|
+
document: partitionDoc,
|
|
3037
|
+
requestId: `replicate-update-${docId}`,
|
|
3038
|
+
removedSpellings: prepared.removedSpellings
|
|
3039
|
+
});
|
|
3040
|
+
};
|
|
3041
|
+
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
3042
|
+
}
|
|
3043
|
+
async function updateDocument(ctx, indexName, docId, document, options) {
|
|
3044
|
+
ctx.guardShutdown();
|
|
3045
|
+
const entry = ctx.requireIndex(indexName);
|
|
3046
|
+
const prepared = await prepareUpdate(ctx, indexName, entry, docId, document);
|
|
3047
|
+
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
3048
|
+
if (ctx.durability) {
|
|
3049
|
+
try {
|
|
3050
|
+
await ctx.durability.recordInsertOrUpdate(indexName, docId, document, apply);
|
|
3051
|
+
} catch (err) {
|
|
3052
|
+
await rollbackUpdatedDocument(
|
|
3053
|
+
ctx,
|
|
3054
|
+
indexName,
|
|
3055
|
+
docId,
|
|
3056
|
+
prepared.rollbackDoc,
|
|
3057
|
+
err,
|
|
3058
|
+
prepared.preparedText,
|
|
3059
|
+
prepared.appliedRef,
|
|
3060
|
+
prepared.rollbackRef
|
|
3061
|
+
);
|
|
3062
|
+
throw err;
|
|
3063
|
+
}
|
|
3064
|
+
} else {
|
|
3065
|
+
await apply();
|
|
3066
|
+
}
|
|
3067
|
+
if (!prepared.buffered) await settleUpdate(ctx, indexName, prepared);
|
|
3068
|
+
ctx.checkHeapPressure(indexName);
|
|
3069
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3070
|
+
}
|
|
3071
|
+
function reportedDocIdOf(update) {
|
|
3072
|
+
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
3073
|
+
return typeof docId === "string" ? docId : "";
|
|
3074
|
+
}
|
|
3075
|
+
function requireUpdateEntry(update) {
|
|
3076
|
+
if (update !== null && typeof update === "object" && !Array.isArray(update)) {
|
|
3077
|
+
const { docId, document } = update;
|
|
3078
|
+
if (typeof docId === "string" && document !== null && typeof document === "object" && !Array.isArray(document)) {
|
|
3079
|
+
return { document };
|
|
3080
|
+
}
|
|
3081
|
+
}
|
|
3082
|
+
throw new NarsilError(
|
|
3083
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
3084
|
+
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
3085
|
+
);
|
|
3086
|
+
}
|
|
3087
|
+
async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
3088
|
+
ctx.guardShutdown();
|
|
3089
|
+
const entry = ctx.requireIndex(indexName);
|
|
3090
|
+
validateBatchList(updates, "updates");
|
|
3091
|
+
const succeeded = [];
|
|
3092
|
+
const failed = [];
|
|
3093
|
+
const updateBatchManager = ctx.requireManager(indexName);
|
|
3094
|
+
const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
|
|
3095
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
3096
|
+
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3097
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
3098
|
+
const prepared = [];
|
|
3099
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3100
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3101
|
+
const update = updates[i];
|
|
3102
|
+
const docId = reportedDocIdOf(update);
|
|
3103
|
+
try {
|
|
3104
|
+
const { document } = requireUpdateEntry(update);
|
|
3105
|
+
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document));
|
|
3106
|
+
} catch (err) {
|
|
3107
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3108
|
+
}
|
|
3109
|
+
}
|
|
3110
|
+
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3111
|
+
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
3112
|
+
for (let i = prepared.length - 1; i >= 0; i--) {
|
|
3113
|
+
const failure = failures[i];
|
|
3114
|
+
if (failure === null) continue;
|
|
3115
|
+
const update = prepared[i];
|
|
3116
|
+
try {
|
|
3117
|
+
await rollbackUpdatedDocument(
|
|
3118
|
+
ctx,
|
|
3119
|
+
indexName,
|
|
3120
|
+
update.docId,
|
|
3121
|
+
update.rollbackDoc,
|
|
3122
|
+
failure.error,
|
|
3123
|
+
update.preparedText,
|
|
3124
|
+
update.appliedRef,
|
|
3125
|
+
update.rollbackRef
|
|
3126
|
+
);
|
|
3127
|
+
} catch (error) {
|
|
3128
|
+
failure.error = error;
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
3131
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
3132
|
+
const update = prepared[i];
|
|
3133
|
+
const failure = failures[i];
|
|
3134
|
+
if (failure !== null) {
|
|
3135
|
+
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
3136
|
+
continue;
|
|
3137
|
+
}
|
|
3138
|
+
if (!update.buffered) await settleUpdate(ctx, indexName, update);
|
|
3139
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3140
|
+
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
3141
|
+
}
|
|
3142
|
+
succeeded.push(update.docId);
|
|
3143
|
+
}
|
|
3144
|
+
if (chunkEnd < updates.length) {
|
|
3145
|
+
ctx.checkHeapPressure(indexName);
|
|
3146
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
for (const fieldPath of touchedVectorFields) {
|
|
3150
|
+
updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
3151
|
+
}
|
|
3152
|
+
ctx.checkHeapPressure(indexName);
|
|
3153
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3154
|
+
return { succeeded, failed };
|
|
3155
|
+
}
|
|
3156
|
+
|
|
3157
|
+
// src/engine/partition-config.ts
|
|
3158
|
+
async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
|
|
3159
|
+
const entry = core.requireIndex(indexName);
|
|
3160
|
+
const manager = core.requireManager(indexName);
|
|
3161
|
+
if (core.rebalancingIndexes.has(indexName)) {
|
|
3162
|
+
throw new NarsilError(
|
|
3163
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
3164
|
+
`Index "${indexName}" is currently being rebalanced`
|
|
3165
|
+
);
|
|
3166
|
+
}
|
|
3167
|
+
validatePartitionConfig(partitionConfig);
|
|
3168
|
+
if (partitionConfig.maxPartitions !== void 0 && partitionConfig.maxPartitions < manager.partitionCount) {
|
|
3169
|
+
throw new NarsilError(
|
|
3170
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
3171
|
+
`maxPartitions (${partitionConfig.maxPartitions}) is less than the current partition count (${manager.partitionCount})`,
|
|
3172
|
+
{ maxPartitions: partitionConfig.maxPartitions, partitionCount: manager.partitionCount }
|
|
3173
|
+
);
|
|
3174
|
+
}
|
|
3175
|
+
const currentDocCount = manager.countDocuments();
|
|
3176
|
+
const newMaxDocs = partitionConfig.maxDocsPerPartition ?? entry.config.partitions?.maxDocsPerPartition;
|
|
3177
|
+
if (newMaxDocs !== void 0) {
|
|
3178
|
+
const newTotalCapacity = newMaxDocs * manager.partitionCount;
|
|
3179
|
+
if (newTotalCapacity < currentDocCount) {
|
|
3180
|
+
throw new NarsilError(
|
|
3181
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
3182
|
+
`New capacity (${newTotalCapacity}) is less than current document count (${currentDocCount})`,
|
|
3183
|
+
{ newTotalCapacity, currentDocCount }
|
|
3184
|
+
);
|
|
3185
|
+
}
|
|
3186
|
+
}
|
|
3187
|
+
if (!entry.config.partitions) entry.config.partitions = {};
|
|
3188
|
+
if (partitionConfig.maxDocsPerPartition !== void 0)
|
|
3189
|
+
entry.config.partitions.maxDocsPerPartition = partitionConfig.maxDocsPerPartition;
|
|
3190
|
+
if (partitionConfig.maxPartitions !== void 0) entry.config.partitions.maxPartitions = partitionConfig.maxPartitions;
|
|
3191
|
+
if (partitionConfig.watermark !== void 0) entry.config.partitions.watermark = partitionConfig.watermark;
|
|
3192
|
+
if (core.durability) await core.durability.manager.persistMetadata(indexName);
|
|
3193
|
+
core.watermarkNotifier.check(indexName);
|
|
3194
|
+
}
|
|
3195
|
+
|
|
3196
|
+
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|