@delali/narsil 0.3.1-next.99 → 0.4.1-next.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +30 -11
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +404 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -0,0 +1,3603 @@
|
|
|
1
|
+
// src/core/frozen-writes/kernel-host.browser.ts
|
|
2
|
+
var PENDING_DOCUMENTS;
|
|
3
|
+
var PENDING_TOKEN_WORDS;
|
|
4
|
+
var PENDING_TOKENS;
|
|
5
|
+
var PENDING_UNITS;
|
|
6
|
+
var SegmentRefusal;
|
|
7
|
+
var checkText;
|
|
8
|
+
var compareBytes;
|
|
9
|
+
var packedTokens;
|
|
10
|
+
var pendingSettled;
|
|
11
|
+
var readFrozenParts;
|
|
12
|
+
var requireRoom;
|
|
13
|
+
var requireSegment;
|
|
14
|
+
var resolveStopWords;
|
|
15
|
+
var storedText;
|
|
16
|
+
var textAt;
|
|
17
|
+
var transformToken;
|
|
18
|
+
var visitDocumentSurfaceSpellings;
|
|
19
|
+
function bindFrozenKernelHost(host) {
|
|
20
|
+
({
|
|
21
|
+
PENDING_DOCUMENTS,
|
|
22
|
+
PENDING_TOKEN_WORDS,
|
|
23
|
+
PENDING_TOKENS,
|
|
24
|
+
PENDING_UNITS,
|
|
25
|
+
SegmentRefusal,
|
|
26
|
+
checkText,
|
|
27
|
+
compareBytes,
|
|
28
|
+
packedTokens,
|
|
29
|
+
pendingSettled,
|
|
30
|
+
readFrozenParts,
|
|
31
|
+
requireRoom,
|
|
32
|
+
requireSegment,
|
|
33
|
+
resolveStopWords,
|
|
34
|
+
storedText,
|
|
35
|
+
textAt,
|
|
36
|
+
transformToken,
|
|
37
|
+
visitDocumentSurfaceSpellings
|
|
38
|
+
} = host);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// src/core/frozen-writes/constants.ts
|
|
42
|
+
var PENDING_REMOVAL_DOCUMENTS = 64;
|
|
43
|
+
var PENDING_REMOVAL_TOKENS = 1024;
|
|
44
|
+
var PENDING_REMOVAL_UNITS = 16384;
|
|
45
|
+
var PENDING_REMOVAL_GROWTH_FACTOR = 2;
|
|
46
|
+
var PENDING_REMOVAL_GROWTH_LIMIT_BYTES = 67108864;
|
|
47
|
+
var PENDING_REMOVAL_LOW_MEMORY_LIMIT_BYTES = 65536;
|
|
48
|
+
var SETTLE_KEY_SLOTS_PER_KEY = 2;
|
|
49
|
+
var SETTLE_ORDER_CHECKS_PER_BUFFER = 8;
|
|
50
|
+
var REMOVAL_SCRATCH_START_TOKEN_WORDS = 2048;
|
|
51
|
+
var REMOVAL_SCRATCH_START_UNITS = 8192;
|
|
52
|
+
var REMOVAL_SCRATCH_START_KEYS = 256;
|
|
53
|
+
var REMOVAL_SCRATCH_START_KEY_UNITS = 2048;
|
|
54
|
+
var REMOVAL_SCRATCH_SLOTS_PER_KEY = 2;
|
|
55
|
+
var REMOVAL_SCRATCH_GROWTH_FACTOR = 2;
|
|
56
|
+
|
|
57
|
+
// src/core/frozen-writes/pending-writes.ts
|
|
58
|
+
var NO_SURFACE = 4294967295;
|
|
59
|
+
var FIELD_TOKEN_WORDS = 8;
|
|
60
|
+
function pendingNeeds(fields) {
|
|
61
|
+
let tokens = 0;
|
|
62
|
+
let units = 0;
|
|
63
|
+
for (const field of fields) {
|
|
64
|
+
for (let at = 0; at < field.tokens.length; at += FIELD_TOKEN_WORDS) {
|
|
65
|
+
tokens++;
|
|
66
|
+
units += field.tokens[at + 2];
|
|
67
|
+
if (field.tokens[at + 3] !== NO_SURFACE) units += field.tokens[at + 4];
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return { documents: 1, tokens, units };
|
|
71
|
+
}
|
|
72
|
+
function pendingFits(state, needed, empty) {
|
|
73
|
+
const header = state.pendingHeader;
|
|
74
|
+
const held = (word) => empty ? 0 : header[word];
|
|
75
|
+
return held(PENDING_DOCUMENTS) + needed.documents <= state.pendingDocuments.length && held(PENDING_TOKENS) + needed.tokens <= state.pendingTokens.length / PENDING_TOKEN_WORDS && held(PENDING_UNITS) + needed.units <= state.pendingUnits.length;
|
|
76
|
+
}
|
|
77
|
+
function grownLength(held, required, smallest) {
|
|
78
|
+
return held >= required ? held : Math.max(smallest, required * PENDING_REMOVAL_GROWTH_FACTOR);
|
|
79
|
+
}
|
|
80
|
+
function grown(held, length, create) {
|
|
81
|
+
if (held.length >= length) return held;
|
|
82
|
+
const next = new create(length);
|
|
83
|
+
next.set(held);
|
|
84
|
+
return next;
|
|
85
|
+
}
|
|
86
|
+
function growTo(state, documents, tokens, units, processHasRoomFor) {
|
|
87
|
+
const documentLength = grownLength(state.pendingDocuments.length, documents, PENDING_REMOVAL_DOCUMENTS);
|
|
88
|
+
const tokenLength = grownLength(
|
|
89
|
+
state.pendingTokens.length,
|
|
90
|
+
tokens * PENDING_TOKEN_WORDS,
|
|
91
|
+
PENDING_REMOVAL_TOKENS * PENDING_TOKEN_WORDS
|
|
92
|
+
);
|
|
93
|
+
const unitLength = grownLength(state.pendingUnits.length, units, PENDING_REMOVAL_UNITS);
|
|
94
|
+
const bytes = documentLength * Uint32Array.BYTES_PER_ELEMENT + tokenLength * Uint32Array.BYTES_PER_ELEMENT + unitLength * Uint16Array.BYTES_PER_ELEMENT;
|
|
95
|
+
if (bytes > PENDING_REMOVAL_GROWTH_LIMIT_BYTES) return false;
|
|
96
|
+
const heldBytes = state.pendingDocuments.byteLength + state.pendingTokens.byteLength + state.pendingUnits.byteLength;
|
|
97
|
+
if (bytes > PENDING_REMOVAL_LOW_MEMORY_LIMIT_BYTES && !processHasRoomFor(bytes - heldBytes)) return false;
|
|
98
|
+
state.pendingDocuments = grown(state.pendingDocuments, documentLength, Uint32Array);
|
|
99
|
+
state.pendingTokens = grown(state.pendingTokens, tokenLength, Uint32Array);
|
|
100
|
+
state.pendingUnits = grown(state.pendingUnits, unitLength, Uint16Array);
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
var growFrozenRemovalRegions = (_segment, state, fields, processHasRoomFor) => {
|
|
104
|
+
const needed = pendingNeeds(fields);
|
|
105
|
+
if (pendingFits(state, needed, false)) return;
|
|
106
|
+
const header = state.pendingHeader;
|
|
107
|
+
const heldGrows = growTo(
|
|
108
|
+
state,
|
|
109
|
+
header[PENDING_DOCUMENTS] + needed.documents,
|
|
110
|
+
header[PENDING_TOKENS] + needed.tokens,
|
|
111
|
+
header[PENDING_UNITS] + needed.units,
|
|
112
|
+
processHasRoomFor
|
|
113
|
+
);
|
|
114
|
+
if (heldGrows || pendingFits(state, needed, true)) return;
|
|
115
|
+
state.pendingDocuments = grown(state.pendingDocuments, needed.documents, Uint32Array);
|
|
116
|
+
state.pendingTokens = grown(state.pendingTokens, needed.tokens * PENDING_TOKEN_WORDS, Uint32Array);
|
|
117
|
+
state.pendingUnits = grown(state.pendingUnits, needed.units, Uint16Array);
|
|
118
|
+
};
|
|
119
|
+
function appendPending(state, ordinal, fields) {
|
|
120
|
+
const header = state.pendingHeader;
|
|
121
|
+
state.pendingDocuments[header[PENDING_DOCUMENTS]] = ordinal;
|
|
122
|
+
let written = header[PENDING_TOKENS];
|
|
123
|
+
let units = header[PENDING_UNITS];
|
|
124
|
+
for (const field of fields) {
|
|
125
|
+
for (let at = 0; at < field.tokens.length; at += FIELD_TOKEN_WORDS) {
|
|
126
|
+
const word = written++ * PENDING_TOKEN_WORDS;
|
|
127
|
+
const surface = field.tokens[at + 3];
|
|
128
|
+
state.pendingTokens[word] = field.field;
|
|
129
|
+
state.pendingTokens[word + 1] = units;
|
|
130
|
+
state.pendingTokens[word + 2] = field.tokens[at + 2];
|
|
131
|
+
units = copyUnits(state.pendingUnits, units, field.units, field.tokens[at + 1], field.tokens[at + 2]);
|
|
132
|
+
state.pendingTokens[word + 3] = surface === NO_SURFACE ? NO_SURFACE : units;
|
|
133
|
+
state.pendingTokens[word + 4] = surface === NO_SURFACE ? 0 : field.tokens[at + 4];
|
|
134
|
+
if (surface !== NO_SURFACE)
|
|
135
|
+
units = copyUnits(state.pendingUnits, units, field.units, surface, field.tokens[at + 4]);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
function copyUnits(target, into, source, start, length) {
|
|
140
|
+
for (let at = 0; at < length; at++) target[into + at] = source[start + at];
|
|
141
|
+
return into + length;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// src/core/frozen-writes/counts.ts
|
|
145
|
+
var termChangesByCounts = /* @__PURE__ */ new WeakMap();
|
|
146
|
+
function termChangesOf(segment) {
|
|
147
|
+
const counts = segment.terms.counts;
|
|
148
|
+
const held = termChangesByCounts.get(counts);
|
|
149
|
+
if (held !== void 0) return held;
|
|
150
|
+
const terms = counts.length;
|
|
151
|
+
const created = {
|
|
152
|
+
documents: new Float64Array(terms),
|
|
153
|
+
amounts: new Float64Array(terms),
|
|
154
|
+
marks: new Float64Array(terms),
|
|
155
|
+
touched: [],
|
|
156
|
+
mark: 0,
|
|
157
|
+
seen: new Float64Array(segment.documentCount),
|
|
158
|
+
seenMark: 0
|
|
159
|
+
};
|
|
160
|
+
termChangesByCounts.set(counts, created);
|
|
161
|
+
return created;
|
|
162
|
+
}
|
|
163
|
+
function clearTermChanges(changes) {
|
|
164
|
+
for (const term of changes.touched) {
|
|
165
|
+
changes.documents[term] = 0;
|
|
166
|
+
changes.amounts[term] = 0;
|
|
167
|
+
}
|
|
168
|
+
changes.touched.length = 0;
|
|
169
|
+
}
|
|
170
|
+
function slotOfRow(offsets, row) {
|
|
171
|
+
let low = 0;
|
|
172
|
+
let high = offsets.length - 1;
|
|
173
|
+
while (low < high) {
|
|
174
|
+
const middle = low + (high - low >>> 1);
|
|
175
|
+
if (offsets[middle + 1] <= row) low = middle + 1;
|
|
176
|
+
else high = middle;
|
|
177
|
+
}
|
|
178
|
+
return low;
|
|
179
|
+
}
|
|
180
|
+
function addTerm(changes, term, documents, amount) {
|
|
181
|
+
if (changes.documents[term] === 0) changes.touched.push(term);
|
|
182
|
+
changes.documents[term] += documents;
|
|
183
|
+
changes.amounts[term] += amount;
|
|
184
|
+
}
|
|
185
|
+
function addOccurrences(changes, term, amount) {
|
|
186
|
+
if (changes.marks[term] === changes.mark) {
|
|
187
|
+
changes.amounts[term] += amount;
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
changes.marks[term] = changes.mark;
|
|
191
|
+
addTerm(changes, term, 1, amount);
|
|
192
|
+
}
|
|
193
|
+
function addIndexedDocument(changes, segment, ordinal) {
|
|
194
|
+
const { removalIndex, postingDocuments, postingFrequencies, postingFields, postingOffsets, payloadSlots, terms } = segment;
|
|
195
|
+
if (removalIndex === null) return;
|
|
196
|
+
const start = removalIndex.starts[ordinal];
|
|
197
|
+
const end = removalIndex.starts[ordinal + 1];
|
|
198
|
+
if (start > end || end > postingDocuments.length) return;
|
|
199
|
+
changes.mark++;
|
|
200
|
+
let previous = -1;
|
|
201
|
+
for (let at = start; at < end; at++) {
|
|
202
|
+
const row = removalIndex.rows[at];
|
|
203
|
+
if (row >= postingDocuments.length || row <= previous) continue;
|
|
204
|
+
previous = row;
|
|
205
|
+
if (postingDocuments[row] !== ordinal || postingFields[row] >= segment.partialFields.length) continue;
|
|
206
|
+
const slot = slotOfRow(postingOffsets, row);
|
|
207
|
+
if (slot >= terms.counts.length || postingOffsets[slot] > row || postingOffsets[slot + 1] > postingDocuments.length)
|
|
208
|
+
continue;
|
|
209
|
+
const term = removalIndex.termOfSlot[slot];
|
|
210
|
+
if (term >= terms.counts.length || payloadSlots[term] !== slot) continue;
|
|
211
|
+
addOccurrences(changes, term, postingFrequencies[row]);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
var UNCHECKED_INDEX = -2;
|
|
215
|
+
var MALFORMED_INDEX = -3;
|
|
216
|
+
var indexFieldLimits = /* @__PURE__ */ new WeakMap();
|
|
217
|
+
function rowsAreOrdered(segment, index) {
|
|
218
|
+
const { postingDocuments, postingOffsets, postingFields, payloadSlots, terms } = segment;
|
|
219
|
+
const { starts, rows, termOfSlot } = index;
|
|
220
|
+
const slots = terms.counts.length;
|
|
221
|
+
const postings = postingDocuments.length;
|
|
222
|
+
if (postingOffsets[slots] > postings) return MALFORMED_INDEX;
|
|
223
|
+
for (let slot = 0; slot < slots; slot++) {
|
|
224
|
+
const term = termOfSlot[slot];
|
|
225
|
+
if (postingOffsets[slot] > postingOffsets[slot + 1] || term >= slots || payloadSlots[term] !== slot)
|
|
226
|
+
return MALFORMED_INDEX;
|
|
227
|
+
}
|
|
228
|
+
let widestField = -1;
|
|
229
|
+
for (let ordinal = 0; ordinal < segment.documentCount; ordinal++) {
|
|
230
|
+
const end = starts[ordinal + 1];
|
|
231
|
+
if (starts[ordinal] > end || end > postings) return MALFORMED_INDEX;
|
|
232
|
+
let previous = -1;
|
|
233
|
+
for (let at = starts[ordinal]; at < end; at++) {
|
|
234
|
+
const row = rows[at];
|
|
235
|
+
if (row <= previous || row < postingOffsets[0] || row >= postingOffsets[slots]) return MALFORMED_INDEX;
|
|
236
|
+
if (postingDocuments[row] !== ordinal) return MALFORMED_INDEX;
|
|
237
|
+
if (postingFields[row] > widestField) widestField = postingFields[row];
|
|
238
|
+
previous = row;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return widestField;
|
|
242
|
+
}
|
|
243
|
+
var checkedOccurrences = /* @__PURE__ */ new WeakMap();
|
|
244
|
+
function occurrencesAreCounts(occurrences) {
|
|
245
|
+
const held = checkedOccurrences.get(occurrences);
|
|
246
|
+
if (held !== void 0) return held;
|
|
247
|
+
let counts = true;
|
|
248
|
+
for (let term = 0; term < occurrences.length && counts; term++)
|
|
249
|
+
counts = Number.isFinite(occurrences[term]) && occurrences[term] >= 0;
|
|
250
|
+
checkedOccurrences.set(occurrences, counts);
|
|
251
|
+
return counts;
|
|
252
|
+
}
|
|
253
|
+
function trustedIndexOf(segment, state) {
|
|
254
|
+
const index = segment.removalIndex;
|
|
255
|
+
if (index === null || !occurrencesAreCounts(state.removedOccurrences)) return null;
|
|
256
|
+
let widestField = indexFieldLimits.get(segment) ?? UNCHECKED_INDEX;
|
|
257
|
+
if (widestField === UNCHECKED_INDEX) {
|
|
258
|
+
widestField = rowsAreOrdered(segment, index);
|
|
259
|
+
indexFieldLimits.set(segment, widestField);
|
|
260
|
+
}
|
|
261
|
+
return widestField !== MALFORMED_INDEX && widestField < segment.partialFields.length ? index : null;
|
|
262
|
+
}
|
|
263
|
+
function subtractTrusted(segment, state, index) {
|
|
264
|
+
const { postingOffsets, postingFrequencies } = segment;
|
|
265
|
+
const stored = segment.terms.counts;
|
|
266
|
+
const { starts, rows, termOfSlot } = index;
|
|
267
|
+
const { removedDocuments, removedOccurrences, pendingDocuments } = state;
|
|
268
|
+
const pending = state.pendingHeader[PENDING_DOCUMENTS];
|
|
269
|
+
for (let document = 0; document < pending; document++) {
|
|
270
|
+
const ordinal = pendingDocuments[document];
|
|
271
|
+
let previousSlot = -1;
|
|
272
|
+
let term = 0;
|
|
273
|
+
for (let at = starts[ordinal]; at < starts[ordinal + 1]; at++) {
|
|
274
|
+
const row = rows[at];
|
|
275
|
+
const slot = slotOfRow(postingOffsets, row);
|
|
276
|
+
if (slot !== previousSlot) {
|
|
277
|
+
previousSlot = slot;
|
|
278
|
+
term = termOfSlot[slot];
|
|
279
|
+
removedDocuments[term] = Math.min(stored[term], removedDocuments[term] + 1);
|
|
280
|
+
}
|
|
281
|
+
removedOccurrences[term] += postingFrequencies[row];
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
function addPagedDocument(changes, segment, paged, ordinal) {
|
|
286
|
+
const pairs = paged.read(ordinal);
|
|
287
|
+
changes.mark++;
|
|
288
|
+
for (let at = 0; at + 1 < pairs.length; at += 2) {
|
|
289
|
+
requireSegment(pairs[at] < segment.terms.counts.length);
|
|
290
|
+
addOccurrences(changes, pairs[at], pairs[at + 1]);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
function scanPostings(changes, segment, marks) {
|
|
294
|
+
const { postingDocuments, postingFrequencies, postingFields, postingOffsets, payloadSlots, terms } = segment;
|
|
295
|
+
const stamps = new Uint32Array(marks.length);
|
|
296
|
+
for (let term = 0; term < terms.counts.length; term++) {
|
|
297
|
+
const slot = payloadSlots[term];
|
|
298
|
+
if (slot >= terms.counts.length) continue;
|
|
299
|
+
const start = postingOffsets[slot];
|
|
300
|
+
const end = postingOffsets[slot + 1];
|
|
301
|
+
if (start > end || end > postingDocuments.length) continue;
|
|
302
|
+
const stamp = term + 1;
|
|
303
|
+
let documents = 0;
|
|
304
|
+
let amount = 0;
|
|
305
|
+
for (let row = start; row < end; row++) {
|
|
306
|
+
const ordinal = postingDocuments[row];
|
|
307
|
+
if (ordinal >= marks.length || marks[ordinal] === 0 || postingFields[row] >= segment.partialFields.length)
|
|
308
|
+
continue;
|
|
309
|
+
amount += postingFrequencies[row];
|
|
310
|
+
if (stamps[ordinal] !== stamp) {
|
|
311
|
+
stamps[ordinal] = stamp;
|
|
312
|
+
documents++;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (documents > 0) addTerm(changes, term, documents, amount);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function requirePendingDocuments(changes, segment, state) {
|
|
319
|
+
const pending = state.pendingHeader[PENDING_DOCUMENTS];
|
|
320
|
+
if (changes.seen.length !== segment.documentCount) changes.seen = new Float64Array(segment.documentCount);
|
|
321
|
+
const { seen } = changes;
|
|
322
|
+
const stamp = ++changes.seenMark;
|
|
323
|
+
for (let at = 0; at < pending; at++) {
|
|
324
|
+
const ordinal = state.pendingDocuments[at];
|
|
325
|
+
requireSegment(
|
|
326
|
+
ordinal < segment.documentCount && (state.tombstones[ordinal >>> 5] & 1 << (ordinal & 31)) !== 0 && seen[ordinal] !== stamp
|
|
327
|
+
);
|
|
328
|
+
seen[ordinal] = stamp;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
function settleTermCounts(changes, segment, state, paged) {
|
|
332
|
+
const pending = state.pendingHeader[PENDING_DOCUMENTS];
|
|
333
|
+
if (paged === null && segment.removalIndex === null) {
|
|
334
|
+
const marks = new Uint8Array(segment.documentCount);
|
|
335
|
+
for (let at = 0; at < pending; at++) marks[state.pendingDocuments[at]] = 1;
|
|
336
|
+
scanPostings(changes, segment, marks);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
for (let at = 0; at < pending; at++) {
|
|
340
|
+
const ordinal = state.pendingDocuments[at];
|
|
341
|
+
if (paged === null) addIndexedDocument(changes, segment, ordinal);
|
|
342
|
+
else addPagedDocument(changes, segment, paged, ordinal);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// src/core/frozen-writes/removal-request.ts
|
|
347
|
+
function copyField(value) {
|
|
348
|
+
requireSegment(typeof value === "object" && value !== null);
|
|
349
|
+
return { ...value };
|
|
350
|
+
}
|
|
351
|
+
function readFrozenFields(input, output, ordinal, rawFields) {
|
|
352
|
+
const { segment, state, regions } = readFrozenParts(input, output);
|
|
353
|
+
requireSegment(Array.isArray(rawFields) && rawFields.length <= 65536);
|
|
354
|
+
const fields = rawFields.map(copyField);
|
|
355
|
+
requireSegment(ordinal === null || ordinal < segment.documentCount);
|
|
356
|
+
const fieldCount = segment.partialFields.length;
|
|
357
|
+
requireSegment(fields.length <= fieldCount);
|
|
358
|
+
let previous = -1;
|
|
359
|
+
for (const field of fields) {
|
|
360
|
+
requireSegment(Number.isInteger(field.field) && field.field > previous && field.field < fieldCount);
|
|
361
|
+
field.tokens = regions.array(field.tokens, "Uint32Array");
|
|
362
|
+
field.units = regions.array(field.units, "Uint16Array");
|
|
363
|
+
requireSegment(
|
|
364
|
+
field.tokens.length % FIELD_TOKEN_WORDS === 0 && field.tokens.length / FIELD_TOKEN_WORDS <= 4294967295 && field.units.length <= 4294967295
|
|
365
|
+
);
|
|
366
|
+
for (let at = 0; at < field.tokens.length; at += FIELD_TOKEN_WORDS) {
|
|
367
|
+
checkText(field.units, field.tokens[at + 1], field.tokens[at + 2]);
|
|
368
|
+
if (field.tokens[at + 3] !== NO_SURFACE) checkText(field.units, field.tokens[at + 3], field.tokens[at + 4]);
|
|
369
|
+
}
|
|
370
|
+
previous = field.field;
|
|
371
|
+
}
|
|
372
|
+
regions.check();
|
|
373
|
+
return {
|
|
374
|
+
segment,
|
|
375
|
+
state,
|
|
376
|
+
fields,
|
|
377
|
+
readOutput(rawEntries, rawValues) {
|
|
378
|
+
const entries = regions.array(rawEntries, "Uint32Array", void 0, true);
|
|
379
|
+
const values = regions.array(rawValues, "Float64Array", void 0, true);
|
|
380
|
+
requireSegment(entries.length % 2 === 0);
|
|
381
|
+
regions.check();
|
|
382
|
+
return { entries, values };
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
function readFrozenRequest(input, output, ordinal, fields) {
|
|
387
|
+
requireSegment(Number.isInteger(ordinal) && ordinal >= 0);
|
|
388
|
+
return readFrozenFields(input, output, ordinal, fields);
|
|
389
|
+
}
|
|
390
|
+
function readFrozenSettle(input, output) {
|
|
391
|
+
return readFrozenFields(input, output, null, []);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// src/core/frozen-writes/unit-order.ts
|
|
395
|
+
var UTF8_LEADS = [0, 0, 192, 224, 240];
|
|
396
|
+
function holdsLoneSurrogate(units, start, length) {
|
|
397
|
+
const end = start + length;
|
|
398
|
+
for (let at = start; at < end; at++) {
|
|
399
|
+
const unit = units[at];
|
|
400
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
401
|
+
if (unit >= 56320 || at + 1 >= end || units[at + 1] < 56320 || units[at + 1] >= 57344) return true;
|
|
402
|
+
at++;
|
|
403
|
+
}
|
|
404
|
+
return false;
|
|
405
|
+
}
|
|
406
|
+
function utf8Width(point) {
|
|
407
|
+
if (point < 128) return 1;
|
|
408
|
+
if (point < 2048) return 2;
|
|
409
|
+
return point < 65536 ? 3 : 4;
|
|
410
|
+
}
|
|
411
|
+
function utf8Byte(point, width, index) {
|
|
412
|
+
if (width === 1) return point;
|
|
413
|
+
const shift = 6 * (width - 1 - index);
|
|
414
|
+
return index === 0 ? UTF8_LEADS[width] | point >> shift : 128 | point >> shift & 63;
|
|
415
|
+
}
|
|
416
|
+
function compareTableToUnits(table, from, to, units, start, length) {
|
|
417
|
+
let byte = from;
|
|
418
|
+
const end = start + length;
|
|
419
|
+
for (let at = start; at < end; at++) {
|
|
420
|
+
let point = units[at];
|
|
421
|
+
if (point >= 55296 && point < 56320) point = 65536 + (point - 55296 << 10) + (units[++at] - 56320);
|
|
422
|
+
const width = utf8Width(point);
|
|
423
|
+
for (let index = 0; index < width; index++) {
|
|
424
|
+
if (byte >= to) return -1;
|
|
425
|
+
const order = table[byte++] - utf8Byte(point, width, index);
|
|
426
|
+
if (order !== 0) return order;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return byte < to ? 1 : 0;
|
|
430
|
+
}
|
|
431
|
+
function codePointRank(unit) {
|
|
432
|
+
if (unit < 55296) return unit;
|
|
433
|
+
return unit < 57344 ? unit + 8192 : unit - 2048;
|
|
434
|
+
}
|
|
435
|
+
function compareUnits(units, left, leftLength, right, rightLength) {
|
|
436
|
+
const shared = leftLength < rightLength ? leftLength : rightLength;
|
|
437
|
+
for (let at = 0; at < shared; at++) {
|
|
438
|
+
const order = codePointRank(units[left + at]) - codePointRank(units[right + at]);
|
|
439
|
+
if (order !== 0) return order;
|
|
440
|
+
}
|
|
441
|
+
return leftLength - rightLength;
|
|
442
|
+
}
|
|
443
|
+
function sameUnits(units, left, leftLength, right, rightLength) {
|
|
444
|
+
if (leftLength !== rightLength) return false;
|
|
445
|
+
for (let at = 0; at < leftLength; at++) if (units[left + at] !== units[right + at]) return false;
|
|
446
|
+
return true;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// src/core/frozen-writes/table-order.ts
|
|
450
|
+
var heldOrders = /* @__PURE__ */ new WeakMap();
|
|
451
|
+
function sameView(left, right) {
|
|
452
|
+
if (left === void 0 || right === void 0) return left === right;
|
|
453
|
+
return left.buffer === right.buffer && left.byteOffset === right.byteOffset && left.byteLength === right.byteLength;
|
|
454
|
+
}
|
|
455
|
+
function compareEntries(text, leftFrom, leftTo, rightFrom, rightTo) {
|
|
456
|
+
const leftLength = leftTo - leftFrom;
|
|
457
|
+
const rightLength = rightTo - rightFrom;
|
|
458
|
+
const shared = leftLength < rightLength ? leftLength : rightLength;
|
|
459
|
+
for (let at = 0; at < shared; at++) {
|
|
460
|
+
const order = text[leftFrom + at] - text[rightFrom + at];
|
|
461
|
+
if (order !== 0) return order;
|
|
462
|
+
}
|
|
463
|
+
return leftLength - rightLength;
|
|
464
|
+
}
|
|
465
|
+
function checkAscends(table, stride, tokens) {
|
|
466
|
+
const { offsets, text, counts } = table;
|
|
467
|
+
const size = counts.length;
|
|
468
|
+
if (size === 0) return true;
|
|
469
|
+
if ((size - 1) * stride + 1 >= offsets.length) return false;
|
|
470
|
+
if (tokens !== void 0 && tokens.length < size) return false;
|
|
471
|
+
for (let entry = 0; entry < size; entry++) {
|
|
472
|
+
const from = offsets[entry * stride];
|
|
473
|
+
const to = offsets[entry * stride + 1];
|
|
474
|
+
if (from > to || to > text.length) return false;
|
|
475
|
+
if (entry === 0) continue;
|
|
476
|
+
const before = (entry - 1) * stride;
|
|
477
|
+
const order = compareEntries(text, offsets[before], offsets[before + 1], from, to) || (tokens === void 0 ? 0 : tokens[entry - 1] - tokens[entry]);
|
|
478
|
+
if (order >= 0) return false;
|
|
479
|
+
}
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
function tableAscends(table, stride, tokens) {
|
|
483
|
+
const buffer = table.offsets.buffer;
|
|
484
|
+
const held = heldOrders.get(buffer);
|
|
485
|
+
if (held !== void 0) {
|
|
486
|
+
for (const order of held)
|
|
487
|
+
if (order.stride === stride && sameView(order.offsets, table.offsets) && sameView(order.text, table.text) && sameView(order.counts, table.counts) && sameView(order.tokens, tokens))
|
|
488
|
+
return order.ascends;
|
|
489
|
+
}
|
|
490
|
+
const ascends = checkAscends(table, stride, tokens);
|
|
491
|
+
const orders = held ?? [];
|
|
492
|
+
if (orders.length >= SETTLE_ORDER_CHECKS_PER_BUFFER) orders.shift();
|
|
493
|
+
orders.push({ stride, offsets: table.offsets, text: table.text, counts: table.counts, tokens, ascends });
|
|
494
|
+
if (held === void 0) heldOrders.set(buffer, orders);
|
|
495
|
+
return ascends;
|
|
496
|
+
}
|
|
497
|
+
function entryOrder(table, stride, tokens, entry, units, start, length, token) {
|
|
498
|
+
return compareTableToUnits(
|
|
499
|
+
table.text,
|
|
500
|
+
table.offsets[entry * stride],
|
|
501
|
+
table.offsets[entry * stride + 1],
|
|
502
|
+
units,
|
|
503
|
+
start,
|
|
504
|
+
length
|
|
505
|
+
) || (tokens === void 0 ? 0 : tokens[entry] - token);
|
|
506
|
+
}
|
|
507
|
+
function findUnits(table, units, start, length, stride, tokens, token = 0) {
|
|
508
|
+
if (holdsLoneSurrogate(units, start, length)) return -1;
|
|
509
|
+
let low = 0;
|
|
510
|
+
let high = table.counts.length;
|
|
511
|
+
while (low < high) {
|
|
512
|
+
const middle = low + (high - low >>> 1);
|
|
513
|
+
const from = table.offsets[middle * stride];
|
|
514
|
+
const to = table.offsets[middle * stride + 1];
|
|
515
|
+
if (from > to || to > table.text.length) break;
|
|
516
|
+
const order = compareTableToUnits(table.text, from, to, units, start, length) || (tokens === void 0 ? 0 : tokens[middle] - token);
|
|
517
|
+
if (order === 0) return middle;
|
|
518
|
+
if (order < 0) low = middle + 1;
|
|
519
|
+
else high = middle;
|
|
520
|
+
}
|
|
521
|
+
if (low === high) return -1;
|
|
522
|
+
for (let at = 0; at < table.counts.length; at++) {
|
|
523
|
+
const from = table.offsets[at * stride];
|
|
524
|
+
const to = table.offsets[at * stride + 1];
|
|
525
|
+
if ((tokens === void 0 || tokens[at] === token) && from <= to && to <= table.text.length) {
|
|
526
|
+
if (compareTableToUnits(table.text, from, to, units, start, length) === 0) return at;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
return -1;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// src/core/frozen-writes/distinct-units.ts
|
|
533
|
+
var EMPTY_SLOT = -1;
|
|
534
|
+
var MISSING_ENTRY = -1;
|
|
535
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
536
|
+
var FNV_PRIME = 16777619;
|
|
537
|
+
function slotCountFor(keys) {
|
|
538
|
+
let slots = 1;
|
|
539
|
+
while (slots < keys * SETTLE_KEY_SLOTS_PER_KEY) slots *= 2;
|
|
540
|
+
return slots;
|
|
541
|
+
}
|
|
542
|
+
var DistinctUnits = class {
|
|
543
|
+
count = 0;
|
|
544
|
+
hits = new Uint32Array(0);
|
|
545
|
+
entries = new Int32Array(0);
|
|
546
|
+
starts = new Uint32Array(0);
|
|
547
|
+
lengths = new Uint32Array(0);
|
|
548
|
+
tags = new Uint32Array(0);
|
|
549
|
+
order = new Uint32Array(0);
|
|
550
|
+
slots = new Int32Array(0);
|
|
551
|
+
mask = 0;
|
|
552
|
+
units = new Uint16Array(0);
|
|
553
|
+
byUnits = (left, right) => compareUnits(this.units, this.starts[left], this.lengths[left], this.starts[right], this.lengths[right]) || this.tags[left] - this.tags[right];
|
|
554
|
+
begin(units, capacity) {
|
|
555
|
+
this.units = units;
|
|
556
|
+
this.count = 0;
|
|
557
|
+
if (this.starts.length < capacity) {
|
|
558
|
+
this.hits = new Uint32Array(capacity);
|
|
559
|
+
this.entries = new Int32Array(capacity);
|
|
560
|
+
this.starts = new Uint32Array(capacity);
|
|
561
|
+
this.lengths = new Uint32Array(capacity);
|
|
562
|
+
this.tags = new Uint32Array(capacity);
|
|
563
|
+
this.order = new Uint32Array(capacity);
|
|
564
|
+
}
|
|
565
|
+
const slots = slotCountFor(capacity);
|
|
566
|
+
if (this.slots.length < slots) this.slots = new Int32Array(slots);
|
|
567
|
+
this.slots.fill(EMPTY_SLOT, 0, slots);
|
|
568
|
+
this.mask = slots - 1;
|
|
569
|
+
}
|
|
570
|
+
add(start, length, tag) {
|
|
571
|
+
const { units, slots, starts, lengths, tags } = this;
|
|
572
|
+
const mask = this.mask;
|
|
573
|
+
let hash = FNV_OFFSET_BASIS;
|
|
574
|
+
for (let at = start; at < start + length; at++) hash = Math.imul(hash ^ units[at], FNV_PRIME);
|
|
575
|
+
let slot = Math.imul(hash ^ tag, FNV_PRIME) & mask;
|
|
576
|
+
let key = slots[slot];
|
|
577
|
+
while (key !== EMPTY_SLOT && !(tags[key] === tag && sameUnits(units, starts[key], lengths[key], start, length))) {
|
|
578
|
+
slot = slot + 1 & mask;
|
|
579
|
+
key = slots[slot];
|
|
580
|
+
}
|
|
581
|
+
if (key === EMPTY_SLOT) {
|
|
582
|
+
key = this.count++;
|
|
583
|
+
slots[slot] = key;
|
|
584
|
+
starts[key] = start;
|
|
585
|
+
lengths[key] = length;
|
|
586
|
+
tags[key] = tag;
|
|
587
|
+
this.hits[key] = 0;
|
|
588
|
+
}
|
|
589
|
+
this.hits[key]++;
|
|
590
|
+
return key;
|
|
591
|
+
}
|
|
592
|
+
resolve(table, stride, tokens) {
|
|
593
|
+
const { count, starts, lengths, tags, entries, units } = this;
|
|
594
|
+
if (!tableAscends(table, stride, tokens)) {
|
|
595
|
+
for (let key = 0; key < count; key++)
|
|
596
|
+
entries[key] = findUnits(table, units, starts[key], lengths[key], stride, tokens, tags[key]);
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const order = this.order.subarray(0, count);
|
|
600
|
+
for (let key = 0; key < count; key++) order[key] = key;
|
|
601
|
+
order.sort(this.byUnits);
|
|
602
|
+
const size = table.counts.length;
|
|
603
|
+
let cursor = 0;
|
|
604
|
+
for (let rank = 0; rank < count; rank++) {
|
|
605
|
+
const key = order[rank];
|
|
606
|
+
const start = starts[key];
|
|
607
|
+
const length = lengths[key];
|
|
608
|
+
const tag = tags[key];
|
|
609
|
+
cursor = lowerBound(table, stride, tokens, units, start, length, tag, cursor, size);
|
|
610
|
+
if (cursor < size && entryOrder(table, stride, tokens, cursor, units, start, length, tag) === 0)
|
|
611
|
+
entries[key] = cursor++;
|
|
612
|
+
else entries[key] = MISSING_ENTRY;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
function lowerBound(table, stride, tokens, units, start, length, tag, cursor, size) {
|
|
617
|
+
if (cursor >= size || entryOrder(table, stride, tokens, cursor, units, start, length, tag) >= 0) return cursor;
|
|
618
|
+
let below = cursor;
|
|
619
|
+
let step = 1;
|
|
620
|
+
let above = cursor + 1;
|
|
621
|
+
while (above < size && entryOrder(table, stride, tokens, above, units, start, length, tag) < 0) {
|
|
622
|
+
below = above;
|
|
623
|
+
step *= 2;
|
|
624
|
+
above = below + step;
|
|
625
|
+
}
|
|
626
|
+
if (above > size) above = size;
|
|
627
|
+
while (above - below > 1) {
|
|
628
|
+
const middle = below + (above - below >>> 1);
|
|
629
|
+
if (entryOrder(table, stride, tokens, middle, units, start, length, tag) < 0) below = middle;
|
|
630
|
+
else above = middle;
|
|
631
|
+
}
|
|
632
|
+
return above;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// src/core/frozen-writes/string-counts.ts
|
|
636
|
+
var SURFACE_STRIDE = 2;
|
|
637
|
+
var STRING_STRIDE = 1;
|
|
638
|
+
var UNTAGGED = 0;
|
|
639
|
+
var NO_TERM_KEY = -1;
|
|
640
|
+
var surfaceKeys = new DistinctUnits();
|
|
641
|
+
var termKeys = new DistinctUnits();
|
|
642
|
+
var tokenTerms = new Int32Array(0);
|
|
643
|
+
function requireUnits(state, start, length) {
|
|
644
|
+
const units = state.pendingHeader[PENDING_UNITS];
|
|
645
|
+
requireSegment(start <= units && length <= units - start);
|
|
646
|
+
}
|
|
647
|
+
function add(counts, entry, amount) {
|
|
648
|
+
counts.set(entry, (counts.get(entry) ?? 0) + amount);
|
|
649
|
+
}
|
|
650
|
+
function addResolved(counts, keys) {
|
|
651
|
+
for (let key = 0; key < keys.count; key++) if (keys.entries[key] >= 0) add(counts, keys.entries[key], keys.hits[key]);
|
|
652
|
+
}
|
|
653
|
+
function countUnorderedSurfaces(changes, surfaces, written) {
|
|
654
|
+
for (let at = 0; at < surfaces.counts.length; at++) {
|
|
655
|
+
const held = written.get(textAt(surfaces, at * 2) ?? "");
|
|
656
|
+
if (held !== void 0) add(changes.surfaces, at, held);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
function countSpellings(changes, segment, state, spellings, count) {
|
|
660
|
+
const units = state.pendingUnits;
|
|
661
|
+
termKeys.resolve(segment.terms, STRING_STRIDE, void 0);
|
|
662
|
+
surfaceKeys.begin(units, count);
|
|
663
|
+
for (let token = 0; token < count; token++) {
|
|
664
|
+
const termKey = tokenTerms[token];
|
|
665
|
+
if (termKey === NO_TERM_KEY) continue;
|
|
666
|
+
const termEntry = termKeys.entries[termKey];
|
|
667
|
+
if (termEntry < 0) continue;
|
|
668
|
+
const at = token * PENDING_TOKEN_WORDS;
|
|
669
|
+
const start = state.pendingTokens[at + 3];
|
|
670
|
+
const length = start === NO_SURFACE ? 0 : state.pendingTokens[at + 4];
|
|
671
|
+
const spellingStart = length === 0 ? state.pendingTokens[at + 1] : start;
|
|
672
|
+
const spellingLength = length === 0 ? state.pendingTokens[at + 2] : length;
|
|
673
|
+
if (!holdsLoneSurrogate(units, spellingStart, spellingLength))
|
|
674
|
+
surfaceKeys.add(spellingStart, spellingLength, termEntry);
|
|
675
|
+
}
|
|
676
|
+
surfaceKeys.resolve(spellings, STRING_STRIDE, segment.spellingTokens);
|
|
677
|
+
addResolved(changes.spellings, surfaceKeys);
|
|
678
|
+
}
|
|
679
|
+
function settleStringCounts(changes, segment, state) {
|
|
680
|
+
const { surfaces, spellings, partialFields } = segment;
|
|
681
|
+
const ordered = surfaces === null || tableAscends(surfaces, SURFACE_STRIDE, void 0);
|
|
682
|
+
let unordered = null;
|
|
683
|
+
const units = state.pendingUnits;
|
|
684
|
+
const count = state.pendingHeader[PENDING_TOKENS];
|
|
685
|
+
surfaceKeys.begin(units, count);
|
|
686
|
+
termKeys.begin(units, count);
|
|
687
|
+
if (tokenTerms.length < count) tokenTerms = new Int32Array(count);
|
|
688
|
+
for (let token = 0; token < count; token++) {
|
|
689
|
+
const at = token * PENDING_TOKEN_WORDS;
|
|
690
|
+
const field = state.pendingTokens[at];
|
|
691
|
+
const termStart = state.pendingTokens[at + 1];
|
|
692
|
+
const termLength = state.pendingTokens[at + 2];
|
|
693
|
+
const start = state.pendingTokens[at + 3];
|
|
694
|
+
const length = start === NO_SURFACE ? 0 : state.pendingTokens[at + 4];
|
|
695
|
+
requireSegment(field < partialFields.length);
|
|
696
|
+
requireUnits(state, termStart, termLength);
|
|
697
|
+
if (start !== NO_SURFACE) requireUnits(state, start, length);
|
|
698
|
+
tokenTerms[token] = NO_TERM_KEY;
|
|
699
|
+
if (surfaces === null && spellings === null) continue;
|
|
700
|
+
if (surfaces !== null && length > 0 && !sameUnits(units, termStart, termLength, start, length)) {
|
|
701
|
+
if (!ordered) {
|
|
702
|
+
unordered ??= /* @__PURE__ */ new Map();
|
|
703
|
+
const text = storedText(units, start, length);
|
|
704
|
+
unordered.set(text, (unordered.get(text) ?? 0) + 1);
|
|
705
|
+
} else if (!holdsLoneSurrogate(units, start, length)) surfaceKeys.add(start, length, UNTAGGED);
|
|
706
|
+
}
|
|
707
|
+
if (spellings === null || partialFields[field] === 0) continue;
|
|
708
|
+
if (!holdsLoneSurrogate(units, termStart, termLength))
|
|
709
|
+
tokenTerms[token] = termKeys.add(termStart, termLength, UNTAGGED);
|
|
710
|
+
}
|
|
711
|
+
if (surfaces !== null) {
|
|
712
|
+
surfaceKeys.resolve(surfaces, SURFACE_STRIDE, void 0);
|
|
713
|
+
addResolved(changes.surfaces, surfaceKeys);
|
|
714
|
+
if (unordered !== null) countUnorderedSurfaces(changes, surfaces, unordered);
|
|
715
|
+
}
|
|
716
|
+
if (spellings !== null) countSpellings(changes, segment, state, spellings, count);
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// src/core/frozen-writes/portable.ts
|
|
720
|
+
var PREPARED_REPLACEMENTS = 5;
|
|
721
|
+
var TOMBSTONE_WORDS = 4;
|
|
722
|
+
var TOMBSTONE_COUNT = 5;
|
|
723
|
+
var PENDING_HEADER = 6;
|
|
724
|
+
function replace(entries, values, at, kind, entry, value) {
|
|
725
|
+
entries[at * 2] = kind;
|
|
726
|
+
entries[at * 2 + 1] = entry;
|
|
727
|
+
values[at] = value;
|
|
728
|
+
}
|
|
729
|
+
function clamped(removed, stored, amount) {
|
|
730
|
+
return Math.min(stored, removed + amount);
|
|
731
|
+
}
|
|
732
|
+
function publishTerms(terms, segment, state, paged) {
|
|
733
|
+
settleTermCounts(terms, segment, state, paged);
|
|
734
|
+
for (const term of terms.touched)
|
|
735
|
+
requireSegment(Number.isFinite(state.removedOccurrences[term]) && state.removedOccurrences[term] >= 0);
|
|
736
|
+
for (const term of terms.touched) {
|
|
737
|
+
state.removedDocuments[term] = clamped(
|
|
738
|
+
state.removedDocuments[term],
|
|
739
|
+
segment.terms.counts[term],
|
|
740
|
+
terms.documents[term]
|
|
741
|
+
);
|
|
742
|
+
state.removedOccurrences[term] += terms.amounts[term];
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
function settle(segment, state, paged) {
|
|
746
|
+
if (pendingSettled(state)) return;
|
|
747
|
+
const changes = { terms: termChangesOf(segment), surfaces: /* @__PURE__ */ new Map(), spellings: /* @__PURE__ */ new Map() };
|
|
748
|
+
settleStringCounts(changes, segment, state);
|
|
749
|
+
requirePendingDocuments(changes.terms, segment, state);
|
|
750
|
+
const trusted = paged === null ? trustedIndexOf(segment, state) : null;
|
|
751
|
+
if (trusted !== null) subtractTrusted(segment, state, trusted);
|
|
752
|
+
else
|
|
753
|
+
try {
|
|
754
|
+
publishTerms(changes.terms, segment, state, paged);
|
|
755
|
+
} finally {
|
|
756
|
+
clearTermChanges(changes.terms);
|
|
757
|
+
}
|
|
758
|
+
if (segment.surfaces !== null)
|
|
759
|
+
for (const [entry, amount] of changes.surfaces)
|
|
760
|
+
state.removedSurfaces[entry] = clamped(state.removedSurfaces[entry], segment.surfaces.counts[entry], amount);
|
|
761
|
+
if (segment.spellings !== null)
|
|
762
|
+
for (const [entry, amount] of changes.spellings)
|
|
763
|
+
state.removedSpellings[entry] = clamped(state.removedSpellings[entry], segment.spellings.counts[entry], amount);
|
|
764
|
+
state.pendingHeader[PENDING_DOCUMENTS] = 0;
|
|
765
|
+
state.pendingHeader[PENDING_TOKENS] = 0;
|
|
766
|
+
state.pendingHeader[PENDING_UNITS] = 0;
|
|
767
|
+
}
|
|
768
|
+
function tombstoned(state, ordinal) {
|
|
769
|
+
return (state.tombstones[ordinal >>> 5] & 1 << (ordinal & 31)) !== 0;
|
|
770
|
+
}
|
|
771
|
+
function makeRoom(request, paged) {
|
|
772
|
+
const { segment, state, fields } = request;
|
|
773
|
+
const needed = pendingNeeds(fields);
|
|
774
|
+
requireRoom(pendingFits(state, needed, true));
|
|
775
|
+
if (!pendingFits(state, needed, false)) settle(segment, state, paged);
|
|
776
|
+
return needed;
|
|
777
|
+
}
|
|
778
|
+
function refusal(error) {
|
|
779
|
+
if (error instanceof SegmentRefusal) return error.status;
|
|
780
|
+
throw error;
|
|
781
|
+
}
|
|
782
|
+
var removeFrozenDocument = (input, output, ordinal, tokens, paged) => {
|
|
783
|
+
try {
|
|
784
|
+
const request = readFrozenRequest(input, output, ordinal, tokens);
|
|
785
|
+
const { segment, state, fields } = request;
|
|
786
|
+
if (tombstoned(state, ordinal)) return 0;
|
|
787
|
+
requireSegment(state.tombstoneCount[0] < segment.documentCount);
|
|
788
|
+
const needed = makeRoom(request, paged);
|
|
789
|
+
appendPending(state, ordinal, fields);
|
|
790
|
+
state.pendingHeader[PENDING_DOCUMENTS] += needed.documents;
|
|
791
|
+
state.pendingHeader[PENDING_TOKENS] += needed.tokens;
|
|
792
|
+
state.pendingHeader[PENDING_UNITS] += needed.units;
|
|
793
|
+
state.tombstones[ordinal >>> 5] |= 1 << (ordinal & 31);
|
|
794
|
+
state.tombstoneCount[0]++;
|
|
795
|
+
return 0;
|
|
796
|
+
} catch (error) {
|
|
797
|
+
return refusal(error);
|
|
798
|
+
}
|
|
799
|
+
};
|
|
800
|
+
var prepareFrozenRemoval = (input, output, ordinal, tokens, rawEntries, rawValues, paged) => {
|
|
801
|
+
try {
|
|
802
|
+
const request = readFrozenRequest(input, output, ordinal, tokens);
|
|
803
|
+
const { entries, values } = request.readOutput(rawEntries, rawValues);
|
|
804
|
+
const { segment, state, fields } = request;
|
|
805
|
+
if (tombstoned(state, ordinal)) return 0;
|
|
806
|
+
requireSegment(state.tombstoneCount[0] < segment.documentCount);
|
|
807
|
+
requireRoom(entries.length / 2 >= PREPARED_REPLACEMENTS && values.length >= PREPARED_REPLACEMENTS);
|
|
808
|
+
const needed = makeRoom(request, paged);
|
|
809
|
+
appendPending(state, ordinal, fields);
|
|
810
|
+
const header = state.pendingHeader;
|
|
811
|
+
replace(
|
|
812
|
+
entries,
|
|
813
|
+
values,
|
|
814
|
+
0,
|
|
815
|
+
TOMBSTONE_WORDS,
|
|
816
|
+
ordinal >>> 5,
|
|
817
|
+
(state.tombstones[ordinal >>> 5] | 1 << (ordinal & 31)) >>> 0
|
|
818
|
+
);
|
|
819
|
+
replace(entries, values, 1, TOMBSTONE_COUNT, 0, state.tombstoneCount[0] + 1);
|
|
820
|
+
replace(entries, values, 2, PENDING_HEADER, PENDING_DOCUMENTS, header[PENDING_DOCUMENTS] + needed.documents);
|
|
821
|
+
replace(entries, values, 3, PENDING_HEADER, PENDING_TOKENS, header[PENDING_TOKENS] + needed.tokens);
|
|
822
|
+
replace(entries, values, 4, PENDING_HEADER, PENDING_UNITS, header[PENDING_UNITS] + needed.units);
|
|
823
|
+
return PREPARED_REPLACEMENTS;
|
|
824
|
+
} catch (error) {
|
|
825
|
+
return -refusal(error);
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
var settleFrozenState = (input, output, paged) => {
|
|
829
|
+
try {
|
|
830
|
+
const { segment, state } = readFrozenSettle(input, output);
|
|
831
|
+
settle(segment, state, paged);
|
|
832
|
+
return 0;
|
|
833
|
+
} catch (error) {
|
|
834
|
+
return refusal(error);
|
|
835
|
+
}
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
// src/core/frozen-writes/removal-scratch.ts
|
|
839
|
+
var NO_SURFACE2 = 4294967295;
|
|
840
|
+
var FIELD_TOKEN_WORDS2 = 8;
|
|
841
|
+
var NO_KEY = -1;
|
|
842
|
+
var ABSENT_TEXT = 4294967295;
|
|
843
|
+
var NO_ENTRY = 4294967295;
|
|
844
|
+
var NO_FIELD = -1;
|
|
845
|
+
var LISTED_TAG = 65535;
|
|
846
|
+
var FNV_OFFSET_BASIS2 = 2166136261;
|
|
847
|
+
var FNV_PRIME2 = 16777619;
|
|
848
|
+
function hashText(text, tag) {
|
|
849
|
+
let hash = FNV_OFFSET_BASIS2;
|
|
850
|
+
for (let at = 0; at < text.length; at++) hash = Math.imul(hash ^ text.charCodeAt(at), FNV_PRIME2);
|
|
851
|
+
return Math.imul(hash ^ tag, FNV_PRIME2);
|
|
852
|
+
}
|
|
853
|
+
function hashUnits(units, start, length, tag) {
|
|
854
|
+
let hash = FNV_OFFSET_BASIS2;
|
|
855
|
+
for (let at = start; at < start + length; at++) hash = Math.imul(hash ^ units[at], FNV_PRIME2);
|
|
856
|
+
return Math.imul(hash ^ tag, FNV_PRIME2);
|
|
857
|
+
}
|
|
858
|
+
function grownUint32(held, used, needed) {
|
|
859
|
+
if (needed <= held.length) return held;
|
|
860
|
+
const grown2 = new Uint32Array(Math.max(held.length * REMOVAL_SCRATCH_GROWTH_FACTOR, needed));
|
|
861
|
+
grown2.set(held.subarray(0, used));
|
|
862
|
+
return grown2;
|
|
863
|
+
}
|
|
864
|
+
function grownUint16(held, used, needed) {
|
|
865
|
+
if (needed <= held.length) return held;
|
|
866
|
+
const grown2 = new Uint16Array(Math.max(held.length * REMOVAL_SCRATCH_GROWTH_FACTOR, needed));
|
|
867
|
+
grown2.set(held.subarray(0, used));
|
|
868
|
+
return grown2;
|
|
869
|
+
}
|
|
870
|
+
var RemovalScratch = class {
|
|
871
|
+
tokens = new Uint32Array(REMOVAL_SCRATCH_START_TOKEN_WORDS);
|
|
872
|
+
units = new Uint16Array(REMOVAL_SCRATCH_START_UNITS);
|
|
873
|
+
tokenWords = 0;
|
|
874
|
+
unitCount = 0;
|
|
875
|
+
field = NO_FIELD;
|
|
876
|
+
fieldStart = 0;
|
|
877
|
+
keyCount = 0;
|
|
878
|
+
keyUnits = new Uint16Array(REMOVAL_SCRATCH_START_KEY_UNITS);
|
|
879
|
+
keyUnitCount = 0;
|
|
880
|
+
keyStarts = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
881
|
+
keyLengths = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
882
|
+
keyTags = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
883
|
+
keyFields = new Int32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
884
|
+
termAt = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
885
|
+
surfaceAt = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
886
|
+
listedAt = new Uint32Array(REMOVAL_SCRATCH_START_KEYS);
|
|
887
|
+
terms = [];
|
|
888
|
+
surfaces = [];
|
|
889
|
+
listed = [];
|
|
890
|
+
listedCount = 0;
|
|
891
|
+
listedCounts = [];
|
|
892
|
+
slots = new Int32Array(REMOVAL_SCRATCH_START_KEYS * REMOVAL_SCRATCH_SLOTS_PER_KEY).fill(NO_KEY);
|
|
893
|
+
reset() {
|
|
894
|
+
this.tokenWords = 0;
|
|
895
|
+
this.unitCount = 0;
|
|
896
|
+
this.field = NO_FIELD;
|
|
897
|
+
if (this.keyCount > 0) this.slots.fill(NO_KEY);
|
|
898
|
+
this.keyCount = 0;
|
|
899
|
+
this.keyUnitCount = 0;
|
|
900
|
+
this.listedCount = 0;
|
|
901
|
+
}
|
|
902
|
+
beginField() {
|
|
903
|
+
this.field++;
|
|
904
|
+
this.fieldStart = this.unitCount;
|
|
905
|
+
}
|
|
906
|
+
fieldUnits() {
|
|
907
|
+
return this.units.subarray(this.fieldStart, this.unitCount);
|
|
908
|
+
}
|
|
909
|
+
writeText(text) {
|
|
910
|
+
const start = this.unitCount;
|
|
911
|
+
this.units = grownUint16(this.units, start, start + text.length);
|
|
912
|
+
const units = this.units;
|
|
913
|
+
for (let at = 0; at < text.length; at++) units[start + at] = text.charCodeAt(at);
|
|
914
|
+
this.unitCount = start + text.length;
|
|
915
|
+
return start - this.fieldStart;
|
|
916
|
+
}
|
|
917
|
+
sameText(key, text, tag) {
|
|
918
|
+
if (this.keyTags[key] !== tag || this.keyLengths[key] !== text.length) return false;
|
|
919
|
+
const start = this.keyStarts[key];
|
|
920
|
+
const units = this.keyUnits;
|
|
921
|
+
for (let at = 0; at < text.length; at++) if (units[start + at] !== text.charCodeAt(at)) return false;
|
|
922
|
+
return true;
|
|
923
|
+
}
|
|
924
|
+
slotOf(text, tag) {
|
|
925
|
+
const slots = this.slots;
|
|
926
|
+
const mask = slots.length - 1;
|
|
927
|
+
let slot = hashText(text, tag) & mask;
|
|
928
|
+
let key = slots[slot];
|
|
929
|
+
while (key !== NO_KEY && !this.sameText(key, text, tag)) {
|
|
930
|
+
slot = slot + 1 & mask;
|
|
931
|
+
key = slots[slot];
|
|
932
|
+
}
|
|
933
|
+
return slot;
|
|
934
|
+
}
|
|
935
|
+
find(text, tag) {
|
|
936
|
+
return this.slots[this.slotOf(text, tag)];
|
|
937
|
+
}
|
|
938
|
+
rehash() {
|
|
939
|
+
const slots = new Int32Array(this.slots.length * REMOVAL_SCRATCH_GROWTH_FACTOR).fill(NO_KEY);
|
|
940
|
+
const mask = slots.length - 1;
|
|
941
|
+
for (let key = 0; key < this.keyCount; key++) {
|
|
942
|
+
let slot = hashUnits(this.keyUnits, this.keyStarts[key], this.keyLengths[key], this.keyTags[key]) & mask;
|
|
943
|
+
while (slots[slot] !== NO_KEY) slot = slot + 1 & mask;
|
|
944
|
+
slots[slot] = key;
|
|
945
|
+
}
|
|
946
|
+
this.slots = slots;
|
|
947
|
+
}
|
|
948
|
+
growKeys() {
|
|
949
|
+
const used = this.keyCount;
|
|
950
|
+
const needed = used + 1;
|
|
951
|
+
if (needed <= this.keyStarts.length) return;
|
|
952
|
+
this.keyStarts = grownUint32(this.keyStarts, used, needed);
|
|
953
|
+
this.keyLengths = grownUint32(this.keyLengths, used, needed);
|
|
954
|
+
this.keyTags = grownUint32(this.keyTags, used, needed);
|
|
955
|
+
this.termAt = grownUint32(this.termAt, used, needed);
|
|
956
|
+
this.surfaceAt = grownUint32(this.surfaceAt, used, needed);
|
|
957
|
+
this.listedAt = grownUint32(this.listedAt, used, needed);
|
|
958
|
+
const fields = new Int32Array(this.keyStarts.length);
|
|
959
|
+
fields.set(this.keyFields.subarray(0, used));
|
|
960
|
+
this.keyFields = fields;
|
|
961
|
+
}
|
|
962
|
+
add(text, tag, term, surface) {
|
|
963
|
+
if ((this.keyCount + 1) * REMOVAL_SCRATCH_SLOTS_PER_KEY > this.slots.length) this.rehash();
|
|
964
|
+
this.growKeys();
|
|
965
|
+
const key = this.keyCount++;
|
|
966
|
+
const start = this.keyUnitCount;
|
|
967
|
+
this.keyUnits = grownUint16(this.keyUnits, start, start + text.length);
|
|
968
|
+
for (let at = 0; at < text.length; at++) this.keyUnits[start + at] = text.charCodeAt(at);
|
|
969
|
+
this.keyUnitCount = start + text.length;
|
|
970
|
+
this.keyStarts[key] = start;
|
|
971
|
+
this.keyLengths[key] = text.length;
|
|
972
|
+
this.keyTags[key] = tag;
|
|
973
|
+
this.keyFields[key] = NO_FIELD;
|
|
974
|
+
this.listedAt[key] = NO_ENTRY;
|
|
975
|
+
this.terms[key] = term;
|
|
976
|
+
this.surfaces[key] = surface;
|
|
977
|
+
this.slots[this.slotOf(text, tag)] = key;
|
|
978
|
+
return key;
|
|
979
|
+
}
|
|
980
|
+
appendListed(spelling) {
|
|
981
|
+
const entry = this.listedCount++;
|
|
982
|
+
this.listedCounts[entry] = 0;
|
|
983
|
+
this.listed[entry] = spelling;
|
|
984
|
+
return entry;
|
|
985
|
+
}
|
|
986
|
+
listSpelling(key, spelling, shared) {
|
|
987
|
+
if (!shared) {
|
|
988
|
+
this.listedAt[key] = this.appendListed(spelling);
|
|
989
|
+
return;
|
|
990
|
+
}
|
|
991
|
+
let listing = this.find(spelling, LISTED_TAG);
|
|
992
|
+
if (listing === NO_KEY) {
|
|
993
|
+
listing = this.add(spelling, LISTED_TAG, null, null);
|
|
994
|
+
this.listedAt[listing] = this.appendListed(spelling);
|
|
995
|
+
}
|
|
996
|
+
this.listedAt[key] = this.listedAt[listing];
|
|
997
|
+
}
|
|
998
|
+
countListed(key) {
|
|
999
|
+
const entry = this.listedAt[key];
|
|
1000
|
+
if (entry !== NO_ENTRY) this.listedCounts[entry]++;
|
|
1001
|
+
}
|
|
1002
|
+
recordsTokens(key) {
|
|
1003
|
+
return this.terms[key] !== null;
|
|
1004
|
+
}
|
|
1005
|
+
writeRecord(position, key) {
|
|
1006
|
+
const term = this.terms[key];
|
|
1007
|
+
if (term === null) return;
|
|
1008
|
+
if (this.keyFields[key] !== this.field) {
|
|
1009
|
+
const surface = this.surfaces[key];
|
|
1010
|
+
this.termAt[key] = this.writeText(term);
|
|
1011
|
+
this.surfaceAt[key] = surface === null ? NO_SURFACE2 : this.writeText(surface);
|
|
1012
|
+
this.keyFields[key] = this.field;
|
|
1013
|
+
}
|
|
1014
|
+
const at = this.tokenWords;
|
|
1015
|
+
this.tokens = grownUint32(this.tokens, at, at + FIELD_TOKEN_WORDS2);
|
|
1016
|
+
const tokens = this.tokens;
|
|
1017
|
+
const surfaceAt = this.surfaceAt[key];
|
|
1018
|
+
tokens[at] = position;
|
|
1019
|
+
tokens[at + 1] = this.termAt[key];
|
|
1020
|
+
tokens[at + 2] = term.length;
|
|
1021
|
+
tokens[at + 3] = surfaceAt;
|
|
1022
|
+
tokens[at + 4] = surfaceAt === NO_SURFACE2 ? 0 : this.surfaces[key]?.length ?? 0;
|
|
1023
|
+
tokens[at + 5] = 0;
|
|
1024
|
+
tokens[at + 6] = 0;
|
|
1025
|
+
tokens[at + 7] = ABSENT_TEXT;
|
|
1026
|
+
this.tokenWords = at + FIELD_TOKEN_WORDS2;
|
|
1027
|
+
}
|
|
1028
|
+
copyRecord(source, at) {
|
|
1029
|
+
this.tokens = grownUint32(this.tokens, this.tokenWords, this.tokenWords + FIELD_TOKEN_WORDS2);
|
|
1030
|
+
this.tokens.set(source.subarray(at, at + FIELD_TOKEN_WORDS2), this.tokenWords);
|
|
1031
|
+
this.tokenWords += FIELD_TOKEN_WORDS2;
|
|
1032
|
+
}
|
|
1033
|
+
removedSpellings() {
|
|
1034
|
+
const size = this.listedCount;
|
|
1035
|
+
if (size === 0) return { spellings: [], counts: [] };
|
|
1036
|
+
return { spellings: this.listed.slice(0, size), counts: this.listedCounts.slice(0, size) };
|
|
1037
|
+
}
|
|
1038
|
+
};
|
|
1039
|
+
|
|
1040
|
+
// src/core/frozen-writes/removal-strings.ts
|
|
1041
|
+
var SURFACE_TAG = 0;
|
|
1042
|
+
var PARTIAL_TAG = 1;
|
|
1043
|
+
var NO_SPELLING_TOKENS = new Uint32Array(0);
|
|
1044
|
+
var NO_REMOVED_PARTIAL_SPELLINGS = new Uint32Array(0);
|
|
1045
|
+
var scratch = new RemovalScratch();
|
|
1046
|
+
var visitedFields = [""];
|
|
1047
|
+
var visitedTag = SURFACE_TAG;
|
|
1048
|
+
var visitedPosition = 0;
|
|
1049
|
+
var sharedListing = false;
|
|
1050
|
+
var visitedSurfaces = null;
|
|
1051
|
+
var visitedPartialWords = null;
|
|
1052
|
+
var normalizingLanguage = null;
|
|
1053
|
+
var normalizingStopWords = null;
|
|
1054
|
+
var flaggedFieldNames = null;
|
|
1055
|
+
var flaggedPartialFields;
|
|
1056
|
+
var heldPartialFlags = new Uint8Array(0);
|
|
1057
|
+
function normalizedOf(spelling) {
|
|
1058
|
+
if (normalizingLanguage === null || normalizingStopWords?.has(spelling) === true) return null;
|
|
1059
|
+
return transformToken(spelling, normalizingLanguage, false, false);
|
|
1060
|
+
}
|
|
1061
|
+
function resolveSpelling(written, surfaceToken) {
|
|
1062
|
+
const partial = visitedTag === PARTIAL_TAG;
|
|
1063
|
+
let spelling = written;
|
|
1064
|
+
let token = partial ? visitedPartialWords?.tokenForSpelling(written) : surfaceToken;
|
|
1065
|
+
if (token === void 0 && partial) {
|
|
1066
|
+
const normalized = normalizedOf(written);
|
|
1067
|
+
if (normalized !== null) {
|
|
1068
|
+
token = visitedPartialWords?.tokenForSpelling(normalized);
|
|
1069
|
+
if (token !== void 0) spelling = normalized;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
token ??= visitedSurfaces?.tokenOf(written);
|
|
1073
|
+
const differs = token !== void 0 && spelling !== token && spelling.length > 0;
|
|
1074
|
+
const records = token !== void 0 && (partial || differs);
|
|
1075
|
+
const key = scratch.add(
|
|
1076
|
+
written,
|
|
1077
|
+
visitedTag,
|
|
1078
|
+
records && token !== void 0 ? token : null,
|
|
1079
|
+
records && spelling !== token ? spelling : null
|
|
1080
|
+
);
|
|
1081
|
+
if (differs) scratch.listSpelling(key, spelling, sharedListing);
|
|
1082
|
+
return key;
|
|
1083
|
+
}
|
|
1084
|
+
function visitSpelling(spelling) {
|
|
1085
|
+
const surfaceToken = visitedTag === SURFACE_TAG ? visitedSurfaces?.tokenOf(spelling) : void 0;
|
|
1086
|
+
if (visitedTag === SURFACE_TAG && (surfaceToken === void 0 || surfaceToken === spelling)) return;
|
|
1087
|
+
let key = scratch.find(spelling, visitedTag);
|
|
1088
|
+
if (key === NO_KEY) key = resolveSpelling(spelling, surfaceToken);
|
|
1089
|
+
scratch.countListed(key);
|
|
1090
|
+
if (scratch.recordsTokens(key)) scratch.writeRecord(visitedPosition++, key);
|
|
1091
|
+
}
|
|
1092
|
+
function visitedRecords(field, document, analysis, isPartial) {
|
|
1093
|
+
const firstWord = scratch.tokenWords;
|
|
1094
|
+
scratch.beginField();
|
|
1095
|
+
visitedTag = isPartial ? PARTIAL_TAG : SURFACE_TAG;
|
|
1096
|
+
visitedPosition = 0;
|
|
1097
|
+
visitDocumentSurfaceSpellings(document, analysis, visitSpelling);
|
|
1098
|
+
if (scratch.tokenWords === firstWord) return null;
|
|
1099
|
+
return { field, tokens: scratch.tokens.subarray(firstWord, scratch.tokenWords), units: scratch.fieldUnits() };
|
|
1100
|
+
}
|
|
1101
|
+
function preparedRecords(field, isPartial, { tokens, units }) {
|
|
1102
|
+
const firstWord = scratch.tokenWords;
|
|
1103
|
+
for (let at = 0; at < tokens.length; at += FIELD_TOKEN_WORDS2) {
|
|
1104
|
+
const surface = tokens[at + 3] === NO_SURFACE2 || tokens[at + 4] === 0 ? "" : storedText(units, tokens[at + 3], tokens[at + 4]);
|
|
1105
|
+
const differs = surface.length > 0 && surface !== storedText(units, tokens[at + 1], tokens[at + 2]);
|
|
1106
|
+
if (differs) {
|
|
1107
|
+
let key = scratch.find(surface, SURFACE_TAG);
|
|
1108
|
+
if (key === NO_KEY) {
|
|
1109
|
+
key = scratch.add(surface, SURFACE_TAG, null, null);
|
|
1110
|
+
scratch.listSpelling(key, surface, true);
|
|
1111
|
+
}
|
|
1112
|
+
scratch.countListed(key);
|
|
1113
|
+
}
|
|
1114
|
+
if (isPartial || differs) scratch.copyRecord(tokens, at);
|
|
1115
|
+
}
|
|
1116
|
+
if (scratch.tokenWords === firstWord) return null;
|
|
1117
|
+
return { field, tokens: scratch.tokens.subarray(firstWord, scratch.tokenWords), units };
|
|
1118
|
+
}
|
|
1119
|
+
function beginVisit(surfaces, partialWords, analysis, partialFields) {
|
|
1120
|
+
scratch.reset();
|
|
1121
|
+
visitedSurfaces = surfaces;
|
|
1122
|
+
visitedPartialWords = partialWords;
|
|
1123
|
+
sharedListing = partialFields !== void 0;
|
|
1124
|
+
const { language, options } = analysis;
|
|
1125
|
+
const normalizes = partialFields !== void 0 && options?.customTokenizer === void 0 && language.normalizer !== void 0 && language.tokenizer?.normalizeDiacritics !== true;
|
|
1126
|
+
normalizingLanguage = normalizes ? language : null;
|
|
1127
|
+
normalizingStopWords = normalizes ? resolveStopWords(language, options?.stopWordOverride) : null;
|
|
1128
|
+
}
|
|
1129
|
+
function endVisit() {
|
|
1130
|
+
visitedSurfaces = null;
|
|
1131
|
+
visitedPartialWords = null;
|
|
1132
|
+
normalizingLanguage = null;
|
|
1133
|
+
normalizingStopWords = null;
|
|
1134
|
+
}
|
|
1135
|
+
function partialFlagsOf(fieldNames, partialFields) {
|
|
1136
|
+
if (fieldNames === flaggedFieldNames && partialFields === flaggedPartialFields) return heldPartialFlags;
|
|
1137
|
+
const flags = new Uint8Array(fieldNames.length);
|
|
1138
|
+
for (const path of partialFields ?? []) {
|
|
1139
|
+
const field = fieldNames.indexOf(path);
|
|
1140
|
+
if (field >= 0) flags[field] = 1;
|
|
1141
|
+
}
|
|
1142
|
+
flaggedFieldNames = fieldNames;
|
|
1143
|
+
flaggedPartialFields = partialFields;
|
|
1144
|
+
heldPartialFlags = flags;
|
|
1145
|
+
return flags;
|
|
1146
|
+
}
|
|
1147
|
+
function collectRecords(fieldNames, textFields, document, analysis, surfaces, partialFields) {
|
|
1148
|
+
const fields = [];
|
|
1149
|
+
const custom = analysis.options?.customTokenizer !== void 0;
|
|
1150
|
+
if (!custom && partialFields === void 0 && !surfaces.holdsAny()) return fields;
|
|
1151
|
+
const visiting = {
|
|
1152
|
+
language: analysis.language,
|
|
1153
|
+
options: analysis.options,
|
|
1154
|
+
fields: visitedFields
|
|
1155
|
+
};
|
|
1156
|
+
for (let field = 0; field < fieldNames.length; field++) {
|
|
1157
|
+
const name = fieldNames[field];
|
|
1158
|
+
const isPartial = partialFields?.includes(name) === true;
|
|
1159
|
+
let records = null;
|
|
1160
|
+
if (custom) {
|
|
1161
|
+
const prepared = analysis.text?.fields.get(name);
|
|
1162
|
+
if (prepared !== void 0) records = preparedRecords(field, isPartial, packedTokens(prepared.records));
|
|
1163
|
+
} else if (textFields.includes(name)) {
|
|
1164
|
+
visitedFields[0] = name;
|
|
1165
|
+
records = visitedRecords(field, document, visiting, isPartial);
|
|
1166
|
+
}
|
|
1167
|
+
if (records !== null) fields.push(records);
|
|
1168
|
+
}
|
|
1169
|
+
return fields;
|
|
1170
|
+
}
|
|
1171
|
+
function prepareFrozenStrings(fieldNames, textFields, ordinal, document, analysis, surfaces, partialWords) {
|
|
1172
|
+
const partial = partialWords.removalState();
|
|
1173
|
+
const partialFields = partial?.table.fields;
|
|
1174
|
+
beginVisit(surfaces, partialWords, analysis, partialFields);
|
|
1175
|
+
let fields;
|
|
1176
|
+
try {
|
|
1177
|
+
fields = collectRecords(fieldNames, textFields, document, analysis, surfaces, partialFields);
|
|
1178
|
+
} finally {
|
|
1179
|
+
endVisit();
|
|
1180
|
+
}
|
|
1181
|
+
const spellings = scratch.removedSpellings();
|
|
1182
|
+
const surface = surfaces.prepareCoreRemoval(ordinal, spellings);
|
|
1183
|
+
const partialFlags = partialFlagsOf(fieldNames, partialFields);
|
|
1184
|
+
return {
|
|
1185
|
+
spellings,
|
|
1186
|
+
core: {
|
|
1187
|
+
fields,
|
|
1188
|
+
partialFields: partialFlags,
|
|
1189
|
+
surfaces: surface.table,
|
|
1190
|
+
removedSurfaces: surface.removed,
|
|
1191
|
+
spellings: partial === null ? null : {
|
|
1192
|
+
text: partial.table.spellings,
|
|
1193
|
+
offsets: partial.table.spellingOffsets,
|
|
1194
|
+
counts: partial.table.spellingCounts
|
|
1195
|
+
},
|
|
1196
|
+
spellingTokens: partial?.table.spellingTokens ?? NO_SPELLING_TOKENS,
|
|
1197
|
+
removedSpellings: partial?.removed ?? NO_REMOVED_PARTIAL_SPELLINGS
|
|
1198
|
+
},
|
|
1199
|
+
publish: surface.publish
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
// src/core/segment-writes/removal-host.browser.ts
|
|
1204
|
+
var POSTING_LIST_COMPACTION_THRESHOLD;
|
|
1205
|
+
var SegmentRefusal2;
|
|
1206
|
+
var StringPlan;
|
|
1207
|
+
var acquireTokenPlan;
|
|
1208
|
+
var findString;
|
|
1209
|
+
var holdsUnpairedSurrogate;
|
|
1210
|
+
var prepareSpellingRemoval;
|
|
1211
|
+
var preparedSources;
|
|
1212
|
+
var readSegmentView;
|
|
1213
|
+
var releaseSlice;
|
|
1214
|
+
var releaseTokenPlan;
|
|
1215
|
+
var removeString;
|
|
1216
|
+
var requireSegment2;
|
|
1217
|
+
var rowRemoved;
|
|
1218
|
+
var sharedTexts;
|
|
1219
|
+
var stringHash;
|
|
1220
|
+
var trimClass;
|
|
1221
|
+
var trimSlice;
|
|
1222
|
+
function bindSegmentRemovalHost(host) {
|
|
1223
|
+
({
|
|
1224
|
+
POSTING_LIST_COMPACTION_THRESHOLD,
|
|
1225
|
+
SegmentRefusal: SegmentRefusal2,
|
|
1226
|
+
StringPlan,
|
|
1227
|
+
acquireTokenPlan,
|
|
1228
|
+
findString,
|
|
1229
|
+
holdsUnpairedSurrogate,
|
|
1230
|
+
prepareSpellingRemoval,
|
|
1231
|
+
preparedSources,
|
|
1232
|
+
readSegmentView,
|
|
1233
|
+
releaseSlice,
|
|
1234
|
+
releaseTokenPlan,
|
|
1235
|
+
removeString,
|
|
1236
|
+
requireSegment: requireSegment2,
|
|
1237
|
+
rowRemoved,
|
|
1238
|
+
sharedTexts,
|
|
1239
|
+
stringHash,
|
|
1240
|
+
trimClass,
|
|
1241
|
+
trimSlice
|
|
1242
|
+
} = host);
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
// src/core/segment-writes/document-removal.ts
|
|
1246
|
+
var PREPARED_ROW_WORDS = 3;
|
|
1247
|
+
var SURFACE_WORD = 1;
|
|
1248
|
+
var ABSENT_TEXT2 = 4294967295;
|
|
1249
|
+
var LAST_MARK = 4294967295;
|
|
1250
|
+
var ABSENT = -1;
|
|
1251
|
+
var DocumentRemoval = class {
|
|
1252
|
+
size = 0;
|
|
1253
|
+
ordinals = new Uint32Array(0);
|
|
1254
|
+
hashes = new Uint32Array(0);
|
|
1255
|
+
texts = [];
|
|
1256
|
+
surfaces = null;
|
|
1257
|
+
termMarks = new Uint32Array(0);
|
|
1258
|
+
termAt = new Int32Array(0);
|
|
1259
|
+
surfaceMarks = new Uint32Array(0);
|
|
1260
|
+
surfaceAt = new Int32Array(0);
|
|
1261
|
+
mark = 0;
|
|
1262
|
+
surfacePlan = null;
|
|
1263
|
+
collect(segment, texts, fields) {
|
|
1264
|
+
this.reserve(texts.length);
|
|
1265
|
+
this.size = 0;
|
|
1266
|
+
this.texts.length = 0;
|
|
1267
|
+
const mark = this.takeMark();
|
|
1268
|
+
const table = segment.surfaces;
|
|
1269
|
+
const surfaces = table === null ? null : this.surfacePlanFor(table.strings);
|
|
1270
|
+
this.surfaces = surfaces;
|
|
1271
|
+
const termMarks = this.termMarks;
|
|
1272
|
+
const termAt = this.termAt;
|
|
1273
|
+
for (const { text } of fields) {
|
|
1274
|
+
const rows = text.rows;
|
|
1275
|
+
const end = text.count * PREPARED_ROW_WORDS;
|
|
1276
|
+
for (let row = 0; row < end; row += PREPARED_ROW_WORDS) {
|
|
1277
|
+
const termId = rows[row];
|
|
1278
|
+
if (termMarks[termId] !== mark) {
|
|
1279
|
+
termMarks[termId] = mark;
|
|
1280
|
+
termAt[termId] = this.addTerm(segment.terms, texts[termId]);
|
|
1281
|
+
}
|
|
1282
|
+
if (surfaces === null || table === null || termAt[termId] === ABSENT) continue;
|
|
1283
|
+
const surfaceId = rows[row + SURFACE_WORD];
|
|
1284
|
+
if (surfaceId !== ABSENT_TEXT2 && surfaceId !== termId)
|
|
1285
|
+
this.countSurface(table, surfaces, texts, surfaceId, termId, mark);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
addTerm(terms, text) {
|
|
1290
|
+
const hash = stringHash(text);
|
|
1291
|
+
const found = findString(terms, text, hash);
|
|
1292
|
+
if (found < 0) return ABSENT;
|
|
1293
|
+
const at = this.size++;
|
|
1294
|
+
this.ordinals[at] = found;
|
|
1295
|
+
this.hashes[at] = hash;
|
|
1296
|
+
this.texts.push(text);
|
|
1297
|
+
return at;
|
|
1298
|
+
}
|
|
1299
|
+
countSurface(table, surfaces, texts, surfaceId, termId, mark) {
|
|
1300
|
+
const surface = texts[surfaceId];
|
|
1301
|
+
if (surface === "" || surface === texts[termId]) return;
|
|
1302
|
+
if (this.surfaceMarks[surfaceId] !== mark) {
|
|
1303
|
+
this.surfaceMarks[surfaceId] = mark;
|
|
1304
|
+
this.surfaceAt[surfaceId] = surfaces.prepareExisting(surface, false);
|
|
1305
|
+
}
|
|
1306
|
+
const at = this.surfaceAt[surfaceId];
|
|
1307
|
+
if (at < 0) return;
|
|
1308
|
+
if (surfaces.counts[at] === 0) {
|
|
1309
|
+
surfaces.tokens[at] = table.token[surfaces.ordinals[at]];
|
|
1310
|
+
surfaces.points[at] = 0;
|
|
1311
|
+
}
|
|
1312
|
+
surfaces.counts[at]++;
|
|
1313
|
+
}
|
|
1314
|
+
surfacePlanFor(strings) {
|
|
1315
|
+
const plan = this.surfacePlan === null ? new StringPlan(strings) : this.surfacePlan.reset(strings);
|
|
1316
|
+
this.surfacePlan = plan;
|
|
1317
|
+
return plan;
|
|
1318
|
+
}
|
|
1319
|
+
takeMark() {
|
|
1320
|
+
if (this.mark === LAST_MARK) {
|
|
1321
|
+
this.termMarks.fill(0);
|
|
1322
|
+
this.surfaceMarks.fill(0);
|
|
1323
|
+
this.mark = 0;
|
|
1324
|
+
}
|
|
1325
|
+
return ++this.mark;
|
|
1326
|
+
}
|
|
1327
|
+
reserve(texts) {
|
|
1328
|
+
if (texts <= this.termMarks.length) return;
|
|
1329
|
+
const capacity = Math.max(texts, this.termMarks.length * 2);
|
|
1330
|
+
this.ordinals = new Uint32Array(capacity);
|
|
1331
|
+
this.hashes = new Uint32Array(capacity);
|
|
1332
|
+
this.termMarks = new Uint32Array(capacity);
|
|
1333
|
+
this.termAt = new Int32Array(capacity);
|
|
1334
|
+
this.surfaceMarks = new Uint32Array(capacity);
|
|
1335
|
+
this.surfaceAt = new Int32Array(capacity);
|
|
1336
|
+
}
|
|
1337
|
+
};
|
|
1338
|
+
|
|
1339
|
+
// src/core/segment-writes/removal.ts
|
|
1340
|
+
var NO_SLICE_CLASS = 255;
|
|
1341
|
+
var ORDERED_FLAG = 2;
|
|
1342
|
+
var POSITIONS_FLAG = 4;
|
|
1343
|
+
function firstRow(docIds, start, length, ordered, document) {
|
|
1344
|
+
let low = start;
|
|
1345
|
+
if (!ordered) return low;
|
|
1346
|
+
let high = low + length;
|
|
1347
|
+
while (low < high) {
|
|
1348
|
+
const middle = low + (high - low >>> 1);
|
|
1349
|
+
if (docIds[middle] < document) low = middle + 1;
|
|
1350
|
+
else high = middle;
|
|
1351
|
+
}
|
|
1352
|
+
return low;
|
|
1353
|
+
}
|
|
1354
|
+
var RemovalPlan = class {
|
|
1355
|
+
found = new Uint8Array(0);
|
|
1356
|
+
compacts = new Uint8Array(0);
|
|
1357
|
+
ordered = new Uint8Array(0);
|
|
1358
|
+
sliceClasses = new Uint8Array(0);
|
|
1359
|
+
kept = new Uint32Array(0);
|
|
1360
|
+
removedFrequencies = new Float64Array(0);
|
|
1361
|
+
freedPositions = 0;
|
|
1362
|
+
freedRows = 0;
|
|
1363
|
+
prepare(segment, document, terms) {
|
|
1364
|
+
this.reserve(terms.size);
|
|
1365
|
+
const allocation = segment.postings.allocation;
|
|
1366
|
+
requireSegment2(allocation[0] <= segment.postings.docIds.length && allocation[1] <= allocation[0]);
|
|
1367
|
+
this.freedRows = allocation[1];
|
|
1368
|
+
this.freedPositions = 0;
|
|
1369
|
+
const positions = segment.positions;
|
|
1370
|
+
if (positions !== null)
|
|
1371
|
+
requireSegment2(
|
|
1372
|
+
positions.allocation[0] > 0 && positions.allocation[0] <= positions.values.length && positions.allocation[1] < positions.allocation[0]
|
|
1373
|
+
);
|
|
1374
|
+
for (let at = 0; at < terms.size; at++) this.prepareTerm(segment, document, at, terms.ordinals[at]);
|
|
1375
|
+
if (positions !== null) requireSegment2(this.freedPositions <= positions.allocation[0] - positions.allocation[1]);
|
|
1376
|
+
}
|
|
1377
|
+
prepareTerm(segment, document, at, token) {
|
|
1378
|
+
const { columns, postings } = segment;
|
|
1379
|
+
const start = columns.sliceStart[token];
|
|
1380
|
+
const length = columns.lengths[token];
|
|
1381
|
+
const kind = columns.sliceClass[token];
|
|
1382
|
+
const end = postings.allocation[0];
|
|
1383
|
+
this.found[at] = 0;
|
|
1384
|
+
this.compacts[at] = 0;
|
|
1385
|
+
this.removedFrequencies[at] = 0;
|
|
1386
|
+
const docFrequency = columns.docFrequency[token];
|
|
1387
|
+
const totalTermFrequency = columns.totalTermFrequency[token];
|
|
1388
|
+
const surfaceOccurrences = columns.surfaceOccurrences[token];
|
|
1389
|
+
requireSegment2(
|
|
1390
|
+
docFrequency >= columns.deadDocuments[token] && docFrequency <= length && Number.isFinite(totalTermFrequency) && totalTermFrequency >= 0 && Number.isFinite(surfaceOccurrences) && surfaceOccurrences >= 0 && start <= end
|
|
1391
|
+
);
|
|
1392
|
+
if (kind === NO_SLICE_CLASS && length === 0) return;
|
|
1393
|
+
requireSegment2(
|
|
1394
|
+
kind < postings.sliceCapacities.length && postings.sliceCapacities[kind] <= end - start && length <= postings.sliceCapacities[kind]
|
|
1395
|
+
);
|
|
1396
|
+
const ordered = (columns.flags[token] & ORDERED_FLAG) !== 0;
|
|
1397
|
+
const { docIds, fieldIndices, frequencies } = postings;
|
|
1398
|
+
const ordinalCapacity = segment.ordinalCapacity;
|
|
1399
|
+
const fieldCount = segment.partialFields.length;
|
|
1400
|
+
let removedFrequency = 0;
|
|
1401
|
+
let found = false;
|
|
1402
|
+
for (let row = firstRow(docIds, start, length, ordered, document); row < start + length; row++) {
|
|
1403
|
+
const doc = docIds[row];
|
|
1404
|
+
if (ordered && doc > document) break;
|
|
1405
|
+
requireSegment2(doc < ordinalCapacity && fieldIndices[row] < fieldCount);
|
|
1406
|
+
if (doc === document) {
|
|
1407
|
+
found = true;
|
|
1408
|
+
removedFrequency += frequencies[row];
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
if (!found) return;
|
|
1412
|
+
this.found[at] = 1;
|
|
1413
|
+
this.removedFrequencies[at] = removedFrequency;
|
|
1414
|
+
const dead = columns.deadDocuments[token];
|
|
1415
|
+
const live = docFrequency - dead;
|
|
1416
|
+
requireSegment2(live > 0);
|
|
1417
|
+
if (live !== 1 && (dead + 1) / length <= POSTING_LIST_COMPACTION_THRESHOLD) return;
|
|
1418
|
+
this.compacts[at] = 1;
|
|
1419
|
+
this.prepareCompaction(segment, document, at, token, live);
|
|
1420
|
+
}
|
|
1421
|
+
prepareCompaction(segment, document, at, token, live) {
|
|
1422
|
+
const { columns, postings } = segment;
|
|
1423
|
+
const start = columns.sliceStart[token];
|
|
1424
|
+
const length = columns.lengths[token];
|
|
1425
|
+
const kind = columns.sliceClass[token];
|
|
1426
|
+
let kept = 0;
|
|
1427
|
+
let last = 0;
|
|
1428
|
+
let ordered = 1;
|
|
1429
|
+
for (let row = start; row < start + length; row++) {
|
|
1430
|
+
const doc = postings.docIds[row];
|
|
1431
|
+
requireSegment2(doc < segment.ordinalCapacity && postings.fieldIndices[row] < segment.partialFields.length);
|
|
1432
|
+
if (doc === document || rowRemoved(segment, doc)) this.preparePositions(segment, token, row);
|
|
1433
|
+
else {
|
|
1434
|
+
if (kept > 0 && doc < last) ordered = 0;
|
|
1435
|
+
last = doc;
|
|
1436
|
+
kept++;
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
requireSegment2(live === 1 === (kept === 0) && live - 1 <= kept);
|
|
1440
|
+
const capacities = postings.sliceCapacities;
|
|
1441
|
+
const sliceClass = kept > 0 ? trimClass(capacities, kind, kept) : kind;
|
|
1442
|
+
const retained = kept === 0 ? 0 : capacities[sliceClass];
|
|
1443
|
+
const released = capacities[kind] - retained;
|
|
1444
|
+
requireSegment2(released <= postings.allocation[0] - this.freedRows);
|
|
1445
|
+
this.freedRows += released;
|
|
1446
|
+
this.kept[at] = kept;
|
|
1447
|
+
this.ordered[at] = ordered;
|
|
1448
|
+
this.sliceClasses[at] = sliceClass;
|
|
1449
|
+
}
|
|
1450
|
+
preparePositions(segment, token, row) {
|
|
1451
|
+
const { positions, postings, columns } = segment;
|
|
1452
|
+
if (positions === null || postings.positionStarts === null || (columns.flags[token] & POSITIONS_FLAG) === 0) return;
|
|
1453
|
+
const start = postings.positionStarts[row];
|
|
1454
|
+
if (start === 0) return;
|
|
1455
|
+
const end = positions.allocation[0];
|
|
1456
|
+
requireSegment2(start < end && positions.values[start] < end - start);
|
|
1457
|
+
const count = positions.values[start] + 1;
|
|
1458
|
+
requireSegment2(count <= 4294967295 - this.freedPositions);
|
|
1459
|
+
this.freedPositions += count;
|
|
1460
|
+
}
|
|
1461
|
+
publish(segment, document, terms) {
|
|
1462
|
+
const columns = segment.columns;
|
|
1463
|
+
for (let at = 0; at < terms.size; at++) {
|
|
1464
|
+
if (this.found[at] === 0) continue;
|
|
1465
|
+
const token = terms.ordinals[at];
|
|
1466
|
+
columns.revision[token]++;
|
|
1467
|
+
columns.deadDocuments[token]++;
|
|
1468
|
+
columns.totalTermFrequency[token] = Math.max(0, columns.totalTermFrequency[token] - this.removedFrequencies[at]);
|
|
1469
|
+
if (this.compacts[at] === 1) this.publishCompaction(segment, document, terms, at, token);
|
|
1470
|
+
}
|
|
1471
|
+
if (segment.positions !== null) segment.positions.allocation[1] += this.freedPositions;
|
|
1472
|
+
}
|
|
1473
|
+
publishCompaction(segment, document, terms, at, token) {
|
|
1474
|
+
const { columns, postings } = segment;
|
|
1475
|
+
this.compactRows(segment, document, at, token);
|
|
1476
|
+
const kept = this.kept[at];
|
|
1477
|
+
const sliceClass = this.sliceClasses[at];
|
|
1478
|
+
const kind = columns.sliceClass[token];
|
|
1479
|
+
if (kept > 0 && kind > sliceClass) {
|
|
1480
|
+
const pool2 = { ...postings, values: postings.docIds };
|
|
1481
|
+
trimSlice(
|
|
1482
|
+
pool2,
|
|
1483
|
+
{ start: columns.sliceStart[token], length: pool2.sliceCapacities[kind], sliceClass: kind },
|
|
1484
|
+
sliceClass
|
|
1485
|
+
);
|
|
1486
|
+
columns.sliceClass[token] = sliceClass;
|
|
1487
|
+
}
|
|
1488
|
+
if (kept > 0) return;
|
|
1489
|
+
const pool = { ...postings, values: postings.docIds };
|
|
1490
|
+
releaseSlice(pool, { start: columns.sliceStart[token], length: pool.sliceCapacities[kind], sliceClass: kind });
|
|
1491
|
+
removeString(segment.terms, terms.texts[at], token, terms.hashes[at]);
|
|
1492
|
+
columns.sliceClass[token] = NO_SLICE_CLASS;
|
|
1493
|
+
columns.flags[token] = 0;
|
|
1494
|
+
columns.totalTermFrequency[token] = 0;
|
|
1495
|
+
}
|
|
1496
|
+
compactRows(segment, document, at, token) {
|
|
1497
|
+
const { columns, postings } = segment;
|
|
1498
|
+
const start = columns.sliceStart[token];
|
|
1499
|
+
let into = start;
|
|
1500
|
+
for (let row = start; row < start + columns.lengths[token]; row++) {
|
|
1501
|
+
const doc = postings.docIds[row];
|
|
1502
|
+
if (doc === document || rowRemoved(segment, doc)) continue;
|
|
1503
|
+
postings.docIds[into] = doc;
|
|
1504
|
+
postings.frequencies[into] = postings.frequencies[row];
|
|
1505
|
+
postings.fieldIndices[into] = postings.fieldIndices[row];
|
|
1506
|
+
if (postings.positionStarts !== null) postings.positionStarts[into] = postings.positionStarts[row];
|
|
1507
|
+
into++;
|
|
1508
|
+
}
|
|
1509
|
+
columns.lengths[token] = this.kept[at];
|
|
1510
|
+
columns.docFrequency[token] -= columns.deadDocuments[token];
|
|
1511
|
+
columns.deadDocuments[token] = 0;
|
|
1512
|
+
columns.flags[token] = this.ordered[at] === 1 ? columns.flags[token] | ORDERED_FLAG : columns.flags[token] & ~ORDERED_FLAG;
|
|
1513
|
+
}
|
|
1514
|
+
reserve(size) {
|
|
1515
|
+
if (size <= this.found.length) return;
|
|
1516
|
+
const capacity = Math.max(size, this.found.length * 2);
|
|
1517
|
+
this.found = new Uint8Array(capacity);
|
|
1518
|
+
this.compacts = new Uint8Array(capacity);
|
|
1519
|
+
this.ordered = new Uint8Array(capacity);
|
|
1520
|
+
this.sliceClasses = new Uint8Array(capacity);
|
|
1521
|
+
this.kept = new Uint32Array(capacity);
|
|
1522
|
+
this.removedFrequencies = new Float64Array(capacity);
|
|
1523
|
+
}
|
|
1524
|
+
};
|
|
1525
|
+
|
|
1526
|
+
// src/core/segment-writes/remove.ts
|
|
1527
|
+
var removalPlan = new RemovalPlan();
|
|
1528
|
+
var documentRemoval = new DocumentRemoval();
|
|
1529
|
+
function nonnegative(value) {
|
|
1530
|
+
return value > 0 ? value : 0;
|
|
1531
|
+
}
|
|
1532
|
+
function requireOrdinal(segment, ordinal) {
|
|
1533
|
+
requireSegment2(Number.isInteger(ordinal) && ordinal >= 0 && ordinal < segment.ordinalCapacity);
|
|
1534
|
+
}
|
|
1535
|
+
function requireFields(segment, fields) {
|
|
1536
|
+
const fieldCount = segment.partialFields.length;
|
|
1537
|
+
requireSegment2(fields.length <= fieldCount);
|
|
1538
|
+
let previous = -1;
|
|
1539
|
+
for (const { field } of fields) {
|
|
1540
|
+
requireSegment2(Number.isInteger(field) && field > previous && field < fieldCount);
|
|
1541
|
+
previous = field;
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
function removeSurfaces(plan, table, segment) {
|
|
1545
|
+
for (let at = 0; at < plan.size; at++) {
|
|
1546
|
+
const ordinal = plan.ordinals[at];
|
|
1547
|
+
const token = plan.tokens[at];
|
|
1548
|
+
const removed = Math.min(plan.counts[at], table.count[ordinal]);
|
|
1549
|
+
table.count[ordinal] -= removed;
|
|
1550
|
+
if (table === segment.surfaces && token < segment.terms.header[0])
|
|
1551
|
+
segment.columns.surfaceOccurrences[token] = nonnegative(segment.columns.surfaceOccurrences[token] - removed);
|
|
1552
|
+
if (table.count[ordinal] === 0) removeString(table.strings, plan.texts[at], ordinal, plan.hashes[at]);
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
function removeSharedTexts(segment, ordinal, texts, fields) {
|
|
1556
|
+
for (const text of texts) requireSegment2(!holdsUnpairedSurrogate(text));
|
|
1557
|
+
documentRemoval.collect(segment, texts, fields);
|
|
1558
|
+
removalPlan.prepare(segment, ordinal, documentRemoval);
|
|
1559
|
+
removalPlan.publish(segment, ordinal, documentRemoval);
|
|
1560
|
+
const surfaces = documentRemoval.surfaces;
|
|
1561
|
+
if (surfaces !== null && segment.surfaces !== null) removeSurfaces(surfaces, segment.surfaces, segment);
|
|
1562
|
+
}
|
|
1563
|
+
function removeTokens(segment, ordinal, tokens) {
|
|
1564
|
+
removalPlan.prepare(segment, ordinal, tokens.terms);
|
|
1565
|
+
const spellings = tokens.spellings;
|
|
1566
|
+
if (spellings !== null && segment.spellings !== null) {
|
|
1567
|
+
for (let at = 0; at < spellings.size; at++)
|
|
1568
|
+
spellings.added[at] = spellings.counts[at] >= segment.spellings.count[spellings.ordinals[at]] ? 1 : 0;
|
|
1569
|
+
}
|
|
1570
|
+
const runs = spellings === null || segment.spellings === null ? null : prepareSpellingRemoval(segment.spellings.runs, spellings);
|
|
1571
|
+
removalPlan.publish(segment, ordinal, tokens.terms);
|
|
1572
|
+
if (tokens.surfaces !== null && segment.surfaces !== null) removeSurfaces(tokens.surfaces, segment.surfaces, segment);
|
|
1573
|
+
if (spellings !== null && segment.spellings !== null && runs !== null) {
|
|
1574
|
+
runs();
|
|
1575
|
+
removeSurfaces(spellings, segment.spellings, segment);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
function retirable(segment, ordinal) {
|
|
1579
|
+
if (rowRemoved(segment, ordinal)) return false;
|
|
1580
|
+
requireSegment2(segment.totalDocuments[0] > 0 && segment.totalDocuments[0] <= segment.ordinalCapacity);
|
|
1581
|
+
for (let field = 0; field < segment.totalFieldLengths.length; field++) {
|
|
1582
|
+
const total = segment.totalFieldLengths[field];
|
|
1583
|
+
requireSegment2(Number.isFinite(total) && total >= segment.fieldLengths[field * segment.ordinalCapacity + ordinal]);
|
|
1584
|
+
}
|
|
1585
|
+
return true;
|
|
1586
|
+
}
|
|
1587
|
+
function retire(segment, ordinal) {
|
|
1588
|
+
for (let field = 0; field < segment.partialFields.length; field++) {
|
|
1589
|
+
const at = field * segment.ordinalCapacity + ordinal;
|
|
1590
|
+
segment.totalFieldLengths[field] = nonnegative(segment.totalFieldLengths[field] - segment.fieldLengths[at]);
|
|
1591
|
+
segment.fieldLengths[at] = 0;
|
|
1592
|
+
}
|
|
1593
|
+
segment.retired[ordinal >>> 5] |= 1 << (ordinal & 31);
|
|
1594
|
+
segment.totalDocuments[0]--;
|
|
1595
|
+
}
|
|
1596
|
+
function holdsText(fields) {
|
|
1597
|
+
for (const { text } of fields) if (text.count > 0) return true;
|
|
1598
|
+
return false;
|
|
1599
|
+
}
|
|
1600
|
+
function removeDocument(input, generation, ordinal, fields) {
|
|
1601
|
+
try {
|
|
1602
|
+
const segment = readSegmentView(input, generation);
|
|
1603
|
+
requireOrdinal(segment, ordinal);
|
|
1604
|
+
requireFields(segment, fields);
|
|
1605
|
+
if (!retirable(segment, ordinal)) return 0;
|
|
1606
|
+
if (!holdsText(fields)) {
|
|
1607
|
+
retire(segment, ordinal);
|
|
1608
|
+
return 0;
|
|
1609
|
+
}
|
|
1610
|
+
const texts = sharedTexts(fields);
|
|
1611
|
+
if (texts !== null && segment.spellings === null) removeSharedTexts(segment, ordinal, texts, fields);
|
|
1612
|
+
else {
|
|
1613
|
+
const tokens = acquireTokenPlan(segment, preparedSources(fields), true);
|
|
1614
|
+
try {
|
|
1615
|
+
removeTokens(segment, ordinal, tokens);
|
|
1616
|
+
} finally {
|
|
1617
|
+
releaseTokenPlan(tokens);
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
retire(segment, ordinal);
|
|
1621
|
+
return 0;
|
|
1622
|
+
} catch (error) {
|
|
1623
|
+
if (error instanceof SegmentRefusal2) return error.status;
|
|
1624
|
+
throw error;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
// src/engine/optional-writes-host.browser.ts
|
|
1629
|
+
var BATCH_CHUNK_SIZE;
|
|
1630
|
+
var ErrorCodes;
|
|
1631
|
+
var MAX_CALLS_ACROSS_INDEX_CHANGES;
|
|
1632
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS;
|
|
1633
|
+
var NarsilError;
|
|
1634
|
+
var admitBufferedUpsert;
|
|
1635
|
+
var admitInsert;
|
|
1636
|
+
var arrayPositionsAreStale;
|
|
1637
|
+
var awaitWriteVisibility;
|
|
1638
|
+
var capsEachPartition;
|
|
1639
|
+
var changeIndexOutsideExternalWaits;
|
|
1640
|
+
var createFrozenSegment;
|
|
1641
|
+
var deleteNestedValue;
|
|
1642
|
+
var emitSyncChange;
|
|
1643
|
+
var extractVectorFromDoc;
|
|
1644
|
+
var fnv1a;
|
|
1645
|
+
var generateId;
|
|
1646
|
+
var getLanguage;
|
|
1647
|
+
var getNestedValue;
|
|
1648
|
+
var getVectorFieldPaths;
|
|
1649
|
+
var indexChangeOutsideExternalWaits;
|
|
1650
|
+
var insertDocument;
|
|
1651
|
+
var insertDocumentVectors;
|
|
1652
|
+
var isSyncEpoch;
|
|
1653
|
+
var loadEmbedCode;
|
|
1654
|
+
var loadSchemaCodeFor;
|
|
1655
|
+
var loadStorageCode;
|
|
1656
|
+
var lookupOfStoredSynonyms;
|
|
1657
|
+
var newSyncEpoch;
|
|
1658
|
+
var oneAtATime;
|
|
1659
|
+
var prepareDocumentVectors;
|
|
1660
|
+
var prepareStoredSynonyms;
|
|
1661
|
+
var providedDocId;
|
|
1662
|
+
var readHeapStatistics;
|
|
1663
|
+
var removeDocumentVectors;
|
|
1664
|
+
var requireDocumentObject;
|
|
1665
|
+
var requireDocumentWithinSizeLimit;
|
|
1666
|
+
var requireMatchingDocId;
|
|
1667
|
+
var requireSurfaceFormsForPartialFields;
|
|
1668
|
+
var storedBM25Params;
|
|
1669
|
+
var turnIsSpent;
|
|
1670
|
+
var turnsAreBudgeted;
|
|
1671
|
+
var updateDocumentVectors;
|
|
1672
|
+
var validateBM25Params;
|
|
1673
|
+
var validateBatchList;
|
|
1674
|
+
var validateDocId;
|
|
1675
|
+
var validateDocument;
|
|
1676
|
+
var validateDocumentStrict;
|
|
1677
|
+
var validateEmbeddingConfig;
|
|
1678
|
+
var validatePartitionConfig;
|
|
1679
|
+
var validatePatternValueLimit;
|
|
1680
|
+
var validateRequiredFields;
|
|
1681
|
+
var validateSchema;
|
|
1682
|
+
var validateTextStorage;
|
|
1683
|
+
var validateVectorDimensions;
|
|
1684
|
+
var validateVectorStorage;
|
|
1685
|
+
var withOpenIndex;
|
|
1686
|
+
var yieldTurn;
|
|
1687
|
+
var createSharedFrozenSegment;
|
|
1688
|
+
var reconstructSchemaFromMetadata;
|
|
1689
|
+
function bindOptionalWritesHost(host, segmentCodec) {
|
|
1690
|
+
bindFrozenKernelHost(host);
|
|
1691
|
+
host.bindFrozenRemovalKernel({
|
|
1692
|
+
growFrozenRemovalRegions,
|
|
1693
|
+
prepareFrozenRemoval,
|
|
1694
|
+
prepareFrozenStrings,
|
|
1695
|
+
removeFrozenDocument,
|
|
1696
|
+
settleFrozenState
|
|
1697
|
+
});
|
|
1698
|
+
bindSegmentRemovalHost(host);
|
|
1699
|
+
host.bindSegmentRemoval(removeDocument);
|
|
1700
|
+
({
|
|
1701
|
+
BATCH_CHUNK_SIZE,
|
|
1702
|
+
ErrorCodes,
|
|
1703
|
+
MAX_CALLS_ACROSS_INDEX_CHANGES,
|
|
1704
|
+
MIN_DOCUMENTS_FOR_SEGMENTS,
|
|
1705
|
+
NarsilError,
|
|
1706
|
+
admitBufferedUpsert,
|
|
1707
|
+
admitInsert,
|
|
1708
|
+
arrayPositionsAreStale,
|
|
1709
|
+
awaitWriteVisibility,
|
|
1710
|
+
capsEachPartition,
|
|
1711
|
+
changeIndexOutsideExternalWaits,
|
|
1712
|
+
createFrozenSegment,
|
|
1713
|
+
deleteNestedValue,
|
|
1714
|
+
emitSyncChange,
|
|
1715
|
+
extractVectorFromDoc,
|
|
1716
|
+
fnv1a,
|
|
1717
|
+
generateId,
|
|
1718
|
+
getLanguage,
|
|
1719
|
+
getNestedValue,
|
|
1720
|
+
getVectorFieldPaths,
|
|
1721
|
+
indexChangeOutsideExternalWaits,
|
|
1722
|
+
insertDocument,
|
|
1723
|
+
insertDocumentVectors,
|
|
1724
|
+
isSyncEpoch,
|
|
1725
|
+
loadEmbedCode,
|
|
1726
|
+
loadSchemaCodeFor,
|
|
1727
|
+
loadStorageCode,
|
|
1728
|
+
lookupOfStoredSynonyms,
|
|
1729
|
+
newSyncEpoch,
|
|
1730
|
+
oneAtATime,
|
|
1731
|
+
prepareDocumentVectors,
|
|
1732
|
+
prepareStoredSynonyms,
|
|
1733
|
+
providedDocId,
|
|
1734
|
+
readHeapStatistics,
|
|
1735
|
+
removeDocumentVectors,
|
|
1736
|
+
requireDocumentObject,
|
|
1737
|
+
requireDocumentWithinSizeLimit,
|
|
1738
|
+
requireMatchingDocId,
|
|
1739
|
+
requireSurfaceFormsForPartialFields,
|
|
1740
|
+
storedBM25Params,
|
|
1741
|
+
turnIsSpent,
|
|
1742
|
+
turnsAreBudgeted,
|
|
1743
|
+
updateDocumentVectors,
|
|
1744
|
+
validateBM25Params,
|
|
1745
|
+
validateBatchList,
|
|
1746
|
+
validateDocId,
|
|
1747
|
+
validateDocument,
|
|
1748
|
+
validateDocumentStrict,
|
|
1749
|
+
validateEmbeddingConfig,
|
|
1750
|
+
validatePartitionConfig,
|
|
1751
|
+
validatePatternValueLimit,
|
|
1752
|
+
validateRequiredFields,
|
|
1753
|
+
validateSchema,
|
|
1754
|
+
validateTextStorage,
|
|
1755
|
+
validateVectorDimensions,
|
|
1756
|
+
validateVectorStorage,
|
|
1757
|
+
withOpenIndex,
|
|
1758
|
+
yieldTurn
|
|
1759
|
+
} = host);
|
|
1760
|
+
({ createSharedFrozenSegment, reconstructSchemaFromMetadata } = segmentCodec);
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// src/runtime/process-memory.ts
|
|
1764
|
+
function readProcessMemory() {
|
|
1765
|
+
const proc = globalThis.process;
|
|
1766
|
+
if (!proc || typeof proc.memoryUsage !== "function") {
|
|
1767
|
+
return null;
|
|
1768
|
+
}
|
|
1769
|
+
try {
|
|
1770
|
+
const usage = proc.memoryUsage();
|
|
1771
|
+
const heap = readHeapStatistics();
|
|
1772
|
+
if (!Number.isFinite(usage.heapUsed) || !Number.isFinite(usage.heapTotal) || !Number.isFinite(usage.external) || !Number.isFinite(usage.rss)) {
|
|
1773
|
+
return null;
|
|
1774
|
+
}
|
|
1775
|
+
return {
|
|
1776
|
+
heapUsed: usage.heapUsed,
|
|
1777
|
+
heapTotal: usage.heapTotal,
|
|
1778
|
+
heapLimit: heap === null ? null : heap.configuredLimitBytes ?? heap.limitBytes,
|
|
1779
|
+
external: usage.external,
|
|
1780
|
+
rss: usage.rss
|
|
1781
|
+
};
|
|
1782
|
+
} catch {
|
|
1783
|
+
return null;
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
// src/engine/bm25-update.ts
|
|
1788
|
+
async function replaceNow(core, indexName, bm25) {
|
|
1789
|
+
const { config } = core.requireIndex(indexName);
|
|
1790
|
+
const manager = core.requireManager(indexName);
|
|
1791
|
+
const previous = config.bm25;
|
|
1792
|
+
await manager.publication.write(() => {
|
|
1793
|
+
config.bm25 = bm25;
|
|
1794
|
+
core.orchestrator.publishToWorkers({
|
|
1795
|
+
type: "replaceBm25",
|
|
1796
|
+
indexName,
|
|
1797
|
+
requestId: `replace-bm25-${indexName}`,
|
|
1798
|
+
bm25
|
|
1799
|
+
});
|
|
1800
|
+
});
|
|
1801
|
+
try {
|
|
1802
|
+
if (core.durability) await core.durability.manager.persistMetadata(indexName);
|
|
1803
|
+
} catch (err) {
|
|
1804
|
+
await manager.publication.write(() => {
|
|
1805
|
+
config.bm25 = previous;
|
|
1806
|
+
core.orchestrator.publishToWorkers({
|
|
1807
|
+
type: "replaceBm25",
|
|
1808
|
+
indexName,
|
|
1809
|
+
requestId: `restore-bm25-${indexName}`,
|
|
1810
|
+
bm25: previous
|
|
1811
|
+
});
|
|
1812
|
+
});
|
|
1813
|
+
throw err;
|
|
1814
|
+
}
|
|
1815
|
+
await core.orchestrator.awaitWrites(indexName);
|
|
1816
|
+
if (core.requireIndex(indexName).sync !== null) emitSyncChange(core.indexRegistry, { kind: "settings", indexName });
|
|
1817
|
+
}
|
|
1818
|
+
function updateIndexBm25(core, indexName, value) {
|
|
1819
|
+
return withOpenIndex(
|
|
1820
|
+
core,
|
|
1821
|
+
indexName,
|
|
1822
|
+
() => oneAtATime(core, indexName, () => {
|
|
1823
|
+
const entry = core.requireIndex(indexName);
|
|
1824
|
+
validateBM25Params(value, entry.config.schema, ErrorCodes.CONFIG_INVALID);
|
|
1825
|
+
return replaceNow(core, indexName, storedBM25Params(value));
|
|
1826
|
+
})
|
|
1827
|
+
);
|
|
1828
|
+
}
|
|
1829
|
+
function applyStoredIndexBm25(core, indexName, load) {
|
|
1830
|
+
return withOpenIndex(
|
|
1831
|
+
core,
|
|
1832
|
+
indexName,
|
|
1833
|
+
() => oneAtATime(core, indexName, async () => {
|
|
1834
|
+
const loaded = load();
|
|
1835
|
+
if (loaded === null) return false;
|
|
1836
|
+
await replaceNow(core, indexName, loaded.bm25);
|
|
1837
|
+
return true;
|
|
1838
|
+
})
|
|
1839
|
+
);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// src/engine/sync-state.ts
|
|
1843
|
+
function epochStatesOfMetadata(metadata) {
|
|
1844
|
+
const syncOn = metadata.sync?.enabled === true;
|
|
1845
|
+
const changesOn = metadata.changes?.enabled === true;
|
|
1846
|
+
if (!syncOn && !changesOn) return { sync: null, changes: null };
|
|
1847
|
+
const stored = isSyncEpoch(metadata.syncEpoch) ? metadata.syncEpoch : null;
|
|
1848
|
+
const epoch = stored ?? newSyncEpoch();
|
|
1849
|
+
const sync = metadata.sync;
|
|
1850
|
+
const changes = metadata.changes;
|
|
1851
|
+
return {
|
|
1852
|
+
sync: syncOn && sync !== void 0 ? {
|
|
1853
|
+
...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
|
|
1854
|
+
...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {},
|
|
1855
|
+
active: stored !== null && metadata.syncActive === true,
|
|
1856
|
+
epoch
|
|
1857
|
+
} : null,
|
|
1858
|
+
changes: changesOn && changes !== void 0 ? {
|
|
1859
|
+
previousDocument: changes.previousDocument === true,
|
|
1860
|
+
...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
|
|
1861
|
+
...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {},
|
|
1862
|
+
epoch
|
|
1863
|
+
} : null
|
|
1864
|
+
};
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
// src/engine/index-from-metadata.ts
|
|
1868
|
+
function isRecoveryConfigError(err) {
|
|
1869
|
+
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
1870
|
+
}
|
|
1871
|
+
function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
1872
|
+
const name = metadata.embedding?.adapter ?? null;
|
|
1873
|
+
if (!metadata.embedding) return { adapter: null, name };
|
|
1874
|
+
const candidate = name !== null ? deps.embeddingAdapters.get(name) ?? null : deps.config?.embedding ?? null;
|
|
1875
|
+
if (!candidate) return { adapter: null, name };
|
|
1876
|
+
try {
|
|
1877
|
+
validateEmbeddingConfig({ fields: metadata.embedding.fields, adapter: candidate }, schema, void 0);
|
|
1878
|
+
} catch (err) {
|
|
1879
|
+
if (err instanceof NarsilError) {
|
|
1880
|
+
throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
1881
|
+
indexName: metadata.indexName,
|
|
1882
|
+
adapter: name ?? void 0
|
|
1883
|
+
});
|
|
1884
|
+
}
|
|
1885
|
+
throw err;
|
|
1886
|
+
}
|
|
1887
|
+
return { adapter: candidate, name };
|
|
1888
|
+
}
|
|
1889
|
+
function recoveryFailure(metadata, err) {
|
|
1890
|
+
if (!isRecoveryConfigError(err)) return err;
|
|
1891
|
+
return new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
1892
|
+
...err.details,
|
|
1893
|
+
indexName: metadata.indexName,
|
|
1894
|
+
tokenizer: metadata.tokenizer,
|
|
1895
|
+
stopWords: metadata.stopWords
|
|
1896
|
+
});
|
|
1897
|
+
}
|
|
1898
|
+
async function savedPartitionCountOf(metadata) {
|
|
1899
|
+
const count = metadata.partitionCount;
|
|
1900
|
+
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 1) {
|
|
1901
|
+
const held = typeof count === "number" ? String(count) : `a ${typeof count} as its count of`;
|
|
1902
|
+
throw new NarsilError(
|
|
1903
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1904
|
+
`Recovery of index "${metadata.indexName}" failed: its metadata holds ${held} partitions, and an index holds a whole number above zero`,
|
|
1905
|
+
{ indexName: metadata.indexName, partitionCount: typeof count === "number" ? count : typeof count }
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1908
|
+
const { requireSavedPartitionCount } = await loadStorageCode();
|
|
1909
|
+
requireSavedPartitionCount(count, "metadata", metadata.indexName);
|
|
1910
|
+
return count;
|
|
1911
|
+
}
|
|
1912
|
+
async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
1913
|
+
const loadingSchemaCode = loadSchemaCodeFor(
|
|
1914
|
+
loadData ? metadata.schema : null,
|
|
1915
|
+
metadata.tokenizer,
|
|
1916
|
+
metadata.pendingAnalysis?.tokenizer
|
|
1917
|
+
);
|
|
1918
|
+
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
1919
|
+
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
1920
|
+
if (existing !== void 0) {
|
|
1921
|
+
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
1922
|
+
const { adoptSavedAnalysis } = await loadStorageCode();
|
|
1923
|
+
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
1924
|
+
}
|
|
1925
|
+
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
1926
|
+
const partitionCount2 = await savedPartitionCountOf(metadata);
|
|
1927
|
+
deps.executor.createIndex(
|
|
1928
|
+
metadata.indexName,
|
|
1929
|
+
existing.config,
|
|
1930
|
+
existing.language,
|
|
1931
|
+
false,
|
|
1932
|
+
lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language),
|
|
1933
|
+
partitionCount2
|
|
1934
|
+
);
|
|
1935
|
+
}
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1938
|
+
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
1939
|
+
const language = getLanguage(indexConfig.language ?? "english");
|
|
1940
|
+
try {
|
|
1941
|
+
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
1942
|
+
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
1943
|
+
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
1944
|
+
} catch (err) {
|
|
1945
|
+
throw recoveryFailure(metadata, err);
|
|
1946
|
+
}
|
|
1947
|
+
const partitionCount = await savedPartitionCountOf(metadata);
|
|
1948
|
+
const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
|
|
1949
|
+
const prepared = prepareStoredSynonyms(
|
|
1950
|
+
metadata.synonyms,
|
|
1951
|
+
indexConfig,
|
|
1952
|
+
language,
|
|
1953
|
+
"The synonym set in the index metadata is invalid: "
|
|
1954
|
+
);
|
|
1955
|
+
const synonyms = prepared?.encoded ?? null;
|
|
1956
|
+
try {
|
|
1957
|
+
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
1958
|
+
validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
|
|
1959
|
+
if (loadData) {
|
|
1960
|
+
deps.executor.createIndex(
|
|
1961
|
+
metadata.indexName,
|
|
1962
|
+
indexConfig,
|
|
1963
|
+
language,
|
|
1964
|
+
false,
|
|
1965
|
+
prepared?.lookup ?? null,
|
|
1966
|
+
partitionCount
|
|
1967
|
+
);
|
|
1968
|
+
}
|
|
1969
|
+
} catch (err) {
|
|
1970
|
+
throw recoveryFailure(metadata, err);
|
|
1971
|
+
}
|
|
1972
|
+
deps.indexRegistry.set(metadata.indexName, {
|
|
1973
|
+
storedSizes: deps.indexRegistry.get(metadata.indexName)?.storedSizes,
|
|
1974
|
+
config: indexConfig,
|
|
1975
|
+
language,
|
|
1976
|
+
embeddingAdapter: embedding.adapter,
|
|
1977
|
+
embeddingAdapterName: embedding.name,
|
|
1978
|
+
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema),
|
|
1979
|
+
indexUuid: metadata.indexUuid ?? null,
|
|
1980
|
+
heldPartitions: metadata.heldPartitions ?? null,
|
|
1981
|
+
documentCount: metadata.documentCount ?? 0,
|
|
1982
|
+
savedBytes: metadata.savedBytes,
|
|
1983
|
+
partitionCount,
|
|
1984
|
+
analysisRevision: metadata.analysisRevision ?? null,
|
|
1985
|
+
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
1986
|
+
synonyms,
|
|
1987
|
+
analysisGeneration: metadata.analysisGeneration ?? 0,
|
|
1988
|
+
pendingAnalysis: metadata.pendingAnalysis ?? null,
|
|
1989
|
+
durableAnalysis: null,
|
|
1990
|
+
...epochStatesOfMetadata(metadata),
|
|
1991
|
+
querySuggestions: metadata.querySuggestions ?? null
|
|
1992
|
+
});
|
|
1993
|
+
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
1994
|
+
else deps.indexState().registerClosed(metadata.indexName);
|
|
1995
|
+
if (metadata.analysisRevision !== language.revision || arrayPositionsAreStale(indexConfig, metadata.arrayPositionGap ?? null)) {
|
|
1996
|
+
deps.analysisRebuild().markStale({
|
|
1997
|
+
indexName: metadata.indexName,
|
|
1998
|
+
language: language.name,
|
|
1999
|
+
storedRevision: metadata.analysisRevision ?? null,
|
|
2000
|
+
currentRevision: language.revision
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
// src/engine/mutations/batch-admissions.ts
|
|
2006
|
+
function createPendingAdmissions() {
|
|
2007
|
+
return { total: 0, byPartition: /* @__PURE__ */ new Map() };
|
|
2008
|
+
}
|
|
2009
|
+
function recordPendingAdmission(ctx, pending, manager, docId) {
|
|
2010
|
+
pending.total += 1;
|
|
2011
|
+
if (!capsEachPartition(ctx, manager)) return;
|
|
2012
|
+
const partitionId = manager.routePartition(docId);
|
|
2013
|
+
pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
|
|
2014
|
+
}
|
|
2015
|
+
function asBatchInsertError(err) {
|
|
2016
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
// src/engine/mutations/outside-hold.ts
|
|
2020
|
+
function runOutside(hold, work) {
|
|
2021
|
+
return hold === void 0 ? work() : hold.outside(work);
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
// src/engine/mutations/write-preparation.ts
|
|
2025
|
+
function settingsOf(entry) {
|
|
2026
|
+
return { required: entry.config.required, embedding: entry.config.embedding, adapter: entry.embeddingAdapter };
|
|
2027
|
+
}
|
|
2028
|
+
function embedsThroughAdapter(entry) {
|
|
2029
|
+
return entry.config.embedding !== void 0 && entry.embeddingAdapter !== null;
|
|
2030
|
+
}
|
|
2031
|
+
function sourcesOf(sources) {
|
|
2032
|
+
return typeof sources === "string" ? [sources] : sources;
|
|
2033
|
+
}
|
|
2034
|
+
function sameMapping(left, right) {
|
|
2035
|
+
if (left === right) return true;
|
|
2036
|
+
const targets = Object.keys(left);
|
|
2037
|
+
if (targets.length !== Object.keys(right).length) return false;
|
|
2038
|
+
for (const target of targets) {
|
|
2039
|
+
if (!Object.hasOwn(right, target)) return false;
|
|
2040
|
+
const leftSources = sourcesOf(left[target]);
|
|
2041
|
+
const rightSources = sourcesOf(right[target]);
|
|
2042
|
+
if (leftSources.length !== rightSources.length) return false;
|
|
2043
|
+
if (leftSources.some((source, position) => source !== rightSources[position])) return false;
|
|
2044
|
+
}
|
|
2045
|
+
return true;
|
|
2046
|
+
}
|
|
2047
|
+
function embeddingFits(entry, embedding) {
|
|
2048
|
+
const embeddingConfig = entry.config.embedding;
|
|
2049
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter === null) return embedding === null;
|
|
2050
|
+
return embedding !== null && embedding.adapter === entry.embeddingAdapter && sameMapping(embedding.mapping, embeddingConfig.fields);
|
|
2051
|
+
}
|
|
2052
|
+
function embeddingOfDocuments(embedding, documents) {
|
|
2053
|
+
if (embedding === null) return null;
|
|
2054
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2055
|
+
for (const document of documents) {
|
|
2056
|
+
const embedded = embedding.fields.get(document);
|
|
2057
|
+
if (embedded !== void 0) fields.set(document, embedded);
|
|
2058
|
+
}
|
|
2059
|
+
return { adapter: embedding.adapter, mapping: embedding.mapping, fields };
|
|
2060
|
+
}
|
|
2061
|
+
function writable(preparations) {
|
|
2062
|
+
return preparations.filter((preparation) => preparation.failure === void 0);
|
|
2063
|
+
}
|
|
2064
|
+
function failAll(preparations, failure) {
|
|
2065
|
+
for (const preparation of writable(preparations)) preparation.failure = failure;
|
|
2066
|
+
}
|
|
2067
|
+
function requireFieldsOf(required, preparations) {
|
|
2068
|
+
if (required === void 0 || required.length === 0) return;
|
|
2069
|
+
for (const preparation of writable(preparations)) {
|
|
2070
|
+
try {
|
|
2071
|
+
validateRequiredFields(preparation.document, required);
|
|
2072
|
+
} catch (err) {
|
|
2073
|
+
preparation.failure = err;
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
2077
|
+
async function requireMappedVectors(entry, preparations) {
|
|
2078
|
+
const embeddingConfig = entry.config.embedding;
|
|
2079
|
+
if (embeddingConfig === void 0 || entry.embeddingAdapter !== null) return;
|
|
2080
|
+
const { assertDocumentCarriesMappedVectors } = await loadEmbedCode();
|
|
2081
|
+
for (const preparation of writable(preparations)) {
|
|
2082
|
+
try {
|
|
2083
|
+
assertDocumentCarriesMappedVectors(
|
|
2084
|
+
preparation.document,
|
|
2085
|
+
embeddingConfig,
|
|
2086
|
+
entry.embeddingAdapterName
|
|
2087
|
+
);
|
|
2088
|
+
} catch (err) {
|
|
2089
|
+
preparation.failure = err;
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
async function embedWith(settings, signal, preparations, earlier, together) {
|
|
2094
|
+
requireFieldsOf(settings.required, preparations);
|
|
2095
|
+
const { clearWriteEmbedding, embedDocumentsForWrite, lacksMappedVectors } = await loadEmbedCode();
|
|
2096
|
+
if (earlier !== null) clearWriteEmbedding(earlier);
|
|
2097
|
+
const { embedding: embeddingConfig, adapter } = settings;
|
|
2098
|
+
const embeddable = writable(preparations);
|
|
2099
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
2100
|
+
if (embeddingConfig === void 0 || adapter === null || embeddable.length === 0) {
|
|
2101
|
+
return { embedding: null, called: false, documentFailures };
|
|
2102
|
+
}
|
|
2103
|
+
const fields = /* @__PURE__ */ new Map();
|
|
2104
|
+
const embedding = { adapter, mapping: embeddingConfig.fields, fields };
|
|
2105
|
+
const documents = embeddable.map((preparation) => preparation.document);
|
|
2106
|
+
if (!lacksMappedVectors(documents, embeddingConfig)) return { embedding, called: false, documentFailures };
|
|
2107
|
+
for (let chunkStart = 0; chunkStart < embeddable.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2108
|
+
const chunk = embeddable.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2109
|
+
const embedded = await embedDocumentsForWrite(
|
|
2110
|
+
chunk.map((preparation) => preparation.document),
|
|
2111
|
+
embeddingConfig,
|
|
2112
|
+
adapter,
|
|
2113
|
+
signal,
|
|
2114
|
+
together
|
|
2115
|
+
);
|
|
2116
|
+
for (const [document, embeddedFields] of embedded.embedding.fields) fields.set(document, embeddedFields);
|
|
2117
|
+
for (const [position, error] of embedded.failed) chunk[position].failure = error;
|
|
2118
|
+
for (const [position, error] of embedded.documentFailures) documentFailures.set(chunk[position], error);
|
|
2119
|
+
}
|
|
2120
|
+
return { embedding, called: true, documentFailures };
|
|
2121
|
+
}
|
|
2122
|
+
function settleDocumentFailures(entry, pass) {
|
|
2123
|
+
if (!embeddingFits(entry, pass.embedding)) return [...pass.documentFailures.keys()];
|
|
2124
|
+
for (const [preparation, failure] of pass.documentFailures) preparation.failure = failure;
|
|
2125
|
+
return [];
|
|
2126
|
+
}
|
|
2127
|
+
async function runBeforeWrites(preparations) {
|
|
2128
|
+
for (const preparation of preparations) {
|
|
2129
|
+
if (preparation.beforeWrite === null || preparation.failure !== void 0) continue;
|
|
2130
|
+
try {
|
|
2131
|
+
await preparation.beforeWrite();
|
|
2132
|
+
} catch (err) {
|
|
2133
|
+
preparation.failure = err;
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
async function embedThenRunHooks(settings, signal, preparations, earlier) {
|
|
2138
|
+
const pass = await embedWith(settings, signal, preparations, earlier, true);
|
|
2139
|
+
await runBeforeWrites(preparations.filter((preparation) => !pass.documentFailures.has(preparation)));
|
|
2140
|
+
return pass;
|
|
2141
|
+
}
|
|
2142
|
+
async function runHooksThenEmbedEach(ctx, indexName, preparations, earlier) {
|
|
2143
|
+
let settings = null;
|
|
2144
|
+
let embedding = null;
|
|
2145
|
+
let called = false;
|
|
2146
|
+
let uncleared = earlier;
|
|
2147
|
+
const documentFailures = /* @__PURE__ */ new Map();
|
|
2148
|
+
for (const preparation of preparations) {
|
|
2149
|
+
await runBeforeWrites([preparation]);
|
|
2150
|
+
settings ??= settingsOf(ctx.requireIndex(indexName));
|
|
2151
|
+
const pass = await embedWith(settings, ctx.abortController.signal, [preparation], uncleared, false);
|
|
2152
|
+
uncleared = null;
|
|
2153
|
+
called ||= pass.called;
|
|
2154
|
+
for (const [failed, failure] of pass.documentFailures) documentFailures.set(failed, failure);
|
|
2155
|
+
if (pass.embedding === null) continue;
|
|
2156
|
+
if (embedding === null) embedding = pass.embedding;
|
|
2157
|
+
else for (const [document, fields] of pass.embedding.fields) embedding.fields.set(document, fields);
|
|
2158
|
+
}
|
|
2159
|
+
return { embedding, called, documentFailures };
|
|
2160
|
+
}
|
|
2161
|
+
function adapterChangesRefusal(indexName, embeddings) {
|
|
2162
|
+
return new NarsilError(
|
|
2163
|
+
ErrorCodes.WORKER_BUSY,
|
|
2164
|
+
`Changes to index "${indexName}" replaced its embedding adapter after each of ${embeddings} embeddings of this write`,
|
|
2165
|
+
{ indexName, embeddings }
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
async function prepareDocumentWrites(ctx, indexName, preparations, hold, together, earlier = null) {
|
|
2169
|
+
let entry = ctx.requireIndex(indexName);
|
|
2170
|
+
let embedding = earlier;
|
|
2171
|
+
if (writable(preparations).length === 0) return { entry, embedding };
|
|
2172
|
+
const signal = ctx.abortController.signal;
|
|
2173
|
+
let embeddings = 0;
|
|
2174
|
+
let unhooked = [];
|
|
2175
|
+
const hooked = preparations.some((preparation) => preparation.beforeWrite !== null);
|
|
2176
|
+
if (hooked || embedsThroughAdapter(entry) && !embeddingFits(entry, embedding)) {
|
|
2177
|
+
if (together && !embedsThroughAdapter(entry)) {
|
|
2178
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
2179
|
+
await requireMappedVectors(entry, preparations);
|
|
2180
|
+
}
|
|
2181
|
+
const settings = settingsOf(entry);
|
|
2182
|
+
const uncleared = embedding;
|
|
2183
|
+
const pass = await runOutside(
|
|
2184
|
+
hold,
|
|
2185
|
+
() => together ? embedThenRunHooks(settings, signal, preparations, uncleared) : runHooksThenEmbedEach(ctx, indexName, preparations, uncleared)
|
|
2186
|
+
);
|
|
2187
|
+
embedding = pass.embedding;
|
|
2188
|
+
if (pass.called) embeddings += 1;
|
|
2189
|
+
entry = ctx.requireIndex(indexName);
|
|
2190
|
+
unhooked = settleDocumentFailures(entry, pass);
|
|
2191
|
+
}
|
|
2192
|
+
while (writable(preparations).length > 0 && !embeddingFits(entry, embedding)) {
|
|
2193
|
+
if (embeddings >= MAX_CALLS_ACROSS_INDEX_CHANGES) {
|
|
2194
|
+
failAll(preparations, adapterChangesRefusal(indexName, embeddings));
|
|
2195
|
+
break;
|
|
2196
|
+
}
|
|
2197
|
+
const uncleared = embedding;
|
|
2198
|
+
const settings = settingsOf(entry);
|
|
2199
|
+
const hooking = unhooked;
|
|
2200
|
+
const pass = await runOutside(hold, async () => {
|
|
2201
|
+
const embedded = await embedWith(settings, signal, preparations, uncleared, together);
|
|
2202
|
+
await runBeforeWrites(hooking.filter((preparation) => !embedded.documentFailures.has(preparation)));
|
|
2203
|
+
return embedded;
|
|
2204
|
+
});
|
|
2205
|
+
embedding = pass.embedding;
|
|
2206
|
+
if (pass.called) embeddings += 1;
|
|
2207
|
+
entry = ctx.requireIndex(indexName);
|
|
2208
|
+
const stale = settleDocumentFailures(entry, pass);
|
|
2209
|
+
unhooked = hooking.filter((preparation) => stale.includes(preparation));
|
|
2210
|
+
}
|
|
2211
|
+
requireFieldsOf(entry.config.required, preparations);
|
|
2212
|
+
await requireMappedVectors(entry, preparations);
|
|
2213
|
+
return { entry, embedding };
|
|
2214
|
+
}
|
|
2215
|
+
async function prepareDocumentWrite(ctx, indexName, document, hold, beforeWrite) {
|
|
2216
|
+
const preparation = { document, beforeWrite, failure: void 0 };
|
|
2217
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
2218
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
2219
|
+
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2220
|
+
return entry;
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
// src/engine/mutations/insert-batch-admission.ts
|
|
2224
|
+
function pendingInsertOf(ctx, indexName, docId, document, hooked) {
|
|
2225
|
+
const pending = {
|
|
2226
|
+
docId,
|
|
2227
|
+
preparation: { document, beforeWrite: null, failure: void 0 },
|
|
2228
|
+
refusedByHook: false
|
|
2229
|
+
};
|
|
2230
|
+
if (hooked && ctx.pluginRegistry.hasHooks("beforeInsert")) {
|
|
2231
|
+
pending.preparation.beforeWrite = async () => {
|
|
2232
|
+
try {
|
|
2233
|
+
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document });
|
|
2234
|
+
} catch (err) {
|
|
2235
|
+
pending.refusedByHook = true;
|
|
2236
|
+
throw err;
|
|
2237
|
+
}
|
|
2238
|
+
};
|
|
2239
|
+
}
|
|
2240
|
+
return pending;
|
|
2241
|
+
}
|
|
2242
|
+
async function prepareInsertDocuments(ctx, indexName, documents, start, end, hold, hooked, failed, earlier = null) {
|
|
2243
|
+
const pending = [];
|
|
2244
|
+
for (let i = start; i < end; i++) {
|
|
2245
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2246
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2247
|
+
const document = documents[i];
|
|
2248
|
+
const docId = providedDocId(document) ?? ctx.idGenerator();
|
|
2249
|
+
try {
|
|
2250
|
+
const sizing = requireDocumentObject(document);
|
|
2251
|
+
if (sizing !== null) await sizing;
|
|
2252
|
+
validateDocId(docId);
|
|
2253
|
+
pending.push(pendingInsertOf(ctx, indexName, docId, document, hooked));
|
|
2254
|
+
} catch (err) {
|
|
2255
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
const { entry, embedding } = await prepareDocumentWrites(
|
|
2259
|
+
ctx,
|
|
2260
|
+
indexName,
|
|
2261
|
+
pending.map((item) => item.preparation),
|
|
2262
|
+
hold,
|
|
2263
|
+
true,
|
|
2264
|
+
earlier
|
|
2265
|
+
);
|
|
2266
|
+
const candidates = [];
|
|
2267
|
+
for (const item of pending) {
|
|
2268
|
+
const { document, failure } = item.preparation;
|
|
2269
|
+
if (failure === void 0) {
|
|
2270
|
+
candidates.push({ docId: item.docId, document });
|
|
2271
|
+
continue;
|
|
2272
|
+
}
|
|
2273
|
+
const docId = item.refusedByHook ? item.docId : providedDocId(document) ?? "";
|
|
2274
|
+
failed.push({ docId, error: asBatchInsertError(failure) });
|
|
2275
|
+
}
|
|
2276
|
+
return { entry, candidates, embedding };
|
|
2277
|
+
}
|
|
2278
|
+
async function admitBatchDocuments(ctx, indexName, documents, failed, hold) {
|
|
2279
|
+
const { entry, candidates, embedding } = await prepareInsertDocuments(
|
|
2280
|
+
ctx,
|
|
2281
|
+
indexName,
|
|
2282
|
+
documents,
|
|
2283
|
+
0,
|
|
2284
|
+
documents.length,
|
|
2285
|
+
hold,
|
|
2286
|
+
true,
|
|
2287
|
+
failed
|
|
2288
|
+
);
|
|
2289
|
+
const manager = ctx.requireManager(indexName);
|
|
2290
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2291
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2292
|
+
const strict = entry.config.strict === true;
|
|
2293
|
+
const admitted = [];
|
|
2294
|
+
const admittedIds = /* @__PURE__ */ new Set();
|
|
2295
|
+
const pending = createPendingAdmissions();
|
|
2296
|
+
for (let position = 0; position < candidates.length; position++) {
|
|
2297
|
+
if (position > 0 && position % BATCH_CHUNK_SIZE === 0) await new Promise((r) => setTimeout(r, 0));
|
|
2298
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2299
|
+
const { docId, document } = candidates[position];
|
|
2300
|
+
try {
|
|
2301
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2302
|
+
document,
|
|
2303
|
+
vectorFieldPaths
|
|
2304
|
+
);
|
|
2305
|
+
if (extractedVectors.size > 0) {
|
|
2306
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2307
|
+
}
|
|
2308
|
+
admitInsert(ctx, indexName, manager, docId, pending);
|
|
2309
|
+
if (admittedIds.has(docId) || manager.has(docId)) {
|
|
2310
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
|
|
2311
|
+
}
|
|
2312
|
+
validateDocument(partitionDoc, entry.config.schema, entry.config.patternValueLimit);
|
|
2313
|
+
if (strict) {
|
|
2314
|
+
validateDocumentStrict(partitionDoc, entry.config.schema);
|
|
2315
|
+
}
|
|
2316
|
+
admittedIds.add(docId);
|
|
2317
|
+
recordPendingAdmission(ctx, pending, manager, docId);
|
|
2318
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2319
|
+
} catch (err) {
|
|
2320
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
return { admitted, embedding };
|
|
2324
|
+
}
|
|
2325
|
+
|
|
2326
|
+
// src/engine/mutations/durable-rollback.ts
|
|
2327
|
+
function holdVectorsForRollback(indexes, docId) {
|
|
2328
|
+
const releases = [];
|
|
2329
|
+
for (const index of indexes.values()) releases.push(index.holdForRollback(docId));
|
|
2330
|
+
return () => {
|
|
2331
|
+
for (const release of releases) release();
|
|
2332
|
+
};
|
|
2333
|
+
}
|
|
2334
|
+
function restoreDocumentVectors(docId, vectors, indexes, partitionId) {
|
|
2335
|
+
for (const [fieldPath, index] of indexes) {
|
|
2336
|
+
const vector = vectors.get(fieldPath);
|
|
2337
|
+
if (vector !== void 0) index.restore(docId, vector, partitionId);
|
|
2338
|
+
else if (index.has(docId)) index.remove(docId);
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
function rollbackFailed(docId, originalError, rollbackError) {
|
|
2342
|
+
throw new NarsilError(
|
|
2343
|
+
ErrorCodes.REPLICATION_ROLLBACK_FAILED,
|
|
2344
|
+
`Durable write failed for document "${docId}" and the in-memory rollback also failed; the partition may expose an unacknowledged mutation`,
|
|
2345
|
+
{
|
|
2346
|
+
docId,
|
|
2347
|
+
originalError: originalError instanceof Error ? originalError.message : String(originalError),
|
|
2348
|
+
rollbackError: rollbackError instanceof Error ? rollbackError.message : String(rollbackError)
|
|
2349
|
+
}
|
|
2350
|
+
);
|
|
2351
|
+
}
|
|
2352
|
+
async function rollbackInsertedDocument(ctx, indexName, docId, inserted, originalError, preparedText, appliedRef, removedSpellings) {
|
|
2353
|
+
if (!inserted) {
|
|
2354
|
+
return;
|
|
2355
|
+
}
|
|
2356
|
+
try {
|
|
2357
|
+
const manager = ctx.requireManager(indexName);
|
|
2358
|
+
await manager.publication.write(
|
|
2359
|
+
() => changeIndexOutsideExternalWaits(async () => {
|
|
2360
|
+
if (appliedRef !== void 0 && manager.getRef(docId) !== appliedRef) return;
|
|
2361
|
+
manager.remove(docId, removedSpellings, { preparedText: { before: preparedText?.after } });
|
|
2362
|
+
removeDocumentVectors(docId, manager.getVectorIndexes());
|
|
2363
|
+
ctx.orchestrator.publishToWorkers({
|
|
2364
|
+
type: "remove",
|
|
2365
|
+
indexName,
|
|
2366
|
+
docId,
|
|
2367
|
+
removedSpellings,
|
|
2368
|
+
requestId: `rollback-${docId}`
|
|
2369
|
+
});
|
|
2370
|
+
})
|
|
2371
|
+
);
|
|
2372
|
+
} catch (rollbackError) {
|
|
2373
|
+
rollbackFailed(docId, originalError, rollbackError);
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, originalError, preparedText) {
|
|
2377
|
+
if (restoreDoc === void 0) {
|
|
2378
|
+
return;
|
|
2379
|
+
}
|
|
2380
|
+
const manager = ctx.requireManager(indexName);
|
|
2381
|
+
try {
|
|
2382
|
+
await manager.publication.write(
|
|
2383
|
+
() => changeIndexOutsideExternalWaits(async () => {
|
|
2384
|
+
if (manager.has(docId)) return;
|
|
2385
|
+
const previous = prepareDocumentVectors(restoreDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
2386
|
+
manager.insert(docId, previous.partitionDoc, {
|
|
2387
|
+
preparedText: { after: preparedText?.before }
|
|
2388
|
+
});
|
|
2389
|
+
restoreDocumentVectors(
|
|
2390
|
+
docId,
|
|
2391
|
+
previous.extractedVectors,
|
|
2392
|
+
manager.getVectorIndexes(),
|
|
2393
|
+
manager.partitionIdOf(docId)
|
|
2394
|
+
);
|
|
2395
|
+
ctx.orchestrator.publishToWorkers({
|
|
2396
|
+
type: "insert",
|
|
2397
|
+
indexName,
|
|
2398
|
+
docId,
|
|
2399
|
+
document: previous.partitionDoc,
|
|
2400
|
+
requestId: `rollback-${docId}`
|
|
2401
|
+
});
|
|
2402
|
+
})
|
|
2403
|
+
);
|
|
2404
|
+
} catch (rollbackError) {
|
|
2405
|
+
rollbackFailed(docId, originalError, rollbackError);
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, originalError, preparedText, appliedRef, rollbackRef) {
|
|
2409
|
+
if (restoreDoc === void 0 || appliedRef === void 0) {
|
|
2410
|
+
return;
|
|
2411
|
+
}
|
|
2412
|
+
try {
|
|
2413
|
+
const manager = ctx.requireManager(indexName);
|
|
2414
|
+
await manager.publication.write(
|
|
2415
|
+
() => changeIndexOutsideExternalWaits(async () => {
|
|
2416
|
+
if (manager.getRef(docId) !== appliedRef) return;
|
|
2417
|
+
const fields = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
2418
|
+
const previous = prepareDocumentVectors(restoreDoc, fields);
|
|
2419
|
+
manager.update(docId, rollbackRef ?? previous.partitionDoc, {
|
|
2420
|
+
skipClone: rollbackRef !== void 0,
|
|
2421
|
+
preparedText: { before: preparedText?.after, after: preparedText?.before }
|
|
2422
|
+
});
|
|
2423
|
+
restoreDocumentVectors(
|
|
2424
|
+
docId,
|
|
2425
|
+
previous.extractedVectors,
|
|
2426
|
+
manager.getVectorIndexes(),
|
|
2427
|
+
manager.partitionIdOf(docId)
|
|
2428
|
+
);
|
|
2429
|
+
ctx.orchestrator.publishToWorkers({
|
|
2430
|
+
type: "update",
|
|
2431
|
+
indexName,
|
|
2432
|
+
docId,
|
|
2433
|
+
document: previous.partitionDoc,
|
|
2434
|
+
requestId: `rollback-${docId}`
|
|
2435
|
+
});
|
|
2436
|
+
})
|
|
2437
|
+
);
|
|
2438
|
+
} catch (rollbackError) {
|
|
2439
|
+
rollbackFailed(docId, originalError, rollbackError);
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
|
|
2443
|
+
// src/engine/mutations/record-batch.ts
|
|
2444
|
+
async function recordChunk(ctx, indexName, mutations, applies) {
|
|
2445
|
+
if (mutations.length === 0) return [];
|
|
2446
|
+
if (ctx.durability) {
|
|
2447
|
+
const outcomes = await ctx.durability.recordMutationBatch(
|
|
2448
|
+
indexName,
|
|
2449
|
+
mutations.map((mutation, i) => ({ docId: mutation.docId, document: mutation.document, apply: applies[i] }))
|
|
2450
|
+
);
|
|
2451
|
+
return outcomes.map((outcome) => outcome.ok ? null : { error: outcome.error });
|
|
2452
|
+
}
|
|
2453
|
+
const failures = [];
|
|
2454
|
+
for (const apply of applies) {
|
|
2455
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2456
|
+
try {
|
|
2457
|
+
await apply();
|
|
2458
|
+
failures.push(null);
|
|
2459
|
+
} catch (error) {
|
|
2460
|
+
failures.push({ error });
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
return failures;
|
|
2464
|
+
}
|
|
2465
|
+
|
|
2466
|
+
// src/engine/mutations/insert-batch-apply.ts
|
|
2467
|
+
function applyInsertDocument(ctx, indexName, doc, options, progress) {
|
|
2468
|
+
const manager = ctx.requireManager(indexName);
|
|
2469
|
+
const insertIntoIndex = () => {
|
|
2470
|
+
if (ctx.abortController.signal.aborted) {
|
|
2471
|
+
progress.skipped = true;
|
|
2472
|
+
throw ctx.abortController.signal.reason;
|
|
2473
|
+
}
|
|
2474
|
+
admitInsert(ctx, indexName, manager, doc.docId);
|
|
2475
|
+
const queued = ctx.bufferIfRebalancing(indexName, {
|
|
2476
|
+
action: "insert",
|
|
2477
|
+
docId: doc.docId,
|
|
2478
|
+
document: doc.document,
|
|
2479
|
+
indexName
|
|
2480
|
+
});
|
|
2481
|
+
if (queued !== void 0) {
|
|
2482
|
+
progress.buffered = true;
|
|
2483
|
+
return queued;
|
|
2484
|
+
}
|
|
2485
|
+
const stored = manager.insert(doc.docId, doc.partitionDoc, {
|
|
2486
|
+
skipClone: doc.extractedVectors.size > 0 ? true : options?.skipClone,
|
|
2487
|
+
onPreparedText: doc.extractedVectors.size > 0 || ctx.durability !== null ? (change) => {
|
|
2488
|
+
progress.preparedText = change;
|
|
2489
|
+
} : void 0
|
|
2490
|
+
});
|
|
2491
|
+
progress.inserted = true;
|
|
2492
|
+
try {
|
|
2493
|
+
insertDocumentVectors(
|
|
2494
|
+
doc.docId,
|
|
2495
|
+
doc.extractedVectors,
|
|
2496
|
+
manager.getVectorIndexes(),
|
|
2497
|
+
manager.partitionIdOf(doc.docId)
|
|
2498
|
+
);
|
|
2499
|
+
} catch (vecErr) {
|
|
2500
|
+
try {
|
|
2501
|
+
manager.remove(doc.docId, void 0, { preparedText: { before: progress.preparedText?.after } });
|
|
2502
|
+
progress.inserted = false;
|
|
2503
|
+
} catch (rollbackErr) {
|
|
2504
|
+
console.warn(
|
|
2505
|
+
`Rollback failed for doc "${doc.docId}" during batch insert atomicity:`,
|
|
2506
|
+
rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
|
|
2507
|
+
);
|
|
2508
|
+
}
|
|
2509
|
+
throw vecErr;
|
|
2510
|
+
}
|
|
2511
|
+
if (ctx.durability === null) progress.preparedText = void 0;
|
|
2512
|
+
else progress.appliedRef = stored;
|
|
2513
|
+
ctx.orchestrator.publishToWorkers({
|
|
2514
|
+
type: "insert",
|
|
2515
|
+
indexName,
|
|
2516
|
+
docId: doc.docId,
|
|
2517
|
+
document: doc.partitionDoc,
|
|
2518
|
+
requestId: `replicate-insert-${doc.docId}`,
|
|
2519
|
+
skipClone: options?.skipClone
|
|
2520
|
+
});
|
|
2521
|
+
};
|
|
2522
|
+
return () => manager.publication.write(indexChangeOutsideExternalWaits(insertIntoIndex));
|
|
2523
|
+
}
|
|
2524
|
+
async function applyInsertChunk(ctx, indexName, docs, options) {
|
|
2525
|
+
const progress = docs.map(() => ({ inserted: false, buffered: false, skipped: false }));
|
|
2526
|
+
const applies = docs.map((doc, i) => applyInsertDocument(ctx, indexName, doc, options, progress[i]));
|
|
2527
|
+
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2528
|
+
await ctx.orchestrator.settlePublications(indexName);
|
|
2529
|
+
const applications = [];
|
|
2530
|
+
for (let i = 0; i < docs.length; i++) {
|
|
2531
|
+
if (progress[i].skipped) {
|
|
2532
|
+
applications.push({ status: "skipped" });
|
|
2533
|
+
continue;
|
|
2534
|
+
}
|
|
2535
|
+
const failure = failures[i];
|
|
2536
|
+
if (failure !== null) {
|
|
2537
|
+
let error = failure.error;
|
|
2538
|
+
if (ctx.durability) {
|
|
2539
|
+
try {
|
|
2540
|
+
await rollbackInsertedDocument(
|
|
2541
|
+
ctx,
|
|
2542
|
+
indexName,
|
|
2543
|
+
docs[i].docId,
|
|
2544
|
+
progress[i].inserted,
|
|
2545
|
+
error,
|
|
2546
|
+
progress[i].preparedText,
|
|
2547
|
+
progress[i].appliedRef
|
|
2548
|
+
);
|
|
2549
|
+
} catch (rollbackError) {
|
|
2550
|
+
error = rollbackError;
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
applications.push({ status: "failed", error });
|
|
2554
|
+
continue;
|
|
2555
|
+
}
|
|
2556
|
+
applications.push({ status: progress[i].buffered ? "buffered" : "inserted" });
|
|
2557
|
+
}
|
|
2558
|
+
return applications;
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
// src/engine/mutations/insert-batch-documents.ts
|
|
2562
|
+
async function runAfterInsertHooks(ctx, indexName, inserted, hold) {
|
|
2563
|
+
if (inserted.length === 0 || !ctx.pluginRegistry.hasHooks("afterInsert")) return false;
|
|
2564
|
+
const manager = ctx.executor.getManager(indexName);
|
|
2565
|
+
await runOutside(hold, async () => {
|
|
2566
|
+
for (const doc of inserted) {
|
|
2567
|
+
try {
|
|
2568
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: doc.docId, document: doc.document });
|
|
2569
|
+
} catch (err) {
|
|
2570
|
+
console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
2571
|
+
}
|
|
2572
|
+
}
|
|
2573
|
+
});
|
|
2574
|
+
return ctx.executor.getManager(indexName) !== manager;
|
|
2575
|
+
}
|
|
2576
|
+
async function applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed) {
|
|
2577
|
+
const succeeded = [];
|
|
2578
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2579
|
+
for (let chunkStart = 0; chunkStart < admitted.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2580
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2581
|
+
const chunk = admitted.slice(chunkStart, chunkStart + BATCH_CHUNK_SIZE);
|
|
2582
|
+
try {
|
|
2583
|
+
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2584
|
+
} catch (err) {
|
|
2585
|
+
const error = asBatchInsertError(err);
|
|
2586
|
+
for (let i = chunkStart; i < admitted.length; i++) failed.push({ docId: admitted[i].docId, error });
|
|
2587
|
+
break;
|
|
2588
|
+
}
|
|
2589
|
+
const applications = await applyInsertChunk(ctx, indexName, chunk, options);
|
|
2590
|
+
const inserted = [];
|
|
2591
|
+
for (let i = 0; i < chunk.length; i++) {
|
|
2592
|
+
const doc = chunk[i];
|
|
2593
|
+
const application = applications[i];
|
|
2594
|
+
if (application.status === "skipped") continue;
|
|
2595
|
+
if (application.status === "failed") {
|
|
2596
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2597
|
+
continue;
|
|
2598
|
+
}
|
|
2599
|
+
succeeded.push(doc.docId);
|
|
2600
|
+
if (application.status === "buffered") continue;
|
|
2601
|
+
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2602
|
+
touchedVectorFields.add(fieldPath);
|
|
2603
|
+
}
|
|
2604
|
+
inserted.push(doc);
|
|
2605
|
+
}
|
|
2606
|
+
const applied = chunkStart + BATCH_CHUNK_SIZE;
|
|
2607
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
2608
|
+
return { succeeded, touchedVectorFields, unrecorded: admitted.slice(applied) };
|
|
2609
|
+
}
|
|
2610
|
+
if (applied < admitted.length) {
|
|
2611
|
+
ctx.checkHeapPressure(indexName);
|
|
2612
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
return { succeeded, touchedVectorFields, unrecorded: [] };
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
// src/engine/mutations/insert-batch-chunks.ts
|
|
2619
|
+
async function admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier) {
|
|
2620
|
+
const { entry, candidates } = await prepareInsertDocuments(
|
|
2621
|
+
ctx,
|
|
2622
|
+
indexName,
|
|
2623
|
+
documents,
|
|
2624
|
+
chunkStart,
|
|
2625
|
+
chunkEnd,
|
|
2626
|
+
hold,
|
|
2627
|
+
hooked,
|
|
2628
|
+
failed,
|
|
2629
|
+
earlier
|
|
2630
|
+
);
|
|
2631
|
+
const manager = ctx.requireManager(indexName);
|
|
2632
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
2633
|
+
const vectorFieldPaths = vecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
2634
|
+
const admitted = [];
|
|
2635
|
+
for (const { docId, document } of candidates) {
|
|
2636
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
2637
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2638
|
+
try {
|
|
2639
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
2640
|
+
document,
|
|
2641
|
+
vectorFieldPaths
|
|
2642
|
+
);
|
|
2643
|
+
if (extractedVectors.size > 0) {
|
|
2644
|
+
validateVectorDimensions(extractedVectors, vecIndexes);
|
|
2645
|
+
}
|
|
2646
|
+
admitInsert(ctx, indexName, manager, docId);
|
|
2647
|
+
admitted.push({ docId, document, partitionDoc, extractedVectors });
|
|
2648
|
+
} catch (err) {
|
|
2649
|
+
failed.push({ docId, error: asBatchInsertError(err) });
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
return admitted;
|
|
2653
|
+
}
|
|
2654
|
+
async function insertInChunks(ctx, indexName, documents, options, hold, hooked, failed, embedded = null) {
|
|
2655
|
+
const succeeded = [];
|
|
2656
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2657
|
+
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
2658
|
+
if (ctx.abortController.signal.aborted) break;
|
|
2659
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
2660
|
+
const earlier = embeddingOfDocuments(embedded, documents.slice(chunkStart, chunkEnd));
|
|
2661
|
+
const prepared = await admitChunk(ctx, indexName, documents, chunkStart, chunkEnd, hold, hooked, failed, earlier);
|
|
2662
|
+
try {
|
|
2663
|
+
if (prepared.length > 0) ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2664
|
+
} catch (err) {
|
|
2665
|
+
const error = asBatchInsertError(err);
|
|
2666
|
+
for (const doc of prepared) failed.push({ docId: doc.docId, error });
|
|
2667
|
+
for (let i = chunkEnd; i < documents.length; i++) {
|
|
2668
|
+
failed.push({ docId: providedDocId(documents[i]) ?? "", error });
|
|
2669
|
+
}
|
|
2670
|
+
break;
|
|
2671
|
+
}
|
|
2672
|
+
if (prepared.length > 0) {
|
|
2673
|
+
const applications = await applyInsertChunk(ctx, indexName, prepared, options);
|
|
2674
|
+
const inserted = [];
|
|
2675
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
2676
|
+
const doc = prepared[i];
|
|
2677
|
+
const application = applications[i];
|
|
2678
|
+
if (application.status === "skipped") continue;
|
|
2679
|
+
if (application.status === "failed") {
|
|
2680
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(application.error) });
|
|
2681
|
+
continue;
|
|
2682
|
+
}
|
|
2683
|
+
succeeded.push(doc.docId);
|
|
2684
|
+
if (application.status === "buffered") continue;
|
|
2685
|
+
for (const fieldPath of doc.extractedVectors.keys()) touchedVectorFields.add(fieldPath);
|
|
2686
|
+
inserted.push(doc);
|
|
2687
|
+
}
|
|
2688
|
+
await runAfterInsertHooks(ctx, indexName, inserted, hold);
|
|
2689
|
+
}
|
|
2690
|
+
if (chunkEnd < documents.length) {
|
|
2691
|
+
ctx.checkHeapPressure(indexName);
|
|
2692
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2693
|
+
}
|
|
2694
|
+
}
|
|
2695
|
+
return { succeeded, touchedVectorFields };
|
|
2696
|
+
}
|
|
2697
|
+
async function finishInsertBatch(ctx, indexName, inserts, options) {
|
|
2698
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
2699
|
+
for (const fieldPath of inserts.touchedVectorFields) {
|
|
2700
|
+
vecIndexes?.get(fieldPath)?.scheduleBuild();
|
|
2701
|
+
}
|
|
2702
|
+
ctx.checkWatermark(indexName, inserts.succeeded);
|
|
2703
|
+
ctx.checkHeapPressure(indexName);
|
|
2704
|
+
await ctx.orchestrator.scaleOutReadyIndexes();
|
|
2705
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
2706
|
+
}
|
|
2707
|
+
|
|
2708
|
+
// src/engine/mutations/segment-publication.ts
|
|
2709
|
+
function applySegment(ctx, indexName, attachment, doc, options) {
|
|
2710
|
+
const manager = ctx.requireManager(indexName);
|
|
2711
|
+
const publish = async () => {
|
|
2712
|
+
if (attachment.failure !== void 0) throw attachment.failure;
|
|
2713
|
+
if (attachment.attached) return;
|
|
2714
|
+
if (ctx.abortController.signal.aborted) throw ctx.abortController.signal.reason;
|
|
2715
|
+
if (ctx.isRebalancing(indexName) || manager.partitionCount !== attachment.partitionCount) {
|
|
2716
|
+
attachment.individual = true;
|
|
2717
|
+
}
|
|
2718
|
+
if (attachment.individual) return;
|
|
2719
|
+
const inserted = [];
|
|
2720
|
+
const vectors = manager.getVectorIndexes();
|
|
2721
|
+
try {
|
|
2722
|
+
const pending = createPendingAdmissions();
|
|
2723
|
+
for (const doc2 of attachment.members) {
|
|
2724
|
+
if (manager.has(doc2.docId)) {
|
|
2725
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${doc2.docId}" already exists`);
|
|
2726
|
+
}
|
|
2727
|
+
admitInsert(ctx, indexName, manager, doc2.docId, pending);
|
|
2728
|
+
recordPendingAdmission(ctx, pending, manager, doc2.docId);
|
|
2729
|
+
}
|
|
2730
|
+
if (ctx.durability !== null) {
|
|
2731
|
+
attachment.segment.removals.bind(manager.textAnalysis);
|
|
2732
|
+
for (const doc2 of attachment.members) {
|
|
2733
|
+
attachment.rollback.set(doc2.docId, {
|
|
2734
|
+
inserted: true,
|
|
2735
|
+
buffered: false,
|
|
2736
|
+
skipped: false,
|
|
2737
|
+
removedSpellings: attachment.segment.removedSpellingsOf(doc2.docId)
|
|
2738
|
+
});
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
for (const doc2 of attachment.members) {
|
|
2742
|
+
insertDocumentVectors(doc2.docId, doc2.extractedVectors, vectors, attachment.partitionId);
|
|
2743
|
+
inserted.push(doc2.docId);
|
|
2744
|
+
}
|
|
2745
|
+
manager.attachFrozenSegment(attachment.partitionId, attachment.segment);
|
|
2746
|
+
attachment.attached = true;
|
|
2747
|
+
for (const doc2 of attachment.members) attachment.applied.add(doc2.docId);
|
|
2748
|
+
ctx.orchestrator.publishToWorkers(attachment.action);
|
|
2749
|
+
} catch (error) {
|
|
2750
|
+
for (const docId of inserted) removeDocumentVectors(docId, vectors);
|
|
2751
|
+
attachment.failure = error;
|
|
2752
|
+
throw error;
|
|
2753
|
+
}
|
|
2754
|
+
};
|
|
2755
|
+
return async () => {
|
|
2756
|
+
await manager.publication.write(indexChangeOutsideExternalWaits(publish));
|
|
2757
|
+
if (!attachment.individual) return;
|
|
2758
|
+
const progress = { inserted: false, buffered: false, skipped: false };
|
|
2759
|
+
attachment.rollback.set(doc.docId, progress);
|
|
2760
|
+
try {
|
|
2761
|
+
return await applyInsertDocument(ctx, indexName, doc, options, progress)() ?? void 0;
|
|
2762
|
+
} finally {
|
|
2763
|
+
if (progress.inserted) attachment.applied.add(doc.docId);
|
|
2764
|
+
if (progress.buffered) attachment.buffered.add(doc.docId);
|
|
2765
|
+
}
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2769
|
+
// src/engine/mutations/segment-replication.ts
|
|
2770
|
+
function groupByPartition(docIds, partitionCount) {
|
|
2771
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2772
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
2773
|
+
const partitionId = partitionCount <= 1 ? 0 : fnv1a(docIds[i]) % partitionCount;
|
|
2774
|
+
let group = groups.get(partitionId);
|
|
2775
|
+
if (group === void 0) {
|
|
2776
|
+
group = [];
|
|
2777
|
+
groups.set(partitionId, group);
|
|
2778
|
+
}
|
|
2779
|
+
group.push(i);
|
|
2780
|
+
}
|
|
2781
|
+
return groups;
|
|
2782
|
+
}
|
|
2783
|
+
function buildSegmentRequests(indexName, docIds, documents, partitionCount, skipClone) {
|
|
2784
|
+
const groups = groupByPartition(docIds, partitionCount);
|
|
2785
|
+
const requests = [];
|
|
2786
|
+
const memberIndexes = [];
|
|
2787
|
+
for (const [partitionId, indexes] of groups) {
|
|
2788
|
+
requests.push({
|
|
2789
|
+
partitionId,
|
|
2790
|
+
action: {
|
|
2791
|
+
type: "buildSegment",
|
|
2792
|
+
indexName,
|
|
2793
|
+
segmentId: generateId(),
|
|
2794
|
+
documents: indexes.map((i) => ({ docId: docIds[i], document: documents[i] })),
|
|
2795
|
+
options: skipClone === true ? { skipClone: true } : void 0,
|
|
2796
|
+
requestId: `build-segment-${indexName}-${partitionId}`
|
|
2797
|
+
},
|
|
2798
|
+
documents: indexes.map((i) => documents[i])
|
|
2799
|
+
});
|
|
2800
|
+
memberIndexes.push(indexes);
|
|
2801
|
+
}
|
|
2802
|
+
return { requests, memberIndexes };
|
|
2803
|
+
}
|
|
2804
|
+
|
|
2805
|
+
// src/engine/mutations/insert-batch-segments.ts
|
|
2806
|
+
function mainStoreDocument(doc, options) {
|
|
2807
|
+
if (doc.extractedVectors.size > 0 || options?.skipClone === true) return doc.partitionDoc;
|
|
2808
|
+
return structuredClone(doc.partitionDoc);
|
|
2809
|
+
}
|
|
2810
|
+
function chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted) {
|
|
2811
|
+
const chunks = [];
|
|
2812
|
+
let chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
2813
|
+
for (let i = 0; i < built.length; i++) {
|
|
2814
|
+
const builtSegment = built[i];
|
|
2815
|
+
const requestId = `attach-${builtSegment.segmentId}`;
|
|
2816
|
+
const action = builtSegment.snapshot !== null ? {
|
|
2817
|
+
type: "attachSegments",
|
|
2818
|
+
indexName,
|
|
2819
|
+
requestId,
|
|
2820
|
+
segments: [{ partitionId: builtSegment.partitionId, snapshot: builtSegment.snapshot }]
|
|
2821
|
+
} : {
|
|
2822
|
+
type: "mergeSegments",
|
|
2823
|
+
indexName,
|
|
2824
|
+
requestId,
|
|
2825
|
+
segments: builtSegment.payload === null ? [] : [
|
|
2826
|
+
{
|
|
2827
|
+
partitionId: builtSegment.partitionId,
|
|
2828
|
+
payload: builtSegment.payload,
|
|
2829
|
+
documents: builtSegment.documents
|
|
2830
|
+
}
|
|
2831
|
+
]
|
|
2832
|
+
};
|
|
2833
|
+
const attachment = {
|
|
2834
|
+
partitionId: built[i].partitionId,
|
|
2835
|
+
partitionCount,
|
|
2836
|
+
segment: frozen[i],
|
|
2837
|
+
members: memberIndexes[i].map((member) => admitted[member]),
|
|
2838
|
+
applied: /* @__PURE__ */ new Set(),
|
|
2839
|
+
rollback: /* @__PURE__ */ new Map(),
|
|
2840
|
+
buffered: /* @__PURE__ */ new Set(),
|
|
2841
|
+
individual: false,
|
|
2842
|
+
attached: false,
|
|
2843
|
+
failure: void 0,
|
|
2844
|
+
action
|
|
2845
|
+
};
|
|
2846
|
+
for (const member of memberIndexes[i]) {
|
|
2847
|
+
const doc = admitted[member];
|
|
2848
|
+
chunk.docs.push(doc);
|
|
2849
|
+
chunk.attachmentOf.set(doc.docId, attachment);
|
|
2850
|
+
}
|
|
2851
|
+
if (chunk.docs.length >= BATCH_CHUNK_SIZE && i + 1 < built.length) {
|
|
2852
|
+
chunks.push(chunk);
|
|
2853
|
+
chunk = { docs: [], attachmentOf: /* @__PURE__ */ new Map() };
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
if (chunk.docs.length > 0) chunks.push(chunk);
|
|
2857
|
+
return chunks;
|
|
2858
|
+
}
|
|
2859
|
+
async function recordMergedDocuments(ctx, indexName, partitionCount, built, frozen, memberIndexes, admitted, options, hold, failed) {
|
|
2860
|
+
const succeededIds = /* @__PURE__ */ new Set();
|
|
2861
|
+
const errorOf = /* @__PURE__ */ new Map();
|
|
2862
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
2863
|
+
const chunks = chunksOfWholeSegments(indexName, partitionCount, built, frozen, memberIndexes, admitted);
|
|
2864
|
+
const unrecorded = [];
|
|
2865
|
+
for (let index = 0; index < chunks.length; index++) {
|
|
2866
|
+
const { docs, attachmentOf } = chunks[index];
|
|
2867
|
+
try {
|
|
2868
|
+
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
2869
|
+
} catch (error) {
|
|
2870
|
+
for (const later of chunks.slice(index)) {
|
|
2871
|
+
for (const doc of later.docs) errorOf.set(doc.docId, error);
|
|
2872
|
+
}
|
|
2873
|
+
break;
|
|
2874
|
+
}
|
|
2875
|
+
const applies = docs.map((doc) => {
|
|
2876
|
+
const attachment = attachmentOf.get(doc.docId);
|
|
2877
|
+
if (attachment === void 0) return async () => void 0;
|
|
2878
|
+
return applySegment(ctx, indexName, attachment, doc, options);
|
|
2879
|
+
});
|
|
2880
|
+
const failures = await recordChunk(ctx, indexName, docs, applies);
|
|
2881
|
+
await ctx.orchestrator.settlePublications(indexName, "segments");
|
|
2882
|
+
const inserted = [];
|
|
2883
|
+
for (let i = 0; i < docs.length; i++) {
|
|
2884
|
+
const doc = docs[i];
|
|
2885
|
+
const failure = failures[i];
|
|
2886
|
+
if (failure !== null) {
|
|
2887
|
+
let error = failure.error;
|
|
2888
|
+
const attachment = attachmentOf.get(doc.docId);
|
|
2889
|
+
const rollback = attachment?.rollback.get(doc.docId);
|
|
2890
|
+
try {
|
|
2891
|
+
await rollbackInsertedDocument(
|
|
2892
|
+
ctx,
|
|
2893
|
+
indexName,
|
|
2894
|
+
doc.docId,
|
|
2895
|
+
attachment?.applied.has(doc.docId) === true,
|
|
2896
|
+
error,
|
|
2897
|
+
rollback?.preparedText,
|
|
2898
|
+
rollback?.appliedRef,
|
|
2899
|
+
rollback?.removedSpellings
|
|
2900
|
+
);
|
|
2901
|
+
} catch (rollbackError) {
|
|
2902
|
+
error = rollbackError;
|
|
2903
|
+
}
|
|
2904
|
+
errorOf.set(doc.docId, error);
|
|
2905
|
+
continue;
|
|
2906
|
+
}
|
|
2907
|
+
if (attachmentOf.get(doc.docId)?.buffered.has(doc.docId) === true) {
|
|
2908
|
+
succeededIds.add(doc.docId);
|
|
2909
|
+
continue;
|
|
2910
|
+
}
|
|
2911
|
+
for (const fieldPath of doc.extractedVectors.keys()) {
|
|
2912
|
+
touchedVectorFields.add(fieldPath);
|
|
2913
|
+
}
|
|
2914
|
+
inserted.push(doc);
|
|
2915
|
+
succeededIds.add(doc.docId);
|
|
2916
|
+
}
|
|
2917
|
+
if (await runAfterInsertHooks(ctx, indexName, inserted, hold)) {
|
|
2918
|
+
for (const later of chunks.slice(index + 1)) unrecorded.push(...later.docs);
|
|
2919
|
+
break;
|
|
2920
|
+
}
|
|
2921
|
+
if (index + 1 < chunks.length) {
|
|
2922
|
+
ctx.checkHeapPressure(indexName);
|
|
2923
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
const succeeded = [];
|
|
2927
|
+
for (const doc of admitted) {
|
|
2928
|
+
if (succeededIds.has(doc.docId)) {
|
|
2929
|
+
succeeded.push(doc.docId);
|
|
2930
|
+
continue;
|
|
2931
|
+
}
|
|
2932
|
+
if (!errorOf.has(doc.docId)) continue;
|
|
2933
|
+
failed.push({ docId: doc.docId, error: asBatchInsertError(errorOf.get(doc.docId)) });
|
|
2934
|
+
}
|
|
2935
|
+
return { succeeded, touchedVectorFields, unrecorded };
|
|
2936
|
+
}
|
|
2937
|
+
function frozenSegmentOf(segment, members, options) {
|
|
2938
|
+
if (segment.snapshot !== null) return createSharedFrozenSegment(segment.snapshot);
|
|
2939
|
+
if (segment.payload === null) return null;
|
|
2940
|
+
return createFrozenSegment(
|
|
2941
|
+
segment.payload,
|
|
2942
|
+
members.map((doc) => mainStoreDocument(doc, options)),
|
|
2943
|
+
segment.segmentId
|
|
2944
|
+
);
|
|
2945
|
+
}
|
|
2946
|
+
async function ingestAdmitted(ctx, indexName, admitted, options, hold, failed) {
|
|
2947
|
+
if (admitted.length === 0) {
|
|
2948
|
+
return { succeeded: [], touchedVectorFields: /* @__PURE__ */ new Set(), unrecorded: [] };
|
|
2949
|
+
}
|
|
2950
|
+
const applyIndividually = () => applyAdmittedDocuments(ctx, indexName, admitted, options, hold, failed);
|
|
2951
|
+
const manager = ctx.requireManager(indexName);
|
|
2952
|
+
const workers = ctx.orchestrator.segmentBuildConcurrency(indexName);
|
|
2953
|
+
if (admitted.length < MIN_DOCUMENTS_FOR_SEGMENTS || workers <= 0) {
|
|
2954
|
+
return applyIndividually();
|
|
2955
|
+
}
|
|
2956
|
+
const docIds = admitted.map((doc) => doc.docId);
|
|
2957
|
+
const partitionDocuments = admitted.map((doc) => doc.partitionDoc);
|
|
2958
|
+
const partitionCount = manager.partitionCount;
|
|
2959
|
+
const { requests } = buildSegmentRequests(indexName, docIds, partitionDocuments, partitionCount, options?.skipClone);
|
|
2960
|
+
let built;
|
|
2961
|
+
try {
|
|
2962
|
+
built = await ctx.orchestrator.buildSegments(requests);
|
|
2963
|
+
} catch {
|
|
2964
|
+
built = null;
|
|
2965
|
+
}
|
|
2966
|
+
if (built === null) {
|
|
2967
|
+
return applyIndividually();
|
|
2968
|
+
}
|
|
2969
|
+
const positions = new Map(admitted.map((document, index) => [document.docId, index]));
|
|
2970
|
+
const used = /* @__PURE__ */ new Set();
|
|
2971
|
+
const memberIndexes = [];
|
|
2972
|
+
for (const segment of built) {
|
|
2973
|
+
const members = [];
|
|
2974
|
+
for (const docId of segment.docIds) {
|
|
2975
|
+
const position = positions.get(docId);
|
|
2976
|
+
if (position === void 0 || used.has(position)) return applyIndividually();
|
|
2977
|
+
used.add(position);
|
|
2978
|
+
members.push(position);
|
|
2979
|
+
}
|
|
2980
|
+
memberIndexes.push(members);
|
|
2981
|
+
}
|
|
2982
|
+
if (used.size !== admitted.length) return applyIndividually();
|
|
2983
|
+
if (ctx.isRebalancing(indexName)) {
|
|
2984
|
+
return applyIndividually();
|
|
2985
|
+
}
|
|
2986
|
+
const frozen = [];
|
|
2987
|
+
for (let i = 0; i < built.length; i++) {
|
|
2988
|
+
const segment = frozenSegmentOf(
|
|
2989
|
+
built[i],
|
|
2990
|
+
memberIndexes[i].map((m) => admitted[m]),
|
|
2991
|
+
options
|
|
2992
|
+
);
|
|
2993
|
+
if (segment === null) return applyIndividually();
|
|
2994
|
+
frozen.push(segment);
|
|
2995
|
+
}
|
|
2996
|
+
for (const doc of admitted) {
|
|
2997
|
+
if (manager.has(doc.docId)) return applyIndividually();
|
|
2998
|
+
}
|
|
2999
|
+
return recordMergedDocuments(
|
|
3000
|
+
ctx,
|
|
3001
|
+
indexName,
|
|
3002
|
+
partitionCount,
|
|
3003
|
+
built,
|
|
3004
|
+
frozen,
|
|
3005
|
+
memberIndexes,
|
|
3006
|
+
admitted,
|
|
3007
|
+
options,
|
|
3008
|
+
hold,
|
|
3009
|
+
failed
|
|
3010
|
+
);
|
|
3011
|
+
}
|
|
3012
|
+
async function insertBatchViaSegments(ctx, indexName, documents, options, hold) {
|
|
3013
|
+
const failed = [];
|
|
3014
|
+
const { admitted, embedding } = await admitBatchDocuments(ctx, indexName, documents, failed, hold);
|
|
3015
|
+
const outcome = await ingestAdmitted(ctx, indexName, admitted, options, hold, failed);
|
|
3016
|
+
if (outcome.unrecorded.length > 0) {
|
|
3017
|
+
const unrecorded = outcome.unrecorded.map((doc) => doc.document);
|
|
3018
|
+
const rest = await insertInChunks(ctx, indexName, unrecorded, options, hold, false, failed, embedding);
|
|
3019
|
+
outcome.succeeded.push(...rest.succeeded);
|
|
3020
|
+
for (const fieldPath of rest.touchedVectorFields) outcome.touchedVectorFields.add(fieldPath);
|
|
3021
|
+
}
|
|
3022
|
+
await finishInsertBatch(ctx, indexName, outcome, options);
|
|
3023
|
+
return { succeeded: outcome.succeeded, failed };
|
|
3024
|
+
}
|
|
3025
|
+
|
|
3026
|
+
// src/engine/mutations/insert-batch.ts
|
|
3027
|
+
async function insertDocumentBatch(ctx, indexName, documents, options, hold) {
|
|
3028
|
+
ctx.guardShutdown();
|
|
3029
|
+
ctx.requireIndex(indexName);
|
|
3030
|
+
validateBatchList(documents, "documents");
|
|
3031
|
+
await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
|
|
3032
|
+
const copiesBuildSegments = ctx.orchestrator.segmentBuildConcurrency(indexName) > 0;
|
|
3033
|
+
if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && copiesBuildSegments) {
|
|
3034
|
+
return insertBatchViaSegments(ctx, indexName, documents, options, hold);
|
|
3035
|
+
}
|
|
3036
|
+
const failed = [];
|
|
3037
|
+
const inserts = await insertInChunks(ctx, indexName, documents, options, hold, true, failed);
|
|
3038
|
+
await finishInsertBatch(ctx, indexName, inserts, options);
|
|
3039
|
+
return { succeeded: inserts.succeeded, failed };
|
|
3040
|
+
}
|
|
3041
|
+
|
|
3042
|
+
// src/engine/mutations/remove.ts
|
|
3043
|
+
function asRemoveError(err) {
|
|
3044
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
3045
|
+
}
|
|
3046
|
+
function validateRemoval(ctx, docId) {
|
|
3047
|
+
ctx.guardShutdown();
|
|
3048
|
+
validateDocId(docId);
|
|
3049
|
+
}
|
|
3050
|
+
function prepareRemoval(ctx, indexName, docId) {
|
|
3051
|
+
ctx.guardShutdown();
|
|
3052
|
+
const rebalancing = ctx.isRebalancing(indexName);
|
|
3053
|
+
if (ctx.durability && !rebalancing && !ctx.requireManager(indexName).has(docId)) {
|
|
3054
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
3055
|
+
}
|
|
3056
|
+
return {
|
|
3057
|
+
docId,
|
|
3058
|
+
document: null,
|
|
3059
|
+
restoreDoc: void 0,
|
|
3060
|
+
removedSpellings: void 0,
|
|
3061
|
+
buffered: false
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
function applyOfRemoval(ctx, indexName, prepared) {
|
|
3065
|
+
const { docId } = prepared;
|
|
3066
|
+
const manager = ctx.requireManager(indexName);
|
|
3067
|
+
const removeFromIndex = () => {
|
|
3068
|
+
if (ctx.isRebalancing(indexName)) {
|
|
3069
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
3070
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : ctx.requireManager(indexName).has(docId);
|
|
3071
|
+
if (!exists) {
|
|
3072
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
const queued = ctx.bufferIfRebalancing(indexName, { action: "remove", docId, indexName });
|
|
3076
|
+
if (queued !== void 0) {
|
|
3077
|
+
prepared.buffered = true;
|
|
3078
|
+
return queued;
|
|
3079
|
+
}
|
|
3080
|
+
prepared.restoreDoc = ctx.durability ? manager.get(docId) : void 0;
|
|
3081
|
+
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
3082
|
+
manager.remove(docId, void 0, {
|
|
3083
|
+
onRemovedSpellings: (spellings) => {
|
|
3084
|
+
prepared.removedSpellings = spellings;
|
|
3085
|
+
},
|
|
3086
|
+
onPreparedText: ctx.durability === null ? void 0 : (change) => {
|
|
3087
|
+
prepared.preparedText = change;
|
|
3088
|
+
}
|
|
3089
|
+
});
|
|
3090
|
+
removeDocumentVectors(docId, ctx.requireManager(indexName).getVectorIndexes());
|
|
3091
|
+
ctx.orchestrator.publishToWorkers({
|
|
3092
|
+
type: "remove",
|
|
3093
|
+
indexName,
|
|
3094
|
+
docId,
|
|
3095
|
+
requestId: `replicate-remove-${docId}`,
|
|
3096
|
+
removedSpellings: prepared.removedSpellings
|
|
3097
|
+
});
|
|
3098
|
+
};
|
|
3099
|
+
return () => manager.publication.writeNow(indexChangeOutsideExternalWaits(removeFromIndex));
|
|
3100
|
+
}
|
|
3101
|
+
async function settleRemovals(ctx, indexName, docIds, hold) {
|
|
3102
|
+
if (docIds.length === 0) return;
|
|
3103
|
+
const settling = ctx.orchestrator.settlePublications(indexName, "none");
|
|
3104
|
+
if (settling !== void 0) await settling;
|
|
3105
|
+
if (!ctx.pluginRegistry.hasHooks("afterRemove")) return;
|
|
3106
|
+
await runOutside(hold, async () => {
|
|
3107
|
+
for (const docId of docIds) {
|
|
3108
|
+
try {
|
|
3109
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
3110
|
+
} catch (err) {
|
|
3111
|
+
console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3112
|
+
}
|
|
3113
|
+
}
|
|
3114
|
+
});
|
|
3115
|
+
}
|
|
3116
|
+
async function removeDocument2(ctx, indexName, docId, options, hold) {
|
|
3117
|
+
ctx.guardShutdown();
|
|
3118
|
+
ctx.requireIndex(indexName);
|
|
3119
|
+
validateRemoval(ctx, docId);
|
|
3120
|
+
if (ctx.pluginRegistry.hasHooks("beforeRemove")) {
|
|
3121
|
+
await runOutside(hold, async () => {
|
|
3122
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
3123
|
+
});
|
|
3124
|
+
}
|
|
3125
|
+
const prepared = prepareRemoval(ctx, indexName, docId);
|
|
3126
|
+
const apply = applyOfRemoval(ctx, indexName, prepared);
|
|
3127
|
+
if (ctx.durability) {
|
|
3128
|
+
try {
|
|
3129
|
+
await ctx.durability.recordRemove(indexName, docId, apply);
|
|
3130
|
+
} catch (err) {
|
|
3131
|
+
await rollbackRemovedDocument(ctx, indexName, docId, prepared.restoreDoc, err, prepared.preparedText);
|
|
3132
|
+
throw err;
|
|
3133
|
+
} finally {
|
|
3134
|
+
prepared.releaseVectorHolds?.();
|
|
3135
|
+
}
|
|
3136
|
+
} else {
|
|
3137
|
+
await apply();
|
|
3138
|
+
}
|
|
3139
|
+
if (!prepared.buffered) await settleRemovals(ctx, indexName, [docId], hold);
|
|
3140
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3141
|
+
}
|
|
3142
|
+
async function refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold) {
|
|
3143
|
+
const refused = /* @__PURE__ */ new Map();
|
|
3144
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3145
|
+
try {
|
|
3146
|
+
validateRemoval(ctx, docIds[i]);
|
|
3147
|
+
} catch (err) {
|
|
3148
|
+
refused.set(i, err);
|
|
3149
|
+
}
|
|
3150
|
+
}
|
|
3151
|
+
if (!ctx.pluginRegistry.hasHooks("beforeRemove")) return refused;
|
|
3152
|
+
await runOutside(hold, async () => {
|
|
3153
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3154
|
+
if (refused.has(i)) continue;
|
|
3155
|
+
try {
|
|
3156
|
+
await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId: docIds[i] });
|
|
3157
|
+
} catch (err) {
|
|
3158
|
+
refused.set(i, err);
|
|
3159
|
+
}
|
|
3160
|
+
}
|
|
3161
|
+
});
|
|
3162
|
+
return refused;
|
|
3163
|
+
}
|
|
3164
|
+
async function removeDocumentBatch(ctx, indexName, docIds, options, hold) {
|
|
3165
|
+
ctx.guardShutdown();
|
|
3166
|
+
ctx.requireIndex(indexName);
|
|
3167
|
+
validateBatchList(docIds, "document IDs");
|
|
3168
|
+
const succeeded = [];
|
|
3169
|
+
const failed = [];
|
|
3170
|
+
const first = ctx.requireManager(indexName);
|
|
3171
|
+
await first.publication.write(() => first.beginBatchRemove());
|
|
3172
|
+
let manager = first;
|
|
3173
|
+
try {
|
|
3174
|
+
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3175
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
3176
|
+
const refused = await refusedRemovalsOf(ctx, indexName, docIds, chunkStart, chunkEnd, hold);
|
|
3177
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3178
|
+
const removed = [];
|
|
3179
|
+
const prepared = [];
|
|
3180
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3181
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3182
|
+
try {
|
|
3183
|
+
if (refused.has(i)) throw refused.get(i);
|
|
3184
|
+
prepared.push(prepareRemoval(ctx, indexName, docIds[i]));
|
|
3185
|
+
} catch (err) {
|
|
3186
|
+
failed.push({ docId: docIds[i], error: asRemoveError(err) });
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3189
|
+
const applies = prepared.map((removal) => applyOfRemoval(ctx, indexName, removal));
|
|
3190
|
+
try {
|
|
3191
|
+
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
3192
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
3193
|
+
const removal = prepared[i];
|
|
3194
|
+
const failure = failures[i];
|
|
3195
|
+
if (failure !== null) {
|
|
3196
|
+
let error = failure.error;
|
|
3197
|
+
try {
|
|
3198
|
+
await rollbackRemovedDocument(
|
|
3199
|
+
ctx,
|
|
3200
|
+
indexName,
|
|
3201
|
+
removal.docId,
|
|
3202
|
+
removal.restoreDoc,
|
|
3203
|
+
error,
|
|
3204
|
+
removal.preparedText
|
|
3205
|
+
);
|
|
3206
|
+
} catch (rollbackError) {
|
|
3207
|
+
error = rollbackError;
|
|
3208
|
+
}
|
|
3209
|
+
removal.releaseVectorHolds?.();
|
|
3210
|
+
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
3211
|
+
continue;
|
|
3212
|
+
}
|
|
3213
|
+
removal.releaseVectorHolds?.();
|
|
3214
|
+
if (!removal.buffered) removed.push(removal.docId);
|
|
3215
|
+
succeeded.push(removal.docId);
|
|
3216
|
+
}
|
|
3217
|
+
} finally {
|
|
3218
|
+
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
3219
|
+
}
|
|
3220
|
+
await settleRemovals(ctx, indexName, removed, hold);
|
|
3221
|
+
manager = await batchManagerAfterHooks(ctx, indexName, manager);
|
|
3222
|
+
if (chunkEnd < docIds.length) {
|
|
3223
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
3224
|
+
}
|
|
3225
|
+
}
|
|
3226
|
+
} finally {
|
|
3227
|
+
const ending = manager;
|
|
3228
|
+
await ending?.publication.write(() => ending.endBatchRemove());
|
|
3229
|
+
}
|
|
3230
|
+
ctx.checkHeapPressure(indexName);
|
|
3231
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3232
|
+
return { succeeded, failed };
|
|
3233
|
+
}
|
|
3234
|
+
async function batchManagerAfterHooks(ctx, indexName, batchManager) {
|
|
3235
|
+
const current = ctx.executor.getManager(indexName);
|
|
3236
|
+
if (current === batchManager) return batchManager;
|
|
3237
|
+
await batchManager?.publication.write(() => batchManager.endBatchRemove());
|
|
3238
|
+
await current?.publication.write(() => current.beginBatchRemove());
|
|
3239
|
+
return current;
|
|
3240
|
+
}
|
|
3241
|
+
|
|
3242
|
+
// src/engine/mutations/update.ts
|
|
3243
|
+
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
3244
|
+
if (extractedVectors.size === 0) {
|
|
3245
|
+
return { partitionDoc: document };
|
|
3246
|
+
}
|
|
3247
|
+
let source = document;
|
|
3248
|
+
const prototype = Object.getPrototypeOf(document);
|
|
3249
|
+
let topLevelVectors = prototype === Object.prototype || prototype === null;
|
|
3250
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
3251
|
+
if (fieldPath.includes(".") || vector === null) {
|
|
3252
|
+
topLevelVectors = false;
|
|
3253
|
+
break;
|
|
3254
|
+
}
|
|
3255
|
+
const value = document[fieldPath];
|
|
3256
|
+
if (Array.isArray(value) && Object.keys(value).length !== value.length) {
|
|
3257
|
+
topLevelVectors = false;
|
|
3258
|
+
break;
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
if (topLevelVectors) {
|
|
3262
|
+
source = { ...document };
|
|
3263
|
+
for (const fieldPath of extractedVectors.keys()) delete source[fieldPath];
|
|
3264
|
+
}
|
|
3265
|
+
const partitionDoc = structuredClone(source);
|
|
3266
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
3267
|
+
if (vector !== null || getNestedValue(partitionDoc, fieldPath) == null) deleteNestedValue(partitionDoc, fieldPath);
|
|
3268
|
+
}
|
|
3269
|
+
return { partitionDoc };
|
|
3270
|
+
}
|
|
3271
|
+
function asUpdateError(err) {
|
|
3272
|
+
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err));
|
|
3273
|
+
}
|
|
3274
|
+
async function runAfterUpdateHooks(ctx, indexName, updates) {
|
|
3275
|
+
for (const update of updates) {
|
|
3276
|
+
try {
|
|
3277
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
3278
|
+
indexName,
|
|
3279
|
+
docId: update.docId,
|
|
3280
|
+
oldDocument: update.oldDocument ?? {},
|
|
3281
|
+
newDocument: update.document
|
|
3282
|
+
});
|
|
3283
|
+
} catch (err) {
|
|
3284
|
+
console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
}
|
|
3288
|
+
async function settleUpdates(ctx, indexName, updates, hold) {
|
|
3289
|
+
if (updates.length === 0) return;
|
|
3290
|
+
await ctx.orchestrator.settlePublications(indexName);
|
|
3291
|
+
if (ctx.pluginRegistry.hasHooks("afterUpdate")) {
|
|
3292
|
+
await runOutside(hold, () => runAfterUpdateHooks(ctx, indexName, updates));
|
|
3293
|
+
}
|
|
3294
|
+
const vecIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3295
|
+
if (vecIndexes === void 0) return;
|
|
3296
|
+
for (const update of updates) {
|
|
3297
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3298
|
+
if (vec !== null) vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
3299
|
+
}
|
|
3300
|
+
}
|
|
3301
|
+
}
|
|
3302
|
+
function updatePreparationOf(ctx, indexName, docId, document) {
|
|
3303
|
+
ctx.guardShutdown();
|
|
3304
|
+
validateDocId(docId);
|
|
3305
|
+
requireMatchingDocId(docId, document);
|
|
3306
|
+
if (!ctx.pluginRegistry.hasHooks("beforeUpdate")) return { document, beforeWrite: null, failure: void 0 };
|
|
3307
|
+
const oldDocument = ctx.requireManager(indexName).get(docId) ?? {};
|
|
3308
|
+
const beforeWrite = () => ctx.pluginRegistry.runHook("beforeUpdate", { indexName, docId, oldDocument, newDocument: document });
|
|
3309
|
+
return { document, beforeWrite, failure: void 0 };
|
|
3310
|
+
}
|
|
3311
|
+
async function finishUpdate(ctx, indexName, entry, docId, document) {
|
|
3312
|
+
const sizing = requireDocumentWithinSizeLimit(document);
|
|
3313
|
+
if (sizing !== null) await sizing;
|
|
3314
|
+
const manager = ctx.requireManager(indexName);
|
|
3315
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
3316
|
+
const extractedVectors = /* @__PURE__ */ new Map();
|
|
3317
|
+
if (vecIndexes.size > 0) {
|
|
3318
|
+
const dimensionCheckVectors = /* @__PURE__ */ new Map();
|
|
3319
|
+
for (const fieldPath of entry.vectorFieldPaths) {
|
|
3320
|
+
const newVec = extractVectorFromDoc(document, fieldPath);
|
|
3321
|
+
extractedVectors.set(fieldPath, newVec);
|
|
3322
|
+
if (newVec) {
|
|
3323
|
+
dimensionCheckVectors.set(fieldPath, newVec);
|
|
3324
|
+
}
|
|
3325
|
+
}
|
|
3326
|
+
if (dimensionCheckVectors.size > 0) {
|
|
3327
|
+
validateVectorDimensions(dimensionCheckVectors, vecIndexes);
|
|
3328
|
+
}
|
|
3329
|
+
}
|
|
3330
|
+
const { partitionDoc } = prepareUpdatePartitionDoc(document, extractedVectors);
|
|
3331
|
+
return {
|
|
3332
|
+
docId,
|
|
3333
|
+
document,
|
|
3334
|
+
partitionDoc,
|
|
3335
|
+
oldDocument: void 0,
|
|
3336
|
+
rollbackDoc: void 0,
|
|
3337
|
+
extractedVectors,
|
|
3338
|
+
removedSpellings: void 0,
|
|
3339
|
+
buffered: false
|
|
3340
|
+
};
|
|
3341
|
+
}
|
|
3342
|
+
function applyOfUpdate(ctx, indexName, prepared) {
|
|
3343
|
+
const manager = ctx.requireManager(indexName);
|
|
3344
|
+
const { docId, document, partitionDoc, extractedVectors } = prepared;
|
|
3345
|
+
const updateInIndex = () => {
|
|
3346
|
+
if (ctx.isRebalancing(indexName)) {
|
|
3347
|
+
const bufferedState = ctx.bufferedDocState(indexName, docId);
|
|
3348
|
+
const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
|
|
3349
|
+
if (!exists) {
|
|
3350
|
+
admitBufferedUpsert(ctx, indexName, manager, docId);
|
|
3351
|
+
}
|
|
3352
|
+
}
|
|
3353
|
+
const queued = ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName });
|
|
3354
|
+
if (queued !== void 0) {
|
|
3355
|
+
prepared.buffered = true;
|
|
3356
|
+
return queued;
|
|
3357
|
+
}
|
|
3358
|
+
if (manager.has(docId)) {
|
|
3359
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
3360
|
+
if (vector !== null) manager.getVectorIndexes().get(fieldPath)?.requireRoomFor(docId, vector);
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
prepared.oldDocument = extractedVectors.size > 0 || ctx.durability !== null || ctx.pluginRegistry.hasHooks("afterUpdate") ? manager.get(docId) : void 0;
|
|
3364
|
+
prepared.rollbackDoc = prepared.oldDocument;
|
|
3365
|
+
prepared.rollbackRef = ctx.durability === null ? void 0 : manager.getRef(docId);
|
|
3366
|
+
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
3367
|
+
const stored = manager.update(docId, partitionDoc, {
|
|
3368
|
+
onRemovedSpellings: (spellings) => {
|
|
3369
|
+
prepared.removedSpellings = spellings;
|
|
3370
|
+
},
|
|
3371
|
+
onPreparedText: extractedVectors.size > 0 || ctx.durability !== null ? (change) => {
|
|
3372
|
+
prepared.preparedText = change;
|
|
3373
|
+
} : void 0
|
|
3374
|
+
});
|
|
3375
|
+
try {
|
|
3376
|
+
updateDocumentVectors(docId, extractedVectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
|
|
3377
|
+
} catch (err) {
|
|
3378
|
+
if (prepared.rollbackDoc === void 0) {
|
|
3379
|
+
manager.remove(docId, void 0, { preparedText: { before: prepared.preparedText?.after } });
|
|
3380
|
+
} else {
|
|
3381
|
+
try {
|
|
3382
|
+
const previous = prepareDocumentVectors(prepared.rollbackDoc, ctx.requireIndex(indexName).vectorFieldPaths);
|
|
3383
|
+
manager.update(docId, previous.partitionDoc, {
|
|
3384
|
+
preparedText: { before: prepared.preparedText?.after, after: prepared.preparedText?.before }
|
|
3385
|
+
});
|
|
3386
|
+
} catch (rollbackErr) {
|
|
3387
|
+
console.warn(
|
|
3388
|
+
`Rollback failed for doc "${docId}" during update atomicity:`,
|
|
3389
|
+
rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
|
|
3390
|
+
);
|
|
3391
|
+
}
|
|
3392
|
+
}
|
|
3393
|
+
throw err;
|
|
3394
|
+
}
|
|
3395
|
+
if (ctx.durability === null) prepared.preparedText = void 0;
|
|
3396
|
+
else prepared.appliedRef = stored;
|
|
3397
|
+
ctx.orchestrator.publishToWorkers({
|
|
3398
|
+
type: "update",
|
|
3399
|
+
indexName,
|
|
3400
|
+
docId,
|
|
3401
|
+
document: partitionDoc,
|
|
3402
|
+
requestId: `replicate-update-${docId}`,
|
|
3403
|
+
removedSpellings: prepared.removedSpellings
|
|
3404
|
+
});
|
|
3405
|
+
};
|
|
3406
|
+
return () => manager.publication.write(indexChangeOutsideExternalWaits(updateInIndex));
|
|
3407
|
+
}
|
|
3408
|
+
async function updateDocument(ctx, indexName, docId, document, options, hold) {
|
|
3409
|
+
ctx.guardShutdown();
|
|
3410
|
+
ctx.requireIndex(indexName);
|
|
3411
|
+
const preparation = updatePreparationOf(ctx, indexName, docId, document);
|
|
3412
|
+
const { entry } = await prepareDocumentWrites(ctx, indexName, [preparation], hold, false);
|
|
3413
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3414
|
+
const prepared = await finishUpdate(ctx, indexName, entry, docId, document);
|
|
3415
|
+
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
3416
|
+
const apply = applyOfUpdate(ctx, indexName, prepared);
|
|
3417
|
+
if (ctx.durability) {
|
|
3418
|
+
try {
|
|
3419
|
+
await ctx.durability.recordInsertOrUpdate(indexName, docId, document, apply);
|
|
3420
|
+
} catch (err) {
|
|
3421
|
+
await rollbackUpdatedDocument(
|
|
3422
|
+
ctx,
|
|
3423
|
+
indexName,
|
|
3424
|
+
docId,
|
|
3425
|
+
prepared.rollbackDoc,
|
|
3426
|
+
err,
|
|
3427
|
+
prepared.preparedText,
|
|
3428
|
+
prepared.appliedRef,
|
|
3429
|
+
prepared.rollbackRef
|
|
3430
|
+
);
|
|
3431
|
+
throw err;
|
|
3432
|
+
} finally {
|
|
3433
|
+
prepared.releaseVectorHolds?.();
|
|
3434
|
+
}
|
|
3435
|
+
} else {
|
|
3436
|
+
await apply();
|
|
3437
|
+
}
|
|
3438
|
+
if (!prepared.buffered) await settleUpdates(ctx, indexName, [prepared], hold);
|
|
3439
|
+
ctx.checkHeapPressure(indexName);
|
|
3440
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3441
|
+
}
|
|
3442
|
+
|
|
3443
|
+
// src/engine/mutations/update-batch.ts
|
|
3444
|
+
function reportedDocIdOf(update) {
|
|
3445
|
+
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
3446
|
+
return typeof docId === "string" ? docId : "";
|
|
3447
|
+
}
|
|
3448
|
+
function requireUpdateEntry(update) {
|
|
3449
|
+
if (update !== null && typeof update === "object" && !Array.isArray(update)) {
|
|
3450
|
+
const { docId, document } = update;
|
|
3451
|
+
if (typeof docId === "string" && document !== null && typeof document === "object" && !Array.isArray(document)) {
|
|
3452
|
+
return { document };
|
|
3453
|
+
}
|
|
3454
|
+
}
|
|
3455
|
+
throw new NarsilError(
|
|
3456
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
3457
|
+
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
3458
|
+
);
|
|
3459
|
+
}
|
|
3460
|
+
async function prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed) {
|
|
3461
|
+
const preparations = [];
|
|
3462
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
3463
|
+
const update = updates[i];
|
|
3464
|
+
const docId = reportedDocIdOf(update);
|
|
3465
|
+
try {
|
|
3466
|
+
const { document } = requireUpdateEntry(update);
|
|
3467
|
+
preparations.push({ docId, preparation: updatePreparationOf(ctx, indexName, docId, document) });
|
|
3468
|
+
} catch (err) {
|
|
3469
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3470
|
+
}
|
|
3471
|
+
}
|
|
3472
|
+
const { entry } = await prepareDocumentWrites(
|
|
3473
|
+
ctx,
|
|
3474
|
+
indexName,
|
|
3475
|
+
preparations.map((pending) => pending.preparation),
|
|
3476
|
+
hold,
|
|
3477
|
+
false
|
|
3478
|
+
);
|
|
3479
|
+
const prepared = [];
|
|
3480
|
+
for (const { docId, preparation } of preparations) {
|
|
3481
|
+
if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
|
|
3482
|
+
try {
|
|
3483
|
+
if (preparation.failure !== void 0) throw preparation.failure;
|
|
3484
|
+
prepared.push(await finishUpdate(ctx, indexName, entry, docId, preparation.document));
|
|
3485
|
+
} catch (err) {
|
|
3486
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
3487
|
+
}
|
|
3488
|
+
}
|
|
3489
|
+
return prepared;
|
|
3490
|
+
}
|
|
3491
|
+
async function updateDocumentBatch(ctx, indexName, updates, options, hold) {
|
|
3492
|
+
ctx.guardShutdown();
|
|
3493
|
+
ctx.requireIndex(indexName);
|
|
3494
|
+
validateBatchList(updates, "updates");
|
|
3495
|
+
const succeeded = [];
|
|
3496
|
+
const failed = [];
|
|
3497
|
+
const touchedVectorFields = /* @__PURE__ */ new Set();
|
|
3498
|
+
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
3499
|
+
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
3500
|
+
const prepared = await prepareChunk(ctx, indexName, updates, chunkStart, chunkEnd, hold, failed);
|
|
3501
|
+
try {
|
|
3502
|
+
ctx.memoryWriteBlock?.refuseWhileFull(indexName);
|
|
3503
|
+
} catch (err) {
|
|
3504
|
+
const error = asUpdateError(err);
|
|
3505
|
+
for (const update of prepared) failed.push({ docId: update.docId, error });
|
|
3506
|
+
for (let i = chunkEnd; i < updates.length; i++) failed.push({ docId: reportedDocIdOf(updates[i]), error });
|
|
3507
|
+
break;
|
|
3508
|
+
}
|
|
3509
|
+
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3510
|
+
let failures;
|
|
3511
|
+
try {
|
|
3512
|
+
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
3513
|
+
for (let i = prepared.length - 1; i >= 0; i--) {
|
|
3514
|
+
const failure = failures[i];
|
|
3515
|
+
if (failure === null) continue;
|
|
3516
|
+
const update = prepared[i];
|
|
3517
|
+
try {
|
|
3518
|
+
await rollbackUpdatedDocument(
|
|
3519
|
+
ctx,
|
|
3520
|
+
indexName,
|
|
3521
|
+
update.docId,
|
|
3522
|
+
update.rollbackDoc,
|
|
3523
|
+
failure.error,
|
|
3524
|
+
update.preparedText,
|
|
3525
|
+
update.appliedRef,
|
|
3526
|
+
update.rollbackRef
|
|
3527
|
+
);
|
|
3528
|
+
} catch (error) {
|
|
3529
|
+
failure.error = error;
|
|
3530
|
+
}
|
|
3531
|
+
}
|
|
3532
|
+
} finally {
|
|
3533
|
+
for (const update of prepared) update.releaseVectorHolds?.();
|
|
3534
|
+
}
|
|
3535
|
+
const settled = [];
|
|
3536
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
3537
|
+
const update = prepared[i];
|
|
3538
|
+
const failure = failures[i];
|
|
3539
|
+
if (failure !== null) {
|
|
3540
|
+
failed.push({ docId: update.docId, error: asUpdateError(failure.error) });
|
|
3541
|
+
continue;
|
|
3542
|
+
}
|
|
3543
|
+
if (!update.buffered) settled.push(update);
|
|
3544
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
3545
|
+
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
3546
|
+
}
|
|
3547
|
+
succeeded.push(update.docId);
|
|
3548
|
+
}
|
|
3549
|
+
await settleUpdates(ctx, indexName, settled, hold);
|
|
3550
|
+
if (chunkEnd < updates.length) {
|
|
3551
|
+
ctx.checkHeapPressure(indexName);
|
|
3552
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
const vectorIndexes = ctx.executor.getManager(indexName)?.getVectorIndexes();
|
|
3556
|
+
for (const fieldPath of touchedVectorFields) {
|
|
3557
|
+
vectorIndexes?.get(fieldPath)?.scheduleBuild();
|
|
3558
|
+
}
|
|
3559
|
+
ctx.checkHeapPressure(indexName);
|
|
3560
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
3561
|
+
return { succeeded, failed };
|
|
3562
|
+
}
|
|
3563
|
+
|
|
3564
|
+
// src/engine/partition-config.ts
|
|
3565
|
+
async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
|
|
3566
|
+
const entry = core.requireIndex(indexName);
|
|
3567
|
+
const manager = core.requireManager(indexName);
|
|
3568
|
+
if (core.rebalancingIndexes.has(indexName)) {
|
|
3569
|
+
throw new NarsilError(
|
|
3570
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
3571
|
+
`Index "${indexName}" is currently being rebalanced`
|
|
3572
|
+
);
|
|
3573
|
+
}
|
|
3574
|
+
validatePartitionConfig(partitionConfig);
|
|
3575
|
+
if (partitionConfig.maxPartitions !== void 0 && partitionConfig.maxPartitions < manager.partitionCount) {
|
|
3576
|
+
throw new NarsilError(
|
|
3577
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
3578
|
+
`maxPartitions (${partitionConfig.maxPartitions}) is less than the current partition count (${manager.partitionCount})`,
|
|
3579
|
+
{ maxPartitions: partitionConfig.maxPartitions, partitionCount: manager.partitionCount }
|
|
3580
|
+
);
|
|
3581
|
+
}
|
|
3582
|
+
const currentDocCount = manager.countDocuments();
|
|
3583
|
+
const newMaxDocs = partitionConfig.maxDocsPerPartition ?? entry.config.partitions?.maxDocsPerPartition;
|
|
3584
|
+
if (newMaxDocs !== void 0) {
|
|
3585
|
+
const newTotalCapacity = newMaxDocs * manager.partitionCount;
|
|
3586
|
+
if (newTotalCapacity < currentDocCount) {
|
|
3587
|
+
throw new NarsilError(
|
|
3588
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
3589
|
+
`New capacity (${newTotalCapacity}) is less than current document count (${currentDocCount})`,
|
|
3590
|
+
{ newTotalCapacity, currentDocCount }
|
|
3591
|
+
);
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3594
|
+
if (!entry.config.partitions) entry.config.partitions = {};
|
|
3595
|
+
if (partitionConfig.maxDocsPerPartition !== void 0)
|
|
3596
|
+
entry.config.partitions.maxDocsPerPartition = partitionConfig.maxDocsPerPartition;
|
|
3597
|
+
if (partitionConfig.maxPartitions !== void 0) entry.config.partitions.maxPartitions = partitionConfig.maxPartitions;
|
|
3598
|
+
if (partitionConfig.watermark !== void 0) entry.config.partitions.watermark = partitionConfig.watermark;
|
|
3599
|
+
if (core.durability) await core.durability.manager.persistMetadata(indexName);
|
|
3600
|
+
core.watermarkNotifier.check(indexName);
|
|
3601
|
+
}
|
|
3602
|
+
|
|
3603
|
+
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|