@delali/narsil 0.3.1-next.98 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -0,0 +1,1636 @@
|
|
|
1
|
+
// src/core/segment-merge-host.browser.ts
|
|
2
|
+
var ErrorCodes;
|
|
3
|
+
var GrowingSegment;
|
|
4
|
+
var LivePostings;
|
|
5
|
+
var MergeRegions;
|
|
6
|
+
var NarsilError;
|
|
7
|
+
var SegmentRefusal;
|
|
8
|
+
var bitsetIterator;
|
|
9
|
+
var buildPartialWordTableFrom;
|
|
10
|
+
var captureInputs;
|
|
11
|
+
var createBooleanIndex;
|
|
12
|
+
var createEnumIndex;
|
|
13
|
+
var createFieldMerge;
|
|
14
|
+
var createLoadedGeoIndex;
|
|
15
|
+
var createNumericIndex;
|
|
16
|
+
var createSpellingTable;
|
|
17
|
+
var createSurfaceRegistry;
|
|
18
|
+
var documentsOf;
|
|
19
|
+
var embedded;
|
|
20
|
+
var encodeExternalIdTableData;
|
|
21
|
+
var encodeFieldIndexes;
|
|
22
|
+
var encodeFrozenTokenTableData;
|
|
23
|
+
var encodePartialSpellingsOf;
|
|
24
|
+
var encodeStringBlob;
|
|
25
|
+
var encodeSurfaceTable;
|
|
26
|
+
var generateId;
|
|
27
|
+
var getFlatSchema;
|
|
28
|
+
var getNestedValue;
|
|
29
|
+
var isWordIndexedBase;
|
|
30
|
+
var mergeFieldRegionsWith;
|
|
31
|
+
var mergeKeywordSources;
|
|
32
|
+
var mergeStoredFields;
|
|
33
|
+
var mergeTextAt;
|
|
34
|
+
var numbers;
|
|
35
|
+
var partialSpellingsOfDocuments;
|
|
36
|
+
var patternIndexOf;
|
|
37
|
+
var prepareDocumentText;
|
|
38
|
+
var rebuildSizeIndexes;
|
|
39
|
+
var recordValueSize;
|
|
40
|
+
var regionLength;
|
|
41
|
+
var regionsOverlap;
|
|
42
|
+
var requirePatternSearch;
|
|
43
|
+
var requireRoom;
|
|
44
|
+
var requireSegment;
|
|
45
|
+
var sameFields;
|
|
46
|
+
var schemaFieldsOf;
|
|
47
|
+
var storedPlan;
|
|
48
|
+
var storedText;
|
|
49
|
+
var typedRegion;
|
|
50
|
+
var uint32;
|
|
51
|
+
var validateDocument;
|
|
52
|
+
var validateDocumentStrict;
|
|
53
|
+
var words;
|
|
54
|
+
var wrapFrozenTokenTable;
|
|
55
|
+
var encodeDocument;
|
|
56
|
+
var encodeDocumentTableData;
|
|
57
|
+
var storedDocumentsOf;
|
|
58
|
+
var topLevelMapKeys;
|
|
59
|
+
function bindSegmentMergeHost(host, msgpack, segmentCodec) {
|
|
60
|
+
({
|
|
61
|
+
ErrorCodes,
|
|
62
|
+
GrowingSegment,
|
|
63
|
+
LivePostings,
|
|
64
|
+
MergeRegions,
|
|
65
|
+
NarsilError,
|
|
66
|
+
SegmentRefusal,
|
|
67
|
+
bitsetIterator,
|
|
68
|
+
buildPartialWordTableFrom,
|
|
69
|
+
captureInputs,
|
|
70
|
+
createBooleanIndex,
|
|
71
|
+
createEnumIndex,
|
|
72
|
+
createFieldMerge,
|
|
73
|
+
createLoadedGeoIndex,
|
|
74
|
+
createNumericIndex,
|
|
75
|
+
createSpellingTable,
|
|
76
|
+
createSurfaceRegistry,
|
|
77
|
+
documentsOf,
|
|
78
|
+
embedded,
|
|
79
|
+
encodeExternalIdTableData,
|
|
80
|
+
encodeFieldIndexes,
|
|
81
|
+
encodeFrozenTokenTableData,
|
|
82
|
+
encodePartialSpellingsOf,
|
|
83
|
+
encodeStringBlob,
|
|
84
|
+
encodeSurfaceTable,
|
|
85
|
+
generateId,
|
|
86
|
+
getFlatSchema,
|
|
87
|
+
getNestedValue,
|
|
88
|
+
isWordIndexedBase,
|
|
89
|
+
mergeFieldRegionsWith,
|
|
90
|
+
mergeKeywordSources,
|
|
91
|
+
mergeStoredFields,
|
|
92
|
+
mergeTextAt,
|
|
93
|
+
numbers,
|
|
94
|
+
partialSpellingsOfDocuments,
|
|
95
|
+
patternIndexOf,
|
|
96
|
+
prepareDocumentText,
|
|
97
|
+
rebuildSizeIndexes,
|
|
98
|
+
recordValueSize,
|
|
99
|
+
regionLength,
|
|
100
|
+
regionsOverlap,
|
|
101
|
+
requirePatternSearch,
|
|
102
|
+
requireRoom,
|
|
103
|
+
requireSegment,
|
|
104
|
+
sameFields,
|
|
105
|
+
schemaFieldsOf,
|
|
106
|
+
storedPlan,
|
|
107
|
+
storedText,
|
|
108
|
+
typedRegion,
|
|
109
|
+
uint32,
|
|
110
|
+
validateDocument,
|
|
111
|
+
validateDocumentStrict,
|
|
112
|
+
words,
|
|
113
|
+
wrapFrozenTokenTable
|
|
114
|
+
} = host);
|
|
115
|
+
({ encodeDocument } = msgpack);
|
|
116
|
+
({ encodeDocumentTableData, storedDocumentsOf, topLevelMapKeys } = segmentCodec);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// src/engine/orchestration/eligibility.ts
|
|
120
|
+
function toError(value) {
|
|
121
|
+
return value instanceof Error ? value : new Error(String(value));
|
|
122
|
+
}
|
|
123
|
+
function isDeterministicFailure(error) {
|
|
124
|
+
return error instanceof NarsilError && error.code === ErrorCodes.CONFIG_INVALID;
|
|
125
|
+
}
|
|
126
|
+
function alreadyPresentOnWorker(reason) {
|
|
127
|
+
return reason instanceof NarsilError && reason.code === ErrorCodes.DOC_ALREADY_EXISTS;
|
|
128
|
+
}
|
|
129
|
+
function assertConfigReachesWorker(indexName, config, bootstrapModule) {
|
|
130
|
+
if (config.tokenizer !== void 0 && typeof config.tokenizer !== "string") {
|
|
131
|
+
throw new NarsilError(
|
|
132
|
+
ErrorCodes.CONFIG_INVALID,
|
|
133
|
+
`Index "${indexName}" holds a tokenizer instance, and no worker thread can receive one. Register the tokenizer with registerTokenizer and name it in the index config`,
|
|
134
|
+
{ indexName }
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
if (typeof config.stopWords === "function") {
|
|
138
|
+
throw new NarsilError(
|
|
139
|
+
ErrorCodes.CONFIG_INVALID,
|
|
140
|
+
`Index "${indexName}" holds a stop word function, and no worker thread can receive one. Register the function with registerStopWords and name it in the index config`,
|
|
141
|
+
{ indexName }
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
if (bootstrapModule !== void 0) return;
|
|
145
|
+
const language = config.language ?? "english";
|
|
146
|
+
const requireRegistered = (setting, name) => {
|
|
147
|
+
throw new NarsilError(
|
|
148
|
+
ErrorCodes.CONFIG_INVALID,
|
|
149
|
+
`Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
|
|
150
|
+
{ indexName, [setting]: name }
|
|
151
|
+
);
|
|
152
|
+
};
|
|
153
|
+
if (language !== "english") requireRegistered("language", language);
|
|
154
|
+
if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
|
|
155
|
+
if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
|
|
156
|
+
}
|
|
157
|
+
function workerIneligibility(indexName, config, bootstrapModule) {
|
|
158
|
+
try {
|
|
159
|
+
assertConfigReachesWorker(indexName, config, bootstrapModule);
|
|
160
|
+
return null;
|
|
161
|
+
} catch (err) {
|
|
162
|
+
if (err instanceof NarsilError && err.code === ErrorCodes.CONFIG_INVALID) {
|
|
163
|
+
return err;
|
|
164
|
+
}
|
|
165
|
+
throw err;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function reportIneligible(state, indexName, error) {
|
|
169
|
+
if (state.reportedIneligible.has(indexName)) return;
|
|
170
|
+
state.reportedIneligible.add(indexName);
|
|
171
|
+
state.callbacks?.onCopyLoadFailure?.("index-excluded", error, false);
|
|
172
|
+
}
|
|
173
|
+
function eligibleIndexNames(state) {
|
|
174
|
+
const names = [];
|
|
175
|
+
for (const [name, entry] of state.indexRegistry) {
|
|
176
|
+
const ineligibility = workerIneligibility(name, entry.config, state.bootstrapModule);
|
|
177
|
+
if (ineligibility) {
|
|
178
|
+
reportIneligible(state, name, ineligibility);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
names.push(name);
|
|
182
|
+
}
|
|
183
|
+
return names;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// src/workers/protocol.ts
|
|
187
|
+
var nextRequestSequence = 0;
|
|
188
|
+
function createRequestId() {
|
|
189
|
+
nextRequestSequence += 1;
|
|
190
|
+
return String(nextRequestSequence);
|
|
191
|
+
}
|
|
192
|
+
function isValidWorkerResponse(msg) {
|
|
193
|
+
if (msg === null || msg === void 0 || typeof msg !== "object") {
|
|
194
|
+
return false;
|
|
195
|
+
}
|
|
196
|
+
const candidate = msg;
|
|
197
|
+
if (typeof candidate.requestId !== "string") {
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
if (candidate.type !== "success" && candidate.type !== "error") {
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// src/core/segment-merges/host.ts
|
|
207
|
+
function mergeSourceSurfaces(input, analysis) {
|
|
208
|
+
const registry = createSurfaceRegistry();
|
|
209
|
+
input.segment.surfaces.addKeptForms(registry, input.remap, analysis);
|
|
210
|
+
const forms = registry.serialize();
|
|
211
|
+
for (const value of Object.values(forms)) {
|
|
212
|
+
if (Array.isArray(value) && value[0] > 4294967295) throw new SegmentRefusal(8);
|
|
213
|
+
}
|
|
214
|
+
const table = encodeSurfaceTable(forms);
|
|
215
|
+
return { text: table.blob, offsets: table.offsets, counts: table.counts };
|
|
216
|
+
}
|
|
217
|
+
function mergeSourceSpellings(input, analysis) {
|
|
218
|
+
if (analysis === null || analysis.fields.length === 0) return null;
|
|
219
|
+
const counts = /* @__PURE__ */ new Map();
|
|
220
|
+
input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
221
|
+
const values = encodePartialSpellingsOf(analysis.fields, counts);
|
|
222
|
+
const pairs = [];
|
|
223
|
+
for (let at = 0; at < values.counts.length; at++) {
|
|
224
|
+
if (values.counts[at] > 4294967295) throw new SegmentRefusal(8);
|
|
225
|
+
pairs.push(values.spellings[at], values.tokens[at]);
|
|
226
|
+
}
|
|
227
|
+
const table = encodeStringBlob(pairs);
|
|
228
|
+
return { text: table.blob, offsets: table.offsets, counts: Uint32Array.from(values.counts) };
|
|
229
|
+
}
|
|
230
|
+
function source(input, analysis, textAnalysis) {
|
|
231
|
+
const arrays = input.segment.arrays;
|
|
232
|
+
const terms = arrays.tokenTable.data;
|
|
233
|
+
return {
|
|
234
|
+
terms: { text: terms.blob, offsets: terms.offsets, counts: terms.documentFrequencies },
|
|
235
|
+
payloadSlots: terms.payloadSlots,
|
|
236
|
+
postingOffsets: arrays.postingOffsets,
|
|
237
|
+
postingDocuments: arrays.postingDocIds,
|
|
238
|
+
postingFrequencies: arrays.postingFrequencies,
|
|
239
|
+
postingFields: arrays.postingFieldIndices,
|
|
240
|
+
positionOffsets: arrays.positionOffsets,
|
|
241
|
+
positionValues: arrays.positionValues,
|
|
242
|
+
mapping: new Uint32Array(input.remap.buffer, input.remap.byteOffset, input.remap.length),
|
|
243
|
+
fieldMapping: input.fieldIndexRemap,
|
|
244
|
+
fieldLengths: arrays.fieldNames.map((field) => {
|
|
245
|
+
const at = arrays.fieldLengthNames.indexOf(field);
|
|
246
|
+
return at < 0 ? null : arrays.fieldLengthColumns[at];
|
|
247
|
+
}),
|
|
248
|
+
documentCount: arrays.documentCount,
|
|
249
|
+
surfaces: mergeSourceSurfaces(input, textAnalysis),
|
|
250
|
+
spellings: mergeSourceSpellings(input, analysis)
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
function mergedPartialSpellings(table, analysis) {
|
|
254
|
+
if (analysis === null || analysis.fields.length === 0) return null;
|
|
255
|
+
const values = { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
256
|
+
for (let at = 0; at < table.counts.length; at++) {
|
|
257
|
+
values.spellings.push(mergeTextAt(table, at * 2));
|
|
258
|
+
values.tokens.push(mergeTextAt(table, at * 2 + 1));
|
|
259
|
+
values.counts.push(table.counts[at]);
|
|
260
|
+
}
|
|
261
|
+
return values;
|
|
262
|
+
}
|
|
263
|
+
function mergeKeywordColumns(inputs, documentCount, fieldNames, analysis, textAnalysis) {
|
|
264
|
+
const sources = inputs.map((input) => source(input, analysis, textAnalysis));
|
|
265
|
+
const output = mergeKeywordSources(sources, documentCount, fieldNames.length);
|
|
266
|
+
const termCount = output.terms.counts.length;
|
|
267
|
+
const payloadSlots = new Uint32Array(termCount);
|
|
268
|
+
for (let at = 0; at < termCount; at++) payloadSlots[at] = at;
|
|
269
|
+
const fieldLengthNames = [];
|
|
270
|
+
const fieldLengthColumns = [];
|
|
271
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
272
|
+
for (const input of inputs) {
|
|
273
|
+
for (const name of input.segment.arrays.fieldLengthNames) {
|
|
274
|
+
if (fieldLengthNames.includes(name)) continue;
|
|
275
|
+
const field = fieldNames.indexOf(name);
|
|
276
|
+
fieldLengthNames.push(name);
|
|
277
|
+
fieldLengthColumns.push(output.fieldLengths.subarray(field * documentCount, (field + 1) * documentCount));
|
|
278
|
+
totalFieldLengths[name] = output.totalFieldLengths[field];
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
tokenTable: {
|
|
283
|
+
blob: output.terms.text,
|
|
284
|
+
offsets: output.terms.offsets,
|
|
285
|
+
payloadSlots,
|
|
286
|
+
documentFrequencies: output.terms.counts
|
|
287
|
+
},
|
|
288
|
+
fieldLengthNames,
|
|
289
|
+
fieldLengthColumns,
|
|
290
|
+
totalFieldLengths,
|
|
291
|
+
postingOffsets: output.postingOffsets,
|
|
292
|
+
postingDocIds: output.postingDocuments,
|
|
293
|
+
postingFrequencies: output.postingFrequencies,
|
|
294
|
+
postingFieldIndices: output.postingFields,
|
|
295
|
+
positionOffsets: output.positionOffsets,
|
|
296
|
+
positionValues: output.positionValues,
|
|
297
|
+
surfaceTable: output.surfaces.counts.length === 0 ? null : { blob: output.surfaces.text, offsets: output.surfaces.offsets, counts: output.surfaces.counts },
|
|
298
|
+
partialSpellings: mergedPartialSpellings(output.spellings, analysis)
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// src/core/field-merges/dictionary.ts
|
|
303
|
+
function compareKeys(left, right) {
|
|
304
|
+
if (left.run !== null && right.run !== null) {
|
|
305
|
+
for (let at = 0; at < 3; at++) {
|
|
306
|
+
if (left.run[at] !== right.run[at]) return left.run[at] - right.run[at];
|
|
307
|
+
}
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
return left.text < right.text ? -1 : left.text > right.text ? 1 : 0;
|
|
311
|
+
}
|
|
312
|
+
function compareSuffix(left, right) {
|
|
313
|
+
for (let at = 1; at <= Math.min(left.length, right.length); at++) {
|
|
314
|
+
const difference = left.charCodeAt(left.length - at) - right.charCodeAt(right.length - at);
|
|
315
|
+
if (difference !== 0) return difference;
|
|
316
|
+
}
|
|
317
|
+
return left.length - right.length;
|
|
318
|
+
}
|
|
319
|
+
function dictionaryPlan(sources, pattern2) {
|
|
320
|
+
const keys = /* @__PURE__ */ new Map();
|
|
321
|
+
const present = /* @__PURE__ */ new Set();
|
|
322
|
+
let inputKeyCount = 0;
|
|
323
|
+
let inputRowCount = 0;
|
|
324
|
+
for (const source2 of sources) {
|
|
325
|
+
const offsets = words(source2, 2);
|
|
326
|
+
const postings = words(source2, 3);
|
|
327
|
+
const textOffsets = words(source2, 1);
|
|
328
|
+
const count = pattern2 ? textOffsets.length / 3 : textOffsets.length - 1;
|
|
329
|
+
inputKeyCount += count;
|
|
330
|
+
embedded(inputKeyCount <= 4294967295 && offsets[0] === 0 && offsets[count] === postings.length);
|
|
331
|
+
if (pattern2) {
|
|
332
|
+
for (const point of textOffsets) embedded(point <= 1114111);
|
|
333
|
+
for (const ordinal of words(source2, 0)) {
|
|
334
|
+
embedded(ordinal < source2.mapping.length);
|
|
335
|
+
if (source2.mapping[ordinal] !== 4294967295) present.add(source2.mapping[ordinal]);
|
|
336
|
+
}
|
|
337
|
+
} else {
|
|
338
|
+
embedded(textOffsets[0] === 0 && textOffsets[count] === source2.regions[0].length);
|
|
339
|
+
}
|
|
340
|
+
for (let at = 0; at < count; at++) {
|
|
341
|
+
embedded(offsets[at] <= offsets[at + 1] && offsets[at + 1] <= postings.length);
|
|
342
|
+
let text;
|
|
343
|
+
let run = null;
|
|
344
|
+
if (pattern2) {
|
|
345
|
+
run = [textOffsets[at * 3], textOffsets[at * 3 + 1], textOffsets[at * 3 + 2]];
|
|
346
|
+
text = `${run[0]},${run[1]},${run[2]}`;
|
|
347
|
+
} else {
|
|
348
|
+
embedded(
|
|
349
|
+
textOffsets[at] <= textOffsets[at + 1] && textOffsets[at + 1] <= source2.regions[0].length && words(source2, 4)[at] < count
|
|
350
|
+
);
|
|
351
|
+
text = storedText(source2.regions[0], textOffsets[at], textOffsets[at + 1] - textOffsets[at]);
|
|
352
|
+
}
|
|
353
|
+
let key = keys.get(text);
|
|
354
|
+
if (key === void 0) {
|
|
355
|
+
key = { text, run, ordinals: /* @__PURE__ */ new Set() };
|
|
356
|
+
keys.set(text, key);
|
|
357
|
+
}
|
|
358
|
+
for (let row = offsets[at]; row < offsets[at + 1]; row++) {
|
|
359
|
+
const ordinal = postings[row];
|
|
360
|
+
embedded(ordinal < source2.mapping.length);
|
|
361
|
+
const target = source2.mapping[ordinal];
|
|
362
|
+
if (target === 4294967295) continue;
|
|
363
|
+
uint32(++inputRowCount);
|
|
364
|
+
key.ordinals.add(target);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
const ordered = [...keys.values()].filter((key) => !pattern2 || key.ordinals.size > 0).sort(compareKeys);
|
|
369
|
+
uint32(ordered.length + 1);
|
|
370
|
+
const rows = ordered.map((key) => [...key.ordinals].sort((left, right) => left - right));
|
|
371
|
+
const suffixes = ordered.map((_, at) => at).sort((left, right) => compareSuffix(ordered[left].text, ordered[right].text));
|
|
372
|
+
const ordinals = [...present].sort((left, right) => left - right);
|
|
373
|
+
let textLength = 0;
|
|
374
|
+
let rowCount = 0;
|
|
375
|
+
for (let at = 0; at < ordered.length; at++) {
|
|
376
|
+
textLength += pattern2 ? 3 : ordered[at].text.length;
|
|
377
|
+
rowCount += rows[at].length;
|
|
378
|
+
uint32(textLength);
|
|
379
|
+
uint32(rowCount);
|
|
380
|
+
}
|
|
381
|
+
const lengths = new Uint32Array(8);
|
|
382
|
+
lengths[0] = pattern2 ? ordinals.length : textLength;
|
|
383
|
+
lengths[1] = pattern2 ? textLength : ordered.length + 1;
|
|
384
|
+
lengths[2] = ordered.length + 1;
|
|
385
|
+
lengths[3] = rowCount;
|
|
386
|
+
if (!pattern2) lengths[4] = ordered.length;
|
|
387
|
+
return {
|
|
388
|
+
lengths,
|
|
389
|
+
publish(output) {
|
|
390
|
+
const offsets = output[2];
|
|
391
|
+
const postings = output[3];
|
|
392
|
+
const textOffsets = output[1];
|
|
393
|
+
const text = output[0];
|
|
394
|
+
let row = 0;
|
|
395
|
+
let unit = 0;
|
|
396
|
+
offsets[0] = 0;
|
|
397
|
+
if (!pattern2) textOffsets[0] = 0;
|
|
398
|
+
for (let at = 0; at < ordered.length; at++) {
|
|
399
|
+
const key = ordered[at];
|
|
400
|
+
if (key.run !== null) {
|
|
401
|
+
for (let point = 0; point < 3; point++) textOffsets[at * 3 + point] = key.run[point];
|
|
402
|
+
} else {
|
|
403
|
+
for (let index = 0; index < key.text.length; index++) text[unit++] = key.text.charCodeAt(index);
|
|
404
|
+
textOffsets[at + 1] = unit;
|
|
405
|
+
output[4][at] = suffixes[at];
|
|
406
|
+
}
|
|
407
|
+
for (const ordinal of rows[at]) postings[row++] = ordinal;
|
|
408
|
+
offsets[at + 1] = row;
|
|
409
|
+
}
|
|
410
|
+
if (pattern2) {
|
|
411
|
+
const present2 = output[0];
|
|
412
|
+
for (let at = 0; at < ordinals.length; at++) present2[at] = ordinals[at];
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// src/core/field-merges/rows.ts
|
|
419
|
+
function rowPlan(sources, geo2) {
|
|
420
|
+
const rows = [];
|
|
421
|
+
for (const source2 of sources) {
|
|
422
|
+
const ordinals = words(source2, geo2 ? 3 : 1);
|
|
423
|
+
const values = numbers(source2, geo2 ? 1 : 0);
|
|
424
|
+
const codes = geo2 ? source2.regions[0] : null;
|
|
425
|
+
const raw = new BigUint64Array(values.buffer, values.byteOffset, values.length);
|
|
426
|
+
for (let at = 0; at < ordinals.length; at++) {
|
|
427
|
+
const ordinal = ordinals[at];
|
|
428
|
+
embedded(ordinal < source2.mapping.length);
|
|
429
|
+
if (!geo2) embedded(!Number.isNaN(values[at]));
|
|
430
|
+
if (geo2) {
|
|
431
|
+
const longitude = numbers(source2, 2)[at];
|
|
432
|
+
embedded(
|
|
433
|
+
Number.isFinite(values[at]) && values[at] >= -90 && values[at] <= 90 && Number.isFinite(longitude) && longitude >= -180 && longitude <= 180
|
|
434
|
+
);
|
|
435
|
+
}
|
|
436
|
+
const target = source2.mapping[ordinal];
|
|
437
|
+
if (target !== 4294967295)
|
|
438
|
+
rows.push({ source: source2, at, ordinal: target, value: values[at], code: codes?.[at] ?? 0n, bits: raw[at] });
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
uint32(rows.length);
|
|
442
|
+
rows.sort((left, right) => {
|
|
443
|
+
if (geo2) {
|
|
444
|
+
if (left.code !== right.code) return left.code < right.code ? -1 : 1;
|
|
445
|
+
} else {
|
|
446
|
+
if (Number.isNaN(left.value) !== Number.isNaN(right.value)) return Number.isNaN(left.value) ? 1 : -1;
|
|
447
|
+
if (left.value < right.value) return -1;
|
|
448
|
+
if (left.value > right.value) return 1;
|
|
449
|
+
}
|
|
450
|
+
return left.ordinal - right.ordinal;
|
|
451
|
+
});
|
|
452
|
+
const lengths = new Uint32Array(8);
|
|
453
|
+
lengths.fill(rows.length, 0, geo2 ? 4 : 2);
|
|
454
|
+
return {
|
|
455
|
+
lengths,
|
|
456
|
+
publish(output) {
|
|
457
|
+
const values = output[geo2 ? 1 : 0];
|
|
458
|
+
const valueBits = new BigUint64Array(values.buffer, values.byteOffset, values.length);
|
|
459
|
+
const ordinals = output[geo2 ? 3 : 1];
|
|
460
|
+
for (let at = 0; at < rows.length; at++) {
|
|
461
|
+
const row = rows[at];
|
|
462
|
+
valueBits[at] = row.bits;
|
|
463
|
+
ordinals[at] = row.ordinal;
|
|
464
|
+
if (geo2) {
|
|
465
|
+
output[0][at] = row.code;
|
|
466
|
+
output[2][at] = numbers(row.source, 2)[row.at];
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
function booleanPlan(documents) {
|
|
473
|
+
const lengths = new Uint32Array(8);
|
|
474
|
+
lengths[0] = lengths[1] = Math.max(1, Math.ceil(documents.length / 32));
|
|
475
|
+
return {
|
|
476
|
+
lengths,
|
|
477
|
+
publish(output) {
|
|
478
|
+
for (let value = 0; value < 2; value++) {
|
|
479
|
+
const bits = output[value];
|
|
480
|
+
bits.fill(0, 0, lengths[value]);
|
|
481
|
+
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
482
|
+
const document = documents[ordinal];
|
|
483
|
+
const input = words(document.source, value);
|
|
484
|
+
if ((input[document.ordinal >>> 5] & 1 << (document.ordinal & 31)) !== 0)
|
|
485
|
+
bits[ordinal >>> 5] |= 1 << (ordinal & 31);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
};
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// src/core/field-merges/portable.ts
|
|
493
|
+
function plan(kind, sources, count) {
|
|
494
|
+
uint32(count);
|
|
495
|
+
const documents = documentsOf(sources, count);
|
|
496
|
+
switch (kind) {
|
|
497
|
+
case 1:
|
|
498
|
+
return rowPlan(sources, false);
|
|
499
|
+
case 2:
|
|
500
|
+
return booleanPlan(documents);
|
|
501
|
+
case 3:
|
|
502
|
+
return dictionaryPlan(sources, false);
|
|
503
|
+
case 4:
|
|
504
|
+
return rowPlan(sources, true);
|
|
505
|
+
case 5:
|
|
506
|
+
return storedPlan(sources, documents);
|
|
507
|
+
case 6:
|
|
508
|
+
return dictionaryPlan(sources, true);
|
|
509
|
+
default:
|
|
510
|
+
throw new SegmentRefusal(1);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
var planned = null;
|
|
514
|
+
function plannedFieldMerge() {
|
|
515
|
+
planned ??= createFieldMerge(plan);
|
|
516
|
+
return planned;
|
|
517
|
+
}
|
|
518
|
+
var measureFieldMerge = (...args) => plannedFieldMerge().measureFieldMerge(...args);
|
|
519
|
+
var writeFieldMerge = (...args) => plannedFieldMerge().writeFieldMerge(...args);
|
|
520
|
+
|
|
521
|
+
// src/core/field-merges/host.ts
|
|
522
|
+
function mergeFieldRegions(kind, sources, count) {
|
|
523
|
+
return mergeFieldRegionsWith(kind, sources, count, { measureFieldMerge, writeFieldMerge });
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// src/core/partition/frozen/merge-fields.ts
|
|
527
|
+
function fieldsOf(inputs, kind) {
|
|
528
|
+
const fields = /* @__PURE__ */ new Set();
|
|
529
|
+
for (const input of inputs) for (const entry of input.segment.arrays[kind]) fields.add(entry.fieldPath);
|
|
530
|
+
return [...fields];
|
|
531
|
+
}
|
|
532
|
+
function sourceOf(input, regions) {
|
|
533
|
+
return {
|
|
534
|
+
mapping: new Uint32Array(input.remap.buffer, input.remap.byteOffset, input.remap.length),
|
|
535
|
+
regions
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
function numeric(inputs, count, kind) {
|
|
539
|
+
return fieldsOf(inputs, kind).map((fieldPath) => {
|
|
540
|
+
const sources = inputs.map((input) => {
|
|
541
|
+
const regions = (kind === "numeric" ? input.segment.numericIndexes : input.segment.sizeIndexes).get(fieldPath)?.regions();
|
|
542
|
+
return sourceOf(input, [regions?.values ?? new Float64Array(0), regions?.ordinals ?? new Uint32Array(0)]);
|
|
543
|
+
});
|
|
544
|
+
const output = mergeFieldRegions(1, sources, count);
|
|
545
|
+
return { fieldPath, values: output[0], docIds: output[1] };
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
function boolean(inputs, count) {
|
|
549
|
+
return fieldsOf(inputs, "boolean").map((fieldPath) => {
|
|
550
|
+
const sources = inputs.map((input) => {
|
|
551
|
+
const regions = input.segment.booleanIndexes.get(fieldPath)?.regions(input.remap.length);
|
|
552
|
+
const words2 = Math.max(1, Math.ceil(input.remap.length / 32));
|
|
553
|
+
return sourceOf(input, [
|
|
554
|
+
regions?.trueBits ?? new Uint32Array(words2),
|
|
555
|
+
regions?.falseBits ?? new Uint32Array(words2)
|
|
556
|
+
]);
|
|
557
|
+
});
|
|
558
|
+
const output = mergeFieldRegions(2, sources, count);
|
|
559
|
+
return {
|
|
560
|
+
fieldPath,
|
|
561
|
+
trueDocs: Uint32Array.from(bitsetIterator(output[0])),
|
|
562
|
+
falseDocs: Uint32Array.from(bitsetIterator(output[1]))
|
|
563
|
+
};
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
function enums(inputs, count) {
|
|
567
|
+
return fieldsOf(inputs, "enums").map((fieldPath) => {
|
|
568
|
+
const sources = inputs.map((input) => {
|
|
569
|
+
const regions = input.segment.enumIndexes.get(fieldPath)?.regions();
|
|
570
|
+
return sourceOf(input, [
|
|
571
|
+
regions?.units ?? new Uint16Array(0),
|
|
572
|
+
regions?.offsets ?? new Uint32Array(1),
|
|
573
|
+
regions?.postingOffsets ?? new Uint32Array(1),
|
|
574
|
+
regions?.postings ?? new Uint32Array(0),
|
|
575
|
+
regions?.suffixOrder ?? new Uint32Array(0)
|
|
576
|
+
]);
|
|
577
|
+
});
|
|
578
|
+
const output = mergeFieldRegions(3, sources, count);
|
|
579
|
+
const units = output[0];
|
|
580
|
+
const offsets = output[1];
|
|
581
|
+
const values = [];
|
|
582
|
+
for (let at = 0; at < offsets.length - 1; at++)
|
|
583
|
+
values.push(storedText(units, offsets[at], offsets[at + 1] - offsets[at]));
|
|
584
|
+
return { fieldPath, values, offsets: output[2], docIds: output[3] };
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
function geo(inputs, count) {
|
|
588
|
+
return fieldsOf(inputs, "geo").map((fieldPath) => {
|
|
589
|
+
const sources = inputs.map((input) => {
|
|
590
|
+
const regions = input.segment.geoIndexes.get(fieldPath)?.regions();
|
|
591
|
+
return sourceOf(input, [
|
|
592
|
+
regions?.codes ?? new BigUint64Array(0),
|
|
593
|
+
regions?.lat ?? new Float64Array(0),
|
|
594
|
+
regions?.lon ?? new Float64Array(0),
|
|
595
|
+
regions?.ordinals ?? new Uint32Array(0)
|
|
596
|
+
]);
|
|
597
|
+
});
|
|
598
|
+
const output = mergeFieldRegions(4, sources, count);
|
|
599
|
+
return {
|
|
600
|
+
fieldPath,
|
|
601
|
+
latitudes: output[1],
|
|
602
|
+
longitudes: output[2],
|
|
603
|
+
docIds: output[3]
|
|
604
|
+
};
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
function pattern(inputs, count) {
|
|
608
|
+
return fieldsOf(inputs, "pattern").map((fieldPath) => {
|
|
609
|
+
const sources = inputs.map((input) => {
|
|
610
|
+
const regions = input.segment.arrays.pattern.find((entry) => entry.fieldPath === fieldPath);
|
|
611
|
+
return sourceOf(input, [
|
|
612
|
+
regions?.docIds ?? new Uint32Array(0),
|
|
613
|
+
regions?.runs ?? new Uint32Array(0),
|
|
614
|
+
regions?.offsets ?? new Uint32Array(1),
|
|
615
|
+
regions?.postings ?? new Uint32Array(0)
|
|
616
|
+
]);
|
|
617
|
+
});
|
|
618
|
+
const output = mergeFieldRegions(6, sources, count);
|
|
619
|
+
return {
|
|
620
|
+
fieldPath,
|
|
621
|
+
docIds: output[0],
|
|
622
|
+
runs: output[1],
|
|
623
|
+
offsets: output[2],
|
|
624
|
+
postings: output[3]
|
|
625
|
+
};
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
function mergeFieldIndexes(inputs, documentCount) {
|
|
629
|
+
return {
|
|
630
|
+
numeric: numeric(inputs, documentCount, "numeric"),
|
|
631
|
+
boolean: boolean(inputs, documentCount),
|
|
632
|
+
enums: enums(inputs, documentCount),
|
|
633
|
+
geo: geo(inputs, documentCount),
|
|
634
|
+
pattern: pattern(inputs, documentCount),
|
|
635
|
+
sizes: numeric(inputs, documentCount, "sizes")
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
// src/core/partition/frozen/shared-snapshot.ts
|
|
640
|
+
function encodedPayloadOf(payload) {
|
|
641
|
+
const { tokens, docFrequencies, surfaceForms, ...columns } = payload;
|
|
642
|
+
return {
|
|
643
|
+
...columns,
|
|
644
|
+
tokenTable: encodeFrozenTokenTableData(tokens, docFrequencies),
|
|
645
|
+
surfaceTable: surfaceForms === null ? null : encodeSurfaceTable(surfaceForms)
|
|
646
|
+
};
|
|
647
|
+
}
|
|
648
|
+
function sharedUint32(source2) {
|
|
649
|
+
const copy = new Uint32Array(new SharedArrayBuffer(source2.length * 4));
|
|
650
|
+
copy.set(source2);
|
|
651
|
+
return copy;
|
|
652
|
+
}
|
|
653
|
+
function sharedUint16(source2) {
|
|
654
|
+
const copy = new Uint16Array(new SharedArrayBuffer(source2.length * 2));
|
|
655
|
+
copy.set(source2);
|
|
656
|
+
return copy;
|
|
657
|
+
}
|
|
658
|
+
function sharedUint8(source2) {
|
|
659
|
+
const copy = new Uint8Array(new SharedArrayBuffer(source2.length));
|
|
660
|
+
copy.set(source2);
|
|
661
|
+
return copy;
|
|
662
|
+
}
|
|
663
|
+
function sharedFloat64(source2) {
|
|
664
|
+
const copy = new Float64Array(new SharedArrayBuffer(source2.length * 8));
|
|
665
|
+
copy.set(source2);
|
|
666
|
+
return copy;
|
|
667
|
+
}
|
|
668
|
+
function sharedSurfaceTable(table) {
|
|
669
|
+
if (table === null || table.counts.length === 0) return null;
|
|
670
|
+
return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
|
|
671
|
+
}
|
|
672
|
+
function sharePartialWordTable(table) {
|
|
673
|
+
if (typeof SharedArrayBuffer !== "function" || table.runEntries.buffer instanceof SharedArrayBuffer) return table;
|
|
674
|
+
return {
|
|
675
|
+
fields: [...table.fields],
|
|
676
|
+
spellings: sharedUint8(table.spellings),
|
|
677
|
+
spellingOffsets: sharedUint32(table.spellingOffsets),
|
|
678
|
+
spellingTokens: sharedUint32(table.spellingTokens),
|
|
679
|
+
spellingCounts: sharedUint32(table.spellingCounts),
|
|
680
|
+
runs: sharedUint32(table.runs),
|
|
681
|
+
runOffsets: sharedUint32(table.runOffsets),
|
|
682
|
+
runEntries: sharedUint32(table.runEntries)
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
686
|
+
let source2 = spellings;
|
|
687
|
+
if (source2 === null && analysis !== null && analysis.fields.length > 0) {
|
|
688
|
+
source2 = partialSpellingsOfDocuments(storedDocumentsOf(documentData), analysis);
|
|
689
|
+
}
|
|
690
|
+
if (source2 === null || source2.fields.length === 0) return null;
|
|
691
|
+
const table = buildPartialWordTableFrom(source2, wrapFrozenTokenTable(tokenData));
|
|
692
|
+
return table === null ? null : sharePartialWordTable(table);
|
|
693
|
+
}
|
|
694
|
+
function partialWordsMatch(table, analysis) {
|
|
695
|
+
if (table === null) return analysis === null || analysis.fields.length === 0;
|
|
696
|
+
return analysis === null || sameFields(table.fields, analysis.fields);
|
|
697
|
+
}
|
|
698
|
+
function shareSegmentArrays(arrays, analysis, segmentId) {
|
|
699
|
+
const partialWords = partialWordsMatch(arrays.partialWords, analysis) ? arrays.partialWords : sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis);
|
|
700
|
+
return { ...arrays, segmentId: segmentId ?? generateId(), partialWords };
|
|
701
|
+
}
|
|
702
|
+
function segmentArraysOf(payload, documentData, reusedPartialWords = null) {
|
|
703
|
+
const { tokenTable, surfaceTable } = payload;
|
|
704
|
+
const idTable = encodeExternalIdTableData(payload.docIds);
|
|
705
|
+
const partialWords = reusedPartialWords ?? (payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, wrapFrozenTokenTable(tokenTable)));
|
|
706
|
+
return {
|
|
707
|
+
documentCount: payload.documentCount,
|
|
708
|
+
fieldNames: payload.fieldNames,
|
|
709
|
+
fieldLengthNames: payload.fieldLengthNames,
|
|
710
|
+
fieldLengthColumns: payload.fieldLengthColumns,
|
|
711
|
+
totalFieldLengths: payload.totalFieldLengths,
|
|
712
|
+
tokenTable,
|
|
713
|
+
idTable,
|
|
714
|
+
documentTable: documentData,
|
|
715
|
+
postingOffsets: payload.postingOffsets,
|
|
716
|
+
postingDocIds: payload.postingDocIds,
|
|
717
|
+
postingFrequencies: payload.postingFrequencies,
|
|
718
|
+
postingFieldIndices: payload.postingFieldIndices,
|
|
719
|
+
positionOffsets: payload.positionOffsets,
|
|
720
|
+
positionValues: payload.positionValues,
|
|
721
|
+
numeric: payload.numeric,
|
|
722
|
+
sizes: payload.sizes,
|
|
723
|
+
boolean: payload.boolean,
|
|
724
|
+
enums: payload.enums,
|
|
725
|
+
geo: payload.geo,
|
|
726
|
+
pattern: payload.pattern,
|
|
727
|
+
storedFields: payload.storedFields,
|
|
728
|
+
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable,
|
|
729
|
+
partialWords
|
|
730
|
+
};
|
|
731
|
+
}
|
|
732
|
+
function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
|
|
733
|
+
if (typeof SharedArrayBuffer !== "function") return null;
|
|
734
|
+
return freezeEncodedSegmentShared(encodedPayloadOf(payload), encodeDocumentTableData(documents), segmentId, analysis);
|
|
735
|
+
}
|
|
736
|
+
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null, reusedPartialWords = null) {
|
|
737
|
+
if (typeof SharedArrayBuffer !== "function") return null;
|
|
738
|
+
const tokenData = payload.tokenTable;
|
|
739
|
+
const idData = encodeExternalIdTableData(payload.docIds);
|
|
740
|
+
return {
|
|
741
|
+
segmentId: segmentId ?? generateId(),
|
|
742
|
+
documentCount: payload.documentCount,
|
|
743
|
+
fieldNames: [...payload.fieldNames],
|
|
744
|
+
fieldLengthNames: [...payload.fieldLengthNames],
|
|
745
|
+
fieldLengthColumns: payload.fieldLengthColumns.map(sharedUint32),
|
|
746
|
+
totalFieldLengths: { ...payload.totalFieldLengths },
|
|
747
|
+
tokenTable: {
|
|
748
|
+
blob: sharedUint8(tokenData.blob),
|
|
749
|
+
offsets: sharedUint32(tokenData.offsets),
|
|
750
|
+
payloadSlots: sharedUint32(tokenData.payloadSlots),
|
|
751
|
+
documentFrequencies: sharedUint32(tokenData.documentFrequencies)
|
|
752
|
+
},
|
|
753
|
+
idTable: {
|
|
754
|
+
blob: sharedUint8(idData.blob),
|
|
755
|
+
offsets: sharedUint32(idData.offsets),
|
|
756
|
+
sortedOrdinals: sharedUint32(idData.sortedOrdinals)
|
|
757
|
+
},
|
|
758
|
+
documentTable: {
|
|
759
|
+
blob: sharedUint8(documentData.blob),
|
|
760
|
+
offsets: sharedUint32(documentData.offsets),
|
|
761
|
+
...documentData.discardedFields === void 0 ? {} : { discardedFields: documentData.discardedFields }
|
|
762
|
+
},
|
|
763
|
+
postingOffsets: sharedUint32(payload.postingOffsets),
|
|
764
|
+
postingDocIds: sharedUint32(payload.postingDocIds),
|
|
765
|
+
postingFrequencies: sharedUint16(payload.postingFrequencies),
|
|
766
|
+
postingFieldIndices: sharedUint16(payload.postingFieldIndices),
|
|
767
|
+
positionOffsets: payload.positionOffsets === null ? null : sharedUint32(payload.positionOffsets),
|
|
768
|
+
positionValues: payload.positionValues === null ? null : sharedUint32(payload.positionValues),
|
|
769
|
+
numeric: payload.numeric.map((entry) => ({
|
|
770
|
+
fieldPath: entry.fieldPath,
|
|
771
|
+
docIds: sharedUint32(entry.docIds),
|
|
772
|
+
values: sharedFloat64(entry.values)
|
|
773
|
+
})),
|
|
774
|
+
sizes: payload.sizes.map((entry) => ({
|
|
775
|
+
fieldPath: entry.fieldPath,
|
|
776
|
+
docIds: sharedUint32(entry.docIds),
|
|
777
|
+
values: sharedFloat64(entry.values)
|
|
778
|
+
})),
|
|
779
|
+
boolean: payload.boolean.map((entry) => ({
|
|
780
|
+
fieldPath: entry.fieldPath,
|
|
781
|
+
trueDocs: sharedUint32(entry.trueDocs),
|
|
782
|
+
falseDocs: sharedUint32(entry.falseDocs)
|
|
783
|
+
})),
|
|
784
|
+
enums: payload.enums.map((entry) => ({
|
|
785
|
+
fieldPath: entry.fieldPath,
|
|
786
|
+
values: [...entry.values],
|
|
787
|
+
offsets: sharedUint32(entry.offsets),
|
|
788
|
+
docIds: sharedUint32(entry.docIds)
|
|
789
|
+
})),
|
|
790
|
+
geo: payload.geo.map((entry) => ({
|
|
791
|
+
fieldPath: entry.fieldPath,
|
|
792
|
+
docIds: sharedUint32(entry.docIds),
|
|
793
|
+
latitudes: sharedFloat64(entry.latitudes),
|
|
794
|
+
longitudes: sharedFloat64(entry.longitudes)
|
|
795
|
+
})),
|
|
796
|
+
pattern: payload.pattern.map((entry) => ({
|
|
797
|
+
fieldPath: entry.fieldPath,
|
|
798
|
+
docIds: sharedUint32(entry.docIds),
|
|
799
|
+
runs: sharedUint32(entry.runs),
|
|
800
|
+
offsets: sharedUint32(entry.offsets),
|
|
801
|
+
postings: sharedUint32(entry.postings)
|
|
802
|
+
})),
|
|
803
|
+
storedFields: payload.storedFields?.map(({ field, values }) => ({
|
|
804
|
+
field,
|
|
805
|
+
values: {
|
|
806
|
+
shapes: sharedUint8(values.shapes),
|
|
807
|
+
starts: sharedUint32(values.starts),
|
|
808
|
+
kinds: sharedUint8(values.kinds),
|
|
809
|
+
numbers: sharedFloat64(values.numbers),
|
|
810
|
+
textStart: sharedUint32(values.textStart),
|
|
811
|
+
textLength: sharedUint32(values.textLength),
|
|
812
|
+
units: sharedUint16(values.units),
|
|
813
|
+
bytes: sharedUint8(values.bytes)
|
|
814
|
+
}
|
|
815
|
+
})),
|
|
816
|
+
surfaceTable: sharedSurfaceTable(payload.surfaceTable),
|
|
817
|
+
partialWords: reusedPartialWords === null ? sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis) : sharePartialWordTable(reusedPartialWords)
|
|
818
|
+
};
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// src/core/partition/frozen/merge.ts
|
|
822
|
+
function selectionKeeps(selection, ordinal) {
|
|
823
|
+
return selection.keep === void 0 ? !selection.segment.isTombstoned(ordinal) : selection.keep(ordinal);
|
|
824
|
+
}
|
|
825
|
+
function remapOf(selection, docIds) {
|
|
826
|
+
const { segment } = selection;
|
|
827
|
+
const count = segment.arrays.documentCount;
|
|
828
|
+
const remap = new Int32Array(count).fill(-1);
|
|
829
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
830
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
831
|
+
remap[ordinal] = docIds.length;
|
|
832
|
+
docIds.push(segment.arrays.idTable.idAt(ordinal));
|
|
833
|
+
}
|
|
834
|
+
return remap;
|
|
835
|
+
}
|
|
836
|
+
function fieldNameIndex(fieldNames, name) {
|
|
837
|
+
const at = fieldNames.indexOf(name);
|
|
838
|
+
if (at >= 0) return at;
|
|
839
|
+
fieldNames.push(name);
|
|
840
|
+
return fieldNames.length - 1;
|
|
841
|
+
}
|
|
842
|
+
function fieldIndexRemapOf(segment, fieldNames) {
|
|
843
|
+
const remap = new Uint16Array(segment.arrays.fieldNames.length);
|
|
844
|
+
for (let i = 0; i < segment.arrays.fieldNames.length; i++) {
|
|
845
|
+
remap[i] = fieldNameIndex(fieldNames, segment.arrays.fieldNames[i]);
|
|
846
|
+
}
|
|
847
|
+
return remap;
|
|
848
|
+
}
|
|
849
|
+
function holdsADiscardedField(bytes, discardedFields) {
|
|
850
|
+
const keys = topLevelMapKeys(bytes);
|
|
851
|
+
if (keys === null) return true;
|
|
852
|
+
for (const fieldPath of discardedFields) {
|
|
853
|
+
const dot = fieldPath.indexOf(".");
|
|
854
|
+
if (keys.has(dot < 0 ? fieldPath : fieldPath.slice(0, dot))) return true;
|
|
855
|
+
}
|
|
856
|
+
return false;
|
|
857
|
+
}
|
|
858
|
+
function encodedDocumentsOf(input) {
|
|
859
|
+
const table = input.segment.arrays.documentTable;
|
|
860
|
+
const encoded = new Array(input.remap.length);
|
|
861
|
+
const encodeOrdinal = (ordinal) => {
|
|
862
|
+
let bytes = encoded[ordinal];
|
|
863
|
+
if (bytes === void 0) {
|
|
864
|
+
bytes = encodeDocument(input.segment.documentSource.docAt(ordinal));
|
|
865
|
+
encoded[ordinal] = bytes;
|
|
866
|
+
}
|
|
867
|
+
return bytes;
|
|
868
|
+
};
|
|
869
|
+
if (table === null) return encodeOrdinal;
|
|
870
|
+
const discardedFields = table.discardedFields;
|
|
871
|
+
return (ordinal) => {
|
|
872
|
+
if (encoded[ordinal] !== void 0) return encodeOrdinal(ordinal);
|
|
873
|
+
const stored = table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]);
|
|
874
|
+
if (discardedFields === void 0 || !holdsADiscardedField(stored, discardedFields)) return stored;
|
|
875
|
+
return encodeOrdinal(ordinal);
|
|
876
|
+
};
|
|
877
|
+
}
|
|
878
|
+
function mergeDocuments(inputs, documentCount) {
|
|
879
|
+
const readers = inputs.map(encodedDocumentsOf);
|
|
880
|
+
const offsets = new Uint32Array(documentCount + 1);
|
|
881
|
+
for (let i = 0; i < inputs.length; i++) {
|
|
882
|
+
const { remap } = inputs[i];
|
|
883
|
+
for (let ordinal = 0; ordinal < remap.length; ordinal++) {
|
|
884
|
+
if (remap[ordinal] >= 0) offsets[remap[ordinal] + 1] = readers[i](ordinal).length;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
for (let ordinal = 0; ordinal < documentCount; ordinal++) {
|
|
888
|
+
if (offsets[ordinal + 1] > 4294967295 - offsets[ordinal]) throw new SegmentRefusal(8);
|
|
889
|
+
offsets[ordinal + 1] += offsets[ordinal];
|
|
890
|
+
}
|
|
891
|
+
const blob = new Uint8Array(offsets[documentCount]);
|
|
892
|
+
for (let i = 0; i < inputs.length; i++) {
|
|
893
|
+
const { remap } = inputs[i];
|
|
894
|
+
for (let ordinal = 0; ordinal < remap.length; ordinal++) {
|
|
895
|
+
const target = remap[ordinal];
|
|
896
|
+
if (target >= 0) blob.set(readers[i](ordinal), offsets[target]);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
return { blob, offsets };
|
|
900
|
+
}
|
|
901
|
+
function mergeSegmentContents(selections, analysis = null, textAnalysis = null) {
|
|
902
|
+
const docIds = [];
|
|
903
|
+
const fieldNames = [];
|
|
904
|
+
const inputs = selections.map((selection) => ({
|
|
905
|
+
segment: selection.segment,
|
|
906
|
+
remap: remapOf(selection, docIds),
|
|
907
|
+
fieldIndexRemap: fieldIndexRemapOf(selection.segment, fieldNames)
|
|
908
|
+
}));
|
|
909
|
+
const documentCount = docIds.length;
|
|
910
|
+
const keyword = mergeKeywordColumns(inputs, documentCount, fieldNames, analysis, textAnalysis);
|
|
911
|
+
const payload = {
|
|
912
|
+
documentCount,
|
|
913
|
+
docIds,
|
|
914
|
+
fieldNames,
|
|
915
|
+
...keyword,
|
|
916
|
+
...mergeFieldIndexes(inputs, documentCount),
|
|
917
|
+
storedFields: mergeStoredFields(
|
|
918
|
+
inputs.map((input) => ({ store: input.segment.docStore, mapping: new Uint32Array(input.remap.buffer) })),
|
|
919
|
+
documentCount
|
|
920
|
+
)
|
|
921
|
+
};
|
|
922
|
+
return { payload, documents: mergeDocuments(inputs, documentCount), reusedPartialWords: null };
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
// src/core/partition/segment-builder/keywords.ts
|
|
926
|
+
var SegmentKeywords = class {
|
|
927
|
+
constructor(fields, trackPositions) {
|
|
928
|
+
this.trackPositions = trackPositions;
|
|
929
|
+
this.segment = new GrowingSegment(new LivePostings(), trackPositions);
|
|
930
|
+
this.partial = new Set(fields.filter((field) => field.partial).map((field) => field.path));
|
|
931
|
+
if (this.partial.size > 0) this.segment.spellings = createSpellingTable();
|
|
932
|
+
}
|
|
933
|
+
segment;
|
|
934
|
+
fields = [];
|
|
935
|
+
fieldIndexes = /* @__PURE__ */ new Map();
|
|
936
|
+
partial;
|
|
937
|
+
add(prepared, ordinal) {
|
|
938
|
+
const fields = [];
|
|
939
|
+
for (const [path, text] of prepared.fields) {
|
|
940
|
+
let field = this.fieldIndexes.get(path);
|
|
941
|
+
if (field === void 0) {
|
|
942
|
+
field = this.fields.length;
|
|
943
|
+
this.fields.push(path);
|
|
944
|
+
this.fieldIndexes.set(path, field);
|
|
945
|
+
}
|
|
946
|
+
fields.push({ field, text: text.records });
|
|
947
|
+
}
|
|
948
|
+
fields.sort((left, right) => left.field - right.field);
|
|
949
|
+
this.segment.reserveFields(ordinal + 1, this.fields.length);
|
|
950
|
+
for (const field of fields)
|
|
951
|
+
this.segment.fields.partial[field.field] = Number(this.partial.has(this.fields[field.field]));
|
|
952
|
+
return this.segment.insert(ordinal, fields);
|
|
953
|
+
}
|
|
954
|
+
encode() {
|
|
955
|
+
const { terms, pool, positions } = this.segment.postings;
|
|
956
|
+
let postings = 0;
|
|
957
|
+
let positionCount = 0;
|
|
958
|
+
for (let token = 0; token < terms.count; token++) {
|
|
959
|
+
postings += terms.lengths[token];
|
|
960
|
+
if (!this.trackPositions || pool.positionStarts === null) continue;
|
|
961
|
+
const end = terms.sliceStart[token] + terms.lengths[token];
|
|
962
|
+
for (let row2 = terms.sliceStart[token]; row2 < end; row2++)
|
|
963
|
+
positionCount += positions.values[pool.positionStarts[row2]];
|
|
964
|
+
}
|
|
965
|
+
const postingOffsets = new Uint32Array(terms.count + 1);
|
|
966
|
+
const postingDocIds = new Uint32Array(postings);
|
|
967
|
+
const postingFrequencies = new Uint16Array(postings);
|
|
968
|
+
const postingFieldIndices = new Uint16Array(postings);
|
|
969
|
+
const positionOffsets = this.trackPositions && postings > 0 ? new Uint32Array(postings + 1) : null;
|
|
970
|
+
const positionValues = positionOffsets === null ? null : new Uint32Array(positionCount);
|
|
971
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
972
|
+
let row = 0;
|
|
973
|
+
let position = 0;
|
|
974
|
+
for (let token = 0; token < terms.count; token++) {
|
|
975
|
+
const start = terms.sliceStart[token];
|
|
976
|
+
const length = terms.lengths[token];
|
|
977
|
+
postingOffsets[token] = row;
|
|
978
|
+
postingDocIds.set(pool.docIds.subarray(start, start + length), row);
|
|
979
|
+
postingFrequencies.set(pool.frequencies.subarray(start, start + length), row);
|
|
980
|
+
postingFieldIndices.set(pool.fieldIndices.subarray(start, start + length), row);
|
|
981
|
+
if (positionOffsets !== null && positionValues !== null && pool.positionStarts !== null) {
|
|
982
|
+
for (let at = start; at < start + length; at++) {
|
|
983
|
+
const offset = pool.positionStarts[at];
|
|
984
|
+
const count = positions.values[offset];
|
|
985
|
+
positionOffsets[row + at - start] = position;
|
|
986
|
+
positionValues.set(positions.values.subarray(offset + 1, offset + 1 + count), position);
|
|
987
|
+
position += count;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
row += length;
|
|
991
|
+
docFrequencies[terms.tokens[token]] = terms.frequencyAt(token);
|
|
992
|
+
}
|
|
993
|
+
postingOffsets[terms.count] = row;
|
|
994
|
+
if (positionOffsets !== null) positionOffsets[row] = position;
|
|
995
|
+
const fieldLengthColumns = [];
|
|
996
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
997
|
+
const lengths = this.segment.fields;
|
|
998
|
+
for (let field = 0; field < this.fields.length; field++) {
|
|
999
|
+
const start = field * lengths.ordinalCapacity;
|
|
1000
|
+
fieldLengthColumns.push(lengths.values.slice(start, start + lengths.documents[0]));
|
|
1001
|
+
totalFieldLengths[this.fields[field]] = lengths.totals[field];
|
|
1002
|
+
}
|
|
1003
|
+
let surfaceForms = null;
|
|
1004
|
+
if (this.segment.surfaces !== null) {
|
|
1005
|
+
const table = this.segment.surfaces;
|
|
1006
|
+
surfaceForms = /* @__PURE__ */ Object.create(null);
|
|
1007
|
+
for (const entry of table.liveEntries())
|
|
1008
|
+
surfaceForms[table.textAt(entry)] = [table.count[entry], terms.tokens[table.token[entry]]];
|
|
1009
|
+
}
|
|
1010
|
+
let partialSpellings = null;
|
|
1011
|
+
if (this.segment.spellings !== null) {
|
|
1012
|
+
const table = this.segment.spellings;
|
|
1013
|
+
partialSpellings = { fields: [...this.partial], tokens: [], spellings: [], counts: [] };
|
|
1014
|
+
for (const entry of table.liveEntries()) {
|
|
1015
|
+
partialSpellings.tokens.push(terms.tokens[table.token[entry]]);
|
|
1016
|
+
partialSpellings.spellings.push(table.textAt(entry));
|
|
1017
|
+
partialSpellings.counts.push(table.count[entry]);
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
return {
|
|
1021
|
+
tokens: [...terms.tokens],
|
|
1022
|
+
fieldNames: [...this.fields],
|
|
1023
|
+
postingOffsets,
|
|
1024
|
+
postingDocIds,
|
|
1025
|
+
postingFrequencies,
|
|
1026
|
+
postingFieldIndices,
|
|
1027
|
+
positionOffsets,
|
|
1028
|
+
positionValues,
|
|
1029
|
+
docFrequencies,
|
|
1030
|
+
fieldLengthNames: [...this.fields],
|
|
1031
|
+
fieldLengthColumns,
|
|
1032
|
+
totalFieldLengths,
|
|
1033
|
+
surfaceForms,
|
|
1034
|
+
partialSpellings
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
};
|
|
1038
|
+
|
|
1039
|
+
// src/core/partition/segment-builder/index.ts
|
|
1040
|
+
function indexOf(indexes, fieldPath, create) {
|
|
1041
|
+
let index = indexes.get(fieldPath);
|
|
1042
|
+
if (index === void 0) {
|
|
1043
|
+
index = create();
|
|
1044
|
+
indexes.set(fieldPath, index);
|
|
1045
|
+
}
|
|
1046
|
+
return index;
|
|
1047
|
+
}
|
|
1048
|
+
function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
1049
|
+
if (fieldType === "number" || fieldType === "number[]") {
|
|
1050
|
+
const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
|
|
1051
|
+
for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
|
|
1052
|
+
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
1053
|
+
const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
|
|
1054
|
+
for (const entry of fieldType === "boolean" ? [value] : value)
|
|
1055
|
+
index.insert(ordinal, entry);
|
|
1056
|
+
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
1057
|
+
const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
|
|
1058
|
+
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
1059
|
+
} else if (fieldType === "geopoint") {
|
|
1060
|
+
const point = value;
|
|
1061
|
+
indexOf(indexes.geoIndexes, fieldPath, createLoadedGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
function refuseRepeatedDocIds(documents) {
|
|
1065
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1066
|
+
for (const { docId } of documents) {
|
|
1067
|
+
if (seen.has(docId)) {
|
|
1068
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
|
|
1069
|
+
docId,
|
|
1070
|
+
partitionId: 0
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
seen.add(docId);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
function validateAsAnInsertDoes(documents, schema, options) {
|
|
1077
|
+
if (options?.validate === false) return;
|
|
1078
|
+
for (const { document } of documents) {
|
|
1079
|
+
validateDocument(document, schema, options?.patternValueLimit);
|
|
1080
|
+
if (options?.strict) validateDocumentStrict(document, schema);
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
function emptyFilters() {
|
|
1084
|
+
return {
|
|
1085
|
+
numericIndexes: /* @__PURE__ */ new Map(),
|
|
1086
|
+
booleanIndexes: /* @__PURE__ */ new Map(),
|
|
1087
|
+
enumIndexes: /* @__PURE__ */ new Map(),
|
|
1088
|
+
geoIndexes: /* @__PURE__ */ new Map(),
|
|
1089
|
+
patternIndexes: /* @__PURE__ */ new Map(),
|
|
1090
|
+
sizeIndexes: /* @__PURE__ */ new Map()
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
function buildSegmentPayloads(documents, schema, language, options, trackPositions) {
|
|
1094
|
+
refuseRepeatedDocIds(documents);
|
|
1095
|
+
validateAsAnInsertDoes(documents, schema, options);
|
|
1096
|
+
const fields = schemaFieldsOf(schema);
|
|
1097
|
+
let keywords = new SegmentKeywords(fields, trackPositions);
|
|
1098
|
+
let filters = emptyFilters();
|
|
1099
|
+
let start = 0;
|
|
1100
|
+
const outputs = [];
|
|
1101
|
+
const finish = (end) => {
|
|
1102
|
+
const members = documents.slice(start, end);
|
|
1103
|
+
const identity = Int32Array.from({ length: members.length }, (_, ordinal) => ordinal);
|
|
1104
|
+
outputs.push({
|
|
1105
|
+
documents: members,
|
|
1106
|
+
payload: {
|
|
1107
|
+
documentCount: members.length,
|
|
1108
|
+
docIds: members.map((entry) => entry.docId),
|
|
1109
|
+
...keywords.encode(),
|
|
1110
|
+
...encodeFieldIndexes(filters, identity)
|
|
1111
|
+
}
|
|
1112
|
+
});
|
|
1113
|
+
};
|
|
1114
|
+
for (let at = 0; at < documents.length; at++) {
|
|
1115
|
+
const document = documents[at].document;
|
|
1116
|
+
const text = prepareDocumentText(document, fields, language, options);
|
|
1117
|
+
if (!keywords.add(text, at - start)) {
|
|
1118
|
+
if (at === start) throw new SegmentRefusal(8);
|
|
1119
|
+
finish(at);
|
|
1120
|
+
start = at;
|
|
1121
|
+
keywords = new SegmentKeywords(fields, trackPositions);
|
|
1122
|
+
filters = emptyFilters();
|
|
1123
|
+
if (!keywords.add(text, 0)) throw new SegmentRefusal(8);
|
|
1124
|
+
}
|
|
1125
|
+
const ordinal = at - start;
|
|
1126
|
+
for (const { path: fieldPath, base: fieldType, pattern: pattern2 } of fields) {
|
|
1127
|
+
const value = getNestedValue(document, fieldPath);
|
|
1128
|
+
if (value === void 0 || value === null) continue;
|
|
1129
|
+
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
1130
|
+
if (pattern2) patternIndexOf(filters.patternIndexes, fieldPath).add(ordinal, value);
|
|
1131
|
+
if (!isWordIndexedBase(fieldType)) indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
if (start < documents.length || outputs.length === 0) finish(documents.length);
|
|
1135
|
+
return outputs;
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
// src/core/partition/pattern-snapshot.ts
|
|
1139
|
+
function serializePatternIndexes(state, schema) {
|
|
1140
|
+
const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
|
|
1141
|
+
if (patternFields.length === 0) return void 0;
|
|
1142
|
+
const resolver = state.docStore.resolver();
|
|
1143
|
+
const toExternal = (ordinal) => resolver.toExternal(ordinal);
|
|
1144
|
+
const serialized = /* @__PURE__ */ Object.create(null);
|
|
1145
|
+
for (const { path: fieldPath } of patternFields) {
|
|
1146
|
+
const index = state.patternIndexes.get(fieldPath);
|
|
1147
|
+
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : requirePatternSearch(fieldPath).serializeIndex(fieldPath, index, toExternal);
|
|
1148
|
+
}
|
|
1149
|
+
return serialized;
|
|
1150
|
+
}
|
|
1151
|
+
function rebuildPatternIndex(state, fieldPath) {
|
|
1152
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
1153
|
+
const value = getNestedValue(stored.fields, fieldPath);
|
|
1154
|
+
if (value === void 0 || value === null) continue;
|
|
1155
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
1156
|
+
if (internalId !== void 0) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
function loadPatternIndexes(state, serialized, schema) {
|
|
1160
|
+
const resolver = state.docStore.resolver();
|
|
1161
|
+
for (const { path: fieldPath, pattern: pattern2 } of schemaFieldsOf(schema)) {
|
|
1162
|
+
if (!pattern2) continue;
|
|
1163
|
+
const entry = serialized !== void 0 && Object.hasOwn(serialized, fieldPath) ? serialized[fieldPath] : void 0;
|
|
1164
|
+
if (entry === void 0) {
|
|
1165
|
+
rebuildPatternIndex(state, fieldPath);
|
|
1166
|
+
continue;
|
|
1167
|
+
}
|
|
1168
|
+
const decoded = requirePatternSearch(fieldPath).decodeIndex(fieldPath, entry);
|
|
1169
|
+
const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
|
|
1170
|
+
patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
// src/core/partition/serialization.ts
|
|
1175
|
+
function serializePartition(state, partitionId, indexName, totalPartitions, language, schema) {
|
|
1176
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
1177
|
+
const flatSchemaStrings = {};
|
|
1178
|
+
for (const [k, v] of Object.entries(flatSchema)) {
|
|
1179
|
+
flatSchemaStrings[k] = v;
|
|
1180
|
+
}
|
|
1181
|
+
const serializedDocs = /* @__PURE__ */ Object.create(null);
|
|
1182
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
1183
|
+
serializedDocs[docId] = {
|
|
1184
|
+
fields: stored.fields,
|
|
1185
|
+
fieldLengths: { ...stored.fieldLengths }
|
|
1186
|
+
};
|
|
1187
|
+
}
|
|
1188
|
+
const resolver = state.docStore.resolver();
|
|
1189
|
+
const fieldNames = state.fieldNameTable.names;
|
|
1190
|
+
const serializedInvertedIndex = /* @__PURE__ */ Object.create(null);
|
|
1191
|
+
for (const token of state.invertedIdx.tokens()) {
|
|
1192
|
+
const list = state.invertedIdx.lookup(token);
|
|
1193
|
+
if (!list) continue;
|
|
1194
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, positions, length } = list;
|
|
1195
|
+
const hasDeleted = deletedDocs.size > 0;
|
|
1196
|
+
const postings = [];
|
|
1197
|
+
for (let i = 0; i < length; i++) {
|
|
1198
|
+
if (hasDeleted && deletedDocs.has(docIds[i])) continue;
|
|
1199
|
+
const externalId = resolver.toExternal(docIds[i]);
|
|
1200
|
+
if (externalId === void 0) continue;
|
|
1201
|
+
postings.push({
|
|
1202
|
+
docId: externalId,
|
|
1203
|
+
termFrequency: termFrequencies[i],
|
|
1204
|
+
field: fieldNames[fieldNameIndices[i]],
|
|
1205
|
+
positions: positions === null ? [] : Array.from(positions[i] ?? [])
|
|
1206
|
+
});
|
|
1207
|
+
}
|
|
1208
|
+
if (postings.length > 0) {
|
|
1209
|
+
serializedInvertedIndex[token] = { docFrequency: list.docIdSet.size, postings };
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
const serializedNumeric = {};
|
|
1213
|
+
for (const [path, idx] of state.numericIndexes) {
|
|
1214
|
+
const raw = idx.serialize();
|
|
1215
|
+
serializedNumeric[path] = raw.map((e) => {
|
|
1216
|
+
const externalId = resolver.toExternal(e.docId);
|
|
1217
|
+
return { value: e.value, docId: externalId ?? "" };
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
const serializedBoolean = {};
|
|
1221
|
+
for (const [path, idx] of state.booleanIndexes) {
|
|
1222
|
+
const raw = idx.serialize();
|
|
1223
|
+
serializedBoolean[path] = {
|
|
1224
|
+
trueDocs: raw.trueDocs.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== ""),
|
|
1225
|
+
falseDocs: raw.falseDocs.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== "")
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1228
|
+
const serializedEnum = {};
|
|
1229
|
+
for (const [path, idx] of state.enumIndexes) {
|
|
1230
|
+
const raw = idx.serialize();
|
|
1231
|
+
const converted = /* @__PURE__ */ Object.create(null);
|
|
1232
|
+
for (const [value, internalIds] of Object.entries(raw)) {
|
|
1233
|
+
converted[value] = internalIds.map((id) => resolver.toExternal(id) ?? "").filter((id) => id !== "");
|
|
1234
|
+
}
|
|
1235
|
+
serializedEnum[path] = converted;
|
|
1236
|
+
}
|
|
1237
|
+
const serializedGeo = {};
|
|
1238
|
+
for (const [path, idx] of state.geoIndexes) {
|
|
1239
|
+
const raw = idx.serialize();
|
|
1240
|
+
serializedGeo[path] = raw.map((e) => ({
|
|
1241
|
+
lat: e.lat,
|
|
1242
|
+
lon: e.lon,
|
|
1243
|
+
docId: resolver.toExternal(e.docId) ?? ""
|
|
1244
|
+
}));
|
|
1245
|
+
}
|
|
1246
|
+
const serializedStats = state.stats.serialize();
|
|
1247
|
+
const serializedPattern = serializePatternIndexes(state, schema);
|
|
1248
|
+
return {
|
|
1249
|
+
indexName,
|
|
1250
|
+
partitionId,
|
|
1251
|
+
totalPartitions,
|
|
1252
|
+
language,
|
|
1253
|
+
schema: flatSchemaStrings,
|
|
1254
|
+
docCount: state.docStore.count(),
|
|
1255
|
+
avgDocLength: Object.values(serializedStats.averageFieldLengths).reduce((sum, v) => sum + v, 0),
|
|
1256
|
+
documents: serializedDocs,
|
|
1257
|
+
invertedIndex: serializedInvertedIndex,
|
|
1258
|
+
fieldIndexes: {
|
|
1259
|
+
numeric: serializedNumeric,
|
|
1260
|
+
boolean: serializedBoolean,
|
|
1261
|
+
enum: serializedEnum,
|
|
1262
|
+
geopoint: serializedGeo,
|
|
1263
|
+
...serializedPattern === void 0 ? {} : { pattern: serializedPattern }
|
|
1264
|
+
},
|
|
1265
|
+
surfaceForms: state.surfaceRegistry.serialize(),
|
|
1266
|
+
statistics: serializedStats
|
|
1267
|
+
};
|
|
1268
|
+
}
|
|
1269
|
+
function deserializePartition(state, data, clearFn, schema) {
|
|
1270
|
+
clearFn();
|
|
1271
|
+
const docsData = /* @__PURE__ */ Object.create(null);
|
|
1272
|
+
for (const [docId, doc] of Object.entries(data.documents)) {
|
|
1273
|
+
docsData[docId] = { fields: doc.fields, fieldLengths: doc.fieldLengths };
|
|
1274
|
+
}
|
|
1275
|
+
state.docStore.deserialize(docsData);
|
|
1276
|
+
const resolver = state.docStore.resolver();
|
|
1277
|
+
const invertedData = /* @__PURE__ */ Object.create(null);
|
|
1278
|
+
for (const [token, list] of Object.entries(data.invertedIndex)) {
|
|
1279
|
+
invertedData[token] = {
|
|
1280
|
+
docFrequency: list.docFrequency,
|
|
1281
|
+
postings: list.postings.map((p) => ({
|
|
1282
|
+
docId: p.docId,
|
|
1283
|
+
termFrequency: p.termFrequency,
|
|
1284
|
+
fieldName: p.field,
|
|
1285
|
+
positions: state.trackPositions ? [...p.positions] : []
|
|
1286
|
+
}))
|
|
1287
|
+
};
|
|
1288
|
+
}
|
|
1289
|
+
state.invertedIdx.deserialize(invertedData, resolver);
|
|
1290
|
+
for (const [path, entries] of Object.entries(data.fieldIndexes.numeric)) {
|
|
1291
|
+
const idx = createNumericIndex();
|
|
1292
|
+
const converted = entries.map((e) => {
|
|
1293
|
+
const internalId = resolver.toInternal(e.docId);
|
|
1294
|
+
return { value: e.value, docId: internalId ?? -1 };
|
|
1295
|
+
}).filter((e) => e.docId !== -1);
|
|
1296
|
+
idx.deserialize(converted);
|
|
1297
|
+
state.numericIndexes.set(path, idx);
|
|
1298
|
+
}
|
|
1299
|
+
for (const [path, serialized] of Object.entries(data.fieldIndexes.boolean)) {
|
|
1300
|
+
const idx = createBooleanIndex();
|
|
1301
|
+
const trueDocs = serialized.trueDocs.map((id) => resolver.toInternal(id)).filter((id) => id !== void 0);
|
|
1302
|
+
const falseDocs = serialized.falseDocs.map((id) => resolver.toInternal(id)).filter((id) => id !== void 0);
|
|
1303
|
+
idx.deserialize({ trueDocs, falseDocs });
|
|
1304
|
+
state.booleanIndexes.set(path, idx);
|
|
1305
|
+
}
|
|
1306
|
+
for (const [path, serialized] of Object.entries(data.fieldIndexes.enum)) {
|
|
1307
|
+
const idx = createEnumIndex();
|
|
1308
|
+
const converted = /* @__PURE__ */ Object.create(null);
|
|
1309
|
+
for (const [value, docIds] of Object.entries(serialized)) {
|
|
1310
|
+
converted[value] = docIds.map((id) => resolver.toInternal(id)).filter((id) => id !== void 0);
|
|
1311
|
+
}
|
|
1312
|
+
idx.deserialize(converted);
|
|
1313
|
+
state.enumIndexes.set(path, idx);
|
|
1314
|
+
}
|
|
1315
|
+
for (const [path, entries] of Object.entries(data.fieldIndexes.geopoint)) {
|
|
1316
|
+
const geoIdx = createLoadedGeoIndex();
|
|
1317
|
+
const converted = entries.map((e) => ({
|
|
1318
|
+
lat: e.lat,
|
|
1319
|
+
lon: e.lon,
|
|
1320
|
+
docId: resolver.toInternal(e.docId) ?? -1
|
|
1321
|
+
})).filter((e) => e.docId !== -1);
|
|
1322
|
+
geoIdx.deserialize(converted);
|
|
1323
|
+
state.geoIndexes.set(path, geoIdx);
|
|
1324
|
+
}
|
|
1325
|
+
state.surfaceRegistry.deserialize(data.surfaceForms ?? {});
|
|
1326
|
+
state.stats.deserialize(data.statistics);
|
|
1327
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
1328
|
+
const serializedFields = data.schema;
|
|
1329
|
+
for (const [field, expectedType] of Object.entries(flatSchema)) {
|
|
1330
|
+
if (field in serializedFields && serializedFields[field] !== expectedType) {
|
|
1331
|
+
throw new Error(
|
|
1332
|
+
`Schema mismatch on field "${field}": expected "${expectedType}", found "${serializedFields[field]}"`
|
|
1333
|
+
);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
for (const field of Object.keys(serializedFields)) {
|
|
1337
|
+
if (!(field in flatSchema)) {
|
|
1338
|
+
throw new Error(`Schema mismatch: serialized data contains unknown field "${field}" not present in schema`);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
1342
|
+
rebuildSizeIndexes(state, schema);
|
|
1343
|
+
state.partialWords.forgetSpellings();
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
// src/core/segment-merges/posting-context.ts
|
|
1347
|
+
var contexts = /* @__PURE__ */ new WeakMap();
|
|
1348
|
+
function heldViews(views) {
|
|
1349
|
+
return { views, lengths: views.map((view) => ArrayBuffer.isView(view) ? regionLength(view) : null) };
|
|
1350
|
+
}
|
|
1351
|
+
function lengthsHeld(held) {
|
|
1352
|
+
const { views, lengths } = held;
|
|
1353
|
+
for (let at = 0; at < views.length; at++) {
|
|
1354
|
+
const length = lengths[at];
|
|
1355
|
+
if (length !== null && regionLength(views[at]) !== length) return false;
|
|
1356
|
+
}
|
|
1357
|
+
return true;
|
|
1358
|
+
}
|
|
1359
|
+
function sameOutput(raw, held) {
|
|
1360
|
+
return raw.documents === held.documents && raw.frequencies === held.frequencies && raw.fields === held.fields && raw.positionOffsets === held.positionOffsets && raw.positions === held.positions && raw.totals === held.totals;
|
|
1361
|
+
}
|
|
1362
|
+
function sameStrings(left, right) {
|
|
1363
|
+
if (left === null || right === null) return left === right;
|
|
1364
|
+
return left.text === right.text && left.offsets === right.offsets && left.counts === right.counts;
|
|
1365
|
+
}
|
|
1366
|
+
function sameSegment(raw, held) {
|
|
1367
|
+
if (raw.documentCount !== held.documentCount || raw.fieldMapping !== held.fieldMapping || raw.payloadSlots !== held.payloadSlots || raw.postingOffsets !== held.postingOffsets || raw.mapping !== held.mapping || raw.fieldLengths.length !== held.fieldLengths.length || !sameStrings(raw.terms, held.terms) || !sameStrings(raw.surfaces, held.surfaces) || !sameStrings(raw.spellings, held.spellings)) {
|
|
1368
|
+
return false;
|
|
1369
|
+
}
|
|
1370
|
+
for (let at = 0; at < raw.fieldLengths.length; at++) {
|
|
1371
|
+
if (raw.fieldLengths[at] !== held.fieldLengths[at]) return false;
|
|
1372
|
+
}
|
|
1373
|
+
return true;
|
|
1374
|
+
}
|
|
1375
|
+
function stringViews(strings) {
|
|
1376
|
+
return strings === null ? [] : [strings.text, strings.offsets, strings.counts];
|
|
1377
|
+
}
|
|
1378
|
+
function segmentViews(input) {
|
|
1379
|
+
return [
|
|
1380
|
+
...stringViews(input.terms),
|
|
1381
|
+
input.fieldMapping,
|
|
1382
|
+
input.payloadSlots,
|
|
1383
|
+
input.postingOffsets,
|
|
1384
|
+
input.mapping,
|
|
1385
|
+
...stringViews(input.surfaces),
|
|
1386
|
+
...stringViews(input.spellings),
|
|
1387
|
+
...input.fieldLengths
|
|
1388
|
+
];
|
|
1389
|
+
}
|
|
1390
|
+
function readOutputs(regions, target) {
|
|
1391
|
+
const documents = regions.read(target.documents, "Uint32Array", void 0, true);
|
|
1392
|
+
return {
|
|
1393
|
+
documents,
|
|
1394
|
+
frequencies: regions.read(target.frequencies, "Uint16Array", documents.length, true),
|
|
1395
|
+
fields: regions.read(target.fields, "Uint16Array", documents.length, true),
|
|
1396
|
+
offsets: regions.read(target.positionOffsets, "Uint32Array", void 0, true),
|
|
1397
|
+
positions: regions.read(target.positions, "Uint32Array", void 0, true),
|
|
1398
|
+
totals: regions.read(target.totals, "Float64Array", void 0, true)
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
function contextOf(key, target) {
|
|
1402
|
+
const held = contexts.get(key);
|
|
1403
|
+
if (held !== void 0 && sameOutput(target, held.raw) && lengthsHeld(held)) return held;
|
|
1404
|
+
const regions = new MergeRegions();
|
|
1405
|
+
const outputs = readOutputs(regions, target);
|
|
1406
|
+
regions.check();
|
|
1407
|
+
const writes = [
|
|
1408
|
+
outputs.documents,
|
|
1409
|
+
outputs.frequencies,
|
|
1410
|
+
outputs.fields,
|
|
1411
|
+
outputs.offsets,
|
|
1412
|
+
outputs.positions,
|
|
1413
|
+
outputs.totals
|
|
1414
|
+
];
|
|
1415
|
+
const context = {
|
|
1416
|
+
...heldViews([
|
|
1417
|
+
target.documents,
|
|
1418
|
+
target.frequencies,
|
|
1419
|
+
target.fields,
|
|
1420
|
+
target.positionOffsets,
|
|
1421
|
+
target.positions,
|
|
1422
|
+
target.totals
|
|
1423
|
+
]),
|
|
1424
|
+
raw: target,
|
|
1425
|
+
outputs,
|
|
1426
|
+
writes,
|
|
1427
|
+
buffers: writes.map((output) => output.buffer),
|
|
1428
|
+
sources: /* @__PURE__ */ new Map()
|
|
1429
|
+
};
|
|
1430
|
+
contexts.set(key, context);
|
|
1431
|
+
return context;
|
|
1432
|
+
}
|
|
1433
|
+
function sourceOf2(context, key, input, target) {
|
|
1434
|
+
const held = context.sources.get(key);
|
|
1435
|
+
if (held !== void 0 && sameSegment(input, held.raw) && lengthsHeld(held)) return held.prepared;
|
|
1436
|
+
const regions = new MergeRegions();
|
|
1437
|
+
const prepared = regions.source(input);
|
|
1438
|
+
readOutputs(regions, target);
|
|
1439
|
+
regions.check();
|
|
1440
|
+
context.sources.set(key, { ...heldViews(segmentViews(input)), raw: input, prepared });
|
|
1441
|
+
return prepared;
|
|
1442
|
+
}
|
|
1443
|
+
function tokenRegion(context, value, kind, length) {
|
|
1444
|
+
const view = typedRegion(value, kind);
|
|
1445
|
+
requireSegment(view !== null && (length === void 0 || view.length === length));
|
|
1446
|
+
const { writes, buffers } = context;
|
|
1447
|
+
const buffer = view.buffer;
|
|
1448
|
+
for (let at = 0; at < writes.length; at++) {
|
|
1449
|
+
if (buffers[at] === buffer) requireSegment(!regionsOverlap(writes[at], view));
|
|
1450
|
+
}
|
|
1451
|
+
return view;
|
|
1452
|
+
}
|
|
1453
|
+
function prepareToken(context, input, prepared) {
|
|
1454
|
+
const postingDocuments = tokenRegion(context, input.postingDocuments, "Uint32Array");
|
|
1455
|
+
requireSegment(postingDocuments.length <= 4294967295);
|
|
1456
|
+
requireSegment(input.positionOffsets === null === (input.positionValues === null));
|
|
1457
|
+
const positionOffsets = input.positionOffsets === null ? null : tokenRegion(context, input.positionOffsets, "Uint32Array", postingDocuments.length + 1);
|
|
1458
|
+
const positionValues = input.positionValues === null ? null : tokenRegion(context, input.positionValues, "Uint32Array");
|
|
1459
|
+
requireSegment(positionValues === null || positionValues.length <= 4294967295);
|
|
1460
|
+
prepared.postingDocuments = postingDocuments;
|
|
1461
|
+
prepared.positionOffsets = positionOffsets;
|
|
1462
|
+
prepared.positionValues = positionValues;
|
|
1463
|
+
prepared.postingFrequencies = tokenRegion(context, input.postingFrequencies, "Uint16Array", postingDocuments.length);
|
|
1464
|
+
prepared.postingFields = tokenRegion(context, input.postingFields, "Uint16Array", postingDocuments.length);
|
|
1465
|
+
}
|
|
1466
|
+
function preparePostingsInContext(key, target, rawInputs, inputs) {
|
|
1467
|
+
const context = contextOf(key, target);
|
|
1468
|
+
const prepared = [];
|
|
1469
|
+
for (let at = 0; at < inputs.length; at++) {
|
|
1470
|
+
const input = inputs[at];
|
|
1471
|
+
const held = sourceOf2(context, rawInputs[at], input, target);
|
|
1472
|
+
const source2 = prepared.includes(held) ? { ...held } : held;
|
|
1473
|
+
prepareToken(context, input, source2);
|
|
1474
|
+
prepared.push(source2);
|
|
1475
|
+
}
|
|
1476
|
+
return { inputs: prepared, outputs: context.outputs };
|
|
1477
|
+
}
|
|
1478
|
+
function preparePostings(target, inputs) {
|
|
1479
|
+
const regions = new MergeRegions();
|
|
1480
|
+
const prepared = inputs.map((input) => regions.source(input));
|
|
1481
|
+
const outputs = readOutputs(regions, target);
|
|
1482
|
+
regions.check();
|
|
1483
|
+
return { inputs: prepared, outputs };
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
// src/core/segment-merges/constants.ts
|
|
1487
|
+
var POSTING_MERGE_START_ROWS = 1024;
|
|
1488
|
+
var POSTING_MERGE_GROWTH_FACTOR = 2;
|
|
1489
|
+
|
|
1490
|
+
// src/core/segment-merges/posting-rows.ts
|
|
1491
|
+
var PostingRows = class {
|
|
1492
|
+
targets = new Uint32Array(POSTING_MERGE_START_ROWS);
|
|
1493
|
+
fields = new Uint16Array(POSTING_MERGE_START_ROWS);
|
|
1494
|
+
sources = new Uint32Array(POSTING_MERGE_START_ROWS);
|
|
1495
|
+
rows = new Uint32Array(POSTING_MERGE_START_ROWS);
|
|
1496
|
+
order = new Uint32Array(POSTING_MERGE_START_ROWS);
|
|
1497
|
+
count = 0;
|
|
1498
|
+
ordered = true;
|
|
1499
|
+
reset(capacity) {
|
|
1500
|
+
this.count = 0;
|
|
1501
|
+
this.ordered = true;
|
|
1502
|
+
if (capacity <= this.targets.length) return;
|
|
1503
|
+
const size = Math.max(capacity, this.targets.length * POSTING_MERGE_GROWTH_FACTOR);
|
|
1504
|
+
this.targets = new Uint32Array(size);
|
|
1505
|
+
this.fields = new Uint16Array(size);
|
|
1506
|
+
this.sources = new Uint32Array(size);
|
|
1507
|
+
this.rows = new Uint32Array(size);
|
|
1508
|
+
this.order = new Uint32Array(size);
|
|
1509
|
+
}
|
|
1510
|
+
push(target, field, source2, row) {
|
|
1511
|
+
const at = this.count;
|
|
1512
|
+
if (at > 0 && this.ordered) {
|
|
1513
|
+
const before = at - 1;
|
|
1514
|
+
const order = this.targets[before] - target || this.fields[before] - field || this.sources[before] - source2 || this.rows[before] - row;
|
|
1515
|
+
if (order > 0) this.ordered = false;
|
|
1516
|
+
}
|
|
1517
|
+
this.targets[at] = target;
|
|
1518
|
+
this.fields[at] = field;
|
|
1519
|
+
this.sources[at] = source2;
|
|
1520
|
+
this.rows[at] = row;
|
|
1521
|
+
this.count = at + 1;
|
|
1522
|
+
}
|
|
1523
|
+
sequence() {
|
|
1524
|
+
const order = this.order.subarray(0, this.count);
|
|
1525
|
+
for (let at = 0; at < this.count; at++) order[at] = at;
|
|
1526
|
+
if (this.ordered) return order;
|
|
1527
|
+
const { targets, fields, sources, rows } = this;
|
|
1528
|
+
order.sort(
|
|
1529
|
+
(left, right) => targets[left] - targets[right] || fields[left] - fields[right] || sources[left] - sources[right] || rows[left] - rows[right]
|
|
1530
|
+
);
|
|
1531
|
+
return order;
|
|
1532
|
+
}
|
|
1533
|
+
};
|
|
1534
|
+
|
|
1535
|
+
// src/core/segment-merges/postings.ts
|
|
1536
|
+
var scratch = new PostingRows();
|
|
1537
|
+
function allFrozen(inputs) {
|
|
1538
|
+
for (const input of inputs) if (input.growing != null) return false;
|
|
1539
|
+
return true;
|
|
1540
|
+
}
|
|
1541
|
+
function collect(inputs, documents, fields) {
|
|
1542
|
+
let capacity = 0;
|
|
1543
|
+
for (const input of inputs) {
|
|
1544
|
+
capacity += input.postingDocuments.length;
|
|
1545
|
+
requireSegment(input.growing == null && capacity <= 4294967295);
|
|
1546
|
+
}
|
|
1547
|
+
scratch.reset(capacity);
|
|
1548
|
+
for (let source2 = 0; source2 < inputs.length; source2++) {
|
|
1549
|
+
const { postingDocuments, postingFields, documentCount, fieldMapping, mapping, positionOffsets, positionValues } = inputs[source2];
|
|
1550
|
+
for (let at = 0; at < postingDocuments.length; at++) {
|
|
1551
|
+
const document = postingDocuments[at];
|
|
1552
|
+
const field = postingFields[at];
|
|
1553
|
+
requireSegment(document < documentCount && field < fieldMapping.length);
|
|
1554
|
+
const mapped = fieldMapping[field];
|
|
1555
|
+
requireSegment(mapped < fields);
|
|
1556
|
+
if (positionOffsets !== null && positionValues !== null) {
|
|
1557
|
+
const start = positionOffsets[at];
|
|
1558
|
+
const end = positionOffsets[at + 1];
|
|
1559
|
+
requireSegment(start <= end && end <= positionValues.length);
|
|
1560
|
+
for (let position = start + 1; position < end; position++) {
|
|
1561
|
+
requireSegment(positionValues[position] >= positionValues[position - 1]);
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
const target = mapping[document];
|
|
1565
|
+
if (target === 4294967295) continue;
|
|
1566
|
+
requireSegment(target < documents);
|
|
1567
|
+
scratch.push(target, mapped, source2, at);
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
return scratch;
|
|
1571
|
+
}
|
|
1572
|
+
function positionCountOf(input, row) {
|
|
1573
|
+
const { positionOffsets, positionValues } = input;
|
|
1574
|
+
return positionOffsets === null || positionValues === null ? 0 : positionOffsets[row + 1] - positionOffsets[row];
|
|
1575
|
+
}
|
|
1576
|
+
var mergePostings = (rawInputs, documentCount, fieldCount, rawOutput) => {
|
|
1577
|
+
try {
|
|
1578
|
+
const captured = captureInputs(rawInputs);
|
|
1579
|
+
requireSegment(typeof rawOutput === "object" && rawOutput !== null);
|
|
1580
|
+
const target = { ...rawOutput };
|
|
1581
|
+
uint32(documentCount);
|
|
1582
|
+
uint32(fieldCount);
|
|
1583
|
+
requireSegment(fieldCount <= 65536);
|
|
1584
|
+
const { inputs, outputs } = allFrozen(captured) ? preparePostingsInContext(rawOutput, target, rawInputs, captured) : preparePostings(target, captured);
|
|
1585
|
+
const { documents, frequencies, fields, offsets, positions, totals } = outputs;
|
|
1586
|
+
const rows = collect(inputs, documentCount, fieldCount);
|
|
1587
|
+
const order = rows.sequence();
|
|
1588
|
+
const { targets, sources } = rows;
|
|
1589
|
+
let positionCount = 0;
|
|
1590
|
+
let documentTotal = 0;
|
|
1591
|
+
let frequencyTotal = 0;
|
|
1592
|
+
for (let at = 0; at < order.length; at++) {
|
|
1593
|
+
const row = order[at];
|
|
1594
|
+
const input = inputs[sources[row]];
|
|
1595
|
+
const read = rows.rows[row];
|
|
1596
|
+
positionCount += positionCountOf(input, read);
|
|
1597
|
+
requireSegment(positionCount <= 4294967295);
|
|
1598
|
+
frequencyTotal += input.postingFrequencies[read];
|
|
1599
|
+
if (at === 0 || targets[row] !== targets[order[at - 1]]) documentTotal++;
|
|
1600
|
+
else {
|
|
1601
|
+
const before = order[at - 1];
|
|
1602
|
+
requireSegment(
|
|
1603
|
+
sources[row] === sources[before] && input.postingDocuments[read] === inputs[sources[before]].postingDocuments[rows.rows[before]] && rows.fields[row] !== rows.fields[before]
|
|
1604
|
+
);
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
requireRoom(
|
|
1608
|
+
order.length <= documents.length && order.length < offsets.length && positionCount <= positions.length && totals.length >= 2
|
|
1609
|
+
);
|
|
1610
|
+
let next = 0;
|
|
1611
|
+
for (let at = 0; at < order.length; at++) {
|
|
1612
|
+
const row = order[at];
|
|
1613
|
+
const input = inputs[sources[row]];
|
|
1614
|
+
const read = rows.rows[row];
|
|
1615
|
+
documents[at] = targets[row];
|
|
1616
|
+
frequencies[at] = input.postingFrequencies[read];
|
|
1617
|
+
fields[at] = rows.fields[row];
|
|
1618
|
+
offsets[at] = next;
|
|
1619
|
+
const { positionOffsets, positionValues } = input;
|
|
1620
|
+
if (positionOffsets !== null && positionValues !== null) {
|
|
1621
|
+
const end = positionOffsets[read + 1];
|
|
1622
|
+
for (let position = positionOffsets[read]; position < end; position++)
|
|
1623
|
+
positions[next++] = positionValues[position];
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
offsets[order.length] = next;
|
|
1627
|
+
totals[0] = documentTotal;
|
|
1628
|
+
totals[1] = frequencyTotal;
|
|
1629
|
+
return order.length;
|
|
1630
|
+
} catch (error) {
|
|
1631
|
+
if (error instanceof SegmentRefusal) return -error.status;
|
|
1632
|
+
throw error;
|
|
1633
|
+
}
|
|
1634
|
+
};
|
|
1635
|
+
|
|
1636
|
+
export { alreadyPresentOnWorker, bindSegmentMergeHost, buildSegmentPayloads, createRequestId, deserializePartition, eligibleIndexNames, fieldIndexRemapOf, freezeEncodedSegmentShared, freezeSegmentShared, holdsADiscardedField, isDeterministicFailure, isValidWorkerResponse, mergeFieldIndexes, mergePostings, mergeSegmentContents, mergeSourceSpellings, mergeSourceSurfaces, mergedPartialSpellings, reportIneligible, segmentArraysOf, selectionKeeps, serializePartition, shareSegmentArrays, toError, workerIneligibility };
|