@delali/narsil 0.3.1-next.99 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
package/dist/chunk-NKRBEGJZ.mjs
DELETED
|
@@ -1,1705 +0,0 @@
|
|
|
1
|
-
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex } from './chunk-K25LZA54.mjs';
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-LXWDQLTW.mjs';
|
|
3
|
-
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QY7GC2MW.mjs';
|
|
4
|
-
import { flattenSchema, partialFieldsOf } from './chunk-LXPIXDUJ.mjs';
|
|
5
|
-
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-E74YLAIG.mjs';
|
|
6
|
-
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-OOEXLUGR.mjs';
|
|
8
|
-
import { tokenize } from './chunk-LU2CE5AE.mjs';
|
|
9
|
-
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
|
-
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
13
|
-
|
|
14
|
-
// src/search/cursor-binding.ts
|
|
15
|
-
var ABSENT_TAG = 0;
|
|
16
|
-
var NULL_TAG = 1;
|
|
17
|
-
var FALSE_TAG = 2;
|
|
18
|
-
var TRUE_TAG = 3;
|
|
19
|
-
var NUMBER_TAG = 4;
|
|
20
|
-
var STRING_TAG = 5;
|
|
21
|
-
var LIST_TAG = 6;
|
|
22
|
-
var MAP_TAG = 7;
|
|
23
|
-
var LENGTH_BYTES = 4;
|
|
24
|
-
var LAST_ASCII_UNIT = 127;
|
|
25
|
-
var encoder = new TextEncoder();
|
|
26
|
-
function flush(stream) {
|
|
27
|
-
stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
|
|
28
|
-
stream.length = 0;
|
|
29
|
-
}
|
|
30
|
-
function ensureCapacity(stream, extra) {
|
|
31
|
-
if (stream.length + extra <= stream.bytes.length) return;
|
|
32
|
-
const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
|
|
33
|
-
grown.set(stream.bytes.subarray(0, stream.length));
|
|
34
|
-
stream.bytes = grown;
|
|
35
|
-
}
|
|
36
|
-
function writeByte(stream, value) {
|
|
37
|
-
ensureCapacity(stream, 1);
|
|
38
|
-
stream.bytes[stream.length] = value;
|
|
39
|
-
stream.length += 1;
|
|
40
|
-
}
|
|
41
|
-
function writeUint32(stream, value) {
|
|
42
|
-
ensureCapacity(stream, 4);
|
|
43
|
-
stream.bytes[stream.length] = value >>> 24 & 255;
|
|
44
|
-
stream.bytes[stream.length + 1] = value >>> 16 & 255;
|
|
45
|
-
stream.bytes[stream.length + 2] = value >>> 8 & 255;
|
|
46
|
-
stream.bytes[stream.length + 3] = value & 255;
|
|
47
|
-
stream.length += 4;
|
|
48
|
-
}
|
|
49
|
-
var numberBytes = new DataView(new ArrayBuffer(8));
|
|
50
|
-
function writeNumber(stream, value) {
|
|
51
|
-
numberBytes.setFloat64(0, value, false);
|
|
52
|
-
ensureCapacity(stream, 8);
|
|
53
|
-
for (let i = 0; i < 8; i++) {
|
|
54
|
-
stream.bytes[stream.length + i] = numberBytes.getUint8(i);
|
|
55
|
-
}
|
|
56
|
-
stream.length += 8;
|
|
57
|
-
}
|
|
58
|
-
function writeEncodedString(stream, value) {
|
|
59
|
-
const encoded = encoder.encode(value);
|
|
60
|
-
writeUint32(stream, encoded.length);
|
|
61
|
-
ensureCapacity(stream, encoded.length);
|
|
62
|
-
stream.bytes.set(encoded, stream.length);
|
|
63
|
-
stream.length += encoded.length;
|
|
64
|
-
}
|
|
65
|
-
function writeString(stream, value) {
|
|
66
|
-
ensureCapacity(stream, LENGTH_BYTES + value.length);
|
|
67
|
-
const start = stream.length + LENGTH_BYTES;
|
|
68
|
-
const bytes = stream.bytes;
|
|
69
|
-
for (let at = 0; at < value.length; at++) {
|
|
70
|
-
const unit = value.charCodeAt(at);
|
|
71
|
-
if (unit > LAST_ASCII_UNIT) {
|
|
72
|
-
writeEncodedString(stream, value);
|
|
73
|
-
return;
|
|
74
|
-
}
|
|
75
|
-
bytes[start + at] = unit;
|
|
76
|
-
}
|
|
77
|
-
writeUint32(stream, value.length);
|
|
78
|
-
stream.length += value.length;
|
|
79
|
-
}
|
|
80
|
-
var NumberList = class {
|
|
81
|
-
constructor(members) {
|
|
82
|
-
this.members = members;
|
|
83
|
-
}
|
|
84
|
-
};
|
|
85
|
-
function writeNumberList(stream, list) {
|
|
86
|
-
writeByte(stream, LIST_TAG);
|
|
87
|
-
writeUint32(stream, list.members.length);
|
|
88
|
-
flush(stream);
|
|
89
|
-
stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
|
|
90
|
-
}
|
|
91
|
-
function writeValue(stream, value) {
|
|
92
|
-
if (value instanceof NumberList) {
|
|
93
|
-
writeNumberList(stream, value);
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
96
|
-
if (value === void 0) {
|
|
97
|
-
writeByte(stream, ABSENT_TAG);
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
if (value === null) {
|
|
101
|
-
writeByte(stream, NULL_TAG);
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
if (typeof value === "boolean") {
|
|
105
|
-
writeByte(stream, value ? TRUE_TAG : FALSE_TAG);
|
|
106
|
-
return;
|
|
107
|
-
}
|
|
108
|
-
if (typeof value === "number") {
|
|
109
|
-
writeByte(stream, NUMBER_TAG);
|
|
110
|
-
writeNumber(stream, value);
|
|
111
|
-
return;
|
|
112
|
-
}
|
|
113
|
-
if (typeof value === "string") {
|
|
114
|
-
writeByte(stream, STRING_TAG);
|
|
115
|
-
writeString(stream, value);
|
|
116
|
-
return;
|
|
117
|
-
}
|
|
118
|
-
if (Array.isArray(value)) {
|
|
119
|
-
writeByte(stream, LIST_TAG);
|
|
120
|
-
writeUint32(stream, value.length);
|
|
121
|
-
for (const member of value) {
|
|
122
|
-
writeValue(stream, member ?? null);
|
|
123
|
-
}
|
|
124
|
-
return;
|
|
125
|
-
}
|
|
126
|
-
const record = value;
|
|
127
|
-
const keys = Object.keys(record).sort(compareCodePoints);
|
|
128
|
-
writeByte(stream, MAP_TAG);
|
|
129
|
-
writeUint32(stream, keys.length);
|
|
130
|
-
for (const key of keys) {
|
|
131
|
-
writeByte(stream, STRING_TAG);
|
|
132
|
-
writeString(stream, key);
|
|
133
|
-
writeValue(stream, record[key] ?? null);
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
function boundVectorOf(vector) {
|
|
137
|
-
if (vector === void 0) return void 0;
|
|
138
|
-
const bound = { ...vector };
|
|
139
|
-
if (vector.value !== void 0) {
|
|
140
|
-
bound.value = new NumberList(vector.value);
|
|
141
|
-
}
|
|
142
|
-
if (vector.text !== void 0) {
|
|
143
|
-
delete bound.value;
|
|
144
|
-
}
|
|
145
|
-
return bound;
|
|
146
|
-
}
|
|
147
|
-
function boundRerankOf(rerank) {
|
|
148
|
-
if (typeof rerank !== "object" || rerank === null || Array.isArray(rerank)) return rerank;
|
|
149
|
-
const present = {};
|
|
150
|
-
for (const [key, value] of Object.entries(rerank)) {
|
|
151
|
-
if (value !== void 0) present[key] = value;
|
|
152
|
-
}
|
|
153
|
-
return present;
|
|
154
|
-
}
|
|
155
|
-
var FILTERS_SLOT = 2;
|
|
156
|
-
var BOUND_SLOT_COUNT = 16;
|
|
157
|
-
function bindingOf(values) {
|
|
158
|
-
const stream = { bytes: new Uint8Array(256), length: 0, hash: FNV_OFFSET_BASIS };
|
|
159
|
-
for (const value of values) {
|
|
160
|
-
writeValue(stream, value);
|
|
161
|
-
}
|
|
162
|
-
flush(stream);
|
|
163
|
-
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
164
|
-
}
|
|
165
|
-
function queryBindingOf(params) {
|
|
166
|
-
const values = [
|
|
167
|
-
params.term,
|
|
168
|
-
params.fields,
|
|
169
|
-
params.filters,
|
|
170
|
-
params.boost,
|
|
171
|
-
params.minScore,
|
|
172
|
-
params.termMatch,
|
|
173
|
-
params.tolerance,
|
|
174
|
-
params.prefixLength,
|
|
175
|
-
params.prefix,
|
|
176
|
-
params.partialWords,
|
|
177
|
-
params.exact,
|
|
178
|
-
params.pinned,
|
|
179
|
-
params.mode,
|
|
180
|
-
params.hybrid,
|
|
181
|
-
boundVectorOf(params.vector),
|
|
182
|
-
params.scoring ?? "local"
|
|
183
|
-
];
|
|
184
|
-
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
185
|
-
return bindingOf(values);
|
|
186
|
-
}
|
|
187
|
-
function listBindingOf(filters) {
|
|
188
|
-
const values = new Array(BOUND_SLOT_COUNT).fill(void 0);
|
|
189
|
-
values[FILTERS_SLOT] = filters;
|
|
190
|
-
return bindingOf(values);
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
// src/search/query-fields.ts
|
|
194
|
-
function requireBoostableFields(boost, flatSchema) {
|
|
195
|
-
for (const [field, weight] of Object.entries(boost)) {
|
|
196
|
-
const fieldType = flatSchema[field];
|
|
197
|
-
if (fieldType === void 0 || !isWordIndexedFieldType(fieldType)) {
|
|
198
|
-
throw new NarsilError(
|
|
199
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
200
|
-
fieldType === void 0 ? `The engine boosts a text field that the schema declares, and the schema declares no field "${field}"` : `The engine boosts a text field alone, and the schema declares field "${field}" as "${fieldType}"`,
|
|
201
|
-
{ field, ...fieldType === void 0 ? {} : { fieldType } }
|
|
202
|
-
);
|
|
203
|
-
}
|
|
204
|
-
if (typeof weight !== "number" || !Number.isFinite(weight)) {
|
|
205
|
-
throw new NarsilError(
|
|
206
|
-
ErrorCodes.CONFIG_INVALID,
|
|
207
|
-
`The boost on field "${field}" must be a finite number, and this query sets ${String(weight)}`,
|
|
208
|
-
{ field, boost: String(weight) }
|
|
209
|
-
);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
function requireFacetableFields(facets, flatSchema, strict) {
|
|
214
|
-
for (const field of Object.keys(facets)) {
|
|
215
|
-
const fieldType = flatSchema[field];
|
|
216
|
-
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "facet");
|
|
217
|
-
if (isGeopointOrVectorType(fieldType)) {
|
|
218
|
-
throw new NarsilError(
|
|
219
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
220
|
-
`The engine counts the values of a text, number, boolean, or enum field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
221
|
-
{ field, fieldType }
|
|
222
|
-
);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
function requireGroupableFields(fields, flatSchema, strict) {
|
|
227
|
-
for (const field of fields) {
|
|
228
|
-
const fieldType = flatSchema[field];
|
|
229
|
-
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(field, "group");
|
|
230
|
-
if (fieldType !== void 0 && VECTOR_PATTERN.test(fieldType)) {
|
|
231
|
-
throw new NarsilError(
|
|
232
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
233
|
-
`The engine groups hits by a text, number, boolean, enum, or geopoint field, and the schema declares field "${field}" as "${fieldType}"`,
|
|
234
|
-
{ field, fieldType }
|
|
235
|
-
);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
function requireUsableQueryFields(params, schema, strict = false) {
|
|
240
|
-
const groupFields = params.group?.fields;
|
|
241
|
-
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
242
|
-
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
243
|
-
const flatSchema = flattenSchema(schema);
|
|
244
|
-
if (params.boost !== void 0) requireBoostableFields(params.boost, flatSchema);
|
|
245
|
-
if (params.facets !== void 0) requireFacetableFields(params.facets, flatSchema, strict);
|
|
246
|
-
if (checksGroup) requireGroupableFields(groupFields, flatSchema, strict);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// src/search/typo-tolerance.ts
|
|
250
|
-
function requireIntegerUpTo(option, value, maximum) {
|
|
251
|
-
if (value === void 0 || Number.isInteger(value) && value >= 0 && value <= maximum) return;
|
|
252
|
-
throw new NarsilError(
|
|
253
|
-
ErrorCodes.CONFIG_INVALID,
|
|
254
|
-
`The query option "${option}" takes an integer from 0 to ${maximum}, and this query sets ${String(value)}`,
|
|
255
|
-
{ option, value: String(value), maximum }
|
|
256
|
-
);
|
|
257
|
-
}
|
|
258
|
-
function requireValidTypoTolerance(params) {
|
|
259
|
-
requireIntegerUpTo("tolerance", params.tolerance, MAX_TOLERANCE);
|
|
260
|
-
requireIntegerUpTo("prefixLength", params.prefixLength, MAX_PREFIX_LENGTH);
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
// src/engine/query/shared.ts
|
|
264
|
-
function withPatternWork(context) {
|
|
265
|
-
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
|
|
266
|
-
}
|
|
267
|
-
function partitionsFor(manager, partitionIds) {
|
|
268
|
-
return partitionsIn(manager, partitionIds);
|
|
269
|
-
}
|
|
270
|
-
function coverageFor(manager, partitionIds) {
|
|
271
|
-
if (partitionIds === void 0) {
|
|
272
|
-
return {
|
|
273
|
-
totalPartitions: manager.partitionCount,
|
|
274
|
-
queriedPartitions: manager.partitionCount,
|
|
275
|
-
timedOutPartitions: 0,
|
|
276
|
-
failedPartitions: 0
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
|
-
const queriedPartitions = partitionsFor(manager, partitionIds).length;
|
|
280
|
-
return {
|
|
281
|
-
totalPartitions: partitionIds.length,
|
|
282
|
-
queriedPartitions,
|
|
283
|
-
timedOutPartitions: 0,
|
|
284
|
-
failedPartitions: partitionIds.length - queriedPartitions
|
|
285
|
-
};
|
|
286
|
-
}
|
|
287
|
-
function partitionsForVectorSearch(manager, vecIndex, partitionIds) {
|
|
288
|
-
if (partitionIds === void 0) {
|
|
289
|
-
return void 0;
|
|
290
|
-
}
|
|
291
|
-
if (!vecIndex.partitionsKnown()) {
|
|
292
|
-
vecIndex.assignPartitions((docId) => manager.partitionIdOf(docId));
|
|
293
|
-
}
|
|
294
|
-
return new Set(partitionIds);
|
|
295
|
-
}
|
|
296
|
-
function scoringConfigFor(params, context) {
|
|
297
|
-
const scoringMode = params.scoring ?? context.config.defaultScoring ?? "local";
|
|
298
|
-
if (scoringMode !== "broadcast") {
|
|
299
|
-
return { scoringMode, partitionIds: context.partitionIds };
|
|
300
|
-
}
|
|
301
|
-
return { scoringMode, globalStats: context.broadcastStats?.(context.indexName), partitionIds: context.partitionIds };
|
|
302
|
-
}
|
|
303
|
-
function broadcastStatsForWorker(params, context, scoring) {
|
|
304
|
-
if (scoring.scoringMode !== "broadcast" || scoring.globalStats === void 0) {
|
|
305
|
-
return void 0;
|
|
306
|
-
}
|
|
307
|
-
const term = params.term;
|
|
308
|
-
if (term === void 0 || term.trim().length === 0) {
|
|
309
|
-
return void 0;
|
|
310
|
-
}
|
|
311
|
-
const { manager } = context;
|
|
312
|
-
const analysis = { ...manager.analysis, synonyms: manager.synonyms };
|
|
313
|
-
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, analysis, {
|
|
314
|
-
partitions: partitionsFor(manager, context.partitionIds),
|
|
315
|
-
expansion: { options: params, schema: context.config.schema }
|
|
316
|
-
});
|
|
317
|
-
}
|
|
318
|
-
function searchOptionsFor(manager, patternWork) {
|
|
319
|
-
return {
|
|
320
|
-
bm25Params: manager.config.bm25,
|
|
321
|
-
stopWords: manager.analysis.stopWords,
|
|
322
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
323
|
-
synonyms: manager.synonyms,
|
|
324
|
-
patternWork
|
|
325
|
-
};
|
|
326
|
-
}
|
|
327
|
-
function vectorResultsToScored(results) {
|
|
328
|
-
return results.map((r) => ({
|
|
329
|
-
docId: r.docId,
|
|
330
|
-
score: r.score,
|
|
331
|
-
termFrequencies: {},
|
|
332
|
-
fieldLengths: {},
|
|
333
|
-
idf: {}
|
|
334
|
-
}));
|
|
335
|
-
}
|
|
336
|
-
function vectorSearchersOf(context) {
|
|
337
|
-
return context.vectorSearchers ?? context.manager.getVectorIndexes();
|
|
338
|
-
}
|
|
339
|
-
function resolveVectorIndex(context, fieldName) {
|
|
340
|
-
return vectorSearchersOf(context).get(fieldName);
|
|
341
|
-
}
|
|
342
|
-
var SEARCH_MODES = /* @__PURE__ */ new Set(["fulltext", "vector", "hybrid"]);
|
|
343
|
-
function requireKnownMode(params) {
|
|
344
|
-
if (params.mode === void 0 || SEARCH_MODES.has(params.mode)) return;
|
|
345
|
-
throw new NarsilError(
|
|
346
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
347
|
-
`A search takes the mode "fulltext", "vector", or "hybrid", while this query names "${String(params.mode)}"`,
|
|
348
|
-
{ mode: String(params.mode), modes: [...SEARCH_MODES] }
|
|
349
|
-
);
|
|
350
|
-
}
|
|
351
|
-
function requireBooleanPartialWords(params) {
|
|
352
|
-
const setting = params.partialWords;
|
|
353
|
-
if (setting === void 0 || typeof setting === "boolean") return;
|
|
354
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The "partialWords" query option takes true or false', {
|
|
355
|
-
option: "partialWords",
|
|
356
|
-
received: setting === null ? "null" : typeof setting
|
|
357
|
-
});
|
|
358
|
-
}
|
|
359
|
-
function validQueryParams(params, config) {
|
|
360
|
-
const strict = config.strict === true;
|
|
361
|
-
requireValidTypoTolerance(params);
|
|
362
|
-
requireBooleanPartialWords(params);
|
|
363
|
-
requireSortableFields(params.sort, config.schema, strict);
|
|
364
|
-
const filters = validQueryFilter(params.filters, config.schema);
|
|
365
|
-
requireUsableQueryFields(params, config.schema, strict);
|
|
366
|
-
return filters === params.filters ? params : { ...params, filters };
|
|
367
|
-
}
|
|
368
|
-
function validQueryFilter(filters, schema) {
|
|
369
|
-
if (filters === void 0 || filters === null) return filters;
|
|
370
|
-
return validatedFilterOf(filters, flattenSchema(schema));
|
|
371
|
-
}
|
|
372
|
-
function requireVectorSearchable(params, context, needsVector) {
|
|
373
|
-
if (!needsVector) return;
|
|
374
|
-
const field = params.vector?.field;
|
|
375
|
-
if (field === void 0) {
|
|
376
|
-
throw new NarsilError(
|
|
377
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
378
|
-
`A "${params.mode}" query needs a vector clause naming the field to search`,
|
|
379
|
-
{ mode: params.mode }
|
|
380
|
-
);
|
|
381
|
-
}
|
|
382
|
-
if (vectorSearchersOf(context).has(field)) return;
|
|
383
|
-
throw new NarsilError(
|
|
384
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
385
|
-
`Field "${field}" has no vector index, so a vector search cannot use it`,
|
|
386
|
-
{ field, vectorFields: [...vectorSearchersOf(context).keys()] }
|
|
387
|
-
);
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
// src/engine/validation.ts
|
|
391
|
-
function now() {
|
|
392
|
-
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
393
|
-
return performance.now();
|
|
394
|
-
}
|
|
395
|
-
return Date.now();
|
|
396
|
-
}
|
|
397
|
-
function validateIndexName(name) {
|
|
398
|
-
const given = name;
|
|
399
|
-
if (typeof given !== "string" || name.length === 0) {
|
|
400
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index name must be a string of at least one character", {
|
|
401
|
-
received: given === null ? "null" : typeof given
|
|
402
|
-
});
|
|
403
|
-
}
|
|
404
|
-
if (name.length > MAX_INDEX_NAME_LENGTH) {
|
|
405
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`, {
|
|
406
|
-
indexName: name,
|
|
407
|
-
length: name.length
|
|
408
|
-
});
|
|
409
|
-
}
|
|
410
|
-
if (!INDEX_NAME_PATTERN.test(name)) {
|
|
411
|
-
throw new NarsilError(
|
|
412
|
-
ErrorCodes.CONFIG_INVALID,
|
|
413
|
-
`Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
|
|
414
|
-
{ indexName: name }
|
|
415
|
-
);
|
|
416
|
-
}
|
|
417
|
-
if (name.includes("..")) {
|
|
418
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Index name "${name}" must not contain ".."`, { indexName: name });
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
function validateDocId(docId) {
|
|
422
|
-
const given = docId;
|
|
423
|
-
if (typeof given !== "string") {
|
|
424
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must be a string", {
|
|
425
|
-
received: given === null ? "null" : Array.isArray(given) ? "array" : typeof given
|
|
426
|
-
});
|
|
427
|
-
}
|
|
428
|
-
if (docId.length === 0) {
|
|
429
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
430
|
-
}
|
|
431
|
-
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
432
|
-
throw new NarsilError(
|
|
433
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
434
|
-
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
435
|
-
{
|
|
436
|
-
docId,
|
|
437
|
-
length: docId.length
|
|
438
|
-
}
|
|
439
|
-
);
|
|
440
|
-
}
|
|
441
|
-
if (docId.includes("\0")) {
|
|
442
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
443
|
-
}
|
|
444
|
-
if (holdsUnpairedSurrogate(docId)) {
|
|
445
|
-
throw new NarsilError(
|
|
446
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
447
|
-
"Document ID holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8",
|
|
448
|
-
{ docId }
|
|
449
|
-
);
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
function validatePartitionConfig(partitions) {
|
|
453
|
-
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
454
|
-
if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
455
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxDocsPerPartition must be a positive integer", {
|
|
456
|
-
maxDocsPerPartition
|
|
457
|
-
});
|
|
458
|
-
}
|
|
459
|
-
if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
460
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxPartitions must be a positive integer", {
|
|
461
|
-
maxPartitions
|
|
462
|
-
});
|
|
463
|
-
}
|
|
464
|
-
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
465
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
|
|
466
|
-
watermark
|
|
467
|
-
});
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
function requireSurfaceFormsForPartialFields(schema, surfaceForms) {
|
|
471
|
-
if (surfaceForms !== false) return;
|
|
472
|
-
const partialFields = partialFieldsOf(schema);
|
|
473
|
-
if (partialFields.length === 0) return;
|
|
474
|
-
throw new NarsilError(
|
|
475
|
-
ErrorCodes.CONFIG_INVALID,
|
|
476
|
-
`Field "${partialFields[0]}" declares "partial", which compares words as the documents wrote them, so its index needs surfaceForms left on`,
|
|
477
|
-
{ fields: [...partialFields], surfaceForms: false }
|
|
478
|
-
);
|
|
479
|
-
}
|
|
480
|
-
function validatePatternValueLimit(limit) {
|
|
481
|
-
if (limit === void 0) return;
|
|
482
|
-
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < MIN_PATTERN_VALUE_LIMIT || limit > MAX_PATTERN_VALUE_LIMIT) {
|
|
483
|
-
throw new NarsilError(
|
|
484
|
-
ErrorCodes.CONFIG_INVALID,
|
|
485
|
-
`patternValueLimit must be a whole number of code points from ${MIN_PATTERN_VALUE_LIMIT} to ${MAX_PATTERN_VALUE_LIMIT}`,
|
|
486
|
-
{ patternValueLimit: typeof limit === "number" ? limit : String(limit) }
|
|
487
|
-
);
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
function requirePositiveSafeInteger(field, value) {
|
|
491
|
-
if (value === void 0) return;
|
|
492
|
-
if (!Number.isSafeInteger(value) || value < 1) {
|
|
493
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${field} must be a positive safe integer`, { field, value });
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
function validateBootstrapModule(bootstrapModule) {
|
|
497
|
-
if (bootstrapModule === void 0) return;
|
|
498
|
-
const absolute = bootstrapModule.startsWith("/") || ABSOLUTE_WINDOWS_PATH_PATTERN.test(bootstrapModule) || MODULE_URL_SCHEME_PATTERN.test(bootstrapModule);
|
|
499
|
-
if (absolute) return;
|
|
500
|
-
throw new NarsilError(
|
|
501
|
-
ErrorCodes.CONFIG_INVALID,
|
|
502
|
-
`workers.bootstrapModule is "${bootstrapModule}", which a worker would resolve against Narsil's own directory, so pass an absolute path or a URL such as new URL('./bootstrap.js', import.meta.url).href`,
|
|
503
|
-
{ bootstrapModule }
|
|
504
|
-
);
|
|
505
|
-
}
|
|
506
|
-
function validateWorkerConfig(workers, lifecycle) {
|
|
507
|
-
if (workers === void 0) return;
|
|
508
|
-
requirePositiveSafeInteger("workers.count", workers.count);
|
|
509
|
-
requirePositiveSafeInteger("workers.promotionThreshold", workers.promotionThreshold);
|
|
510
|
-
requirePositiveSafeInteger("workers.idleTimeoutMs", workers.idleTimeoutMs);
|
|
511
|
-
if (workers.mainCopyQueries !== void 0 && workers.mainCopyQueries !== "lone" && workers.mainCopyQueries !== "none") {
|
|
512
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "workers.mainCopyQueries must be 'lone' or 'none'", {
|
|
513
|
-
mainCopyQueries: workers.mainCopyQueries
|
|
514
|
-
});
|
|
515
|
-
}
|
|
516
|
-
validateBootstrapModule(workers.bootstrapModule);
|
|
517
|
-
const closeAfter = lifecycle?.idleTimeoutMs;
|
|
518
|
-
if (workers.idleTimeoutMs !== void 0 && closeAfter !== void 0 && workers.idleTimeoutMs > closeAfter) {
|
|
519
|
-
throw new NarsilError(
|
|
520
|
-
ErrorCodes.CONFIG_INVALID,
|
|
521
|
-
"workers.idleTimeoutMs must be at most lifecycle.idleTimeoutMs, so that an index drops its copies before it closes",
|
|
522
|
-
{ workersIdleTimeoutMs: workers.idleTimeoutMs, lifecycleIdleTimeoutMs: closeAfter }
|
|
523
|
-
);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
function validateBM25Params(bm25) {
|
|
527
|
-
if (bm25 === void 0) return;
|
|
528
|
-
const { k1, b } = bm25;
|
|
529
|
-
if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
|
|
530
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
|
|
531
|
-
}
|
|
532
|
-
if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
|
|
533
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
|
-
function clampLimit(limit) {
|
|
537
|
-
return clampRowCount(limit, DEFAULT_LIMIT);
|
|
538
|
-
}
|
|
539
|
-
function clampOffset(offset) {
|
|
540
|
-
return clampRowCount(offset, DEFAULT_OFFSET);
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
// src/engine/list-documents/index.ts
|
|
544
|
-
function clampListLimit(limit) {
|
|
545
|
-
const clamped = Math.max(1, clampLimit(limit));
|
|
546
|
-
requireWithinResultWindow(clamped, 0);
|
|
547
|
-
return clamped;
|
|
548
|
-
}
|
|
549
|
-
function firstIdAfter(sorted, anchor) {
|
|
550
|
-
let low = 0;
|
|
551
|
-
let high = sorted.length;
|
|
552
|
-
while (low < high) {
|
|
553
|
-
const mid = low + high >>> 1;
|
|
554
|
-
if (compareCodePoints(sorted[mid], anchor) <= 0) low = mid + 1;
|
|
555
|
-
else high = mid;
|
|
556
|
-
}
|
|
557
|
-
return low;
|
|
558
|
-
}
|
|
559
|
-
function collectFilterMatches(partitions, filters, schema, patternWorkCap) {
|
|
560
|
-
const meter = createPatternWorkMeter(patternWorkCap);
|
|
561
|
-
const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema, meter));
|
|
562
|
-
let total = 0;
|
|
563
|
-
for (const matches of perPartition) total += matches.count;
|
|
564
|
-
return { matchesFor: (partitionIndex) => perPartition[partitionIndex] ?? null, total };
|
|
565
|
-
}
|
|
566
|
-
function collectCandidates(partition, anchor, matches, wanted, out) {
|
|
567
|
-
const sorted = partition.sortedDocIds();
|
|
568
|
-
let index = anchor === null ? 0 : firstIdAfter(sorted, anchor);
|
|
569
|
-
let taken = 0;
|
|
570
|
-
while (index < sorted.length && taken < wanted) {
|
|
571
|
-
const docId = sorted[index];
|
|
572
|
-
index++;
|
|
573
|
-
if (matches !== null && !matches.hasExternal(docId)) continue;
|
|
574
|
-
out.push(docId);
|
|
575
|
-
taken++;
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
function pageInIdOrder(partitions, cursor, filtered, limit, binding) {
|
|
579
|
-
const anchor = cursor === null ? null : cursor.anchor;
|
|
580
|
-
const candidates = [];
|
|
581
|
-
for (let index = 0; index < partitions.length; index++) {
|
|
582
|
-
collectCandidates(
|
|
583
|
-
partitions[index],
|
|
584
|
-
anchor,
|
|
585
|
-
filtered === null ? null : filtered.matchesFor(index),
|
|
586
|
-
limit + 1,
|
|
587
|
-
candidates
|
|
588
|
-
);
|
|
589
|
-
}
|
|
590
|
-
candidates.sort(compareCodePoints);
|
|
591
|
-
const hasMore = candidates.length > limit;
|
|
592
|
-
const ids = hasMore ? candidates.slice(0, limit) : candidates;
|
|
593
|
-
const nextCursor = hasMore && ids.length > 0 ? { anchor: ids[ids.length - 1], score: null, sortKey: null, sortSignature: null, binding } : null;
|
|
594
|
-
return { ids, nextCursor };
|
|
595
|
-
}
|
|
596
|
-
function pageInSortOrder(partitions, cursor, filtered, limit, sort, signature, schema, binding) {
|
|
597
|
-
const normalized = normalizeSort(sort);
|
|
598
|
-
const fields = normalized.map((entry) => entry.field);
|
|
599
|
-
const directions = normalized.map((entry) => entry.direction);
|
|
600
|
-
const flatSchema = flattenSchema(schema);
|
|
601
|
-
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
602
|
-
const request = {
|
|
603
|
-
fields,
|
|
604
|
-
directions,
|
|
605
|
-
fieldTypes,
|
|
606
|
-
modes: sortModesOf(normalized),
|
|
607
|
-
limit: limit + 1,
|
|
608
|
-
anchorKey: cursor === null ? null : cursor.sortKey,
|
|
609
|
-
anchorId: cursor === null ? null : cursor.anchor
|
|
610
|
-
};
|
|
611
|
-
const merged = [];
|
|
612
|
-
for (let index = 0; index < partitions.length; index++) {
|
|
613
|
-
const matches = filtered === null ? null : filtered.matchesFor(index);
|
|
614
|
-
for (const entry of partitions[index].sortedPage({ ...request, matches })) merged.push(entry);
|
|
615
|
-
}
|
|
616
|
-
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
617
|
-
const hasMore = merged.length > limit;
|
|
618
|
-
const page = hasMore ? merged.slice(0, limit) : merged;
|
|
619
|
-
const last = page[page.length - 1];
|
|
620
|
-
const nextCursor = hasMore && last !== void 0 ? { anchor: last.id, score: null, sortKey: last.key, sortSignature: signature, binding } : null;
|
|
621
|
-
return { ids: page.map((entry) => entry.id), nextCursor };
|
|
622
|
-
}
|
|
623
|
-
function countAcrossPartitions(manager, partitionIds, partitions) {
|
|
624
|
-
if (partitionIds === void 0) {
|
|
625
|
-
return manager.countDocuments();
|
|
626
|
-
}
|
|
627
|
-
let total = 0;
|
|
628
|
-
for (const partition of partitions) {
|
|
629
|
-
total += partition.count();
|
|
630
|
-
}
|
|
631
|
-
return total;
|
|
632
|
-
}
|
|
633
|
-
function executeListDocuments(params, context) {
|
|
634
|
-
const { manager, schema } = context;
|
|
635
|
-
const startTime = now();
|
|
636
|
-
const limit = clampListLimit(params.limit);
|
|
637
|
-
const signature = sortSignatureOf(params.sort);
|
|
638
|
-
requireSortableFields(params.sort, schema, context.strict === true);
|
|
639
|
-
const filters = validQueryFilter(params.filters, schema);
|
|
640
|
-
const projection = resolveProjection(params.document);
|
|
641
|
-
const binding = listBindingOf(params.filters);
|
|
642
|
-
let cursor = null;
|
|
643
|
-
if (params.cursor !== void 0) {
|
|
644
|
-
cursor = decodePageCursor(params.cursor);
|
|
645
|
-
requireMatchingCursor(cursor, params.cursor, signature, false, binding);
|
|
646
|
-
}
|
|
647
|
-
const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
|
|
648
|
-
const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
|
|
649
|
-
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
650
|
-
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
651
|
-
const documents = [];
|
|
652
|
-
for (const docId of page.ids) {
|
|
653
|
-
if (projection.kind === "none") {
|
|
654
|
-
documents.push({ id: docId, document: {} });
|
|
655
|
-
continue;
|
|
656
|
-
}
|
|
657
|
-
const stored = manager.get(docId, projection);
|
|
658
|
-
if (stored === void 0) continue;
|
|
659
|
-
documents.push({ id: docId, document: stored });
|
|
660
|
-
}
|
|
661
|
-
if (page.nextCursor === null) {
|
|
662
|
-
for (const partition of partitions) partition.releaseSortedDocIds();
|
|
663
|
-
}
|
|
664
|
-
return {
|
|
665
|
-
documents,
|
|
666
|
-
cursor: page.nextCursor === null ? null : encodePageCursor(page.nextCursor),
|
|
667
|
-
total,
|
|
668
|
-
elapsed: now() - startTime
|
|
669
|
-
};
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
// src/search/grouping.ts
|
|
673
|
-
function hitsKeptPerGroup(maxPerGroup) {
|
|
674
|
-
if (maxPerGroup === void 0 || !Number.isFinite(maxPerGroup) || maxPerGroup < 1) return DEFAULT_MAX_PER_GROUP;
|
|
675
|
-
return Math.floor(maxPerGroup);
|
|
676
|
-
}
|
|
677
|
-
function applyGrouping(hits, group, documents) {
|
|
678
|
-
if (!group.fields || group.fields.length === 0) {
|
|
679
|
-
return [{ values: {}, hits }];
|
|
680
|
-
}
|
|
681
|
-
const groupMap = /* @__PURE__ */ new Map();
|
|
682
|
-
for (const hit of hits) {
|
|
683
|
-
const stored = documents.fieldValues(hit.id, group.fields);
|
|
684
|
-
const fieldValues = [];
|
|
685
|
-
const valuesRecord = {};
|
|
686
|
-
for (let at = 0; at < group.fields.length; at++) {
|
|
687
|
-
const value = stored?.[at];
|
|
688
|
-
fieldValues.push(value);
|
|
689
|
-
valuesRecord[group.fields[at]] = value;
|
|
690
|
-
}
|
|
691
|
-
const compositeKey = JSON.stringify(fieldValues);
|
|
692
|
-
const existing = groupMap.get(compositeKey);
|
|
693
|
-
if (existing) {
|
|
694
|
-
existing.hits.push(hit);
|
|
695
|
-
} else {
|
|
696
|
-
groupMap.set(compositeKey, { values: valuesRecord, hits: [hit] });
|
|
697
|
-
}
|
|
698
|
-
}
|
|
699
|
-
const groups = [];
|
|
700
|
-
const keptPerGroup = hitsKeptPerGroup(group.maxPerGroup);
|
|
701
|
-
for (const entry of groupMap.values()) {
|
|
702
|
-
const everyHit = entry.hits;
|
|
703
|
-
const result = { values: entry.values, hits: everyHit.slice(0, keptPerGroup) };
|
|
704
|
-
if (group.reduce) {
|
|
705
|
-
const folded = foldGroupReducer(
|
|
706
|
-
group.reduce,
|
|
707
|
-
everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
|
|
708
|
-
);
|
|
709
|
-
if (folded.reducerError !== void 0) {
|
|
710
|
-
result.reducerError = folded.reducerError;
|
|
711
|
-
} else {
|
|
712
|
-
result.reduced = folded.reduced;
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
groups.push(result);
|
|
716
|
-
}
|
|
717
|
-
groups.sort((a, b) => {
|
|
718
|
-
const aScore = a.hits.length > 0 ? a.hits[0].score ?? 0 : 0;
|
|
719
|
-
const bScore = b.hits.length > 0 ? b.hits[0].score ?? 0 : 0;
|
|
720
|
-
if (aScore !== bScore) return bScore - aScore;
|
|
721
|
-
const aId = a.hits.length > 0 ? a.hits[0].id : "";
|
|
722
|
-
const bId = b.hits.length > 0 ? b.hits[0].id : "";
|
|
723
|
-
return compareCodePoints(aId, bId);
|
|
724
|
-
});
|
|
725
|
-
if (group.limit !== void 0 && Number.isFinite(group.limit) && group.limit >= 0) {
|
|
726
|
-
const cap = Math.floor(group.limit);
|
|
727
|
-
if (groups.length > cap) {
|
|
728
|
-
groups.length = cap;
|
|
729
|
-
}
|
|
730
|
-
}
|
|
731
|
-
return groups;
|
|
732
|
-
}
|
|
733
|
-
function foldGroupReducer(reduce, rows) {
|
|
734
|
-
try {
|
|
735
|
-
let accumulator = reduce.initialValue();
|
|
736
|
-
for (const row of rows) {
|
|
737
|
-
if (row.document !== void 0) {
|
|
738
|
-
accumulator = reduce.reducer(accumulator, row.document, row.score);
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
return { reduced: accumulator };
|
|
742
|
-
} catch (err) {
|
|
743
|
-
return { reducerError: err instanceof Error ? err.message : String(err) };
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
// src/search/pinning.ts
|
|
748
|
-
function dedupePinnedEntries(pinned) {
|
|
749
|
-
const deduped = [];
|
|
750
|
-
const seen = /* @__PURE__ */ new Set();
|
|
751
|
-
for (const entry of pinned) {
|
|
752
|
-
if (seen.has(entry.docId)) continue;
|
|
753
|
-
seen.add(entry.docId);
|
|
754
|
-
deduped.push(entry);
|
|
755
|
-
}
|
|
756
|
-
return deduped;
|
|
757
|
-
}
|
|
758
|
-
function placePinned(hits, pinned, resolveDoc) {
|
|
759
|
-
const result = hits.slice();
|
|
760
|
-
const deduped = dedupePinnedEntries(pinned);
|
|
761
|
-
const pinnedDocIds = new Set(deduped.map((entry) => entry.docId));
|
|
762
|
-
const listedPins = /* @__PURE__ */ new Set();
|
|
763
|
-
for (let i = result.length - 1; i >= 0; i--) {
|
|
764
|
-
if (pinnedDocIds.has(result[i].id)) {
|
|
765
|
-
listedPins.add(result[i].id);
|
|
766
|
-
result.splice(i, 1);
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
const sorted = deduped.sort((a, b) => a.position - b.position);
|
|
770
|
-
let placedFromOutside = 0;
|
|
771
|
-
for (const entry of sorted) {
|
|
772
|
-
const doc = resolveDoc(entry.docId);
|
|
773
|
-
if (!doc) continue;
|
|
774
|
-
if (!listedPins.has(entry.docId)) placedFromOutside++;
|
|
775
|
-
const pos = Math.max(0, Math.min(entry.position, result.length));
|
|
776
|
-
result.splice(pos, 0, doc);
|
|
777
|
-
}
|
|
778
|
-
return { hits: result, placedFromOutside };
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
// src/highlighting/highlighter.ts
|
|
782
|
-
function buildTokenCharOffsets(text, originalTokens) {
|
|
783
|
-
const offsets = [];
|
|
784
|
-
let searchFrom = 0;
|
|
785
|
-
for (const token of originalTokens) {
|
|
786
|
-
const idx = text.indexOf(token, searchFrom);
|
|
787
|
-
if (idx === -1) continue;
|
|
788
|
-
offsets.push({ start: idx, end: idx + token.length });
|
|
789
|
-
searchFrom = idx + token.length;
|
|
790
|
-
}
|
|
791
|
-
return offsets;
|
|
792
|
-
}
|
|
793
|
-
function mergeOverlapping(ranges) {
|
|
794
|
-
if (ranges.length === 0) return [];
|
|
795
|
-
const sorted = [...ranges].sort((a, b) => a.start - b.start);
|
|
796
|
-
const merged = [sorted[0]];
|
|
797
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
798
|
-
const prev = merged[merged.length - 1];
|
|
799
|
-
const curr = sorted[i];
|
|
800
|
-
if (curr.start <= prev.end) {
|
|
801
|
-
prev.end = Math.max(prev.end, curr.end);
|
|
802
|
-
} else {
|
|
803
|
-
merged.push(curr);
|
|
804
|
-
}
|
|
805
|
-
}
|
|
806
|
-
return merged;
|
|
807
|
-
}
|
|
808
|
-
function findDensestWindow(positions, text, windowSize) {
|
|
809
|
-
if (positions.length === 0) {
|
|
810
|
-
return { start: 0, end: Math.min(windowSize, text.length) };
|
|
811
|
-
}
|
|
812
|
-
let bestStart = 0;
|
|
813
|
-
let bestCount = 0;
|
|
814
|
-
for (const pos of positions) {
|
|
815
|
-
const windowStart = Math.max(0, pos.start - Math.floor(windowSize / 2));
|
|
816
|
-
const windowEnd = Math.min(text.length, windowStart + windowSize);
|
|
817
|
-
const adjustedStart = Math.max(0, windowEnd - windowSize);
|
|
818
|
-
let count = 0;
|
|
819
|
-
for (const p of positions) {
|
|
820
|
-
if (p.start >= adjustedStart && p.end <= windowEnd) {
|
|
821
|
-
count++;
|
|
822
|
-
}
|
|
823
|
-
}
|
|
824
|
-
if (count > bestCount) {
|
|
825
|
-
bestCount = count;
|
|
826
|
-
bestStart = adjustedStart;
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
return { start: bestStart, end: Math.min(text.length, bestStart + windowSize) };
|
|
830
|
-
}
|
|
831
|
-
function applySnippetTruncation(taggedText, originalText, positions, maxLen, preTag, postTag) {
|
|
832
|
-
if (maxLen <= 0 || originalText.length <= maxLen) {
|
|
833
|
-
return { snippet: taggedText, visiblePositions: positions };
|
|
834
|
-
}
|
|
835
|
-
const window = findDensestWindow(positions, originalText, maxLen);
|
|
836
|
-
const snippetRaw = originalText.substring(window.start, window.end);
|
|
837
|
-
const visiblePositions = [];
|
|
838
|
-
const withinSnippet = [];
|
|
839
|
-
for (const pos of positions) {
|
|
840
|
-
if (pos.start < window.start || pos.end > window.end) continue;
|
|
841
|
-
visiblePositions.push(pos);
|
|
842
|
-
withinSnippet.push({ start: pos.start - window.start, end: pos.end - window.start });
|
|
843
|
-
}
|
|
844
|
-
let snippet = snippetRaw;
|
|
845
|
-
const sortedDesc = [...withinSnippet].sort((a, b) => b.start - a.start);
|
|
846
|
-
for (const pos of sortedDesc) {
|
|
847
|
-
snippet = snippet.substring(0, pos.start) + preTag + snippet.substring(pos.start, pos.end) + postTag + snippet.substring(pos.end);
|
|
848
|
-
}
|
|
849
|
-
const needsLeadingEllipsis = window.start > 0;
|
|
850
|
-
const needsTrailingEllipsis = window.end < originalText.length;
|
|
851
|
-
if (needsLeadingEllipsis) snippet = `...${snippet}`;
|
|
852
|
-
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
853
|
-
return { snippet, visiblePositions };
|
|
854
|
-
}
|
|
855
|
-
function highlightField(text, queryTokens, language, options) {
|
|
856
|
-
const preTag = options?.preTag ?? "<mark>";
|
|
857
|
-
const postTag = options?.postTag ?? "</mark>";
|
|
858
|
-
const maxSnippetLength = options?.maxSnippetLength ?? 200;
|
|
859
|
-
if (text.length === 0) {
|
|
860
|
-
return { snippet: text, positions: [] };
|
|
861
|
-
}
|
|
862
|
-
const fieldResult = tokenize(text, language, { stem: false, removeStopWords: false });
|
|
863
|
-
if (fieldResult.tokens.length === 0) {
|
|
864
|
-
return { snippet: text, positions: [] };
|
|
865
|
-
}
|
|
866
|
-
const lowercaseText = text.normalize("NFC").toLowerCase();
|
|
867
|
-
const charOffsets = buildTokenCharOffsets(lowercaseText, fieldResult.originalTokens);
|
|
868
|
-
const stemmedQueryTokens = queryTokens.map((qt) => {
|
|
869
|
-
if (language.stemmer) {
|
|
870
|
-
return language.stemmer(qt.token.toLowerCase());
|
|
871
|
-
}
|
|
872
|
-
return qt.token.toLowerCase();
|
|
873
|
-
});
|
|
874
|
-
const matchedRanges = [];
|
|
875
|
-
const prefixToken = options?.prefixToken;
|
|
876
|
-
const tolerance = options?.tolerance ?? 0;
|
|
877
|
-
const prefixLength = options?.prefixLength ?? 0;
|
|
878
|
-
const completedToken = options?.completedToken;
|
|
879
|
-
const queryTolerances = queryTokens.map((qt) => qt.token === completedToken ? 0 : tolerance);
|
|
880
|
-
const synonyms = options?.synonyms;
|
|
881
|
-
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
882
|
-
const matchesQuery = (fieldToken) => {
|
|
883
|
-
const known = matchByFieldToken.get(fieldToken);
|
|
884
|
-
if (known !== void 0) return known;
|
|
885
|
-
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
886
|
-
const matches = synonyms?.includes(stemmedField) === true || stemmedQueryTokens.some(
|
|
887
|
-
(stemmedQuery, index) => fuzzyTermMatches(stemmedQuery, stemmedField, queryTolerances[index], prefixLength)
|
|
888
|
-
);
|
|
889
|
-
matchByFieldToken.set(fieldToken, matches);
|
|
890
|
-
return matches;
|
|
891
|
-
};
|
|
892
|
-
const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
|
|
893
|
-
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
894
|
-
if (i >= charOffsets.length) break;
|
|
895
|
-
const fieldToken = fieldResult.tokens[i].token;
|
|
896
|
-
const prefixMatch = prefixToken !== void 0 && fieldToken.startsWith(prefixToken);
|
|
897
|
-
const insideMatch = matchesInside?.(fieldResult.tokens[i].position, fieldToken) === true;
|
|
898
|
-
if (prefixMatch || insideMatch || matchesQuery(fieldToken)) {
|
|
899
|
-
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
const mergedPositions = mergeOverlapping(matchedRanges);
|
|
903
|
-
if (mergedPositions.length === 0) {
|
|
904
|
-
if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
|
|
905
|
-
const truncated = text.substring(0, maxSnippetLength);
|
|
906
|
-
return { snippet: `${truncated}...`, positions: [] };
|
|
907
|
-
}
|
|
908
|
-
return { snippet: text, positions: [] };
|
|
909
|
-
}
|
|
910
|
-
if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
|
|
911
|
-
const { snippet, visiblePositions } = applySnippetTruncation(
|
|
912
|
-
text,
|
|
913
|
-
text,
|
|
914
|
-
mergedPositions,
|
|
915
|
-
maxSnippetLength,
|
|
916
|
-
preTag,
|
|
917
|
-
postTag
|
|
918
|
-
);
|
|
919
|
-
return { snippet, positions: visiblePositions };
|
|
920
|
-
}
|
|
921
|
-
let result = text;
|
|
922
|
-
const sortedDesc = [...mergedPositions].sort((a, b) => b.start - a.start);
|
|
923
|
-
for (const pos of sortedDesc) {
|
|
924
|
-
result = result.substring(0, pos.start) + preTag + result.substring(pos.start, pos.end) + postTag + result.substring(pos.end);
|
|
925
|
-
}
|
|
926
|
-
return { snippet: result, positions: mergedPositions };
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
// src/engine/query/highlight.ts
|
|
930
|
-
function writtenWordsOf(params, language, analysis) {
|
|
931
|
-
return tokenize(params.term ?? "", language, {
|
|
932
|
-
stem: false,
|
|
933
|
-
removeStopWords: true,
|
|
934
|
-
customTokenizer: analysis.customTokenizer,
|
|
935
|
-
stopWordOverride: analysis.stopWords
|
|
936
|
-
}).tokens;
|
|
937
|
-
}
|
|
938
|
-
function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
939
|
-
if (params.partialWords !== true || params.exact === true) return null;
|
|
940
|
-
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
941
|
-
if (fields.length === 0) return null;
|
|
942
|
-
const words = partialWordsToHighlight(queryTokens, written);
|
|
943
|
-
if (words.length === 0) return null;
|
|
944
|
-
return {
|
|
945
|
-
fields: new Set(fields),
|
|
946
|
-
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
947
|
-
};
|
|
948
|
-
}
|
|
949
|
-
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
950
|
-
if (!params.highlight) return;
|
|
951
|
-
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
952
|
-
stem: true,
|
|
953
|
-
removeStopWords: true,
|
|
954
|
-
customTokenizer: analysis.customTokenizer,
|
|
955
|
-
stopWordOverride: analysis.stopWords
|
|
956
|
-
});
|
|
957
|
-
const queryTokens = queryTokenResult.tokens;
|
|
958
|
-
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
959
|
-
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
960
|
-
const written = usesPrefix || usesPartialWords ? writtenWordsOf(params, language, analysis) : [];
|
|
961
|
-
let prefixToken;
|
|
962
|
-
let completedToken;
|
|
963
|
-
if (usesPrefix) {
|
|
964
|
-
const lastToken = queryTokens[queryTokens.length - 1];
|
|
965
|
-
for (const t of written) {
|
|
966
|
-
if (t.position === lastToken.position) {
|
|
967
|
-
prefixToken = t.token;
|
|
968
|
-
completedToken = lastToken.token;
|
|
969
|
-
break;
|
|
970
|
-
}
|
|
971
|
-
}
|
|
972
|
-
}
|
|
973
|
-
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
974
|
-
const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
975
|
-
for (const hit of hits) {
|
|
976
|
-
const highlights = {};
|
|
977
|
-
const projected = hit.document;
|
|
978
|
-
let stored;
|
|
979
|
-
for (const field of params.highlight.fields) {
|
|
980
|
-
let fieldValue = projected[field];
|
|
981
|
-
if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
|
|
982
|
-
stored ??= readStoredDocument(hit.id);
|
|
983
|
-
fieldValue = stored?.[field];
|
|
984
|
-
}
|
|
985
|
-
if (typeof fieldValue === "string") {
|
|
986
|
-
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
987
|
-
preTag: params.highlight.preTag,
|
|
988
|
-
postTag: params.highlight.postTag,
|
|
989
|
-
maxSnippetLength: params.highlight.maxSnippetLength,
|
|
990
|
-
prefixToken,
|
|
991
|
-
completedToken,
|
|
992
|
-
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
993
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
994
|
-
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0,
|
|
995
|
-
synonyms
|
|
996
|
-
});
|
|
997
|
-
}
|
|
998
|
-
}
|
|
999
|
-
hit.highlights = highlights;
|
|
1000
|
-
}
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
// src/search/fusion.ts
|
|
1004
|
-
function rankConstantOf(k) {
|
|
1005
|
-
if (k === void 0 || k === null) return DEFAULT_RANK_CONSTANT;
|
|
1006
|
-
if (!Number.isInteger(k) || k < MIN_RANK_CONSTANT) {
|
|
1007
|
-
throw new NarsilError(
|
|
1008
|
-
ErrorCodes.CONFIG_INVALID,
|
|
1009
|
-
`hybrid.k must be a whole number of at least ${MIN_RANK_CONSTANT}`,
|
|
1010
|
-
{ k }
|
|
1011
|
-
);
|
|
1012
|
-
}
|
|
1013
|
-
return k;
|
|
1014
|
-
}
|
|
1015
|
-
function linearAlphaOf(alpha) {
|
|
1016
|
-
if (alpha === void 0 || alpha === null) return DEFAULT_LINEAR_ALPHA;
|
|
1017
|
-
if (!Number.isFinite(alpha) || alpha < MIN_LINEAR_ALPHA || alpha > MAX_LINEAR_ALPHA) {
|
|
1018
|
-
throw new NarsilError(
|
|
1019
|
-
ErrorCodes.CONFIG_INVALID,
|
|
1020
|
-
`hybrid.alpha must be a number from ${MIN_LINEAR_ALPHA} to ${MAX_LINEAR_ALPHA}`,
|
|
1021
|
-
{ alpha }
|
|
1022
|
-
);
|
|
1023
|
-
}
|
|
1024
|
-
return alpha;
|
|
1025
|
-
}
|
|
1026
|
-
function resolveHybridFusion(hybrid) {
|
|
1027
|
-
const strategy = hybrid?.strategy ?? "rrf";
|
|
1028
|
-
if (strategy !== "rrf" && strategy !== "linear") {
|
|
1029
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `hybrid.strategy must be "rrf" or "linear"`, { strategy });
|
|
1030
|
-
}
|
|
1031
|
-
return { strategy, k: rankConstantOf(hybrid?.k), alpha: linearAlphaOf(hybrid?.alpha) };
|
|
1032
|
-
}
|
|
1033
|
-
function reciprocalRankFusion(lists, options) {
|
|
1034
|
-
const k = options.k;
|
|
1035
|
-
const positions = /* @__PURE__ */ new Map();
|
|
1036
|
-
const result = [];
|
|
1037
|
-
for (const list of lists) {
|
|
1038
|
-
for (let rank = 0; rank < list.length; rank++) {
|
|
1039
|
-
const doc = list[rank];
|
|
1040
|
-
const rrfContribution = 1 / (k + rank + 1);
|
|
1041
|
-
const position = positions.get(doc.docId);
|
|
1042
|
-
if (position === void 0) {
|
|
1043
|
-
positions.set(doc.docId, result.length);
|
|
1044
|
-
result.push({
|
|
1045
|
-
docId: doc.docId,
|
|
1046
|
-
score: rrfContribution,
|
|
1047
|
-
termFrequencies: doc.termFrequencies,
|
|
1048
|
-
fieldLengths: doc.fieldLengths,
|
|
1049
|
-
idf: doc.idf
|
|
1050
|
-
});
|
|
1051
|
-
} else {
|
|
1052
|
-
result[position].score += rrfContribution;
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
result.sort((a, b) => b.score - a.score || compareCodePoints(a.docId, b.docId));
|
|
1057
|
-
return result;
|
|
1058
|
-
}
|
|
1059
|
-
function linearCombination(textResults, vectorResults, options) {
|
|
1060
|
-
const { alpha } = options;
|
|
1061
|
-
const normalizedText = minMaxNormalize(textResults);
|
|
1062
|
-
const normalizedVector = minMaxNormalize(vectorResults);
|
|
1063
|
-
const textScoreMap = /* @__PURE__ */ new Map();
|
|
1064
|
-
for (const doc of normalizedText) {
|
|
1065
|
-
textScoreMap.set(doc.docId, doc.score);
|
|
1066
|
-
}
|
|
1067
|
-
const textDocMap = /* @__PURE__ */ new Map();
|
|
1068
|
-
for (const doc of textResults) {
|
|
1069
|
-
textDocMap.set(doc.docId, doc);
|
|
1070
|
-
}
|
|
1071
|
-
const vectorScoreMap = /* @__PURE__ */ new Map();
|
|
1072
|
-
for (const doc of normalizedVector) {
|
|
1073
|
-
vectorScoreMap.set(doc.docId, doc.score);
|
|
1074
|
-
}
|
|
1075
|
-
const allDocIds = /* @__PURE__ */ new Set();
|
|
1076
|
-
for (const doc of textResults) allDocIds.add(doc.docId);
|
|
1077
|
-
for (const doc of vectorResults) allDocIds.add(doc.docId);
|
|
1078
|
-
const result = [];
|
|
1079
|
-
for (const docId of allDocIds) {
|
|
1080
|
-
const tScore = textScoreMap.get(docId) ?? 0;
|
|
1081
|
-
const vScore = vectorScoreMap.get(docId) ?? 0;
|
|
1082
|
-
const combined = alpha * vScore + (1 - alpha) * tScore;
|
|
1083
|
-
const original = textDocMap.get(docId);
|
|
1084
|
-
result.push({
|
|
1085
|
-
docId,
|
|
1086
|
-
score: combined,
|
|
1087
|
-
termFrequencies: original?.termFrequencies ?? {},
|
|
1088
|
-
fieldLengths: original?.fieldLengths ?? {},
|
|
1089
|
-
idf: original?.idf ?? {}
|
|
1090
|
-
});
|
|
1091
|
-
}
|
|
1092
|
-
result.sort((a, b) => b.score - a.score || compareCodePoints(a.docId, b.docId));
|
|
1093
|
-
return result;
|
|
1094
|
-
}
|
|
1095
|
-
function minMaxNormalize(docs) {
|
|
1096
|
-
if (docs.length === 0) return [];
|
|
1097
|
-
let min = docs[0].score;
|
|
1098
|
-
let max = docs[0].score;
|
|
1099
|
-
for (let i = 1; i < docs.length; i++) {
|
|
1100
|
-
if (docs[i].score < min) min = docs[i].score;
|
|
1101
|
-
if (docs[i].score > max) max = docs[i].score;
|
|
1102
|
-
}
|
|
1103
|
-
const range = max - min;
|
|
1104
|
-
if (range === 0) {
|
|
1105
|
-
return docs.map((d) => ({ docId: d.docId, score: 1 }));
|
|
1106
|
-
}
|
|
1107
|
-
return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
|
|
1108
|
-
}
|
|
1109
|
-
|
|
1110
|
-
// src/engine/query/filter-subsets.ts
|
|
1111
|
-
function collectFilterSubsets(manager, filters, schema, partitionIds, patternWork) {
|
|
1112
|
-
const subsets = [];
|
|
1113
|
-
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
1114
|
-
for (const state of partition.readStates()) {
|
|
1115
|
-
subsets.push({
|
|
1116
|
-
owner: state,
|
|
1117
|
-
bits: applyPartitionFiltersBitset(state, filters, schema, patternWork),
|
|
1118
|
-
numberingGeneration: state.docStore.numberingGeneration()
|
|
1119
|
-
});
|
|
1120
|
-
}
|
|
1121
|
-
}
|
|
1122
|
-
return subsets;
|
|
1123
|
-
}
|
|
1124
|
-
function filterSubsetsMatchNothing(subsets) {
|
|
1125
|
-
for (const subset of subsets) {
|
|
1126
|
-
for (let wordIndex = 0; wordIndex < subset.bits.length; wordIndex++) {
|
|
1127
|
-
if (subset.bits[wordIndex] !== 0) return false;
|
|
1128
|
-
}
|
|
1129
|
-
}
|
|
1130
|
-
return true;
|
|
1131
|
-
}
|
|
1132
|
-
function filterSubsetsRenumbered(subsets) {
|
|
1133
|
-
return subsets.some((subset) => subset.owner.docStore.numberingGeneration() !== subset.numberingGeneration);
|
|
1134
|
-
}
|
|
1135
|
-
async function searchCurrentSubsets(fieldName, first, collectAgain, search) {
|
|
1136
|
-
let subsets = first;
|
|
1137
|
-
for (let attempt = 1; ; attempt++) {
|
|
1138
|
-
const answer = await search(subsets);
|
|
1139
|
-
if (!filterSubsetsRenumbered(subsets)) return answer;
|
|
1140
|
-
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
1141
|
-
throw new NarsilError(
|
|
1142
|
-
ErrorCodes.WORKER_BUSY,
|
|
1143
|
-
`The index renumbered its documents during each of ${attempt} filtered searches of vector field "${fieldName}"`,
|
|
1144
|
-
{ fieldName, attempts: attempt }
|
|
1145
|
-
);
|
|
1146
|
-
}
|
|
1147
|
-
subsets = collectAgain();
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
// src/engine/query/vector.ts
|
|
1152
|
-
function oversampleOf(vectorConfig) {
|
|
1153
|
-
const oversample = vectorConfig.oversample;
|
|
1154
|
-
if (oversample === void 0) return void 0;
|
|
1155
|
-
if (typeof oversample !== "number" || !Number.isFinite(oversample) || oversample < MIN_OVERSAMPLE) {
|
|
1156
|
-
throw new NarsilError(
|
|
1157
|
-
ErrorCodes.CONFIG_INVALID,
|
|
1158
|
-
`vector.oversample must be a finite number of at least ${MIN_OVERSAMPLE}`,
|
|
1159
|
-
{ oversample }
|
|
1160
|
-
);
|
|
1161
|
-
}
|
|
1162
|
-
return oversample;
|
|
1163
|
-
}
|
|
1164
|
-
function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
1165
|
-
const reach = cursorDepth + offset + limit;
|
|
1166
|
-
if (reach > RESULT_WINDOW) {
|
|
1167
|
-
throw new NarsilError(
|
|
1168
|
-
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
1169
|
-
`The engine pages a vector search through the first ${RESULT_WINDOW} results, and this page ends at ${reach}. Narrow the search with a filter or a similarity floor`,
|
|
1170
|
-
{ limit, offset, cursorDepth, window: RESULT_WINDOW }
|
|
1171
|
-
);
|
|
1172
|
-
}
|
|
1173
|
-
return Math.min(reach + 1, RESULT_WINDOW);
|
|
1174
|
-
}
|
|
1175
|
-
function filteredSearchOf(params, context) {
|
|
1176
|
-
const filters = params.filters;
|
|
1177
|
-
if (!filters) return void 0;
|
|
1178
|
-
const { manager, config, partitionIds, patternWork } = context;
|
|
1179
|
-
const collectAgain = () => collectFilterSubsets(manager, filters, config.schema, partitionIds, patternWork);
|
|
1180
|
-
return { first: collectAgain(), collectAgain };
|
|
1181
|
-
}
|
|
1182
|
-
function searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow) {
|
|
1183
|
-
const options = {
|
|
1184
|
-
metric: vectorConfig.metric ?? "cosine",
|
|
1185
|
-
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
1186
|
-
efSearch: vectorConfig.efSearch,
|
|
1187
|
-
oversample: oversampleOf(vectorConfig),
|
|
1188
|
-
...rescoreWindow !== void 0 ? { rescoreWindow } : {}
|
|
1189
|
-
};
|
|
1190
|
-
if (filtered !== void 0) {
|
|
1191
|
-
return searchCurrentSubsets(
|
|
1192
|
-
vecIndex.fieldName,
|
|
1193
|
-
filtered.first,
|
|
1194
|
-
filtered.collectAgain,
|
|
1195
|
-
(filterSubsets) => vecIndex.searchParallel(queryVector, k, { ...options, filterSubsets })
|
|
1196
|
-
);
|
|
1197
|
-
}
|
|
1198
|
-
const filterPartitions = partitionsForVectorSearch(context.manager, vecIndex, context.partitionIds);
|
|
1199
|
-
return vecIndex.searchParallel(
|
|
1200
|
-
queryVector,
|
|
1201
|
-
k,
|
|
1202
|
-
filterPartitions === void 0 ? options : { ...options, filterPartitions }
|
|
1203
|
-
);
|
|
1204
|
-
}
|
|
1205
|
-
async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1206
|
-
const vectorConfig = params.vector;
|
|
1207
|
-
if (!vectorConfig || !vectorConfig.value) {
|
|
1208
|
-
return { scored: [], totalMatched: 0 };
|
|
1209
|
-
}
|
|
1210
|
-
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1211
|
-
if (!vecIndex) {
|
|
1212
|
-
return { scored: [], totalMatched: 0 };
|
|
1213
|
-
}
|
|
1214
|
-
const filtered = filteredSearchOf(params, context);
|
|
1215
|
-
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1216
|
-
return { scored: [], totalMatched: 0 };
|
|
1217
|
-
}
|
|
1218
|
-
const queryVec = new Float32Array(vectorConfig.value);
|
|
1219
|
-
const k = vectorFetchDepth(limit, offset, cursorDepth);
|
|
1220
|
-
const outcome = await searchField(vecIndex, vectorConfig, queryVec, k, context, filtered, context.rescoreWindow);
|
|
1221
|
-
const scored = vectorResultsToScored(outcome.results);
|
|
1222
|
-
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
1223
|
-
}
|
|
1224
|
-
async function textLeg(textOnlyParams, context) {
|
|
1225
|
-
const { manager, language, config, workerSearch, indexName } = context;
|
|
1226
|
-
const scoring = scoringConfigFor(textOnlyParams, context);
|
|
1227
|
-
if (workerSearch) {
|
|
1228
|
-
const viaWorker = await workerSearch(
|
|
1229
|
-
indexName,
|
|
1230
|
-
textOnlyParams,
|
|
1231
|
-
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
1232
|
-
context.partitionIds,
|
|
1233
|
-
context.patternWork
|
|
1234
|
-
);
|
|
1235
|
-
if (viaWorker) return viaWorker;
|
|
1236
|
-
}
|
|
1237
|
-
return fanOutQuery(
|
|
1238
|
-
manager,
|
|
1239
|
-
textOnlyParams,
|
|
1240
|
-
language,
|
|
1241
|
-
config.schema,
|
|
1242
|
-
scoring,
|
|
1243
|
-
searchOptionsFor(manager, context.patternWork)
|
|
1244
|
-
);
|
|
1245
|
-
}
|
|
1246
|
-
async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
1247
|
-
const vecIndex = resolveVectorIndex(context, vectorConfig.field);
|
|
1248
|
-
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
1249
|
-
const outcome = await searchField(vecIndex, vectorConfig, queryVector, k, context, filtered, rescoreWindow);
|
|
1250
|
-
return {
|
|
1251
|
-
scored: vectorResultsToScored(outcome.results),
|
|
1252
|
-
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
1253
|
-
};
|
|
1254
|
-
}
|
|
1255
|
-
async function executeHybridSearch(params, context, limit, offset, cursorDepth = 0) {
|
|
1256
|
-
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
1257
|
-
const fusion = resolveHybridFusion(params.hybrid);
|
|
1258
|
-
const filtered = filteredSearchOf(params, context);
|
|
1259
|
-
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
1260
|
-
return { scored: [], totalMatched: 0 };
|
|
1261
|
-
}
|
|
1262
|
-
if (!vectorConfig || !vectorConfig.value) {
|
|
1263
|
-
const textOnly = await textLeg(textOnlyParams, context);
|
|
1264
|
-
return { scored: textOnly.scored, totalMatched: textOnly.totalMatched, facets: textOnly.facets };
|
|
1265
|
-
}
|
|
1266
|
-
const vectorPending = vectorLeg(
|
|
1267
|
-
vectorConfig,
|
|
1268
|
-
new Float32Array(vectorConfig.value),
|
|
1269
|
-
context,
|
|
1270
|
-
filtered,
|
|
1271
|
-
vectorFetchDepth(limit, offset, cursorDepth)
|
|
1272
|
-
);
|
|
1273
|
-
const [textFanOutResult, vectorOutcome] = await Promise.all([textLeg(textOnlyParams, context), vectorPending]);
|
|
1274
|
-
const vectorScored = vectorOutcome.scored;
|
|
1275
|
-
let fusedScored;
|
|
1276
|
-
if (fusion.strategy === "rrf") {
|
|
1277
|
-
fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: fusion.k });
|
|
1278
|
-
} else {
|
|
1279
|
-
fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha: fusion.alpha });
|
|
1280
|
-
}
|
|
1281
|
-
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
1282
|
-
const threshold = params.minScore;
|
|
1283
|
-
fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
|
|
1284
|
-
}
|
|
1285
|
-
const textHoldsEveryMatch = textFanOutResult.scored.length === textFanOutResult.totalMatched;
|
|
1286
|
-
return {
|
|
1287
|
-
scored: fusedScored,
|
|
1288
|
-
totalMatched: fusedScored.length,
|
|
1289
|
-
matchedExact: vectorOutcome.holdsEveryMatch && textHoldsEveryMatch,
|
|
1290
|
-
facets: textFanOutResult.facets
|
|
1291
|
-
};
|
|
1292
|
-
}
|
|
1293
|
-
|
|
1294
|
-
// src/engine/query/runtime-host.ts
|
|
1295
|
-
var runtimeHost = {
|
|
1296
|
-
now,
|
|
1297
|
-
compareCodePoints,
|
|
1298
|
-
clampRowCount,
|
|
1299
|
-
resolveProjection,
|
|
1300
|
-
projectionKeepsField,
|
|
1301
|
-
decodePageCursor,
|
|
1302
|
-
encodePageCursor,
|
|
1303
|
-
requireMatchingCursor,
|
|
1304
|
-
dedupePinnedEntries,
|
|
1305
|
-
placePinned,
|
|
1306
|
-
normalizeSort,
|
|
1307
|
-
flattenSchema,
|
|
1308
|
-
parsedTypeOf,
|
|
1309
|
-
isStringValuedBase,
|
|
1310
|
-
applyHighlights,
|
|
1311
|
-
textLeg,
|
|
1312
|
-
vectorLeg,
|
|
1313
|
-
filteredSearchOf,
|
|
1314
|
-
filterSubsetsMatchNothing,
|
|
1315
|
-
reciprocalRankFusion,
|
|
1316
|
-
linearCombination,
|
|
1317
|
-
resolveHybridFusion
|
|
1318
|
-
};
|
|
1319
|
-
|
|
1320
|
-
// src/engine/query/sorted.ts
|
|
1321
|
-
function sortsWithoutScores(params) {
|
|
1322
|
-
return params.includeScores !== true && params.includeScoreComponents !== true && params.minScore === void 0 && params.group === void 0 && params.pinned === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
|
|
1323
|
-
}
|
|
1324
|
-
function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
1325
|
-
const { manager, language, config } = context;
|
|
1326
|
-
const schema = config.schema;
|
|
1327
|
-
let anchorKey = null;
|
|
1328
|
-
let anchorId = null;
|
|
1329
|
-
if (params.searchAfter !== void 0) {
|
|
1330
|
-
const decoded = decodePageCursor(params.searchAfter);
|
|
1331
|
-
requireMatchingCursor(decoded, params.searchAfter, sortSignature, true, context.cursorBinding);
|
|
1332
|
-
anchorKey = decoded.sortKey;
|
|
1333
|
-
anchorId = decoded.anchor;
|
|
1334
|
-
}
|
|
1335
|
-
const normalized = normalizeSort(params.sort);
|
|
1336
|
-
const fields = normalized.map((entry) => entry.field);
|
|
1337
|
-
const directions = normalized.map((entry) => entry.direction);
|
|
1338
|
-
const flatSchema = flattenSchema(schema);
|
|
1339
|
-
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
1340
|
-
const modes = sortModesOf(normalized);
|
|
1341
|
-
const options = searchOptionsFor(manager, context.patternWork);
|
|
1342
|
-
const partitions = partitionsFor(manager, context.partitionIds);
|
|
1343
|
-
const partitionLimit = offset + limit + 1;
|
|
1344
|
-
const merged = [];
|
|
1345
|
-
const partitionFacets = [];
|
|
1346
|
-
const partitionFacetConfig = params.facets !== void 0 && partitions.length > 1 ? everyValueFacetConfig(params.facets) : params.facets;
|
|
1347
|
-
let count = 0;
|
|
1348
|
-
for (const partition of partitions) {
|
|
1349
|
-
const matches = fulltextMatches(partition, params, language, schema, options);
|
|
1350
|
-
if (partitionFacetConfig !== void 0) {
|
|
1351
|
-
partitionFacets.push(
|
|
1352
|
-
partition.computeFacets(
|
|
1353
|
-
matches === null ? /* @__PURE__ */ new Set() : { ordinalBitset: matches.ordinalBitset() },
|
|
1354
|
-
partitionFacetConfig,
|
|
1355
|
-
schema
|
|
1356
|
-
)
|
|
1357
|
-
);
|
|
1358
|
-
}
|
|
1359
|
-
if (matches === null || matches.count === 0) continue;
|
|
1360
|
-
count += matches.count;
|
|
1361
|
-
const page2 = partition.sortedPage({
|
|
1362
|
-
fields,
|
|
1363
|
-
directions,
|
|
1364
|
-
fieldTypes,
|
|
1365
|
-
modes,
|
|
1366
|
-
limit: partitionLimit,
|
|
1367
|
-
anchorKey,
|
|
1368
|
-
anchorId,
|
|
1369
|
-
matches
|
|
1370
|
-
});
|
|
1371
|
-
for (const entry of page2) merged.push(entry);
|
|
1372
|
-
}
|
|
1373
|
-
merged.sort((a, b) => compareComparableKeys(a.key, b.key, directions) || compareCodePoints(a.id, b.id));
|
|
1374
|
-
const hasMore = merged.length > offset + limit;
|
|
1375
|
-
const page = merged.slice(offset, offset + limit);
|
|
1376
|
-
const last = page[page.length - 1];
|
|
1377
|
-
let cursor;
|
|
1378
|
-
if (hasMore && last !== void 0) {
|
|
1379
|
-
cursor = encodePageCursor({
|
|
1380
|
-
anchor: last.id,
|
|
1381
|
-
score: null,
|
|
1382
|
-
sortKey: last.key,
|
|
1383
|
-
sortSignature,
|
|
1384
|
-
binding: context.cursorBinding
|
|
1385
|
-
});
|
|
1386
|
-
}
|
|
1387
|
-
return {
|
|
1388
|
-
hits: page.map((entry) => ({ id: entry.id, document: void 0 })),
|
|
1389
|
-
count,
|
|
1390
|
-
cursor,
|
|
1391
|
-
facets: params.facets !== void 0 && partitionFacets.length > 0 ? mergeFacets(partitionFacets, params.facets) : void 0
|
|
1392
|
-
};
|
|
1393
|
-
}
|
|
1394
|
-
|
|
1395
|
-
// src/engine/query/index.ts
|
|
1396
|
-
function requireCursorDepth(cursor, signature, binding) {
|
|
1397
|
-
if (cursor === void 0) return 0;
|
|
1398
|
-
const decoded = decodePageCursor(cursor);
|
|
1399
|
-
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
1400
|
-
return decoded.depth ?? 0;
|
|
1401
|
-
}
|
|
1402
|
-
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1403
|
-
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1404
|
-
return import('./runtime-ZKYZFJJC.mjs').then(
|
|
1405
|
-
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1406
|
-
);
|
|
1407
|
-
}
|
|
1408
|
-
function executeQuery(request, callerContext) {
|
|
1409
|
-
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
1410
|
-
return import('./runtime-ZKYZFJJC.mjs').then(
|
|
1411
|
-
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1412
|
-
);
|
|
1413
|
-
}
|
|
1414
|
-
async function runQuery(request, callerContext) {
|
|
1415
|
-
const context = withPatternWork(callerContext);
|
|
1416
|
-
const { manager, language, config, workerSearch, indexName } = context;
|
|
1417
|
-
const startTime = now();
|
|
1418
|
-
const limit = clampLimit(request.limit);
|
|
1419
|
-
const offset = clampOffset(request.offset);
|
|
1420
|
-
requireWithinResultWindow(limit, offset);
|
|
1421
|
-
const sortFields = normalizeSort(request.sort);
|
|
1422
|
-
requireKnownMode(request);
|
|
1423
|
-
const sortSignature = sortSignatureOf(request.sort);
|
|
1424
|
-
const hasTerm = request.term !== void 0 && request.term.trim().length > 0;
|
|
1425
|
-
const hasVector = request.vector !== void 0 && request.vector.value !== void 0;
|
|
1426
|
-
const isHybridMode = request.mode === "hybrid" || hasTerm && hasVector;
|
|
1427
|
-
const isVectorOnly = (request.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
1428
|
-
if (sortSignature !== null && (isHybridMode || request.hybrid !== void 0)) {
|
|
1429
|
-
throw new NarsilError(
|
|
1430
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
1431
|
-
"A hybrid query cannot carry a sort, because fusion defines the order of hybrid results",
|
|
1432
|
-
{ sort: sortFields.map((entry) => entry.field) }
|
|
1433
|
-
);
|
|
1434
|
-
}
|
|
1435
|
-
const params = validQueryParams(request, config);
|
|
1436
|
-
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1437
|
-
const projection = resolveProjection(params.document);
|
|
1438
|
-
let paginated;
|
|
1439
|
-
let nextCursor;
|
|
1440
|
-
let count;
|
|
1441
|
-
let countExact = true;
|
|
1442
|
-
let facets;
|
|
1443
|
-
let groups;
|
|
1444
|
-
if (sortSignature !== null && hasTerm && !isVectorOnly && !isHybridMode && sortsWithoutScores(params)) {
|
|
1445
|
-
const page = executeSortedQueryPage(params, context, limit, offset, sortSignature);
|
|
1446
|
-
paginated = page.hits;
|
|
1447
|
-
nextCursor = page.cursor;
|
|
1448
|
-
count = page.count;
|
|
1449
|
-
facets = page.facets;
|
|
1450
|
-
} else {
|
|
1451
|
-
const requestedVectorField = params.vector?.field;
|
|
1452
|
-
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
1453
|
-
let fanOutResult;
|
|
1454
|
-
if (isVectorOnly && hasGlobalVectorIndex) {
|
|
1455
|
-
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
1456
|
-
fanOutResult = await executeVectorSearch(params, context, limit, offset, depth);
|
|
1457
|
-
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1458
|
-
const depth = requireCursorDepth(params.searchAfter, sortSignature, context.cursorBinding);
|
|
1459
|
-
fanOutResult = await hybridSearchOf(params, context, limit, offset, depth);
|
|
1460
|
-
} else {
|
|
1461
|
-
const scoring = scoringConfigFor(params, context);
|
|
1462
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
1463
|
-
indexName,
|
|
1464
|
-
params,
|
|
1465
|
-
broadcastStatsForWorker(params, context, scoring),
|
|
1466
|
-
context.partitionIds,
|
|
1467
|
-
context.patternWork
|
|
1468
|
-
) : null;
|
|
1469
|
-
if (workerResult) {
|
|
1470
|
-
fanOutResult = workerResult;
|
|
1471
|
-
} else {
|
|
1472
|
-
fanOutResult = await fanOutQuery(
|
|
1473
|
-
manager,
|
|
1474
|
-
params,
|
|
1475
|
-
language,
|
|
1476
|
-
config.schema,
|
|
1477
|
-
scoring,
|
|
1478
|
-
searchOptionsFor(manager, context.patternWork)
|
|
1479
|
-
);
|
|
1480
|
-
}
|
|
1481
|
-
}
|
|
1482
|
-
const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
1483
|
-
let hits;
|
|
1484
|
-
if (needsFullHits) {
|
|
1485
|
-
hits = fanOutResult.scored.map((scored) => ({
|
|
1486
|
-
id: scored.docId,
|
|
1487
|
-
score: scored.score,
|
|
1488
|
-
document: void 0,
|
|
1489
|
-
scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
|
|
1490
|
-
}));
|
|
1491
|
-
} else {
|
|
1492
|
-
const end = Math.min(offset + limit + 1, fanOutResult.scored.length);
|
|
1493
|
-
hits = new Array(end);
|
|
1494
|
-
for (let i = 0; i < end; i++) {
|
|
1495
|
-
const scored = fanOutResult.scored[i];
|
|
1496
|
-
hits[i] = {
|
|
1497
|
-
id: scored.docId,
|
|
1498
|
-
score: scored.score,
|
|
1499
|
-
document: void 0,
|
|
1500
|
-
scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
|
|
1501
|
-
};
|
|
1502
|
-
}
|
|
1503
|
-
}
|
|
1504
|
-
const sortFieldNames = sortFields.map((entry) => entry.field);
|
|
1505
|
-
const sortFlatSchema = sortFieldNames.length === 0 ? {} : flattenSchema(config.schema);
|
|
1506
|
-
const sortFieldTypes = sortFieldNames.map((field) => sortFlatSchema[field]);
|
|
1507
|
-
const sortModes = sortModesOf(sortFields);
|
|
1508
|
-
const sortKeyCache = /* @__PURE__ */ new Map();
|
|
1509
|
-
const sortKeyOf = (docId) => {
|
|
1510
|
-
let key = sortKeyCache.get(docId);
|
|
1511
|
-
if (key === void 0) {
|
|
1512
|
-
key = manager.sortValues(docId, sortFieldNames, sortFieldTypes, sortModes);
|
|
1513
|
-
sortKeyCache.set(docId, key);
|
|
1514
|
-
}
|
|
1515
|
-
return key;
|
|
1516
|
-
};
|
|
1517
|
-
if (params.sort) {
|
|
1518
|
-
hits = applySorting(hits, params.sort, sortKeyOf);
|
|
1519
|
-
}
|
|
1520
|
-
if (params.group) {
|
|
1521
|
-
groups = applyGrouping(hits, params.group, {
|
|
1522
|
-
fieldValues: (docId, fieldPaths) => manager.fieldValues(docId, fieldPaths),
|
|
1523
|
-
document: (docId) => manager.getRef(docId)
|
|
1524
|
-
});
|
|
1525
|
-
}
|
|
1526
|
-
let pinsFromOutside = 0;
|
|
1527
|
-
if (params.pinned && params.searchAfter === void 0 && context.partitionIds === void 0) {
|
|
1528
|
-
const placement = placePinned(hits, params.pinned, (docId) => {
|
|
1529
|
-
const doc = manager.getRef(docId);
|
|
1530
|
-
if (!doc) return void 0;
|
|
1531
|
-
return { id: docId, score: 0, document: doc };
|
|
1532
|
-
});
|
|
1533
|
-
hits = placement.hits;
|
|
1534
|
-
pinsFromOutside = placement.placedFromOutside;
|
|
1535
|
-
}
|
|
1536
|
-
let sortContext;
|
|
1537
|
-
if (sortSignature !== null) {
|
|
1538
|
-
sortContext = {
|
|
1539
|
-
signature: sortSignature,
|
|
1540
|
-
directions: sortFields.map((entry) => entry.direction),
|
|
1541
|
-
sortKeyOf
|
|
1542
|
-
};
|
|
1543
|
-
}
|
|
1544
|
-
const pinnedIds = params.pinned !== void 0 && params.searchAfter === void 0 ? new Set(params.pinned.map((entry) => entry.docId)) : void 0;
|
|
1545
|
-
const paged = (fanOutResult.paginate ?? applyPagination)(
|
|
1546
|
-
hits,
|
|
1547
|
-
limit,
|
|
1548
|
-
offset,
|
|
1549
|
-
context.cursorBinding,
|
|
1550
|
-
params.searchAfter,
|
|
1551
|
-
sortContext,
|
|
1552
|
-
pinnedIds
|
|
1553
|
-
);
|
|
1554
|
-
paginated = paged.paginated;
|
|
1555
|
-
nextCursor = paged.nextCursor;
|
|
1556
|
-
const holdsEveryMatch = fanOutResult.matchedExact !== false && fanOutResult.scored.length === fanOutResult.totalMatched;
|
|
1557
|
-
count = fanOutResult.totalMatched + (holdsEveryMatch ? pinsFromOutside : 0);
|
|
1558
|
-
countExact = fanOutResult.matchedExact !== false && (holdsEveryMatch || pinsFromOutside === 0);
|
|
1559
|
-
facets = fanOutResult.facets;
|
|
1560
|
-
if (sortSignature !== null && params.includeScores !== true) {
|
|
1561
|
-
for (const hit of hits) hit.score = void 0;
|
|
1562
|
-
}
|
|
1563
|
-
}
|
|
1564
|
-
if (projection.kind === "none") {
|
|
1565
|
-
for (const hit of paginated) {
|
|
1566
|
-
hit.document = {};
|
|
1567
|
-
}
|
|
1568
|
-
} else {
|
|
1569
|
-
for (const hit of paginated) {
|
|
1570
|
-
hit.document = manager.get(hit.id, projection) ?? {};
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
if (groups) {
|
|
1574
|
-
for (const group of groups) {
|
|
1575
|
-
for (const hit of group.hits) {
|
|
1576
|
-
if (projection.kind === "none") {
|
|
1577
|
-
hit.document = {};
|
|
1578
|
-
continue;
|
|
1579
|
-
}
|
|
1580
|
-
hit.document = manager.get(hit.id, projection) ?? {};
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1583
|
-
}
|
|
1584
|
-
if (params.highlight) {
|
|
1585
|
-
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
1586
|
-
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
1587
|
-
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument, manager.synonyms);
|
|
1588
|
-
}
|
|
1589
|
-
const elapsed = now() - startTime;
|
|
1590
|
-
return {
|
|
1591
|
-
hits: paginated,
|
|
1592
|
-
count,
|
|
1593
|
-
countExact,
|
|
1594
|
-
elapsed,
|
|
1595
|
-
cursor: nextCursor,
|
|
1596
|
-
facets,
|
|
1597
|
-
groups,
|
|
1598
|
-
coverage: coverageFor(manager, context.partitionIds)
|
|
1599
|
-
};
|
|
1600
|
-
}
|
|
1601
|
-
async function executePreflight(request, callerContext) {
|
|
1602
|
-
const context = withPatternWork(callerContext);
|
|
1603
|
-
const { manager, language, config, workerSearch, indexName } = context;
|
|
1604
|
-
const startTime = now();
|
|
1605
|
-
requireKnownMode(request);
|
|
1606
|
-
const params = validQueryParams(request, config);
|
|
1607
|
-
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
1608
|
-
const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
|
|
1609
|
-
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
1610
|
-
const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
1611
|
-
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1612
|
-
const requestedVectorField = params.vector?.field;
|
|
1613
|
-
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
1614
|
-
let totalMatched;
|
|
1615
|
-
let countExact = true;
|
|
1616
|
-
const preflightLimit = 1e3;
|
|
1617
|
-
const preflightOffset = 0;
|
|
1618
|
-
if (isVectorOnly && hasGlobalVectorIndex) {
|
|
1619
|
-
const result = await executeVectorSearch(params, context, preflightLimit, preflightOffset);
|
|
1620
|
-
totalMatched = result.totalMatched;
|
|
1621
|
-
countExact = result.matchedExact !== false;
|
|
1622
|
-
} else if (isHybridMode && hasGlobalVectorIndex) {
|
|
1623
|
-
const result = await hybridSearchOf(params, context, preflightLimit, preflightOffset);
|
|
1624
|
-
totalMatched = result.totalMatched;
|
|
1625
|
-
countExact = result.matchedExact !== false;
|
|
1626
|
-
} else if (countsWithoutScores(params)) {
|
|
1627
|
-
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
1628
|
-
searchOptions: searchOptionsFor(manager, context.patternWork),
|
|
1629
|
-
partitionIds: context.partitionIds
|
|
1630
|
-
});
|
|
1631
|
-
} else {
|
|
1632
|
-
const scoring = scoringConfigFor(params, context);
|
|
1633
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
1634
|
-
indexName,
|
|
1635
|
-
params,
|
|
1636
|
-
broadcastStatsForWorker(params, context, scoring),
|
|
1637
|
-
context.partitionIds,
|
|
1638
|
-
context.patternWork
|
|
1639
|
-
) : null;
|
|
1640
|
-
if (workerResult) {
|
|
1641
|
-
totalMatched = workerResult.totalMatched;
|
|
1642
|
-
} else {
|
|
1643
|
-
const fanOutResult = await fanOutQuery(
|
|
1644
|
-
manager,
|
|
1645
|
-
params,
|
|
1646
|
-
language,
|
|
1647
|
-
config.schema,
|
|
1648
|
-
scoring,
|
|
1649
|
-
searchOptionsFor(manager, context.patternWork)
|
|
1650
|
-
);
|
|
1651
|
-
totalMatched = fanOutResult.totalMatched;
|
|
1652
|
-
}
|
|
1653
|
-
}
|
|
1654
|
-
const elapsed = now() - startTime;
|
|
1655
|
-
return { count: totalMatched, countExact, elapsed };
|
|
1656
|
-
}
|
|
1657
|
-
|
|
1658
|
-
// src/engine/suggest.ts
|
|
1659
|
-
function executeSuggest(manager, language, params, partitionIds) {
|
|
1660
|
-
const t0 = performance.now();
|
|
1661
|
-
const ceiling = partitionIds === void 0 ? MAX_SUGGEST_LIMIT : MAX_SUGGEST_SCATTER_LIMIT;
|
|
1662
|
-
const limit = Math.max(1, Math.min(clampRowCount(params.limit, DEFAULT_SUGGEST_LIMIT), ceiling));
|
|
1663
|
-
const rawPrefix = params.prefix.trim();
|
|
1664
|
-
if (rawPrefix.length === 0) {
|
|
1665
|
-
return { terms: [], elapsed: performance.now() - t0 };
|
|
1666
|
-
}
|
|
1667
|
-
const unstemmed = tokenize(rawPrefix, language, { stem: false, removeStopWords: false });
|
|
1668
|
-
const lastToken = unstemmed.tokens.length > 0 ? unstemmed.tokens[unstemmed.tokens.length - 1].token : rawPrefix.toLowerCase();
|
|
1669
|
-
if (lastToken.length === 0) {
|
|
1670
|
-
return { terms: [], elapsed: performance.now() - t0 };
|
|
1671
|
-
}
|
|
1672
|
-
const stemmed = language.stemmer ? language.stemmer(lastToken) : lastToken;
|
|
1673
|
-
const merged = /* @__PURE__ */ new Map();
|
|
1674
|
-
const perPartitionLimit = limit * 2;
|
|
1675
|
-
const partitions = partitionIds === void 0 ? manager.getAllPartitions() : partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
|
|
1676
|
-
for (const partition of partitions) {
|
|
1677
|
-
for (const suggestion of partition.suggestTerms(lastToken, stemmed, perPartitionLimit)) {
|
|
1678
|
-
let entry = merged.get(suggestion.token);
|
|
1679
|
-
if (!entry) {
|
|
1680
|
-
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
1681
|
-
merged.set(suggestion.token, entry);
|
|
1682
|
-
}
|
|
1683
|
-
entry.documentFrequency += suggestion.documentFrequency;
|
|
1684
|
-
for (const s of suggestion.surfaces) {
|
|
1685
|
-
entry.surfaceOccurrences.set(s.surface, (entry.surfaceOccurrences.get(s.surface) ?? 0) + s.occurrences);
|
|
1686
|
-
}
|
|
1687
|
-
}
|
|
1688
|
-
}
|
|
1689
|
-
const terms = Array.from(merged.values()).map((entry) => ({ term: pickDisplaySurface(entry.surfaceOccurrences), documentFrequency: entry.documentFrequency })).sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.term, b.term));
|
|
1690
|
-
if (terms.length > limit) terms.length = limit;
|
|
1691
|
-
return { terms, elapsed: performance.now() - t0 };
|
|
1692
|
-
}
|
|
1693
|
-
function pickDisplaySurface(surfaceOccurrences) {
|
|
1694
|
-
let best = "";
|
|
1695
|
-
let bestCount = -1;
|
|
1696
|
-
for (const [surface, occurrences] of surfaceOccurrences) {
|
|
1697
|
-
if (occurrences > bestCount || occurrences === bestCount && compareCodePoints(surface, best) < 0) {
|
|
1698
|
-
best = surface;
|
|
1699
|
-
bestCount = occurrences;
|
|
1700
|
-
}
|
|
1701
|
-
}
|
|
1702
|
-
return best;
|
|
1703
|
-
}
|
|
1704
|
-
|
|
1705
|
-
export { clampLimit, dedupePinnedEntries, executeListDocuments, executePreflight, executeQuery, executeSuggest, foldGroupReducer, hitsKeptPerGroup, listBindingOf, now, queryBindingOf, requireSurfaceFormsForPartialFields, resolveHybridFusion, runtimeHost, validateBM25Params, validateDocId, validateIndexName, validatePartitionConfig, validatePatternValueLimit, validateWorkerConfig };
|