@delali/narsil 0.3.1-next.99 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -0,0 +1,1430 @@
|
|
|
1
|
+
// src/query-suggestions/suggestions-host.browser.ts
|
|
2
|
+
var ErrorCodes;
|
|
3
|
+
var GROUP_FORMAT;
|
|
4
|
+
var NarsilError;
|
|
5
|
+
var SUGGESTIONS_INDEX_INFIX;
|
|
6
|
+
var compareCodePoints;
|
|
7
|
+
var createEngineIndex;
|
|
8
|
+
var emitEngineEvent;
|
|
9
|
+
var emitWriteEventAfter;
|
|
10
|
+
var loadDigest;
|
|
11
|
+
var spawnNodeWorker;
|
|
12
|
+
var withOpenIndex;
|
|
13
|
+
var decode;
|
|
14
|
+
var encode;
|
|
15
|
+
var BUILD_LIST_PAGE;
|
|
16
|
+
var BUILD_WRITE_BATCH;
|
|
17
|
+
var BYTES_PER_MIB;
|
|
18
|
+
var DAY_MS;
|
|
19
|
+
var DEFAULT_PREVIEW_LIMIT;
|
|
20
|
+
var DayStore;
|
|
21
|
+
var HISTORY_SWEEP_INTERVAL_MS;
|
|
22
|
+
var KEY_FINGERPRINT_BYTES;
|
|
23
|
+
var KEY_FINGERPRINT_LABEL;
|
|
24
|
+
var MAX_MANUAL_SUGGESTION_COUNT;
|
|
25
|
+
var MAX_PREVIEW_LIMIT;
|
|
26
|
+
var MAX_SOURCE_INDEX_NAME_LENGTH;
|
|
27
|
+
var MAX_TIMER_DELAY_MS;
|
|
28
|
+
var MIN_KEY_BYTES;
|
|
29
|
+
var MS_PER_SECOND;
|
|
30
|
+
var RestoredHistory;
|
|
31
|
+
var SCHEDULE_OFF;
|
|
32
|
+
var SCRAMBLED_ID_BYTES;
|
|
33
|
+
var acceptedPhrase;
|
|
34
|
+
var attachRuntime;
|
|
35
|
+
var attachedRuntimeOf;
|
|
36
|
+
var banListFromInput;
|
|
37
|
+
var buildRecordKey;
|
|
38
|
+
var crc32;
|
|
39
|
+
var dayOf;
|
|
40
|
+
var historyFileOf;
|
|
41
|
+
var historyFilesOf;
|
|
42
|
+
var installSuggestionHooks;
|
|
43
|
+
var invalidSetting;
|
|
44
|
+
var isGroup;
|
|
45
|
+
var isPerson;
|
|
46
|
+
var latestMatchAtOrBefore;
|
|
47
|
+
var listHistoryFolder;
|
|
48
|
+
var nextMatchAfter;
|
|
49
|
+
var normaliseBan;
|
|
50
|
+
var oneSyncChangeAtATime;
|
|
51
|
+
var openChecksummedEnvelope;
|
|
52
|
+
var openCountsFile;
|
|
53
|
+
var packEnvelopeWithChecksum;
|
|
54
|
+
var parseSchedule;
|
|
55
|
+
var removeHistoryFolder;
|
|
56
|
+
var removeRestoreLeftovers;
|
|
57
|
+
var requireFieldsInSchema;
|
|
58
|
+
var resolveSetting;
|
|
59
|
+
var sameBytes;
|
|
60
|
+
var unionOfBans;
|
|
61
|
+
var validateSetting;
|
|
62
|
+
var withState;
|
|
63
|
+
function bindSuggestionsHost(host, msgpack, suggestionStore) {
|
|
64
|
+
({
|
|
65
|
+
ErrorCodes,
|
|
66
|
+
GROUP_FORMAT,
|
|
67
|
+
NarsilError,
|
|
68
|
+
SUGGESTIONS_INDEX_INFIX,
|
|
69
|
+
compareCodePoints,
|
|
70
|
+
createEngineIndex,
|
|
71
|
+
emitEngineEvent,
|
|
72
|
+
emitWriteEventAfter,
|
|
73
|
+
loadDigest,
|
|
74
|
+
spawnNodeWorker,
|
|
75
|
+
withOpenIndex
|
|
76
|
+
} = host);
|
|
77
|
+
({ decode, encode } = msgpack);
|
|
78
|
+
({
|
|
79
|
+
BUILD_LIST_PAGE,
|
|
80
|
+
BUILD_WRITE_BATCH,
|
|
81
|
+
BYTES_PER_MIB,
|
|
82
|
+
DAY_MS,
|
|
83
|
+
DEFAULT_PREVIEW_LIMIT,
|
|
84
|
+
DayStore,
|
|
85
|
+
HISTORY_SWEEP_INTERVAL_MS,
|
|
86
|
+
KEY_FINGERPRINT_BYTES,
|
|
87
|
+
KEY_FINGERPRINT_LABEL,
|
|
88
|
+
MAX_MANUAL_SUGGESTION_COUNT,
|
|
89
|
+
MAX_PREVIEW_LIMIT,
|
|
90
|
+
MAX_SOURCE_INDEX_NAME_LENGTH,
|
|
91
|
+
MAX_TIMER_DELAY_MS,
|
|
92
|
+
MIN_KEY_BYTES,
|
|
93
|
+
MS_PER_SECOND,
|
|
94
|
+
RestoredHistory,
|
|
95
|
+
SCHEDULE_OFF,
|
|
96
|
+
SCRAMBLED_ID_BYTES,
|
|
97
|
+
acceptedPhrase,
|
|
98
|
+
attachRuntime,
|
|
99
|
+
attachedRuntimeOf,
|
|
100
|
+
banListFromInput,
|
|
101
|
+
buildRecordKey,
|
|
102
|
+
crc32,
|
|
103
|
+
dayOf,
|
|
104
|
+
historyFileOf,
|
|
105
|
+
historyFilesOf,
|
|
106
|
+
installSuggestionHooks,
|
|
107
|
+
invalidSetting,
|
|
108
|
+
isGroup,
|
|
109
|
+
isPerson,
|
|
110
|
+
latestMatchAtOrBefore,
|
|
111
|
+
listHistoryFolder,
|
|
112
|
+
nextMatchAfter,
|
|
113
|
+
normaliseBan,
|
|
114
|
+
oneSyncChangeAtATime,
|
|
115
|
+
openChecksummedEnvelope,
|
|
116
|
+
openCountsFile,
|
|
117
|
+
packEnvelopeWithChecksum,
|
|
118
|
+
parseSchedule,
|
|
119
|
+
removeHistoryFolder,
|
|
120
|
+
removeRestoreLeftovers,
|
|
121
|
+
requireFieldsInSchema,
|
|
122
|
+
resolveSetting,
|
|
123
|
+
sameBytes,
|
|
124
|
+
unionOfBans,
|
|
125
|
+
validateSetting,
|
|
126
|
+
withState
|
|
127
|
+
} = suggestionStore);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// src/query-suggestions/build-record.ts
|
|
131
|
+
var BUILD_RECORD_FORMAT_VERSION = 1;
|
|
132
|
+
var SOURCES_DIGEST_BYTES = 32;
|
|
133
|
+
var TRIGGERS = /* @__PURE__ */ new Set(["schedule", "missed", "caller"]);
|
|
134
|
+
var STATUSES = /* @__PURE__ */ new Set(["running", "completed", "failed", "skipped"]);
|
|
135
|
+
var WHAT = "query suggestions build record";
|
|
136
|
+
async function sourcesDigestOf(sources) {
|
|
137
|
+
const sorted = [...sources].sort(
|
|
138
|
+
(left, right) => compareCodePoints(left.nodeId, right.nodeId) || compareCodePoints(left.key, right.key)
|
|
139
|
+
);
|
|
140
|
+
const { sha256 } = await loadDigest();
|
|
141
|
+
return sha256(encode(sorted.map((source) => [source.nodeId, source.key, source.savedAt])));
|
|
142
|
+
}
|
|
143
|
+
function isCount(value) {
|
|
144
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
145
|
+
}
|
|
146
|
+
function buildRecordToWire(record) {
|
|
147
|
+
return encode({
|
|
148
|
+
buildId: record.buildId,
|
|
149
|
+
trigger: record.trigger,
|
|
150
|
+
status: record.status,
|
|
151
|
+
startedAt: record.startedAt,
|
|
152
|
+
...record.endedAt !== void 0 ? { endedAt: record.endedAt } : {},
|
|
153
|
+
published: record.published,
|
|
154
|
+
removed: record.removed,
|
|
155
|
+
discarded: record.discarded,
|
|
156
|
+
sourcesDigest: record.sourcesDigest,
|
|
157
|
+
...record.errorCode !== void 0 ? { errorCode: record.errorCode } : {}
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
function buildRecordFromWire(bytes) {
|
|
161
|
+
let raw;
|
|
162
|
+
try {
|
|
163
|
+
raw = decode(bytes);
|
|
164
|
+
} catch {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
168
|
+
const value = raw;
|
|
169
|
+
const { buildId, trigger, status: status2, startedAt, endedAt, published, removed, discarded, sourcesDigest, errorCode } = value;
|
|
170
|
+
if (typeof buildId !== "string" || typeof trigger !== "string" || !TRIGGERS.has(trigger)) return null;
|
|
171
|
+
if (typeof status2 !== "string" || !STATUSES.has(status2)) return null;
|
|
172
|
+
if (!isCount(startedAt) || !isCount(published) || !isCount(removed) || !isCount(discarded)) return null;
|
|
173
|
+
if (endedAt !== void 0 && !isCount(endedAt)) return null;
|
|
174
|
+
if (errorCode !== void 0 && typeof errorCode !== "string") return null;
|
|
175
|
+
if (!(sourcesDigest instanceof Uint8Array) || sourcesDigest.byteLength !== SOURCES_DIGEST_BYTES) return null;
|
|
176
|
+
return {
|
|
177
|
+
buildId,
|
|
178
|
+
trigger,
|
|
179
|
+
status: status2,
|
|
180
|
+
startedAt,
|
|
181
|
+
...endedAt !== void 0 ? { endedAt } : {},
|
|
182
|
+
published,
|
|
183
|
+
removed,
|
|
184
|
+
discarded,
|
|
185
|
+
sourcesDigest: new Uint8Array(sourcesDigest),
|
|
186
|
+
...errorCode !== void 0 ? { errorCode } : {}
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function encodeBuildRecordFile(record) {
|
|
190
|
+
const payload = buildRecordToWire(record);
|
|
191
|
+
const packed = packEnvelopeWithChecksum([payload], BUILD_RECORD_FORMAT_VERSION, WHAT, crc32(payload));
|
|
192
|
+
const bytes = new Uint8Array(packed.byteLength);
|
|
193
|
+
let offset = 0;
|
|
194
|
+
for (const chunk of packed.chunks) {
|
|
195
|
+
bytes.set(chunk, offset);
|
|
196
|
+
offset += chunk.byteLength;
|
|
197
|
+
}
|
|
198
|
+
return bytes;
|
|
199
|
+
}
|
|
200
|
+
async function decodeBuildRecordFile(bytes) {
|
|
201
|
+
try {
|
|
202
|
+
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
203
|
+
envelopeFormatVersion: BUILD_RECORD_FORMAT_VERSION,
|
|
204
|
+
what: WHAT,
|
|
205
|
+
allowsCompression: false
|
|
206
|
+
});
|
|
207
|
+
return buildRecordFromWire(payload);
|
|
208
|
+
} catch {
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function publicBuildOf(record) {
|
|
213
|
+
return {
|
|
214
|
+
buildId: record.buildId,
|
|
215
|
+
trigger: record.trigger,
|
|
216
|
+
status: record.status,
|
|
217
|
+
startedAt: record.startedAt,
|
|
218
|
+
endedAt: record.endedAt ?? null,
|
|
219
|
+
published: record.published,
|
|
220
|
+
removed: record.removed,
|
|
221
|
+
discarded: record.discarded,
|
|
222
|
+
errorCode: record.errorCode ?? null
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// src/query-suggestions/merge.ts
|
|
227
|
+
var SEPARATOR = "\0";
|
|
228
|
+
function idKey(source, offset) {
|
|
229
|
+
let key = "";
|
|
230
|
+
for (let i = 0; i < SCRAMBLED_ID_BYTES; i++) key += String.fromCharCode(source[offset + i]);
|
|
231
|
+
return key;
|
|
232
|
+
}
|
|
233
|
+
var Merger = class {
|
|
234
|
+
settings;
|
|
235
|
+
banned;
|
|
236
|
+
entries = /* @__PURE__ */ new Map();
|
|
237
|
+
groups;
|
|
238
|
+
discarded = 0;
|
|
239
|
+
constructor(settings) {
|
|
240
|
+
this.settings = settings;
|
|
241
|
+
this.banned = new Set(settings.banned);
|
|
242
|
+
this.groups = new Set(settings.markedGroups);
|
|
243
|
+
}
|
|
244
|
+
async addFile(bytes, expectedDay) {
|
|
245
|
+
const opened = await openCountsFile(bytes, expectedDay, this.settings.fingerprint);
|
|
246
|
+
if (opened.kind !== "counts") return opened;
|
|
247
|
+
if (this.settings.clustered && (opened.indexUuid === null || opened.indexUuid !== this.settings.indexUuid)) {
|
|
248
|
+
return { kind: "other-index" };
|
|
249
|
+
}
|
|
250
|
+
const { minPeople, maxMergePhrases, maxGroups } = this.settings;
|
|
251
|
+
opened.visit((group, phrase, count, people) => {
|
|
252
|
+
if (this.banned.has(phrase)) return;
|
|
253
|
+
const key = `${group}${SEPARATOR}${phrase}`;
|
|
254
|
+
let entry = this.entries.get(key);
|
|
255
|
+
if (entry === void 0) {
|
|
256
|
+
const newGroup = !this.groups.has(group);
|
|
257
|
+
if (newGroup && this.groups.size >= maxGroups || this.entries.size >= maxMergePhrases) {
|
|
258
|
+
this.discarded += 1;
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
if (newGroup) this.groups.add(group);
|
|
262
|
+
entry = { count: 0, people: /* @__PURE__ */ new Set() };
|
|
263
|
+
this.entries.set(key, entry);
|
|
264
|
+
}
|
|
265
|
+
entry.count += count;
|
|
266
|
+
if (entry.people === null) return;
|
|
267
|
+
for (const offset of people.offsets) {
|
|
268
|
+
entry.people.add(idKey(people.source, offset));
|
|
269
|
+
if (entry.people.size >= minPeople) {
|
|
270
|
+
entry.people = null;
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
});
|
|
275
|
+
return { kind: "merged", day: opened.day, savedAt: opened.savedAt };
|
|
276
|
+
}
|
|
277
|
+
result() {
|
|
278
|
+
const groups = {};
|
|
279
|
+
for (const [key, entry] of this.entries) {
|
|
280
|
+
if (entry.people !== null) continue;
|
|
281
|
+
const split = key.indexOf(SEPARATOR);
|
|
282
|
+
const group = key.slice(0, split);
|
|
283
|
+
const list = groups[group] ?? [];
|
|
284
|
+
list.push({ phrase: key.slice(split + 1), count: entry.count });
|
|
285
|
+
groups[group] = list;
|
|
286
|
+
}
|
|
287
|
+
for (const list of Object.values(groups)) {
|
|
288
|
+
list.sort((left, right) => right.count - left.count || (left.phrase < right.phrase ? -1 : 1));
|
|
289
|
+
}
|
|
290
|
+
return { groups, discarded: this.discarded };
|
|
291
|
+
}
|
|
292
|
+
};
|
|
293
|
+
function inThreadSession(settings) {
|
|
294
|
+
const merger = new Merger(settings);
|
|
295
|
+
return {
|
|
296
|
+
addFile: (input) => merger.addFile(input.bytes, input.day),
|
|
297
|
+
finish: async () => merger.result(),
|
|
298
|
+
abandon() {
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
async function startMerge(settings, memoryBytes) {
|
|
303
|
+
return inThreadSession(settings);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// src/query-suggestions/suggestions-writer.ts
|
|
307
|
+
var ORIGIN_POPULAR = "popular";
|
|
308
|
+
var ORIGIN_MANUAL = "manual";
|
|
309
|
+
function suggestionsIndexName(indexName, group) {
|
|
310
|
+
return `${indexName}${SUGGESTIONS_INDEX_INFIX}${group}`;
|
|
311
|
+
}
|
|
312
|
+
function suggestionDocument(phrase, count, origin) {
|
|
313
|
+
return { id: phrase, query: phrase, count, origin };
|
|
314
|
+
}
|
|
315
|
+
function markedSuggestionIndexes(host, indexName) {
|
|
316
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
317
|
+
for (const name of host.indexNames()) {
|
|
318
|
+
const source = host.view(name)?.state?.querySuggestionsSource;
|
|
319
|
+
if (source?.index === indexName) byGroup.set(source.group, name);
|
|
320
|
+
}
|
|
321
|
+
return byGroup;
|
|
322
|
+
}
|
|
323
|
+
async function ensureSuggestionsIndex(host, indexName, group, created = []) {
|
|
324
|
+
const name = suggestionsIndexName(indexName, group);
|
|
325
|
+
const source = host.view(indexName);
|
|
326
|
+
if (source === null) {
|
|
327
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, { indexName });
|
|
328
|
+
}
|
|
329
|
+
const existing = host.view(name);
|
|
330
|
+
if (existing === null) {
|
|
331
|
+
await host.createSuggestionsIndex(name, { index: indexName, group }, source.language);
|
|
332
|
+
created.push(name);
|
|
333
|
+
return name;
|
|
334
|
+
}
|
|
335
|
+
const marker = existing.state?.querySuggestionsSource;
|
|
336
|
+
if (marker?.index !== indexName || marker.group !== group) {
|
|
337
|
+
throw new NarsilError(
|
|
338
|
+
ErrorCodes.CONFIG_INVALID,
|
|
339
|
+
`Index "${name}" exists without the marker of a suggestions index for "${indexName}", so the build leaves it alone`,
|
|
340
|
+
{ indexName: name }
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
if (existing.language !== source.language) {
|
|
344
|
+
await host.engine.changeAnalysis(name, { language: source.language });
|
|
345
|
+
}
|
|
346
|
+
return name;
|
|
347
|
+
}
|
|
348
|
+
async function readSuggestions(host, name) {
|
|
349
|
+
const stored = /* @__PURE__ */ new Map();
|
|
350
|
+
let cursor;
|
|
351
|
+
do {
|
|
352
|
+
const page = await host.engine.listDocuments(name, {
|
|
353
|
+
limit: BUILD_LIST_PAGE,
|
|
354
|
+
...cursor !== void 0 ? { cursor } : {}
|
|
355
|
+
});
|
|
356
|
+
for (const { id, document } of page.documents) {
|
|
357
|
+
const count = typeof document.count === "number" ? document.count : 0;
|
|
358
|
+
const origin = typeof document.origin === "string" ? document.origin : ORIGIN_POPULAR;
|
|
359
|
+
stored.set(id, { count, origin });
|
|
360
|
+
}
|
|
361
|
+
cursor = page.cursor ?? void 0;
|
|
362
|
+
} while (cursor !== void 0);
|
|
363
|
+
return stored;
|
|
364
|
+
}
|
|
365
|
+
function requireWritten(result) {
|
|
366
|
+
const first = result.failed[0];
|
|
367
|
+
if (first !== void 0) throw first.error;
|
|
368
|
+
}
|
|
369
|
+
async function removeBatchOf(host, name, ids) {
|
|
370
|
+
const result = await host.engine.removeBatch(name, [...ids]);
|
|
371
|
+
for (const failure of result.failed) {
|
|
372
|
+
if (failure.error.code !== ErrorCodes.DOC_NOT_FOUND) throw failure.error;
|
|
373
|
+
}
|
|
374
|
+
return result.succeeded;
|
|
375
|
+
}
|
|
376
|
+
async function removeSuggestions(host, name, ids) {
|
|
377
|
+
let removed = 0;
|
|
378
|
+
for (let start = 0; start < ids.length; start += BUILD_WRITE_BATCH) {
|
|
379
|
+
removed += (await removeBatchOf(host, name, ids.slice(start, start + BUILD_WRITE_BATCH))).length;
|
|
380
|
+
}
|
|
381
|
+
return removed;
|
|
382
|
+
}
|
|
383
|
+
function restoreDocument(entry) {
|
|
384
|
+
return suggestionDocument(entry.phrase, entry.stored.count, entry.stored.origin);
|
|
385
|
+
}
|
|
386
|
+
async function undoWrites(host, steps, banned) {
|
|
387
|
+
for (const step of [...steps].reverse()) {
|
|
388
|
+
await removeSuggestions(host, step.name, step.remove);
|
|
389
|
+
const restore = step.restore.filter((entry) => !banned.has(entry.phrase));
|
|
390
|
+
for (let start = 0; start < restore.length; start += BUILD_WRITE_BATCH) {
|
|
391
|
+
const batch = restore.slice(start, start + BUILD_WRITE_BATCH);
|
|
392
|
+
await removeSuggestions(
|
|
393
|
+
host,
|
|
394
|
+
step.name,
|
|
395
|
+
batch.map((entry) => entry.phrase)
|
|
396
|
+
);
|
|
397
|
+
requireWritten(await host.engine.insertBatch(step.name, batch.map(restoreDocument)));
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
function planWrites(stored, due, banned, onlyAdd) {
|
|
402
|
+
const plan = { inserts: [], updates: [], stale: [], banned: [] };
|
|
403
|
+
for (const [phrase, count] of due) {
|
|
404
|
+
if (banned.has(phrase)) continue;
|
|
405
|
+
const current = stored.get(phrase);
|
|
406
|
+
if (current === void 0) plan.inserts.push(suggestionDocument(phrase, count, ORIGIN_POPULAR));
|
|
407
|
+
else if (current.origin === ORIGIN_POPULAR && current.count !== count) {
|
|
408
|
+
plan.updates.push({ phrase, count, previous: current });
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
for (const [phrase, current] of stored) {
|
|
412
|
+
if (banned.has(phrase)) plan.banned.push(phrase);
|
|
413
|
+
else if (!onlyAdd && current.origin === ORIGIN_POPULAR && !due.has(phrase)) {
|
|
414
|
+
plan.stale.push({ phrase, stored: current });
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
return plan;
|
|
418
|
+
}
|
|
419
|
+
async function writeGroup(host, name, due, banned, onlyAdd, undo) {
|
|
420
|
+
const plan = planWrites(await readSuggestions(host, name), due, banned, onlyAdd);
|
|
421
|
+
const step = { name, remove: [], restore: [] };
|
|
422
|
+
undo.push(step);
|
|
423
|
+
for (let start = 0; start < plan.inserts.length; start += BUILD_WRITE_BATCH) {
|
|
424
|
+
const result = await host.engine.insertBatch(name, plan.inserts.slice(start, start + BUILD_WRITE_BATCH));
|
|
425
|
+
step.remove.push(...result.succeeded);
|
|
426
|
+
requireWritten(result);
|
|
427
|
+
}
|
|
428
|
+
for (let start = 0; start < plan.updates.length; start += BUILD_WRITE_BATCH) {
|
|
429
|
+
const batch = plan.updates.slice(start, start + BUILD_WRITE_BATCH);
|
|
430
|
+
const result = await host.engine.updateBatch(
|
|
431
|
+
name,
|
|
432
|
+
batch.map((entry) => ({
|
|
433
|
+
docId: entry.phrase,
|
|
434
|
+
document: suggestionDocument(entry.phrase, entry.count, ORIGIN_POPULAR)
|
|
435
|
+
}))
|
|
436
|
+
);
|
|
437
|
+
const succeeded = new Set(result.succeeded);
|
|
438
|
+
for (const entry of batch) {
|
|
439
|
+
if (succeeded.has(entry.phrase)) step.restore.push({ phrase: entry.phrase, stored: entry.previous });
|
|
440
|
+
}
|
|
441
|
+
requireWritten(result);
|
|
442
|
+
}
|
|
443
|
+
let removed = 0;
|
|
444
|
+
for (let start = 0; start < plan.stale.length; start += BUILD_WRITE_BATCH) {
|
|
445
|
+
const batch = plan.stale.slice(start, start + BUILD_WRITE_BATCH);
|
|
446
|
+
const succeeded = new Set(
|
|
447
|
+
await removeBatchOf(
|
|
448
|
+
host,
|
|
449
|
+
name,
|
|
450
|
+
batch.map((entry) => entry.phrase)
|
|
451
|
+
)
|
|
452
|
+
);
|
|
453
|
+
removed += succeeded.size;
|
|
454
|
+
for (const entry of batch) if (succeeded.has(entry.phrase)) step.restore.push(entry);
|
|
455
|
+
}
|
|
456
|
+
removed += await removeSuggestions(host, name, plan.banned);
|
|
457
|
+
return { published: plan.inserts.length + plan.updates.length, removed };
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// src/query-suggestions/build.ts
|
|
461
|
+
function enabledSetting(host, indexName) {
|
|
462
|
+
const view = host.view(indexName);
|
|
463
|
+
if (view === null)
|
|
464
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, { indexName });
|
|
465
|
+
const setting = view.state?.querySuggestions;
|
|
466
|
+
if (setting === void 0 || !setting.enabled) {
|
|
467
|
+
throw new NarsilError(
|
|
468
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
469
|
+
`Index "${indexName}" has popular query suggestions off`,
|
|
470
|
+
{ indexName }
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
return resolveSetting(setting);
|
|
474
|
+
}
|
|
475
|
+
function cutoffDay(now, retentionDays) {
|
|
476
|
+
return dayOf(now - retentionDays * DAY_MS);
|
|
477
|
+
}
|
|
478
|
+
function warn(host, indexName, reason, file) {
|
|
479
|
+
host.emit("querySuggestionsWarning", { indexName, reason, file });
|
|
480
|
+
}
|
|
481
|
+
async function mergeFile(ctx, session, merged, indexName, file, removeLocal) {
|
|
482
|
+
const outcome = await session.addFile({ day: file.day, bytes: file.bytes });
|
|
483
|
+
if (outcome.kind === "merged") {
|
|
484
|
+
merged.sources.push({ nodeId: file.nodeId, key: file.key, savedAt: outcome.savedAt });
|
|
485
|
+
if (merged.oldestDay === null || outcome.day < merged.oldestDay) merged.oldestDay = outcome.day;
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
if (outcome.kind === "other-index") {
|
|
489
|
+
warn(ctx.host, indexName, "damaged-file", file.key);
|
|
490
|
+
await removeLocal?.();
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
warn(ctx.host, indexName, outcome.kind === "other-key" ? "other-key" : "damaged-file", file.key);
|
|
494
|
+
}
|
|
495
|
+
async function mergeHistory(ctx, indexName, setting, now) {
|
|
496
|
+
const { host } = ctx;
|
|
497
|
+
const view = host.view(indexName);
|
|
498
|
+
await ctx.saveDay(indexName);
|
|
499
|
+
const cutoff = cutoffDay(now, setting.retentionDays);
|
|
500
|
+
const session = await startMerge(
|
|
501
|
+
{
|
|
502
|
+
minPeople: setting.minPeople,
|
|
503
|
+
maxMergePhrases: setting.maxMergePhrases,
|
|
504
|
+
maxGroups: setting.maxGroups,
|
|
505
|
+
markedGroups: [...markedSuggestionIndexes(host, indexName).keys()],
|
|
506
|
+
fingerprint: ctx.scrambler.fingerprint,
|
|
507
|
+
indexUuid: view?.indexUuid ?? null,
|
|
508
|
+
clustered: host.clustered,
|
|
509
|
+
banned: view?.state?.bannedQueries ?? []
|
|
510
|
+
},
|
|
511
|
+
setting.mergeMemoryBytes
|
|
512
|
+
);
|
|
513
|
+
const merged = { result: { groups: {}, discarded: 0 }, sources: [], oldestDay: null, silentNodes: 0 };
|
|
514
|
+
try {
|
|
515
|
+
let keys;
|
|
516
|
+
try {
|
|
517
|
+
keys = await listHistoryFolder(host.files, indexName);
|
|
518
|
+
} catch (error) {
|
|
519
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "The node cannot read its query history folder", {
|
|
520
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
const files = [];
|
|
524
|
+
for (const key of keys) {
|
|
525
|
+
const file = historyFileOf(indexName, key);
|
|
526
|
+
if (file !== null && file.day >= cutoff) files.push(file);
|
|
527
|
+
}
|
|
528
|
+
files.sort((left, right) => compareCodePoints(left.key, right.key));
|
|
529
|
+
let localMerged = false;
|
|
530
|
+
const mergeLocalFiles = async () => {
|
|
531
|
+
if (localMerged) return;
|
|
532
|
+
localMerged = true;
|
|
533
|
+
for (const file of files) {
|
|
534
|
+
const bytes = await host.files.read(file.key);
|
|
535
|
+
if (bytes === null) continue;
|
|
536
|
+
if (bytes === "too-long") {
|
|
537
|
+
warn(host, indexName, "damaged-file", file.key);
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
await mergeFile(
|
|
541
|
+
ctx,
|
|
542
|
+
session,
|
|
543
|
+
merged,
|
|
544
|
+
indexName,
|
|
545
|
+
{ nodeId: host.nodeId, key: file.key, day: file.day, bytes },
|
|
546
|
+
() => host.files.remove(file.key)
|
|
547
|
+
);
|
|
548
|
+
}
|
|
549
|
+
};
|
|
550
|
+
for await (const remote of host.remoteHistory(indexName)) {
|
|
551
|
+
if (remote.kind === "silent") {
|
|
552
|
+
merged.silentNodes += 1;
|
|
553
|
+
continue;
|
|
554
|
+
}
|
|
555
|
+
if (compareCodePoints(remote.nodeId, host.nodeId) > 0) await mergeLocalFiles();
|
|
556
|
+
const name = historyFileOf(indexName, remote.key);
|
|
557
|
+
if (name === null || name.day < cutoff) continue;
|
|
558
|
+
await mergeFile(ctx, session, merged, indexName, { ...remote, day: name.day }, null);
|
|
559
|
+
}
|
|
560
|
+
await mergeLocalFiles();
|
|
561
|
+
merged.result = await session.finish();
|
|
562
|
+
return merged;
|
|
563
|
+
} catch (error) {
|
|
564
|
+
session.abandon();
|
|
565
|
+
throw error;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
async function pace(checksPerSecond) {
|
|
569
|
+
await new Promise((resolve) => setTimeout(resolve, MS_PER_SECOND / checksPerSecond));
|
|
570
|
+
}
|
|
571
|
+
async function meetsMinResults(ctx, indexName, setting, phrase) {
|
|
572
|
+
if (ctx.isStopping()) throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "The engine is shutting down");
|
|
573
|
+
const checked = await ctx.host.engine.preflight(indexName, { term: phrase, fields: setting.fields, termMatch: "all" });
|
|
574
|
+
await pace(setting.resultChecksPerSecond);
|
|
575
|
+
return checked.count >= setting.minResults;
|
|
576
|
+
}
|
|
577
|
+
async function choose(ctx, indexName, setting, phrases) {
|
|
578
|
+
const due = /* @__PURE__ */ new Map();
|
|
579
|
+
for (const { phrase, count } of phrases) {
|
|
580
|
+
if (await meetsMinResults(ctx, indexName, setting, phrase)) due.set(phrase, count);
|
|
581
|
+
}
|
|
582
|
+
return due;
|
|
583
|
+
}
|
|
584
|
+
function groupsToWrite(ctx, indexName, setting, result) {
|
|
585
|
+
const existing = markedSuggestionIndexes(ctx.host, indexName);
|
|
586
|
+
const groups = [...existing.keys()];
|
|
587
|
+
let discarded = 0;
|
|
588
|
+
for (const group of Object.keys(result.groups).sort()) {
|
|
589
|
+
if (existing.has(group)) continue;
|
|
590
|
+
if (groups.length < setting.maxGroups) groups.push(group);
|
|
591
|
+
else discarded += result.groups[group]?.length ?? 0;
|
|
592
|
+
}
|
|
593
|
+
return { groups, discarded };
|
|
594
|
+
}
|
|
595
|
+
async function runBuild(ctx, indexName, lastCompleted, now) {
|
|
596
|
+
const setting = enabledSetting(ctx.host, indexName);
|
|
597
|
+
const merged = await mergeHistory(ctx, indexName, setting, now);
|
|
598
|
+
const sourcesDigest = await sourcesDigestOf(merged.sources);
|
|
599
|
+
if (lastCompleted !== null && sameBytes(lastCompleted.sourcesDigest, sourcesDigest)) {
|
|
600
|
+
return { status: "skipped", published: 0, removed: 0, discarded: 0, sourcesDigest };
|
|
601
|
+
}
|
|
602
|
+
const onlyAdd = merged.silentNodes > 0 || merged.oldestDay === null || merged.oldestDay > cutoffDay(now, setting.retentionDays);
|
|
603
|
+
const { groups, discarded: groupsDiscarded } = groupsToWrite(ctx, indexName, setting, merged.result);
|
|
604
|
+
const dueByGroup = /* @__PURE__ */ new Map();
|
|
605
|
+
for (const group of groups) {
|
|
606
|
+
dueByGroup.set(group, await choose(ctx, indexName, setting, merged.result.groups[group] ?? []));
|
|
607
|
+
}
|
|
608
|
+
let published = 0;
|
|
609
|
+
let removed = 0;
|
|
610
|
+
const undo = [];
|
|
611
|
+
const created = [];
|
|
612
|
+
const bannedNow = () => new Set(ctx.host.view(indexName)?.state?.bannedQueries ?? []);
|
|
613
|
+
try {
|
|
614
|
+
for (const [group, due] of dueByGroup) {
|
|
615
|
+
const changes = await ctx.withWriteLock(indexName, async () => {
|
|
616
|
+
const name = await ensureSuggestionsIndex(ctx.host, indexName, group, created);
|
|
617
|
+
return writeGroup(ctx.host, name, due, bannedNow(), onlyAdd, undo);
|
|
618
|
+
});
|
|
619
|
+
published += changes.published;
|
|
620
|
+
removed += changes.removed;
|
|
621
|
+
}
|
|
622
|
+
} catch (error) {
|
|
623
|
+
await ctx.withWriteLock(indexName, () => undoWrites(ctx.host, undo, bannedNow())).catch(() => void 0);
|
|
624
|
+
for (const name of created) await ctx.host.engine.dropIndex(name).catch(() => void 0);
|
|
625
|
+
throw error;
|
|
626
|
+
}
|
|
627
|
+
return {
|
|
628
|
+
status: "completed",
|
|
629
|
+
published,
|
|
630
|
+
removed,
|
|
631
|
+
discarded: merged.result.discarded + groupsDiscarded,
|
|
632
|
+
sourcesDigest
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// src/query-suggestions/preview.ts
|
|
637
|
+
function previewOrder(left, right) {
|
|
638
|
+
if (left.count !== right.count) return right.count - left.count;
|
|
639
|
+
if (left.group !== right.group) return left.group < right.group ? -1 : 1;
|
|
640
|
+
return left.query < right.query ? -1 : left.query > right.query ? 1 : 0;
|
|
641
|
+
}
|
|
642
|
+
async function previewBuild(ctx, indexName, max) {
|
|
643
|
+
const setting = enabledSetting(ctx.host, indexName);
|
|
644
|
+
const merged = await mergeHistory(ctx, indexName, setting, Date.now());
|
|
645
|
+
const { groups } = groupsToWrite(ctx, indexName, setting, merged.result);
|
|
646
|
+
const banned = new Set(ctx.host.view(indexName)?.state?.bannedQueries ?? []);
|
|
647
|
+
const candidates = [];
|
|
648
|
+
for (const group of groups) {
|
|
649
|
+
for (const { phrase, count } of merged.result.groups[group] ?? []) {
|
|
650
|
+
if (!banned.has(phrase)) candidates.push({ group, query: phrase, count });
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
candidates.sort(previewOrder);
|
|
654
|
+
const checked = /* @__PURE__ */ new Map();
|
|
655
|
+
const entries = [];
|
|
656
|
+
let next = 0;
|
|
657
|
+
while (next < candidates.length && entries.length < max) {
|
|
658
|
+
const candidate = candidates[next];
|
|
659
|
+
next += 1;
|
|
660
|
+
if (candidate === void 0) break;
|
|
661
|
+
let passes = checked.get(candidate.query);
|
|
662
|
+
if (passes === void 0) {
|
|
663
|
+
passes = await meetsMinResults(ctx, indexName, setting, candidate.query);
|
|
664
|
+
checked.set(candidate.query, passes);
|
|
665
|
+
}
|
|
666
|
+
if (passes) entries.push(candidate);
|
|
667
|
+
}
|
|
668
|
+
return { entries, truncated: next < candidates.length };
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// src/query-suggestions/operations.ts
|
|
672
|
+
function requireView(runtime, indexName) {
|
|
673
|
+
const view = runtime.host.view(indexName);
|
|
674
|
+
if (view === null) {
|
|
675
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, { indexName });
|
|
676
|
+
}
|
|
677
|
+
return view;
|
|
678
|
+
}
|
|
679
|
+
function requireSourceView(runtime, indexName) {
|
|
680
|
+
const view = requireView(runtime, indexName);
|
|
681
|
+
if (view.state?.querySuggestionsSource !== void 0) {
|
|
682
|
+
invalidSetting(`Index "${indexName}" is a suggestions index, so it takes no query suggestions of its own`, {
|
|
683
|
+
indexName
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
if (indexName.length > MAX_SOURCE_INDEX_NAME_LENGTH) {
|
|
687
|
+
invalidSetting(
|
|
688
|
+
`Index "${indexName}" has a name longer than ${MAX_SOURCE_INDEX_NAME_LENGTH} characters, which leaves no room for the names of its suggestions indexes`,
|
|
689
|
+
{ indexName }
|
|
690
|
+
);
|
|
691
|
+
}
|
|
692
|
+
return view;
|
|
693
|
+
}
|
|
694
|
+
function requireAcceptable(runtime, indexName, setting) {
|
|
695
|
+
const view = requireSourceView(runtime, indexName);
|
|
696
|
+
requireFieldsInSchema(setting, view.schema);
|
|
697
|
+
if (!setting.enabled) return;
|
|
698
|
+
if (runtime.scrambler === null) {
|
|
699
|
+
invalidSetting("This engine has no query suggestions key, so it cannot turn the feature on", { indexName });
|
|
700
|
+
}
|
|
701
|
+
if (view.hasSync) {
|
|
702
|
+
invalidSetting(
|
|
703
|
+
`Index "${indexName}" has device sync on, and a device receives the whole index of one user while a suggestion comes from the searches of several people`,
|
|
704
|
+
{ indexName }
|
|
705
|
+
);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
async function updateSetting(runtime, indexName, raw) {
|
|
709
|
+
const setting = validateSetting(raw);
|
|
710
|
+
requireAcceptable(runtime, indexName, setting);
|
|
711
|
+
await runtime.host.updateState(indexName, (current) => {
|
|
712
|
+
requireAcceptable(runtime, indexName, setting);
|
|
713
|
+
return withState(current, { querySuggestions: setting });
|
|
714
|
+
});
|
|
715
|
+
if (!setting.enabled) await runtime.discardHistory(indexName);
|
|
716
|
+
runtime.settingChanged(indexName);
|
|
717
|
+
}
|
|
718
|
+
async function status(runtime, indexName) {
|
|
719
|
+
const view = requireView(runtime, indexName);
|
|
720
|
+
return {
|
|
721
|
+
indexName,
|
|
722
|
+
setting: view.state?.querySuggestions ?? null,
|
|
723
|
+
lastBuild: await runtime.builds.lastBuild(indexName),
|
|
724
|
+
nextBuildAt: runtime.timers.nextBuildAt(indexName),
|
|
725
|
+
suggestionIndexes: [...markedSuggestionIndexes(runtime.host, indexName).values()].sort()
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
function previewLimitOf(limit) {
|
|
729
|
+
if (limit === void 0) return DEFAULT_PREVIEW_LIMIT;
|
|
730
|
+
if (typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_PREVIEW_LIMIT) {
|
|
731
|
+
invalidSetting(`The preview limit must be a whole number from 1 to ${MAX_PREVIEW_LIMIT}`);
|
|
732
|
+
}
|
|
733
|
+
return limit;
|
|
734
|
+
}
|
|
735
|
+
async function preview(runtime, indexName, limit) {
|
|
736
|
+
const max = previewLimitOf(limit);
|
|
737
|
+
runtime.enabledSetting(indexName);
|
|
738
|
+
const scrambler = runtime.scrambler;
|
|
739
|
+
if (scrambler === null) invalidSetting("This engine has no query suggestions key, so it holds no counts");
|
|
740
|
+
const context = {
|
|
741
|
+
host: runtime.host,
|
|
742
|
+
scrambler,
|
|
743
|
+
saveDay: (name) => runtime.saveDay(name),
|
|
744
|
+
withWriteLock: (name, work) => runtime.withWriteLock(name, work),
|
|
745
|
+
isStopping: () => runtime.isStopping
|
|
746
|
+
};
|
|
747
|
+
return previewBuild(context, indexName, max);
|
|
748
|
+
}
|
|
749
|
+
async function banPhrases(runtime, indexName, next) {
|
|
750
|
+
requireSourceView(runtime, indexName);
|
|
751
|
+
let added = [];
|
|
752
|
+
let stored = [];
|
|
753
|
+
await runtime.host.updateState(indexName, (current) => {
|
|
754
|
+
requireSourceView(runtime, indexName);
|
|
755
|
+
const previous = current?.bannedQueries ?? [];
|
|
756
|
+
stored = next(previous);
|
|
757
|
+
const before = new Set(previous);
|
|
758
|
+
added = stored.filter((phrase) => !before.has(phrase));
|
|
759
|
+
return withState(current, { bannedQueries: stored });
|
|
760
|
+
});
|
|
761
|
+
return { stored, added };
|
|
762
|
+
}
|
|
763
|
+
async function removeFromSuggestions(runtime, indexName, phrases) {
|
|
764
|
+
let holders = 0;
|
|
765
|
+
if (phrases.length === 0) return holders;
|
|
766
|
+
for (const name of markedSuggestionIndexes(runtime.host, indexName).values()) {
|
|
767
|
+
if (await removeSuggestions(runtime.host, name, phrases) > 0) holders += 1;
|
|
768
|
+
}
|
|
769
|
+
return holders;
|
|
770
|
+
}
|
|
771
|
+
function updateBans(runtime, indexName, raw) {
|
|
772
|
+
const phrases = banListFromInput(raw);
|
|
773
|
+
return runtime.withWriteLock(indexName, async () => {
|
|
774
|
+
const { stored, added } = await banPhrases(runtime, indexName, () => phrases);
|
|
775
|
+
await removeFromSuggestions(runtime, indexName, added);
|
|
776
|
+
return stored;
|
|
777
|
+
});
|
|
778
|
+
}
|
|
779
|
+
function removeQuery(runtime, indexName, raw) {
|
|
780
|
+
const phrase = normaliseBan(raw);
|
|
781
|
+
return runtime.withWriteLock(indexName, async () => {
|
|
782
|
+
await banPhrases(runtime, indexName, (current) => unionOfBans(current, [phrase]));
|
|
783
|
+
return removeFromSuggestions(runtime, indexName, [phrase]);
|
|
784
|
+
});
|
|
785
|
+
}
|
|
786
|
+
function manualCountOf(count) {
|
|
787
|
+
if (count === void 0) return 0;
|
|
788
|
+
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0 || count > MAX_MANUAL_SUGGESTION_COUNT) {
|
|
789
|
+
invalidSetting(`A suggestion's count must be a whole number from 0 to ${MAX_MANUAL_SUGGESTION_COUNT}`);
|
|
790
|
+
}
|
|
791
|
+
return count;
|
|
792
|
+
}
|
|
793
|
+
function addSuggestion(runtime, indexName, raw) {
|
|
794
|
+
const { group, query, count } = raw ?? {};
|
|
795
|
+
if (!isGroup(group)) invalidSetting(`A suggestion needs a group of ${GROUP_FORMAT}`);
|
|
796
|
+
if (typeof query !== "string") invalidSetting("A suggestion needs its query as a string");
|
|
797
|
+
const documentCount = manualCountOf(count);
|
|
798
|
+
return runtime.withWriteLock(indexName, async () => {
|
|
799
|
+
const view = requireSourceView(runtime, indexName);
|
|
800
|
+
const setting = resolveSetting(view.state?.querySuggestions ?? { enabled: false, fields: [] });
|
|
801
|
+
const phrase = acceptedPhrase(query, setting);
|
|
802
|
+
if (phrase === null) invalidSetting("The suggestion is not a phrase that the setting of the index accepts");
|
|
803
|
+
if ((view.state?.bannedQueries ?? []).includes(phrase)) {
|
|
804
|
+
invalidSetting("The ban list of the index holds this phrase", { indexName });
|
|
805
|
+
}
|
|
806
|
+
const marked = markedSuggestionIndexes(runtime.host, indexName);
|
|
807
|
+
if (!marked.has(group) && marked.size >= setting.maxGroups) {
|
|
808
|
+
invalidSetting(`Index "${indexName}" already has ${setting.maxGroups} groups`, { indexName });
|
|
809
|
+
}
|
|
810
|
+
const name = await ensureSuggestionsIndex(runtime.host, indexName, group);
|
|
811
|
+
const document = suggestionDocument(phrase, documentCount, ORIGIN_MANUAL);
|
|
812
|
+
if (await runtime.host.engine.has(name, phrase)) await runtime.host.engine.update(name, phrase, document);
|
|
813
|
+
else await runtime.host.engine.insert(name, document);
|
|
814
|
+
return name;
|
|
815
|
+
});
|
|
816
|
+
}
|
|
817
|
+
function querySuggestionsOf(runtime) {
|
|
818
|
+
return {
|
|
819
|
+
recordQuery: async (indexName, query) => (await runtime()).record(indexName, query),
|
|
820
|
+
updateSetting: async (indexName, setting) => updateSetting(await runtime(), indexName, setting),
|
|
821
|
+
status: async (indexName) => status(await runtime(), indexName),
|
|
822
|
+
build: async (indexName) => (await runtime()).builds.start(indexName, "caller"),
|
|
823
|
+
preview: async (indexName, limit) => preview(await runtime(), indexName, limit),
|
|
824
|
+
bans: async (indexName) => [...requireView(await runtime(), indexName).state?.bannedQueries ?? []],
|
|
825
|
+
updateBans: async (indexName, phrases) => updateBans(await runtime(), indexName, phrases),
|
|
826
|
+
removeQuery: async (indexName, query) => removeQuery(await runtime(), indexName, query),
|
|
827
|
+
addSuggestion: async (indexName, suggestion) => addSuggestion(await runtime(), indexName, suggestion),
|
|
828
|
+
suggestionsIndexName: (indexName, group) => {
|
|
829
|
+
if (!isGroup(group)) invalidSetting(`A group holds ${GROUP_FORMAT}`);
|
|
830
|
+
return suggestionsIndexName(indexName, group);
|
|
831
|
+
}
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
// src/query-suggestions/local-host.ts
|
|
836
|
+
var SUGGESTIONS_SCHEMA = { query: "string", count: "number", origin: "enum" };
|
|
837
|
+
async function* noRemoteHistory() {
|
|
838
|
+
}
|
|
839
|
+
function persistState(core, indexName, next) {
|
|
840
|
+
const entry = core.requireIndex(indexName);
|
|
841
|
+
const previous = entry.querySuggestions;
|
|
842
|
+
entry.querySuggestions = next;
|
|
843
|
+
const persisted = core.durability?.manager.persistMetadata(indexName) ?? Promise.resolve();
|
|
844
|
+
return persisted.then(
|
|
845
|
+
() => next,
|
|
846
|
+
(error) => {
|
|
847
|
+
if (core.indexRegistry.get(indexName) === entry) entry.querySuggestions = previous;
|
|
848
|
+
throw error;
|
|
849
|
+
}
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
function createLocalHost(core, engine, config) {
|
|
853
|
+
const files = historyFilesOf(core.durability?.manager.historyStorage?.() ?? null);
|
|
854
|
+
return {
|
|
855
|
+
engine,
|
|
856
|
+
nodeId: "",
|
|
857
|
+
files,
|
|
858
|
+
clustered: false,
|
|
859
|
+
view(indexName) {
|
|
860
|
+
const entry = core.indexRegistry.get(indexName);
|
|
861
|
+
if (entry === void 0) return null;
|
|
862
|
+
return {
|
|
863
|
+
state: entry.querySuggestions,
|
|
864
|
+
schema: entry.config.schema,
|
|
865
|
+
language: entry.language.name,
|
|
866
|
+
indexUuid: entry.indexUuid,
|
|
867
|
+
hasSync: entry.sync !== null
|
|
868
|
+
};
|
|
869
|
+
},
|
|
870
|
+
indexNames: () => [...core.indexRegistry.keys()],
|
|
871
|
+
updateState(indexName, change) {
|
|
872
|
+
return withOpenIndex(
|
|
873
|
+
core,
|
|
874
|
+
indexName,
|
|
875
|
+
() => oneSyncChangeAtATime(
|
|
876
|
+
core.indexRegistry,
|
|
877
|
+
indexName,
|
|
878
|
+
() => persistState(core, indexName, change(core.requireIndex(indexName).querySuggestions))
|
|
879
|
+
)
|
|
880
|
+
);
|
|
881
|
+
},
|
|
882
|
+
async createSuggestionsIndex(name, source, language) {
|
|
883
|
+
const indexConfig = {
|
|
884
|
+
schema: { ...SUGGESTIONS_SCHEMA },
|
|
885
|
+
language,
|
|
886
|
+
stopWords: /* @__PURE__ */ new Set(),
|
|
887
|
+
partitions: { maxPartitions: 1 }
|
|
888
|
+
};
|
|
889
|
+
await emitWriteEventAfter(
|
|
890
|
+
core,
|
|
891
|
+
name,
|
|
892
|
+
createEngineIndex(core, config, name, indexConfig, {
|
|
893
|
+
appliesCreationLimits: true,
|
|
894
|
+
querySuggestions: { querySuggestionsSource: source }
|
|
895
|
+
})
|
|
896
|
+
);
|
|
897
|
+
},
|
|
898
|
+
remoteHistory: noRemoteHistory,
|
|
899
|
+
async readBuildRecord(indexName) {
|
|
900
|
+
const bytes = await files.read(buildRecordKey(indexName));
|
|
901
|
+
if (bytes === null || bytes === "too-long") return null;
|
|
902
|
+
return decodeBuildRecordFile(bytes);
|
|
903
|
+
},
|
|
904
|
+
async writeBuildRecord(indexName, record) {
|
|
905
|
+
await files.write(buildRecordKey(indexName), encodeBuildRecordFile(record));
|
|
906
|
+
},
|
|
907
|
+
mayBuild: () => true,
|
|
908
|
+
emit(name, payload) {
|
|
909
|
+
if (emitEngineEvent(core.eventHandlers, name, payload) > 0 || name !== "querySuggestionsWarning") return;
|
|
910
|
+
console.warn("Popular query suggestions warning:", payload);
|
|
911
|
+
}
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// src/query-suggestions/build-runner.ts
|
|
916
|
+
function errorCodeOf(error) {
|
|
917
|
+
return error instanceof NarsilError ? error.code : ErrorCodes.WORKER_ACTION_FAILED;
|
|
918
|
+
}
|
|
919
|
+
function runningRecordOf(identity) {
|
|
920
|
+
return {
|
|
921
|
+
...identity,
|
|
922
|
+
status: "running",
|
|
923
|
+
published: 0,
|
|
924
|
+
removed: 0,
|
|
925
|
+
discarded: 0,
|
|
926
|
+
sourcesDigest: new Uint8Array(SOURCES_DIGEST_BYTES)
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
function newBuild(trigger, status2) {
|
|
930
|
+
return {
|
|
931
|
+
buildId: globalThis.crypto.randomUUID(),
|
|
932
|
+
trigger,
|
|
933
|
+
status: status2,
|
|
934
|
+
startedAt: Date.now(),
|
|
935
|
+
endedAt: null,
|
|
936
|
+
published: 0,
|
|
937
|
+
removed: 0,
|
|
938
|
+
discarded: 0,
|
|
939
|
+
errorCode: null
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
var BuildRunner = class {
|
|
943
|
+
runtime;
|
|
944
|
+
running = /* @__PURE__ */ new Map();
|
|
945
|
+
finished = /* @__PURE__ */ new Map();
|
|
946
|
+
constructor(runtime) {
|
|
947
|
+
this.runtime = runtime;
|
|
948
|
+
}
|
|
949
|
+
async lastBuild(indexName) {
|
|
950
|
+
const running = this.running.get(indexName);
|
|
951
|
+
if (running !== void 0) return { ...running.build };
|
|
952
|
+
const finished = this.finished.get(indexName);
|
|
953
|
+
if (finished !== void 0) return { ...finished };
|
|
954
|
+
const record = await this.runtime.host.readBuildRecord(indexName).catch(() => null);
|
|
955
|
+
return record === null ? null : publicBuildOf(record);
|
|
956
|
+
}
|
|
957
|
+
isRunning(indexName) {
|
|
958
|
+
return this.running.has(indexName);
|
|
959
|
+
}
|
|
960
|
+
start(indexName, trigger) {
|
|
961
|
+
return this.begin(indexName, trigger).done;
|
|
962
|
+
}
|
|
963
|
+
begin(indexName, trigger) {
|
|
964
|
+
this.runtime.enabledSetting(indexName);
|
|
965
|
+
if (this.runtime.scrambler === null) {
|
|
966
|
+
throw new NarsilError(
|
|
967
|
+
ErrorCodes.CONFIG_INVALID,
|
|
968
|
+
"This engine has no query suggestions key, so it builds no suggestions",
|
|
969
|
+
{ indexName }
|
|
970
|
+
);
|
|
971
|
+
}
|
|
972
|
+
if (this.runtime.isStopping) throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "The engine is shutting down");
|
|
973
|
+
if (this.running.has(indexName)) {
|
|
974
|
+
const skipped = { ...newBuild(trigger, "skipped"), endedAt: Date.now() };
|
|
975
|
+
this.emitPhase(indexName, skipped);
|
|
976
|
+
return { build: skipped, started: Promise.resolve(), done: Promise.resolve(skipped) };
|
|
977
|
+
}
|
|
978
|
+
const build = newBuild(trigger, "running");
|
|
979
|
+
let markStarted = () => void 0;
|
|
980
|
+
const started = new Promise((resolve) => {
|
|
981
|
+
markStarted = resolve;
|
|
982
|
+
});
|
|
983
|
+
const done = this.run(indexName, build, markStarted);
|
|
984
|
+
void done.finally(markStarted);
|
|
985
|
+
const running = { build, started, done };
|
|
986
|
+
this.running.set(indexName, running);
|
|
987
|
+
void done.finally(() => {
|
|
988
|
+
if (this.running.get(indexName)?.build === build) this.running.delete(indexName);
|
|
989
|
+
});
|
|
990
|
+
return running;
|
|
991
|
+
}
|
|
992
|
+
async settled() {
|
|
993
|
+
await Promise.allSettled([...this.running.values()].map((running) => running.done));
|
|
994
|
+
}
|
|
995
|
+
forget(indexName) {
|
|
996
|
+
this.finished.delete(indexName);
|
|
997
|
+
}
|
|
998
|
+
emitPhase(indexName, build) {
|
|
999
|
+
const phase = build.status === "running" ? "started" : build.status;
|
|
1000
|
+
this.runtime.host.emit("querySuggestionsBuild", {
|
|
1001
|
+
indexName,
|
|
1002
|
+
buildId: build.buildId,
|
|
1003
|
+
phase,
|
|
1004
|
+
trigger: build.trigger,
|
|
1005
|
+
published: build.published,
|
|
1006
|
+
removed: build.removed,
|
|
1007
|
+
discarded: build.discarded,
|
|
1008
|
+
...build.errorCode !== null ? { errorCode: build.errorCode } : {}
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
async lastCompleted(indexName) {
|
|
1012
|
+
const record = await this.runtime.host.readBuildRecord(indexName).catch(() => null);
|
|
1013
|
+
return record !== null && (record.status === "completed" || record.status === "skipped") ? record : null;
|
|
1014
|
+
}
|
|
1015
|
+
async run(indexName, build, markStarted) {
|
|
1016
|
+
await Promise.resolve();
|
|
1017
|
+
this.emitPhase(indexName, build);
|
|
1018
|
+
const runtime = this.runtime;
|
|
1019
|
+
let record;
|
|
1020
|
+
const identity = { buildId: build.buildId, trigger: build.trigger, startedAt: build.startedAt };
|
|
1021
|
+
try {
|
|
1022
|
+
const scrambler = runtime.scrambler;
|
|
1023
|
+
if (scrambler === null)
|
|
1024
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This engine has no query suggestions key");
|
|
1025
|
+
const lastCompleted = await this.lastCompleted(indexName);
|
|
1026
|
+
if (runtime.host.clustered) await runtime.host.writeBuildRecord(indexName, runningRecordOf(identity));
|
|
1027
|
+
markStarted();
|
|
1028
|
+
const outcome = await runBuild(
|
|
1029
|
+
{
|
|
1030
|
+
host: runtime.host,
|
|
1031
|
+
scrambler,
|
|
1032
|
+
saveDay: (name) => runtime.saveDay(name),
|
|
1033
|
+
withWriteLock: (name, work) => runtime.withWriteLock(name, work),
|
|
1034
|
+
isStopping: () => runtime.isStopping
|
|
1035
|
+
},
|
|
1036
|
+
indexName,
|
|
1037
|
+
lastCompleted,
|
|
1038
|
+
build.startedAt
|
|
1039
|
+
);
|
|
1040
|
+
record = { ...identity, ...outcome, endedAt: Date.now() };
|
|
1041
|
+
} catch (error) {
|
|
1042
|
+
record = {
|
|
1043
|
+
...identity,
|
|
1044
|
+
status: "failed",
|
|
1045
|
+
endedAt: Date.now(),
|
|
1046
|
+
published: 0,
|
|
1047
|
+
removed: 0,
|
|
1048
|
+
discarded: 0,
|
|
1049
|
+
sourcesDigest: new Uint8Array(SOURCES_DIGEST_BYTES),
|
|
1050
|
+
errorCode: errorCodeOf(error)
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
const finalRecord = record;
|
|
1054
|
+
await runtime.host.writeBuildRecord(indexName, finalRecord).catch((error) => {
|
|
1055
|
+
runtime.host.emit("querySuggestionsWarning", {
|
|
1056
|
+
indexName,
|
|
1057
|
+
reason: "save-failed",
|
|
1058
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1059
|
+
});
|
|
1060
|
+
});
|
|
1061
|
+
const finished = publicBuildOf(finalRecord);
|
|
1062
|
+
this.finished.set(indexName, finished);
|
|
1063
|
+
this.emitPhase(indexName, finished);
|
|
1064
|
+
return finished;
|
|
1065
|
+
}
|
|
1066
|
+
};
|
|
1067
|
+
|
|
1068
|
+
// src/query-suggestions/scramble.ts
|
|
1069
|
+
var INNER_PAD = 54;
|
|
1070
|
+
var OUTER_PAD = 92;
|
|
1071
|
+
function padState(digest, key, pad, words) {
|
|
1072
|
+
const block = new Uint8Array(digest.SHA256_BLOCK_BYTES);
|
|
1073
|
+
for (let i = 0; i < digest.SHA256_BLOCK_BYTES; i++) block[i] = (key[i] ?? 0) ^ pad;
|
|
1074
|
+
const state = Uint32Array.from(digest.SHA256_INITIAL_STATE);
|
|
1075
|
+
digest.compressSha256Block(state, new DataView(block.buffer), 0, words);
|
|
1076
|
+
return state;
|
|
1077
|
+
}
|
|
1078
|
+
function createPersonScrambler(key, digest) {
|
|
1079
|
+
const blockBytes = digest.SHA256_BLOCK_BYTES;
|
|
1080
|
+
const words = new Uint32Array(64);
|
|
1081
|
+
const blockKey = key.byteLength > blockBytes ? digest.sha256(key) : key;
|
|
1082
|
+
const innerState = padState(digest, blockKey, INNER_PAD, words);
|
|
1083
|
+
const outerState = padState(digest, blockKey, OUTER_PAD, words);
|
|
1084
|
+
const working = new Uint32Array(8);
|
|
1085
|
+
const encoder = new TextEncoder();
|
|
1086
|
+
function digestAfter(prefixState, message) {
|
|
1087
|
+
working.set(prefixState);
|
|
1088
|
+
const view = digest.paddedSha256Message(message, blockBytes);
|
|
1089
|
+
for (let block = 0; block < view.byteLength; block += blockBytes) {
|
|
1090
|
+
digest.compressSha256Block(working, view, block, words);
|
|
1091
|
+
}
|
|
1092
|
+
return digest.sha256Digest(working);
|
|
1093
|
+
}
|
|
1094
|
+
function hmac(message) {
|
|
1095
|
+
return digestAfter(outerState, digestAfter(innerState, message));
|
|
1096
|
+
}
|
|
1097
|
+
return {
|
|
1098
|
+
fingerprint: hmac(encoder.encode(KEY_FINGERPRINT_LABEL)).slice(0, KEY_FINGERPRINT_BYTES),
|
|
1099
|
+
scramble: (person) => hmac(encoder.encode(person)).slice(0, SCRAMBLED_ID_BYTES)
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
// src/query-suggestions/timers.ts
|
|
1104
|
+
function unrefTimer(timer) {
|
|
1105
|
+
timer.unref?.();
|
|
1106
|
+
return timer;
|
|
1107
|
+
}
|
|
1108
|
+
var SuggestionTimers = class {
|
|
1109
|
+
runtime;
|
|
1110
|
+
saves = /* @__PURE__ */ new Map();
|
|
1111
|
+
schedules = /* @__PURE__ */ new Map();
|
|
1112
|
+
sweep = null;
|
|
1113
|
+
midnight = null;
|
|
1114
|
+
stopped = false;
|
|
1115
|
+
constructor(runtime) {
|
|
1116
|
+
this.runtime = runtime;
|
|
1117
|
+
}
|
|
1118
|
+
saveSoon(indexName, intervalMs) {
|
|
1119
|
+
if (this.stopped || this.saves.has(indexName)) return;
|
|
1120
|
+
const timer = setTimeout(() => {
|
|
1121
|
+
this.saves.delete(indexName);
|
|
1122
|
+
void this.runtime.saveDay(indexName).then(() => {
|
|
1123
|
+
if (this.runtime.days?.hasUnsaved(indexName) === true) this.saveSoon(indexName, intervalMs);
|
|
1124
|
+
}).catch(() => void 0);
|
|
1125
|
+
}, intervalMs);
|
|
1126
|
+
this.saves.set(indexName, unrefTimer(timer));
|
|
1127
|
+
}
|
|
1128
|
+
forgetSave(indexName) {
|
|
1129
|
+
const timer = this.saves.get(indexName);
|
|
1130
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
1131
|
+
this.saves.delete(indexName);
|
|
1132
|
+
}
|
|
1133
|
+
forgetIndex(indexName) {
|
|
1134
|
+
this.forgetSave(indexName);
|
|
1135
|
+
const timer = this.schedules.get(indexName);
|
|
1136
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
1137
|
+
this.schedules.delete(indexName);
|
|
1138
|
+
}
|
|
1139
|
+
async start() {
|
|
1140
|
+
await removeRestoreLeftovers(this.runtime.host.files).catch(() => void 0);
|
|
1141
|
+
await this.sweepNow();
|
|
1142
|
+
this.sweep = setInterval(() => void this.sweepNow(), HISTORY_SWEEP_INTERVAL_MS);
|
|
1143
|
+
this.sweep.unref?.();
|
|
1144
|
+
this.armMidnight();
|
|
1145
|
+
for (const indexName of this.runtime.host.indexNames()) {
|
|
1146
|
+
if (this.runtime.settingOf(indexName)?.enabled !== true) continue;
|
|
1147
|
+
await this.startMissedBuild(indexName);
|
|
1148
|
+
this.rearm(indexName);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
async catchUpMissedBuilds() {
|
|
1152
|
+
for (const indexName of this.runtime.host.indexNames()) {
|
|
1153
|
+
if (this.stopped) return;
|
|
1154
|
+
if (this.runtime.settingOf(indexName)?.enabled === true) await this.startMissedBuild(indexName);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
stop() {
|
|
1158
|
+
this.stopped = true;
|
|
1159
|
+
for (const timer of this.saves.values()) clearTimeout(timer);
|
|
1160
|
+
for (const timer of this.schedules.values()) clearTimeout(timer);
|
|
1161
|
+
this.saves.clear();
|
|
1162
|
+
this.schedules.clear();
|
|
1163
|
+
if (this.sweep !== null) clearInterval(this.sweep);
|
|
1164
|
+
if (this.midnight !== null) clearTimeout(this.midnight);
|
|
1165
|
+
this.sweep = null;
|
|
1166
|
+
this.midnight = null;
|
|
1167
|
+
}
|
|
1168
|
+
nextBuildAt(indexName) {
|
|
1169
|
+
const setting = this.runtime.settingOf(indexName);
|
|
1170
|
+
if (setting?.enabled !== true || this.runtime.scrambler === null) return null;
|
|
1171
|
+
const resolved = resolveSetting(setting);
|
|
1172
|
+
if (resolved.schedule === SCHEDULE_OFF) return null;
|
|
1173
|
+
const schedule = parseSchedule(resolved.schedule);
|
|
1174
|
+
return schedule === null ? null : nextMatchAfter(schedule, resolved.timeZone, Date.now());
|
|
1175
|
+
}
|
|
1176
|
+
rearm(indexName) {
|
|
1177
|
+
const previous = this.schedules.get(indexName);
|
|
1178
|
+
if (previous !== void 0) clearTimeout(previous);
|
|
1179
|
+
this.schedules.delete(indexName);
|
|
1180
|
+
if (this.stopped) return;
|
|
1181
|
+
const at = this.nextBuildAt(indexName);
|
|
1182
|
+
if (at !== null) this.arm(indexName, at);
|
|
1183
|
+
}
|
|
1184
|
+
arm(indexName, at) {
|
|
1185
|
+
const delay = Math.min(Math.max(at - Date.now(), 0), MAX_TIMER_DELAY_MS);
|
|
1186
|
+
const timer = setTimeout(() => {
|
|
1187
|
+
this.schedules.delete(indexName);
|
|
1188
|
+
if (this.stopped) return;
|
|
1189
|
+
if (Date.now() < at) {
|
|
1190
|
+
this.arm(indexName, at);
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
this.startBuild(indexName, "schedule");
|
|
1194
|
+
this.rearm(indexName);
|
|
1195
|
+
}, delay);
|
|
1196
|
+
this.schedules.set(indexName, unrefTimer(timer));
|
|
1197
|
+
}
|
|
1198
|
+
startBuild(indexName, trigger) {
|
|
1199
|
+
if (!this.runtime.host.mayBuild()) return;
|
|
1200
|
+
try {
|
|
1201
|
+
void this.runtime.builds.start(indexName, trigger).catch(() => void 0);
|
|
1202
|
+
} catch {
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
async startMissedBuild(indexName) {
|
|
1206
|
+
const setting = this.runtime.settingOf(indexName);
|
|
1207
|
+
if (setting === null) return;
|
|
1208
|
+
const resolved = resolveSetting(setting);
|
|
1209
|
+
if (resolved.schedule === SCHEDULE_OFF) return;
|
|
1210
|
+
const schedule = parseSchedule(resolved.schedule);
|
|
1211
|
+
if (schedule === null) return;
|
|
1212
|
+
const latest = latestMatchAtOrBefore(schedule, resolved.timeZone, Date.now());
|
|
1213
|
+
if (latest === null) return;
|
|
1214
|
+
const record = await this.runtime.host.readBuildRecord(indexName).catch(() => null);
|
|
1215
|
+
const completedSince = record !== null && (record.status === "completed" || record.status === "skipped") && record.startedAt >= latest;
|
|
1216
|
+
if (!completedSince) this.startBuild(indexName, "missed");
|
|
1217
|
+
}
|
|
1218
|
+
armMidnight() {
|
|
1219
|
+
if (this.stopped) return;
|
|
1220
|
+
const now = Date.now();
|
|
1221
|
+
const delay = Math.min(Math.floor(now / DAY_MS + 1) * DAY_MS - now, MAX_TIMER_DELAY_MS);
|
|
1222
|
+
this.midnight = unrefTimer(
|
|
1223
|
+
setTimeout(() => {
|
|
1224
|
+
this.midnight = null;
|
|
1225
|
+
void this.retireDays().finally(() => this.armMidnight());
|
|
1226
|
+
}, delay)
|
|
1227
|
+
);
|
|
1228
|
+
}
|
|
1229
|
+
async retireDays() {
|
|
1230
|
+
const days = this.runtime.days;
|
|
1231
|
+
if (days === null) return;
|
|
1232
|
+
const today = dayOf(Date.now());
|
|
1233
|
+
for (const indexName of days.indexes()) await days.retire(indexName, today).catch(() => void 0);
|
|
1234
|
+
}
|
|
1235
|
+
async sweepNow() {
|
|
1236
|
+
const { host } = this.runtime;
|
|
1237
|
+
const today = Date.now();
|
|
1238
|
+
for (const indexName of host.indexNames()) {
|
|
1239
|
+
const setting = this.runtime.settingOf(indexName);
|
|
1240
|
+
try {
|
|
1241
|
+
if (setting?.enabled !== true) {
|
|
1242
|
+
if (host.view(indexName)?.state?.querySuggestionsSource === void 0) {
|
|
1243
|
+
await removeHistoryFolder(host.files, indexName);
|
|
1244
|
+
}
|
|
1245
|
+
continue;
|
|
1246
|
+
}
|
|
1247
|
+
const cutoff = dayOf(today - resolveSetting(setting).retentionDays * DAY_MS);
|
|
1248
|
+
for (const key of await listHistoryFolder(host.files, indexName)) {
|
|
1249
|
+
const file = historyFileOf(indexName, key);
|
|
1250
|
+
if (file !== null && file.day < cutoff) await host.files.remove(key);
|
|
1251
|
+
}
|
|
1252
|
+
} catch {
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
};
|
|
1257
|
+
|
|
1258
|
+
// src/query-suggestions/runtime.ts
|
|
1259
|
+
function keyBytesOf(key) {
|
|
1260
|
+
const bytes = typeof key === "string" ? new TextEncoder().encode(key) : new Uint8Array(key);
|
|
1261
|
+
if (bytes.byteLength < MIN_KEY_BYTES) {
|
|
1262
|
+
throw new NarsilError(
|
|
1263
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1264
|
+
`The query suggestions key must hold at least ${MIN_KEY_BYTES} bytes, and it holds ${bytes.byteLength}`,
|
|
1265
|
+
{ bytes: bytes.byteLength }
|
|
1266
|
+
);
|
|
1267
|
+
}
|
|
1268
|
+
return bytes;
|
|
1269
|
+
}
|
|
1270
|
+
function notFound(indexName) {
|
|
1271
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, { indexName });
|
|
1272
|
+
}
|
|
1273
|
+
function notEnabled(indexName) {
|
|
1274
|
+
return new NarsilError(
|
|
1275
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
1276
|
+
`Index "${indexName}" has popular query suggestions off`,
|
|
1277
|
+
{ indexName }
|
|
1278
|
+
);
|
|
1279
|
+
}
|
|
1280
|
+
async function createQuerySuggestionsRuntime(host, key) {
|
|
1281
|
+
const scrambler = key === null ? null : createPersonScrambler(key, await loadDigest());
|
|
1282
|
+
return new QuerySuggestionsRuntime(host, scrambler);
|
|
1283
|
+
}
|
|
1284
|
+
var QuerySuggestionsRuntime = class {
|
|
1285
|
+
host;
|
|
1286
|
+
scrambler;
|
|
1287
|
+
days;
|
|
1288
|
+
builds;
|
|
1289
|
+
timers;
|
|
1290
|
+
locks = /* @__PURE__ */ new Map();
|
|
1291
|
+
stopping = false;
|
|
1292
|
+
constructor(host, scrambler) {
|
|
1293
|
+
this.host = host;
|
|
1294
|
+
this.scrambler = scrambler;
|
|
1295
|
+
this.days = this.scrambler === null ? null : new DayStore(host, this.scrambler);
|
|
1296
|
+
this.builds = new BuildRunner(this);
|
|
1297
|
+
this.timers = new SuggestionTimers(this);
|
|
1298
|
+
}
|
|
1299
|
+
get isStopping() {
|
|
1300
|
+
return this.stopping;
|
|
1301
|
+
}
|
|
1302
|
+
enabledSetting(indexName) {
|
|
1303
|
+
const view = this.host.view(indexName);
|
|
1304
|
+
if (view === null) throw notFound(indexName);
|
|
1305
|
+
const setting = view.state?.querySuggestions;
|
|
1306
|
+
if (setting === void 0 || !setting.enabled) throw notEnabled(indexName);
|
|
1307
|
+
return resolveSetting(setting);
|
|
1308
|
+
}
|
|
1309
|
+
settingOf(indexName) {
|
|
1310
|
+
return this.host.view(indexName)?.state?.querySuggestions ?? null;
|
|
1311
|
+
}
|
|
1312
|
+
withWriteLock(indexName, work) {
|
|
1313
|
+
const previous = this.locks.get(indexName) ?? Promise.resolve();
|
|
1314
|
+
const next = previous.then(work, work);
|
|
1315
|
+
const settled = next.catch(() => void 0);
|
|
1316
|
+
this.locks.set(indexName, settled);
|
|
1317
|
+
void settled.then(() => {
|
|
1318
|
+
if (this.locks.get(indexName) === settled) this.locks.delete(indexName);
|
|
1319
|
+
});
|
|
1320
|
+
return next;
|
|
1321
|
+
}
|
|
1322
|
+
async saveDay(indexName) {
|
|
1323
|
+
await this.days?.save(indexName);
|
|
1324
|
+
}
|
|
1325
|
+
groupAllowed(indexName, group, maxGroups, known) {
|
|
1326
|
+
for (const marked of markedSuggestionIndexes(this.host, indexName).keys()) known.add(marked);
|
|
1327
|
+
return known.has(group) || known.size < maxGroups;
|
|
1328
|
+
}
|
|
1329
|
+
async record(indexName, input) {
|
|
1330
|
+
if (this.stopping) throw new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "The engine is shutting down");
|
|
1331
|
+
const setting = this.enabledSetting(indexName);
|
|
1332
|
+
if (this.days === null) return { recorded: false, outcome: "no-key" };
|
|
1333
|
+
const { query, person, group } = input ?? {};
|
|
1334
|
+
if (typeof query !== "string" || !isPerson(person) || !isGroup(group)) {
|
|
1335
|
+
return { recorded: false, outcome: "invalid" };
|
|
1336
|
+
}
|
|
1337
|
+
const phrase = acceptedPhrase(query, setting);
|
|
1338
|
+
if (phrase === null) return { recorded: false, outcome: "invalid" };
|
|
1339
|
+
const isBanned = () => (this.host.view(indexName)?.state?.bannedQueries ?? []).includes(phrase);
|
|
1340
|
+
if (isBanned()) return { recorded: false, outcome: "banned" };
|
|
1341
|
+
const outcome = await this.days.count(indexName, setting, Date.now(), {
|
|
1342
|
+
group,
|
|
1343
|
+
phrase,
|
|
1344
|
+
person,
|
|
1345
|
+
groupAllowed: (known) => this.groupAllowed(indexName, group, setting.maxGroups, known),
|
|
1346
|
+
stillEnabled: () => this.settingOf(indexName)?.enabled === true,
|
|
1347
|
+
isBanned
|
|
1348
|
+
});
|
|
1349
|
+
if (outcome === null) throw notEnabled(indexName);
|
|
1350
|
+
if (outcome === "recorded") this.timers.saveSoon(indexName, setting.saveIntervalMs);
|
|
1351
|
+
return { recorded: outcome === "recorded", outcome };
|
|
1352
|
+
}
|
|
1353
|
+
restoredHistory(indexName, setting) {
|
|
1354
|
+
if (setting?.enabled !== true || this.scrambler === null) return null;
|
|
1355
|
+
return new RestoredHistory(this, indexName, setting);
|
|
1356
|
+
}
|
|
1357
|
+
async discardHistory(indexName) {
|
|
1358
|
+
this.timers.forgetSave(indexName);
|
|
1359
|
+
await this.days?.discard(indexName);
|
|
1360
|
+
await removeHistoryFolder(this.host.files, indexName);
|
|
1361
|
+
}
|
|
1362
|
+
async onDrop(indexName) {
|
|
1363
|
+
this.timers.forgetIndex(indexName);
|
|
1364
|
+
this.builds.forget(indexName);
|
|
1365
|
+
await this.days?.discard(indexName);
|
|
1366
|
+
await removeHistoryFolder(this.host.files, indexName).catch(() => void 0);
|
|
1367
|
+
}
|
|
1368
|
+
warnWithoutKey() {
|
|
1369
|
+
if (this.scrambler !== null) return;
|
|
1370
|
+
for (const indexName of this.host.indexNames()) {
|
|
1371
|
+
if (this.settingOf(indexName)?.enabled === true) {
|
|
1372
|
+
this.host.emit("querySuggestionsWarning", { indexName, reason: "no-key" });
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
async start() {
|
|
1377
|
+
this.warnWithoutKey();
|
|
1378
|
+
if (this.scrambler === null) {
|
|
1379
|
+
await this.timers.sweepNow();
|
|
1380
|
+
return;
|
|
1381
|
+
}
|
|
1382
|
+
await this.timers.start();
|
|
1383
|
+
}
|
|
1384
|
+
settingChanged(indexName) {
|
|
1385
|
+
this.timers.rearm(indexName);
|
|
1386
|
+
}
|
|
1387
|
+
async stop() {
|
|
1388
|
+
if (this.stopping) return;
|
|
1389
|
+
this.stopping = true;
|
|
1390
|
+
this.timers.stop();
|
|
1391
|
+
await this.builds.settled();
|
|
1392
|
+
if (this.days === null) return;
|
|
1393
|
+
for (const indexName of this.days.indexes()) await this.days.save(indexName);
|
|
1394
|
+
}
|
|
1395
|
+
today() {
|
|
1396
|
+
return dayOf(Date.now());
|
|
1397
|
+
}
|
|
1398
|
+
};
|
|
1399
|
+
|
|
1400
|
+
// src/query-suggestions/start.ts
|
|
1401
|
+
var starting = /* @__PURE__ */ new WeakMap();
|
|
1402
|
+
async function startRuntime(core, engine, config) {
|
|
1403
|
+
const keySetting = config?.querySuggestions?.key;
|
|
1404
|
+
const key = keySetting === void 0 ? null : keyBytesOf(keySetting);
|
|
1405
|
+
const runtime = await createQuerySuggestionsRuntime(createLocalHost(core, engine, config), key);
|
|
1406
|
+
attachRuntime(core, runtime);
|
|
1407
|
+
installSuggestionHooks(core);
|
|
1408
|
+
await runtime.start();
|
|
1409
|
+
return runtime;
|
|
1410
|
+
}
|
|
1411
|
+
function localRuntimeOf(core, engine, config) {
|
|
1412
|
+
const attached = attachedRuntimeOf(core);
|
|
1413
|
+
if (attached !== void 0) return Promise.resolve(attached);
|
|
1414
|
+
const pending = starting.get(core);
|
|
1415
|
+
if (pending !== void 0) return pending;
|
|
1416
|
+
const started = startRuntime(core, engine, config);
|
|
1417
|
+
starting.set(core, started);
|
|
1418
|
+
void started.then(
|
|
1419
|
+
() => starting.delete(core),
|
|
1420
|
+
() => starting.delete(core)
|
|
1421
|
+
);
|
|
1422
|
+
return started;
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
// src/query-suggestions/suggestions-leaf.ts
|
|
1426
|
+
function localQuerySuggestionsOf(core, engine) {
|
|
1427
|
+
return querySuggestionsOf(() => localRuntimeOf(core, engine, void 0));
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
export { bindSuggestionsHost, localQuerySuggestionsOf, localRuntimeOf };
|