@delali/narsil 0.3.1-next.99 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
package/dist/chunk-LU2CE5AE.mjs
DELETED
|
@@ -1,507 +0,0 @@
|
|
|
1
|
-
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
2
|
-
import { createNarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
3
|
-
|
|
4
|
-
// src/core/tokenizer/constants.ts
|
|
5
|
-
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
6
|
-
var TOKEN_CACHE_SIZE_CEILING = 2e6;
|
|
7
|
-
var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
|
|
8
|
-
var NODE_TOKEN_CACHE_ENTRIES = 1e6;
|
|
9
|
-
var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
|
|
10
|
-
var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
|
|
11
|
-
var SMALL_DEVICE_MEMORY_GB = 1;
|
|
12
|
-
var MEDIUM_DEVICE_MEMORY_GB = 4;
|
|
13
|
-
var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
|
|
14
|
-
var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
|
|
15
|
-
var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
|
|
16
|
-
var DEFAULT_MIN_TOKEN_LENGTH = 1;
|
|
17
|
-
|
|
18
|
-
// src/core/tokenizer/normalize.ts
|
|
19
|
-
var DOTTED_CAPITAL_I = "\u0130";
|
|
20
|
-
var COMBINING_DOT_ABOVE = "\u0307";
|
|
21
|
-
var ARMENIAN_MARKS = /[\u055B-\u055F]/;
|
|
22
|
-
var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
|
|
23
|
-
var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
|
|
24
|
-
var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
|
|
25
|
-
var FULLWIDTH_ASCII_END = 65374;
|
|
26
|
-
var FULLWIDTH_ASCII_OFFSET = 65248;
|
|
27
|
-
var HALFWIDTH_KANA_START = 65377;
|
|
28
|
-
var HALFWIDTH_VOICED_MARK = 65438;
|
|
29
|
-
var HALFWIDTH_SEMI_VOICED_MARK = 65439;
|
|
30
|
-
var COMBINING_VOICED_MARK = "\u3099";
|
|
31
|
-
var COMBINING_SEMI_VOICED_MARK = "\u309A";
|
|
32
|
-
var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
|
|
33
|
-
function foldForm(match) {
|
|
34
|
-
if (match === DOTTED_CAPITAL_I) return "i";
|
|
35
|
-
if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
|
|
36
|
-
return "'";
|
|
37
|
-
}
|
|
38
|
-
function foldWidth(match) {
|
|
39
|
-
const codePoint = match.charCodeAt(0);
|
|
40
|
-
if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
|
|
41
|
-
if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
|
|
42
|
-
if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
|
|
43
|
-
return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
|
|
44
|
-
}
|
|
45
|
-
function isAscii(text) {
|
|
46
|
-
for (let i = 0; i < text.length; i++) {
|
|
47
|
-
if (text.charCodeAt(i) > 127) return false;
|
|
48
|
-
}
|
|
49
|
-
return true;
|
|
50
|
-
}
|
|
51
|
-
function foldWidthForms(text) {
|
|
52
|
-
if (!WIDTH_FORMS.test(text)) return text;
|
|
53
|
-
return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
|
|
54
|
-
}
|
|
55
|
-
function normalizeForSplitting(text) {
|
|
56
|
-
if (isAscii(text)) return text.toLowerCase();
|
|
57
|
-
return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
|
|
58
|
-
}
|
|
59
|
-
function stripDiacritics(text) {
|
|
60
|
-
return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// src/core/tokenizer/cache.ts
|
|
64
|
-
function clampCacheSize(entries) {
|
|
65
|
-
return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
|
|
66
|
-
}
|
|
67
|
-
function computeDefaultCacheSize() {
|
|
68
|
-
try {
|
|
69
|
-
if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
|
|
70
|
-
const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
71
|
-
if (constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER) {
|
|
72
|
-
const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
|
|
73
|
-
return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
|
|
74
|
-
}
|
|
75
|
-
return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
|
|
76
|
-
}
|
|
77
|
-
if (typeof navigator !== "undefined") {
|
|
78
|
-
const mem = navigator.deviceMemory;
|
|
79
|
-
if (typeof mem === "number" && mem > 0) {
|
|
80
|
-
let entries;
|
|
81
|
-
if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
|
|
82
|
-
else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
|
|
83
|
-
else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
|
|
84
|
-
return clampCacheSize(entries);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
} catch {
|
|
88
|
-
return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
|
|
89
|
-
}
|
|
90
|
-
return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
|
|
91
|
-
}
|
|
92
|
-
var normalizationCache = /* @__PURE__ */ new Map();
|
|
93
|
-
var normalizationCacheSize = 0;
|
|
94
|
-
var maxCacheSize = computeDefaultCacheSize();
|
|
95
|
-
var cachedLangName = "";
|
|
96
|
-
var cachedFlags = "";
|
|
97
|
-
var cachedBucket = null;
|
|
98
|
-
function configureNormalizationCache(maxSize) {
|
|
99
|
-
if (!Number.isFinite(maxSize)) {
|
|
100
|
-
throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
|
|
101
|
-
received: maxSize
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
if (maxSize < 0) {
|
|
105
|
-
throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
|
|
106
|
-
received: maxSize
|
|
107
|
-
});
|
|
108
|
-
}
|
|
109
|
-
if (maxSize === 0) {
|
|
110
|
-
throw createNarsilError(
|
|
111
|
-
ErrorCodes.CONFIG_INVALID,
|
|
112
|
-
"tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
|
|
113
|
-
{
|
|
114
|
-
received: maxSize
|
|
115
|
-
}
|
|
116
|
-
);
|
|
117
|
-
}
|
|
118
|
-
maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
|
|
119
|
-
if (normalizationCacheSize > maxCacheSize) {
|
|
120
|
-
evictOldestEntries(normalizationCacheSize - maxCacheSize);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
function getCacheBucket(language, flags) {
|
|
124
|
-
const langKey = `${language.name}@${language.revision}`;
|
|
125
|
-
if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
|
|
126
|
-
const key = `${langKey}:${flags}`;
|
|
127
|
-
let bucket = normalizationCache.get(key);
|
|
128
|
-
if (!bucket) {
|
|
129
|
-
bucket = /* @__PURE__ */ new Map();
|
|
130
|
-
normalizationCache.set(key, bucket);
|
|
131
|
-
}
|
|
132
|
-
cachedLangName = langKey;
|
|
133
|
-
cachedFlags = flags;
|
|
134
|
-
cachedBucket = bucket;
|
|
135
|
-
return bucket;
|
|
136
|
-
}
|
|
137
|
-
function evictOldestEntries(count) {
|
|
138
|
-
let remaining = count;
|
|
139
|
-
for (const [key, bucket] of normalizationCache) {
|
|
140
|
-
if (remaining <= 0) break;
|
|
141
|
-
if (bucket.size <= remaining) {
|
|
142
|
-
remaining -= bucket.size;
|
|
143
|
-
normalizationCacheSize -= bucket.size;
|
|
144
|
-
normalizationCache.delete(key);
|
|
145
|
-
if (bucket === cachedBucket) cachedBucket = null;
|
|
146
|
-
} else {
|
|
147
|
-
let deleted = 0;
|
|
148
|
-
for (const raw of bucket.keys()) {
|
|
149
|
-
if (deleted >= remaining) break;
|
|
150
|
-
bucket.delete(raw);
|
|
151
|
-
deleted++;
|
|
152
|
-
}
|
|
153
|
-
normalizationCacheSize -= deleted;
|
|
154
|
-
remaining = 0;
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
function transformToken(raw, language, stem, removeDiacritics) {
|
|
159
|
-
const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
|
|
160
|
-
let bucket = getCacheBucket(language, flags);
|
|
161
|
-
const cached = bucket.get(raw);
|
|
162
|
-
if (cached !== void 0) return cached;
|
|
163
|
-
let normalized = raw;
|
|
164
|
-
if (language.normalizer) {
|
|
165
|
-
normalized = language.normalizer(normalized);
|
|
166
|
-
}
|
|
167
|
-
if (removeDiacritics) {
|
|
168
|
-
normalized = stripDiacritics(normalized);
|
|
169
|
-
}
|
|
170
|
-
if (stem && language.stemmer) {
|
|
171
|
-
normalized = language.stemmer(normalized);
|
|
172
|
-
}
|
|
173
|
-
if (normalizationCacheSize >= maxCacheSize) {
|
|
174
|
-
evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
|
|
175
|
-
bucket = getCacheBucket(language, flags);
|
|
176
|
-
}
|
|
177
|
-
bucket.set(raw, normalized);
|
|
178
|
-
normalizationCacheSize++;
|
|
179
|
-
return normalized;
|
|
180
|
-
}
|
|
181
|
-
function clearNormalizationCache() {
|
|
182
|
-
normalizationCache.clear();
|
|
183
|
-
normalizationCacheSize = 0;
|
|
184
|
-
cachedBucket = null;
|
|
185
|
-
}
|
|
186
|
-
function resetNormalizationCache() {
|
|
187
|
-
clearNormalizationCache();
|
|
188
|
-
maxCacheSize = computeDefaultCacheSize();
|
|
189
|
-
}
|
|
190
|
-
function getNormalizationCacheSize() {
|
|
191
|
-
return normalizationCacheSize;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
// src/core/tokenizer/ngrams.ts
|
|
195
|
-
var OTHER_SCRIPT = 0;
|
|
196
|
-
var HAN_SCRIPT = 1;
|
|
197
|
-
var HIRAGANA_SCRIPT = 2;
|
|
198
|
-
var KATAKANA_SCRIPT = 3;
|
|
199
|
-
var HANGUL_SCRIPT = 4;
|
|
200
|
-
var THAI_SCRIPT = 5;
|
|
201
|
-
var LAO_SCRIPT = 6;
|
|
202
|
-
var KHMER_SCRIPT = 7;
|
|
203
|
-
var MYANMAR_SCRIPT = 8;
|
|
204
|
-
var PROLONGED_SOUND_MARK = 12540;
|
|
205
|
-
var IDEOGRAPHIC_ITERATION_MARK = 12293;
|
|
206
|
-
var LOWEST_NGRAM_CODE_POINT = 3584;
|
|
207
|
-
var LOWEST_MARK_CODE_POINT = 768;
|
|
208
|
-
var THAI_BLOCK_END = 3711;
|
|
209
|
-
var LAO_BLOCK_END = 3839;
|
|
210
|
-
var MYANMAR_BLOCK_START = 4096;
|
|
211
|
-
var MYANMAR_BLOCK_END = 4255;
|
|
212
|
-
var KHMER_BLOCK_START = 6016;
|
|
213
|
-
var KHMER_BLOCK_END = 6655;
|
|
214
|
-
var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
|
|
215
|
-
var HIRAGANA_BLOCK_START = 12352;
|
|
216
|
-
var HIRAGANA_BLOCK_END = 12447;
|
|
217
|
-
var COMBINING_VOICED_MARK2 = 12441;
|
|
218
|
-
var COMBINING_SEMI_VOICED_MARK2 = 12442;
|
|
219
|
-
var KATAKANA_BLOCK_START = 12448;
|
|
220
|
-
var KATAKANA_BLOCK_END = 12543;
|
|
221
|
-
var HAN_EXTENSION_A_START = 13312;
|
|
222
|
-
var HAN_EXTENSION_A_END = 19903;
|
|
223
|
-
var HAN_UNIFIED_START = 19968;
|
|
224
|
-
var HAN_UNIFIED_END = 40959;
|
|
225
|
-
var HANGUL_SYLLABLE_START = 44032;
|
|
226
|
-
var HANGUL_SYLLABLE_END = 55203;
|
|
227
|
-
var HAN_COMPATIBILITY_START = 63744;
|
|
228
|
-
var HAN_COMPATIBILITY_END = 64255;
|
|
229
|
-
var SUPPLEMENTARY_PLANE_START = 65536;
|
|
230
|
-
var MARK = /\p{M}/uy;
|
|
231
|
-
var HAN = /\p{Script=Han}/uy;
|
|
232
|
-
var HIRAGANA = /\p{Script=Hiragana}/uy;
|
|
233
|
-
var KATAKANA = /\p{Script=Katakana}/uy;
|
|
234
|
-
var HANGUL = /\p{Script=Hangul}/uy;
|
|
235
|
-
var THAI = /\p{Script=Thai}/uy;
|
|
236
|
-
var LAO = /\p{Script=Lao}/uy;
|
|
237
|
-
var KHMER = /\p{Script=Khmer}/uy;
|
|
238
|
-
var MYANMAR = /\p{Script=Myanmar}/uy;
|
|
239
|
-
function matchesAt(pattern, part, index) {
|
|
240
|
-
pattern.lastIndex = index;
|
|
241
|
-
return pattern.test(part);
|
|
242
|
-
}
|
|
243
|
-
function isMark(part, index, codePoint) {
|
|
244
|
-
if (codePoint < LOWEST_MARK_CODE_POINT) return false;
|
|
245
|
-
if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
|
|
246
|
-
if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
|
|
247
|
-
return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
|
|
248
|
-
}
|
|
249
|
-
if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
|
|
250
|
-
if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
|
|
251
|
-
return matchesAt(MARK, part, index);
|
|
252
|
-
}
|
|
253
|
-
function classifyOutsideCommonBlocks(part, index) {
|
|
254
|
-
if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
|
|
255
|
-
if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
|
|
256
|
-
if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
|
|
257
|
-
if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
|
|
258
|
-
if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
|
|
259
|
-
if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
|
|
260
|
-
if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
|
|
261
|
-
if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
|
|
262
|
-
return OTHER_SCRIPT;
|
|
263
|
-
}
|
|
264
|
-
function classifySoutheastAsian(part, index, codePoint) {
|
|
265
|
-
if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
|
|
266
|
-
if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
|
|
267
|
-
if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
|
|
268
|
-
return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
|
|
269
|
-
}
|
|
270
|
-
if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
|
|
271
|
-
return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
|
|
272
|
-
}
|
|
273
|
-
return classifyOutsideCommonBlocks(part, index);
|
|
274
|
-
}
|
|
275
|
-
function classify(part, index, codePoint, current) {
|
|
276
|
-
if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
|
|
277
|
-
if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
|
|
278
|
-
if (codePoint === PROLONGED_SOUND_MARK) {
|
|
279
|
-
return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
|
|
280
|
-
}
|
|
281
|
-
if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
|
|
282
|
-
if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
|
|
283
|
-
if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
|
|
284
|
-
if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
|
|
285
|
-
if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
|
|
286
|
-
if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
|
|
287
|
-
if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
|
|
288
|
-
return classifyOutsideCommonBlocks(part, index);
|
|
289
|
-
}
|
|
290
|
-
function emitRun(part, offsets, kind, size, out) {
|
|
291
|
-
const count = offsets.length - 1;
|
|
292
|
-
if (count < 1) return;
|
|
293
|
-
if (kind === OTHER_SCRIPT || count <= size) {
|
|
294
|
-
out.push(part.slice(offsets[0], offsets[count]));
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
for (let start = 0; start + size <= count; start++) {
|
|
298
|
-
out.push(part.slice(offsets[start], offsets[start + size]));
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
function expandNgrams(part, size, out) {
|
|
302
|
-
const length = part.length;
|
|
303
|
-
if (length === 0) return;
|
|
304
|
-
const offsets = [];
|
|
305
|
-
let kind = OTHER_SCRIPT;
|
|
306
|
-
let index = 0;
|
|
307
|
-
while (index < length) {
|
|
308
|
-
const codePoint = part.codePointAt(index);
|
|
309
|
-
if (codePoint === void 0) break;
|
|
310
|
-
const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
|
|
311
|
-
if (offsets.length > 0 && isMark(part, index, codePoint)) {
|
|
312
|
-
index += width;
|
|
313
|
-
continue;
|
|
314
|
-
}
|
|
315
|
-
const next = classify(part, index, codePoint, kind);
|
|
316
|
-
if (offsets.length > 0 && next !== kind) {
|
|
317
|
-
offsets.push(index);
|
|
318
|
-
emitRun(part, offsets, kind, size, out);
|
|
319
|
-
offsets.length = 0;
|
|
320
|
-
}
|
|
321
|
-
kind = next;
|
|
322
|
-
offsets.push(index);
|
|
323
|
-
index += width;
|
|
324
|
-
}
|
|
325
|
-
offsets.push(length);
|
|
326
|
-
emitRun(part, offsets, kind, size, out);
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
// src/core/tokenizer/tokenize.ts
|
|
330
|
-
function producesSurfaceForms(language, options) {
|
|
331
|
-
if (!options?.collectSurfaces || options.customTokenizer) return false;
|
|
332
|
-
return (options.stem ?? true) && language.stemmer !== null;
|
|
333
|
-
}
|
|
334
|
-
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
|
|
335
|
-
function resolveStopWords(language, override) {
|
|
336
|
-
if (!override) return language.stopWords;
|
|
337
|
-
if (override instanceof Set) return override;
|
|
338
|
-
return override(language.stopWords);
|
|
339
|
-
}
|
|
340
|
-
function splitText(text, language) {
|
|
341
|
-
const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
|
|
342
|
-
const parts = text.split(pattern);
|
|
343
|
-
const ngramSize = language.tokenizer?.ngramSize;
|
|
344
|
-
if (ngramSize === void 0 || ngramSize < 1) return parts;
|
|
345
|
-
const size = Math.floor(ngramSize);
|
|
346
|
-
const expanded = [];
|
|
347
|
-
for (const part of parts) {
|
|
348
|
-
expandNgrams(part, size, expanded);
|
|
349
|
-
}
|
|
350
|
-
return expanded;
|
|
351
|
-
}
|
|
352
|
-
var CHAR_APOSTROPHE = 39;
|
|
353
|
-
var CHAR_S = 115;
|
|
354
|
-
function stripPossessive(token) {
|
|
355
|
-
const len = token.length;
|
|
356
|
-
if (len >= 2 && token.charCodeAt(len - 1) === CHAR_S && token.charCodeAt(len - 2) === CHAR_APOSTROPHE) {
|
|
357
|
-
return token.slice(0, -2);
|
|
358
|
-
}
|
|
359
|
-
if (len >= 1 && token.charCodeAt(len - 1) === CHAR_APOSTROPHE) {
|
|
360
|
-
return token.slice(0, -1);
|
|
361
|
-
}
|
|
362
|
-
return token;
|
|
363
|
-
}
|
|
364
|
-
function visitTokens(text, language, options, visit) {
|
|
365
|
-
const {
|
|
366
|
-
stem = true,
|
|
367
|
-
removeStopWords = true,
|
|
368
|
-
removeDiacritics = false,
|
|
369
|
-
collectSurfaces = false,
|
|
370
|
-
customTokenizer,
|
|
371
|
-
stopWordOverride
|
|
372
|
-
} = options ?? {};
|
|
373
|
-
if (customTokenizer) {
|
|
374
|
-
for (const entry of customTokenizer.tokenize(text)) visit(entry.token, entry.position, void 0, entry.token);
|
|
375
|
-
return;
|
|
376
|
-
}
|
|
377
|
-
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
378
|
-
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
379
|
-
const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
|
|
380
|
-
const wantSurfaces = collectSurfaces && stem && language.stemmer !== null;
|
|
381
|
-
const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : null;
|
|
382
|
-
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
383
|
-
for (let position = 0; position < rawParts.length; position++) {
|
|
384
|
-
const part = rawParts[position];
|
|
385
|
-
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
386
|
-
if (candidate.length < minLength || stopWords?.has(candidate)) continue;
|
|
387
|
-
const processed = transformToken(candidate, language, stem, effectiveDiacritics);
|
|
388
|
-
if (processed.length === 0) continue;
|
|
389
|
-
if (!wantSurfaces) {
|
|
390
|
-
visit(processed, position, void 0, part);
|
|
391
|
-
continue;
|
|
392
|
-
}
|
|
393
|
-
const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
394
|
-
visit(processed, position, surface === processed ? void 0 : surface, part);
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
function visitSurfaceSpellings(text, language, visit) {
|
|
398
|
-
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
399
|
-
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
400
|
-
const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
|
|
401
|
-
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
402
|
-
for (const part of rawParts) {
|
|
403
|
-
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
404
|
-
if (candidate.length < minLength) continue;
|
|
405
|
-
visit(removeDiacritics ? transformToken(candidate, language, false, true) : candidate);
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
function tokenize(text, language, options) {
|
|
409
|
-
if (options?.customTokenizer) {
|
|
410
|
-
const customResult = options.customTokenizer.tokenize(text);
|
|
411
|
-
return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
|
|
412
|
-
}
|
|
413
|
-
const tokens = [];
|
|
414
|
-
const originalTokens = [];
|
|
415
|
-
const surfaces = producesSurfaceForms(language, options) ? [] : void 0;
|
|
416
|
-
visitTokens(text, language, options, (token, position, surface, original) => {
|
|
417
|
-
tokens.push({ token, position });
|
|
418
|
-
originalTokens.push(original);
|
|
419
|
-
surfaces?.push(surface);
|
|
420
|
-
});
|
|
421
|
-
return { tokens, originalTokens, surfaces };
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
// src/core/partition/constants.ts
|
|
425
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
426
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
427
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
428
|
-
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
429
|
-
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
430
|
-
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
431
|
-
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
432
|
-
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
433
|
-
var FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION = 0.9;
|
|
434
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
435
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
436
|
-
|
|
437
|
-
// src/core/partition/frozen/string-blob.ts
|
|
438
|
-
var SURROGATE = /[\ud800-\udfff]/;
|
|
439
|
-
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
440
|
-
var encoder = new TextEncoder();
|
|
441
|
-
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
442
|
-
function encodeLookupKey(text) {
|
|
443
|
-
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder.encode(text);
|
|
444
|
-
const { written } = encoder.encodeInto(text, lookupKeyBytes);
|
|
445
|
-
return lookupKeyBytes.subarray(0, written);
|
|
446
|
-
}
|
|
447
|
-
function encodeOneByOne(strings) {
|
|
448
|
-
const encoded = new Array(strings.length);
|
|
449
|
-
let blobLength = 0;
|
|
450
|
-
for (let i = 0; i < strings.length; i++) {
|
|
451
|
-
encoded[i] = encoder.encode(strings[i]);
|
|
452
|
-
blobLength += encoded[i].length;
|
|
453
|
-
}
|
|
454
|
-
const blob = new Uint8Array(blobLength);
|
|
455
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
456
|
-
let cursor = 0;
|
|
457
|
-
for (let i = 0; i < strings.length; i++) {
|
|
458
|
-
blob.set(encoded[i], cursor);
|
|
459
|
-
cursor += encoded[i].length;
|
|
460
|
-
offsets[i + 1] = cursor;
|
|
461
|
-
}
|
|
462
|
-
return { blob, offsets };
|
|
463
|
-
}
|
|
464
|
-
function basicPlaneUtf8Bytes(text) {
|
|
465
|
-
let bytes = text.length;
|
|
466
|
-
for (let i = 0; i < text.length; i++) {
|
|
467
|
-
const unit = text.charCodeAt(i);
|
|
468
|
-
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
469
|
-
}
|
|
470
|
-
return bytes;
|
|
471
|
-
}
|
|
472
|
-
function encodeStringBlob(strings) {
|
|
473
|
-
const joined = strings.join("");
|
|
474
|
-
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
475
|
-
const blob = encoder.encode(joined);
|
|
476
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
477
|
-
let cursor = 0;
|
|
478
|
-
for (let i = 0; i < strings.length; i++) {
|
|
479
|
-
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
480
|
-
offsets[i + 1] = cursor;
|
|
481
|
-
}
|
|
482
|
-
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
483
|
-
}
|
|
484
|
-
function orderByComparator(strings) {
|
|
485
|
-
const order = new Array(strings.length);
|
|
486
|
-
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
487
|
-
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
488
|
-
}
|
|
489
|
-
function alreadyInCodePointOrder(strings) {
|
|
490
|
-
for (let i = 1; i < strings.length; i++) {
|
|
491
|
-
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
492
|
-
}
|
|
493
|
-
return true;
|
|
494
|
-
}
|
|
495
|
-
function codePointOrder(strings) {
|
|
496
|
-
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
497
|
-
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
498
|
-
const positions = /* @__PURE__ */ new Map();
|
|
499
|
-
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
500
|
-
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
501
|
-
const order = new Array(strings.length);
|
|
502
|
-
const sorted = [...strings].sort();
|
|
503
|
-
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
504
|
-
return order;
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
export { FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, getNormalizationCacheSize, resetNormalizationCache, resolveStopWords, tokenize, transformToken, visitSurfaceSpellings, visitTokens };
|
package/dist/chunk-LXWDQLTW.mjs
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { DEFAULT_PAGE_SIZE } from './chunk-E74YLAIG.mjs';
|
|
2
|
-
|
|
3
|
-
// src/engine/constants.ts
|
|
4
|
-
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
5
|
-
var BATCH_CHUNK_SIZE = 1e3;
|
|
6
|
-
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
7
|
-
var DEFAULT_OFFSET = 0;
|
|
8
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
9
|
-
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
10
|
-
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
11
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
12
|
-
var MAX_DOC_ID_LENGTH = 512;
|
|
13
|
-
var EXPORT_CHUNK_BYTES = 1048576;
|
|
14
|
-
var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
15
|
-
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
16
|
-
var IMPORT_BATCH_DOCUMENTS = 1e3;
|
|
17
|
-
var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
|
|
18
|
-
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
19
|
-
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
20
|
-
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
21
|
-
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
22
|
-
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
23
|
-
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
24
|
-
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
25
|
-
var MAX_SUGGEST_LIMIT = 100;
|
|
26
|
-
var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
27
|
-
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
28
|
-
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
29
|
-
|
|
30
|
-
export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
|