@delali/narsil 0.3.1-next.99 → 0.4.1-next.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +30 -11
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +404 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
package/dist/chunk-BKB3XTMI.mjs
DELETED
|
@@ -1,707 +0,0 @@
|
|
|
1
|
-
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-ZG477IPV.mjs';
|
|
2
|
-
import { manifestKey, segmentsPrefix } from './chunk-LH4NZNJJ.mjs';
|
|
3
|
-
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-VWULPWRJ.mjs';
|
|
4
|
-
import { VALID_HNSW_METRICS } from './chunk-AY3AQ3UH.mjs';
|
|
5
|
-
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-LXPIXDUJ.mjs';
|
|
6
|
-
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
8
|
-
import { decode, encode } from '@msgpack/msgpack';
|
|
9
|
-
|
|
10
|
-
// src/serialization/stored-vector-values.ts
|
|
11
|
-
function dropStoredVectorValues(documents, schema) {
|
|
12
|
-
const vectorFieldPaths = [];
|
|
13
|
-
for (const [fieldPath, fieldType] of Object.entries(schema)) {
|
|
14
|
-
if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
|
|
15
|
-
vectorFieldPaths.push(fieldPath);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
if (vectorFieldPaths.length === 0) {
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
for (const document of Object.values(documents)) {
|
|
22
|
-
for (const fieldPath of vectorFieldPaths) {
|
|
23
|
-
removeFieldAtPath(document.fields, fieldPath);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
function removeFieldAtPath(fields, fieldPath) {
|
|
28
|
-
if (!fieldPath.includes(".")) {
|
|
29
|
-
delete fields[fieldPath];
|
|
30
|
-
return;
|
|
31
|
-
}
|
|
32
|
-
const segments = fieldPath.split(".");
|
|
33
|
-
let current = fields;
|
|
34
|
-
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
35
|
-
const next = current[segments[i]];
|
|
36
|
-
if (next === null || typeof next !== "object") {
|
|
37
|
-
return;
|
|
38
|
-
}
|
|
39
|
-
current = next;
|
|
40
|
-
}
|
|
41
|
-
delete current[segments[segments.length - 1]];
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// src/serialization/payload-v1/partition.ts
|
|
45
|
-
function sanitizeSurfaceForms(raw) {
|
|
46
|
-
if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
47
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
48
|
-
let count = 0;
|
|
49
|
-
for (const [surface, value] of Object.entries(raw)) {
|
|
50
|
-
if (typeof value === "number") {
|
|
51
|
-
result[surface] = value;
|
|
52
|
-
count++;
|
|
53
|
-
} else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
|
|
54
|
-
result[surface] = [value[0], value[1]];
|
|
55
|
-
count++;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
return count > 0 ? result : void 0;
|
|
59
|
-
}
|
|
60
|
-
function isWireRecord(value) {
|
|
61
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
62
|
-
}
|
|
63
|
-
function corruptPatternEntry(field) {
|
|
64
|
-
return new NarsilError(
|
|
65
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
66
|
-
`The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
|
|
67
|
-
{ field }
|
|
68
|
-
);
|
|
69
|
-
}
|
|
70
|
-
function patternIndexesFromWire(raw) {
|
|
71
|
-
if (raw === void 0 || raw === null) return {};
|
|
72
|
-
if (!isWireRecord(raw)) {
|
|
73
|
-
throw new NarsilError(
|
|
74
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
75
|
-
"The pattern entry of this partition must be a map of fields"
|
|
76
|
-
);
|
|
77
|
-
}
|
|
78
|
-
const pattern = /* @__PURE__ */ Object.create(null);
|
|
79
|
-
for (const [field, index] of Object.entries(raw)) {
|
|
80
|
-
if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
|
|
81
|
-
throw corruptPatternEntry(field);
|
|
82
|
-
}
|
|
83
|
-
const docIds = [];
|
|
84
|
-
for (const docId of index.doc_ids) {
|
|
85
|
-
if (typeof docId !== "string") throw corruptPatternEntry(field);
|
|
86
|
-
docIds.push(docId);
|
|
87
|
-
}
|
|
88
|
-
pattern[field] = { docIds, runs: index.runs };
|
|
89
|
-
}
|
|
90
|
-
return { pattern };
|
|
91
|
-
}
|
|
92
|
-
function validateHnswMetric(value) {
|
|
93
|
-
if (value === void 0 || value === null) return void 0;
|
|
94
|
-
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
95
|
-
return value;
|
|
96
|
-
}
|
|
97
|
-
return void 0;
|
|
98
|
-
}
|
|
99
|
-
function wireToPartition(raw) {
|
|
100
|
-
const documents = {};
|
|
101
|
-
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
102
|
-
documents[docId] = {
|
|
103
|
-
fields: doc.fields,
|
|
104
|
-
fieldLengths: doc.field_lengths ?? {}
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
requireValidFieldTypes(raw.schema ?? {});
|
|
108
|
-
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
109
|
-
const invertedIndex = {};
|
|
110
|
-
for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
|
|
111
|
-
invertedIndex[token] = {
|
|
112
|
-
docFrequency: list.doc_freq,
|
|
113
|
-
postings: (list.postings ?? []).map((p) => ({
|
|
114
|
-
docId: p.doc_id,
|
|
115
|
-
termFrequency: p.term_freq,
|
|
116
|
-
field: p.field,
|
|
117
|
-
positions: p.positions ?? []
|
|
118
|
-
}))
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
const numeric = {};
|
|
122
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
123
|
-
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
124
|
-
}
|
|
125
|
-
const boolean = {};
|
|
126
|
-
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
127
|
-
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
128
|
-
}
|
|
129
|
-
const geopoint = {};
|
|
130
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
131
|
-
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
132
|
-
}
|
|
133
|
-
const vectorData = {};
|
|
134
|
-
for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
|
|
135
|
-
vectorData[field] = {
|
|
136
|
-
dimension: data.dimension,
|
|
137
|
-
vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
138
|
-
hnswGraph: data.hnsw_graph ? {
|
|
139
|
-
entryPoint: data.hnsw_graph.entry_point,
|
|
140
|
-
maxLayer: data.hnsw_graph.max_layer,
|
|
141
|
-
m: data.hnsw_graph.m,
|
|
142
|
-
efConstruction: data.hnsw_graph.ef_construction,
|
|
143
|
-
metric: validateHnswMetric(data.hnsw_graph.metric),
|
|
144
|
-
nodes: data.hnsw_graph.nodes
|
|
145
|
-
} : null,
|
|
146
|
-
codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
|
|
147
|
-
};
|
|
148
|
-
}
|
|
149
|
-
return {
|
|
150
|
-
indexName: raw.index_name,
|
|
151
|
-
partitionId: raw.partition_id,
|
|
152
|
-
totalPartitions: raw.total_partitions,
|
|
153
|
-
language: raw.language ?? "english",
|
|
154
|
-
schema: raw.schema ?? {},
|
|
155
|
-
docCount: raw.doc_count ?? 0,
|
|
156
|
-
avgDocLength: raw.avg_doc_length ?? 0,
|
|
157
|
-
documents,
|
|
158
|
-
invertedIndex,
|
|
159
|
-
fieldIndexes: {
|
|
160
|
-
numeric,
|
|
161
|
-
boolean,
|
|
162
|
-
enum: raw.field_indexes?.enum ?? {},
|
|
163
|
-
geopoint,
|
|
164
|
-
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
165
|
-
},
|
|
166
|
-
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
167
|
-
vectorData,
|
|
168
|
-
statistics: {
|
|
169
|
-
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
170
|
-
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
171
|
-
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
172
|
-
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
173
|
-
}
|
|
174
|
-
};
|
|
175
|
-
}
|
|
176
|
-
function deserializePayloadV1(data) {
|
|
177
|
-
const raw = decode(data);
|
|
178
|
-
return wireToPartition(raw);
|
|
179
|
-
}
|
|
180
|
-
function metadataToWire(meta) {
|
|
181
|
-
const wire = {
|
|
182
|
-
index_name: meta.indexName,
|
|
183
|
-
schema: meta.schema,
|
|
184
|
-
language: meta.language,
|
|
185
|
-
partition_count: meta.partitionCount,
|
|
186
|
-
bm25_params: meta.bm25Params,
|
|
187
|
-
created_at: meta.createdAt,
|
|
188
|
-
engine_version: meta.engineVersion
|
|
189
|
-
};
|
|
190
|
-
if (meta.vectorFields) {
|
|
191
|
-
wire.vector_fields = meta.vectorFields;
|
|
192
|
-
}
|
|
193
|
-
if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
|
|
194
|
-
wire.document_count = meta.documentCount;
|
|
195
|
-
}
|
|
196
|
-
if (meta.embedding) {
|
|
197
|
-
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
198
|
-
}
|
|
199
|
-
if (meta.surfaceForms !== void 0) {
|
|
200
|
-
wire.surface_forms_enabled = meta.surfaceForms;
|
|
201
|
-
}
|
|
202
|
-
if (meta.analysisRevision !== void 0) {
|
|
203
|
-
wire.analysis_revision = meta.analysisRevision;
|
|
204
|
-
}
|
|
205
|
-
if (meta.tokenizer !== void 0) {
|
|
206
|
-
wire.tokenizer = meta.tokenizer;
|
|
207
|
-
}
|
|
208
|
-
if (meta.stopWords !== void 0) {
|
|
209
|
-
wire.stop_words = meta.stopWords;
|
|
210
|
-
}
|
|
211
|
-
if (meta.stopWordList !== void 0) {
|
|
212
|
-
wire.stop_word_list = meta.stopWordList;
|
|
213
|
-
}
|
|
214
|
-
if (meta.partitionLimits !== void 0) {
|
|
215
|
-
wire.partition_limits = {
|
|
216
|
-
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
217
|
-
...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
|
|
218
|
-
...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
|
|
219
|
-
};
|
|
220
|
-
}
|
|
221
|
-
if (meta.defaultScoring !== void 0) {
|
|
222
|
-
wire.default_scoring = meta.defaultScoring;
|
|
223
|
-
}
|
|
224
|
-
if (meta.trackPositions !== void 0) {
|
|
225
|
-
wire.track_positions = meta.trackPositions;
|
|
226
|
-
}
|
|
227
|
-
if (meta.strict !== void 0) {
|
|
228
|
-
wire.strict = meta.strict;
|
|
229
|
-
}
|
|
230
|
-
if (meta.required !== void 0) {
|
|
231
|
-
wire.required = meta.required;
|
|
232
|
-
}
|
|
233
|
-
if (meta.vectorPromotion !== void 0) {
|
|
234
|
-
const promotion = meta.vectorPromotion;
|
|
235
|
-
wire.vector_promotion = {
|
|
236
|
-
...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
|
|
237
|
-
...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
|
|
238
|
-
...promotion.hnswConfig !== void 0 ? {
|
|
239
|
-
hnsw_config: {
|
|
240
|
-
...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
|
|
241
|
-
...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
|
|
242
|
-
...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
|
|
243
|
-
}
|
|
244
|
-
} : {},
|
|
245
|
-
...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
|
|
246
|
-
...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
|
-
if (meta.patternValueLimit !== void 0) {
|
|
250
|
-
wire.pattern_value_limit = meta.patternValueLimit;
|
|
251
|
-
}
|
|
252
|
-
if (meta.indexUuid !== void 0) {
|
|
253
|
-
wire.index_uuid = meta.indexUuid;
|
|
254
|
-
}
|
|
255
|
-
if (meta.heldPartitions !== void 0) {
|
|
256
|
-
wire.held_partitions = meta.heldPartitions;
|
|
257
|
-
}
|
|
258
|
-
if (Array.isArray(meta.synonyms) && meta.synonyms.length > 0) {
|
|
259
|
-
wire.synonyms = meta.synonyms;
|
|
260
|
-
}
|
|
261
|
-
return wire;
|
|
262
|
-
}
|
|
263
|
-
function wireToMetadata(raw) {
|
|
264
|
-
const meta = {
|
|
265
|
-
indexName: raw.index_name,
|
|
266
|
-
schema: raw.schema ?? {},
|
|
267
|
-
language: raw.language ?? "english",
|
|
268
|
-
partitionCount: raw.partition_count ?? 1,
|
|
269
|
-
bm25Params: raw.bm25_params ?? { k1: 1.2, b: 0.75 },
|
|
270
|
-
createdAt: raw.created_at ?? 0,
|
|
271
|
-
engineVersion: raw.engine_version ?? "0.0.0"
|
|
272
|
-
};
|
|
273
|
-
if (raw.vector_fields) {
|
|
274
|
-
for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
|
|
275
|
-
if (!isQuantizationMode(field.quantization)) {
|
|
276
|
-
throw new NarsilError(
|
|
277
|
-
ErrorCodes.CONFIG_INVALID,
|
|
278
|
-
`Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
|
|
279
|
-
{ fieldPath, quantization: field.quantization }
|
|
280
|
-
);
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
meta.vectorFields = raw.vector_fields;
|
|
284
|
-
}
|
|
285
|
-
if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
|
|
286
|
-
meta.documentCount = raw.document_count;
|
|
287
|
-
}
|
|
288
|
-
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
289
|
-
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
290
|
-
}
|
|
291
|
-
if (typeof raw.surface_forms_enabled === "boolean") {
|
|
292
|
-
meta.surfaceForms = raw.surface_forms_enabled;
|
|
293
|
-
}
|
|
294
|
-
if (typeof raw.analysis_revision === "string") {
|
|
295
|
-
meta.analysisRevision = raw.analysis_revision;
|
|
296
|
-
}
|
|
297
|
-
if (typeof raw.tokenizer === "string") {
|
|
298
|
-
meta.tokenizer = raw.tokenizer;
|
|
299
|
-
}
|
|
300
|
-
if (typeof raw.stop_words === "string") {
|
|
301
|
-
meta.stopWords = raw.stop_words;
|
|
302
|
-
}
|
|
303
|
-
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
304
|
-
meta.stopWordList = raw.stop_word_list;
|
|
305
|
-
}
|
|
306
|
-
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
307
|
-
const limits = {
|
|
308
|
-
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
309
|
-
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
|
|
310
|
-
...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
|
|
311
|
-
};
|
|
312
|
-
if (Object.keys(limits).length > 0) {
|
|
313
|
-
meta.partitionLimits = limits;
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
|
|
317
|
-
meta.defaultScoring = raw.default_scoring;
|
|
318
|
-
}
|
|
319
|
-
if (typeof raw.track_positions === "boolean") {
|
|
320
|
-
meta.trackPositions = raw.track_positions;
|
|
321
|
-
}
|
|
322
|
-
if (typeof raw.strict === "boolean") {
|
|
323
|
-
meta.strict = raw.strict;
|
|
324
|
-
}
|
|
325
|
-
if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
|
|
326
|
-
meta.required = raw.required;
|
|
327
|
-
}
|
|
328
|
-
if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
|
|
329
|
-
const promotion = raw.vector_promotion;
|
|
330
|
-
const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
|
|
331
|
-
const metric = hnsw?.metric;
|
|
332
|
-
const restored = {};
|
|
333
|
-
const hnswConfig = {};
|
|
334
|
-
if (isPositiveInteger(hnsw?.m)) {
|
|
335
|
-
hnswConfig.m = hnsw?.m;
|
|
336
|
-
}
|
|
337
|
-
if (isPositiveInteger(hnsw?.ef_construction)) {
|
|
338
|
-
hnswConfig.efConstruction = hnsw?.ef_construction;
|
|
339
|
-
}
|
|
340
|
-
if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
|
|
341
|
-
hnswConfig.metric = metric;
|
|
342
|
-
}
|
|
343
|
-
if (isPositiveInteger(promotion.threshold)) {
|
|
344
|
-
restored.threshold = promotion.threshold;
|
|
345
|
-
}
|
|
346
|
-
if (isSelectivityRatio(promotion.filter_threshold)) {
|
|
347
|
-
restored.filterThreshold = promotion.filter_threshold;
|
|
348
|
-
}
|
|
349
|
-
if (Object.keys(hnswConfig).length > 0) {
|
|
350
|
-
restored.hnswConfig = hnswConfig;
|
|
351
|
-
}
|
|
352
|
-
if (promotion.quantization !== void 0) {
|
|
353
|
-
if (!isQuantizationMode(promotion.quantization)) {
|
|
354
|
-
throw new NarsilError(
|
|
355
|
-
ErrorCodes.CONFIG_INVALID,
|
|
356
|
-
`Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
|
|
357
|
-
{ quantization: promotion.quantization }
|
|
358
|
-
);
|
|
359
|
-
}
|
|
360
|
-
restored.quantization = promotion.quantization;
|
|
361
|
-
}
|
|
362
|
-
if (promotion.storage !== void 0) {
|
|
363
|
-
if (!isStorageMode(promotion.storage)) {
|
|
364
|
-
throw new NarsilError(
|
|
365
|
-
ErrorCodes.CONFIG_INVALID,
|
|
366
|
-
`Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
|
|
367
|
-
{ storage: promotion.storage }
|
|
368
|
-
);
|
|
369
|
-
}
|
|
370
|
-
restored.storage = promotion.storage;
|
|
371
|
-
}
|
|
372
|
-
if (Object.keys(restored).length > 0) {
|
|
373
|
-
meta.vectorPromotion = restored;
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
if (raw.pattern_value_limit !== void 0) {
|
|
377
|
-
if (!isPositiveInteger(raw.pattern_value_limit)) {
|
|
378
|
-
throw new NarsilError(
|
|
379
|
-
ErrorCodes.CONFIG_INVALID,
|
|
380
|
-
`The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
|
|
381
|
-
{ patternValueLimit: String(raw.pattern_value_limit) }
|
|
382
|
-
);
|
|
383
|
-
}
|
|
384
|
-
meta.patternValueLimit = raw.pattern_value_limit;
|
|
385
|
-
}
|
|
386
|
-
if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
|
|
387
|
-
meta.indexUuid = raw.index_uuid;
|
|
388
|
-
}
|
|
389
|
-
if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
|
|
390
|
-
meta.heldPartitions = raw.held_partitions;
|
|
391
|
-
}
|
|
392
|
-
if (raw.synonyms !== void 0 && raw.synonyms !== null) meta.synonyms = raw.synonyms;
|
|
393
|
-
return meta;
|
|
394
|
-
}
|
|
395
|
-
function isNonNegativeInteger(value) {
|
|
396
|
-
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
397
|
-
}
|
|
398
|
-
function isPositiveInteger(value) {
|
|
399
|
-
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
400
|
-
}
|
|
401
|
-
function isSelectivityRatio(value) {
|
|
402
|
-
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
403
|
-
}
|
|
404
|
-
function isWatermarkFraction(value) {
|
|
405
|
-
return typeof value === "number" && value > 0 && value <= 1;
|
|
406
|
-
}
|
|
407
|
-
function serializeMetadata(meta) {
|
|
408
|
-
const wire = metadataToWire(meta);
|
|
409
|
-
return encode(wire);
|
|
410
|
-
}
|
|
411
|
-
function deserializeMetadata(data) {
|
|
412
|
-
const raw = decode(data);
|
|
413
|
-
return wireToMetadata(raw);
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
// src/serialization/envelope.ts
|
|
417
|
-
var CURRENT_ENVELOPE_VERSION = 1;
|
|
418
|
-
var SNAPSHOT_ENVELOPE_VERSION = 2;
|
|
419
|
-
var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
|
|
420
|
-
async function collectStream(stream) {
|
|
421
|
-
const reader = stream.getReader();
|
|
422
|
-
const chunks = [];
|
|
423
|
-
let totalLength = 0;
|
|
424
|
-
for (; ; ) {
|
|
425
|
-
const { done, value } = await reader.read();
|
|
426
|
-
if (done) break;
|
|
427
|
-
chunks.push(value);
|
|
428
|
-
totalLength += value.length;
|
|
429
|
-
}
|
|
430
|
-
const result = new Uint8Array(totalLength);
|
|
431
|
-
let offset = 0;
|
|
432
|
-
for (const chunk of chunks) {
|
|
433
|
-
result.set(chunk, offset);
|
|
434
|
-
offset += chunk.length;
|
|
435
|
-
}
|
|
436
|
-
return result;
|
|
437
|
-
}
|
|
438
|
-
function toArrayBuffer(data) {
|
|
439
|
-
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
440
|
-
}
|
|
441
|
-
async function compressGzip(data) {
|
|
442
|
-
const blob = new Blob([toArrayBuffer(data)]);
|
|
443
|
-
const compressed = blob.stream().pipeThrough(new CompressionStream("gzip"));
|
|
444
|
-
return collectStream(compressed);
|
|
445
|
-
}
|
|
446
|
-
async function decompressGzip(data) {
|
|
447
|
-
const blob = new Blob([toArrayBuffer(data)]);
|
|
448
|
-
const decompressed = blob.stream().pipeThrough(new DecompressionStream("gzip"));
|
|
449
|
-
return collectStream(decompressed);
|
|
450
|
-
}
|
|
451
|
-
function buildFlags(options) {
|
|
452
|
-
const useCompression = options.compression === "gzip";
|
|
453
|
-
return {
|
|
454
|
-
compressionEnabled: useCompression,
|
|
455
|
-
compressionAlgorithm: useCompression ? "gzip" : "none",
|
|
456
|
-
checksumPresent: options.checksum ?? false,
|
|
457
|
-
encryptionEnabled: false
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
async function packEnvelope(payloadBytes, options) {
|
|
461
|
-
const flags = buildFlags(options);
|
|
462
|
-
let finalPayload = payloadBytes;
|
|
463
|
-
if (flags.compressionEnabled) {
|
|
464
|
-
finalPayload = await compressGzip(payloadBytes);
|
|
465
|
-
}
|
|
466
|
-
const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
|
|
467
|
-
const [major, minor, patch] = engineVersionParts();
|
|
468
|
-
const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
|
|
469
|
-
const header = {
|
|
470
|
-
magic: "NRSL",
|
|
471
|
-
envelopeFormatVersion,
|
|
472
|
-
engineVersionMajor: major,
|
|
473
|
-
engineVersionMinor: minor,
|
|
474
|
-
engineVersionPatch: patch,
|
|
475
|
-
payloadLength: finalPayload.length,
|
|
476
|
-
flags,
|
|
477
|
-
checksum,
|
|
478
|
-
reserved: new Uint8Array(14)
|
|
479
|
-
};
|
|
480
|
-
const headerBytes = writeHeader(header);
|
|
481
|
-
const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
|
|
482
|
-
envelope.set(headerBytes, 0);
|
|
483
|
-
envelope.set(finalPayload, HEADER_SIZE);
|
|
484
|
-
return envelope;
|
|
485
|
-
}
|
|
486
|
-
async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
|
|
487
|
-
const header = readHeader(data);
|
|
488
|
-
if (header.envelopeFormatVersion > maxAcceptedVersion) {
|
|
489
|
-
throw new NarsilError(
|
|
490
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
491
|
-
`This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
|
|
492
|
-
);
|
|
493
|
-
}
|
|
494
|
-
if (header.flags.encryptionEnabled) {
|
|
495
|
-
throw new NarsilError(
|
|
496
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
497
|
-
"Encrypted envelopes are not supported in this version of Narsil"
|
|
498
|
-
);
|
|
499
|
-
}
|
|
500
|
-
const compAlgo = header.flags.compressionAlgorithm;
|
|
501
|
-
if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
|
|
502
|
-
throw new NarsilError(
|
|
503
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
504
|
-
`Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
|
|
505
|
-
);
|
|
506
|
-
}
|
|
507
|
-
const availablePayload = data.length - HEADER_SIZE;
|
|
508
|
-
if (availablePayload < header.payloadLength) {
|
|
509
|
-
throw new NarsilError(
|
|
510
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
511
|
-
`Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
|
|
512
|
-
);
|
|
513
|
-
}
|
|
514
|
-
let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
|
|
515
|
-
if (header.flags.checksumPresent) {
|
|
516
|
-
const computed = crc32(payloadBytes);
|
|
517
|
-
if (computed !== header.checksum) {
|
|
518
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
|
|
519
|
-
expected: header.checksum,
|
|
520
|
-
computed
|
|
521
|
-
});
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
if (header.flags.compressionEnabled) {
|
|
525
|
-
payloadBytes = await decompressGzip(payloadBytes);
|
|
526
|
-
}
|
|
527
|
-
return { header, payloadBytes };
|
|
528
|
-
}
|
|
529
|
-
async function unpackEnvelopeBytes(data) {
|
|
530
|
-
return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
|
|
531
|
-
}
|
|
532
|
-
async function unpackIndexSnapshotEnvelope(data) {
|
|
533
|
-
const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
|
|
534
|
-
if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
|
|
535
|
-
throw new NarsilError(
|
|
536
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
537
|
-
`Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
|
|
538
|
-
);
|
|
539
|
-
}
|
|
540
|
-
return payloadBytes;
|
|
541
|
-
}
|
|
542
|
-
async function writeMetadataEnvelope(meta, options = {}) {
|
|
543
|
-
const payloadBytes = serializeMetadata(meta);
|
|
544
|
-
return packEnvelope(payloadBytes, options);
|
|
545
|
-
}
|
|
546
|
-
async function readMetadataEnvelope(data) {
|
|
547
|
-
const { header, payloadBytes } = await unpackEnvelope(data);
|
|
548
|
-
const metadata = deserializeMetadata(payloadBytes);
|
|
549
|
-
return { header, metadata };
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
// src/persistence/durability/segment/load.ts
|
|
553
|
-
async function readSegmentManifest(store, indexName) {
|
|
554
|
-
const bytes = await store.read(manifestKey(indexName));
|
|
555
|
-
if (bytes === null) return null;
|
|
556
|
-
return decodeSegmentManifest(bytes, indexName);
|
|
557
|
-
}
|
|
558
|
-
async function readCurrentManifest(store, indexName) {
|
|
559
|
-
const decoded = await readSegmentManifest(store, indexName);
|
|
560
|
-
if (decoded === null) return null;
|
|
561
|
-
if (decoded.kind === "current") return decoded.manifest;
|
|
562
|
-
throw new NarsilError(
|
|
563
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
564
|
-
`Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
|
|
565
|
-
{ indexName }
|
|
566
|
-
);
|
|
567
|
-
}
|
|
568
|
-
function growPartitionsTo(manager, highestPartitionId) {
|
|
569
|
-
while (manager.partitionCount <= highestPartitionId) {
|
|
570
|
-
const before = manager.partitionCount;
|
|
571
|
-
try {
|
|
572
|
-
manager.addPartition();
|
|
573
|
-
} catch {
|
|
574
|
-
break;
|
|
575
|
-
}
|
|
576
|
-
if (manager.partitionCount === before) break;
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
580
|
-
let highestPartitionId = manifest.partitions.length - 1;
|
|
581
|
-
for (const partition of manifest.partitions) {
|
|
582
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
583
|
-
}
|
|
584
|
-
growPartitionsTo(deps.manager, highestPartitionId);
|
|
585
|
-
for (const partition of manifest.partitions) {
|
|
586
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
587
|
-
throw new NarsilError(
|
|
588
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
589
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
590
|
-
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
591
|
-
);
|
|
592
|
-
}
|
|
593
|
-
await loadPartitionSegments(store, partition, deps.manager, { schema: manifest.schema, decodesEveryDocument });
|
|
594
|
-
}
|
|
595
|
-
for (const vector of manifest.vectors) {
|
|
596
|
-
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
597
|
-
if (vectorIndex === void 0) continue;
|
|
598
|
-
await loadVectorField(store, vector, vectorIndex);
|
|
599
|
-
}
|
|
600
|
-
return manifest.checkpoint;
|
|
601
|
-
}
|
|
602
|
-
async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
603
|
-
for (const key of await store.list(segmentsPrefix(indexName))) {
|
|
604
|
-
if (!referenced.has(key)) await store.remove(key);
|
|
605
|
-
}
|
|
606
|
-
}
|
|
607
|
-
function validateHnswMetric2(value) {
|
|
608
|
-
if (value === void 0 || value === null) return void 0;
|
|
609
|
-
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
610
|
-
return value;
|
|
611
|
-
}
|
|
612
|
-
return void 0;
|
|
613
|
-
}
|
|
614
|
-
function storedDocumentsOf(raw) {
|
|
615
|
-
requireValidFieldTypes(raw.schema ?? {});
|
|
616
|
-
const documents = {};
|
|
617
|
-
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
618
|
-
documents[docId] = {
|
|
619
|
-
fields: doc.fields,
|
|
620
|
-
fieldLengths: doc.field_lengths ?? {}
|
|
621
|
-
};
|
|
622
|
-
}
|
|
623
|
-
dropStoredVectorValues(documents, raw.schema ?? {});
|
|
624
|
-
return documents;
|
|
625
|
-
}
|
|
626
|
-
function documentsOfPayloadV2(data) {
|
|
627
|
-
const raw = decode(data);
|
|
628
|
-
return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
|
|
629
|
-
}
|
|
630
|
-
function deserializePayloadV2(data) {
|
|
631
|
-
const raw = decode(data);
|
|
632
|
-
const documents = storedDocumentsOf(raw);
|
|
633
|
-
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
634
|
-
const invertedIndex = {};
|
|
635
|
-
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
636
|
-
const count = list.ids.length;
|
|
637
|
-
const postings = new Array(count);
|
|
638
|
-
for (let i = 0; i < count; i++) {
|
|
639
|
-
postings[i] = {
|
|
640
|
-
docId: list.ids[i],
|
|
641
|
-
termFrequency: list.tf[i],
|
|
642
|
-
field: fieldNames[list.fi[i]] ?? "",
|
|
643
|
-
positions: list.pos ? list.pos[i] : []
|
|
644
|
-
};
|
|
645
|
-
}
|
|
646
|
-
invertedIndex[token] = {
|
|
647
|
-
docFrequency: list.df,
|
|
648
|
-
postings
|
|
649
|
-
};
|
|
650
|
-
}
|
|
651
|
-
const numeric = {};
|
|
652
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
653
|
-
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
654
|
-
}
|
|
655
|
-
const boolean = {};
|
|
656
|
-
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
657
|
-
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
658
|
-
}
|
|
659
|
-
const geopoint = {};
|
|
660
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
661
|
-
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
662
|
-
}
|
|
663
|
-
const vectorData = {};
|
|
664
|
-
for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
|
|
665
|
-
vectorData[field] = {
|
|
666
|
-
dimension: data2.dimension,
|
|
667
|
-
vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
668
|
-
hnswGraph: data2.hnsw_graph ? {
|
|
669
|
-
entryPoint: data2.hnsw_graph.entry_point,
|
|
670
|
-
maxLayer: data2.hnsw_graph.max_layer,
|
|
671
|
-
m: data2.hnsw_graph.m,
|
|
672
|
-
efConstruction: data2.hnsw_graph.ef_construction,
|
|
673
|
-
metric: validateHnswMetric2(data2.hnsw_graph.metric),
|
|
674
|
-
nodes: data2.hnsw_graph.nodes
|
|
675
|
-
} : null,
|
|
676
|
-
codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
|
|
677
|
-
};
|
|
678
|
-
}
|
|
679
|
-
return {
|
|
680
|
-
indexName: raw.index_name,
|
|
681
|
-
partitionId: raw.partition_id,
|
|
682
|
-
totalPartitions: raw.total_partitions,
|
|
683
|
-
language: raw.language ?? "english",
|
|
684
|
-
schema: raw.schema ?? {},
|
|
685
|
-
docCount: raw.doc_count ?? 0,
|
|
686
|
-
avgDocLength: raw.avg_doc_length ?? 0,
|
|
687
|
-
documents,
|
|
688
|
-
invertedIndex,
|
|
689
|
-
fieldIndexes: {
|
|
690
|
-
numeric,
|
|
691
|
-
boolean,
|
|
692
|
-
enum: raw.field_indexes?.enum ?? {},
|
|
693
|
-
geopoint,
|
|
694
|
-
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
695
|
-
},
|
|
696
|
-
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
697
|
-
vectorData,
|
|
698
|
-
statistics: {
|
|
699
|
-
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
700
|
-
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
701
|
-
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
702
|
-
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
703
|
-
}
|
|
704
|
-
};
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
export { deserializePayloadV1, deserializePayloadV2, documentsOfPayloadV2, growPartitionsTo, loadSegmentedCheckpoint, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeMetadataEnvelope };
|