@delali/narsil 0.3.1-next.98 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
package/dist/chunk-C7SLA6CM.mjs
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
// src/workers/node-worker.ts
|
|
2
|
-
async function spawnNodeWorker(entryPoint, resourceLimits) {
|
|
3
|
-
try {
|
|
4
|
-
const workerThreads = await import('worker_threads');
|
|
5
|
-
return new workerThreads.Worker(entryPoint, { resourceLimits });
|
|
6
|
-
} catch {
|
|
7
|
-
return null;
|
|
8
|
-
}
|
|
9
|
-
}
|
|
10
|
-
async function nodeThreadId() {
|
|
11
|
-
try {
|
|
12
|
-
const workerThreads = await import('worker_threads');
|
|
13
|
-
return workerThreads.threadId;
|
|
14
|
-
} catch {
|
|
15
|
-
return 0;
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export { nodeThreadId, spawnNodeWorker };
|
package/dist/chunk-COVCQ73F.mjs
DELETED
|
@@ -1,587 +0,0 @@
|
|
|
1
|
-
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-WCJKUGHB.mjs';
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes } from './chunk-HXJBSHXF.mjs';
|
|
3
|
-
import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
|
|
4
|
-
import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
5
|
-
import { MAX_DOC_ID_LENGTH } from './chunk-LXWDQLTW.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
7
|
-
|
|
8
|
-
// src/vector/filter-ordinals.ts
|
|
9
|
-
var UNRESOLVED_ORDINAL = -1;
|
|
10
|
-
var NO_VECTOR_MARK = -2;
|
|
11
|
-
function freshOrdinals(length) {
|
|
12
|
-
return new Int32Array(length).fill(UNRESOLVED_ORDINAL);
|
|
13
|
-
}
|
|
14
|
-
function mapsOf(owner) {
|
|
15
|
-
owner.vectorOrdinalMaps ??= { bytes: 0, byField: /* @__PURE__ */ new Map() };
|
|
16
|
-
return owner.vectorOrdinalMaps;
|
|
17
|
-
}
|
|
18
|
-
function mapFor(maps, docStore, space, numberingGeneration) {
|
|
19
|
-
const held = maps.byField.get(space.fieldName);
|
|
20
|
-
if (held !== void 0 && held.header === space.header && held.layoutGeneration === space.layoutGeneration && held.numberingGeneration === numberingGeneration) {
|
|
21
|
-
return held;
|
|
22
|
-
}
|
|
23
|
-
const map = {
|
|
24
|
-
header: space.header,
|
|
25
|
-
layoutGeneration: space.layoutGeneration,
|
|
26
|
-
numberingGeneration,
|
|
27
|
-
ordinals: freshOrdinals(docStore.internalIdCapacity())
|
|
28
|
-
};
|
|
29
|
-
maps.bytes += map.ordinals.byteLength - (held?.ordinals.byteLength ?? 0);
|
|
30
|
-
maps.byField.set(space.fieldName, map);
|
|
31
|
-
return map;
|
|
32
|
-
}
|
|
33
|
-
function growTo(maps, map, length) {
|
|
34
|
-
const grown = freshOrdinals(Math.max(length, map.ordinals.length * 2));
|
|
35
|
-
grown.set(map.ordinals);
|
|
36
|
-
maps.bytes += grown.byteLength - map.ordinals.byteLength;
|
|
37
|
-
map.ordinals = grown;
|
|
38
|
-
}
|
|
39
|
-
function ordinalAt(maps, map, internalId, docStore, space) {
|
|
40
|
-
if (internalId >= map.ordinals.length) growTo(maps, map, internalId + 1);
|
|
41
|
-
const held = map.ordinals[internalId];
|
|
42
|
-
if (held >= 0 && space.holdsCurrentOrdinal(held)) return held;
|
|
43
|
-
if (held <= NO_VECTOR_MARK && NO_VECTOR_MARK - held === space.slots) return UNRESOLVED_ORDINAL;
|
|
44
|
-
const docId = docStore.getExternalId(internalId);
|
|
45
|
-
const resolved = docId === void 0 ? void 0 : space.ordinalOf(docId);
|
|
46
|
-
map.ordinals[internalId] = resolved ?? NO_VECTOR_MARK - space.slots;
|
|
47
|
-
return resolved ?? UNRESOLVED_ORDINAL;
|
|
48
|
-
}
|
|
49
|
-
function ordinalFilterForSubsets(subsets, space) {
|
|
50
|
-
const filter = createOrdinalFilter(space.slots);
|
|
51
|
-
for (const subset of subsets) {
|
|
52
|
-
const docStore = subset.owner.docStore;
|
|
53
|
-
if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
|
|
54
|
-
const resolver = docStore.resolver();
|
|
55
|
-
const maps = mapsOf(subset.owner);
|
|
56
|
-
const map = mapFor(maps, docStore, space, subset.numberingGeneration);
|
|
57
|
-
const bits = subset.bits;
|
|
58
|
-
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
59
|
-
let word = bits[wordIndex];
|
|
60
|
-
if (word === 0) continue;
|
|
61
|
-
const base = wordIndex << 5;
|
|
62
|
-
while (word !== 0) {
|
|
63
|
-
const internalId = base + (Math.clz32(word & -word) ^ 31);
|
|
64
|
-
word &= word - 1;
|
|
65
|
-
if (!resolver.isLive(internalId)) continue;
|
|
66
|
-
const ordinal = ordinalAt(maps, map, internalId, docStore, space);
|
|
67
|
-
if (ordinal < 0 || space.isRemoved(ordinal)) continue;
|
|
68
|
-
addToOrdinalFilter(filter, ordinal);
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
return filter;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// src/vector/vector-index/constants.ts
|
|
76
|
-
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
77
|
-
var DEFAULT_FILTER_THRESHOLD = 0.03;
|
|
78
|
-
var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
|
|
79
|
-
var ESTIMATED_MS_PER_VECTOR_REBUILD = 0.15;
|
|
80
|
-
var WORKER_COPY_MIN_VECTORS = 1024;
|
|
81
|
-
var BUILD_CHUNK_SIZE = 100;
|
|
82
|
-
var REBUILD_REMOVED_RATIO = 0.2;
|
|
83
|
-
var VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE = 0.2;
|
|
84
|
-
var VECTOR_FILE_FIXED_OVERHEAD_BYTES = 65536;
|
|
85
|
-
var VECTOR_FILE_STRING_HEADER_BYTES = 5;
|
|
86
|
-
var VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT = 9;
|
|
87
|
-
var OSQ4_MIN_DIMENSION = 384;
|
|
88
|
-
var DISK_ADOPTION_YIELD_INTERVAL = 4096;
|
|
89
|
-
|
|
90
|
-
// src/vector/vector-index/shared.ts
|
|
91
|
-
var VECTOR_WORKER_COPIES_ALLOWED = { enabled: true };
|
|
92
|
-
function liveSize(state) {
|
|
93
|
-
return state.store.size - state.tombstones.size;
|
|
94
|
-
}
|
|
95
|
-
function threadsHoldCurrentLayout(state) {
|
|
96
|
-
return state.sharedLayoutRevision === state.store.handles.layoutRevision;
|
|
97
|
-
}
|
|
98
|
-
function assignStorePartitions(state, resolve) {
|
|
99
|
-
for (let ordinal = 0; ordinal < state.store.slots; ordinal += 1) {
|
|
100
|
-
if (state.store.partitionOfOrdinal(ordinal) !== void 0) continue;
|
|
101
|
-
const docId = state.store.docIdForOrdinal(ordinal);
|
|
102
|
-
if (docId === void 0) continue;
|
|
103
|
-
const partitionId = resolve(docId);
|
|
104
|
-
if (partitionId === void 0) {
|
|
105
|
-
state.store.forgetPartition(docId);
|
|
106
|
-
continue;
|
|
107
|
-
}
|
|
108
|
-
state.store.setPartition(docId, partitionId);
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
function emptyFieldBeforeRestore(state) {
|
|
112
|
-
state.store.clear();
|
|
113
|
-
state.tombstones.clear();
|
|
114
|
-
state.buffer.clear();
|
|
115
|
-
state.pendingLocations.clear();
|
|
116
|
-
adoptGraph(state, null);
|
|
117
|
-
state.osq?.clear();
|
|
118
|
-
state.savedFiles = [];
|
|
119
|
-
state.savedSignature = null;
|
|
120
|
-
}
|
|
121
|
-
function releaseReplacedVectors(state) {
|
|
122
|
-
for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
|
|
123
|
-
}
|
|
124
|
-
function adoptGraph(state, graph) {
|
|
125
|
-
if (state.hnsw !== null && state.hnsw !== graph) state.hnsw.retire();
|
|
126
|
-
if (graph === null && state.hnsw !== null) {
|
|
127
|
-
state.hnsw.clear();
|
|
128
|
-
}
|
|
129
|
-
state.hnsw = graph;
|
|
130
|
-
state.compactedNodeCount = 0;
|
|
131
|
-
if (graph === null) return;
|
|
132
|
-
for (const docId of state.tombstones) {
|
|
133
|
-
if (graph.has(docId)) {
|
|
134
|
-
graph.markTombstone(docId);
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
function graphNeedsRebuild(state) {
|
|
139
|
-
const graph = state.hnsw;
|
|
140
|
-
if (graph === null) return false;
|
|
141
|
-
const removed = graph.tombstoneCount + state.compactedNodeCount;
|
|
142
|
-
const held = graph.size + removed;
|
|
143
|
-
if (held === 0) return false;
|
|
144
|
-
return removed / held > REBUILD_REMOVED_RATIO;
|
|
145
|
-
}
|
|
146
|
-
function yieldToEventLoop() {
|
|
147
|
-
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
148
|
-
}
|
|
149
|
-
function* allLiveDocIds(state) {
|
|
150
|
-
for (const docId of state.store.docIds()) {
|
|
151
|
-
if (state.tombstones.has(docId)) continue;
|
|
152
|
-
yield docId;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
function ordinalFilterForPartitions(state, partitionIds) {
|
|
156
|
-
const filter = state.store.partitionFilter(partitionIds);
|
|
157
|
-
for (const docId of state.tombstones) {
|
|
158
|
-
const ordinal = state.store.getOrdinal(docId);
|
|
159
|
-
if (ordinal === void 0) continue;
|
|
160
|
-
removeFromOrdinalFilter(filter, ordinal);
|
|
161
|
-
}
|
|
162
|
-
return filter;
|
|
163
|
-
}
|
|
164
|
-
function ordinalSpaceOf(state) {
|
|
165
|
-
const store = state.store;
|
|
166
|
-
const handles = store.handles;
|
|
167
|
-
const removed = new Uint8Array(handles.removed);
|
|
168
|
-
return {
|
|
169
|
-
fieldName: state.fieldName,
|
|
170
|
-
slots: store.slots,
|
|
171
|
-
header: handles.header,
|
|
172
|
-
layoutGeneration: Atomics.load(handles.header, STORE_ORDINAL_LAYOUT_GENERATION),
|
|
173
|
-
ordinalOf: (docId) => store.getOrdinal(docId),
|
|
174
|
-
holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal),
|
|
175
|
-
isRemoved: (ordinal) => ordinal < removed.length && removed[ordinal] === 1
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
function filterForOptions(state, options) {
|
|
179
|
-
if (options.filterSubsets !== void 0) {
|
|
180
|
-
if (state.store.dimension === 0) return createOrdinalFilter(0);
|
|
181
|
-
return ordinalFilterForSubsets(options.filterSubsets, ordinalSpaceOf(state));
|
|
182
|
-
}
|
|
183
|
-
if (options.filterPartitions !== void 0) {
|
|
184
|
-
return ordinalFilterForPartitions(state, options.filterPartitions);
|
|
185
|
-
}
|
|
186
|
-
return void 0;
|
|
187
|
-
}
|
|
188
|
-
function liveOrdinals(state) {
|
|
189
|
-
const ordinals = [];
|
|
190
|
-
for (let ordinal = 0; ordinal < state.store.slots; ordinal++) {
|
|
191
|
-
const docId = state.store.docIdForOrdinal(ordinal);
|
|
192
|
-
if (docId !== void 0 && !state.tombstones.has(docId)) ordinals.push(ordinal);
|
|
193
|
-
}
|
|
194
|
-
return Int32Array.from(ordinals);
|
|
195
|
-
}
|
|
196
|
-
function calibrateQuantizer(state) {
|
|
197
|
-
if (state.osq === null || state.store.size === 0) return;
|
|
198
|
-
state.osq.calibrate(liveOrdinals(state));
|
|
199
|
-
}
|
|
200
|
-
function recalibrateFromStore(state) {
|
|
201
|
-
if (state.osq === null) return;
|
|
202
|
-
state.osq.recalibrate(liveOrdinals(state));
|
|
203
|
-
}
|
|
204
|
-
function fieldHandlesOf(state, graph, searchable) {
|
|
205
|
-
return {
|
|
206
|
-
dimension: state.dimension,
|
|
207
|
-
quantization: state.osq === null ? "none" : state.quantizationMode,
|
|
208
|
-
metric: state.metric,
|
|
209
|
-
store: state.store.handles,
|
|
210
|
-
graph: graph === null ? null : graph.handles,
|
|
211
|
-
filterThreshold: state.filterThreshold,
|
|
212
|
-
searchable
|
|
213
|
-
};
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
// src/vector/vector-index/persistence.ts
|
|
217
|
-
function recordFor(state, quantizer, docId) {
|
|
218
|
-
const record = quantizer.recordOf(docId);
|
|
219
|
-
if (record !== void 0) return record;
|
|
220
|
-
if (!state.store.has(docId)) return void 0;
|
|
221
|
-
quantizer.quantize(docId);
|
|
222
|
-
return quantizer.recordOf(docId);
|
|
223
|
-
}
|
|
224
|
-
function codedCentroid(state) {
|
|
225
|
-
const centroid = state.osq?.centroid ?? null;
|
|
226
|
-
if (centroid === null || state.store.handles.codeLayout === null || state.hnsw === null) return null;
|
|
227
|
-
return Array.from(centroid);
|
|
228
|
-
}
|
|
229
|
-
function sameCentroid(state, planned) {
|
|
230
|
-
const current = state.osq?.centroid ?? null;
|
|
231
|
-
if (current === null || current.length !== planned.length) return false;
|
|
232
|
-
for (let i = 0; i < planned.length; i++) {
|
|
233
|
-
if (current[i] !== planned[i]) return false;
|
|
234
|
-
}
|
|
235
|
-
return true;
|
|
236
|
-
}
|
|
237
|
-
function partCodes(state, docIds, centroid, tombstonedAt) {
|
|
238
|
-
const quantizer = state.osq;
|
|
239
|
-
const layout = state.store.handles.codeLayout;
|
|
240
|
-
if (centroid === null || quantizer === null || layout === null) return null;
|
|
241
|
-
if (!sameCentroid(state, centroid)) {
|
|
242
|
-
throw new NarsilError(
|
|
243
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
244
|
-
`The vector field "${state.fieldName}" was recalibrated while a save was writing its vectors`,
|
|
245
|
-
{ fieldName: state.fieldName }
|
|
246
|
-
);
|
|
247
|
-
}
|
|
248
|
-
const records = new Uint8Array(docIds.length * layout.slotStride);
|
|
249
|
-
for (let i = 0; i < docIds.length; i++) {
|
|
250
|
-
const tombstoned = tombstonedAt.get(i);
|
|
251
|
-
const record = tombstoned === void 0 ? recordFor(state, quantizer, docIds[i]) : quantizer.recordAtOrdinal(tombstoned);
|
|
252
|
-
if (record !== void 0) records.set(record, i * layout.slotStride);
|
|
253
|
-
}
|
|
254
|
-
return { bits: quantizer.bits, centroid, records };
|
|
255
|
-
}
|
|
256
|
-
function splitSequences(parts) {
|
|
257
|
-
const sequences = [];
|
|
258
|
-
let current = null;
|
|
259
|
-
for (const part of parts) {
|
|
260
|
-
if (current === null || part.part === 0) {
|
|
261
|
-
current = { docIds: [], graphs: [], codes: null, codesComplete: true };
|
|
262
|
-
sequences.push(current);
|
|
263
|
-
}
|
|
264
|
-
current.docIds.push(...part.docIds);
|
|
265
|
-
for (let index = 0; index < part.graphs.length; index++) {
|
|
266
|
-
const graph = part.graphs[index];
|
|
267
|
-
const held = current.graphs[index];
|
|
268
|
-
if (held === void 0) current.graphs[index] = { ...graph, nodes: [...graph.nodes] };
|
|
269
|
-
else held.nodes.push(...graph.nodes);
|
|
270
|
-
}
|
|
271
|
-
if (part.codes === null) {
|
|
272
|
-
if (part.docIds.length > 0) current.codesComplete = false;
|
|
273
|
-
continue;
|
|
274
|
-
}
|
|
275
|
-
if (current.codes === null) {
|
|
276
|
-
current.codes = { bits: part.codes.bits, centroid: part.codes.centroid, records: part.codes.records };
|
|
277
|
-
continue;
|
|
278
|
-
}
|
|
279
|
-
if (current.codes.bits !== part.codes.bits) {
|
|
280
|
-
current.codesComplete = false;
|
|
281
|
-
continue;
|
|
282
|
-
}
|
|
283
|
-
const joined = new Uint8Array(current.codes.records.length + part.codes.records.length);
|
|
284
|
-
joined.set(current.codes.records, 0);
|
|
285
|
-
joined.set(part.codes.records, current.codes.records.length);
|
|
286
|
-
current.codes = { ...current.codes, records: joined };
|
|
287
|
-
}
|
|
288
|
-
return sequences;
|
|
289
|
-
}
|
|
290
|
-
function restoreCodes(state, sequences) {
|
|
291
|
-
const quantizer = state.osq;
|
|
292
|
-
const layout = state.store.handles.codeLayout;
|
|
293
|
-
if (quantizer === null || layout === null) return;
|
|
294
|
-
const usable = sequences.filter((sequence) => sequence.docIds.length > 0);
|
|
295
|
-
const single = usable.length === 1 ? usable[0] : null;
|
|
296
|
-
if (single === null || single.codes === null || !single.codesComplete || single.codes.bits !== quantizer.bits) {
|
|
297
|
-
recalibrateFromStore(state);
|
|
298
|
-
return;
|
|
299
|
-
}
|
|
300
|
-
quantizer.restoreCentroid(Float32Array.from(single.codes.centroid));
|
|
301
|
-
const recordBytes = layout.slotStride;
|
|
302
|
-
for (let i = 0; i < single.docIds.length; i++) {
|
|
303
|
-
quantizer.restoreRecord(single.docIds[i], single.codes.records.subarray(i * recordBytes, (i + 1) * recordBytes));
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
function restoreGraphs(state, graphs) {
|
|
307
|
-
if (graphs.length === 0) {
|
|
308
|
-
for (const docId of state.store.docIds()) {
|
|
309
|
-
if (!state.tombstones.has(docId)) state.buffer.add(docId);
|
|
310
|
-
}
|
|
311
|
-
return;
|
|
312
|
-
}
|
|
313
|
-
const graphData = graphs[0];
|
|
314
|
-
const restoredHnsw = createHNSWIndex(
|
|
315
|
-
state.dimension,
|
|
316
|
-
state.store,
|
|
317
|
-
{
|
|
318
|
-
m: graphData.m ?? state.hnswConfig?.m,
|
|
319
|
-
efConstruction: graphData.efConstruction ?? state.hnswConfig?.efConstruction,
|
|
320
|
-
metric: graphData.metric ?? state.hnswConfig?.metric
|
|
321
|
-
},
|
|
322
|
-
state.osq ?? void 0
|
|
323
|
-
);
|
|
324
|
-
restoredHnsw.deserialize(graphData);
|
|
325
|
-
adoptGraph(state, restoredHnsw);
|
|
326
|
-
for (let i = 1; i < graphs.length; i++) {
|
|
327
|
-
for (const [nodeDocId] of graphs[i].nodes) {
|
|
328
|
-
if (!restoredHnsw.has(nodeDocId) && state.store.has(nodeDocId)) restoredHnsw.insertNode(nodeDocId);
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
for (const docId of state.store.docIds()) {
|
|
332
|
-
if (state.tombstones.has(docId)) continue;
|
|
333
|
-
if (!restoredHnsw.has(docId)) state.buffer.add(docId);
|
|
334
|
-
}
|
|
335
|
-
if (state.buffer.size > restoredHnsw.size) {
|
|
336
|
-
adoptGraph(state, null);
|
|
337
|
-
state.buffer.clear();
|
|
338
|
-
for (const docId of state.store.docIds()) {
|
|
339
|
-
if (!state.tombstones.has(docId)) state.buffer.add(docId);
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
function insertPart(state, part) {
|
|
344
|
-
const dimension = state.dimension;
|
|
345
|
-
const vectors = bytesToVectors(part.vectors);
|
|
346
|
-
for (let i = 0; i < part.docIds.length; i++) {
|
|
347
|
-
state.store.insert(part.docIds[i], vectors.subarray(i * dimension, (i + 1) * dimension));
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
function deserialize(state, parts) {
|
|
351
|
-
for (const part of parts) {
|
|
352
|
-
if (part.dimension !== state.dimension) {
|
|
353
|
-
throw new NarsilError(
|
|
354
|
-
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
355
|
-
`Payload dimension ${part.dimension} does not match index dimension ${state.dimension}`,
|
|
356
|
-
{ expected: state.dimension, received: part.dimension }
|
|
357
|
-
);
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
const sequences = splitSequences(parts);
|
|
361
|
-
const graphs = [];
|
|
362
|
-
for (const sequence of sequences) {
|
|
363
|
-
for (const graph of sequence.graphs) {
|
|
364
|
-
if (graph.nodes.length > 0) graphs.push(graph);
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
emptyFieldBeforeRestore(state);
|
|
368
|
-
for (const part of parts) insertPart(state, part);
|
|
369
|
-
if (state.osq !== null && osqBitsOf(state.quantizationMode) !== null && graphs.length > 0) {
|
|
370
|
-
restoreCodes(state, sequences);
|
|
371
|
-
}
|
|
372
|
-
restoreGraphs(state, graphs);
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
// src/vector/vector-index/checkpoint-plan.ts
|
|
376
|
-
var ORDINAL_HOLDS_NO_NUMBER = -1;
|
|
377
|
-
function vectorsPerFile(dimension, codeBits) {
|
|
378
|
-
const recordBytes = codeBits === 1 || codeBits === 2 || codeBits === 4 || codeBits === 8 ? osqRecordBytes(dimension, codeBits) : 0;
|
|
379
|
-
const docIdBytes = VECTOR_FILE_STRING_HEADER_BYTES + MAX_DOC_ID_LENGTH * 3;
|
|
380
|
-
const perVector = dimension * 4 + recordBytes + docIdBytes;
|
|
381
|
-
const overhead = VECTOR_FILE_FIXED_OVERHEAD_BYTES + dimension * VECTOR_FILE_CENTROID_BYTES_PER_COMPONENT;
|
|
382
|
-
const fitting = Math.floor((MAX_FILE_BYTES - overhead) / perVector);
|
|
383
|
-
return Math.max(1, Math.min(VECTOR_FILE_MAX_VECTORS, fitting));
|
|
384
|
-
}
|
|
385
|
-
function fileCapacityOf(state) {
|
|
386
|
-
return vectorsPerFile(state.dimension, osqBitsOf(state.quantizationMode));
|
|
387
|
-
}
|
|
388
|
-
function calibrationOf(state) {
|
|
389
|
-
if (codedCentroid(state) === null) return null;
|
|
390
|
-
return Atomics.load(state.store.handles.header, STORE_CALIBRATION_GENERATION);
|
|
391
|
-
}
|
|
392
|
-
function signatureOf(state) {
|
|
393
|
-
return { revision: state.revision, graphSize: state.hnsw?.size ?? -1, calibration: calibrationOf(state) };
|
|
394
|
-
}
|
|
395
|
-
function sameSignature(a, b) {
|
|
396
|
-
return a !== null && a.revision === b.revision && a.graphSize === b.graphSize && a.calibration === b.calibration;
|
|
397
|
-
}
|
|
398
|
-
function registryMatches(state, listedKeys) {
|
|
399
|
-
const listed = listedKeys ?? [];
|
|
400
|
-
if (listed.length !== state.savedFiles.length) return false;
|
|
401
|
-
return state.savedFiles.every((file, index) => file.key === listed[index]);
|
|
402
|
-
}
|
|
403
|
-
function isLive(state, ordinal) {
|
|
404
|
-
if (ordinal === NO_LIVE_VECTOR_AT_POSITION) return false;
|
|
405
|
-
const docId = state.store.docIdForOrdinal(ordinal);
|
|
406
|
-
return docId !== void 0 && !state.tombstones.has(docId);
|
|
407
|
-
}
|
|
408
|
-
function isGraphTombstone(state, ordinal) {
|
|
409
|
-
return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
|
|
410
|
-
}
|
|
411
|
-
function isCarried(state, ordinal) {
|
|
412
|
-
return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
|
|
413
|
-
}
|
|
414
|
-
function carriedCountOf(state, file) {
|
|
415
|
-
let carried = 0;
|
|
416
|
-
for (let i = 0; i < file.ordinals.length; i++) if (isCarried(state, file.ordinals[i])) carried += 1;
|
|
417
|
-
return carried;
|
|
418
|
-
}
|
|
419
|
-
function deadBitsOf(state, file) {
|
|
420
|
-
return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
|
|
421
|
-
}
|
|
422
|
-
function filesWorthKeeping(state, listed, calibration, carriedCounts) {
|
|
423
|
-
const kept = [];
|
|
424
|
-
for (const file of listed) {
|
|
425
|
-
const carried = carriedCountOf(state, file);
|
|
426
|
-
const deadWithoutNodeShare = file.ordinals.length === 0 ? 1 : 1 - carried / file.ordinals.length;
|
|
427
|
-
if (file.calibration !== calibration || deadWithoutNodeShare > VECTOR_FILE_REPLACED_ABOVE_DEAD_SHARE) continue;
|
|
428
|
-
kept.push(file);
|
|
429
|
-
carriedCounts.push(carried);
|
|
430
|
-
}
|
|
431
|
-
return kept;
|
|
432
|
-
}
|
|
433
|
-
function absorbSmallerPartialFiles(kept, carriedCounts, vectorsToWrite, capacity) {
|
|
434
|
-
let pending = vectorsToWrite;
|
|
435
|
-
for (let index = kept.length - 1; index >= 0 && pending > 0; index--) {
|
|
436
|
-
if (kept[index].ordinals.length >= capacity) continue;
|
|
437
|
-
if (carriedCounts[index] > pending) return;
|
|
438
|
-
pending += carriedCounts[index];
|
|
439
|
-
kept.splice(index, 1);
|
|
440
|
-
carriedCounts.splice(index, 1);
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
function ordinalsOutsideOf(state, kept) {
|
|
444
|
-
const slots = state.store.slots;
|
|
445
|
-
const saved = new Uint8Array(slots);
|
|
446
|
-
for (const file of kept) {
|
|
447
|
-
for (let i = 0; i < file.ordinals.length; i++) {
|
|
448
|
-
const ordinal = file.ordinals[i];
|
|
449
|
-
if (ordinal !== NO_LIVE_VECTOR_AT_POSITION && ordinal < slots) saved[ordinal] = 1;
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
const outside = [];
|
|
453
|
-
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
454
|
-
if (saved[ordinal] === 0 && isCarried(state, ordinal)) outside.push(ordinal);
|
|
455
|
-
}
|
|
456
|
-
return outside;
|
|
457
|
-
}
|
|
458
|
-
function countOutside(state, carriedCounts) {
|
|
459
|
-
let held = 0;
|
|
460
|
-
for (const carried of carriedCounts) held += carried;
|
|
461
|
-
const tombstonedNodes = state.hnsw?.tombstoneCount ?? 0;
|
|
462
|
-
return Math.max(0, state.store.size - state.tombstones.size + tombstonedNodes - held);
|
|
463
|
-
}
|
|
464
|
-
function numberEveryVector(state, kept, written) {
|
|
465
|
-
let total = written.length;
|
|
466
|
-
for (const file of kept) total += file.ordinals.length;
|
|
467
|
-
const numberOfOrdinal = new Int32Array(state.store.slots).fill(ORDINAL_HOLDS_NO_NUMBER);
|
|
468
|
-
const ordinalOfNumber = new Int32Array(total).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
469
|
-
let number = 0;
|
|
470
|
-
for (const file of kept) {
|
|
471
|
-
for (let i = 0; i < file.ordinals.length; i++, number++) {
|
|
472
|
-
const ordinal = file.ordinals[i];
|
|
473
|
-
if (!isCarried(state, ordinal)) continue;
|
|
474
|
-
numberOfOrdinal[ordinal] = number;
|
|
475
|
-
ordinalOfNumber[number] = ordinal;
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
for (const ordinal of written) {
|
|
479
|
-
numberOfOrdinal[ordinal] = number;
|
|
480
|
-
ordinalOfNumber[number] = ordinal;
|
|
481
|
-
number += 1;
|
|
482
|
-
}
|
|
483
|
-
return { numberOfOrdinal, ordinalOfNumber };
|
|
484
|
-
}
|
|
485
|
-
function graphPartsOf(state, numbers) {
|
|
486
|
-
if (state.hnsw === null || numbers.ordinalOfNumber.length === 0) return null;
|
|
487
|
-
const parts = state.hnsw.serializeNumbered(
|
|
488
|
-
numbers.numberOfOrdinal,
|
|
489
|
-
numbers.ordinalOfNumber,
|
|
490
|
-
VECTOR_GRAPH_PART_VECTORS
|
|
491
|
-
);
|
|
492
|
-
return parts.map((graph) => ({ v: VECTOR_GRAPH_PART_PAYLOAD_VERSION, fieldName: state.fieldName, graphs: [graph] }));
|
|
493
|
-
}
|
|
494
|
-
function graphPartsWithoutNodes(parts, numbers) {
|
|
495
|
-
return parts.map((part, index) => ({
|
|
496
|
-
...part,
|
|
497
|
-
graphs: part.graphs.map((graph) => graphWithoutNodes(graph, index * VECTOR_GRAPH_PART_VECTORS, numbers))
|
|
498
|
-
}));
|
|
499
|
-
}
|
|
500
|
-
function plannedVectorOf(state, ordinal) {
|
|
501
|
-
const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
|
|
502
|
-
return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
|
|
503
|
-
}
|
|
504
|
-
function readFile(state, planned, centroid) {
|
|
505
|
-
const dimension = state.dimension;
|
|
506
|
-
const vectors = new Float32Array(planned.length * dimension);
|
|
507
|
-
const ordinals = new Int32Array(planned.length).fill(NO_LIVE_VECTOR_AT_POSITION);
|
|
508
|
-
const tombstonedAt = /* @__PURE__ */ new Map();
|
|
509
|
-
const docIds = planned.map((vector) => vector.docId);
|
|
510
|
-
for (let i = 0; i < planned.length; i++) {
|
|
511
|
-
const { ordinal, docId, tombstoned } = planned[i];
|
|
512
|
-
const held = tombstoned || state.store.holdsOrdinal(ordinal) ? ordinal : state.store.getOrdinal(docId);
|
|
513
|
-
const entry = held === void 0 ? void 0 : state.store.entryForOrdinal(held);
|
|
514
|
-
if (held === void 0 || entry === void 0) continue;
|
|
515
|
-
vectors.set(entry.vector, i * dimension);
|
|
516
|
-
ordinals[i] = held;
|
|
517
|
-
if (tombstoned) tombstonedAt.set(i, held);
|
|
518
|
-
}
|
|
519
|
-
return {
|
|
520
|
-
payload: {
|
|
521
|
-
v: VECTOR_FILE_PAYLOAD_VERSION,
|
|
522
|
-
fieldName: state.fieldName,
|
|
523
|
-
dimension,
|
|
524
|
-
docIds,
|
|
525
|
-
codes: partCodes(state, docIds, centroid, tombstonedAt),
|
|
526
|
-
vectors: vectorsToBytes(vectors)
|
|
527
|
-
},
|
|
528
|
-
ordinals,
|
|
529
|
-
dead: deadBitsWhere(planned.length, (i) => planned[i].tombstoned || ordinals[i] === NO_LIVE_VECTOR_AT_POSITION)
|
|
530
|
-
};
|
|
531
|
-
}
|
|
532
|
-
function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
533
|
-
const signature = signatureOf(state);
|
|
534
|
-
const calibration = signature.calibration;
|
|
535
|
-
const trusted = registryMatches(state, listedKeys);
|
|
536
|
-
const capacity = fileCapacityOf(state);
|
|
537
|
-
const carriedCounts = [];
|
|
538
|
-
const keptFiles = filesWorthKeeping(state, trusted ? state.savedFiles : [], calibration, carriedCounts);
|
|
539
|
-
absorbSmallerPartialFiles(keptFiles, carriedCounts, countOutside(state, carriedCounts), capacity);
|
|
540
|
-
const written = ordinalsOutsideOf(state, keptFiles);
|
|
541
|
-
const planned = written.map((ordinal) => plannedVectorOf(state, ordinal));
|
|
542
|
-
const centroid = codedCentroid(state);
|
|
543
|
-
const listedBefore = trusted && listedKeys !== null;
|
|
544
|
-
const unchanged = !rewritesGraph && listedBefore && sameSignature(state.savedSignature, signature) && written.length === 0;
|
|
545
|
-
let firstWrittenNumber = 0;
|
|
546
|
-
for (const file of keptFiles) firstWrittenNumber += file.ordinals.length;
|
|
547
|
-
const graphParts = unchanged ? null : graphPartsOf(state, numberEveryVector(state, keptFiles, written));
|
|
548
|
-
const vanished = /* @__PURE__ */ new Set();
|
|
549
|
-
let vanishedWhenStripped = 0;
|
|
550
|
-
let graphPartsToWrite = graphParts;
|
|
551
|
-
return {
|
|
552
|
-
unchanged,
|
|
553
|
-
keptFiles,
|
|
554
|
-
kept: keptFiles.map((file) => ({
|
|
555
|
-
id: file.id,
|
|
556
|
-
key: file.key,
|
|
557
|
-
count: file.ordinals.length,
|
|
558
|
-
dead: deadBitsOf(state, file)
|
|
559
|
-
})),
|
|
560
|
-
newFiles: Math.ceil(written.length / capacity),
|
|
561
|
-
get graphParts() {
|
|
562
|
-
if (graphParts !== null && vanished.size !== vanishedWhenStripped) {
|
|
563
|
-
graphPartsToWrite = graphPartsWithoutNodes(graphParts, vanished);
|
|
564
|
-
vanishedWhenStripped = vanished.size;
|
|
565
|
-
}
|
|
566
|
-
return graphPartsToWrite;
|
|
567
|
-
},
|
|
568
|
-
signature,
|
|
569
|
-
readNewFile(index) {
|
|
570
|
-
const start = index * capacity;
|
|
571
|
-
const end = start + capacity;
|
|
572
|
-
const file = readFile(state, planned.slice(start, end), centroid);
|
|
573
|
-
for (let i = 0; i < file.ordinals.length; i++) {
|
|
574
|
-
if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
|
|
575
|
-
}
|
|
576
|
-
return file;
|
|
577
|
-
}
|
|
578
|
-
};
|
|
579
|
-
}
|
|
580
|
-
function recordCheckpoint(state, plan, written) {
|
|
581
|
-
if (plan.unchanged) return;
|
|
582
|
-
const calibration = plan.signature.calibration;
|
|
583
|
-
state.savedFiles = [...plan.keptFiles, ...written.map((file) => ({ ...file, calibration }))];
|
|
584
|
-
state.savedSignature = plan.signature;
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
export { BUILD_CHUNK_SIZE, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, OSQ4_MIN_DIMENSION, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, adoptGraph, allLiveDocIds, assignStorePartitions, calibrateQuantizer, deserialize, emptyFieldBeforeRestore, fieldHandlesOf, filterForOptions, graphNeedsRebuild, liveSize, ordinalFilterForSubsets, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseReplacedVectors, signatureOf, threadsHoldCurrentLayout, vectorsPerFile, yieldToEventLoop };
|
package/dist/chunk-DJHHF2L5.mjs
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
// src/vector/native/backend.ts
|
|
2
|
-
var nativeSearchCoreEnabled = true;
|
|
3
|
-
function nativeSearchCoreIsEnabled() {
|
|
4
|
-
return nativeSearchCoreEnabled;
|
|
5
|
-
}
|
|
6
|
-
function stopUsingTheNativeSearchCore(error) {
|
|
7
|
-
if (!nativeSearchCoreEnabled) return;
|
|
8
|
-
nativeSearchCoreEnabled = false;
|
|
9
|
-
console.warn(
|
|
10
|
-
"The native search core raised an error, so this thread searches through WebAssembly from now on:",
|
|
11
|
-
error instanceof Error ? error.message : String(error)
|
|
12
|
-
);
|
|
13
|
-
}
|
|
14
|
-
function noteFallback(operation) {
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export { nativeSearchCoreIsEnabled, noteFallback, stopUsingTheNativeSearchCore };
|
package/dist/chunk-E74YLAIG.mjs
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
// src/search/constants.ts
|
|
2
|
-
var RESULT_WINDOW = 1e4;
|
|
3
|
-
var DEFAULT_PAGE_SIZE = 10;
|
|
4
|
-
var MAX_CURSOR_LENGTH = 40960;
|
|
5
|
-
var MAX_SORT_FIELDS = 8;
|
|
6
|
-
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
7
|
-
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
8
|
-
var PREFIX_MAX_EXPANSIONS = 50;
|
|
9
|
-
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
10
|
-
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
11
|
-
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
12
|
-
var MAX_TOLERANCE = 10;
|
|
13
|
-
var MAX_PREFIX_LENGTH = 1024;
|
|
14
|
-
var DEFAULT_MAX_PER_GROUP = 1;
|
|
15
|
-
var DEFAULT_RANK_CONSTANT = 60;
|
|
16
|
-
var MIN_RANK_CONSTANT = 1;
|
|
17
|
-
var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
18
|
-
var MIN_LINEAR_ALPHA = 0;
|
|
19
|
-
var MAX_LINEAR_ALPHA = 1;
|
|
20
|
-
var MIN_HYBRID_DEPTH = 1;
|
|
21
|
-
var MAX_HYBRID_DEPTH = 1e4;
|
|
22
|
-
var HYBRID_RESCORE_WINDOW = 10;
|
|
23
|
-
|
|
24
|
-
export { DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, HYBRID_RESCORE_WINDOW, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_HYBRID_DEPTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|