@delali/narsil 0.3.1-next.98 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -1,56 +1,6 @@
|
|
|
1
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf,
|
|
2
|
-
import {
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
-
import { readSync, closeSync, openSync } from 'fs';
|
|
5
|
-
|
|
6
|
-
function createVectorFileReader() {
|
|
7
|
-
const descriptors = /* @__PURE__ */ new Map();
|
|
8
|
-
function descriptorOf(path) {
|
|
9
|
-
const open = descriptors.get(path);
|
|
10
|
-
if (open !== void 0) return open;
|
|
11
|
-
try {
|
|
12
|
-
const descriptor = openSync(path, "r");
|
|
13
|
-
descriptors.set(path, descriptor);
|
|
14
|
-
return descriptor;
|
|
15
|
-
} catch {
|
|
16
|
-
return null;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
function closeDescriptor(path, descriptor) {
|
|
20
|
-
descriptors.delete(path);
|
|
21
|
-
try {
|
|
22
|
-
closeSync(descriptor);
|
|
23
|
-
} catch {
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
return {
|
|
27
|
-
readInto(path, offset, target) {
|
|
28
|
-
const descriptor = descriptorOf(path);
|
|
29
|
-
if (descriptor === null) return false;
|
|
30
|
-
let read = 0;
|
|
31
|
-
while (read < target.byteLength) {
|
|
32
|
-
let chunk;
|
|
33
|
-
try {
|
|
34
|
-
chunk = readSync(descriptor, target, read, target.byteLength - read, offset + read);
|
|
35
|
-
} catch {
|
|
36
|
-
closeDescriptor(path, descriptor);
|
|
37
|
-
return false;
|
|
38
|
-
}
|
|
39
|
-
if (chunk <= 0) return false;
|
|
40
|
-
read += chunk;
|
|
41
|
-
}
|
|
42
|
-
return true;
|
|
43
|
-
},
|
|
44
|
-
retainOnly(paths) {
|
|
45
|
-
for (const [path, descriptor] of descriptors) {
|
|
46
|
-
if (!paths.has(path)) closeDescriptor(path, descriptor);
|
|
47
|
-
}
|
|
48
|
-
},
|
|
49
|
-
close() {
|
|
50
|
-
for (const [path, descriptor] of descriptors) closeDescriptor(path, descriptor);
|
|
51
|
-
}
|
|
52
|
-
};
|
|
53
|
-
}
|
|
1
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, withTransientAdmission, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_LIVE_COUNT, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_SLOTS, writeSequenceOf, openGraphState, fixedView, createSharedVectorStoreHandles, sequenceWritesOf, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, oldestHeldPin, coverOrdinal, nextSequence, publishSequence, createOrdinalFilter, addToOrdinalFilter, magnitude, stampBorn, mayFree, retiredAtOf, requireBytesWithinLimit, stampRetired, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorFileReader, grownByteLength, growBufferTo, resetStoreHeader, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createVectorBlock, ensureBlockSlots } from './chunk-R724DJSG.mjs';
|
|
2
|
+
import { createDocIdMap } from './chunk-FJXJZUWF.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
54
4
|
|
|
55
5
|
// src/vector/vector-store/hot-block-distance.ts
|
|
56
6
|
function jsDistance(a, b, magA, magB, metric) {
|
|
@@ -63,6 +13,11 @@ function jsDistance(a, b, magA, magB, metric) {
|
|
|
63
13
|
return euclideanDistance(a, b);
|
|
64
14
|
}
|
|
65
15
|
}
|
|
16
|
+
function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
|
|
17
|
+
if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
|
|
18
|
+
const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
|
|
19
|
+
return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
|
|
20
|
+
}
|
|
66
21
|
function inMemory(reads, ordinal) {
|
|
67
22
|
return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
|
|
68
23
|
}
|
|
@@ -159,9 +114,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
159
114
|
}
|
|
160
115
|
return diskFile[ordinal];
|
|
161
116
|
}
|
|
162
|
-
|
|
163
|
-
return fileOf(ordinal) !== IN_MEMORY;
|
|
164
|
-
}
|
|
117
|
+
const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
|
|
165
118
|
function holdState(ordinal) {
|
|
166
119
|
if (ordinal < 0) return NOT_HELD;
|
|
167
120
|
if (ordinal >= present.length) {
|
|
@@ -173,9 +126,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
173
126
|
const index = Math.floor(ordinal / capacity);
|
|
174
127
|
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
175
128
|
}
|
|
176
|
-
|
|
177
|
-
return holdState(ordinal) !== NOT_HELD;
|
|
178
|
-
}
|
|
129
|
+
const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
|
|
179
130
|
function readFromFile(file, offset, target) {
|
|
180
131
|
const path = handles.vectorFiles[file];
|
|
181
132
|
if (path === void 0 || path.length === 0) return false;
|
|
@@ -383,6 +334,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
383
334
|
}
|
|
384
335
|
return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
|
|
385
336
|
},
|
|
337
|
+
distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
|
|
386
338
|
hotBlockReads,
|
|
387
339
|
queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
|
|
388
340
|
ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
|
|
@@ -474,31 +426,19 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
474
426
|
return outcome;
|
|
475
427
|
},
|
|
476
428
|
searchOrdinals(query, k, metric, minSimilarity, options) {
|
|
477
|
-
|
|
478
|
-
|
|
429
|
+
const current = graph;
|
|
430
|
+
if (current === null) return [];
|
|
431
|
+
if (options.admission !== void 0) {
|
|
432
|
+
return searchOrdinals(current, docIdOf, query, k, metric, minSimilarity, options);
|
|
433
|
+
}
|
|
434
|
+
return withTransientAdmission(
|
|
435
|
+
store,
|
|
436
|
+
(admission) => searchOrdinals(current, docIdOf, query, k, metric, minSimilarity, { ...options, admission })
|
|
437
|
+
);
|
|
479
438
|
}
|
|
480
439
|
};
|
|
481
440
|
}
|
|
482
441
|
|
|
483
|
-
// src/vector/vector-store/doc-ids.ts
|
|
484
|
-
var encoder = new TextEncoder();
|
|
485
|
-
function appendDocId(handles, ordinal, docId) {
|
|
486
|
-
const bytes = encoder.encode(docId);
|
|
487
|
-
const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
488
|
-
const end = start + bytes.length;
|
|
489
|
-
growBufferTo(handles.docIdBytes, end);
|
|
490
|
-
growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
|
|
491
|
-
new Uint8Array(handles.docIdBytes).set(bytes, start);
|
|
492
|
-
const offsets = new Uint32Array(handles.docIdOffsets);
|
|
493
|
-
offsets[ordinal] = start;
|
|
494
|
-
offsets[ordinal + 1] = end;
|
|
495
|
-
Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
|
|
496
|
-
}
|
|
497
|
-
function resetDocIds(handles) {
|
|
498
|
-
new Uint32Array(handles.docIdOffsets).fill(0);
|
|
499
|
-
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
500
|
-
}
|
|
501
|
-
|
|
502
442
|
// src/vector/vector-store/ordinal-reads.ts
|
|
503
443
|
function entryAt(store, ordinal) {
|
|
504
444
|
return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
|
|
@@ -526,6 +466,11 @@ function createOrdinalReads(openStore, readable) {
|
|
|
526
466
|
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
527
467
|
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
528
468
|
},
|
|
469
|
+
distanceFromQuery(query, queryMagnitude, ordinal, metric) {
|
|
470
|
+
const open = openStore();
|
|
471
|
+
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
472
|
+
return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
|
|
473
|
+
},
|
|
529
474
|
queryDistance(prepared, metric) {
|
|
530
475
|
const open = openStore();
|
|
531
476
|
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
@@ -571,6 +516,25 @@ function snapshotOf(open, documents, replacedOrdinals) {
|
|
|
571
516
|
return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
|
|
572
517
|
}
|
|
573
518
|
|
|
519
|
+
// src/vector/vector-store/doc-ids.ts
|
|
520
|
+
var encoder = new TextEncoder();
|
|
521
|
+
function appendDocId(handles, ordinal, docId) {
|
|
522
|
+
const bytes = encoder.encode(docId);
|
|
523
|
+
const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
524
|
+
const end = start + bytes.length;
|
|
525
|
+
growBufferTo(handles.docIdBytes, end);
|
|
526
|
+
growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
|
|
527
|
+
new Uint8Array(handles.docIdBytes).set(bytes, start);
|
|
528
|
+
const offsets = new Uint32Array(handles.docIdOffsets);
|
|
529
|
+
offsets[ordinal] = start;
|
|
530
|
+
offsets[ordinal + 1] = end;
|
|
531
|
+
Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
|
|
532
|
+
}
|
|
533
|
+
function resetDocIds(handles) {
|
|
534
|
+
new Uint32Array(handles.docIdOffsets, 0, handles.docIdOffsets.byteLength / 4).fill(0);
|
|
535
|
+
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
536
|
+
}
|
|
537
|
+
|
|
574
538
|
// src/vector/vector-store/open-store.ts
|
|
575
539
|
function openHandles(dimension, codeBits, blockBytes) {
|
|
576
540
|
const handles = createSharedVectorStoreHandles(dimension, codeBits, blockBytes);
|
|
@@ -581,7 +545,9 @@ function openHandles(dimension, codeBits, blockBytes) {
|
|
|
581
545
|
present: new Uint8Array(handles.present),
|
|
582
546
|
partitions: new Int32Array(handles.partitions),
|
|
583
547
|
diskFile: new Int32Array(handles.diskFile),
|
|
584
|
-
diskOffset: new Uint32Array(handles.diskOffset)
|
|
548
|
+
diskOffset: new Uint32Array(handles.diskOffset),
|
|
549
|
+
longestDocIdBytes: 0,
|
|
550
|
+
sequences: sequenceWritesOf(handles)
|
|
585
551
|
};
|
|
586
552
|
}
|
|
587
553
|
function relayout(store) {
|
|
@@ -592,17 +558,23 @@ function claimSlot(store, docId, cold) {
|
|
|
592
558
|
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
593
559
|
ensureOrdinal(store, ordinal, cold);
|
|
594
560
|
store.diskFile[ordinal] = IN_MEMORY;
|
|
561
|
+
const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
|
|
595
562
|
appendDocId(store.handles, ordinal, docId);
|
|
563
|
+
const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
|
|
564
|
+
if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
|
|
596
565
|
return ordinal;
|
|
597
566
|
}
|
|
598
567
|
function publishOrdinal(store, ordinal) {
|
|
599
568
|
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
600
569
|
Atomics.store(store.present, ordinal, 1);
|
|
601
570
|
}
|
|
602
|
-
function
|
|
571
|
+
function placeOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
603
572
|
store.magnitudes[ordinal] = vectorMagnitude;
|
|
604
573
|
store.diskOffset[ordinal] = location.offset;
|
|
605
574
|
store.diskFile[ordinal] = location.fileIndex;
|
|
575
|
+
}
|
|
576
|
+
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
577
|
+
placeOnDisk(store, ordinal, vectorMagnitude, location);
|
|
606
578
|
publishOrdinal(store, ordinal);
|
|
607
579
|
}
|
|
608
580
|
function relocateFiles(store, rewrite) {
|
|
@@ -643,14 +615,26 @@ function ensureFloatSlot(store, ordinal) {
|
|
|
643
615
|
const block = handles.blocks[blockIndex];
|
|
644
616
|
if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
|
|
645
617
|
}
|
|
618
|
+
var LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT = 3;
|
|
619
|
+
var encoder2 = new TextEncoder();
|
|
620
|
+
function requireOrdinalWithinLimit(ordinal) {
|
|
621
|
+
const maxOrdinals = VECTOR_MEMORY_LIMITS.maxOrdinals;
|
|
622
|
+
if (ordinal < maxOrdinals) return;
|
|
623
|
+
throw new NarsilError(
|
|
624
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
625
|
+
`A vector field holds at most ${maxOrdinals} vectors, deleted ones included, until it is rebuilt`,
|
|
626
|
+
{ ordinal, maxOrdinals }
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
function requireRoomForSlot(store, docId) {
|
|
630
|
+
const { header, docIdBytes } = store.handles;
|
|
631
|
+
requireOrdinalWithinLimit(Atomics.load(header, STORE_SLOTS));
|
|
632
|
+
const heldBytes = Atomics.load(header, STORE_DOC_ID_BYTES);
|
|
633
|
+
if (heldBytes + docId.length * LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT <= docIdBytes.maxByteLength) return;
|
|
634
|
+
requireBytesWithinLimit(heldBytes + encoder2.encode(docId).length, docIdBytes.maxByteLength);
|
|
635
|
+
}
|
|
646
636
|
function ensureOrdinal(store, ordinal, cold) {
|
|
647
|
-
|
|
648
|
-
throw new NarsilError(
|
|
649
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
650
|
-
`A vector field holds at most ${MAX_VECTOR_ORDINALS} vectors, deleted ones included, until it is rebuilt`,
|
|
651
|
-
{ ordinal, maxOrdinals: MAX_VECTOR_ORDINALS }
|
|
652
|
-
);
|
|
653
|
-
}
|
|
637
|
+
requireOrdinalWithinLimit(ordinal);
|
|
654
638
|
const { handles } = store;
|
|
655
639
|
growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
|
|
656
640
|
growBufferTo(handles.present, ordinal + 1);
|
|
@@ -662,12 +646,27 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
662
646
|
const diskFileBytes = handles.diskFile.byteLength;
|
|
663
647
|
growBufferTo(handles.diskFile, (ordinal + 1) * 4);
|
|
664
648
|
if (handles.diskFile.byteLength > diskFileBytes) {
|
|
665
|
-
new Int32Array(handles.diskFile).fill(IN_MEMORY, diskFileBytes / 4);
|
|
649
|
+
new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY, diskFileBytes / 4);
|
|
666
650
|
}
|
|
667
651
|
growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
|
|
652
|
+
growBufferTo(handles.bornAt, (ordinal + 1) * 8);
|
|
653
|
+
growBufferTo(handles.retiredAt, (ordinal + 1) * 8);
|
|
668
654
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
669
655
|
ensureCodeSlot(store, ordinal);
|
|
670
656
|
}
|
|
657
|
+
function bufferGrowthBytes(buffer, bytes) {
|
|
658
|
+
return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
|
|
659
|
+
}
|
|
660
|
+
function growthBytesFor(store, vectors) {
|
|
661
|
+
const { handles } = store;
|
|
662
|
+
const slots = Atomics.load(handles.header, STORE_SLOTS);
|
|
663
|
+
const last = Math.min(slots + vectors, VECTOR_MEMORY_LIMITS.maxOrdinals) - 1;
|
|
664
|
+
if (last < slots) return 0;
|
|
665
|
+
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
666
|
+
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.bornAt, (last + 1) * 8) + bufferGrowthBytes(handles.retiredAt, (last + 1) * 8) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
667
|
+
if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
|
|
668
|
+
return bytes;
|
|
669
|
+
}
|
|
671
670
|
function writeVector(store, ordinal, vector) {
|
|
672
671
|
const block = store.view.blockOf(ordinal);
|
|
673
672
|
const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
|
|
@@ -685,13 +684,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
|
685
684
|
}
|
|
686
685
|
return true;
|
|
687
686
|
}
|
|
688
|
-
function fileHoldsSameBytes(store, ordinal, location) {
|
|
689
|
-
const fromFile = new Float32Array(store.handles.dimension);
|
|
687
|
+
function fileHoldsSameBytes(store, ordinal, location, fromFile) {
|
|
690
688
|
if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
|
|
691
689
|
const held = store.view.vectorAt(ordinal);
|
|
692
|
-
const
|
|
693
|
-
const
|
|
694
|
-
for (let i = 0; i <
|
|
690
|
+
const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
|
|
691
|
+
const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
|
|
692
|
+
for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
|
|
695
693
|
return true;
|
|
696
694
|
}
|
|
697
695
|
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
@@ -731,89 +729,331 @@ function partitionFilterOf(store, documents, partitionIds) {
|
|
|
731
729
|
return filter;
|
|
732
730
|
}
|
|
733
731
|
|
|
734
|
-
// src/vector/vector-store/
|
|
732
|
+
// src/vector/vector-store/retirement.ts
|
|
735
733
|
var UNKNOWN_PARTITION = -1;
|
|
736
734
|
var NO_PARTITION = -2;
|
|
735
|
+
var LOADED_SEQUENCE = 1;
|
|
736
|
+
var RETIRED_BEFORE_LOAD = 1;
|
|
737
|
+
function oldestPinOf(store) {
|
|
738
|
+
return oldestHeldPin(store.handles.pins);
|
|
739
|
+
}
|
|
740
|
+
function freeable(store, ordinal, oldestPin) {
|
|
741
|
+
return mayFree(retiredAtOf(store.sequences, ordinal), oldestPin);
|
|
742
|
+
}
|
|
743
|
+
function recordPartition(state, store, ordinal, partitionId) {
|
|
744
|
+
const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
|
|
745
|
+
store.partitions[ordinal] = given;
|
|
746
|
+
if (given === UNKNOWN_PARTITION) state.unknownPartitions += 1;
|
|
747
|
+
}
|
|
748
|
+
function freeRetired(state, store, ordinal) {
|
|
749
|
+
state.awaitingFree.delete(ordinal);
|
|
750
|
+
Atomics.store(store.present, ordinal, 0);
|
|
751
|
+
state.checkpointPins.note(ordinal);
|
|
752
|
+
}
|
|
753
|
+
function retireOrdinal(state, store, ordinal, oldestPin) {
|
|
754
|
+
state.ordToDoc[ordinal] = void 0;
|
|
755
|
+
if (store.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
756
|
+
if (freeable(store, ordinal, oldestPin)) {
|
|
757
|
+
freeRetired(state, store, ordinal);
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
state.awaitingFree.add(ordinal);
|
|
761
|
+
}
|
|
762
|
+
function keepReplacedOrdinal(state, store, ordinal) {
|
|
763
|
+
state.ordToDoc[ordinal] = void 0;
|
|
764
|
+
if (store.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
765
|
+
state.replacedOrdinals.add(ordinal);
|
|
766
|
+
}
|
|
767
|
+
function freeAwaitingOrdinals(state) {
|
|
768
|
+
const store = state.open;
|
|
769
|
+
if (store === null || state.awaitingFree.size === 0) return;
|
|
770
|
+
const oldestPin = oldestPinOf(store);
|
|
771
|
+
for (const ordinal of state.awaitingFree) {
|
|
772
|
+
if (freeable(store, ordinal, oldestPin)) freeRetired(state, store, ordinal);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
function claimOrdinal(state, store, docId, cold, partitionId) {
|
|
776
|
+
const ordinal = claimSlot(store, docId, cold);
|
|
777
|
+
recordPartition(state, store, ordinal, partitionId);
|
|
778
|
+
state.ordToDoc[ordinal] = docId;
|
|
779
|
+
state.docToOrd.set(docId, ordinal);
|
|
780
|
+
return ordinal;
|
|
781
|
+
}
|
|
782
|
+
function appendOrdinal(state, store, docId, vector, sequence, partitionId) {
|
|
783
|
+
const ordinal = claimOrdinal(state, store, docId, false, partitionId);
|
|
784
|
+
writeVector(store, ordinal, vector);
|
|
785
|
+
stampBorn(store.sequences, ordinal, sequence);
|
|
786
|
+
publishOrdinal(store, ordinal);
|
|
787
|
+
return ordinal;
|
|
788
|
+
}
|
|
789
|
+
function settleReplacement(state, store, previous, keepsReplacedVector) {
|
|
790
|
+
if (previous === void 0) {
|
|
791
|
+
state.liveCount++;
|
|
792
|
+
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
if (keepsReplacedVector) keepReplacedOrdinal(state, store, previous);
|
|
796
|
+
else retireOrdinal(state, store, previous, oldestPinOf(store));
|
|
797
|
+
}
|
|
798
|
+
function beginWrite(state, store) {
|
|
799
|
+
freeAwaitingOrdinals(state);
|
|
800
|
+
return nextSequence(store.sequences);
|
|
801
|
+
}
|
|
802
|
+
function retireInWrite(store, ordinal, sequence) {
|
|
803
|
+
stampRetired(store.sequences, ordinal, sequence);
|
|
804
|
+
}
|
|
805
|
+
function endWrite(store, sequence) {
|
|
806
|
+
publishSequence(store.sequences, sequence);
|
|
807
|
+
}
|
|
808
|
+
var NOT_RETIRED = 0n;
|
|
809
|
+
function clearRetirement(store, ordinal) {
|
|
810
|
+
const writes = store.sequences;
|
|
811
|
+
coverOrdinal(writes, ordinal);
|
|
812
|
+
Atomics.store(writes.retired, ordinal, NOT_RETIRED);
|
|
813
|
+
}
|
|
814
|
+
function reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector) {
|
|
815
|
+
const store = state.open;
|
|
816
|
+
if (store === null) return false;
|
|
817
|
+
const current = state.docToOrd.get(docId);
|
|
818
|
+
if (current === ordinal) {
|
|
819
|
+
clearRetirement(store, ordinal);
|
|
820
|
+
return true;
|
|
821
|
+
}
|
|
822
|
+
if (state.ordToDoc[ordinal] !== void 0) return false;
|
|
823
|
+
if (!state.replacedOrdinals.delete(ordinal) && !state.awaitingFree.delete(ordinal)) return false;
|
|
824
|
+
const sequence = beginWrite(state, store);
|
|
825
|
+
clearRetirement(store, ordinal);
|
|
826
|
+
recordPartition(state, store, ordinal, partitionId);
|
|
827
|
+
state.ordToDoc[ordinal] = docId;
|
|
828
|
+
state.docToOrd.set(docId, ordinal);
|
|
829
|
+
if (current !== void 0) retireInWrite(store, current, sequence);
|
|
830
|
+
endWrite(store, sequence);
|
|
831
|
+
settleReplacement(state, store, current, keepsReplacedVector);
|
|
832
|
+
return true;
|
|
833
|
+
}
|
|
834
|
+
function retireStoredOrdinal(state, ordinal) {
|
|
835
|
+
const store = state.open;
|
|
836
|
+
if (store === null) return false;
|
|
837
|
+
if (state.awaitingFree.size > 0) freeAwaitingOrdinals(state);
|
|
838
|
+
const writes = store.sequences;
|
|
839
|
+
if (ordinal >= writes.retired.length) coverOrdinal(writes, ordinal);
|
|
840
|
+
const retiredHalves = writes.retiredHalves;
|
|
841
|
+
const retiredLow = ordinal * 2;
|
|
842
|
+
if (retiredHalves[retiredLow] !== 0 || retiredHalves[retiredLow + 1] !== 0) return false;
|
|
843
|
+
const sequence = nextSequence(writes);
|
|
844
|
+
Atomics.store(writes.retired, ordinal, BigInt(sequence));
|
|
845
|
+
publishSequence(writes, sequence);
|
|
846
|
+
return true;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
// src/vector/vector-store/load.ts
|
|
850
|
+
function retiredBeforeLoad(store, ordinal) {
|
|
851
|
+
stampRetired(store.sequences, ordinal, RETIRED_BEFORE_LOAD);
|
|
852
|
+
}
|
|
853
|
+
function settleLoadedReplacement(state, store, previous) {
|
|
854
|
+
if (previous === void 0) {
|
|
855
|
+
state.liveCount++;
|
|
856
|
+
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
stampRetired(store.sequences, previous, RETIRED_BEFORE_LOAD);
|
|
860
|
+
retireOrdinal(state, store, previous, Number.POSITIVE_INFINITY);
|
|
861
|
+
}
|
|
862
|
+
function loadColdVector(state, store, docId, vectorMagnitude, location, partitionId) {
|
|
863
|
+
const previous = state.docToOrd.get(docId);
|
|
864
|
+
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
865
|
+
const ordinal = claimOrdinal(state, store, docId, true, partitionId);
|
|
866
|
+
placeOnDisk(store, ordinal, vectorMagnitude, location);
|
|
867
|
+
stampBorn(store.sequences, ordinal, LOADED_SEQUENCE);
|
|
868
|
+
publishOrdinal(store, ordinal);
|
|
869
|
+
settleLoadedReplacement(state, store, previous);
|
|
870
|
+
return ordinal;
|
|
871
|
+
}
|
|
872
|
+
function loadReplacedVector(state, store, docId, vector, location) {
|
|
873
|
+
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
874
|
+
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
875
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
876
|
+
state.ordToDoc[ordinal] = void 0;
|
|
877
|
+
state.replacedOrdinals.add(ordinal);
|
|
878
|
+
retiredBeforeLoad(store, ordinal);
|
|
879
|
+
if (location !== void 0) {
|
|
880
|
+
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
881
|
+
return ordinal;
|
|
882
|
+
}
|
|
883
|
+
writeVector(store, ordinal, vector);
|
|
884
|
+
publishOrdinal(store, ordinal);
|
|
885
|
+
return ordinal;
|
|
886
|
+
}
|
|
887
|
+
function restoreReplacedVector(state, store, ordinal, vector) {
|
|
888
|
+
store.diskFile[ordinal] = IN_MEMORY;
|
|
889
|
+
retiredBeforeLoad(store, ordinal);
|
|
890
|
+
writeVector(store, ordinal, vector);
|
|
891
|
+
Atomics.store(store.present, ordinal, 1);
|
|
892
|
+
state.replacedOrdinals.add(ordinal);
|
|
893
|
+
}
|
|
894
|
+
function loadSnapshot(state, store, snapshot) {
|
|
895
|
+
const replaced = new Set(snapshot.replacedOrdinals);
|
|
896
|
+
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
897
|
+
const docId = snapshot.docIds[ordinal];
|
|
898
|
+
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
899
|
+
if (docId === null || docId === void 0) {
|
|
900
|
+
ensureOrdinal(store, ordinal, false);
|
|
901
|
+
state.ordToDoc[ordinal] = void 0;
|
|
902
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
903
|
+
if (replaced.has(ordinal)) restoreReplacedVector(state, store, ordinal, vector);
|
|
904
|
+
else retiredBeforeLoad(store, ordinal);
|
|
905
|
+
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
906
|
+
continue;
|
|
907
|
+
}
|
|
908
|
+
appendOrdinal(state, store, docId, vector, LOADED_SEQUENCE);
|
|
909
|
+
state.liveCount++;
|
|
910
|
+
}
|
|
911
|
+
Atomics.store(store.handles.header, STORE_LIVE_COUNT, state.liveCount);
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
// src/vector/vector-store/retired-pins.ts
|
|
915
|
+
function createRetiredVectorPins() {
|
|
916
|
+
let pins = 0;
|
|
917
|
+
let lost = false;
|
|
918
|
+
const retired = /* @__PURE__ */ new Set();
|
|
919
|
+
return {
|
|
920
|
+
pin() {
|
|
921
|
+
pins += 1;
|
|
922
|
+
},
|
|
923
|
+
unpin() {
|
|
924
|
+
if (pins === 0) return;
|
|
925
|
+
pins -= 1;
|
|
926
|
+
if (pins > 0) return;
|
|
927
|
+
retired.clear();
|
|
928
|
+
lost = false;
|
|
929
|
+
},
|
|
930
|
+
note(ordinal) {
|
|
931
|
+
if (pins > 0) retired.add(ordinal);
|
|
932
|
+
},
|
|
933
|
+
holds(ordinal) {
|
|
934
|
+
return retired.has(ordinal);
|
|
935
|
+
},
|
|
936
|
+
keptWith(replaced) {
|
|
937
|
+
if (retired.size === 0) return replaced;
|
|
938
|
+
const kept = new Set(replaced);
|
|
939
|
+
for (const ordinal of retired) kept.add(ordinal);
|
|
940
|
+
return kept;
|
|
941
|
+
},
|
|
942
|
+
forget() {
|
|
943
|
+
retired.clear();
|
|
944
|
+
if (pins > 0) lost = true;
|
|
945
|
+
},
|
|
946
|
+
requireIntact() {
|
|
947
|
+
if (!lost) return;
|
|
948
|
+
throw new NarsilError(
|
|
949
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
950
|
+
"The vector store was emptied while a checkpoint read the vectors it captured, so the checkpoint stops"
|
|
951
|
+
);
|
|
952
|
+
}
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
function pinnedReads(pins, openStore, readable) {
|
|
956
|
+
return {
|
|
957
|
+
pinRetiredVectors: () => pins.pin(),
|
|
958
|
+
unpinRetiredVectors: () => pins.unpin(),
|
|
959
|
+
entryForPinnedOrdinal(ordinal) {
|
|
960
|
+
pins.requireIntact();
|
|
961
|
+
const open = openStore();
|
|
962
|
+
if (open === null || !(readable(ordinal) || pins.holds(ordinal))) return void 0;
|
|
963
|
+
return entryAt(open, ordinal);
|
|
964
|
+
}
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
// src/vector/vector-store/store-clear.ts
|
|
969
|
+
function retireEveryOrdinal(store) {
|
|
970
|
+
const sequences = store.sequences;
|
|
971
|
+
const sequence = nextSequence(sequences);
|
|
972
|
+
const slots = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
973
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) stampRetired(sequences, ordinal, sequence);
|
|
974
|
+
publishSequence(sequences, sequence);
|
|
975
|
+
}
|
|
976
|
+
function forgetCodes(store) {
|
|
977
|
+
const { header, codePresent } = store.handles;
|
|
978
|
+
new Uint8Array(codePresent, 0, codePresent.byteLength).fill(0);
|
|
979
|
+
Atomics.store(header, STORE_LIVE_COUNT, 0);
|
|
980
|
+
Atomics.store(header, STORE_CODE_COUNT, 0);
|
|
981
|
+
Atomics.store(header, STORE_CALIBRATED, 0);
|
|
982
|
+
Atomics.add(header, STORE_CALIBRATION_GENERATION, 1);
|
|
983
|
+
}
|
|
984
|
+
function resetInPlace(store) {
|
|
985
|
+
const { handles } = store;
|
|
986
|
+
store.present.fill(0, 0, handles.present.byteLength);
|
|
987
|
+
new Uint8Array(handles.codePresent, 0, handles.codePresent.byteLength).fill(0);
|
|
988
|
+
new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY);
|
|
989
|
+
handles.centroid.fill(0);
|
|
990
|
+
if (handles.vectorFiles.length > 0) {
|
|
991
|
+
handles.vectorFiles.length = 0;
|
|
992
|
+
relayout(store);
|
|
993
|
+
}
|
|
994
|
+
resetDocIds(handles);
|
|
995
|
+
resetStoreHeader(handles.header);
|
|
996
|
+
}
|
|
997
|
+
function clearStore(store) {
|
|
998
|
+
retireEveryOrdinal(store);
|
|
999
|
+
if (oldestHeldPin(store.handles.pins) === Number.POSITIVE_INFINITY) {
|
|
1000
|
+
resetInPlace(store);
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
forgetCodes(store);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
// src/vector/vector-store/index.ts
|
|
737
1007
|
function createVectorStore(options) {
|
|
738
1008
|
const codeBits = options?.codeBits ?? null;
|
|
739
|
-
const blockBytes = options?.blockBytes ??
|
|
740
|
-
const
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
1009
|
+
const blockBytes = Math.min(options?.blockBytes ?? VECTOR_MEMORY_LIMITS.blockBytes, VECTOR_MEMORY_LIMITS.blockBytes);
|
|
1010
|
+
const state = {
|
|
1011
|
+
open: options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes),
|
|
1012
|
+
docToOrd: createDocIdMap(),
|
|
1013
|
+
ordToDoc: [],
|
|
1014
|
+
replacedOrdinals: /* @__PURE__ */ new Set(),
|
|
1015
|
+
awaitingFree: /* @__PURE__ */ new Set(),
|
|
1016
|
+
checkpointPins: createRetiredVectorPins(),
|
|
1017
|
+
unknownPartitions: 0,
|
|
1018
|
+
liveCount: 0
|
|
1019
|
+
};
|
|
1020
|
+
const { docToOrd, ordToDoc, replacedOrdinals, awaitingFree } = state;
|
|
1021
|
+
let readBackVector = null;
|
|
746
1022
|
function forgetDocuments() {
|
|
747
1023
|
docToOrd.clear();
|
|
748
1024
|
ordToDoc.length = 0;
|
|
749
1025
|
replacedOrdinals.clear();
|
|
750
|
-
|
|
751
|
-
|
|
1026
|
+
state.awaitingFree.clear();
|
|
1027
|
+
state.checkpointPins.forget();
|
|
1028
|
+
state.unknownPartitions = 0;
|
|
1029
|
+
state.liveCount = 0;
|
|
752
1030
|
}
|
|
753
|
-
function
|
|
754
|
-
return
|
|
1031
|
+
function heldAfterRemoval(ordinal) {
|
|
1032
|
+
return replacedOrdinals.has(ordinal) || awaitingFree.has(ordinal);
|
|
1033
|
+
}
|
|
1034
|
+
function isReadable(ordinal) {
|
|
1035
|
+
return ordToDoc[ordinal] !== void 0 || heldAfterRemoval(ordinal);
|
|
755
1036
|
}
|
|
756
1037
|
function requireOpen() {
|
|
757
|
-
if (open === null) {
|
|
1038
|
+
if (state.open === null) {
|
|
758
1039
|
throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
|
|
759
1040
|
}
|
|
760
|
-
return open;
|
|
761
|
-
}
|
|
762
|
-
function recordPartition(store, ordinal, partitionId) {
|
|
763
|
-
const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
|
|
764
|
-
store.partitions[ordinal] = given;
|
|
765
|
-
if (given === UNKNOWN_PARTITION) unknownPartitions += 1;
|
|
766
|
-
}
|
|
767
|
-
function retireOrdinal(store, ordinal) {
|
|
768
|
-
ordToDoc[ordinal] = void 0;
|
|
769
|
-
Atomics.store(store.present, ordinal, 0);
|
|
770
|
-
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
771
|
-
}
|
|
772
|
-
function keepReplacedOrdinal(store, ordinal) {
|
|
773
|
-
ordToDoc[ordinal] = void 0;
|
|
774
|
-
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
775
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
776
|
-
replacedOrdinals.add(ordinal);
|
|
777
|
-
}
|
|
778
|
-
function restoreReplacedVector(store, ordinal, vector) {
|
|
779
|
-
store.diskFile[ordinal] = IN_MEMORY;
|
|
780
|
-
writeVector(store, ordinal, vector);
|
|
781
|
-
Atomics.store(store.present, ordinal, 1);
|
|
782
|
-
replacedOrdinals.add(ordinal);
|
|
783
|
-
}
|
|
784
|
-
function claimOrdinal(store, docId, cold, partitionId) {
|
|
785
|
-
const ordinal = claimSlot(store, docId, cold);
|
|
786
|
-
recordPartition(store, ordinal, partitionId);
|
|
787
|
-
ordToDoc[ordinal] = docId;
|
|
788
|
-
docToOrd.set(docId, ordinal);
|
|
789
|
-
return ordinal;
|
|
1041
|
+
return state.open;
|
|
790
1042
|
}
|
|
791
|
-
function
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
publishOrdinal(store, ordinal);
|
|
795
|
-
return ordinal;
|
|
796
|
-
}
|
|
797
|
-
function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
|
|
798
|
-
if (previous !== void 0) {
|
|
799
|
-
if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
|
|
800
|
-
else retireOrdinal(store, previous);
|
|
801
|
-
return ordinal;
|
|
802
|
-
}
|
|
803
|
-
liveCount++;
|
|
804
|
-
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
805
|
-
return ordinal;
|
|
1043
|
+
function openFor(dimension) {
|
|
1044
|
+
state.open ??= openHandles(dimension, codeBits, blockBytes);
|
|
1045
|
+
return state.open;
|
|
806
1046
|
}
|
|
807
1047
|
return {
|
|
808
|
-
...createOrdinalReads(() => open,
|
|
1048
|
+
...createOrdinalReads(() => state.open, isReadable),
|
|
809
1049
|
get size() {
|
|
810
|
-
return liveCount;
|
|
1050
|
+
return state.liveCount;
|
|
811
1051
|
},
|
|
812
1052
|
get slots() {
|
|
813
1053
|
return ordToDoc.length;
|
|
814
1054
|
},
|
|
815
1055
|
get dimension() {
|
|
816
|
-
return open === null ? 0 : open.handles.dimension;
|
|
1056
|
+
return state.open === null ? 0 : state.open.handles.dimension;
|
|
817
1057
|
},
|
|
818
1058
|
get codeBits() {
|
|
819
1059
|
return codeBits;
|
|
@@ -828,8 +1068,7 @@ function createVectorStore(options) {
|
|
|
828
1068
|
return replacedOrdinals.size;
|
|
829
1069
|
},
|
|
830
1070
|
insert(docId, vector, partitionId, keepsReplacedVector = false) {
|
|
831
|
-
|
|
832
|
-
const store = open;
|
|
1071
|
+
const store = openFor(vector.length);
|
|
833
1072
|
if (vector.length !== store.handles.dimension) {
|
|
834
1073
|
throw new NarsilError(
|
|
835
1074
|
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
@@ -838,38 +1077,60 @@ function createVectorStore(options) {
|
|
|
838
1077
|
);
|
|
839
1078
|
}
|
|
840
1079
|
const previous = docToOrd.get(docId);
|
|
841
|
-
const
|
|
842
|
-
|
|
1080
|
+
const sequence = beginWrite(state, store);
|
|
1081
|
+
const ordinal = appendOrdinal(state, store, docId, vector, sequence, partitionId);
|
|
1082
|
+
if (previous !== void 0) retireInWrite(store, previous, sequence);
|
|
1083
|
+
endWrite(store, sequence);
|
|
1084
|
+
settleReplacement(state, store, previous, keepsReplacedVector);
|
|
1085
|
+
return ordinal;
|
|
1086
|
+
},
|
|
1087
|
+
requireRoomFor(docId) {
|
|
1088
|
+
if (state.open !== null) requireRoomForSlot(state.open, docId);
|
|
1089
|
+
},
|
|
1090
|
+
reclaim(docId, ordinal, partitionId, keepsReplacedVector = false) {
|
|
1091
|
+
return reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector);
|
|
843
1092
|
},
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1093
|
+
retire(ordinal) {
|
|
1094
|
+
retireStoredOrdinal(state, ordinal);
|
|
1095
|
+
},
|
|
1096
|
+
freeRetiredVectors() {
|
|
1097
|
+
freeAwaitingOrdinals(state);
|
|
1098
|
+
},
|
|
1099
|
+
oldestPin() {
|
|
1100
|
+
return state.open === null ? Number.POSITIVE_INFINITY : oldestPinOf(state.open);
|
|
1101
|
+
},
|
|
1102
|
+
writeSequence() {
|
|
1103
|
+
return state.open === null ? 0 : writeSequenceOf(state.open.sequences.header);
|
|
1104
|
+
},
|
|
1105
|
+
retiredStampHalves() {
|
|
1106
|
+
return requireOpen().sequences.retiredHalves;
|
|
1107
|
+
},
|
|
1108
|
+
claimedSlots() {
|
|
1109
|
+
return state.open === null ? 0 : state.open.handles.header[STORE_SLOTS];
|
|
1110
|
+
},
|
|
1111
|
+
mayFreeOrdinal(ordinal, oldestPin) {
|
|
1112
|
+
return state.open !== null && freeable(state.open, ordinal, oldestPin);
|
|
1113
|
+
},
|
|
1114
|
+
releaseReplacedVectors(oldestPin) {
|
|
1115
|
+
const store = state.open;
|
|
1116
|
+
const released = [];
|
|
1117
|
+
const oldest = oldestPin ?? (store === null ? Number.POSITIVE_INFINITY : oldestPinOf(store));
|
|
1118
|
+
for (const ordinal of replacedOrdinals) {
|
|
1119
|
+
if (store !== null && !freeable(store, ordinal, oldest)) continue;
|
|
1120
|
+
released.push(ordinal);
|
|
1121
|
+
}
|
|
1122
|
+
for (const ordinal of released) {
|
|
1123
|
+
replacedOrdinals.delete(ordinal);
|
|
1124
|
+
if (store !== null) freeRetired(state, store, ordinal);
|
|
1125
|
+
}
|
|
848
1126
|
return released;
|
|
849
1127
|
},
|
|
1128
|
+
...pinnedReads(state.checkpointPins, () => state.open, isReadable),
|
|
850
1129
|
insertCold(docId, vectorMagnitude, location, partitionId) {
|
|
851
|
-
|
|
852
|
-
const previous = docToOrd.get(docId);
|
|
853
|
-
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
854
|
-
const ordinal = claimOrdinal(store, docId, true, partitionId);
|
|
855
|
-
publishOnDisk(store, ordinal, vectorMagnitude, location);
|
|
856
|
-
return settleReplacement(store, ordinal, previous, false);
|
|
1130
|
+
return loadColdVector(state, requireOpen(), docId, vectorMagnitude, location, partitionId);
|
|
857
1131
|
},
|
|
858
1132
|
insertReplaced(docId, vector, location) {
|
|
859
|
-
|
|
860
|
-
const store = open;
|
|
861
|
-
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
862
|
-
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
863
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
864
|
-
ordToDoc[ordinal] = void 0;
|
|
865
|
-
replacedOrdinals.add(ordinal);
|
|
866
|
-
if (location !== void 0) {
|
|
867
|
-
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
868
|
-
return ordinal;
|
|
869
|
-
}
|
|
870
|
-
writeVector(store, ordinal, vector);
|
|
871
|
-
publishOrdinal(store, ordinal);
|
|
872
|
-
return ordinal;
|
|
1133
|
+
return loadReplacedVector(state, openFor(vector.length), docId, vector, location);
|
|
873
1134
|
},
|
|
874
1135
|
addVectorFile(path) {
|
|
875
1136
|
const store = requireOpen();
|
|
@@ -880,13 +1141,16 @@ function createVectorStore(options) {
|
|
|
880
1141
|
return store.handles.vectorFiles.length - 1;
|
|
881
1142
|
},
|
|
882
1143
|
isCold(ordinal) {
|
|
883
|
-
if (open === null) return false;
|
|
884
|
-
return open.view.isCold(ordinal);
|
|
1144
|
+
if (state.open === null) return false;
|
|
1145
|
+
return state.open.view.isCold(ordinal);
|
|
885
1146
|
},
|
|
886
1147
|
releaseToFile(ordinal, location) {
|
|
887
1148
|
const store = requireOpen();
|
|
888
1149
|
if (ordToDoc[ordinal] === void 0) return false;
|
|
889
|
-
if (store.diskFile[ordinal] === IN_MEMORY
|
|
1150
|
+
if (store.diskFile[ordinal] === IN_MEMORY) {
|
|
1151
|
+
readBackVector ??= new Float32Array(store.handles.dimension);
|
|
1152
|
+
if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
|
|
1153
|
+
}
|
|
890
1154
|
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
891
1155
|
store.diskOffset[ordinal] = location.offset;
|
|
892
1156
|
store.diskFile[ordinal] = location.fileIndex;
|
|
@@ -894,49 +1158,54 @@ function createVectorStore(options) {
|
|
|
894
1158
|
},
|
|
895
1159
|
releaseColdBlocks() {
|
|
896
1160
|
const store = requireOpen();
|
|
1161
|
+
const held = new Set(replacedOrdinals);
|
|
1162
|
+
for (const ordinal of state.awaitingFree) held.add(ordinal);
|
|
1163
|
+
const kept = state.checkpointPins.keptWith(held);
|
|
897
1164
|
let released = 0;
|
|
898
1165
|
for (let index = 0; index < store.handles.blocks.length; index++) {
|
|
899
|
-
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc,
|
|
1166
|
+
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, kept)) continue;
|
|
900
1167
|
store.handles.blocks[index] = null;
|
|
901
1168
|
released += 1;
|
|
902
1169
|
}
|
|
903
|
-
if (released + forgetUnreferencedFiles(store, ordToDoc,
|
|
1170
|
+
if (released + forgetUnreferencedFiles(store, ordToDoc, kept) > 0) relayout(store);
|
|
904
1171
|
return released;
|
|
905
1172
|
},
|
|
906
1173
|
setPartition(docId, partitionId) {
|
|
907
1174
|
const ordinal = docToOrd.get(docId);
|
|
908
|
-
if (ordinal === void 0 || open === null || partitionId < 0) return;
|
|
909
|
-
if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
910
|
-
open.partitions[ordinal] = partitionId;
|
|
1175
|
+
if (ordinal === void 0 || state.open === null || partitionId < 0) return;
|
|
1176
|
+
if (state.open.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
1177
|
+
state.open.partitions[ordinal] = partitionId;
|
|
911
1178
|
},
|
|
912
1179
|
forgetPartition(docId) {
|
|
913
1180
|
const ordinal = docToOrd.get(docId);
|
|
914
|
-
if (ordinal === void 0 || open === null) return;
|
|
915
|
-
if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
916
|
-
open.partitions[ordinal] = NO_PARTITION;
|
|
1181
|
+
if (ordinal === void 0 || state.open === null) return;
|
|
1182
|
+
if (state.open.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
1183
|
+
state.open.partitions[ordinal] = NO_PARTITION;
|
|
917
1184
|
},
|
|
918
1185
|
partitionOfOrdinal(ordinal) {
|
|
919
|
-
if (open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
|
|
920
|
-
const partitionId = open.partitions[ordinal];
|
|
1186
|
+
if (state.open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
|
|
1187
|
+
const partitionId = state.open.partitions[ordinal];
|
|
921
1188
|
return partitionId < 0 ? void 0 : partitionId;
|
|
922
1189
|
},
|
|
923
1190
|
get partitionsKnown() {
|
|
924
|
-
return unknownPartitions === 0;
|
|
1191
|
+
return state.unknownPartitions === 0;
|
|
925
1192
|
},
|
|
926
1193
|
partitionFilter(partitionIds) {
|
|
927
|
-
return partitionFilterOf(open, ordToDoc, partitionIds);
|
|
1194
|
+
return partitionFilterOf(state.open, ordToDoc, partitionIds);
|
|
928
1195
|
},
|
|
929
|
-
remove(docId) {
|
|
1196
|
+
remove(docId, oldestPin) {
|
|
930
1197
|
const ordinal = docToOrd.get(docId);
|
|
931
|
-
|
|
1198
|
+
const store = state.open;
|
|
1199
|
+
if (ordinal === void 0 || store === null) return;
|
|
1200
|
+
const retiredNow = retireStoredOrdinal(state, ordinal);
|
|
932
1201
|
docToOrd.delete(docId);
|
|
933
|
-
retireOrdinal(
|
|
934
|
-
liveCount--;
|
|
935
|
-
Atomics.sub(
|
|
1202
|
+
retireOrdinal(state, store, ordinal, retiredNow || oldestPin === void 0 ? oldestPinOf(store) : oldestPin);
|
|
1203
|
+
state.liveCount--;
|
|
1204
|
+
Atomics.sub(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
936
1205
|
},
|
|
937
1206
|
get(docId) {
|
|
938
1207
|
const ordinal = docToOrd.get(docId);
|
|
939
|
-
return ordinal === void 0 || open === null ? void 0 : entryAt(open, ordinal);
|
|
1208
|
+
return ordinal === void 0 || state.open === null ? void 0 : entryAt(state.open, ordinal);
|
|
940
1209
|
},
|
|
941
1210
|
has(docId) {
|
|
942
1211
|
return docToOrd.has(docId);
|
|
@@ -944,31 +1213,22 @@ function createVectorStore(options) {
|
|
|
944
1213
|
holdsOrdinal(ordinal) {
|
|
945
1214
|
return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
|
|
946
1215
|
},
|
|
947
|
-
entries: () => entriesOf(open, ordToDoc),
|
|
948
|
-
docIds: () => docIdsOf(open, ordToDoc),
|
|
1216
|
+
entries: () => entriesOf(state.open, ordToDoc),
|
|
1217
|
+
docIds: () => docIdsOf(state.open, ordToDoc),
|
|
949
1218
|
clear() {
|
|
1219
|
+
const store = state.open;
|
|
950
1220
|
forgetDocuments();
|
|
951
|
-
if (
|
|
952
|
-
open.present.fill(0);
|
|
953
|
-
new Uint8Array(open.handles.codePresent).fill(0);
|
|
954
|
-
new Int32Array(open.handles.diskFile).fill(IN_MEMORY);
|
|
955
|
-
open.handles.centroid.fill(0);
|
|
956
|
-
if (open.handles.vectorFiles.length > 0) {
|
|
957
|
-
open.handles.vectorFiles.length = 0;
|
|
958
|
-
relayout(open);
|
|
959
|
-
}
|
|
960
|
-
resetDocIds(open.handles);
|
|
961
|
-
resetStoreHeader(open.handles.header);
|
|
1221
|
+
if (store !== null) clearStore(store);
|
|
962
1222
|
},
|
|
963
1223
|
release() {
|
|
964
1224
|
forgetDocuments();
|
|
965
|
-
if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
966
|
-
open?.view.close();
|
|
967
|
-
open = null;
|
|
1225
|
+
if (state.open !== null) Atomics.add(state.open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
1226
|
+
state.open?.view.close();
|
|
1227
|
+
state.open = null;
|
|
968
1228
|
},
|
|
969
|
-
closeFiles: () => open?.view.close(),
|
|
1229
|
+
closeFiles: () => state.open?.view.close(),
|
|
970
1230
|
relocateVectorFiles(rewrite) {
|
|
971
|
-
if (open !== null) relocateFiles(open, rewrite);
|
|
1231
|
+
if (state.open !== null) relocateFiles(state.open, rewrite);
|
|
972
1232
|
},
|
|
973
1233
|
getOrdinal(docId) {
|
|
974
1234
|
return docToOrd.get(docId);
|
|
@@ -977,35 +1237,24 @@ function createVectorStore(options) {
|
|
|
977
1237
|
return ordToDoc[ordinal];
|
|
978
1238
|
},
|
|
979
1239
|
prepareQueryArena(query) {
|
|
980
|
-
return open === null ? null : open.view.prepareQueryArena(query);
|
|
1240
|
+
return state.open === null ? null : state.open.view.prepareQueryArena(query);
|
|
981
1241
|
},
|
|
982
|
-
exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
|
|
1242
|
+
exportSnapshot: () => snapshotOf(state.open, ordToDoc, replacedOrdinals),
|
|
983
1243
|
restoreSnapshot(snapshot) {
|
|
984
1244
|
this.clear();
|
|
985
1245
|
if (snapshot.dimension === 0 || snapshot.slots === 0) return;
|
|
986
|
-
|
|
987
|
-
const store = open;
|
|
988
|
-
const replaced = new Set(snapshot.replacedOrdinals);
|
|
989
|
-
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
990
|
-
const docId = snapshot.docIds[ordinal];
|
|
991
|
-
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
992
|
-
if (docId === null || docId === void 0) {
|
|
993
|
-
ensureOrdinal(store, ordinal, false);
|
|
994
|
-
ordToDoc[ordinal] = void 0;
|
|
995
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
996
|
-
if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
|
|
997
|
-
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
998
|
-
continue;
|
|
999
|
-
}
|
|
1000
|
-
appendOrdinal(store, docId, vector);
|
|
1001
|
-
liveCount++;
|
|
1002
|
-
}
|
|
1003
|
-
Atomics.store(store.handles.header, STORE_LIVE_COUNT, liveCount);
|
|
1246
|
+
loadSnapshot(state, openFor(snapshot.dimension), snapshot);
|
|
1004
1247
|
},
|
|
1005
1248
|
memoryBytes() {
|
|
1006
|
-
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
1249
|
+
return state.open === null ? 0 : sharedVectorStoreBytes(state.open.handles);
|
|
1250
|
+
},
|
|
1251
|
+
growthBytesFor(vectors) {
|
|
1252
|
+
return state.open === null ? 0 : growthBytesFor(state.open, vectors);
|
|
1253
|
+
},
|
|
1254
|
+
visitDocumentTables(visit) {
|
|
1255
|
+
visit("documents", docToOrd.size, true);
|
|
1007
1256
|
}
|
|
1008
1257
|
};
|
|
1009
1258
|
}
|
|
1010
1259
|
|
|
1011
|
-
export { createVectorStore, openSharedVectorField };
|
|
1260
|
+
export { createVectorStore, openSharedVectorField, requireOrdinalWithinLimit };
|