@delali/narsil 0.3.1-next.156 → 0.3.1-next.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-3V7DNNZL.mjs → chunk-4RNSFVWI.mjs} +20997 -19899
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/browser/{durability-config-TIYKW6XX.mjs → durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{geo-index-MRH5JMIR.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/{react-native/chunks/restore-hooks-IBGIY3DR.mjs → browser/restore-queue-S4BWOGMU.mjs} +98 -36
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-SSTPBZZX.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-Da4ywVVl.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-K5QFDYST.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{browser/chunk-23XRXIBW.mjs → chunk-7MWYORQC.mjs} +10 -3
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-XAXKV65G.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-I2KPOLOZ.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-ZKP7G7TV.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-2UV4QBHW.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-25KDNIZZ.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-CRE3G7F3.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-FRBFVQFN.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-6I6AYAXS.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{browser/chunk-SRMZXJSL.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/chunk-QBVTYY4J.mjs +54 -0
- package/dist/{react-native/chunks/chunk-GXX3DQXP.mjs → chunk-R4CE4IVU.mjs} +352 -165
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-FAKFUDZZ.mjs → chunk-SHGN3QIF.mjs} +80 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{react-native/chunks/chunk-EJ33LCHV.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-6MO2XJF2.mjs → chunk-TSLV5P2J.mjs} +397 -79
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-FYOVBEMM.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-DRv7UVeY.d.ts → client-CTp8RRiK.d.ts} +16 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +70 -18
- package/dist/{config-DMwffLKn.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CoSig5Ox.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1192 -1077
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CFxPYqEk.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-HHWGMNIS.mjs → geo-index-2ZMMWWSK.mjs} +190 -92
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -15
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-YA4IAYSY.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-PXW5XAYU.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -58
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -226
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/chunk-FTQ7JNMU.mjs +898 -0
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/{chunk-TBC3XUL2.mjs → react-native/chunks/chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-KV46CT4D.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/chunk-OK3WUA6N.mjs +54 -0
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-DZ7H7D2C.mjs → chunk-TQ26BKBH.mjs} +6 -0
- package/dist/react-native/chunks/{chunk-6VBXNFLE.mjs → chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/react-native/chunks/{geo-index-PX3BIHAH.mjs → geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-UQ4OAXKU.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +127 -50
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +35 -13
- package/dist/server.mjs +90 -101
- package/dist/{browser/durability-wiring-A7BVG2DW.mjs → storage-leaf-HMIIOUYN.mjs} +2348 -902
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +120 -70
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +121 -57
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-qrJojgST.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-BZCGjfLn.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +137 -50
- package/package.json +233 -8
- package/dist/analysis-change-5BQOQ4BF.mjs +0 -55
- package/dist/analysis-change-entry-TXX6JSMX.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-435CUEPZ.mjs +0 -933
- package/dist/browser/analysis-change-entry-HZXKPPGV.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-2CFI5VVV.mjs +0 -241
- package/dist/browser/chunk-2TI4SSUP.mjs +0 -315
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3SSRJMFN.mjs +0 -100
- package/dist/browser/chunk-4NGKC4H4.mjs +0 -2635
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-6LWDPLTB.mjs +0 -177
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-7U3WVJUP.mjs +0 -150
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BHFQATSC.mjs +0 -432
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-EA63E3D2.mjs +0 -857
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-GPPOG6PT.mjs +0 -9
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-IZXVW2ZJ.mjs +0 -217
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K4Q6KX3R.mjs +0 -2528
- package/dist/browser/chunk-KZWSX6BH.mjs +0 -645
- package/dist/browser/chunk-MGI4BCXC.mjs +0 -63
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MOG4OVDK.mjs +0 -51
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NQAME3RK.mjs +0 -21
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-STOWQFT3.mjs +0 -1829
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WI4MR75E.mjs +0 -86
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-2EHCIV4K.mjs +0 -759
- package/dist/browser/engine-hooks-E4UD67JD.mjs +0 -17
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-DXYB6756.mjs +0 -19
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NIYMEA5R.mjs +0 -23
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-hooks-YPN4ZFCY.mjs +0 -114
- package/dist/browser/restore-queue-G3RQGL3Y.mjs +0 -55
- package/dist/browser/retention-ZWVMNC3Q.mjs +0 -210
- package/dist/browser/snapshot-VEQNIE6C.mjs +0 -963
- package/dist/browser/start-BM442C6R.mjs +0 -25
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-2EVMZWDE.mjs +0 -194
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3HYC4JRF.mjs +0 -50
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-46KY2PJD.mjs +0 -905
- package/dist/chunk-4KKXNJM6.mjs +0 -242
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7LXHHCXZ.mjs +0 -1397
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-BU4SAIC4.mjs +0 -196
- package/dist/chunk-BWZ3P4MU.mjs +0 -20
- package/dist/chunk-CSLGFPFQ.mjs +0 -218
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-DP5C7YFQ.mjs +0 -543
- package/dist/chunk-DWEPMFJE.mjs +0 -240
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-EFBDHH5P.mjs +0 -7081
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FNGAM5B7.mjs +0 -2639
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HFPMMRQT.mjs +0 -206
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I53G4JAU.mjs +0 -1974
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JDJBQLRO.mjs +0 -1126
- package/dist/chunk-JWRUUCWD.mjs +0 -151
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KF4EXSM4.mjs +0 -21
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-L6V6BST3.mjs +0 -1435
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MQOJIO5B.mjs +0 -316
- package/dist/chunk-NF4DT3IH.mjs +0 -85
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-P5ZI2P2W.mjs +0 -7481
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PWC4VB3Q.mjs +0 -106
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QOW2JPUI.mjs +0 -634
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RDXO6JST.mjs +0 -63
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SA5LOGO3.mjs +0 -470
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TGS5UXBE.mjs +0 -216
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-TZ3355VS.mjs +0 -121
- package/dist/chunk-TZTRKM33.mjs +0 -74
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VTTRFF2C.mjs +0 -3711
- package/dist/chunk-WX4LAO6M.mjs +0 -656
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-XXHN7J46.mjs +0 -101
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-YTWZVGCB.mjs +0 -382
- package/dist/chunk-Z2KAFCRT.mjs +0 -30
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-FTMYY3TI.mjs +0 -34
- package/dist/durability-wiring-VIF5MOSY.mjs +0 -4064
- package/dist/engine-hooks-EGUGNZYE.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-XKDPJQDP.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-FQ4JWNQJ.mjs +0 -62
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-LT5QBQ3H.mjs +0 -926
- package/dist/react-native/chunks/analysis-change-entry-SCVWK6OU.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3JWDECAZ.mjs +0 -858
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4JYO3JD2.mjs +0 -242
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +0 -85
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DNXOMPAQ.mjs +0 -191
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-G43IPIDG.mjs +0 -151
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MIP4SOR6.mjs +0 -22803
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-MUWD2H2S.mjs +0 -21
- package/dist/react-native/chunks/chunk-NB3LP33V.mjs +0 -315
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PDRWWPKV.mjs +0 -2532
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-POPDAGH6.mjs +0 -2636
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-QUKQNI3H.mjs +0 -647
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TP6KCKER.mjs +0 -63
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-VWUDIOT6.mjs +0 -218
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZLWVW734.mjs +0 -100
- package/dist/react-native/chunks/chunk-ZXZFN7FI.mjs +0 -432
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-KVZYUGRY.mjs +0 -772
- package/dist/react-native/chunks/durability-wiring-34OYO32E.mjs +0 -4942
- package/dist/react-native/chunks/engine-hooks-M7JV46NH.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-H6AQREWI.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-LBZPGOWH.mjs +0 -36
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +0 -33
- package/dist/react-native/chunks/restore-queue-ZZX54AH7.mjs +0 -55
- package/dist/react-native/chunks/retention-EW4TTDFB.mjs +0 -223
- package/dist/react-native/chunks/snapshot-REYVIN3X.mjs +0 -975
- package/dist/react-native/chunks/start-BQK6KN3R.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-FQAYCZF5.mjs +0 -207
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-hooks-CDERDEF4.mjs +0 -153
- package/dist/restore-queue-7NNSTROA.mjs +0 -55
- package/dist/retention-QSVBBNRM.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-RH4NSKIZ.mjs +0 -782
- package/dist/start-MCKCOGIW.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-O4SBJJJS.mjs +0 -50
- package/dist/write-3FK7NG3Y.mjs +0 -232
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
|
@@ -1,21 +1,683 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
1
|
+
// src/vector/vector-host/engine.ts
|
|
2
|
+
var DEFAULT_LINEAR_ALPHA;
|
|
3
|
+
var DEFAULT_RANK_CONSTANT;
|
|
4
|
+
var DISK_STORAGE_BLOCK_BYTES;
|
|
5
|
+
var ErrorCodes;
|
|
6
|
+
var FUSION_TIE_INSERTION_LIMIT;
|
|
7
|
+
var GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER;
|
|
8
|
+
var GRAPH_BUILD_DISPATCH_CHUNK;
|
|
9
|
+
var MAX_LINEAR_ALPHA;
|
|
10
|
+
var MAX_RENUMBERED_SEARCH_ATTEMPTS;
|
|
11
|
+
var MIN_LINEAR_ALPHA;
|
|
12
|
+
var MIN_RANK_CONSTANT;
|
|
13
|
+
var NATIVE_THREAD_SCAN_CHUNK_ORDINALS;
|
|
14
|
+
var NarsilError;
|
|
15
|
+
var RESULT_WINDOW;
|
|
16
|
+
var UNPLACED_FLAG_CHUNK_ORDINALS;
|
|
17
|
+
var VECTOR_INSERT_TIMEOUT_MS;
|
|
18
|
+
var VECTOR_SCRATCH_SLOTS;
|
|
19
|
+
var VECTOR_SEARCH_LOAD_TIMEOUT_MS;
|
|
20
|
+
var VECTOR_SEARCH_TIMEOUT_MS;
|
|
21
|
+
var WORKER_YOUNG_GENERATION_MB;
|
|
22
|
+
var bindSearchSignal;
|
|
23
|
+
var compareCodePoints;
|
|
24
|
+
var createDocIdMap;
|
|
25
|
+
var detectRuntime;
|
|
26
|
+
var isNarsilError;
|
|
27
|
+
var noteThreadSlotGone;
|
|
28
|
+
var noteThreadSlotLive;
|
|
29
|
+
var requirePinnedState;
|
|
30
|
+
var resolveWorkerCount;
|
|
31
|
+
var resolveWorkerEntry;
|
|
32
|
+
var signalOfSearch;
|
|
33
|
+
var spawnNodeWorker;
|
|
34
|
+
var turnIsSpent;
|
|
35
|
+
var yieldTurn;
|
|
36
|
+
function bindEngineNames(host) {
|
|
37
|
+
({
|
|
38
|
+
DEFAULT_LINEAR_ALPHA,
|
|
39
|
+
DEFAULT_RANK_CONSTANT,
|
|
40
|
+
DISK_STORAGE_BLOCK_BYTES,
|
|
41
|
+
ErrorCodes,
|
|
42
|
+
FUSION_TIE_INSERTION_LIMIT,
|
|
43
|
+
GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER,
|
|
44
|
+
GRAPH_BUILD_DISPATCH_CHUNK,
|
|
45
|
+
MAX_LINEAR_ALPHA,
|
|
46
|
+
MAX_RENUMBERED_SEARCH_ATTEMPTS,
|
|
47
|
+
MIN_LINEAR_ALPHA,
|
|
48
|
+
MIN_RANK_CONSTANT,
|
|
49
|
+
NATIVE_THREAD_SCAN_CHUNK_ORDINALS,
|
|
50
|
+
NarsilError,
|
|
51
|
+
RESULT_WINDOW,
|
|
52
|
+
UNPLACED_FLAG_CHUNK_ORDINALS,
|
|
53
|
+
VECTOR_INSERT_TIMEOUT_MS,
|
|
54
|
+
VECTOR_SCRATCH_SLOTS,
|
|
55
|
+
VECTOR_SEARCH_LOAD_TIMEOUT_MS,
|
|
56
|
+
VECTOR_SEARCH_TIMEOUT_MS,
|
|
57
|
+
WORKER_YOUNG_GENERATION_MB,
|
|
58
|
+
bindSearchSignal,
|
|
59
|
+
compareCodePoints,
|
|
60
|
+
createDocIdMap,
|
|
61
|
+
detectRuntime,
|
|
62
|
+
isNarsilError,
|
|
63
|
+
noteThreadSlotGone,
|
|
64
|
+
noteThreadSlotLive,
|
|
65
|
+
requirePinnedState,
|
|
66
|
+
resolveWorkerCount,
|
|
67
|
+
resolveWorkerEntry,
|
|
68
|
+
signalOfSearch,
|
|
69
|
+
spawnNodeWorker,
|
|
70
|
+
turnIsSpent,
|
|
71
|
+
yieldTurn
|
|
72
|
+
} = host);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// src/vector/vector-host/vector-core-1.ts
|
|
76
|
+
var BUILD_CHUNK_SIZE;
|
|
77
|
+
var BUILD_RETRY_FIRST_DELAY_MS;
|
|
78
|
+
var BUILD_RETRY_MAX_DELAY_MS;
|
|
79
|
+
var BUILD_STEPS_BETWEEN_CLOCK_CHECKS;
|
|
80
|
+
var DEFAULT_FILTER_THRESHOLD;
|
|
81
|
+
var DEFAULT_PROMOTION_THRESHOLD;
|
|
82
|
+
var DEFAULT_VECTOR_METRIC;
|
|
83
|
+
var DISK_ADOPTION_YIELD_INTERVAL;
|
|
84
|
+
var DRIFT_CHECK_GROWTH_FACTOR;
|
|
85
|
+
var ESTIMATED_MS_PER_TOMBSTONE;
|
|
86
|
+
var ESTIMATED_MS_PER_VECTOR_REBUILD;
|
|
87
|
+
var GRAPH_RETIRED;
|
|
88
|
+
var IN_MEMORY;
|
|
89
|
+
var MIN_OVERSAMPLE;
|
|
90
|
+
var NATIVE_OK;
|
|
91
|
+
var NOTHING_STAGED;
|
|
92
|
+
var NO_LIVE_VECTOR_AT_POSITION;
|
|
93
|
+
var QUERY_STAGED;
|
|
94
|
+
var RECALIBRATION_CHUNK_VECTORS;
|
|
95
|
+
var RECALIBRATION_DRIFT_RATIO;
|
|
96
|
+
var REFUSAL_COLLECTION_MIN_INTERVAL_MS;
|
|
97
|
+
var REFUSAL_COLLECTION_TIME_MULTIPLE;
|
|
98
|
+
var REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS;
|
|
99
|
+
var SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES;
|
|
100
|
+
var SEARCH_REFILL_RESERVE;
|
|
101
|
+
var SEARCH_SCAN_CHUNK_VECTORS;
|
|
102
|
+
var SHUFFLED_REBUILD_MEMORY_RETRY_MS;
|
|
103
|
+
var STORE_BLOCK_COUNT;
|
|
104
|
+
var STORE_CALIBRATED;
|
|
105
|
+
var STORE_CALIBRATION_GENERATION;
|
|
106
|
+
var STORE_CODE_BLOCK_COUNT;
|
|
107
|
+
var STORE_CODE_COUNT;
|
|
108
|
+
var STORE_DOC_ID_BYTES;
|
|
109
|
+
var STORE_LIVE_COUNT;
|
|
110
|
+
var STORE_ORDINAL_LAYOUT_GENERATION;
|
|
111
|
+
var STORE_RELEASED_VECTORS_TO_DISK;
|
|
112
|
+
var STORE_REMOVED_COUNT;
|
|
113
|
+
var STORE_SLOTS;
|
|
114
|
+
var STORE_UNPLACED_COUNT;
|
|
115
|
+
var STORE_WRITE_SEQUENCE_INT64;
|
|
116
|
+
var UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS;
|
|
117
|
+
var UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS;
|
|
118
|
+
var UI_THREAD_TURN_BUDGET_MS;
|
|
119
|
+
var VECTOR_MEMORY_LIMITS;
|
|
120
|
+
var VECTOR_WORKER_COPIES_ALLOWED;
|
|
121
|
+
var WORKER_COPY_MIN_VECTORS;
|
|
122
|
+
var addToOrdinalFilter;
|
|
123
|
+
var admissionAt;
|
|
124
|
+
var adoptBackgroundScheduler;
|
|
125
|
+
var adoptGraph;
|
|
126
|
+
var adoptLoadedCalibration;
|
|
127
|
+
var allLiveDocIds;
|
|
128
|
+
var arenaFloat32Distance;
|
|
129
|
+
var assignStorePartitions;
|
|
130
|
+
var backgroundTurnIsSpent;
|
|
131
|
+
var blocksGrowthBytes;
|
|
132
|
+
var bytesToVectors;
|
|
133
|
+
var centroidOfCentroidSums;
|
|
134
|
+
var collectAndReturnPages;
|
|
135
|
+
var collectYoungGeneration;
|
|
136
|
+
var cosineSimilarityWithMagnitudes;
|
|
137
|
+
var countOfCentroidSums;
|
|
138
|
+
var createCentroidSums;
|
|
139
|
+
var createHNSWIndex;
|
|
140
|
+
var createOrdinalFilter;
|
|
141
|
+
var createOsqQuantizer;
|
|
142
|
+
var createSharedVectorStoreHandles;
|
|
143
|
+
var createVectorBlock;
|
|
144
|
+
var createVectorFileReader;
|
|
145
|
+
var defaultQuantizationFor;
|
|
146
|
+
var deserialize;
|
|
147
|
+
var detachNativeStore;
|
|
148
|
+
var dotProduct;
|
|
149
|
+
function bindVectorCore1Names(vectorCore) {
|
|
150
|
+
({
|
|
151
|
+
BUILD_CHUNK_SIZE,
|
|
152
|
+
BUILD_RETRY_FIRST_DELAY_MS,
|
|
153
|
+
BUILD_RETRY_MAX_DELAY_MS,
|
|
154
|
+
BUILD_STEPS_BETWEEN_CLOCK_CHECKS,
|
|
155
|
+
DEFAULT_FILTER_THRESHOLD,
|
|
156
|
+
DEFAULT_PROMOTION_THRESHOLD,
|
|
157
|
+
DEFAULT_VECTOR_METRIC,
|
|
158
|
+
DISK_ADOPTION_YIELD_INTERVAL,
|
|
159
|
+
DRIFT_CHECK_GROWTH_FACTOR,
|
|
160
|
+
ESTIMATED_MS_PER_TOMBSTONE,
|
|
161
|
+
ESTIMATED_MS_PER_VECTOR_REBUILD,
|
|
162
|
+
GRAPH_RETIRED,
|
|
163
|
+
IN_MEMORY,
|
|
164
|
+
MIN_OVERSAMPLE,
|
|
165
|
+
NATIVE_OK,
|
|
166
|
+
NOTHING_STAGED,
|
|
167
|
+
NO_LIVE_VECTOR_AT_POSITION,
|
|
168
|
+
QUERY_STAGED,
|
|
169
|
+
RECALIBRATION_CHUNK_VECTORS,
|
|
170
|
+
RECALIBRATION_DRIFT_RATIO,
|
|
171
|
+
REFUSAL_COLLECTION_MIN_INTERVAL_MS,
|
|
172
|
+
REFUSAL_COLLECTION_TIME_MULTIPLE,
|
|
173
|
+
REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS,
|
|
174
|
+
SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES,
|
|
175
|
+
SEARCH_REFILL_RESERVE,
|
|
176
|
+
SEARCH_SCAN_CHUNK_VECTORS,
|
|
177
|
+
SHUFFLED_REBUILD_MEMORY_RETRY_MS,
|
|
178
|
+
STORE_BLOCK_COUNT,
|
|
179
|
+
STORE_CALIBRATED,
|
|
180
|
+
STORE_CALIBRATION_GENERATION,
|
|
181
|
+
STORE_CODE_BLOCK_COUNT,
|
|
182
|
+
STORE_CODE_COUNT,
|
|
183
|
+
STORE_DOC_ID_BYTES,
|
|
184
|
+
STORE_LIVE_COUNT,
|
|
185
|
+
STORE_ORDINAL_LAYOUT_GENERATION,
|
|
186
|
+
STORE_RELEASED_VECTORS_TO_DISK,
|
|
187
|
+
STORE_REMOVED_COUNT,
|
|
188
|
+
STORE_SLOTS,
|
|
189
|
+
STORE_UNPLACED_COUNT,
|
|
190
|
+
STORE_WRITE_SEQUENCE_INT64,
|
|
191
|
+
UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS,
|
|
192
|
+
UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS,
|
|
193
|
+
UI_THREAD_TURN_BUDGET_MS,
|
|
194
|
+
VECTOR_MEMORY_LIMITS,
|
|
195
|
+
VECTOR_WORKER_COPIES_ALLOWED,
|
|
196
|
+
WORKER_COPY_MIN_VECTORS,
|
|
197
|
+
addToOrdinalFilter,
|
|
198
|
+
admissionAt,
|
|
199
|
+
adoptBackgroundScheduler,
|
|
200
|
+
adoptGraph,
|
|
201
|
+
adoptLoadedCalibration,
|
|
202
|
+
allLiveDocIds,
|
|
203
|
+
arenaFloat32Distance,
|
|
204
|
+
assignStorePartitions,
|
|
205
|
+
backgroundTurnIsSpent,
|
|
206
|
+
blocksGrowthBytes,
|
|
207
|
+
bytesToVectors,
|
|
208
|
+
centroidOfCentroidSums,
|
|
209
|
+
collectAndReturnPages,
|
|
210
|
+
collectYoungGeneration,
|
|
211
|
+
cosineSimilarityWithMagnitudes,
|
|
212
|
+
countOfCentroidSums,
|
|
213
|
+
createCentroidSums,
|
|
214
|
+
createHNSWIndex,
|
|
215
|
+
createOrdinalFilter,
|
|
216
|
+
createOsqQuantizer,
|
|
217
|
+
createSharedVectorStoreHandles,
|
|
218
|
+
createVectorBlock,
|
|
219
|
+
createVectorFileReader,
|
|
220
|
+
defaultQuantizationFor,
|
|
221
|
+
deserialize,
|
|
222
|
+
detachNativeStore,
|
|
223
|
+
dotProduct
|
|
224
|
+
} = vectorCore);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// src/vector/vector-host/vector-core-2.ts
|
|
228
|
+
var coverOrdinal;
|
|
229
|
+
var emptyFieldBeforeRestore;
|
|
230
|
+
var ensureBlockSlots;
|
|
231
|
+
var euclideanDistance;
|
|
232
|
+
var fieldHandlesOf;
|
|
233
|
+
var filterForOptions;
|
|
234
|
+
var fixedView;
|
|
235
|
+
var graphNeedsRebuild;
|
|
236
|
+
var growBufferTo;
|
|
237
|
+
var grownByteLength;
|
|
238
|
+
var handOverBuildTurns;
|
|
239
|
+
var holdAdmission;
|
|
240
|
+
var holdWritesAndBackgroundWork;
|
|
241
|
+
var insertNode;
|
|
242
|
+
var isDead;
|
|
243
|
+
var liveOrdinals;
|
|
244
|
+
var liveSize;
|
|
245
|
+
var magnitude;
|
|
246
|
+
var mayFree;
|
|
247
|
+
var meanOfCentroidSums;
|
|
248
|
+
var nativeMetricCode;
|
|
249
|
+
var nativeScoresOf;
|
|
250
|
+
var nativeStoreFor;
|
|
251
|
+
var nativeStoreScratchBytes;
|
|
252
|
+
var nextSequence;
|
|
253
|
+
var noBuildHistory;
|
|
254
|
+
var nodeCountOf;
|
|
255
|
+
var noteFallback;
|
|
256
|
+
var oldestHeldPin;
|
|
257
|
+
var openGraphState;
|
|
258
|
+
var openSharedQuantizer;
|
|
259
|
+
var openVectorBlock;
|
|
260
|
+
var ordinalFilterForSubsets;
|
|
261
|
+
var ordinalFilterHas;
|
|
262
|
+
var ordinalFilterValues;
|
|
263
|
+
var osqBitsOf;
|
|
264
|
+
var planCheckpoint;
|
|
265
|
+
var publishSequence;
|
|
266
|
+
var recalibrateFromStore;
|
|
267
|
+
var recordCheckpoint;
|
|
268
|
+
var releaseLocksHeldBy;
|
|
269
|
+
var releasePinsHeldBy;
|
|
270
|
+
var releaseReplacedVectors;
|
|
271
|
+
var requireWholeNeighbourLists;
|
|
272
|
+
var resetStoreHeader;
|
|
273
|
+
var retiredAtOf;
|
|
274
|
+
var searchOrdinals;
|
|
275
|
+
var sequenceWritesOf;
|
|
276
|
+
var sharedMemoryAvailable;
|
|
277
|
+
var sharedVectorStoreBytes;
|
|
278
|
+
var shuffleInPlace;
|
|
279
|
+
var shuffleStep;
|
|
280
|
+
var signatureOf;
|
|
281
|
+
var slotByteOffset;
|
|
282
|
+
var squaredLengthsOfCentroidSums;
|
|
283
|
+
var stampBorn;
|
|
284
|
+
var stampRetired;
|
|
285
|
+
var storePinsOf;
|
|
286
|
+
var threadedNativeCore;
|
|
287
|
+
var threadsHoldCurrentLayout;
|
|
288
|
+
var toDistance;
|
|
289
|
+
var toScore;
|
|
290
|
+
var tombstoneCountOf;
|
|
291
|
+
var topLayerInParts;
|
|
292
|
+
var turnsAreBudgeted;
|
|
293
|
+
var unplacedChunkCount;
|
|
294
|
+
var withNativeThreadSlot;
|
|
295
|
+
var withTransientAdmission;
|
|
296
|
+
var writeSequenceOf;
|
|
297
|
+
var writeSequenceWords;
|
|
298
|
+
var yieldBackgroundTurn;
|
|
299
|
+
var yieldBuildTurn;
|
|
300
|
+
function bindVectorCore2Names(vectorCore) {
|
|
301
|
+
({
|
|
302
|
+
coverOrdinal,
|
|
303
|
+
emptyFieldBeforeRestore,
|
|
304
|
+
ensureBlockSlots,
|
|
305
|
+
euclideanDistance,
|
|
306
|
+
fieldHandlesOf,
|
|
307
|
+
filterForOptions,
|
|
308
|
+
fixedView,
|
|
309
|
+
graphNeedsRebuild,
|
|
310
|
+
growBufferTo,
|
|
311
|
+
grownByteLength,
|
|
312
|
+
handOverBuildTurns,
|
|
313
|
+
holdAdmission,
|
|
314
|
+
holdWritesAndBackgroundWork,
|
|
315
|
+
insertNode,
|
|
316
|
+
isDead,
|
|
317
|
+
liveOrdinals,
|
|
318
|
+
liveSize,
|
|
319
|
+
magnitude,
|
|
320
|
+
mayFree,
|
|
321
|
+
meanOfCentroidSums,
|
|
322
|
+
nativeMetricCode,
|
|
323
|
+
nativeScoresOf,
|
|
324
|
+
nativeStoreFor,
|
|
325
|
+
nativeStoreScratchBytes,
|
|
326
|
+
nextSequence,
|
|
327
|
+
noBuildHistory,
|
|
328
|
+
nodeCountOf,
|
|
329
|
+
noteFallback,
|
|
330
|
+
oldestHeldPin,
|
|
331
|
+
openGraphState,
|
|
332
|
+
openSharedQuantizer,
|
|
333
|
+
openVectorBlock,
|
|
334
|
+
ordinalFilterForSubsets,
|
|
335
|
+
ordinalFilterHas,
|
|
336
|
+
ordinalFilterValues,
|
|
337
|
+
osqBitsOf,
|
|
338
|
+
planCheckpoint,
|
|
339
|
+
publishSequence,
|
|
340
|
+
recalibrateFromStore,
|
|
341
|
+
recordCheckpoint,
|
|
342
|
+
releaseLocksHeldBy,
|
|
343
|
+
releasePinsHeldBy,
|
|
344
|
+
releaseReplacedVectors,
|
|
345
|
+
requireWholeNeighbourLists,
|
|
346
|
+
resetStoreHeader,
|
|
347
|
+
retiredAtOf,
|
|
348
|
+
searchOrdinals,
|
|
349
|
+
sequenceWritesOf,
|
|
350
|
+
sharedMemoryAvailable,
|
|
351
|
+
sharedVectorStoreBytes,
|
|
352
|
+
shuffleInPlace,
|
|
353
|
+
shuffleStep,
|
|
354
|
+
signatureOf,
|
|
355
|
+
slotByteOffset,
|
|
356
|
+
squaredLengthsOfCentroidSums,
|
|
357
|
+
stampBorn,
|
|
358
|
+
stampRetired,
|
|
359
|
+
storePinsOf,
|
|
360
|
+
threadedNativeCore,
|
|
361
|
+
threadsHoldCurrentLayout,
|
|
362
|
+
toDistance,
|
|
363
|
+
toScore,
|
|
364
|
+
tombstoneCountOf,
|
|
365
|
+
topLayerInParts,
|
|
366
|
+
turnsAreBudgeted,
|
|
367
|
+
unplacedChunkCount,
|
|
368
|
+
withNativeThreadSlot,
|
|
369
|
+
withTransientAdmission,
|
|
370
|
+
writeSequenceOf,
|
|
371
|
+
writeSequenceWords,
|
|
372
|
+
yieldBackgroundTurn,
|
|
373
|
+
yieldBuildTurn
|
|
374
|
+
} = vectorCore);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// src/vector/vector-host/index.browser.ts
|
|
378
|
+
function bindVectorHost(host, vectorCore) {
|
|
379
|
+
bindEngineNames(host);
|
|
380
|
+
bindVectorCore1Names(vectorCore);
|
|
381
|
+
bindVectorCore2Names(vectorCore);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// src/engine/query/subset-search.ts
|
|
385
|
+
function filterSubsetsMatchNothing(subsets) {
|
|
386
|
+
for (const subset of subsets) {
|
|
387
|
+
for (let wordIndex = 0; wordIndex < subset.bits.length; wordIndex++) {
|
|
388
|
+
if (subset.bits[wordIndex] !== 0) return false;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return true;
|
|
392
|
+
}
|
|
393
|
+
function filterSubsetsRenumbered(subsets) {
|
|
394
|
+
return subsets.some((subset) => subset.owner.docStore.numberingGeneration() !== subset.numberingGeneration);
|
|
395
|
+
}
|
|
396
|
+
async function searchCurrentSubsets(fieldName, first, collectAgain, search2) {
|
|
397
|
+
let subsets = first;
|
|
398
|
+
for (let attempt = 1; ; attempt++) {
|
|
399
|
+
const answer = await search2(subsets);
|
|
400
|
+
if (!filterSubsetsRenumbered(subsets)) return answer;
|
|
401
|
+
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
402
|
+
throw new NarsilError(
|
|
403
|
+
ErrorCodes.WORKER_BUSY,
|
|
404
|
+
`The index renumbered its documents during each of ${attempt} filtered searches of vector field "${fieldName}"`,
|
|
405
|
+
{ fieldName, attempts: attempt }
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
subsets = collectAgain();
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// src/search/fusion-rankings.ts
|
|
413
|
+
function compareFused(left, right) {
|
|
414
|
+
return (right.score ?? 0) - (left.score ?? 0) || compareCodePoints(left.docId, right.docId);
|
|
415
|
+
}
|
|
416
|
+
function fuseByRank(lists, k, project) {
|
|
417
|
+
const positions = /* @__PURE__ */ new Map();
|
|
418
|
+
const results = [];
|
|
419
|
+
for (const list of lists) {
|
|
420
|
+
for (let rank = 0; rank < list.length; rank++) {
|
|
421
|
+
const entry = list[rank];
|
|
422
|
+
const contribution = 1 / (k + rank + 1);
|
|
423
|
+
const position = positions.get(entry.docId);
|
|
424
|
+
if (position === void 0) {
|
|
425
|
+
positions.set(entry.docId, results.length);
|
|
426
|
+
results.push(project(entry, contribution));
|
|
427
|
+
} else {
|
|
428
|
+
const result = results[position];
|
|
429
|
+
result.score = (result.score ?? 0) + contribution;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
results.sort(compareFused);
|
|
434
|
+
return results;
|
|
435
|
+
}
|
|
436
|
+
function saturate(score) {
|
|
437
|
+
if (score > Number.MAX_VALUE) return Number.MAX_VALUE;
|
|
438
|
+
if (score < -Number.MAX_VALUE) return -Number.MAX_VALUE;
|
|
439
|
+
return score;
|
|
440
|
+
}
|
|
441
|
+
function normalisedScores(entries) {
|
|
442
|
+
let min = Number.POSITIVE_INFINITY;
|
|
443
|
+
let max = Number.NEGATIVE_INFINITY;
|
|
444
|
+
for (const entry of entries) {
|
|
445
|
+
const score = saturate(entry.score ?? 0);
|
|
446
|
+
if (score < min) min = score;
|
|
447
|
+
if (score > max) max = score;
|
|
448
|
+
}
|
|
449
|
+
const width = max - min;
|
|
450
|
+
const normalised = /* @__PURE__ */ new Map();
|
|
451
|
+
for (const entry of entries) {
|
|
452
|
+
const score = saturate(entry.score ?? 0);
|
|
453
|
+
if (Number.isNaN(score)) normalised.set(entry.docId, 0);
|
|
454
|
+
else if (width === 0) normalised.set(entry.docId, 1);
|
|
455
|
+
else if (Number.isFinite(width)) normalised.set(entry.docId, (score - min) / width);
|
|
456
|
+
else normalised.set(entry.docId, (score / 2 - min / 2) / (max / 2 - min / 2));
|
|
457
|
+
}
|
|
458
|
+
return normalised;
|
|
459
|
+
}
|
|
460
|
+
function fuseByScore(text, vector, alpha, project) {
|
|
461
|
+
const textScores = normalisedScores(text);
|
|
462
|
+
const vectorScores = normalisedScores(vector);
|
|
463
|
+
const results = [];
|
|
464
|
+
const placed = /* @__PURE__ */ new Set();
|
|
465
|
+
for (const list of [text, vector]) {
|
|
466
|
+
for (const entry of list) {
|
|
467
|
+
if (placed.has(entry.docId)) continue;
|
|
468
|
+
placed.add(entry.docId);
|
|
469
|
+
const vectorScore = vectorScores.get(entry.docId) ?? 0;
|
|
470
|
+
const textScore = textScores.get(entry.docId) ?? 0;
|
|
471
|
+
results.push(project(entry, alpha * vectorScore + (1 - alpha) * textScore));
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
results.sort(compareFused);
|
|
475
|
+
return results;
|
|
476
|
+
}
|
|
477
|
+
function fuseRankings(text, vector, strategy, k, alpha, project) {
|
|
478
|
+
{
|
|
479
|
+
return strategy === 0 ? fuseByRank([text, vector], k, project) : fuseByScore(text, vector, alpha, project);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// src/search/fusion.ts
|
|
484
|
+
function rankConstantOf(k) {
|
|
485
|
+
if (k === void 0 || k === null) return DEFAULT_RANK_CONSTANT;
|
|
486
|
+
if (!Number.isInteger(k) || k < MIN_RANK_CONSTANT) {
|
|
487
|
+
throw new NarsilError(
|
|
488
|
+
ErrorCodes.CONFIG_INVALID,
|
|
489
|
+
`hybrid.k must be a whole number of at least ${MIN_RANK_CONSTANT}`,
|
|
490
|
+
{ k }
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
return k;
|
|
494
|
+
}
|
|
495
|
+
function linearAlphaOf(alpha) {
|
|
496
|
+
if (alpha === void 0 || alpha === null) return DEFAULT_LINEAR_ALPHA;
|
|
497
|
+
if (!Number.isFinite(alpha) || alpha < MIN_LINEAR_ALPHA || alpha > MAX_LINEAR_ALPHA) {
|
|
498
|
+
throw new NarsilError(
|
|
499
|
+
ErrorCodes.CONFIG_INVALID,
|
|
500
|
+
`hybrid.alpha must be a number from ${MIN_LINEAR_ALPHA} to ${MAX_LINEAR_ALPHA}`,
|
|
501
|
+
{ alpha }
|
|
502
|
+
);
|
|
503
|
+
}
|
|
504
|
+
return alpha;
|
|
505
|
+
}
|
|
506
|
+
function resolveHybridFusion(hybrid) {
|
|
507
|
+
const strategy = hybrid?.strategy ?? "rrf";
|
|
508
|
+
if (strategy !== "rrf" && strategy !== "linear") {
|
|
509
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `hybrid.strategy must be "rrf" or "linear"`, { strategy });
|
|
510
|
+
}
|
|
511
|
+
return { strategy, k: rankConstantOf(hybrid?.k), alpha: linearAlphaOf(hybrid?.alpha) };
|
|
512
|
+
}
|
|
513
|
+
function reciprocalRankFusion(lists, options) {
|
|
514
|
+
return fuseRankings(lists[0], lists[1], 0, options.k, DEFAULT_LINEAR_ALPHA, withFusedScore);
|
|
515
|
+
}
|
|
516
|
+
function linearCombination(textResults, vectorResults, options) {
|
|
517
|
+
return fuseRankings(textResults, vectorResults, 1, DEFAULT_RANK_CONSTANT, options.alpha, withFusedScore);
|
|
518
|
+
}
|
|
519
|
+
function withFusedScore(entry, score) {
|
|
520
|
+
return {
|
|
521
|
+
docId: entry.docId,
|
|
522
|
+
score,
|
|
523
|
+
termFrequencies: entry.termFrequencies,
|
|
524
|
+
fieldLengths: entry.fieldLengths,
|
|
525
|
+
idf: entry.idf
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// src/engine/query/vector.ts
|
|
530
|
+
function oversampleOf(vectorConfig) {
|
|
531
|
+
const oversample = vectorConfig.oversample;
|
|
532
|
+
if (oversample === void 0) return void 0;
|
|
533
|
+
if (typeof oversample !== "number" || !Number.isFinite(oversample) || oversample < MIN_OVERSAMPLE) {
|
|
534
|
+
throw new NarsilError(
|
|
535
|
+
ErrorCodes.CONFIG_INVALID,
|
|
536
|
+
`vector.oversample must be a finite number of at least ${MIN_OVERSAMPLE}`,
|
|
537
|
+
{ oversample }
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
return oversample;
|
|
541
|
+
}
|
|
542
|
+
function vectorFetchDepth(limit, offset, cursorDepth) {
|
|
543
|
+
const reach = cursorDepth + offset + limit;
|
|
544
|
+
if (reach > RESULT_WINDOW) {
|
|
545
|
+
throw new NarsilError(
|
|
546
|
+
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
547
|
+
`The engine pages a vector search through the first ${RESULT_WINDOW} results, and this page ends at ${reach}. Narrow the search with a filter or a similarity floor`,
|
|
548
|
+
{ limit, offset, cursorDepth, window: RESULT_WINDOW }
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
return Math.min(reach + 1, RESULT_WINDOW);
|
|
552
|
+
}
|
|
553
|
+
function vectorClock(context) {
|
|
554
|
+
const signal = signalOfSearch(context.turns);
|
|
555
|
+
const clock = context.patternWork.clock;
|
|
556
|
+
if (signal === void 0) return clock;
|
|
557
|
+
const nativeWindow = clock?.nativeWindow;
|
|
558
|
+
const bound = {
|
|
559
|
+
check() {
|
|
560
|
+
signal.throwIfAborted();
|
|
561
|
+
clock?.check();
|
|
562
|
+
},
|
|
563
|
+
tick(units) {
|
|
564
|
+
signal.throwIfAborted();
|
|
565
|
+
clock?.tick(units);
|
|
566
|
+
},
|
|
567
|
+
...nativeWindow === void 0 ? {} : { nativeWindow: () => nativeWindow.call(clock) }
|
|
568
|
+
};
|
|
569
|
+
bindSearchSignal(bound, signal);
|
|
570
|
+
return bound;
|
|
571
|
+
}
|
|
572
|
+
async function searchField(search2, k, host) {
|
|
573
|
+
const { vecIndex, vectorConfig, queryVector, context, filtered, rescoreWindow } = search2;
|
|
574
|
+
const options = {
|
|
575
|
+
clock: vectorClock(context),
|
|
576
|
+
metric: vectorConfig.metric ?? "cosine",
|
|
577
|
+
minSimilarity: vectorConfig.similarity ?? -Infinity,
|
|
578
|
+
efSearch: vectorConfig.efSearch,
|
|
579
|
+
oversample: oversampleOf(vectorConfig),
|
|
580
|
+
...rescoreWindow !== void 0 ? { rescoreWindow } : {}
|
|
581
|
+
};
|
|
582
|
+
if (filtered !== void 0) {
|
|
583
|
+
const outcome2 = await searchCurrentSubsets(
|
|
584
|
+
vecIndex.fieldName,
|
|
585
|
+
filtered.first,
|
|
586
|
+
filtered.collectAgain,
|
|
587
|
+
(filterSubsets) => vecIndex.searchParallel(queryVector, k, { ...options, filterSubsets })
|
|
588
|
+
);
|
|
589
|
+
requirePinnedState(context.pin);
|
|
590
|
+
return outcome2;
|
|
591
|
+
}
|
|
592
|
+
const filterPartitions = host.partitionsForVectorSearch(context.manager, vecIndex, context.partitionIds);
|
|
593
|
+
const outcome = await vecIndex.searchParallel(
|
|
594
|
+
queryVector,
|
|
595
|
+
k,
|
|
596
|
+
filterPartitions === void 0 ? options : { ...options, filterPartitions }
|
|
597
|
+
);
|
|
598
|
+
requirePinnedState(context.pin);
|
|
599
|
+
return outcome;
|
|
600
|
+
}
|
|
601
|
+
async function executeVectorSearch(host, params, context, limit, offset, cursorDepth = 0) {
|
|
602
|
+
const vectorConfig = params.vector;
|
|
603
|
+
if (!vectorConfig || !vectorConfig.value) {
|
|
604
|
+
return { scored: [], totalMatched: 0 };
|
|
605
|
+
}
|
|
606
|
+
const vecIndex = host.resolveVectorIndex(context, vectorConfig.field);
|
|
607
|
+
if (!vecIndex) {
|
|
608
|
+
return { scored: [], totalMatched: 0 };
|
|
609
|
+
}
|
|
610
|
+
const filtered = host.filteredSearchOf(params, context);
|
|
611
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
612
|
+
return { scored: [], totalMatched: 0 };
|
|
613
|
+
}
|
|
614
|
+
const search2 = {
|
|
615
|
+
vecIndex,
|
|
616
|
+
vectorConfig,
|
|
617
|
+
queryVector: new Float32Array(vectorConfig.value),
|
|
618
|
+
context,
|
|
619
|
+
filtered,
|
|
620
|
+
rescoreWindow: context.rescoreWindow
|
|
621
|
+
};
|
|
622
|
+
const outcome = await searchField(search2, vectorFetchDepth(limit, offset, cursorDepth), host);
|
|
623
|
+
const scored = host.vectorResultsToScored(outcome.results);
|
|
624
|
+
return { scored, totalMatched: outcome.matched, matchedExact: outcome.matchedExact };
|
|
625
|
+
}
|
|
626
|
+
async function vectorLegWith(host, vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
627
|
+
const vecIndex = host.resolveVectorIndex(context, vectorConfig.field);
|
|
628
|
+
if (!vecIndex) return { scored: [], holdsEveryMatch: true };
|
|
629
|
+
const search2 = { vecIndex, vectorConfig, queryVector, context, filtered, rescoreWindow };
|
|
630
|
+
const outcome = await searchField(search2, k, host);
|
|
631
|
+
return {
|
|
632
|
+
scored: host.vectorResultsToScored(outcome.results),
|
|
633
|
+
holdsEveryMatch: outcome.matchedExact && outcome.results.length === outcome.matched
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
async function executeHybridSearch(host, params, context, limit, offset, cursorDepth = 0) {
|
|
637
|
+
const { vector: vectorConfig, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
|
|
638
|
+
const fusion = resolveHybridFusion(params.hybrid);
|
|
639
|
+
const filtered = host.filteredSearchOf(params, context);
|
|
640
|
+
if (filtered !== void 0 && filterSubsetsMatchNothing(filtered.first)) {
|
|
641
|
+
return { scored: [], totalMatched: 0 };
|
|
642
|
+
}
|
|
643
|
+
if (!vectorConfig || !vectorConfig.value) {
|
|
644
|
+
const textOnly = await host.textLeg(textOnlyParams, context);
|
|
645
|
+
return {
|
|
646
|
+
scored: textOnly.scored,
|
|
647
|
+
totalMatched: textOnly.totalMatched,
|
|
648
|
+
facets: textOnly.facets,
|
|
649
|
+
...textOnly.documents !== void 0 ? { documents: textOnly.documents } : {}
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
const vectorPending = vectorLegWith(
|
|
653
|
+
host,
|
|
654
|
+
vectorConfig,
|
|
655
|
+
new Float32Array(vectorConfig.value),
|
|
656
|
+
context,
|
|
657
|
+
filtered,
|
|
658
|
+
vectorFetchDepth(limit, offset, cursorDepth)
|
|
659
|
+
);
|
|
660
|
+
const [textFanOutResult, vectorOutcome] = await Promise.all([host.textLeg(textOnlyParams, context), vectorPending]);
|
|
661
|
+
const vectorScored = vectorOutcome.scored;
|
|
662
|
+
let fusedScored;
|
|
663
|
+
if (fusion.strategy === "rrf") {
|
|
664
|
+
fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: fusion.k });
|
|
665
|
+
} else {
|
|
666
|
+
fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha: fusion.alpha });
|
|
667
|
+
}
|
|
668
|
+
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
669
|
+
const threshold = params.minScore;
|
|
670
|
+
fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
|
|
671
|
+
}
|
|
672
|
+
const textHoldsEveryMatch = textFanOutResult.scored.length === textFanOutResult.totalMatched;
|
|
673
|
+
return {
|
|
674
|
+
scored: fusedScored,
|
|
675
|
+
totalMatched: fusedScored.length,
|
|
676
|
+
matchedExact: vectorOutcome.holdsEveryMatch && textHoldsEveryMatch,
|
|
677
|
+
facets: textFanOutResult.facets,
|
|
678
|
+
...textFanOutResult.documents !== void 0 ? { documents: textFanOutResult.documents } : {}
|
|
679
|
+
};
|
|
680
|
+
}
|
|
19
681
|
|
|
20
682
|
// src/engine/vector-maintenance.ts
|
|
21
683
|
function resolveVectorIndexes(manager, indexName, fieldName) {
|
|
@@ -232,13 +894,18 @@ function readStoreFlags(store) {
|
|
|
232
894
|
}
|
|
233
895
|
|
|
234
896
|
// src/vector/shared-field/searcher.ts
|
|
235
|
-
function
|
|
897
|
+
function* storeOrdinals(store) {
|
|
898
|
+
const slots = store.slots;
|
|
899
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) yield ordinal;
|
|
900
|
+
}
|
|
901
|
+
function distancesInTypeScript(store, query, metric, ordinals, admission, clock) {
|
|
236
902
|
noteFallback("score");
|
|
237
903
|
const arenaQuery = store.prepareQueryArena(query);
|
|
238
904
|
const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
239
905
|
const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
|
|
240
906
|
for (let i = 0; i < ordinals.length; i++) {
|
|
241
907
|
if (clock !== void 0) clock.tick(query.length);
|
|
908
|
+
if (!admission.admits(ordinals[i])) continue;
|
|
242
909
|
if (arenaQuery) {
|
|
243
910
|
distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
244
911
|
continue;
|
|
@@ -248,11 +915,12 @@ function distancesInTypeScript(store, query, metric, ordinals, clock) {
|
|
|
248
915
|
}
|
|
249
916
|
return distances;
|
|
250
917
|
}
|
|
251
|
-
function scoreCandidates(
|
|
918
|
+
function scoreCandidates(scan, candidates) {
|
|
919
|
+
const { store, query, k, metric, minSimilarity, admission, clock } = scan;
|
|
252
920
|
const { ordinals, docIds } = candidates;
|
|
253
921
|
if (ordinals.length === 0) return { results: [], matched: 0 };
|
|
254
922
|
clock?.check();
|
|
255
|
-
const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals, clock);
|
|
923
|
+
const distances = nativeScoresOf(store.handles, query, metric, ordinals, admission.pin) ?? distancesInTypeScript(store, query, metric, ordinals, admission, clock);
|
|
256
924
|
clock?.check();
|
|
257
925
|
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
258
926
|
let matched = 0;
|
|
@@ -261,7 +929,7 @@ function scoreCandidates(store, candidates, query, k, metric, minSimilarity, clo
|
|
|
261
929
|
const score = toScore(distances[i], metric);
|
|
262
930
|
if (score < minSimilarity) continue;
|
|
263
931
|
matched++;
|
|
264
|
-
heap.push({ docId: docIds[i], score });
|
|
932
|
+
heap.push({ docId: docIds[i], score, ordinal: ordinals[i] });
|
|
265
933
|
}
|
|
266
934
|
return { results: heap.toSortedArray().reverse(), matched };
|
|
267
935
|
}
|
|
@@ -271,44 +939,58 @@ function createSharedVectorSearcher(options) {
|
|
|
271
939
|
function ordinalLayoutGeneration() {
|
|
272
940
|
return Atomics.load(view.store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
|
|
273
941
|
}
|
|
274
|
-
function ordinalSpace() {
|
|
942
|
+
function ordinalSpace(admission) {
|
|
275
943
|
const store = view.store;
|
|
276
944
|
const layoutGeneration2 = ordinalLayoutGeneration();
|
|
945
|
+
let retained;
|
|
946
|
+
function ordinalOf(docId) {
|
|
947
|
+
const current = view.ordinalOf(docId);
|
|
948
|
+
if (admission === void 0 || current !== void 0 && admission.admits(current)) return current;
|
|
949
|
+
if (retained === void 0) {
|
|
950
|
+
retained = /* @__PURE__ */ new Map();
|
|
951
|
+
for (let ordinal = 0; ordinal < store.slots; ordinal++) {
|
|
952
|
+
const id = admission.docIdOf(ordinal);
|
|
953
|
+
if (id !== void 0) retained.set(id, ordinal);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
return retained.get(docId);
|
|
957
|
+
}
|
|
277
958
|
return {
|
|
278
959
|
fieldName,
|
|
279
|
-
layoutGeneration: layoutGeneration2,
|
|
960
|
+
layoutGeneration: admission === void 0 ? layoutGeneration2 : -admission.pin - 1,
|
|
280
961
|
slots: store.slots,
|
|
281
962
|
header: store.handles.header,
|
|
282
|
-
ordinalOf
|
|
283
|
-
holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal) && store.partitionAt(ordinal) !== void 0,
|
|
284
|
-
isRemoved: (ordinal) => flags.isRemoved(ordinal)
|
|
963
|
+
ordinalOf,
|
|
964
|
+
holdsCurrentOrdinal: (ordinal) => admission === void 0 ? store.holdsOrdinal(ordinal) && store.partitionAt(ordinal) !== void 0 : admission.admits(ordinal),
|
|
965
|
+
isRemoved: (ordinal) => admission === void 0 ? flags.isRemoved(ordinal) : !admission.admits(ordinal)
|
|
285
966
|
};
|
|
286
967
|
}
|
|
287
|
-
function filterForPartitions(allowed) {
|
|
968
|
+
function filterForPartitions(allowed, admission) {
|
|
288
969
|
const slots = view.store.slots;
|
|
289
970
|
const filter = createOrdinalFilter(slots);
|
|
290
971
|
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
291
972
|
const partition = view.store.partitionAt(ordinal);
|
|
292
973
|
if (partition === void 0 || !allowed.has(partition)) continue;
|
|
293
|
-
if (!view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal))
|
|
974
|
+
if (admission === void 0 ? !view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal) : !admission.admits(ordinal))
|
|
975
|
+
continue;
|
|
294
976
|
addToOrdinalFilter(filter, ordinal);
|
|
295
977
|
}
|
|
296
978
|
return filter;
|
|
297
979
|
}
|
|
298
|
-
function filterFor(searchOptions) {
|
|
980
|
+
function filterFor(searchOptions, admission) {
|
|
299
981
|
if (searchOptions.filterSubsets !== void 0) {
|
|
300
|
-
return ordinalFilterForSubsets(searchOptions.filterSubsets, ordinalSpace());
|
|
982
|
+
return ordinalFilterForSubsets(searchOptions.filterSubsets, ordinalSpace(admission));
|
|
301
983
|
}
|
|
302
|
-
if (searchOptions.filterPartitions !== void 0)
|
|
984
|
+
if (searchOptions.filterPartitions !== void 0)
|
|
985
|
+
return filterForPartitions(searchOptions.filterPartitions, admission);
|
|
303
986
|
return void 0;
|
|
304
987
|
}
|
|
305
|
-
function candidatesOf(ordinals, filter) {
|
|
988
|
+
function candidatesOf(ordinals, filter, admission) {
|
|
306
989
|
const candidates = { ordinals: [], docIds: [] };
|
|
307
990
|
for (const ordinal of ordinals) {
|
|
308
991
|
if (filter !== void 0 && !ordinalFilterHas(filter, ordinal)) continue;
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
if (docId === void 0 || view.ordinalOf(docId) !== ordinal) continue;
|
|
992
|
+
const docId = admission.docIdOf(ordinal);
|
|
993
|
+
if (docId === void 0) continue;
|
|
312
994
|
candidates.ordinals.push(ordinal);
|
|
313
995
|
candidates.docIds.push(docId);
|
|
314
996
|
}
|
|
@@ -322,9 +1004,10 @@ function createSharedVectorSearcher(options) {
|
|
|
322
1004
|
function scanned(scan) {
|
|
323
1005
|
return { results: scan.results, matched: scan.matched, matchedExact: true, resolvedEveryHit: true };
|
|
324
1006
|
}
|
|
325
|
-
function search2(query, k, searchOptions, filter) {
|
|
1007
|
+
function search2(query, k, searchOptions, filter, admission) {
|
|
326
1008
|
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
|
|
327
1009
|
const clock = searchOptions.clock;
|
|
1010
|
+
const scan = { store: view.store, query, k, metric, minSimilarity, admission, clock };
|
|
328
1011
|
const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
|
|
329
1012
|
const liveDocuments = flags.liveDocuments();
|
|
330
1013
|
if (liveDocuments <= 0) return empty;
|
|
@@ -339,17 +1022,14 @@ function createSharedVectorSearcher(options) {
|
|
|
339
1022
|
if (k <= 0) return fetched([], true);
|
|
340
1023
|
const graph = view.graph;
|
|
341
1024
|
if (graph === null) {
|
|
342
|
-
const ordinals = filter === void 0 ?
|
|
343
|
-
return scanned(
|
|
344
|
-
scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity, clock)
|
|
345
|
-
);
|
|
1025
|
+
const ordinals = filter === void 0 ? storeOrdinals(view.store) : ordinalFilterValues(filter);
|
|
1026
|
+
return scanned(scoreCandidates(scan, candidatesOf(ordinals, filter, admission)));
|
|
346
1027
|
}
|
|
347
1028
|
if (filter !== void 0) {
|
|
348
1029
|
const graphLiveSize = view.liveSize;
|
|
349
1030
|
const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
|
|
350
1031
|
if (selectivity < view.handles.filterThreshold) {
|
|
351
|
-
|
|
352
|
-
return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity, clock));
|
|
1032
|
+
return scanned(scoreCandidates(scan, candidatesOf(ordinalFilterValues(filter), filter, admission)));
|
|
353
1033
|
}
|
|
354
1034
|
}
|
|
355
1035
|
const unplaced = flags.unplacedOrdinals();
|
|
@@ -360,18 +1040,54 @@ function createSharedVectorSearcher(options) {
|
|
|
360
1040
|
efSearch,
|
|
361
1041
|
oversample,
|
|
362
1042
|
rescoreWindow,
|
|
363
|
-
clock
|
|
1043
|
+
clock,
|
|
1044
|
+
admission
|
|
364
1045
|
})) {
|
|
365
|
-
const docId =
|
|
1046
|
+
const docId = admission.docIdOf(hit.ord);
|
|
366
1047
|
if (docId === void 0) {
|
|
367
1048
|
resolvedEveryHit = false;
|
|
368
1049
|
continue;
|
|
369
1050
|
}
|
|
370
|
-
graphResults.push({ docId, score: hit.score });
|
|
1051
|
+
graphResults.push({ docId, score: hit.score, ordinal: hit.ord });
|
|
371
1052
|
}
|
|
372
1053
|
if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
|
|
373
|
-
const
|
|
374
|
-
return fetched(mergeResults(graphResults,
|
|
1054
|
+
const unplacedScan = scoreCandidates(scan, candidatesOf(unplaced, filter, admission));
|
|
1055
|
+
return fetched(mergeResults(graphResults, unplacedScan.results, k), resolvedEveryHit);
|
|
1056
|
+
}
|
|
1057
|
+
function searchAtOnePin(query, k, searchOptions) {
|
|
1058
|
+
const held = searchOptions.admission;
|
|
1059
|
+
if (held !== void 0) {
|
|
1060
|
+
const filter = filterFor(searchOptions, held);
|
|
1061
|
+
const sequence = writeSequenceOf(writeSequenceWords(view.store.handles.header));
|
|
1062
|
+
const outcome = sequence === held.pin && !view.graphRetired ? search2(query, k, searchOptions, filter, held) : void 0;
|
|
1063
|
+
if (outcome !== void 0 && writeSequenceOf(writeSequenceWords(view.store.handles.header)) === held.pin && !view.graphRetired)
|
|
1064
|
+
return outcome;
|
|
1065
|
+
const ordinals = [];
|
|
1066
|
+
for (let ordinal = 0; ordinal < view.store.slots; ordinal++) {
|
|
1067
|
+
if (held.admits(ordinal)) ordinals.push(ordinal);
|
|
1068
|
+
}
|
|
1069
|
+
return scanned(
|
|
1070
|
+
scoreCandidates(
|
|
1071
|
+
{
|
|
1072
|
+
store: view.store,
|
|
1073
|
+
query,
|
|
1074
|
+
k,
|
|
1075
|
+
metric: searchOptions.metric,
|
|
1076
|
+
minSimilarity: searchOptions.minSimilarity,
|
|
1077
|
+
admission: held,
|
|
1078
|
+
clock: searchOptions.clock
|
|
1079
|
+
},
|
|
1080
|
+
candidatesOf(ordinals, filter, held)
|
|
1081
|
+
)
|
|
1082
|
+
);
|
|
1083
|
+
}
|
|
1084
|
+
const pins = storePinsOf(view.store);
|
|
1085
|
+
const pin = pins.take();
|
|
1086
|
+
try {
|
|
1087
|
+
return search2(query, k, searchOptions, filterFor(searchOptions), admissionAt(view.store, pin));
|
|
1088
|
+
} finally {
|
|
1089
|
+
pins.release(pin);
|
|
1090
|
+
}
|
|
375
1091
|
}
|
|
376
1092
|
return {
|
|
377
1093
|
fieldName,
|
|
@@ -386,10 +1102,12 @@ function createSharedVectorSearcher(options) {
|
|
|
386
1102
|
{ expected: view.handles.dimension, received: query.length }
|
|
387
1103
|
);
|
|
388
1104
|
}
|
|
1105
|
+
if (searchOptions.admission !== void 0)
|
|
1106
|
+
return Promise.resolve(answerOf(searchAtOnePin(query, k, searchOptions)));
|
|
389
1107
|
for (let attempt = 1; ; attempt++) {
|
|
390
1108
|
const layoutGeneration2 = ordinalLayoutGeneration();
|
|
391
|
-
const outcome =
|
|
392
|
-
if (view.graphRetired) {
|
|
1109
|
+
const outcome = searchAtOnePin(query, k, searchOptions);
|
|
1110
|
+
if (view.graphRetired && searchOptions.admission === void 0) {
|
|
393
1111
|
throw new NarsilError(
|
|
394
1112
|
ErrorCodes.WORKER_BUSY,
|
|
395
1113
|
`The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
|
|
@@ -833,29 +1551,290 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
833
1551
|
return outcome;
|
|
834
1552
|
},
|
|
835
1553
|
searchOrdinals(query, k, metric, minSimilarity, options) {
|
|
836
|
-
|
|
837
|
-
|
|
1554
|
+
const current = graph;
|
|
1555
|
+
if (current === null) return [];
|
|
1556
|
+
if (options.admission !== void 0) {
|
|
1557
|
+
return searchOrdinals(current, docIdOf, query, k, metric, minSimilarity, options);
|
|
1558
|
+
}
|
|
1559
|
+
return withTransientAdmission(
|
|
1560
|
+
store,
|
|
1561
|
+
(admission) => searchOrdinals(current, docIdOf, query, k, metric, minSimilarity, { ...options, admission })
|
|
1562
|
+
);
|
|
838
1563
|
}
|
|
839
1564
|
};
|
|
840
1565
|
}
|
|
841
1566
|
|
|
842
|
-
// src/
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
1567
|
+
// src/runtime/write-gate.ts
|
|
1568
|
+
function createWriteGate() {
|
|
1569
|
+
let holders = 0;
|
|
1570
|
+
let reopened = Promise.resolve();
|
|
1571
|
+
let reopen = () => void 0;
|
|
1572
|
+
async function enter(write) {
|
|
1573
|
+
while (holders > 0) await reopened;
|
|
1574
|
+
return write();
|
|
1575
|
+
}
|
|
1576
|
+
async function hold(read) {
|
|
1577
|
+
if (holders === 0) {
|
|
1578
|
+
reopened = new Promise((resolve) => {
|
|
1579
|
+
reopen = resolve;
|
|
1580
|
+
});
|
|
1581
|
+
}
|
|
1582
|
+
holders += 1;
|
|
1583
|
+
try {
|
|
1584
|
+
return await read();
|
|
1585
|
+
} finally {
|
|
1586
|
+
holders -= 1;
|
|
1587
|
+
if (holders === 0) reopen();
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
return { enter, hold };
|
|
855
1591
|
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
1592
|
+
|
|
1593
|
+
// src/runtime/turn-scheduler.ts
|
|
1594
|
+
function createTurnScheduler(scheduleHop) {
|
|
1595
|
+
const waiting2 = [];
|
|
1596
|
+
const waitingInBackground = [];
|
|
1597
|
+
let deadline = 0;
|
|
1598
|
+
let lastWriteCheckAt = 0;
|
|
1599
|
+
let slowestWriteUnitMs = 0;
|
|
1600
|
+
let writeUnitRunning = false;
|
|
1601
|
+
let lastBackgroundCheckAt = 0;
|
|
1602
|
+
let slowestBackgroundUnitMs = 0;
|
|
1603
|
+
let backgroundUnitRunning = false;
|
|
1604
|
+
let turnOpen = false;
|
|
1605
|
+
let hopScheduled = false;
|
|
1606
|
+
let backgroundSliceStartedAt = 0;
|
|
1607
|
+
let writesInFlight = 0;
|
|
1608
|
+
let readsPausingBackground = 0;
|
|
1609
|
+
let indexChangesInFlight = 0;
|
|
1610
|
+
let backgroundThisTurn = [];
|
|
1611
|
+
let hopGeneration = 0;
|
|
1612
|
+
let hopFollowsYield = false;
|
|
1613
|
+
let lastTurnWorkAt = Number.NEGATIVE_INFINITY;
|
|
1614
|
+
const indexChangesDrained = [];
|
|
1615
|
+
const writesDrained = [];
|
|
1616
|
+
const writeGate = createWriteGate();
|
|
1617
|
+
const indexChangeGate = createWriteGate();
|
|
1618
|
+
function scheduleNextTurn(followsYield) {
|
|
1619
|
+
hopGeneration += 1;
|
|
1620
|
+
const generation = hopGeneration;
|
|
1621
|
+
hopScheduled = true;
|
|
1622
|
+
hopFollowsYield = followsYield;
|
|
1623
|
+
scheduleHop(() => {
|
|
1624
|
+
if (generation === hopGeneration) runTurn();
|
|
1625
|
+
});
|
|
1626
|
+
}
|
|
1627
|
+
function requestHop() {
|
|
1628
|
+
if (!hopScheduled) scheduleNextTurn(false);
|
|
1629
|
+
}
|
|
1630
|
+
function requestHopAfterYield() {
|
|
1631
|
+
if (!hopScheduled || !hopFollowsYield) scheduleNextTurn(true);
|
|
1632
|
+
}
|
|
1633
|
+
function openTurn(now) {
|
|
1634
|
+
turnOpen = true;
|
|
1635
|
+
deadline = now + UI_THREAD_TURN_BUDGET_MS;
|
|
1636
|
+
slowestWriteUnitMs = 0;
|
|
1637
|
+
writeUnitRunning = false;
|
|
1638
|
+
slowestBackgroundUnitMs = 0;
|
|
1639
|
+
backgroundUnitRunning = false;
|
|
1640
|
+
backgroundSliceStartedAt = now;
|
|
1641
|
+
requestHop();
|
|
1642
|
+
}
|
|
1643
|
+
function startWriteUnit(now) {
|
|
1644
|
+
lastTurnWorkAt = now;
|
|
1645
|
+
lastWriteCheckAt = now;
|
|
1646
|
+
writeUnitRunning = true;
|
|
1647
|
+
backgroundUnitRunning = false;
|
|
1648
|
+
}
|
|
1649
|
+
function startBackgroundUnit(now) {
|
|
1650
|
+
lastTurnWorkAt = now;
|
|
1651
|
+
lastBackgroundCheckAt = now;
|
|
1652
|
+
backgroundUnitRunning = true;
|
|
1653
|
+
writeUnitRunning = false;
|
|
1654
|
+
}
|
|
1655
|
+
function keepBackgroundForNextTurn() {
|
|
1656
|
+
waitingInBackground.unshift(...backgroundThisTurn);
|
|
1657
|
+
backgroundThisTurn = [];
|
|
1658
|
+
}
|
|
1659
|
+
function resumeWhileTurnLasts() {
|
|
1660
|
+
const now = performance.now();
|
|
1661
|
+
if (waiting2.length > 0) {
|
|
1662
|
+
if (now + slowestWriteUnitMs >= deadline) {
|
|
1663
|
+
keepBackgroundForNextTurn();
|
|
1664
|
+
return;
|
|
1665
|
+
}
|
|
1666
|
+
const resumeWrite = waiting2.shift();
|
|
1667
|
+
startWriteUnit(now);
|
|
1668
|
+
resumeWrite?.();
|
|
1669
|
+
void Promise.resolve().then(resumeWhileTurnLasts);
|
|
1670
|
+
return;
|
|
1671
|
+
}
|
|
1672
|
+
if (readsPausingBackground > 0 || now + slowestBackgroundUnitMs >= deadline) {
|
|
1673
|
+
keepBackgroundForNextTurn();
|
|
1674
|
+
return;
|
|
1675
|
+
}
|
|
1676
|
+
const resumeBackground = backgroundThisTurn.shift();
|
|
1677
|
+
if (resumeBackground === void 0) return;
|
|
1678
|
+
backgroundSliceStartedAt = now;
|
|
1679
|
+
startBackgroundUnit(now);
|
|
1680
|
+
resumeBackground();
|
|
1681
|
+
void Promise.resolve().then(resumeWhileTurnLasts);
|
|
1682
|
+
}
|
|
1683
|
+
function runTurn() {
|
|
1684
|
+
hopScheduled = false;
|
|
1685
|
+
hopFollowsYield = false;
|
|
1686
|
+
turnOpen = false;
|
|
1687
|
+
if (waiting2.length === 0 && waitingInBackground.length === 0) return;
|
|
1688
|
+
openTurn(performance.now());
|
|
1689
|
+
backgroundThisTurn = waitingInBackground.splice(0);
|
|
1690
|
+
resumeWhileTurnLasts();
|
|
1691
|
+
}
|
|
1692
|
+
function turnIsSpent2() {
|
|
1693
|
+
const now = performance.now();
|
|
1694
|
+
if (!turnOpen) {
|
|
1695
|
+
if (now - lastTurnWorkAt < UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS) return true;
|
|
1696
|
+
openTurn(now);
|
|
1697
|
+
}
|
|
1698
|
+
if (writeUnitRunning) slowestWriteUnitMs = Math.max(slowestWriteUnitMs, now - lastWriteCheckAt);
|
|
1699
|
+
startWriteUnit(now);
|
|
1700
|
+
return now + slowestWriteUnitMs >= deadline;
|
|
1701
|
+
}
|
|
1702
|
+
function backgroundTurnIsSpent2() {
|
|
1703
|
+
if (!turnOpen || readsPausingBackground > 0) return true;
|
|
1704
|
+
const now = performance.now();
|
|
1705
|
+
if (backgroundUnitRunning) {
|
|
1706
|
+
slowestBackgroundUnitMs = Math.max(slowestBackgroundUnitMs, now - lastBackgroundCheckAt);
|
|
1707
|
+
}
|
|
1708
|
+
startBackgroundUnit(now);
|
|
1709
|
+
const nextUnitEndsAt = now + slowestBackgroundUnitMs;
|
|
1710
|
+
if (nextUnitEndsAt >= deadline || waiting2.length > 0) return true;
|
|
1711
|
+
return writesInFlight > 0 && nextUnitEndsAt - backgroundSliceStartedAt >= UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS;
|
|
1712
|
+
}
|
|
1713
|
+
function waitIn(queue) {
|
|
1714
|
+
return new Promise((resolve) => {
|
|
1715
|
+
queue.push(resolve);
|
|
1716
|
+
requestHopAfterYield();
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1719
|
+
function leaveCountedWrite() {
|
|
1720
|
+
writesInFlight -= 1;
|
|
1721
|
+
if (writesInFlight === 0) for (const drained of writesDrained.splice(0)) drained();
|
|
1722
|
+
}
|
|
1723
|
+
function holdOnceWritesDrain(read) {
|
|
1724
|
+
if (writesInFlight === 0) return writeGate.hold(read);
|
|
1725
|
+
return new Promise((resolve, reject) => {
|
|
1726
|
+
writesDrained.push(() => {
|
|
1727
|
+
writeGate.hold(read).then(resolve, reject);
|
|
1728
|
+
});
|
|
1729
|
+
});
|
|
1730
|
+
}
|
|
1731
|
+
async function countEnteredWrite(write) {
|
|
1732
|
+
writesInFlight += 1;
|
|
1733
|
+
try {
|
|
1734
|
+
return await write();
|
|
1735
|
+
} finally {
|
|
1736
|
+
leaveCountedWrite();
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
async function waitOutsideCountedWrite(wait) {
|
|
1740
|
+
if (writesInFlight === 0) return wait();
|
|
1741
|
+
leaveCountedWrite();
|
|
1742
|
+
try {
|
|
1743
|
+
return await wait();
|
|
1744
|
+
} finally {
|
|
1745
|
+
await writeGate.enter(resumeCountedWrite);
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
function resumeCountedWrite() {
|
|
1749
|
+
writesInFlight += 1;
|
|
1750
|
+
return Promise.resolve();
|
|
1751
|
+
}
|
|
1752
|
+
async function countIndexChange(change) {
|
|
1753
|
+
indexChangesInFlight += 1;
|
|
1754
|
+
try {
|
|
1755
|
+
return await change();
|
|
1756
|
+
} finally {
|
|
1757
|
+
indexChangesInFlight -= 1;
|
|
1758
|
+
if (indexChangesInFlight === 0) for (const drained of indexChangesDrained.splice(0)) drained();
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
function changeIndexOutsideExternalWaits(change) {
|
|
1762
|
+
return indexChangeGate.enter(() => countIndexChange(change));
|
|
1763
|
+
}
|
|
1764
|
+
function untilIndexChangesDrain() {
|
|
1765
|
+
if (indexChangesInFlight === 0) return Promise.resolve();
|
|
1766
|
+
return new Promise((resolve) => {
|
|
1767
|
+
indexChangesDrained.push(resolve);
|
|
1768
|
+
});
|
|
1769
|
+
}
|
|
1770
|
+
async function readOnceIndexChangesDrain(read) {
|
|
1771
|
+
await untilIndexChangesDrain();
|
|
1772
|
+
readsPausingBackground += 1;
|
|
1773
|
+
try {
|
|
1774
|
+
return await read();
|
|
1775
|
+
} finally {
|
|
1776
|
+
readsPausingBackground -= 1;
|
|
1777
|
+
if (readsPausingBackground === 0 && waitingInBackground.length > 0) requestHop();
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
function holdWritesAndBackgroundWork2(read) {
|
|
1781
|
+
return holdOnceWritesDrain(() => indexChangeGate.hold(() => readOnceIndexChangesDrain(read)));
|
|
1782
|
+
}
|
|
1783
|
+
return {
|
|
1784
|
+
turnIsSpent: turnIsSpent2,
|
|
1785
|
+
yieldTurn: () => waitIn(waiting2),
|
|
1786
|
+
backgroundTurnIsSpent: backgroundTurnIsSpent2,
|
|
1787
|
+
yieldBackgroundTurn: () => waitIn(waitingInBackground),
|
|
1788
|
+
countWrite: (write) => writeGate.enter(() => countEnteredWrite(write)),
|
|
1789
|
+
holdWrites: writeGate.hold,
|
|
1790
|
+
holdWritesAndBackgroundWork: holdWritesAndBackgroundWork2,
|
|
1791
|
+
changeIndexOutsideExternalWaits,
|
|
1792
|
+
waitOutsideCountedWrite
|
|
1793
|
+
};
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
// src/runtime/background-turn-sharing.browser.ts
|
|
1797
|
+
function messageChannelHop() {
|
|
1798
|
+
let channel = null;
|
|
1799
|
+
let receiver = null;
|
|
1800
|
+
let pending = null;
|
|
1801
|
+
function openChannel() {
|
|
1802
|
+
const opened = new MessageChannel();
|
|
1803
|
+
const port = opened.port1;
|
|
1804
|
+
port.onmessage = () => {
|
|
1805
|
+
const next = pending;
|
|
1806
|
+
pending = null;
|
|
1807
|
+
next?.();
|
|
1808
|
+
if (pending === null) port.unref?.();
|
|
1809
|
+
};
|
|
1810
|
+
channel = opened;
|
|
1811
|
+
receiver = port;
|
|
1812
|
+
return port;
|
|
1813
|
+
}
|
|
1814
|
+
return {
|
|
1815
|
+
schedule: (run) => {
|
|
1816
|
+
pending = run;
|
|
1817
|
+
const port = receiver ?? openChannel();
|
|
1818
|
+
port.ref?.();
|
|
1819
|
+
channel?.port2.postMessage(null);
|
|
1820
|
+
},
|
|
1821
|
+
closeWhileIdle: () => {
|
|
1822
|
+
if (pending !== null || channel === null) return;
|
|
1823
|
+
channel.port1.close();
|
|
1824
|
+
channel.port2.close();
|
|
1825
|
+
channel = null;
|
|
1826
|
+
receiver = null;
|
|
1827
|
+
}
|
|
1828
|
+
};
|
|
1829
|
+
}
|
|
1830
|
+
var sharedHop = null;
|
|
1831
|
+
function shareTurnsWithBackgroundWork() {
|
|
1832
|
+
if (sharedHop !== null) return;
|
|
1833
|
+
sharedHop = messageChannelHop();
|
|
1834
|
+
adoptBackgroundScheduler(createTurnScheduler(sharedHop.schedule));
|
|
1835
|
+
}
|
|
1836
|
+
function closeIdleTurnHop() {
|
|
1837
|
+
sharedHop?.closeWhileIdle();
|
|
859
1838
|
}
|
|
860
1839
|
|
|
861
1840
|
// src/vector/vector-store/ordinal-reads.ts
|
|
@@ -935,6 +1914,25 @@ function snapshotOf(open, documents, replacedOrdinals) {
|
|
|
935
1914
|
return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
|
|
936
1915
|
}
|
|
937
1916
|
|
|
1917
|
+
// src/vector/vector-store/doc-ids.ts
|
|
1918
|
+
var encoder = new TextEncoder();
|
|
1919
|
+
function appendDocId(handles, ordinal, docId) {
|
|
1920
|
+
const bytes = encoder.encode(docId);
|
|
1921
|
+
const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
1922
|
+
const end = start + bytes.length;
|
|
1923
|
+
growBufferTo(handles.docIdBytes, end);
|
|
1924
|
+
growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
|
|
1925
|
+
new Uint8Array(handles.docIdBytes).set(bytes, start);
|
|
1926
|
+
const offsets = new Uint32Array(handles.docIdOffsets);
|
|
1927
|
+
offsets[ordinal] = start;
|
|
1928
|
+
offsets[ordinal + 1] = end;
|
|
1929
|
+
Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
|
|
1930
|
+
}
|
|
1931
|
+
function resetDocIds(handles) {
|
|
1932
|
+
new Uint32Array(handles.docIdOffsets, 0, handles.docIdOffsets.byteLength / 4).fill(0);
|
|
1933
|
+
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
1934
|
+
}
|
|
1935
|
+
|
|
938
1936
|
// src/vector/vector-store/open-store.ts
|
|
939
1937
|
function openHandles(dimension, codeBits, blockBytes) {
|
|
940
1938
|
const handles = createSharedVectorStoreHandles(dimension, codeBits, blockBytes);
|
|
@@ -946,7 +1944,8 @@ function openHandles(dimension, codeBits, blockBytes) {
|
|
|
946
1944
|
partitions: new Int32Array(handles.partitions),
|
|
947
1945
|
diskFile: new Int32Array(handles.diskFile),
|
|
948
1946
|
diskOffset: new Uint32Array(handles.diskOffset),
|
|
949
|
-
longestDocIdBytes: 0
|
|
1947
|
+
longestDocIdBytes: 0,
|
|
1948
|
+
sequences: sequenceWritesOf(handles)
|
|
950
1949
|
};
|
|
951
1950
|
}
|
|
952
1951
|
function relayout(store) {
|
|
@@ -967,10 +1966,13 @@ function publishOrdinal(store, ordinal) {
|
|
|
967
1966
|
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
968
1967
|
Atomics.store(store.present, ordinal, 1);
|
|
969
1968
|
}
|
|
970
|
-
function
|
|
1969
|
+
function placeOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
971
1970
|
store.magnitudes[ordinal] = vectorMagnitude;
|
|
972
1971
|
store.diskOffset[ordinal] = location.offset;
|
|
973
1972
|
store.diskFile[ordinal] = location.fileIndex;
|
|
1973
|
+
}
|
|
1974
|
+
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
1975
|
+
placeOnDisk(store, ordinal, vectorMagnitude, location);
|
|
974
1976
|
publishOrdinal(store, ordinal);
|
|
975
1977
|
}
|
|
976
1978
|
function relocateFiles(store, rewrite) {
|
|
@@ -1034,6 +2036,8 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
1034
2036
|
new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY, diskFileBytes / 4);
|
|
1035
2037
|
}
|
|
1036
2038
|
growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
|
|
2039
|
+
growBufferTo(handles.bornAt, (ordinal + 1) * 8);
|
|
2040
|
+
growBufferTo(handles.retiredAt, (ordinal + 1) * 8);
|
|
1037
2041
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
1038
2042
|
ensureCodeSlot(store, ordinal);
|
|
1039
2043
|
}
|
|
@@ -1046,7 +2050,7 @@ function growthBytesFor(store, vectors) {
|
|
|
1046
2050
|
const last = Math.min(slots + vectors, VECTOR_MEMORY_LIMITS.maxOrdinals) - 1;
|
|
1047
2051
|
if (last < slots) return 0;
|
|
1048
2052
|
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
1049
|
-
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
2053
|
+
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.bornAt, (last + 1) * 8) + bufferGrowthBytes(handles.retiredAt, (last + 1) * 8) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
1050
2054
|
if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
|
|
1051
2055
|
return bytes;
|
|
1052
2056
|
}
|
|
@@ -1112,6 +2116,162 @@ function partitionFilterOf(store, documents, partitionIds) {
|
|
|
1112
2116
|
return filter;
|
|
1113
2117
|
}
|
|
1114
2118
|
|
|
2119
|
+
// src/vector/vector-store/retirement.ts
|
|
2120
|
+
var UNKNOWN_PARTITION = -1;
|
|
2121
|
+
var NO_PARTITION = -2;
|
|
2122
|
+
var LOADED_SEQUENCE = 1;
|
|
2123
|
+
var RETIRED_BEFORE_LOAD = 1;
|
|
2124
|
+
function oldestPinOf(store) {
|
|
2125
|
+
return oldestHeldPin(store.handles.pins);
|
|
2126
|
+
}
|
|
2127
|
+
function freeable(store, ordinal, oldestPin) {
|
|
2128
|
+
return mayFree(retiredAtOf(store.sequences, ordinal), oldestPin);
|
|
2129
|
+
}
|
|
2130
|
+
function recordPartition(state, store, ordinal, partitionId) {
|
|
2131
|
+
const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
|
|
2132
|
+
store.partitions[ordinal] = given;
|
|
2133
|
+
if (given === UNKNOWN_PARTITION) state.unknownPartitions += 1;
|
|
2134
|
+
}
|
|
2135
|
+
function freeRetired(state, store, ordinal) {
|
|
2136
|
+
state.awaitingFree.delete(ordinal);
|
|
2137
|
+
Atomics.store(store.present, ordinal, 0);
|
|
2138
|
+
state.checkpointPins.note(ordinal);
|
|
2139
|
+
}
|
|
2140
|
+
function retireOrdinal(state, store, ordinal, oldestPin) {
|
|
2141
|
+
state.ordToDoc[ordinal] = void 0;
|
|
2142
|
+
if (store.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
2143
|
+
if (freeable(store, ordinal, oldestPin)) {
|
|
2144
|
+
freeRetired(state, store, ordinal);
|
|
2145
|
+
return;
|
|
2146
|
+
}
|
|
2147
|
+
state.awaitingFree.add(ordinal);
|
|
2148
|
+
}
|
|
2149
|
+
function keepReplacedOrdinal(state, store, ordinal) {
|
|
2150
|
+
state.ordToDoc[ordinal] = void 0;
|
|
2151
|
+
if (store.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
2152
|
+
state.replacedOrdinals.add(ordinal);
|
|
2153
|
+
}
|
|
2154
|
+
function freeAwaitingOrdinals(state) {
|
|
2155
|
+
const store = state.open;
|
|
2156
|
+
if (store === null || state.awaitingFree.size === 0) return;
|
|
2157
|
+
const oldestPin = oldestPinOf(store);
|
|
2158
|
+
for (const ordinal of state.awaitingFree) {
|
|
2159
|
+
if (freeable(store, ordinal, oldestPin)) freeRetired(state, store, ordinal);
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
function claimOrdinal(state, store, docId, cold, partitionId) {
|
|
2163
|
+
const ordinal = claimSlot(store, docId, cold);
|
|
2164
|
+
recordPartition(state, store, ordinal, partitionId);
|
|
2165
|
+
state.ordToDoc[ordinal] = docId;
|
|
2166
|
+
state.docToOrd.set(docId, ordinal);
|
|
2167
|
+
return ordinal;
|
|
2168
|
+
}
|
|
2169
|
+
function appendOrdinal(state, store, docId, vector, sequence, partitionId) {
|
|
2170
|
+
const ordinal = claimOrdinal(state, store, docId, false, partitionId);
|
|
2171
|
+
writeVector(store, ordinal, vector);
|
|
2172
|
+
stampBorn(store.sequences, ordinal, sequence);
|
|
2173
|
+
publishOrdinal(store, ordinal);
|
|
2174
|
+
return ordinal;
|
|
2175
|
+
}
|
|
2176
|
+
function settleReplacement(state, store, previous, keepsReplacedVector) {
|
|
2177
|
+
if (previous === void 0) {
|
|
2178
|
+
state.liveCount++;
|
|
2179
|
+
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
2180
|
+
return;
|
|
2181
|
+
}
|
|
2182
|
+
if (keepsReplacedVector) keepReplacedOrdinal(state, store, previous);
|
|
2183
|
+
else retireOrdinal(state, store, previous, oldestPinOf(store));
|
|
2184
|
+
}
|
|
2185
|
+
function beginWrite(state, store) {
|
|
2186
|
+
freeAwaitingOrdinals(state);
|
|
2187
|
+
return nextSequence(store.sequences);
|
|
2188
|
+
}
|
|
2189
|
+
function retireInWrite(store, ordinal, sequence) {
|
|
2190
|
+
stampRetired(store.sequences, ordinal, sequence);
|
|
2191
|
+
}
|
|
2192
|
+
function endWrite(store, sequence) {
|
|
2193
|
+
publishSequence(store.sequences, sequence);
|
|
2194
|
+
}
|
|
2195
|
+
function retireStoredOrdinal(state, ordinal) {
|
|
2196
|
+
const store = state.open;
|
|
2197
|
+
if (store === null) return false;
|
|
2198
|
+
if (state.awaitingFree.size > 0) freeAwaitingOrdinals(state);
|
|
2199
|
+
const writes = store.sequences;
|
|
2200
|
+
if (ordinal >= writes.retired.length) coverOrdinal(writes, ordinal);
|
|
2201
|
+
const retiredHalves = writes.retiredHalves;
|
|
2202
|
+
const retiredLow = ordinal * 2;
|
|
2203
|
+
if (retiredHalves[retiredLow] !== 0 || retiredHalves[retiredLow + 1] !== 0) return false;
|
|
2204
|
+
const sequence = nextSequence(writes);
|
|
2205
|
+
Atomics.store(writes.retired, ordinal, BigInt(sequence));
|
|
2206
|
+
publishSequence(writes, sequence);
|
|
2207
|
+
return true;
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
// src/vector/vector-store/load.ts
|
|
2211
|
+
function retiredBeforeLoad(store, ordinal) {
|
|
2212
|
+
stampRetired(store.sequences, ordinal, RETIRED_BEFORE_LOAD);
|
|
2213
|
+
}
|
|
2214
|
+
function settleLoadedReplacement(state, store, previous) {
|
|
2215
|
+
if (previous === void 0) {
|
|
2216
|
+
state.liveCount++;
|
|
2217
|
+
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
2218
|
+
return;
|
|
2219
|
+
}
|
|
2220
|
+
stampRetired(store.sequences, previous, RETIRED_BEFORE_LOAD);
|
|
2221
|
+
retireOrdinal(state, store, previous, Number.POSITIVE_INFINITY);
|
|
2222
|
+
}
|
|
2223
|
+
function loadColdVector(state, store, docId, vectorMagnitude, location, partitionId) {
|
|
2224
|
+
const previous = state.docToOrd.get(docId);
|
|
2225
|
+
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
2226
|
+
const ordinal = claimOrdinal(state, store, docId, true, partitionId);
|
|
2227
|
+
placeOnDisk(store, ordinal, vectorMagnitude, location);
|
|
2228
|
+
stampBorn(store.sequences, ordinal, LOADED_SEQUENCE);
|
|
2229
|
+
publishOrdinal(store, ordinal);
|
|
2230
|
+
settleLoadedReplacement(state, store, previous);
|
|
2231
|
+
return ordinal;
|
|
2232
|
+
}
|
|
2233
|
+
function loadReplacedVector(state, store, docId, vector, location) {
|
|
2234
|
+
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
2235
|
+
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
2236
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
2237
|
+
state.ordToDoc[ordinal] = void 0;
|
|
2238
|
+
state.replacedOrdinals.add(ordinal);
|
|
2239
|
+
retiredBeforeLoad(store, ordinal);
|
|
2240
|
+
if (location !== void 0) {
|
|
2241
|
+
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
2242
|
+
return ordinal;
|
|
2243
|
+
}
|
|
2244
|
+
writeVector(store, ordinal, vector);
|
|
2245
|
+
publishOrdinal(store, ordinal);
|
|
2246
|
+
return ordinal;
|
|
2247
|
+
}
|
|
2248
|
+
function restoreReplacedVector(state, store, ordinal, vector) {
|
|
2249
|
+
store.diskFile[ordinal] = IN_MEMORY;
|
|
2250
|
+
retiredBeforeLoad(store, ordinal);
|
|
2251
|
+
writeVector(store, ordinal, vector);
|
|
2252
|
+
Atomics.store(store.present, ordinal, 1);
|
|
2253
|
+
state.replacedOrdinals.add(ordinal);
|
|
2254
|
+
}
|
|
2255
|
+
function loadSnapshot(state, store, snapshot) {
|
|
2256
|
+
const replaced = new Set(snapshot.replacedOrdinals);
|
|
2257
|
+
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
2258
|
+
const docId = snapshot.docIds[ordinal];
|
|
2259
|
+
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
2260
|
+
if (docId === null || docId === void 0) {
|
|
2261
|
+
ensureOrdinal(store, ordinal, false);
|
|
2262
|
+
state.ordToDoc[ordinal] = void 0;
|
|
2263
|
+
store.partitions[ordinal] = NO_PARTITION;
|
|
2264
|
+
if (replaced.has(ordinal)) restoreReplacedVector(state, store, ordinal, vector);
|
|
2265
|
+
else retiredBeforeLoad(store, ordinal);
|
|
2266
|
+
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
2267
|
+
continue;
|
|
2268
|
+
}
|
|
2269
|
+
appendOrdinal(state, store, docId, vector, LOADED_SEQUENCE);
|
|
2270
|
+
state.liveCount++;
|
|
2271
|
+
}
|
|
2272
|
+
Atomics.store(store.handles.header, STORE_LIVE_COUNT, state.liveCount);
|
|
2273
|
+
}
|
|
2274
|
+
|
|
1115
2275
|
// src/vector/vector-store/retired-pins.ts
|
|
1116
2276
|
function createRetiredVectorPins() {
|
|
1117
2277
|
let pins = 0;
|
|
@@ -1166,93 +2326,95 @@ function pinnedReads(pins, openStore, readable) {
|
|
|
1166
2326
|
};
|
|
1167
2327
|
}
|
|
1168
2328
|
|
|
2329
|
+
// src/vector/vector-store/store-clear.ts
|
|
2330
|
+
function retireEveryOrdinal(store) {
|
|
2331
|
+
const sequences = store.sequences;
|
|
2332
|
+
const sequence = nextSequence(sequences);
|
|
2333
|
+
const slots = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
2334
|
+
for (let ordinal = 0; ordinal < slots; ordinal++) stampRetired(sequences, ordinal, sequence);
|
|
2335
|
+
publishSequence(sequences, sequence);
|
|
2336
|
+
}
|
|
2337
|
+
function forgetCodes(store) {
|
|
2338
|
+
const { header, codePresent } = store.handles;
|
|
2339
|
+
new Uint8Array(codePresent, 0, codePresent.byteLength).fill(0);
|
|
2340
|
+
Atomics.store(header, STORE_LIVE_COUNT, 0);
|
|
2341
|
+
Atomics.store(header, STORE_CODE_COUNT, 0);
|
|
2342
|
+
Atomics.store(header, STORE_CALIBRATED, 0);
|
|
2343
|
+
Atomics.add(header, STORE_CALIBRATION_GENERATION, 1);
|
|
2344
|
+
}
|
|
2345
|
+
function resetInPlace(store) {
|
|
2346
|
+
const { handles } = store;
|
|
2347
|
+
store.present.fill(0, 0, handles.present.byteLength);
|
|
2348
|
+
new Uint8Array(handles.codePresent, 0, handles.codePresent.byteLength).fill(0);
|
|
2349
|
+
new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY);
|
|
2350
|
+
handles.centroid.fill(0);
|
|
2351
|
+
if (handles.vectorFiles.length > 0) {
|
|
2352
|
+
handles.vectorFiles.length = 0;
|
|
2353
|
+
relayout(store);
|
|
2354
|
+
}
|
|
2355
|
+
resetDocIds(handles);
|
|
2356
|
+
resetStoreHeader(handles.header);
|
|
2357
|
+
}
|
|
2358
|
+
function clearStore(store) {
|
|
2359
|
+
retireEveryOrdinal(store);
|
|
2360
|
+
if (oldestHeldPin(store.handles.pins) === Number.POSITIVE_INFINITY) {
|
|
2361
|
+
resetInPlace(store);
|
|
2362
|
+
return;
|
|
2363
|
+
}
|
|
2364
|
+
forgetCodes(store);
|
|
2365
|
+
}
|
|
2366
|
+
|
|
1169
2367
|
// src/vector/vector-store/index.ts
|
|
1170
|
-
var UNKNOWN_PARTITION = -1;
|
|
1171
|
-
var NO_PARTITION = -2;
|
|
1172
2368
|
function createVectorStore(options) {
|
|
1173
2369
|
const codeBits = options?.codeBits ?? null;
|
|
1174
2370
|
const blockBytes = Math.min(options?.blockBytes ?? VECTOR_MEMORY_LIMITS.blockBytes, VECTOR_MEMORY_LIMITS.blockBytes);
|
|
1175
|
-
const
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
2371
|
+
const state = {
|
|
2372
|
+
open: options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes),
|
|
2373
|
+
docToOrd: createDocIdMap(),
|
|
2374
|
+
ordToDoc: [],
|
|
2375
|
+
replacedOrdinals: /* @__PURE__ */ new Set(),
|
|
2376
|
+
awaitingFree: /* @__PURE__ */ new Set(),
|
|
2377
|
+
checkpointPins: createRetiredVectorPins(),
|
|
2378
|
+
unknownPartitions: 0,
|
|
2379
|
+
liveCount: 0
|
|
2380
|
+
};
|
|
2381
|
+
const { docToOrd, ordToDoc, replacedOrdinals, awaitingFree } = state;
|
|
1181
2382
|
let readBackVector = null;
|
|
1182
|
-
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
|
|
1183
2383
|
function forgetDocuments() {
|
|
1184
2384
|
docToOrd.clear();
|
|
1185
2385
|
ordToDoc.length = 0;
|
|
1186
2386
|
replacedOrdinals.clear();
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
2387
|
+
state.awaitingFree.clear();
|
|
2388
|
+
state.checkpointPins.forget();
|
|
2389
|
+
state.unknownPartitions = 0;
|
|
2390
|
+
state.liveCount = 0;
|
|
1190
2391
|
}
|
|
1191
|
-
function
|
|
1192
|
-
return
|
|
2392
|
+
function heldAfterRemoval(ordinal) {
|
|
2393
|
+
return replacedOrdinals.has(ordinal) || awaitingFree.has(ordinal);
|
|
2394
|
+
}
|
|
2395
|
+
function isReadable(ordinal) {
|
|
2396
|
+
return ordToDoc[ordinal] !== void 0 || heldAfterRemoval(ordinal);
|
|
1193
2397
|
}
|
|
1194
2398
|
function requireOpen() {
|
|
1195
|
-
if (open === null) {
|
|
2399
|
+
if (state.open === null) {
|
|
1196
2400
|
throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
|
|
1197
2401
|
}
|
|
1198
|
-
return open;
|
|
1199
|
-
}
|
|
1200
|
-
function recordPartition(store, ordinal, partitionId) {
|
|
1201
|
-
const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
|
|
1202
|
-
store.partitions[ordinal] = given;
|
|
1203
|
-
if (given === UNKNOWN_PARTITION) unknownPartitions += 1;
|
|
1204
|
-
}
|
|
1205
|
-
function retireOrdinal(store, ordinal) {
|
|
1206
|
-
ordToDoc[ordinal] = void 0;
|
|
1207
|
-
Atomics.store(store.present, ordinal, 0);
|
|
1208
|
-
pins.note(ordinal);
|
|
1209
|
-
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
1210
|
-
}
|
|
1211
|
-
function keepReplacedOrdinal(store, ordinal) {
|
|
1212
|
-
ordToDoc[ordinal] = void 0;
|
|
1213
|
-
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
1214
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
1215
|
-
replacedOrdinals.add(ordinal);
|
|
1216
|
-
}
|
|
1217
|
-
function restoreReplacedVector(store, ordinal, vector) {
|
|
1218
|
-
store.diskFile[ordinal] = IN_MEMORY;
|
|
1219
|
-
writeVector(store, ordinal, vector);
|
|
1220
|
-
Atomics.store(store.present, ordinal, 1);
|
|
1221
|
-
replacedOrdinals.add(ordinal);
|
|
1222
|
-
}
|
|
1223
|
-
function claimOrdinal(store, docId, cold, partitionId) {
|
|
1224
|
-
const ordinal = claimSlot(store, docId, cold);
|
|
1225
|
-
recordPartition(store, ordinal, partitionId);
|
|
1226
|
-
ordToDoc[ordinal] = docId;
|
|
1227
|
-
docToOrd.set(docId, ordinal);
|
|
1228
|
-
return ordinal;
|
|
2402
|
+
return state.open;
|
|
1229
2403
|
}
|
|
1230
|
-
function
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
publishOrdinal(store, ordinal);
|
|
1234
|
-
return ordinal;
|
|
1235
|
-
}
|
|
1236
|
-
function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
|
|
1237
|
-
if (previous !== void 0) {
|
|
1238
|
-
if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
|
|
1239
|
-
else retireOrdinal(store, previous);
|
|
1240
|
-
return ordinal;
|
|
1241
|
-
}
|
|
1242
|
-
liveCount++;
|
|
1243
|
-
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
1244
|
-
return ordinal;
|
|
2404
|
+
function openFor(dimension) {
|
|
2405
|
+
state.open ??= openHandles(dimension, codeBits, blockBytes);
|
|
2406
|
+
return state.open;
|
|
1245
2407
|
}
|
|
1246
2408
|
return {
|
|
1247
|
-
...createOrdinalReads(() => open,
|
|
2409
|
+
...createOrdinalReads(() => state.open, isReadable),
|
|
1248
2410
|
get size() {
|
|
1249
|
-
return liveCount;
|
|
2411
|
+
return state.liveCount;
|
|
1250
2412
|
},
|
|
1251
2413
|
get slots() {
|
|
1252
2414
|
return ordToDoc.length;
|
|
1253
2415
|
},
|
|
1254
2416
|
get dimension() {
|
|
1255
|
-
return open === null ? 0 : open.handles.dimension;
|
|
2417
|
+
return state.open === null ? 0 : state.open.handles.dimension;
|
|
1256
2418
|
},
|
|
1257
2419
|
get codeBits() {
|
|
1258
2420
|
return codeBits;
|
|
@@ -1267,8 +2429,7 @@ function createVectorStore(options) {
|
|
|
1267
2429
|
return replacedOrdinals.size;
|
|
1268
2430
|
},
|
|
1269
2431
|
insert(docId, vector, partitionId, keepsReplacedVector = false) {
|
|
1270
|
-
|
|
1271
|
-
const store = open;
|
|
2432
|
+
const store = openFor(vector.length);
|
|
1272
2433
|
if (vector.length !== store.handles.dimension) {
|
|
1273
2434
|
throw new NarsilError(
|
|
1274
2435
|
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
@@ -1277,40 +2438,54 @@ function createVectorStore(options) {
|
|
|
1277
2438
|
);
|
|
1278
2439
|
}
|
|
1279
2440
|
const previous = docToOrd.get(docId);
|
|
1280
|
-
const
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
2441
|
+
const sequence = beginWrite(state, store);
|
|
2442
|
+
const ordinal = appendOrdinal(state, store, docId, vector, sequence, partitionId);
|
|
2443
|
+
if (previous !== void 0) retireInWrite(store, previous, sequence);
|
|
2444
|
+
endWrite(store, sequence);
|
|
2445
|
+
settleReplacement(state, store, previous, keepsReplacedVector);
|
|
2446
|
+
return ordinal;
|
|
2447
|
+
},
|
|
2448
|
+
retire(ordinal) {
|
|
2449
|
+
retireStoredOrdinal(state, ordinal);
|
|
2450
|
+
},
|
|
2451
|
+
freeRetiredVectors() {
|
|
2452
|
+
freeAwaitingOrdinals(state);
|
|
2453
|
+
},
|
|
2454
|
+
oldestPin() {
|
|
2455
|
+
return state.open === null ? Number.POSITIVE_INFINITY : oldestPinOf(state.open);
|
|
2456
|
+
},
|
|
2457
|
+
writeSequence() {
|
|
2458
|
+
return state.open === null ? 0 : writeSequenceOf(state.open.sequences.header);
|
|
2459
|
+
},
|
|
2460
|
+
retiredStampHalves() {
|
|
2461
|
+
return requireOpen().sequences.retiredHalves;
|
|
2462
|
+
},
|
|
2463
|
+
claimedSlots() {
|
|
2464
|
+
return state.open === null ? 0 : state.open.handles.header[STORE_SLOTS];
|
|
2465
|
+
},
|
|
2466
|
+
mayFreeOrdinal(ordinal, oldestPin) {
|
|
2467
|
+
return state.open !== null && freeable(state.open, ordinal, oldestPin);
|
|
2468
|
+
},
|
|
2469
|
+
releaseReplacedVectors(oldestPin) {
|
|
2470
|
+
const store = state.open;
|
|
2471
|
+
const released = [];
|
|
2472
|
+
const oldest = oldestPin ?? (store === null ? Number.POSITIVE_INFINITY : oldestPinOf(store));
|
|
2473
|
+
for (const ordinal of replacedOrdinals) {
|
|
2474
|
+
if (store !== null && !freeable(store, ordinal, oldest)) continue;
|
|
2475
|
+
released.push(ordinal);
|
|
2476
|
+
}
|
|
2477
|
+
for (const ordinal of released) {
|
|
2478
|
+
replacedOrdinals.delete(ordinal);
|
|
2479
|
+
if (store !== null) freeRetired(state, store, ordinal);
|
|
2480
|
+
}
|
|
1288
2481
|
return released;
|
|
1289
2482
|
},
|
|
1290
|
-
...pinnedReads(
|
|
2483
|
+
...pinnedReads(state.checkpointPins, () => state.open, isReadable),
|
|
1291
2484
|
insertCold(docId, vectorMagnitude, location, partitionId) {
|
|
1292
|
-
|
|
1293
|
-
const previous = docToOrd.get(docId);
|
|
1294
|
-
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
1295
|
-
const ordinal = claimOrdinal(store, docId, true, partitionId);
|
|
1296
|
-
publishOnDisk(store, ordinal, vectorMagnitude, location);
|
|
1297
|
-
return settleReplacement(store, ordinal, previous, false);
|
|
2485
|
+
return loadColdVector(state, requireOpen(), docId, vectorMagnitude, location, partitionId);
|
|
1298
2486
|
},
|
|
1299
2487
|
insertReplaced(docId, vector, location) {
|
|
1300
|
-
|
|
1301
|
-
const store = open;
|
|
1302
|
-
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
1303
|
-
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
1304
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
1305
|
-
ordToDoc[ordinal] = void 0;
|
|
1306
|
-
replacedOrdinals.add(ordinal);
|
|
1307
|
-
if (location !== void 0) {
|
|
1308
|
-
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
1309
|
-
return ordinal;
|
|
1310
|
-
}
|
|
1311
|
-
writeVector(store, ordinal, vector);
|
|
1312
|
-
publishOrdinal(store, ordinal);
|
|
1313
|
-
return ordinal;
|
|
2488
|
+
return loadReplacedVector(state, openFor(vector.length), docId, vector, location);
|
|
1314
2489
|
},
|
|
1315
2490
|
addVectorFile(path) {
|
|
1316
2491
|
const store = requireOpen();
|
|
@@ -1321,8 +2496,8 @@ function createVectorStore(options) {
|
|
|
1321
2496
|
return store.handles.vectorFiles.length - 1;
|
|
1322
2497
|
},
|
|
1323
2498
|
isCold(ordinal) {
|
|
1324
|
-
if (open === null) return false;
|
|
1325
|
-
return open.view.isCold(ordinal);
|
|
2499
|
+
if (state.open === null) return false;
|
|
2500
|
+
return state.open.view.isCold(ordinal);
|
|
1326
2501
|
},
|
|
1327
2502
|
releaseToFile(ordinal, location) {
|
|
1328
2503
|
const store = requireOpen();
|
|
@@ -1338,7 +2513,9 @@ function createVectorStore(options) {
|
|
|
1338
2513
|
},
|
|
1339
2514
|
releaseColdBlocks() {
|
|
1340
2515
|
const store = requireOpen();
|
|
1341
|
-
const
|
|
2516
|
+
const held = new Set(replacedOrdinals);
|
|
2517
|
+
for (const ordinal of state.awaitingFree) held.add(ordinal);
|
|
2518
|
+
const kept = state.checkpointPins.keptWith(held);
|
|
1342
2519
|
let released = 0;
|
|
1343
2520
|
for (let index = 0; index < store.handles.blocks.length; index++) {
|
|
1344
2521
|
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, kept)) continue;
|
|
@@ -1350,38 +2527,40 @@ function createVectorStore(options) {
|
|
|
1350
2527
|
},
|
|
1351
2528
|
setPartition(docId, partitionId) {
|
|
1352
2529
|
const ordinal = docToOrd.get(docId);
|
|
1353
|
-
if (ordinal === void 0 || open === null || partitionId < 0) return;
|
|
1354
|
-
if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
1355
|
-
open.partitions[ordinal] = partitionId;
|
|
2530
|
+
if (ordinal === void 0 || state.open === null || partitionId < 0) return;
|
|
2531
|
+
if (state.open.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
2532
|
+
state.open.partitions[ordinal] = partitionId;
|
|
1356
2533
|
},
|
|
1357
2534
|
forgetPartition(docId) {
|
|
1358
2535
|
const ordinal = docToOrd.get(docId);
|
|
1359
|
-
if (ordinal === void 0 || open === null) return;
|
|
1360
|
-
if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
1361
|
-
open.partitions[ordinal] = NO_PARTITION;
|
|
2536
|
+
if (ordinal === void 0 || state.open === null) return;
|
|
2537
|
+
if (state.open.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
|
|
2538
|
+
state.open.partitions[ordinal] = NO_PARTITION;
|
|
1362
2539
|
},
|
|
1363
2540
|
partitionOfOrdinal(ordinal) {
|
|
1364
|
-
if (open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
|
|
1365
|
-
const partitionId = open.partitions[ordinal];
|
|
2541
|
+
if (state.open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
|
|
2542
|
+
const partitionId = state.open.partitions[ordinal];
|
|
1366
2543
|
return partitionId < 0 ? void 0 : partitionId;
|
|
1367
2544
|
},
|
|
1368
2545
|
get partitionsKnown() {
|
|
1369
|
-
return unknownPartitions === 0;
|
|
2546
|
+
return state.unknownPartitions === 0;
|
|
1370
2547
|
},
|
|
1371
2548
|
partitionFilter(partitionIds) {
|
|
1372
|
-
return partitionFilterOf(open, ordToDoc, partitionIds);
|
|
2549
|
+
return partitionFilterOf(state.open, ordToDoc, partitionIds);
|
|
1373
2550
|
},
|
|
1374
|
-
remove(docId) {
|
|
2551
|
+
remove(docId, oldestPin) {
|
|
1375
2552
|
const ordinal = docToOrd.get(docId);
|
|
1376
|
-
|
|
2553
|
+
const store = state.open;
|
|
2554
|
+
if (ordinal === void 0 || store === null) return;
|
|
2555
|
+
const retiredNow = retireStoredOrdinal(state, ordinal);
|
|
1377
2556
|
docToOrd.delete(docId);
|
|
1378
|
-
retireOrdinal(
|
|
1379
|
-
liveCount--;
|
|
1380
|
-
Atomics.sub(
|
|
2557
|
+
retireOrdinal(state, store, ordinal, retiredNow || oldestPin === void 0 ? oldestPinOf(store) : oldestPin);
|
|
2558
|
+
state.liveCount--;
|
|
2559
|
+
Atomics.sub(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
1381
2560
|
},
|
|
1382
2561
|
get(docId) {
|
|
1383
2562
|
const ordinal = docToOrd.get(docId);
|
|
1384
|
-
return ordinal === void 0 || open === null ? void 0 : entryAt(open, ordinal);
|
|
2563
|
+
return ordinal === void 0 || state.open === null ? void 0 : entryAt(state.open, ordinal);
|
|
1385
2564
|
},
|
|
1386
2565
|
has(docId) {
|
|
1387
2566
|
return docToOrd.has(docId);
|
|
@@ -1389,31 +2568,22 @@ function createVectorStore(options) {
|
|
|
1389
2568
|
holdsOrdinal(ordinal) {
|
|
1390
2569
|
return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
|
|
1391
2570
|
},
|
|
1392
|
-
entries: () => entriesOf(open, ordToDoc),
|
|
1393
|
-
docIds: () => docIdsOf(open, ordToDoc),
|
|
2571
|
+
entries: () => entriesOf(state.open, ordToDoc),
|
|
2572
|
+
docIds: () => docIdsOf(state.open, ordToDoc),
|
|
1394
2573
|
clear() {
|
|
2574
|
+
const store = state.open;
|
|
1395
2575
|
forgetDocuments();
|
|
1396
|
-
if (
|
|
1397
|
-
open.present.fill(0, 0, open.handles.present.byteLength);
|
|
1398
|
-
new Uint8Array(open.handles.codePresent, 0, open.handles.codePresent.byteLength).fill(0);
|
|
1399
|
-
new Int32Array(open.handles.diskFile, 0, open.handles.diskFile.byteLength / 4).fill(IN_MEMORY);
|
|
1400
|
-
open.handles.centroid.fill(0);
|
|
1401
|
-
if (open.handles.vectorFiles.length > 0) {
|
|
1402
|
-
open.handles.vectorFiles.length = 0;
|
|
1403
|
-
relayout(open);
|
|
1404
|
-
}
|
|
1405
|
-
resetDocIds(open.handles);
|
|
1406
|
-
resetStoreHeader(open.handles.header);
|
|
2576
|
+
if (store !== null) clearStore(store);
|
|
1407
2577
|
},
|
|
1408
2578
|
release() {
|
|
1409
2579
|
forgetDocuments();
|
|
1410
|
-
if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
1411
|
-
open?.view.close();
|
|
1412
|
-
open = null;
|
|
2580
|
+
if (state.open !== null) Atomics.add(state.open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
|
|
2581
|
+
state.open?.view.close();
|
|
2582
|
+
state.open = null;
|
|
1413
2583
|
},
|
|
1414
|
-
closeFiles: () => open?.view.close(),
|
|
2584
|
+
closeFiles: () => state.open?.view.close(),
|
|
1415
2585
|
relocateVectorFiles(rewrite) {
|
|
1416
|
-
if (open !== null) relocateFiles(open, rewrite);
|
|
2586
|
+
if (state.open !== null) relocateFiles(state.open, rewrite);
|
|
1417
2587
|
},
|
|
1418
2588
|
getOrdinal(docId) {
|
|
1419
2589
|
return docToOrd.get(docId);
|
|
@@ -1422,36 +2592,19 @@ function createVectorStore(options) {
|
|
|
1422
2592
|
return ordToDoc[ordinal];
|
|
1423
2593
|
},
|
|
1424
2594
|
prepareQueryArena(query) {
|
|
1425
|
-
return open === null ? null : open.view.prepareQueryArena(query);
|
|
2595
|
+
return state.open === null ? null : state.open.view.prepareQueryArena(query);
|
|
1426
2596
|
},
|
|
1427
|
-
exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
|
|
2597
|
+
exportSnapshot: () => snapshotOf(state.open, ordToDoc, replacedOrdinals),
|
|
1428
2598
|
restoreSnapshot(snapshot) {
|
|
1429
2599
|
this.clear();
|
|
1430
2600
|
if (snapshot.dimension === 0 || snapshot.slots === 0) return;
|
|
1431
|
-
|
|
1432
|
-
const store = open;
|
|
1433
|
-
const replaced = new Set(snapshot.replacedOrdinals);
|
|
1434
|
-
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
1435
|
-
const docId = snapshot.docIds[ordinal];
|
|
1436
|
-
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
1437
|
-
if (docId === null || docId === void 0) {
|
|
1438
|
-
ensureOrdinal(store, ordinal, false);
|
|
1439
|
-
ordToDoc[ordinal] = void 0;
|
|
1440
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
1441
|
-
if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
|
|
1442
|
-
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
1443
|
-
continue;
|
|
1444
|
-
}
|
|
1445
|
-
appendOrdinal(store, docId, vector);
|
|
1446
|
-
liveCount++;
|
|
1447
|
-
}
|
|
1448
|
-
Atomics.store(store.handles.header, STORE_LIVE_COUNT, liveCount);
|
|
2601
|
+
loadSnapshot(state, openFor(snapshot.dimension), snapshot);
|
|
1449
2602
|
},
|
|
1450
2603
|
memoryBytes() {
|
|
1451
|
-
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
2604
|
+
return state.open === null ? 0 : sharedVectorStoreBytes(state.open.handles);
|
|
1452
2605
|
},
|
|
1453
2606
|
growthBytesFor(vectors) {
|
|
1454
|
-
return open === null ? 0 : growthBytesFor(open, vectors);
|
|
2607
|
+
return state.open === null ? 0 : growthBytesFor(state.open, vectors);
|
|
1455
2608
|
}
|
|
1456
2609
|
};
|
|
1457
2610
|
}
|
|
@@ -1548,6 +2701,16 @@ function moveOutOfBuffer(state, docId) {
|
|
|
1548
2701
|
if (!state.buffer.delete(docId)) return;
|
|
1549
2702
|
for (const watch of state.searchWatches) watch.leftBuffer.add(docId);
|
|
1550
2703
|
}
|
|
2704
|
+
function writeWatchesOf(state) {
|
|
2705
|
+
return {
|
|
2706
|
+
watchWrites(listener) {
|
|
2707
|
+
state.writeListeners.add(listener);
|
|
2708
|
+
return () => {
|
|
2709
|
+
state.writeListeners.delete(listener);
|
|
2710
|
+
};
|
|
2711
|
+
}
|
|
2712
|
+
};
|
|
2713
|
+
}
|
|
1551
2714
|
async function watchingSearch(state, search2) {
|
|
1552
2715
|
const watch = { written: /* @__PURE__ */ new Set(), leftBuffer: /* @__PURE__ */ new Set() };
|
|
1553
2716
|
state.searchWatches.add(watch);
|
|
@@ -1558,6 +2721,11 @@ async function watchingSearch(state, search2) {
|
|
|
1558
2721
|
}
|
|
1559
2722
|
}
|
|
1560
2723
|
|
|
2724
|
+
// src/vector/search-pool/worker-entry.ts
|
|
2725
|
+
function resolveWorkerEntryPoint() {
|
|
2726
|
+
return resolveWorkerEntry(import.meta.url, /\/src\/vector\/search-pool\/[^/]+$/, "vector/search-worker.mjs");
|
|
2727
|
+
}
|
|
2728
|
+
|
|
1561
2729
|
// src/vector/search-pool/constants.ts
|
|
1562
2730
|
var VECTOR_POOL_DRAIN_TIMEOUT_MS = 1e4;
|
|
1563
2731
|
|
|
@@ -1570,13 +2738,13 @@ function createSlot(worker, scratchSlot) {
|
|
|
1570
2738
|
pending: /* @__PURE__ */ new Map(),
|
|
1571
2739
|
alive: true,
|
|
1572
2740
|
outstanding: 0,
|
|
1573
|
-
|
|
2741
|
+
fields: /* @__PURE__ */ new Map(),
|
|
1574
2742
|
working: /* @__PURE__ */ new Map(),
|
|
1575
2743
|
idleWaiters: /* @__PURE__ */ new Set()
|
|
1576
2744
|
};
|
|
1577
2745
|
}
|
|
1578
|
-
function
|
|
1579
|
-
|
|
2746
|
+
function noteFieldOpened(slot, handle, handles) {
|
|
2747
|
+
slot.fields.set(handle, handles);
|
|
1580
2748
|
}
|
|
1581
2749
|
function noteWorkStarted(slot, requestId, handle) {
|
|
1582
2750
|
slot.working.set(requestId, handle);
|
|
@@ -1586,8 +2754,8 @@ function stillWorkingOn(slot, handle) {
|
|
|
1586
2754
|
return false;
|
|
1587
2755
|
}
|
|
1588
2756
|
function forgetDroppedGraphs(slot, shared) {
|
|
1589
|
-
for (const handle of [...slot.
|
|
1590
|
-
if (!shared.has(handle) && !stillWorkingOn(slot, handle)) slot.
|
|
2757
|
+
for (const handle of [...slot.fields.keys()]) {
|
|
2758
|
+
if (!shared.has(handle) && !stillWorkingOn(slot, handle)) slot.fields.delete(handle);
|
|
1591
2759
|
}
|
|
1592
2760
|
}
|
|
1593
2761
|
function noteAnswered(slot, requestId, shared) {
|
|
@@ -1610,8 +2778,11 @@ function untilIdle(slot, deadlineMs) {
|
|
|
1610
2778
|
}
|
|
1611
2779
|
function releaseSlotLocks(slot) {
|
|
1612
2780
|
noteThreadSlotGone(slot.scratchSlot);
|
|
1613
|
-
for (const
|
|
1614
|
-
|
|
2781
|
+
for (const field of slot.fields.values()) {
|
|
2782
|
+
releasePinsHeldBy(field.store.pins, slot.scratchSlot);
|
|
2783
|
+
if (field.graph !== null) releaseLocksHeldBy(field.graph, slot.scratchSlot);
|
|
2784
|
+
}
|
|
2785
|
+
slot.fields.clear();
|
|
1615
2786
|
slot.working.clear();
|
|
1616
2787
|
for (const waiter of [...slot.idleWaiters]) waiter();
|
|
1617
2788
|
}
|
|
@@ -1623,8 +2794,31 @@ function rejectPending(slot, err) {
|
|
|
1623
2794
|
slot.pending.clear();
|
|
1624
2795
|
}
|
|
1625
2796
|
|
|
2797
|
+
// src/workers/resource-limits.ts
|
|
2798
|
+
function workerResourceLimits() {
|
|
2799
|
+
return { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
|
|
2800
|
+
}
|
|
2801
|
+
|
|
1626
2802
|
// src/vector/search-pool/spawn.ts
|
|
1627
2803
|
async function spawnWorker(entryPoint) {
|
|
2804
|
+
if (entryPoint === null) return null;
|
|
2805
|
+
const runtime = detectRuntime();
|
|
2806
|
+
if (runtime.supportsWorkerThreads) {
|
|
2807
|
+
try {
|
|
2808
|
+
return await spawnNodeWorker(new URL(entryPoint), workerResourceLimits());
|
|
2809
|
+
} catch {
|
|
2810
|
+
return null;
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
if (runtime.supportsWebWorkers) {
|
|
2814
|
+
try {
|
|
2815
|
+
const WorkerCtor = globalThis.Worker;
|
|
2816
|
+
if (typeof WorkerCtor !== "function") return null;
|
|
2817
|
+
return new WorkerCtor(entryPoint, { type: "module" });
|
|
2818
|
+
} catch {
|
|
2819
|
+
return null;
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
1628
2822
|
return null;
|
|
1629
2823
|
}
|
|
1630
2824
|
function listen(worker, handler) {
|
|
@@ -1658,11 +2852,12 @@ function graphSearchOptionsOf(options) {
|
|
|
1658
2852
|
};
|
|
1659
2853
|
}
|
|
1660
2854
|
async function createVectorSearchPool(requestedCount) {
|
|
2855
|
+
const entryPoint = resolveWorkerEntryPoint();
|
|
1661
2856
|
const count = resolveWorkerCount(requestedCount);
|
|
1662
2857
|
const slots = [];
|
|
1663
2858
|
const sharedFields = /* @__PURE__ */ new Map();
|
|
1664
2859
|
for (let i = 0; i < count; i++) {
|
|
1665
|
-
const worker = await spawnWorker();
|
|
2860
|
+
const worker = await spawnWorker(entryPoint);
|
|
1666
2861
|
if (worker === null) break;
|
|
1667
2862
|
const slot = createSlot(worker, searchPoolScratchSlot(i));
|
|
1668
2863
|
listen(worker, (raw) => {
|
|
@@ -1752,7 +2947,7 @@ async function createVectorSearchPool(requestedCount) {
|
|
|
1752
2947
|
async loadShared(handle, handles) {
|
|
1753
2948
|
sharedFields.set(handle, handles);
|
|
1754
2949
|
const loaded = await broadcast((slot, requestId) => {
|
|
1755
|
-
|
|
2950
|
+
noteFieldOpened(slot, handle, handles);
|
|
1756
2951
|
return { type: "loadShared", requestId, handle, scratchSlot: slot.scratchSlot, handles };
|
|
1757
2952
|
}, VECTOR_SEARCH_LOAD_TIMEOUT_MS);
|
|
1758
2953
|
if (loaded) return true;
|
|
@@ -2460,6 +3655,8 @@ function noteWaitingForMemory(owner, work) {
|
|
|
2460
3655
|
if (work === null) waiting.delete(owner);
|
|
2461
3656
|
else waiting.set(owner, work);
|
|
2462
3657
|
}
|
|
3658
|
+
function chargeHeapBytes(bytes) {
|
|
3659
|
+
}
|
|
2463
3660
|
|
|
2464
3661
|
// src/vector/vector-index/memory-charge.ts
|
|
2465
3662
|
function admitShuffledRebuild(state) {
|
|
@@ -2713,19 +3910,22 @@ function scheduleBuild(state) {
|
|
|
2713
3910
|
if (!buildDue(state)) return;
|
|
2714
3911
|
state.buildScheduled = true;
|
|
2715
3912
|
const turn = turnsAreBudgeted ? yieldBackgroundTurn() : yieldBuildTurn(state);
|
|
2716
|
-
|
|
3913
|
+
const scheduled = turn.then(() => {
|
|
2717
3914
|
state.buildScheduled = false;
|
|
3915
|
+
if (state.scheduledBuild === scheduled) state.scheduledBuild = null;
|
|
2718
3916
|
triggerBuild(state);
|
|
2719
3917
|
});
|
|
3918
|
+
state.scheduledBuild = scheduled;
|
|
2720
3919
|
}
|
|
2721
3920
|
async function awaitBuilds(state) {
|
|
2722
3921
|
for (; ; ) {
|
|
2723
|
-
const running = state.pendingBuild ?? state.shuffledRebuild;
|
|
3922
|
+
const running = state.pendingBuild ?? state.shuffledRebuild ?? state.scheduledBuild;
|
|
2724
3923
|
if (running === null) return;
|
|
2725
3924
|
await running;
|
|
2726
3925
|
}
|
|
2727
3926
|
}
|
|
2728
3927
|
function releaseHeldMemory(state) {
|
|
3928
|
+
state.store.release();
|
|
2729
3929
|
state.hnsw = null;
|
|
2730
3930
|
state.freshGraph = null;
|
|
2731
3931
|
state.osq = null;
|
|
@@ -2734,7 +3934,6 @@ function releaseHeldMemory(state) {
|
|
|
2734
3934
|
state.pendingLocations.clear();
|
|
2735
3935
|
state.savedFiles = [];
|
|
2736
3936
|
state.savedSignature = null;
|
|
2737
|
-
state.store.release();
|
|
2738
3937
|
releaseMemoryCharges(state);
|
|
2739
3938
|
}
|
|
2740
3939
|
function disposeIndex(state) {
|
|
@@ -2745,11 +3944,11 @@ function disposeIndex(state) {
|
|
|
2745
3944
|
state.buildTimer = null;
|
|
2746
3945
|
invalidateWorkerCopies(state);
|
|
2747
3946
|
const release = () => releaseHeldMemory(state);
|
|
2748
|
-
if (state.pendingBuild !== null || state.shuffledRebuild !== null) {
|
|
2749
|
-
|
|
2750
|
-
return;
|
|
3947
|
+
if (state.pendingBuild !== null || state.shuffledRebuild !== null || state.scheduledBuild !== null) {
|
|
3948
|
+
return awaitBuilds(state).then(release, release);
|
|
2751
3949
|
}
|
|
2752
3950
|
release();
|
|
3951
|
+
return Promise.resolve();
|
|
2753
3952
|
}
|
|
2754
3953
|
|
|
2755
3954
|
// src/vector/vector-index/checkpoint-restore.ts
|
|
@@ -3152,12 +4351,16 @@ var FlaggedDocuments = class {
|
|
|
3152
4351
|
}
|
|
3153
4352
|
members = createDocIdMap();
|
|
3154
4353
|
ordinals = createDocIdMap();
|
|
4354
|
+
retired = /* @__PURE__ */ new Set();
|
|
3155
4355
|
handles = null;
|
|
3156
4356
|
flags = new Uint8Array(0);
|
|
3157
4357
|
chunks = null;
|
|
3158
4358
|
get size() {
|
|
3159
4359
|
return this.members.size;
|
|
3160
4360
|
}
|
|
4361
|
+
get retiredCount() {
|
|
4362
|
+
return this.retired.size;
|
|
4363
|
+
}
|
|
3161
4364
|
has(docId) {
|
|
3162
4365
|
return this.members.has(docId);
|
|
3163
4366
|
}
|
|
@@ -3169,29 +4372,56 @@ var FlaggedDocuments = class {
|
|
|
3169
4372
|
const handles = this.openHandles();
|
|
3170
4373
|
if (handles === null) return this;
|
|
3171
4374
|
this.flagCurrentOrdinal(docId);
|
|
3172
|
-
|
|
4375
|
+
this.publishCount(handles);
|
|
3173
4376
|
return this;
|
|
3174
4377
|
}
|
|
3175
4378
|
delete(docId) {
|
|
3176
|
-
|
|
4379
|
+
return this.leave(docId, false);
|
|
4380
|
+
}
|
|
4381
|
+
retire(docId) {
|
|
4382
|
+
return this.leave(docId, this.layout.keepsRetiredFlags);
|
|
4383
|
+
}
|
|
4384
|
+
forgetRetired(mayForget) {
|
|
4385
|
+
if (this.retired.size === 0) return;
|
|
3177
4386
|
const handles = this.openHandles();
|
|
3178
|
-
if (handles === null) return
|
|
3179
|
-
const ordinal
|
|
3180
|
-
|
|
3181
|
-
this.
|
|
4387
|
+
if (handles === null) return;
|
|
4388
|
+
for (const ordinal of this.retired) {
|
|
4389
|
+
if (!mayForget(ordinal)) continue;
|
|
4390
|
+
this.retired.delete(ordinal);
|
|
3182
4391
|
this.unflag(ordinal);
|
|
3183
4392
|
}
|
|
3184
|
-
|
|
3185
|
-
return held;
|
|
4393
|
+
this.publishCount(handles);
|
|
3186
4394
|
}
|
|
3187
4395
|
clear() {
|
|
3188
4396
|
this.members.clear();
|
|
3189
4397
|
const handles = this.openHandles();
|
|
3190
4398
|
if (handles !== null) {
|
|
3191
4399
|
for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
|
|
4400
|
+
for (const ordinal of this.retired) this.unflag(ordinal);
|
|
3192
4401
|
Atomics.store(handles.header, this.layout.countWord, 0);
|
|
3193
4402
|
}
|
|
3194
4403
|
this.ordinals.clear();
|
|
4404
|
+
this.retired.clear();
|
|
4405
|
+
}
|
|
4406
|
+
leave(docId, keepsFlag) {
|
|
4407
|
+
const held = this.members.delete(docId);
|
|
4408
|
+
const store = this.store;
|
|
4409
|
+
if (!held && this.handles !== null && store.dimension !== 0 && store.handles === this.handles) {
|
|
4410
|
+
if (!this.ordinals.has(docId)) return false;
|
|
4411
|
+
}
|
|
4412
|
+
const handles = this.openHandles();
|
|
4413
|
+
if (handles === null) return held;
|
|
4414
|
+
const ordinal = this.ordinals.get(docId);
|
|
4415
|
+
if (ordinal !== void 0) {
|
|
4416
|
+
this.ordinals.delete(docId);
|
|
4417
|
+
if (keepsFlag) this.retired.add(ordinal);
|
|
4418
|
+
else this.unflag(ordinal);
|
|
4419
|
+
}
|
|
4420
|
+
Atomics.store(handles.header, this.layout.countWord, this.members.size + this.retired.size);
|
|
4421
|
+
return held;
|
|
4422
|
+
}
|
|
4423
|
+
publishCount(handles) {
|
|
4424
|
+
Atomics.store(handles.header, this.layout.countWord, this.members.size + this.retired.size);
|
|
3195
4425
|
}
|
|
3196
4426
|
flagCurrentOrdinal(docId) {
|
|
3197
4427
|
const ordinal = this.store.getOrdinal(docId);
|
|
@@ -3200,7 +4430,9 @@ var FlaggedDocuments = class {
|
|
|
3200
4430
|
if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
|
|
3201
4431
|
Atomics.store(this.flags, ordinal, 1);
|
|
3202
4432
|
this.ordinals.set(docId, ordinal);
|
|
3203
|
-
if (previous
|
|
4433
|
+
if (previous === void 0) return;
|
|
4434
|
+
if (this.layout.keepsRetiredFlags) this.retired.add(previous);
|
|
4435
|
+
else this.unflag(previous);
|
|
3204
4436
|
}
|
|
3205
4437
|
unflag(ordinal) {
|
|
3206
4438
|
Atomics.store(this.flags, ordinal, 0);
|
|
@@ -3215,6 +4447,7 @@ var FlaggedDocuments = class {
|
|
|
3215
4447
|
const chunksOf = this.layout.chunksOf;
|
|
3216
4448
|
this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
|
|
3217
4449
|
this.ordinals.clear();
|
|
4450
|
+
this.retired.clear();
|
|
3218
4451
|
for (const docId of this.members.keys()) this.flagCurrentOrdinal(docId);
|
|
3219
4452
|
return handles;
|
|
3220
4453
|
}
|
|
@@ -3223,34 +4456,42 @@ function createUnplacedVectors(store) {
|
|
|
3223
4456
|
return new FlaggedDocuments(store, {
|
|
3224
4457
|
flagsOf: (handles) => handles.unplaced,
|
|
3225
4458
|
chunksOf: (handles) => handles.unplacedChunks,
|
|
3226
|
-
countWord: STORE_UNPLACED_COUNT
|
|
4459
|
+
countWord: STORE_UNPLACED_COUNT,
|
|
4460
|
+
keepsRetiredFlags: true
|
|
3227
4461
|
});
|
|
3228
4462
|
}
|
|
3229
4463
|
function createRemovedDocuments(store) {
|
|
3230
4464
|
return new FlaggedDocuments(store, {
|
|
3231
4465
|
flagsOf: (handles) => handles.removed,
|
|
3232
4466
|
chunksOf: null,
|
|
3233
|
-
countWord: STORE_REMOVED_COUNT
|
|
4467
|
+
countWord: STORE_REMOVED_COUNT,
|
|
4468
|
+
keepsRetiredFlags: false
|
|
3234
4469
|
});
|
|
3235
4470
|
}
|
|
3236
4471
|
|
|
3237
4472
|
// src/vector/vector-index/maintenance.ts
|
|
3238
4473
|
function compact(state) {
|
|
4474
|
+
state.store.freeRetiredVectors();
|
|
4475
|
+
const oldestPin = state.store.oldestPin();
|
|
4476
|
+
state.buffer.forgetRetired((ordinal) => state.store.mayFreeOrdinal(ordinal, oldestPin));
|
|
3239
4477
|
if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
|
|
3240
4478
|
if (state.hnsw) {
|
|
3241
|
-
|
|
3242
|
-
state.hnsw.compactTombstones();
|
|
4479
|
+
const before = state.hnsw.tombstoneCount;
|
|
4480
|
+
state.hnsw.compactTombstones(oldestPin);
|
|
4481
|
+
state.compactedNodeCount += before - state.hnsw.tombstoneCount;
|
|
3243
4482
|
}
|
|
3244
|
-
state.freshGraph?.compactTombstones();
|
|
4483
|
+
state.freshGraph?.compactTombstones(oldestPin);
|
|
3245
4484
|
for (const docId of state.tombstones) {
|
|
3246
|
-
state.store.
|
|
4485
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
4486
|
+
if (ordinal !== void 0 && !state.store.mayFreeOrdinal(ordinal, oldestPin)) continue;
|
|
4487
|
+
state.store.remove(docId, oldestPin);
|
|
3247
4488
|
state.buffer.delete(docId);
|
|
4489
|
+
state.tombstones.delete(docId);
|
|
3248
4490
|
if (state.osq) {
|
|
3249
4491
|
state.osq.remove(docId);
|
|
3250
4492
|
}
|
|
3251
4493
|
}
|
|
3252
|
-
releaseReplacedVectors(state);
|
|
3253
|
-
state.tombstones.clear();
|
|
4494
|
+
releaseReplacedVectors(state, oldestPin);
|
|
3254
4495
|
}
|
|
3255
4496
|
async function insertMissing(state, graph) {
|
|
3256
4497
|
const missingOrReplaced = (docId) => !graph.has(docId) || state.buffer.has(docId);
|
|
@@ -3331,6 +4572,28 @@ function estimateMemoryBytes(state) {
|
|
|
3331
4572
|
return state.store.memoryBytes() + state.hnsw.graphBytes + state.hnsw.searchScratchBytes + freshGraphBytes;
|
|
3332
4573
|
}
|
|
3333
4574
|
|
|
4575
|
+
// src/vector/vector-index/publication.ts
|
|
4576
|
+
var nextStore = 0;
|
|
4577
|
+
function vectorPublicationSource(state) {
|
|
4578
|
+
const handle = `publication-${++nextStore}`;
|
|
4579
|
+
return {
|
|
4580
|
+
prepareCopyPublication: () => untilColdLoaded(state),
|
|
4581
|
+
holdCopyPublication() {
|
|
4582
|
+
const pin = holdAdmission(state.store.view);
|
|
4583
|
+
try {
|
|
4584
|
+
const handles = fieldHandlesOf(state, state.hnsw, true);
|
|
4585
|
+
return {
|
|
4586
|
+
field: { handle, sequence: pin.admission.pin, handles },
|
|
4587
|
+
release: pin.release
|
|
4588
|
+
};
|
|
4589
|
+
} catch (error) {
|
|
4590
|
+
pin.release();
|
|
4591
|
+
throw error;
|
|
4592
|
+
}
|
|
4593
|
+
}
|
|
4594
|
+
};
|
|
4595
|
+
}
|
|
4596
|
+
|
|
3334
4597
|
// src/vector/vector-index/search.ts
|
|
3335
4598
|
function* bufferCandidates(state, buffered, filter) {
|
|
3336
4599
|
for (const docId of buffered) {
|
|
@@ -3400,7 +4663,7 @@ function admitScore(tally, docId, score) {
|
|
|
3400
4663
|
}
|
|
3401
4664
|
function scoreCandidates2(state, query, metric, minSimilarity, docIds, ordinals, tally) {
|
|
3402
4665
|
const count = ordinals.length;
|
|
3403
|
-
const distances = nativeScoresOf(state.store.handles, query, metric, ordinals);
|
|
4666
|
+
const distances = nativeScoresOf(state.store.handles, query, metric, ordinals, state.store.writeSequence());
|
|
3404
4667
|
if (distances !== null) {
|
|
3405
4668
|
for (let i = 0; i < count; i++) {
|
|
3406
4669
|
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
@@ -3495,7 +4758,7 @@ function* searchSteps(state, query, k, options, filter) {
|
|
|
3495
4758
|
function graphResultsOf(state, query, k, options, filter) {
|
|
3496
4759
|
if (state.hnsw === null) return [];
|
|
3497
4760
|
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
|
|
3498
|
-
return state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow })
|
|
4761
|
+
return state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow });
|
|
3499
4762
|
}
|
|
3500
4763
|
function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
|
|
3501
4764
|
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, buffered, filter);
|
|
@@ -3603,8 +4866,10 @@ async function scoredChunk(search2, metric, ordinals, count) {
|
|
|
3603
4866
|
const store = nativeStoreFor(search2.state.store.handles);
|
|
3604
4867
|
if (store === null) return null;
|
|
3605
4868
|
try {
|
|
4869
|
+
const code = nativeMetricCode(metric);
|
|
4870
|
+
const pin = search2.admission.pin;
|
|
3606
4871
|
const scored = await withNativeThreadSlot(
|
|
3607
|
-
(slot) => search2.core.scoreAsync(store.handle, search2.query,
|
|
4872
|
+
(slot) => search2.core.scoreAsync(store.handle, search2.query, code, ordinals, count, pin, slot)
|
|
3608
4873
|
);
|
|
3609
4874
|
return scored.status === NATIVE_OK ? scored.distances : null;
|
|
3610
4875
|
} catch {
|
|
@@ -3669,17 +4934,17 @@ async function outcomeOffThread(search2) {
|
|
|
3669
4934
|
filter,
|
|
3670
4935
|
efSearch,
|
|
3671
4936
|
oversample,
|
|
3672
|
-
rescoreWindow
|
|
4937
|
+
rescoreWindow,
|
|
4938
|
+
admission: search2.admission
|
|
3673
4939
|
});
|
|
3674
4940
|
if (hits === null) return null;
|
|
3675
|
-
|
|
3676
|
-
if (buffered === null) return fetchedOutcome(state, graphResults, minSimilarity, currentFilter(search2));
|
|
4941
|
+
if (buffered === null) return fetchedOutcome(state, hits, minSimilarity, currentFilter(search2));
|
|
3677
4942
|
const scan = await scanOffThread(search2, bufferCandidates(state, buffered, filter));
|
|
3678
4943
|
if (scan === null) return null;
|
|
3679
|
-
return fetchedOutcome(state, mergeResults(
|
|
4944
|
+
return fetchedOutcome(state, mergeResults(hits, scan.results, k), minSimilarity, currentFilter(search2));
|
|
3680
4945
|
}
|
|
3681
4946
|
async function searchOnNativeThreads(search2) {
|
|
3682
|
-
const { state, query, k, options, filter } = search2;
|
|
4947
|
+
const { state, core, query, k, options, filter, confinedAtRevision } = search2;
|
|
3683
4948
|
if (query.length !== state.dimension) {
|
|
3684
4949
|
throw new NarsilError(
|
|
3685
4950
|
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
@@ -3693,9 +4958,25 @@ async function searchOnNativeThreads(search2) {
|
|
|
3693
4958
|
if (state.buffer.size > 0 && !state.building && !state.buildScheduled) scheduleBuild(state);
|
|
3694
4959
|
const generation = layoutGeneration(state);
|
|
3695
4960
|
const settled = await watchingSearch(state, async (watch) => {
|
|
3696
|
-
const
|
|
3697
|
-
|
|
3698
|
-
|
|
4961
|
+
const view = state.store.view;
|
|
4962
|
+
const pins = storePinsOf(view);
|
|
4963
|
+
const pin = pins.take();
|
|
4964
|
+
try {
|
|
4965
|
+
const outcome = await outcomeOffThread({
|
|
4966
|
+
state,
|
|
4967
|
+
core,
|
|
4968
|
+
query,
|
|
4969
|
+
k,
|
|
4970
|
+
options,
|
|
4971
|
+
filter,
|
|
4972
|
+
confinedAtRevision,
|
|
4973
|
+
admission: admissionAt(view, pin)
|
|
4974
|
+
});
|
|
4975
|
+
if (outcome === null || watch.written.size > 0 || layoutGeneration(state) !== generation) return null;
|
|
4976
|
+
return outcome;
|
|
4977
|
+
} finally {
|
|
4978
|
+
pins.release(pin);
|
|
4979
|
+
}
|
|
3699
4980
|
});
|
|
3700
4981
|
return settled ?? searchFiltered(state, query, k, options, filterForOptions(state, options));
|
|
3701
4982
|
}
|
|
@@ -3730,6 +5011,10 @@ function createVectorWrites(state) {
|
|
|
3730
5011
|
for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
|
|
3731
5012
|
return true;
|
|
3732
5013
|
}
|
|
5014
|
+
function forgetRetiredFlags() {
|
|
5015
|
+
const oldestPin = state.store.oldestPin();
|
|
5016
|
+
state.buffer.forgetRetired((ordinal) => state.store.mayFreeOrdinal(ordinal, oldestPin));
|
|
5017
|
+
}
|
|
3733
5018
|
function applyInsert(docId, vector, partitionId) {
|
|
3734
5019
|
handOverBuildTurns(state);
|
|
3735
5020
|
const previous = state.store.getOrdinal(docId);
|
|
@@ -3756,24 +5041,28 @@ function createVectorWrites(state) {
|
|
|
3756
5041
|
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3757
5042
|
}
|
|
3758
5043
|
state.buffer.add(docId);
|
|
5044
|
+
if (state.buffer.retiredCount !== 0) forgetRetiredFlags();
|
|
3759
5045
|
state.pendingLocations.delete(docId);
|
|
3760
5046
|
if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
|
|
3761
5047
|
}
|
|
3762
5048
|
function applyRemove(docId) {
|
|
3763
5049
|
handOverBuildTurns(state);
|
|
3764
|
-
|
|
5050
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
5051
|
+
if (ordinal === void 0) return;
|
|
3765
5052
|
noteWrite(state);
|
|
3766
5053
|
noteSearchedDocWritten(state, docId);
|
|
3767
5054
|
state.tombstones.add(docId);
|
|
3768
|
-
state.
|
|
5055
|
+
state.store.retire(ordinal);
|
|
5056
|
+
state.buffer.retire(docId);
|
|
5057
|
+
if (state.buffer.retiredCount !== 0) forgetRetiredFlags();
|
|
3769
5058
|
state.pendingLocations.delete(docId);
|
|
3770
5059
|
const graph = state.hnsw;
|
|
3771
|
-
|
|
3772
|
-
if (graph !== null && ordinal !== void 0) afterGraphPass(state, () => graph.markTombstoneOrdinal(ordinal));
|
|
5060
|
+
if (graph !== null) afterGraphPass(state, () => graph.markTombstoneOrdinal(ordinal));
|
|
3773
5061
|
}
|
|
3774
5062
|
return {
|
|
3775
5063
|
insert(docId, vector, partitionId) {
|
|
3776
5064
|
validateDimension(vector);
|
|
5065
|
+
for (const listener of state.writeListeners) listener(docId);
|
|
3777
5066
|
const hold = state.cold;
|
|
3778
5067
|
if (hold === null) {
|
|
3779
5068
|
applyInsert(docId, vector, partitionId);
|
|
@@ -3785,6 +5074,7 @@ function createVectorWrites(state) {
|
|
|
3785
5074
|
hold.writes.push(() => applyInsert(docId, held, partitionId));
|
|
3786
5075
|
},
|
|
3787
5076
|
remove(docId) {
|
|
5077
|
+
for (const listener of state.writeListeners) listener(docId);
|
|
3788
5078
|
const hold = state.cold;
|
|
3789
5079
|
if (hold === null) {
|
|
3790
5080
|
applyRemove(docId);
|
|
@@ -3806,6 +5096,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3806
5096
|
{ dimension }
|
|
3807
5097
|
);
|
|
3808
5098
|
}
|
|
5099
|
+
shareTurnsWithBackgroundWork();
|
|
3809
5100
|
const promotionThreshold = config?.threshold ?? DEFAULT_PROMOTION_THRESHOLD;
|
|
3810
5101
|
const filterThreshold = Math.max(0, Math.min(1, config?.filterThreshold ?? DEFAULT_FILTER_THRESHOLD));
|
|
3811
5102
|
const quantizationMode = config?.quantization ?? defaultQuantizationFor(dimension);
|
|
@@ -3848,6 +5139,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3848
5139
|
compactedNodeCount: 0,
|
|
3849
5140
|
building: false,
|
|
3850
5141
|
buildScheduled: false,
|
|
5142
|
+
scheduledBuild: null,
|
|
3851
5143
|
buildTimer: null,
|
|
3852
5144
|
buildRetryDelayMs: 0,
|
|
3853
5145
|
buildTurnWaiters: /* @__PURE__ */ new Map(),
|
|
@@ -3866,7 +5158,8 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3866
5158
|
releaseToFilesInFlight: Promise.resolve(),
|
|
3867
5159
|
cold: null,
|
|
3868
5160
|
graphsHeldOffThread: null,
|
|
3869
|
-
searchWatches: /* @__PURE__ */ new Set()
|
|
5161
|
+
searchWatches: /* @__PURE__ */ new Set(),
|
|
5162
|
+
writeListeners: /* @__PURE__ */ new Set()
|
|
3870
5163
|
};
|
|
3871
5164
|
const { insert, remove } = createVectorWrites(state);
|
|
3872
5165
|
function getVector(docId) {
|
|
@@ -3916,6 +5209,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3916
5209
|
return searchFiltered(state, query, k, options, filter);
|
|
3917
5210
|
}
|
|
3918
5211
|
return {
|
|
5212
|
+
...vectorPublicationSource(state),
|
|
3919
5213
|
get size() {
|
|
3920
5214
|
return liveSize(state);
|
|
3921
5215
|
},
|
|
@@ -3953,13 +5247,14 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3953
5247
|
},
|
|
3954
5248
|
dispose: () => {
|
|
3955
5249
|
cancelColdHold(state);
|
|
3956
|
-
disposeIndex(state);
|
|
5250
|
+
return disposeIndex(state).then(closeIdleTurnHop);
|
|
3957
5251
|
},
|
|
3958
5252
|
search: withGraphFree(
|
|
3959
5253
|
state,
|
|
3960
5254
|
(query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options)
|
|
3961
5255
|
),
|
|
3962
5256
|
searchParallel,
|
|
5257
|
+
...writeWatchesOf(state),
|
|
3963
5258
|
refreshWorkerCopies: () => {
|
|
3964
5259
|
if (state.cold === null) refreshWorkerCopies(state);
|
|
3965
5260
|
},
|
|
@@ -4023,4 +5318,4 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
4023
5318
|
};
|
|
4024
5319
|
}
|
|
4025
5320
|
|
|
4026
|
-
export { compactVectors, createSharedVectorSearcher, createVectorIndex, getVectorMaintenanceStatus, openSharedVectorField, optimizeVectors };
|
|
5321
|
+
export { bindVectorHost, compactVectors, createSharedVectorSearcher, createVectorIndex, executeHybridSearch, executeVectorSearch, filterSubsetsMatchNothing, getVectorMaintenanceStatus, holdAdmission, linearCombination, openSharedVectorField, optimizeVectors, reciprocalRankFusion, releaseLocksHeldBy, releasePinsHeldBy, resolveHybridFusion, vectorLegWith };
|