@delali/narsil 0.3.1-next.145 → 0.3.1-next.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
- package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
- package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
- package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
- package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
- package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
- package/dist/chunk-3MXV3VHR.mjs +46 -0
- package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
- package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
- package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
- package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
- package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
- package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
- package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/chunk-BTPGQC6Z.mjs +27 -0
- package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
- package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
- package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
- package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
- package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
- package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
- package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
- package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
- package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
- package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
- package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
- package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
- package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +2 -0
- package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
- package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
- package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +150 -255
- package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +539 -5
- package/dist/index.mjs +22 -16
- package/dist/index.react-native.mjs +19 -0
- package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
- package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
- package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
- package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
- package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/react-native/partial-words.mjs +733 -0
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +32 -28
- package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.mjs +7 -5
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
- package/package.json +88 -7
- package/dist/analysis-change-D5APH4Z3.mjs +0 -34
- package/dist/browser/import-FTMLTPGQ.mjs +0 -9
- package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
- package/dist/import-6KXFY6XI.mjs +0 -35
- package/dist/thread-OV6T555H.mjs +0 -28
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -1,13 +1,65 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles,
|
|
4
|
-
export { releaseLocksHeldBy } from './chunk-
|
|
5
|
-
import {
|
|
6
|
-
import { createVectorFileReader } from './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES,
|
|
9
|
-
import './chunk-
|
|
10
|
-
import
|
|
1
|
+
import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, yieldBuildTurn, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, emptyFieldBeforeRestore, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, recalibrateFromStore, signatureOf, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, allLiveDocIds, fieldHandlesOf, releaseReplacedVectors, adoptGraph, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, DRIFT_CHECK_GROWTH_FACTOR, adoptLoadedCalibration, planCheckpoint, deserialize, RECALIBRATION_DRIFT_RATIO, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-GV5KFQ6M.mjs';
|
|
2
|
+
import './chunk-PSWKQVVT.mjs';
|
|
3
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, threadedNativeCore, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, grownByteLength, createVectorBlock, ensureBlockSlots, sharedMemoryAvailable, createHNSWIndex, topLayerInParts, shuffleInPlace, requireWholeNeighbourLists, createCentroidSums, shuffleStep, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, nativeStoreFor, withNativeThreadSlot, nativeMetricCode, NATIVE_OK, releaseLocksHeldBy } from './chunk-HURV3ASY.mjs';
|
|
4
|
+
export { releaseLocksHeldBy } from './chunk-HURV3ASY.mjs';
|
|
5
|
+
import { NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, isDead } from './chunk-LQ6Q4L5B.mjs';
|
|
6
|
+
import { createVectorFileReader } from './chunk-UFNCLEQM.mjs';
|
|
7
|
+
import { createDocIdMap, turnsAreBudgeted, yieldBackgroundTurn, chargeHeapBytes, holdWritesAndBackgroundWork, backgroundTurnIsSpent, turnIsSpent, yieldTurn, noteThreadSlotLive, noteThreadSlotGone } from './chunk-LQAKO5QV.mjs';
|
|
8
|
+
import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, resolveWorkerCount, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-L4M7QJNP.mjs';
|
|
9
|
+
import './chunk-AM7RSBPF.mjs';
|
|
10
|
+
import './chunk-TJMSKBEM.mjs';
|
|
11
|
+
import './chunk-KWRW5U65.mjs';
|
|
12
|
+
import { compareCodePoints } from './chunk-PPEPTUVV.mjs';
|
|
13
|
+
import './chunk-AAAETEJK.mjs';
|
|
14
|
+
import './chunk-NFGBOI46.mjs';
|
|
15
|
+
import { NarsilError, ErrorCodes, isNarsilError } from './chunk-YH7W556M.mjs';
|
|
16
|
+
|
|
17
|
+
// src/engine/vector-maintenance.ts
|
|
18
|
+
function resolveVectorIndexes(manager, indexName, fieldName) {
|
|
19
|
+
const all = manager.getVectorIndexes();
|
|
20
|
+
if (fieldName === void 0) return all;
|
|
21
|
+
const idx = all.get(fieldName);
|
|
22
|
+
if (!idx) {
|
|
23
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Vector field "${fieldName}" not found in index "${indexName}"`, {
|
|
24
|
+
indexName,
|
|
25
|
+
fieldName
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
const single = /* @__PURE__ */ new Map();
|
|
29
|
+
single.set(fieldName, idx);
|
|
30
|
+
return single;
|
|
31
|
+
}
|
|
32
|
+
function compactVectors(manager, indexName, fieldName) {
|
|
33
|
+
const indexes = resolveVectorIndexes(manager, indexName, fieldName);
|
|
34
|
+
for (const [, idx] of indexes) {
|
|
35
|
+
idx.compact();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async function optimizeVectors(manager, indexName, fieldName) {
|
|
39
|
+
const indexes = resolveVectorIndexes(manager, indexName, fieldName);
|
|
40
|
+
for (const [, idx] of indexes) {
|
|
41
|
+
await idx.optimize();
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function getVectorMaintenanceStatus(manager) {
|
|
45
|
+
const indexes = manager.getVectorIndexes();
|
|
46
|
+
const results = [];
|
|
47
|
+
for (const [field, idx] of indexes) {
|
|
48
|
+
const status = idx.maintenanceStatus();
|
|
49
|
+
results.push({
|
|
50
|
+
fieldName: field,
|
|
51
|
+
tombstoneRatio: status.tombstoneRatio,
|
|
52
|
+
graphCount: status.graphCount,
|
|
53
|
+
bufferSize: status.bufferSize,
|
|
54
|
+
building: status.building,
|
|
55
|
+
waitingForMemory: status.waitingForMemory,
|
|
56
|
+
loadingFromFiles: status.loadingFromFiles === true,
|
|
57
|
+
estimatedCompactMs: status.estimatedCompactMs,
|
|
58
|
+
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return results;
|
|
62
|
+
}
|
|
11
63
|
|
|
12
64
|
// src/core/heap.ts
|
|
13
65
|
function createMaxHeap(compare) {
|
|
@@ -218,10 +270,10 @@ function createSharedVectorSearcher(options) {
|
|
|
218
270
|
}
|
|
219
271
|
function ordinalSpace() {
|
|
220
272
|
const store = view.store;
|
|
221
|
-
const
|
|
273
|
+
const layoutGeneration2 = ordinalLayoutGeneration();
|
|
222
274
|
return {
|
|
223
275
|
fieldName,
|
|
224
|
-
layoutGeneration,
|
|
276
|
+
layoutGeneration: layoutGeneration2,
|
|
225
277
|
slots: store.slots,
|
|
226
278
|
header: store.handles.header,
|
|
227
279
|
ordinalOf: (docId) => view.ordinalOf(docId),
|
|
@@ -332,7 +384,7 @@ function createSharedVectorSearcher(options) {
|
|
|
332
384
|
);
|
|
333
385
|
}
|
|
334
386
|
for (let attempt = 1; ; attempt++) {
|
|
335
|
-
const
|
|
387
|
+
const layoutGeneration2 = ordinalLayoutGeneration();
|
|
336
388
|
const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
|
|
337
389
|
if (view.graphRetired) {
|
|
338
390
|
throw new NarsilError(
|
|
@@ -341,7 +393,7 @@ function createSharedVectorSearcher(options) {
|
|
|
341
393
|
{ fieldName }
|
|
342
394
|
);
|
|
343
395
|
}
|
|
344
|
-
if (ordinalLayoutGeneration() ===
|
|
396
|
+
if (ordinalLayoutGeneration() === layoutGeneration2) return Promise.resolve(answerOf(outcome));
|
|
345
397
|
if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
|
|
346
398
|
throw new NarsilError(
|
|
347
399
|
ErrorCodes.WORKER_BUSY,
|
|
@@ -719,11 +771,11 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
719
771
|
return store.docIdAt(ordinal);
|
|
720
772
|
}
|
|
721
773
|
function ordinalOf(docId) {
|
|
722
|
-
const
|
|
723
|
-
if (
|
|
774
|
+
const layoutGeneration2 = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
|
|
775
|
+
if (layoutGeneration2 !== scannedLayoutGeneration) {
|
|
724
776
|
ordinals = null;
|
|
725
777
|
scanned = 0;
|
|
726
|
-
scannedLayoutGeneration =
|
|
778
|
+
scannedLayoutGeneration = layoutGeneration2;
|
|
727
779
|
}
|
|
728
780
|
const slots = store.slots;
|
|
729
781
|
if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
|
|
@@ -799,7 +851,7 @@ function appendDocId(handles, ordinal, docId) {
|
|
|
799
851
|
Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
|
|
800
852
|
}
|
|
801
853
|
function resetDocIds(handles) {
|
|
802
|
-
new Uint32Array(handles.docIdOffsets).fill(0);
|
|
854
|
+
new Uint32Array(handles.docIdOffsets, 0, handles.docIdOffsets.byteLength / 4).fill(0);
|
|
803
855
|
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
804
856
|
}
|
|
805
857
|
|
|
@@ -957,11 +1009,12 @@ function ensureFloatSlot(store, ordinal) {
|
|
|
957
1009
|
if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
|
|
958
1010
|
}
|
|
959
1011
|
function ensureOrdinal(store, ordinal, cold) {
|
|
960
|
-
|
|
1012
|
+
const maxOrdinals = VECTOR_MEMORY_LIMITS.maxOrdinals;
|
|
1013
|
+
if (ordinal >= maxOrdinals) {
|
|
961
1014
|
throw new NarsilError(
|
|
962
1015
|
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
963
|
-
`A vector field holds at most ${
|
|
964
|
-
{ ordinal, maxOrdinals
|
|
1016
|
+
`A vector field holds at most ${maxOrdinals} vectors, deleted ones included, until it is rebuilt`,
|
|
1017
|
+
{ ordinal, maxOrdinals }
|
|
965
1018
|
);
|
|
966
1019
|
}
|
|
967
1020
|
const { handles } = store;
|
|
@@ -975,7 +1028,7 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
975
1028
|
const diskFileBytes = handles.diskFile.byteLength;
|
|
976
1029
|
growBufferTo(handles.diskFile, (ordinal + 1) * 4);
|
|
977
1030
|
if (handles.diskFile.byteLength > diskFileBytes) {
|
|
978
|
-
new Int32Array(handles.diskFile).fill(IN_MEMORY, diskFileBytes / 4);
|
|
1031
|
+
new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY, diskFileBytes / 4);
|
|
979
1032
|
}
|
|
980
1033
|
growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
|
|
981
1034
|
if (!cold) ensureFloatSlot(store, ordinal);
|
|
@@ -987,7 +1040,7 @@ function bufferGrowthBytes(buffer, bytes) {
|
|
|
987
1040
|
function growthBytesFor(store, vectors) {
|
|
988
1041
|
const { handles } = store;
|
|
989
1042
|
const slots = Atomics.load(handles.header, STORE_SLOTS);
|
|
990
|
-
const last = Math.min(slots + vectors,
|
|
1043
|
+
const last = Math.min(slots + vectors, VECTOR_MEMORY_LIMITS.maxOrdinals) - 1;
|
|
991
1044
|
if (last < slots) return 0;
|
|
992
1045
|
const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
993
1046
|
let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
|
|
@@ -1115,8 +1168,8 @@ var UNKNOWN_PARTITION = -1;
|
|
|
1115
1168
|
var NO_PARTITION = -2;
|
|
1116
1169
|
function createVectorStore(options) {
|
|
1117
1170
|
const codeBits = options?.codeBits ?? null;
|
|
1118
|
-
const blockBytes = options?.blockBytes ??
|
|
1119
|
-
const docToOrd =
|
|
1171
|
+
const blockBytes = Math.min(options?.blockBytes ?? VECTOR_MEMORY_LIMITS.blockBytes, VECTOR_MEMORY_LIMITS.blockBytes);
|
|
1172
|
+
const docToOrd = createDocIdMap();
|
|
1120
1173
|
const ordToDoc = [];
|
|
1121
1174
|
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
1122
1175
|
const pins = createRetiredVectorPins();
|
|
@@ -1338,9 +1391,9 @@ function createVectorStore(options) {
|
|
|
1338
1391
|
clear() {
|
|
1339
1392
|
forgetDocuments();
|
|
1340
1393
|
if (open === null) return;
|
|
1341
|
-
open.present.fill(0);
|
|
1342
|
-
new Uint8Array(open.handles.codePresent).fill(0);
|
|
1343
|
-
new Int32Array(open.handles.diskFile).fill(IN_MEMORY);
|
|
1394
|
+
open.present.fill(0, 0, open.handles.present.byteLength);
|
|
1395
|
+
new Uint8Array(open.handles.codePresent, 0, open.handles.codePresent.byteLength).fill(0);
|
|
1396
|
+
new Int32Array(open.handles.diskFile, 0, open.handles.diskFile.byteLength / 4).fill(IN_MEMORY);
|
|
1344
1397
|
open.handles.centroid.fill(0);
|
|
1345
1398
|
if (open.handles.vectorFiles.length > 0) {
|
|
1346
1399
|
open.handles.vectorFiles.length = 0;
|
|
@@ -1400,6 +1453,108 @@ function createVectorStore(options) {
|
|
|
1400
1453
|
};
|
|
1401
1454
|
}
|
|
1402
1455
|
|
|
1456
|
+
// src/vector/vector-index/graph-pass.ts
|
|
1457
|
+
function holdGraph(state, graph) {
|
|
1458
|
+
const current = state.graphsHeldOffThread;
|
|
1459
|
+
if (current !== null) {
|
|
1460
|
+
current.graphs.push(graph);
|
|
1461
|
+
return current;
|
|
1462
|
+
}
|
|
1463
|
+
let release = () => void 0;
|
|
1464
|
+
const released = new Promise((resolve) => {
|
|
1465
|
+
release = resolve;
|
|
1466
|
+
});
|
|
1467
|
+
const held = { graphs: [graph], released, release, writesAfter: [] };
|
|
1468
|
+
state.graphsHeldOffThread = held;
|
|
1469
|
+
return held;
|
|
1470
|
+
}
|
|
1471
|
+
function applyWritesHeldBack(held) {
|
|
1472
|
+
const writes = held.writesAfter.splice(0);
|
|
1473
|
+
let firstError = null;
|
|
1474
|
+
for (const write of writes) {
|
|
1475
|
+
try {
|
|
1476
|
+
write();
|
|
1477
|
+
} catch (error) {
|
|
1478
|
+
firstError ??= error;
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
if (firstError !== null) throw firstError;
|
|
1482
|
+
}
|
|
1483
|
+
function releaseGraph(state, held, graph) {
|
|
1484
|
+
const at = held.graphs.indexOf(graph);
|
|
1485
|
+
if (at !== -1) held.graphs.splice(at, 1);
|
|
1486
|
+
if (held.graphs.length > 0) return;
|
|
1487
|
+
if (state.graphsHeldOffThread === held) state.graphsHeldOffThread = null;
|
|
1488
|
+
try {
|
|
1489
|
+
applyWritesHeldBack(held);
|
|
1490
|
+
} finally {
|
|
1491
|
+
held.release();
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
async function connectGraph(state, graph) {
|
|
1495
|
+
const offThread = graph.connectOffThread();
|
|
1496
|
+
if (offThread === null) {
|
|
1497
|
+
await graph.connectOnThisThread(turnsAreBudgeted);
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
const held = holdGraph(state, graph);
|
|
1501
|
+
let connected = false;
|
|
1502
|
+
try {
|
|
1503
|
+
connected = await offThread;
|
|
1504
|
+
} finally {
|
|
1505
|
+
releaseGraph(state, held, graph);
|
|
1506
|
+
}
|
|
1507
|
+
if (!connected && !state.disposed) await graph.connectOnThisThread(true);
|
|
1508
|
+
}
|
|
1509
|
+
async function untilGraphReleased(state) {
|
|
1510
|
+
for (; ; ) {
|
|
1511
|
+
const held = state.graphsHeldOffThread;
|
|
1512
|
+
if (held === null) return;
|
|
1513
|
+
await held.released;
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
function waitOutGraphPass(state) {
|
|
1517
|
+
const held = state.graphsHeldOffThread;
|
|
1518
|
+
if (held === null) return;
|
|
1519
|
+
for (const graph of held.graphs) graph.exclusively(() => void 0);
|
|
1520
|
+
applyWritesHeldBack(held);
|
|
1521
|
+
}
|
|
1522
|
+
function withGraphFree(state, operation) {
|
|
1523
|
+
return (...args) => {
|
|
1524
|
+
waitOutGraphPass(state);
|
|
1525
|
+
return operation(...args);
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1528
|
+
function onceGraphFree(state, operation) {
|
|
1529
|
+
return async (...args) => {
|
|
1530
|
+
await untilGraphReleased(state);
|
|
1531
|
+
return operation(...args);
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
function afterGraphPass(state, write) {
|
|
1535
|
+
const held = state.graphsHeldOffThread;
|
|
1536
|
+
if (held === null) write();
|
|
1537
|
+
else held.writesAfter.push(write);
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
// src/vector/vector-index/search-writes.ts
|
|
1541
|
+
function noteSearchedDocWritten(state, docId) {
|
|
1542
|
+
for (const watch of state.searchWatches) watch.written.add(docId);
|
|
1543
|
+
}
|
|
1544
|
+
function moveOutOfBuffer(state, docId) {
|
|
1545
|
+
if (!state.buffer.delete(docId)) return;
|
|
1546
|
+
for (const watch of state.searchWatches) watch.leftBuffer.add(docId);
|
|
1547
|
+
}
|
|
1548
|
+
async function watchingSearch(state, search2) {
|
|
1549
|
+
const watch = { written: /* @__PURE__ */ new Set(), leftBuffer: /* @__PURE__ */ new Set() };
|
|
1550
|
+
state.searchWatches.add(watch);
|
|
1551
|
+
try {
|
|
1552
|
+
return await search2(watch);
|
|
1553
|
+
} finally {
|
|
1554
|
+
state.searchWatches.delete(watch);
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1403
1558
|
// src/vector/search-pool/constants.ts
|
|
1404
1559
|
var VECTOR_POOL_DRAIN_TIMEOUT_MS = 1e4;
|
|
1405
1560
|
|
|
@@ -1465,34 +1620,8 @@ function rejectPending(slot, err) {
|
|
|
1465
1620
|
slot.pending.clear();
|
|
1466
1621
|
}
|
|
1467
1622
|
|
|
1468
|
-
// src/workers/resource-limits.ts
|
|
1469
|
-
function workerResourceLimits() {
|
|
1470
|
-
return { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
1623
|
// src/vector/search-pool/spawn.ts
|
|
1474
|
-
function resolveWorkerEntryPoint() {
|
|
1475
|
-
return resolveWorkerEntry(import.meta.url, /\/src\/vector\/search-pool\/[^/]+$/, "vector/search-worker.mjs");
|
|
1476
|
-
}
|
|
1477
1624
|
async function spawnWorker(entryPoint) {
|
|
1478
|
-
if (entryPoint === null) return null;
|
|
1479
|
-
const runtime = detectRuntime();
|
|
1480
|
-
if (runtime.supportsWorkerThreads) {
|
|
1481
|
-
try {
|
|
1482
|
-
return await spawnNodeWorker(new URL(entryPoint), workerResourceLimits());
|
|
1483
|
-
} catch {
|
|
1484
|
-
return null;
|
|
1485
|
-
}
|
|
1486
|
-
}
|
|
1487
|
-
if (runtime.supportsWebWorkers) {
|
|
1488
|
-
try {
|
|
1489
|
-
const WorkerCtor = globalThis.Worker;
|
|
1490
|
-
if (typeof WorkerCtor !== "function") return null;
|
|
1491
|
-
return new WorkerCtor(entryPoint, { type: "module" });
|
|
1492
|
-
} catch {
|
|
1493
|
-
return null;
|
|
1494
|
-
}
|
|
1495
|
-
}
|
|
1496
1625
|
return null;
|
|
1497
1626
|
}
|
|
1498
1627
|
function listen(worker, handler) {
|
|
@@ -1526,12 +1655,11 @@ function graphSearchOptionsOf(options) {
|
|
|
1526
1655
|
};
|
|
1527
1656
|
}
|
|
1528
1657
|
async function createVectorSearchPool(requestedCount) {
|
|
1529
|
-
const entryPoint = resolveWorkerEntryPoint();
|
|
1530
1658
|
const count = resolveWorkerCount(requestedCount);
|
|
1531
1659
|
const slots = [];
|
|
1532
1660
|
const sharedFields = /* @__PURE__ */ new Map();
|
|
1533
1661
|
for (let i = 0; i < count; i++) {
|
|
1534
|
-
const worker = await spawnWorker(
|
|
1662
|
+
const worker = await spawnWorker();
|
|
1535
1663
|
if (worker === null) break;
|
|
1536
1664
|
const slot = createSlot(worker, searchPoolScratchSlot(i));
|
|
1537
1665
|
listen(worker, (raw) => {
|
|
@@ -2023,7 +2151,9 @@ async function insertIntoGraph(state, graph, docIds, admit) {
|
|
|
2023
2151
|
await flush();
|
|
2024
2152
|
await Promise.all(pending);
|
|
2025
2153
|
if (state.disposed) return false;
|
|
2026
|
-
|
|
2154
|
+
if (turnsAreBudgeted) await connectGraph(state, graph);
|
|
2155
|
+
else graph.connectUnreached();
|
|
2156
|
+
if (state.disposed) return false;
|
|
2027
2157
|
noteWrite(state);
|
|
2028
2158
|
return true;
|
|
2029
2159
|
}
|
|
@@ -2053,7 +2183,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
|
|
|
2053
2183
|
continue;
|
|
2054
2184
|
}
|
|
2055
2185
|
if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
|
|
2056
|
-
if (graph === state.hnsw) state
|
|
2186
|
+
if (graph === state.hnsw) moveOutOfBuffer(state, docId);
|
|
2057
2187
|
}
|
|
2058
2188
|
if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
|
|
2059
2189
|
fallback.shortAnswerWarned = true;
|
|
@@ -2069,22 +2199,81 @@ async function insertLocally(state, graph, docIds, admit) {
|
|
|
2069
2199
|
if (!admit(docId)) continue;
|
|
2070
2200
|
const ordinal = state.store.getOrdinal(docId);
|
|
2071
2201
|
if (ordinal === void 0) continue;
|
|
2072
|
-
|
|
2073
|
-
|
|
2202
|
+
if (turnsAreBudgeted) {
|
|
2203
|
+
graph.insertOrdinal(ordinal, true);
|
|
2204
|
+
if (graph.holdsUnlinkedNode) await connectGraph(state, graph);
|
|
2205
|
+
if (state.disposed) return false;
|
|
2206
|
+
} else {
|
|
2207
|
+
graph.insertOrdinal(ordinal);
|
|
2208
|
+
}
|
|
2209
|
+
if (graph === state.hnsw) moveOutOfBuffer(state, docId);
|
|
2074
2210
|
count += 1;
|
|
2075
|
-
if (
|
|
2211
|
+
if (turnsAreBudgeted) {
|
|
2212
|
+
if (!backgroundTurnIsSpent()) continue;
|
|
2213
|
+
noteWrite(state);
|
|
2214
|
+
await yieldBackgroundTurn();
|
|
2215
|
+
} else if (count % BUILD_CHUNK_SIZE === 0) {
|
|
2076
2216
|
noteWrite(state);
|
|
2077
2217
|
await yieldBuildTurn(state);
|
|
2078
2218
|
}
|
|
2079
2219
|
}
|
|
2080
2220
|
if (state.disposed) return false;
|
|
2081
2221
|
if (count > 0) {
|
|
2082
|
-
|
|
2222
|
+
if (turnsAreBudgeted) await connectGraph(state, graph);
|
|
2223
|
+
else graph.connectUnreached();
|
|
2083
2224
|
noteWrite(state);
|
|
2084
2225
|
}
|
|
2085
2226
|
return true;
|
|
2086
2227
|
}
|
|
2087
2228
|
|
|
2229
|
+
// src/vector/vector-index/turns.ts
|
|
2230
|
+
function turnDue(steps) {
|
|
2231
|
+
return steps % BUILD_STEPS_BETWEEN_CLOCK_CHECKS === 0 && backgroundTurnIsSpent();
|
|
2232
|
+
}
|
|
2233
|
+
async function forEachInTurns(items, visit, stillWanted) {
|
|
2234
|
+
let steps = 0;
|
|
2235
|
+
for (const item of items) {
|
|
2236
|
+
steps += 1;
|
|
2237
|
+
if (turnDue(steps)) {
|
|
2238
|
+
await yieldBackgroundTurn();
|
|
2239
|
+
if (!stillWanted()) return false;
|
|
2240
|
+
}
|
|
2241
|
+
visit(item);
|
|
2242
|
+
}
|
|
2243
|
+
return true;
|
|
2244
|
+
}
|
|
2245
|
+
async function copyInTurns(items, stillWanted) {
|
|
2246
|
+
const copied = [];
|
|
2247
|
+
const finished = await forEachInTurns(items, (item) => copied.push(item), stillWanted);
|
|
2248
|
+
return finished ? copied : null;
|
|
2249
|
+
}
|
|
2250
|
+
async function shuffledLiveDocIdsInTurns(state) {
|
|
2251
|
+
const stillWanted = () => !state.disposed;
|
|
2252
|
+
const docIds = await copyInTurns(allLiveDocIds(state), stillWanted);
|
|
2253
|
+
if (docIds === null) return null;
|
|
2254
|
+
let stream = docIds.length;
|
|
2255
|
+
for (let i = docIds.length - 1; i > 0; i--) {
|
|
2256
|
+
stream = shuffleStep(docIds, i, stream);
|
|
2257
|
+
if (turnDue(i)) {
|
|
2258
|
+
await yieldBackgroundTurn();
|
|
2259
|
+
if (!stillWanted()) return null;
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
return docIds;
|
|
2263
|
+
}
|
|
2264
|
+
async function liveOrdinalsInTurns(state) {
|
|
2265
|
+
const ordinals = [];
|
|
2266
|
+
for (let ordinal = 0; ordinal < state.store.slots; ordinal++) {
|
|
2267
|
+
if (turnDue(ordinal + 1)) {
|
|
2268
|
+
await yieldBackgroundTurn();
|
|
2269
|
+
if (state.disposed) return null;
|
|
2270
|
+
}
|
|
2271
|
+
const docId = state.store.docIdForOrdinal(ordinal);
|
|
2272
|
+
if (docId !== void 0 && !state.tombstones.has(docId)) ordinals.push(ordinal);
|
|
2273
|
+
}
|
|
2274
|
+
return Int32Array.from(ordinals);
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2088
2277
|
// src/vector/vector-index/drift.ts
|
|
2089
2278
|
function liveVectorMeanOf(sums) {
|
|
2090
2279
|
const centroid = centroidOfCentroidSums(sums);
|
|
@@ -2103,7 +2292,8 @@ function liveVectorMeanOf(sums) {
|
|
|
2103
2292
|
}
|
|
2104
2293
|
async function measureLiveVectors(state, quantizer) {
|
|
2105
2294
|
const sums = createCentroidSums(state.dimension, state.metric);
|
|
2106
|
-
const ordinals = liveOrdinals(state);
|
|
2295
|
+
const ordinals = turnsAreBudgeted ? await liveOrdinalsInTurns(state) : liveOrdinals(state);
|
|
2296
|
+
if (ordinals === null) return null;
|
|
2107
2297
|
for (let start = 0; start < ordinals.length; start += RECALIBRATION_CHUNK_VECTORS) {
|
|
2108
2298
|
if (start > 0) await yieldBuildTurn(state);
|
|
2109
2299
|
if (state.disposed || state.osq !== quantizer) return null;
|
|
@@ -2210,11 +2400,13 @@ async function shareFilesWithEveryThread(state, paths) {
|
|
|
2210
2400
|
}
|
|
2211
2401
|
async function releaseLocations(state, paths, locations) {
|
|
2212
2402
|
const fileIndexes = await shareFilesWithEveryThread(state, paths);
|
|
2403
|
+
await untilGraphReleased(state);
|
|
2213
2404
|
if (state.disposed) return;
|
|
2214
2405
|
let count = 0;
|
|
2215
2406
|
for (const [docId, location] of locations) {
|
|
2216
2407
|
if (count > 0 && count % DISK_ADOPTION_YIELD_INTERVAL === 0) {
|
|
2217
2408
|
await yieldBuildTurn(state);
|
|
2409
|
+
await untilGraphReleased(state);
|
|
2218
2410
|
if (state.disposed) return;
|
|
2219
2411
|
}
|
|
2220
2412
|
count += 1;
|
|
@@ -2265,8 +2457,6 @@ function noteWaitingForMemory(owner, work) {
|
|
|
2265
2457
|
if (work === null) waiting.delete(owner);
|
|
2266
2458
|
else waiting.set(owner, work);
|
|
2267
2459
|
}
|
|
2268
|
-
function chargeHeapBytes(bytes) {
|
|
2269
|
-
}
|
|
2270
2460
|
|
|
2271
2461
|
// src/vector/vector-index/memory-charge.ts
|
|
2272
2462
|
function admitShuffledRebuild(state) {
|
|
@@ -2298,20 +2488,27 @@ function releaseMemoryCharges(state) {
|
|
|
2298
2488
|
}
|
|
2299
2489
|
|
|
2300
2490
|
// src/vector/vector-index/build.ts
|
|
2491
|
+
function bufferIfTheGraphLacks(state, graph, docId) {
|
|
2492
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
2493
|
+
if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) moveOutOfBuffer(state, docId);
|
|
2494
|
+
else state.buffer.add(docId);
|
|
2495
|
+
}
|
|
2301
2496
|
function bufferWhatTheGraphLacks(state, graph) {
|
|
2302
|
-
for (const docId of allLiveDocIds(state))
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2497
|
+
for (const docId of allLiveDocIds(state)) bufferIfTheGraphLacks(state, graph, docId);
|
|
2498
|
+
}
|
|
2499
|
+
async function bufferWhatTheGraphLacksInTurns(state, graph) {
|
|
2500
|
+
const visit = (docId) => bufferIfTheGraphLacks(state, graph, docId);
|
|
2501
|
+
await forEachInTurns(allLiveDocIds(state), visit, () => !state.disposed && state.hnsw === graph);
|
|
2307
2502
|
}
|
|
2308
2503
|
async function buildGraphFromStore(state) {
|
|
2309
2504
|
const previous = state.hnsw;
|
|
2505
|
+
const order = turnsAreBudgeted ? await shuffledLiveDocIdsInTurns(state) : shuffleInPlace([...allLiveDocIds(state)]);
|
|
2506
|
+
if (order === null || state.hnsw !== previous) return false;
|
|
2310
2507
|
const graph = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.osq ?? void 0);
|
|
2311
2508
|
state.freshGraph = graph;
|
|
2312
2509
|
let completed = false;
|
|
2313
2510
|
try {
|
|
2314
|
-
completed = await insertIntoGraph(state, graph,
|
|
2511
|
+
completed = await insertIntoGraph(state, graph, order, () => true);
|
|
2315
2512
|
} finally {
|
|
2316
2513
|
state.freshGraph = null;
|
|
2317
2514
|
}
|
|
@@ -2322,7 +2519,8 @@ async function buildGraphFromStore(state) {
|
|
|
2322
2519
|
adoptGraph(state, graph);
|
|
2323
2520
|
state.shuffledBuildCount = graph.size;
|
|
2324
2521
|
noteWrite(state);
|
|
2325
|
-
|
|
2522
|
+
if (turnsAreBudgeted) await bufferWhatTheGraphLacksInTurns(state, graph);
|
|
2523
|
+
else bufferWhatTheGraphLacks(state, graph);
|
|
2326
2524
|
if (previous !== null) dropSharedGraph(state, previous);
|
|
2327
2525
|
await releasePendingLocations(state);
|
|
2328
2526
|
return true;
|
|
@@ -2426,7 +2624,7 @@ async function growGraph(state, graph, bufferSnapshot) {
|
|
|
2426
2624
|
function triggerBuild(state) {
|
|
2427
2625
|
if (state.building) return;
|
|
2428
2626
|
state.building = true;
|
|
2429
|
-
const bufferSnapshot = new Set(state.buffer);
|
|
2627
|
+
const bufferSnapshot = turnsAreBudgeted ? null : new Set(state.buffer);
|
|
2430
2628
|
const existingGraph = state.hnsw;
|
|
2431
2629
|
let settled = false;
|
|
2432
2630
|
let timedOut = false;
|
|
@@ -2434,7 +2632,9 @@ function triggerBuild(state) {
|
|
|
2434
2632
|
try {
|
|
2435
2633
|
if (state.disposed) return;
|
|
2436
2634
|
if (existingGraph !== null) {
|
|
2437
|
-
await
|
|
2635
|
+
const snapshot = bufferSnapshot ?? await copyInTurns(state.buffer, () => !state.disposed && state.hnsw === existingGraph);
|
|
2636
|
+
if (snapshot === null) return;
|
|
2637
|
+
await growGraph(state, existingGraph, snapshot);
|
|
2438
2638
|
return;
|
|
2439
2639
|
}
|
|
2440
2640
|
await promoteToGraph(state);
|
|
@@ -2509,7 +2709,8 @@ function scheduleBuild(state) {
|
|
|
2509
2709
|
if (state.building || state.buildScheduled || state.disposed) return;
|
|
2510
2710
|
if (!buildDue(state)) return;
|
|
2511
2711
|
state.buildScheduled = true;
|
|
2512
|
-
|
|
2712
|
+
const turn = turnsAreBudgeted ? yieldBackgroundTurn() : yieldBuildTurn(state);
|
|
2713
|
+
void turn.then(() => {
|
|
2513
2714
|
state.buildScheduled = false;
|
|
2514
2715
|
triggerBuild(state);
|
|
2515
2716
|
});
|
|
@@ -2935,26 +3136,35 @@ async function coldSearchConcurrently(state, hold, query, k, options) {
|
|
|
2935
3136
|
}
|
|
2936
3137
|
|
|
2937
3138
|
// src/vector/vector-index/flagged-documents.ts
|
|
2938
|
-
var FlaggedDocuments = class
|
|
3139
|
+
var FlaggedDocuments = class {
|
|
2939
3140
|
constructor(store, layout) {
|
|
2940
|
-
super();
|
|
2941
3141
|
this.store = store;
|
|
2942
3142
|
this.layout = layout;
|
|
2943
3143
|
}
|
|
2944
|
-
|
|
3144
|
+
members = createDocIdMap();
|
|
3145
|
+
ordinals = createDocIdMap();
|
|
2945
3146
|
handles = null;
|
|
2946
3147
|
flags = new Uint8Array(0);
|
|
2947
3148
|
chunks = null;
|
|
3149
|
+
get size() {
|
|
3150
|
+
return this.members.size;
|
|
3151
|
+
}
|
|
3152
|
+
has(docId) {
|
|
3153
|
+
return this.members.has(docId);
|
|
3154
|
+
}
|
|
3155
|
+
[Symbol.iterator]() {
|
|
3156
|
+
return this.members.keys()[Symbol.iterator]();
|
|
3157
|
+
}
|
|
2948
3158
|
add(docId) {
|
|
2949
|
-
|
|
3159
|
+
this.members.set(docId, true);
|
|
2950
3160
|
const handles = this.openHandles();
|
|
2951
3161
|
if (handles === null) return this;
|
|
2952
3162
|
this.flagCurrentOrdinal(docId);
|
|
2953
|
-
Atomics.store(handles.header, this.layout.countWord, this.size);
|
|
3163
|
+
Atomics.store(handles.header, this.layout.countWord, this.members.size);
|
|
2954
3164
|
return this;
|
|
2955
3165
|
}
|
|
2956
3166
|
delete(docId) {
|
|
2957
|
-
const held =
|
|
3167
|
+
const held = this.members.delete(docId);
|
|
2958
3168
|
const handles = this.openHandles();
|
|
2959
3169
|
if (handles === null) return held;
|
|
2960
3170
|
const ordinal = this.ordinals.get(docId);
|
|
@@ -2962,11 +3172,11 @@ var FlaggedDocuments = class extends Set {
|
|
|
2962
3172
|
this.ordinals.delete(docId);
|
|
2963
3173
|
this.unflag(ordinal);
|
|
2964
3174
|
}
|
|
2965
|
-
Atomics.store(handles.header, this.layout.countWord, this.size);
|
|
3175
|
+
Atomics.store(handles.header, this.layout.countWord, this.members.size);
|
|
2966
3176
|
return held;
|
|
2967
3177
|
}
|
|
2968
3178
|
clear() {
|
|
2969
|
-
|
|
3179
|
+
this.members.clear();
|
|
2970
3180
|
const handles = this.openHandles();
|
|
2971
3181
|
if (handles !== null) {
|
|
2972
3182
|
for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
|
|
@@ -2996,7 +3206,7 @@ var FlaggedDocuments = class extends Set {
|
|
|
2996
3206
|
const chunksOf = this.layout.chunksOf;
|
|
2997
3207
|
this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
|
|
2998
3208
|
this.ordinals.clear();
|
|
2999
|
-
for (const docId of this) this.flagCurrentOrdinal(docId);
|
|
3209
|
+
for (const docId of this.members.keys()) this.flagCurrentOrdinal(docId);
|
|
3000
3210
|
return handles;
|
|
3001
3211
|
}
|
|
3002
3212
|
};
|
|
@@ -3141,46 +3351,64 @@ function fetchedOutcome(state, results, minSimilarity, filter) {
|
|
|
3141
3351
|
function scannedOutcome(scan) {
|
|
3142
3352
|
return { results: scan.results, matched: scan.matched, matchedExact: true };
|
|
3143
3353
|
}
|
|
3144
|
-
function
|
|
3354
|
+
function liveOrdinalOf(state, docId, filter) {
|
|
3355
|
+
if (state.tombstones.has(docId)) return void 0;
|
|
3356
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
3357
|
+
if (ordinal === void 0 || filter !== void 0 && !ordinalFilterHas(filter, ordinal)) return void 0;
|
|
3358
|
+
return ordinal;
|
|
3359
|
+
}
|
|
3360
|
+
function liveCandidates(state, candidates, filter) {
|
|
3145
3361
|
const docIds = [];
|
|
3146
3362
|
const ordinals = [];
|
|
3147
3363
|
for (const docId of candidates) {
|
|
3148
|
-
|
|
3149
|
-
const ordinal = state.store.getOrdinal(docId);
|
|
3364
|
+
const ordinal = liveOrdinalOf(state, docId, filter);
|
|
3150
3365
|
if (ordinal === void 0) continue;
|
|
3151
3366
|
docIds.push(docId);
|
|
3152
3367
|
ordinals.push(ordinal);
|
|
3153
3368
|
}
|
|
3154
3369
|
return { docIds, ordinals };
|
|
3155
3370
|
}
|
|
3156
|
-
function
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
const score = toScore(distances[i], metric);
|
|
3164
|
-
if (score < minSimilarity) continue;
|
|
3165
|
-
matched++;
|
|
3166
|
-
heap.push({ docId: scanned.docIds[i], score });
|
|
3167
|
-
}
|
|
3168
|
-
return { results: heap.toSortedArray().reverse(), matched };
|
|
3371
|
+
function createScanTally(capacity) {
|
|
3372
|
+
return {
|
|
3373
|
+
best: createBoundedMaxHeap(highScoreFirst, capacity),
|
|
3374
|
+
capacity,
|
|
3375
|
+
matched: 0,
|
|
3376
|
+
bestDiscardedScore: Number.NEGATIVE_INFINITY
|
|
3377
|
+
};
|
|
3169
3378
|
}
|
|
3170
|
-
function
|
|
3171
|
-
|
|
3172
|
-
const
|
|
3173
|
-
if (
|
|
3379
|
+
function admitScore(tally, docId, score) {
|
|
3380
|
+
tally.matched++;
|
|
3381
|
+
const worst = tally.best.size < tally.capacity ? void 0 : tally.best.peek();
|
|
3382
|
+
if (worst !== void 0 && score < worst.score) {
|
|
3383
|
+
if (score > tally.bestDiscardedScore) tally.bestDiscardedScore = score;
|
|
3384
|
+
return;
|
|
3385
|
+
}
|
|
3386
|
+
const result = { docId, score };
|
|
3387
|
+
tally.best.push(result);
|
|
3388
|
+
if (worst === void 0) return;
|
|
3389
|
+
const discarded = highScoreFirst(result, worst) < 0 ? worst.score : score;
|
|
3390
|
+
if (discarded > tally.bestDiscardedScore) tally.bestDiscardedScore = discarded;
|
|
3391
|
+
}
|
|
3392
|
+
function scoreCandidates2(state, query, metric, minSimilarity, docIds, ordinals, tally) {
|
|
3393
|
+
const count = ordinals.length;
|
|
3394
|
+
const distances = nativeScoresOf(state.store.handles, query, metric, ordinals);
|
|
3395
|
+
if (distances !== null) {
|
|
3396
|
+
for (let i = 0; i < count; i++) {
|
|
3397
|
+
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
3398
|
+
const score = toScore(distances[i], metric);
|
|
3399
|
+
if (score < minSimilarity) continue;
|
|
3400
|
+
admitScore(tally, docIds[i], score);
|
|
3401
|
+
}
|
|
3402
|
+
return;
|
|
3403
|
+
}
|
|
3174
3404
|
noteFallback("score");
|
|
3175
3405
|
const arenaQuery = state.store.prepareQueryArena(query);
|
|
3176
3406
|
const queryMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
for (let i = 0; i < scanned.docIds.length; i++) {
|
|
3180
|
-
const docId = scanned.docIds[i];
|
|
3407
|
+
for (let i = 0; i < count; i++) {
|
|
3408
|
+
const docId = docIds[i];
|
|
3181
3409
|
let score;
|
|
3182
3410
|
if (arenaQuery) {
|
|
3183
|
-
const distance = state.store.distanceFromArena(arenaQuery,
|
|
3411
|
+
const distance = state.store.distanceFromArena(arenaQuery, ordinals[i], metric);
|
|
3184
3412
|
if (distance === Number.POSITIVE_INFINITY) continue;
|
|
3185
3413
|
score = toScore(distance, metric);
|
|
3186
3414
|
} else {
|
|
@@ -3201,16 +3429,29 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
|
|
|
3201
3429
|
}
|
|
3202
3430
|
}
|
|
3203
3431
|
if (score < minSimilarity) continue;
|
|
3204
|
-
|
|
3205
|
-
heap.push({ docId, score });
|
|
3432
|
+
admitScore(tally, docId, score);
|
|
3206
3433
|
}
|
|
3207
|
-
|
|
3434
|
+
}
|
|
3435
|
+
function bruteForceSearch(state, query, k, metric, minSimilarity, candidates, filter) {
|
|
3436
|
+
const scanned = liveCandidates(state, candidates, filter);
|
|
3437
|
+
const tally = createScanTally(k);
|
|
3438
|
+
scoreCandidates2(state, query, metric, minSimilarity, scanned.docIds, scanned.ordinals, tally);
|
|
3439
|
+
return { results: tally.best.toSortedArray().reverse(), matched: tally.matched };
|
|
3208
3440
|
}
|
|
3209
3441
|
function search(state, query, k, options) {
|
|
3210
3442
|
handOverBuildTurns(state);
|
|
3211
3443
|
return searchWithFilter(state, query, k, options, filterForOptions(state, options));
|
|
3212
3444
|
}
|
|
3213
3445
|
function searchWithFilter(state, query, k, options, filter) {
|
|
3446
|
+
const steps = searchSteps(state, query, k, options, filter);
|
|
3447
|
+
let step = steps.next();
|
|
3448
|
+
while (step.done !== true) {
|
|
3449
|
+
const { candidates, filter: remaining } = step.value;
|
|
3450
|
+
step = steps.next(bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates, remaining));
|
|
3451
|
+
}
|
|
3452
|
+
return step.value;
|
|
3453
|
+
}
|
|
3454
|
+
function* searchSteps(state, query, k, options, filter) {
|
|
3214
3455
|
if (query.length !== state.dimension) {
|
|
3215
3456
|
throw new NarsilError(
|
|
3216
3457
|
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
@@ -3219,7 +3460,7 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
3219
3460
|
);
|
|
3220
3461
|
}
|
|
3221
3462
|
const currentLiveSize = liveSize(state);
|
|
3222
|
-
const {
|
|
3463
|
+
const { minSimilarity } = options;
|
|
3223
3464
|
if (currentLiveSize === 0) return { results: [], matched: 0, matchedExact: true };
|
|
3224
3465
|
if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
|
|
3225
3466
|
if (k <= 0) return fetchedOutcome(state, [], minSimilarity, filter);
|
|
@@ -3228,26 +3469,233 @@ function searchWithFilter(state, query, k, options, filter) {
|
|
|
3228
3469
|
}
|
|
3229
3470
|
if (!state.hnsw) {
|
|
3230
3471
|
const candidates = filter ? filteredDocIds(state, filter) : allLiveDocIds(state);
|
|
3231
|
-
return scannedOutcome(
|
|
3472
|
+
return scannedOutcome(yield { candidates, filter: void 0, graphResults: null });
|
|
3232
3473
|
}
|
|
3233
3474
|
if (filter) {
|
|
3234
3475
|
const hnswLiveSize = state.hnsw.size;
|
|
3235
3476
|
const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
|
|
3236
3477
|
if (selectivity < state.filterThreshold) {
|
|
3237
|
-
|
|
3238
|
-
return scannedOutcome(scan);
|
|
3478
|
+
return scannedOutcome(yield { candidates: filteredDocIds(state, filter), filter: void 0, graphResults: null });
|
|
3239
3479
|
}
|
|
3240
3480
|
}
|
|
3241
|
-
const hnswResults = state
|
|
3242
|
-
|
|
3243
|
-
|
|
3481
|
+
const hnswResults = graphResultsOf(state, query, k, options, filter);
|
|
3482
|
+
if (state.buffer.size === 0) return fetchedOutcome(state, hnswResults, minSimilarity, filter);
|
|
3483
|
+
const scan = yield { candidates: state.buffer, filter, graphResults: hnswResults };
|
|
3484
|
+
return fetchedOutcome(state, mergeResults(hnswResults, scan.results, k), minSimilarity, filter);
|
|
3485
|
+
}
|
|
3486
|
+
function graphResultsOf(state, query, k, options, filter) {
|
|
3487
|
+
if (state.hnsw === null) return [];
|
|
3488
|
+
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
|
|
3489
|
+
return state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
|
|
3244
3490
|
}
|
|
3245
3491
|
function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
|
|
3246
|
-
const
|
|
3247
|
-
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
|
|
3492
|
+
const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, buffered, filter);
|
|
3248
3493
|
return mergeResults(graphResults, scan.results, k);
|
|
3249
3494
|
}
|
|
3250
3495
|
|
|
3496
|
+
// src/vector/vector-index/search-in-turns.ts
|
|
3497
|
+
function layoutGeneration(state) {
|
|
3498
|
+
return Atomics.load(state.store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
|
|
3499
|
+
}
|
|
3500
|
+
async function scanInTurns(state, query, capacity, options, request, chunkOfThisSearch) {
|
|
3501
|
+
const tally = createScanTally(capacity);
|
|
3502
|
+
const { docIds, ordinals } = chunkOfThisSearch;
|
|
3503
|
+
let length = 0;
|
|
3504
|
+
const scoreChunk = () => {
|
|
3505
|
+
const chunk = length === SEARCH_SCAN_CHUNK_VECTORS ? ordinals : ordinals.subarray(0, length);
|
|
3506
|
+
scoreCandidates2(state, query, options.metric, options.minSimilarity, docIds, chunk, tally);
|
|
3507
|
+
length = 0;
|
|
3508
|
+
};
|
|
3509
|
+
for (const docId of request.candidates) {
|
|
3510
|
+
const ordinal = liveOrdinalOf(state, docId, request.filter);
|
|
3511
|
+
if (ordinal === void 0) continue;
|
|
3512
|
+
docIds[length] = docId;
|
|
3513
|
+
ordinals[length] = ordinal;
|
|
3514
|
+
length++;
|
|
3515
|
+
if (length < SEARCH_SCAN_CHUNK_VECTORS) continue;
|
|
3516
|
+
scoreChunk();
|
|
3517
|
+
if (turnIsSpent()) await yieldTurn();
|
|
3518
|
+
}
|
|
3519
|
+
if (length > 0) scoreChunk();
|
|
3520
|
+
return {
|
|
3521
|
+
kept: tally.best.toSortedArray().reverse(),
|
|
3522
|
+
matched: tally.matched,
|
|
3523
|
+
bestDiscardedScore: tally.bestDiscardedScore
|
|
3524
|
+
};
|
|
3525
|
+
}
|
|
3526
|
+
async function passInTurns(state, query, k, options, filter, chunk) {
|
|
3527
|
+
const steps = searchSteps(state, query, k, options, filter);
|
|
3528
|
+
let graphResults = null;
|
|
3529
|
+
let scan = null;
|
|
3530
|
+
let step = steps.next();
|
|
3531
|
+
while (step.done !== true) {
|
|
3532
|
+
graphResults = step.value.graphResults;
|
|
3533
|
+
scan = await scanInTurns(state, query, k + SEARCH_REFILL_RESERVE, options, step.value, chunk);
|
|
3534
|
+
step = steps.next({ results: scan.kept.slice(0, k), matched: scan.matched });
|
|
3535
|
+
}
|
|
3536
|
+
return { outcome: step.value, graphResults, scan };
|
|
3537
|
+
}
|
|
3538
|
+
function rescoredSinceTheScan(state, query, k, options, filter, watch) {
|
|
3539
|
+
const touched = new Set(watch.written);
|
|
3540
|
+
for (const docId of watch.leftBuffer) touched.add(docId);
|
|
3541
|
+
return bruteForceSearch(state, query, k, options.metric, options.minSimilarity, touched, filter).results;
|
|
3542
|
+
}
|
|
3543
|
+
function outrunsEveryDiscard(results, k, bestDiscardedScore) {
|
|
3544
|
+
if (bestDiscardedScore === Number.NEGATIVE_INFINITY) return true;
|
|
3545
|
+
const lowest = results[k - 1];
|
|
3546
|
+
return lowest !== void 0 && lowest.score > bestDiscardedScore;
|
|
3547
|
+
}
|
|
3548
|
+
function settledOutcome(state, query, k, options, pass, watch) {
|
|
3549
|
+
const { written, leftBuffer } = watch;
|
|
3550
|
+
const { scan, graphResults } = pass;
|
|
3551
|
+
if (scan === null || written.size === 0 && leftBuffer.size === 0) return pass.outcome;
|
|
3552
|
+
if (graphResults === null) return written.size === 0 ? pass.outcome : null;
|
|
3553
|
+
const filter = filterForOptions(state, options);
|
|
3554
|
+
const graphTouched = graphResults.some((result) => written.has(result.docId));
|
|
3555
|
+
const graphPart = graphTouched ? graphResultsOf(state, query, k, options, filter) : graphResults;
|
|
3556
|
+
const survivors = scan.kept.filter((result) => !written.has(result.docId));
|
|
3557
|
+
const rescored = rescoredSinceTheScan(state, query, k, options, filter, watch);
|
|
3558
|
+
const results = mergeResults(graphPart, mergeResults(survivors, rescored, k), k);
|
|
3559
|
+
const refilled = survivors.length < scan.kept.length;
|
|
3560
|
+
if (refilled && !outrunsEveryDiscard(results, k, scan.bestDiscardedScore)) return null;
|
|
3561
|
+
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
3562
|
+
}
|
|
3563
|
+
function attemptInTurns(state, query, k, options, filter, chunk) {
|
|
3564
|
+
const generation = liveSize(state) === 0 ? null : layoutGeneration(state);
|
|
3565
|
+
return watchingSearch(state, async (watch) => {
|
|
3566
|
+
const pass = await passInTurns(state, query, k, options, filter, chunk);
|
|
3567
|
+
if (state.disposed) return pass.outcome;
|
|
3568
|
+
if (generation !== null && layoutGeneration(state) !== generation) return null;
|
|
3569
|
+
return settledOutcome(state, query, k, options, pass, watch);
|
|
3570
|
+
});
|
|
3571
|
+
}
|
|
3572
|
+
async function searchWithFilterInTurns(state, query, k, options, filter) {
|
|
3573
|
+
let current = filter;
|
|
3574
|
+
const chunk = { docIds: [], ordinals: new Int32Array(SEARCH_SCAN_CHUNK_VECTORS) };
|
|
3575
|
+
for (let attempt = 1; ; attempt++) {
|
|
3576
|
+
const search2 = () => attemptInTurns(state, query, k, options, current, chunk);
|
|
3577
|
+
const outcome = attempt <= SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES ? await search2() : await holdWritesAndBackgroundWork(search2);
|
|
3578
|
+
if (outcome !== null) return outcome;
|
|
3579
|
+
current = filterForOptions(state, options);
|
|
3580
|
+
}
|
|
3581
|
+
}
|
|
3582
|
+
function searchFiltered(state, query, k, options, filter) {
|
|
3583
|
+
if (turnsAreBudgeted) return searchWithFilterInTurns(state, query, k, options, filter);
|
|
3584
|
+
return searchWithFilter(state, query, k, options, filter);
|
|
3585
|
+
}
|
|
3586
|
+
|
|
3587
|
+
// src/vector/vector-index/search-off-thread.ts
|
|
3588
|
+
function currentFilter(search2) {
|
|
3589
|
+
const { state, options, filter, confinedAtRevision } = search2;
|
|
3590
|
+
if (confinedAtRevision === null || state.revision === confinedAtRevision) return filter;
|
|
3591
|
+
return filterForOptions(state, options);
|
|
3592
|
+
}
|
|
3593
|
+
async function scoredChunk(search2, metric, ordinals, count) {
|
|
3594
|
+
const store = nativeStoreFor(search2.state.store.handles);
|
|
3595
|
+
if (store === null) return null;
|
|
3596
|
+
try {
|
|
3597
|
+
const scored = await withNativeThreadSlot(
|
|
3598
|
+
(slot) => search2.core.scoreAsync(store.handle, search2.query, nativeMetricCode(metric), ordinals, count, slot)
|
|
3599
|
+
);
|
|
3600
|
+
return scored.status === NATIVE_OK ? scored.distances : null;
|
|
3601
|
+
} catch {
|
|
3602
|
+
return null;
|
|
3603
|
+
}
|
|
3604
|
+
}
|
|
3605
|
+
async function scanOffThread(search2, candidates) {
|
|
3606
|
+
const { state, k, options } = search2;
|
|
3607
|
+
const { metric, minSimilarity } = options;
|
|
3608
|
+
const heap = createBoundedMaxHeap(highScoreFirst, k);
|
|
3609
|
+
let matched = 0;
|
|
3610
|
+
const docIds = [];
|
|
3611
|
+
const ordinals = new Int32Array(NATIVE_THREAD_SCAN_CHUNK_ORDINALS);
|
|
3612
|
+
const remaining = candidates[Symbol.iterator]();
|
|
3613
|
+
let exhausted = false;
|
|
3614
|
+
while (!exhausted) {
|
|
3615
|
+
docIds.length = 0;
|
|
3616
|
+
while (docIds.length < NATIVE_THREAD_SCAN_CHUNK_ORDINALS) {
|
|
3617
|
+
if (turnIsSpent()) await yieldTurn();
|
|
3618
|
+
const next = remaining.next();
|
|
3619
|
+
if (next.done === true) {
|
|
3620
|
+
exhausted = true;
|
|
3621
|
+
break;
|
|
3622
|
+
}
|
|
3623
|
+
if (state.tombstones.has(next.value)) continue;
|
|
3624
|
+
const ordinal = state.store.getOrdinal(next.value);
|
|
3625
|
+
if (ordinal === void 0) continue;
|
|
3626
|
+
ordinals[docIds.length] = ordinal;
|
|
3627
|
+
docIds.push(next.value);
|
|
3628
|
+
}
|
|
3629
|
+
if (docIds.length === 0) break;
|
|
3630
|
+
const distances = await scoredChunk(search2, metric, ordinals, docIds.length);
|
|
3631
|
+
if (distances === null) return null;
|
|
3632
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
3633
|
+
if (distances[i] === Number.POSITIVE_INFINITY) continue;
|
|
3634
|
+
const score = toScore(distances[i], metric);
|
|
3635
|
+
if (score < minSimilarity) continue;
|
|
3636
|
+
matched++;
|
|
3637
|
+
heap.push({ docId: docIds[i], score });
|
|
3638
|
+
}
|
|
3639
|
+
}
|
|
3640
|
+
return { results: heap.toSortedArray().reverse(), matched };
|
|
3641
|
+
}
|
|
3642
|
+
function scansEveryCandidate(state, filter) {
|
|
3643
|
+
if (state.hnsw === null) return true;
|
|
3644
|
+
if (filter === void 0) return false;
|
|
3645
|
+
const graphSize = state.hnsw.size;
|
|
3646
|
+
const selectivity = graphSize > 0 ? filter.count / graphSize : 1;
|
|
3647
|
+
return selectivity < state.filterThreshold;
|
|
3648
|
+
}
|
|
3649
|
+
async function outcomeOffThread(search2) {
|
|
3650
|
+
const { state, query, k, options, filter } = search2;
|
|
3651
|
+
const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
|
|
3652
|
+
const graph = state.hnsw;
|
|
3653
|
+
if (graph === null || scansEveryCandidate(state, filter)) {
|
|
3654
|
+
const candidates = filter ? filteredDocIds(state, filter) : allLiveDocIds(state);
|
|
3655
|
+
const scan2 = await scanOffThread(search2, candidates);
|
|
3656
|
+
return scan2 === null ? null : { results: scan2.results, matched: scan2.matched, matchedExact: true };
|
|
3657
|
+
}
|
|
3658
|
+
const buffered = state.buffer.size === 0 ? null : [...state.buffer];
|
|
3659
|
+
const hits = await graph.searchOffThread(query, k, metric, minSimilarity, {
|
|
3660
|
+
filter,
|
|
3661
|
+
efSearch,
|
|
3662
|
+
oversample,
|
|
3663
|
+
rescoreWindow
|
|
3664
|
+
});
|
|
3665
|
+
if (hits === null) return null;
|
|
3666
|
+
const graphResults = hits.map((hit) => ({ docId: hit.docId, score: hit.score }));
|
|
3667
|
+
if (buffered === null) return fetchedOutcome(state, graphResults, minSimilarity, currentFilter(search2));
|
|
3668
|
+
const scan = await scanOffThread(search2, bufferCandidates(state, buffered, filter));
|
|
3669
|
+
if (scan === null) return null;
|
|
3670
|
+
return fetchedOutcome(state, mergeResults(graphResults, scan.results, k), minSimilarity, currentFilter(search2));
|
|
3671
|
+
}
|
|
3672
|
+
async function searchOnNativeThreads(search2) {
|
|
3673
|
+
const { state, query, k, options, filter } = search2;
|
|
3674
|
+
if (query.length !== state.dimension) {
|
|
3675
|
+
throw new NarsilError(
|
|
3676
|
+
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
3677
|
+
`Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`,
|
|
3678
|
+
{ expected: state.dimension, received: query.length }
|
|
3679
|
+
);
|
|
3680
|
+
}
|
|
3681
|
+
if (liveSize(state) === 0) return { results: [], matched: 0, matchedExact: true };
|
|
3682
|
+
if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
|
|
3683
|
+
if (k <= 0) return fetchedOutcome(state, [], options.minSimilarity, filter);
|
|
3684
|
+
if (state.buffer.size > 0 && !state.building && !state.buildScheduled) scheduleBuild(state);
|
|
3685
|
+
const generation = layoutGeneration(state);
|
|
3686
|
+
const settled = await watchingSearch(state, async (watch) => {
|
|
3687
|
+
const outcome = await outcomeOffThread(search2);
|
|
3688
|
+
if (outcome === null || watch.written.size > 0 || layoutGeneration(state) !== generation) return null;
|
|
3689
|
+
return outcome;
|
|
3690
|
+
});
|
|
3691
|
+
return settled ?? searchFiltered(state, query, k, options, filterForOptions(state, options));
|
|
3692
|
+
}
|
|
3693
|
+
function searchOffThread(state, query, k, options, filter, confinedAtRevision) {
|
|
3694
|
+
const core = threadedNativeCore();
|
|
3695
|
+
if (core === null) return null;
|
|
3696
|
+
return searchOnNativeThreads({ state, core, query, k, options, filter, confinedAtRevision });
|
|
3697
|
+
}
|
|
3698
|
+
|
|
3251
3699
|
// src/vector/vector-index/writes.ts
|
|
3252
3700
|
function createVectorWrites(state) {
|
|
3253
3701
|
function validateDimension(vector) {
|
|
@@ -3279,17 +3727,23 @@ function createVectorWrites(state) {
|
|
|
3279
3727
|
if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
|
|
3280
3728
|
if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
|
|
3281
3729
|
noteWrite(state);
|
|
3730
|
+
noteSearchedDocWritten(state, docId);
|
|
3282
3731
|
state.store.setPartition(docId, partitionId);
|
|
3283
3732
|
}
|
|
3284
3733
|
return;
|
|
3285
3734
|
}
|
|
3286
3735
|
noteWrite(state);
|
|
3736
|
+
noteSearchedDocWritten(state, docId);
|
|
3287
3737
|
state.tombstones.delete(docId);
|
|
3288
3738
|
const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
|
|
3289
3739
|
state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
|
|
3290
3740
|
if (previous !== void 0) {
|
|
3291
|
-
state.hnsw
|
|
3292
|
-
state.freshGraph
|
|
3741
|
+
const graph = state.hnsw;
|
|
3742
|
+
const freshGraph = state.freshGraph;
|
|
3743
|
+
afterGraphPass(state, () => {
|
|
3744
|
+
graph?.markTombstoneOrdinal(previous);
|
|
3745
|
+
freshGraph?.markTombstoneOrdinal(previous);
|
|
3746
|
+
});
|
|
3293
3747
|
if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
|
|
3294
3748
|
}
|
|
3295
3749
|
state.buffer.add(docId);
|
|
@@ -3300,12 +3754,13 @@ function createVectorWrites(state) {
|
|
|
3300
3754
|
handOverBuildTurns(state);
|
|
3301
3755
|
if (!state.store.has(docId)) return;
|
|
3302
3756
|
noteWrite(state);
|
|
3757
|
+
noteSearchedDocWritten(state, docId);
|
|
3303
3758
|
state.tombstones.add(docId);
|
|
3304
3759
|
state.buffer.delete(docId);
|
|
3305
3760
|
state.pendingLocations.delete(docId);
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3761
|
+
const graph = state.hnsw;
|
|
3762
|
+
const ordinal = state.store.getOrdinal(docId);
|
|
3763
|
+
if (graph !== null && ordinal !== void 0) afterGraphPass(state, () => graph.markTombstoneOrdinal(ordinal));
|
|
3309
3764
|
}
|
|
3310
3765
|
return {
|
|
3311
3766
|
insert(docId, vector, partitionId) {
|
|
@@ -3404,7 +3859,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3404
3859
|
sharedLayoutRevision: 0,
|
|
3405
3860
|
sharing: Promise.resolve(),
|
|
3406
3861
|
releaseToFilesInFlight: Promise.resolve(),
|
|
3407
|
-
cold: null
|
|
3862
|
+
cold: null,
|
|
3863
|
+
graphsHeldOffThread: null,
|
|
3864
|
+
searchWatches: /* @__PURE__ */ new Set()
|
|
3408
3865
|
};
|
|
3409
3866
|
const { insert, remove } = createVectorWrites(state);
|
|
3410
3867
|
function getVector(docId) {
|
|
@@ -3422,22 +3879,24 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3422
3879
|
while (state.cold !== null) await state.cold.loaded;
|
|
3423
3880
|
}
|
|
3424
3881
|
async function searchParallel(query, k, options) {
|
|
3882
|
+
if (state.graphsHeldOffThread !== null) await untilGraphReleased(state);
|
|
3425
3883
|
if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
|
|
3426
3884
|
handOverBuildTurns(state);
|
|
3427
3885
|
const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
|
|
3428
3886
|
let filter = filterForOptions(state, options);
|
|
3887
|
+
const offThread = searchOffThread(state, query, k, options, filter, confined ? state.revision : null);
|
|
3888
|
+
if (offThread !== null) return offThread;
|
|
3429
3889
|
if (filter !== void 0) {
|
|
3430
3890
|
if (state.hnsw) {
|
|
3431
3891
|
const hnswLiveSize = state.hnsw.size;
|
|
3432
3892
|
const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
|
|
3433
|
-
if (selectivity < state.filterThreshold)
|
|
3434
|
-
return searchWithFilter(state, query, k, options, filter);
|
|
3435
|
-
}
|
|
3893
|
+
if (selectivity < state.filterThreshold) return searchFiltered(state, query, k, options, filter);
|
|
3436
3894
|
}
|
|
3437
3895
|
}
|
|
3438
3896
|
const filterRevision = state.revision;
|
|
3439
|
-
const buffered = state.buffer.size === 0 ? null : [...state.buffer];
|
|
3440
3897
|
scheduleWorkerCopyLoad(state);
|
|
3898
|
+
const searchesWorkerCopies = state.workerCopyPool !== null && state.workerCopyHandle !== null;
|
|
3899
|
+
const buffered = state.buffer.size === 0 || !searchesWorkerCopies ? null : [...state.buffer];
|
|
3441
3900
|
const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
|
|
3442
3901
|
filter,
|
|
3443
3902
|
efSearch: options.efSearch,
|
|
@@ -3451,7 +3910,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3451
3910
|
const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
|
|
3452
3911
|
return fetchedOutcome(state, results, options.minSimilarity, filter);
|
|
3453
3912
|
}
|
|
3454
|
-
return
|
|
3913
|
+
return searchFiltered(state, query, k, options, filter);
|
|
3455
3914
|
}
|
|
3456
3915
|
return {
|
|
3457
3916
|
get size() {
|
|
@@ -3493,18 +3952,21 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3493
3952
|
cancelColdHold(state);
|
|
3494
3953
|
disposeIndex(state);
|
|
3495
3954
|
},
|
|
3496
|
-
search: (
|
|
3955
|
+
search: withGraphFree(
|
|
3956
|
+
state,
|
|
3957
|
+
(query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options)
|
|
3958
|
+
),
|
|
3497
3959
|
searchParallel,
|
|
3498
3960
|
refreshWorkerCopies: () => {
|
|
3499
3961
|
if (state.cold === null) refreshWorkerCopies(state);
|
|
3500
3962
|
},
|
|
3501
3963
|
getVector,
|
|
3502
3964
|
has,
|
|
3503
|
-
compact: () => {
|
|
3965
|
+
compact: withGraphFree(state, () => {
|
|
3504
3966
|
if (state.cold !== null) return;
|
|
3505
3967
|
noteWrite(state);
|
|
3506
3968
|
compact(state);
|
|
3507
|
-
},
|
|
3969
|
+
}),
|
|
3508
3970
|
optimize: async () => {
|
|
3509
3971
|
await whenLoaded();
|
|
3510
3972
|
noteWrite(state);
|
|
@@ -3513,18 +3975,21 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3513
3975
|
maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
|
|
3514
3976
|
estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
|
|
3515
3977
|
growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
|
|
3516
|
-
deserialize: (parts) => {
|
|
3978
|
+
deserialize: withGraphFree(state, (parts) => {
|
|
3517
3979
|
cancelColdHold(state);
|
|
3518
3980
|
invalidateWorkerCopies(state);
|
|
3519
3981
|
deserialize(state, parts);
|
|
3520
|
-
},
|
|
3521
|
-
planCheckpoint: (
|
|
3982
|
+
}),
|
|
3983
|
+
planCheckpoint: withGraphFree(
|
|
3984
|
+
state,
|
|
3985
|
+
(listedKeys, rewritesGraph, pinsRetiredVectors) => planCheckpoint(state, listedKeys, rewritesGraph, pinsRetiredVectors)
|
|
3986
|
+
),
|
|
3522
3987
|
recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
|
|
3523
|
-
restoreCheckpoint: (shape) => {
|
|
3988
|
+
restoreCheckpoint: withGraphFree(state, (shape) => {
|
|
3524
3989
|
cancelColdHold(state);
|
|
3525
3990
|
invalidateWorkerCopies(state);
|
|
3526
3991
|
return beginCheckpointRestore(state, shape);
|
|
3527
|
-
},
|
|
3992
|
+
}),
|
|
3528
3993
|
openColdField: (field, load) => openColdField(state, field, load),
|
|
3529
3994
|
startShuffledRebuildWhenDue: () => {
|
|
3530
3995
|
if (state.cold === null) startShuffledRebuild(state);
|
|
@@ -3533,12 +3998,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3533
3998
|
const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
|
|
3534
3999
|
return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
|
|
3535
4000
|
},
|
|
3536
|
-
adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
|
|
3537
|
-
releaseVectorFiles: async () => {
|
|
4001
|
+
adoptDiskLayout: onceGraphFree(state, (layout) => adoptDiskLayout(state, layout)),
|
|
4002
|
+
releaseVectorFiles: onceGraphFree(state, async () => {
|
|
3538
4003
|
await whenLoaded();
|
|
3539
4004
|
await withdrawWorkerCopies(state);
|
|
3540
4005
|
state.store.closeFiles();
|
|
3541
|
-
},
|
|
4006
|
+
}),
|
|
3542
4007
|
relocateVectorFiles: (rewrite) => {
|
|
3543
4008
|
state.store.relocateVectorFiles(rewrite);
|
|
3544
4009
|
for (const [docId, location] of state.pendingLocations) {
|
|
@@ -3555,4 +4020,4 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
|
|
|
3555
4020
|
};
|
|
3556
4021
|
}
|
|
3557
4022
|
|
|
3558
|
-
export { createSharedVectorSearcher, createVectorIndex, openSharedVectorField };
|
|
4023
|
+
export { compactVectors, createSharedVectorSearcher, createVectorIndex, getVectorMaintenanceStatus, openSharedVectorField, optimizeVectors };
|