@delali/narsil 0.3.1-next.85 → 0.3.1-next.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/chunk-3BXQPIXK.mjs +602 -0
- package/dist/browser/chunk-3ZRMZIH4.mjs +3959 -0
- package/dist/browser/chunk-BBDAYUCO.mjs +2703 -0
- package/dist/browser/chunk-FHFFWLEK.mjs +669 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-NQW2YXEX.mjs +346 -0
- package/dist/browser/chunk-TBMNPGFO.mjs +22910 -0
- package/dist/browser/durability-wiring-7RUN7LNS.mjs +3250 -0
- package/dist/browser/module-E25UDADM.mjs +2766 -0
- package/dist/browser/snapshot-A42DRERF.mjs +888 -0
- package/dist/browser/write-4U6Q26TJ.mjs +206 -0
- package/dist/{capabilities-CAdkp2P-.d.ts → capabilities-SJPttcgQ.d.ts} +2 -2
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-AQU2QHSY.mjs +7247 -0
- package/dist/{chunk-OLXM65YH.mjs → chunk-AV4IJKD2.mjs} +1 -18
- package/dist/{chunk-4D6OH4EF.mjs → chunk-AXDAS2VA.mjs} +7 -5
- package/dist/{chunk-LS5IRM23.mjs → chunk-BVOJIYRV.mjs} +2 -25
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-KJ7LB3TR.mjs → chunk-CVQWQVFX.mjs} +3 -22
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-FGMGUDHX.mjs +21 -0
- package/dist/chunk-IKST36MA.mjs +614 -0
- package/dist/{chunk-2453RJYK.mjs → chunk-J2BPXTD7.mjs} +167 -176
- package/dist/chunk-JFJBA3PD.mjs +522 -0
- package/dist/chunk-K4OWQ2M3.mjs +462 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/chunk-MKK4OZ5W.mjs +703 -0
- package/dist/chunk-MXS5HWIH.mjs +515 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PR4O7CAP.mjs +18 -0
- package/dist/{chunk-4VBG6PEH.mjs → chunk-PSUJCOSE.mjs} +3 -2
- package/dist/{chunk-ZCKPRCYI.mjs → chunk-Q5DC6WAB.mjs} +1 -1
- package/dist/{chunk-NFF5TDRG.mjs → chunk-QZN3MIJ3.mjs} +1 -16
- package/dist/chunk-R7OSDAFK.mjs +44 -0
- package/dist/chunk-RVM7AODE.mjs +73 -0
- package/dist/{chunk-XYDP7CCJ.mjs → chunk-SYRFSF6V.mjs} +6 -4
- package/dist/{chunk-SREPGC6O.mjs → chunk-T6CNO72J.mjs} +1 -1
- package/dist/chunk-T7DMUVTL.mjs +1714 -0
- package/dist/chunk-UKSUENHK.mjs +997 -0
- package/dist/{chunk-NIWDQTZB.mjs → chunk-ULRPNYAG.mjs} +1220 -2311
- package/dist/{chunk-GNYXCLJS.mjs → chunk-URGLSNKR.mjs} +4 -3
- package/dist/chunk-WQSHDZ7Z.mjs +631 -0
- package/dist/{chunk-RCSXJCI5.mjs → chunk-XBXKPWMR.mjs} +20 -2664
- package/dist/{chunk-RM6FRYEC.mjs → chunk-XSZVS4S3.mjs} +3 -2
- package/dist/chunk-Y2F66IQJ.mjs +74 -0
- package/dist/chunk-YCJLVNWR.mjs +262 -0
- package/dist/{client-CKUtkLz3.d.ts → client-BMK53IR2.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-CSDEPraV.d.ts → config-BUR-7eVQ.d.ts} +2 -2
- package/dist/{config-BuIkeM5C.d.ts → config-UJx1dyX1.d.ts} +2 -2
- package/dist/direct-executor-LD6JW57R.mjs +18 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +30 -15
- package/dist/durability-wiring-5S32KDDC.mjs +2797 -0
- package/dist/{engine-DMNZJAUq.d.ts → engine-9X6W5naj.d.ts} +2 -2
- package/dist/{index-DzONZUKE.d.ts → index-vCMu8OeQ.d.ts} +1 -1
- package/dist/index.browser.mjs +2 -38121
- package/dist/index.d.ts +23 -9
- package/dist/index.mjs +18 -14
- package/dist/{memory-BPKNx9Tl.d.ts → memory-CV9NAxLY.d.ts} +1 -1
- package/dist/module-WIDYIJLY.mjs +1819 -0
- package/dist/partial-words.mjs +5 -3
- package/dist/pattern.mjs +3 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -13
- package/dist/react.d.ts +7 -7
- package/dist/{schema-DgyDijhf.d.ts → schema-DtsgmNfU.d.ts} +1 -1
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +14 -9
- package/dist/snapshot-6BYSUPWL.mjs +469 -0
- package/dist/{tasks-DGyz586z.d.ts → tasks-BGaZJyoY.d.ts} +1 -1
- package/dist/{thread-EUUNJNKU.mjs → thread-GS65E6RM.mjs} +12 -9
- package/dist/vector/search-worker.d.ts +30 -3
- package/dist/vector/search-worker.mjs +6 -2
- package/dist/workers/entry.mjs +5 -3
- package/dist/write-HLEZTL2L.mjs +233 -0
- package/package.json +11 -7
- package/dist/chunk-67MQF5CV.mjs +0 -15198
- package/dist/direct-executor-NY7ZTZIH.mjs +0 -19
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
|
@@ -1,189 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { readTrailer, unpackLevels, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
2
|
+
import { stopUsingTheNativeSearchCore, noteFallback, nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
3
|
+
import { markSimdUnavailable, MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, NATIVE_ATTACH_RETRY_MS, GRAPH_LOCK_WAIT_SLICE_MS, MAX_DOC_ID_TABLE_BYTES } from './chunk-Y2F66IQJ.mjs';
|
|
4
|
+
import { OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
|
|
5
|
+
import { loadNativeCore } from './chunk-QZN3MIJ3.mjs';
|
|
6
|
+
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
|
|
4
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
8
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
6
|
-
import { readSync, closeSync, openSync } from 'fs';
|
|
7
|
-
|
|
8
|
-
// src/vector/constants.ts
|
|
9
|
-
var WASM_PAGE_BYTES = 65536;
|
|
10
|
-
var MAX_WASM_PAGES = 65536;
|
|
11
|
-
var VECTOR_STORE_INITIAL_CAPACITY = 16;
|
|
12
|
-
var VECTOR_BLOCK_INITIAL_SLOTS = 256;
|
|
13
|
-
var DISK_STORAGE_BLOCK_BYTES = 67108864;
|
|
14
|
-
var MAX_VECTOR_ORDINALS = 16777216;
|
|
15
|
-
var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
|
|
16
|
-
var VECTOR_SCRATCH_SLOTS = 64;
|
|
17
|
-
var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
|
|
18
|
-
var MAX_DOC_ID_TABLE_BYTES = 1073741824;
|
|
19
|
-
var MAX_WORKER_COPIES = 64;
|
|
20
|
-
var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
|
|
21
|
-
var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
|
|
22
|
-
var VECTOR_INSERT_TIMEOUT_MS = 6e5;
|
|
23
|
-
var NATIVE_ATTACH_RETRY_MS = 1e3;
|
|
24
|
-
var LOCK_SPIN_ITERATIONS = 128;
|
|
25
|
-
var LOCK_WAIT_SLICE_MS = 20;
|
|
26
|
-
var GRAPH_LOCK_WAIT_SLICE_MS = 1;
|
|
27
|
-
var GRAPH_BUILD_DISPATCH_CHUNK = 64;
|
|
28
|
-
var GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER = 2;
|
|
29
|
-
|
|
30
|
-
// src/vector/simd-wasm-binary.ts
|
|
31
|
-
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8DDw4AAQAAAgIDAgIEAgQCAgUDAQABB6QBCgZtZW1vcnkCAAtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
32
|
-
var SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8CEgEDZW52Bm1lbW9yeQIDAYCABAMPDgABAAACAgMCAgQCBAICB5sBCQtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
33
|
-
|
|
34
|
-
// src/vector/simd.ts
|
|
35
|
-
var wasmExports = null;
|
|
36
|
-
var f32View = null;
|
|
37
|
-
function decodeBase64(base64) {
|
|
38
|
-
if (typeof Buffer !== "undefined") {
|
|
39
|
-
return new Uint8Array(Buffer.from(base64, "base64"));
|
|
40
|
-
}
|
|
41
|
-
const binary = atob(base64);
|
|
42
|
-
const bytes = new Uint8Array(binary.length);
|
|
43
|
-
for (let i = 0; i < binary.length; i++) {
|
|
44
|
-
bytes[i] = binary.charCodeAt(i);
|
|
45
|
-
}
|
|
46
|
-
return bytes;
|
|
47
|
-
}
|
|
48
|
-
try {
|
|
49
|
-
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
50
|
-
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
51
|
-
if (typeof WebAssembly !== "undefined" && typeof WebAssembly.validate === "function" && WebAssembly.validate(wasmBuffer)) {
|
|
52
|
-
const module = new WebAssembly.Module(wasmBuffer);
|
|
53
|
-
const instance = new WebAssembly.Instance(module);
|
|
54
|
-
wasmExports = instance.exports;
|
|
55
|
-
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
56
|
-
}
|
|
57
|
-
} catch {
|
|
58
|
-
wasmExports = null;
|
|
59
|
-
f32View = null;
|
|
60
|
-
}
|
|
61
|
-
var OFFSET_B_BYTES = 16384;
|
|
62
|
-
function ensureMemory(dim) {
|
|
63
|
-
if (!wasmExports || !f32View) return;
|
|
64
|
-
const needed = OFFSET_B_BYTES + dim * 4;
|
|
65
|
-
const currentSize = wasmExports.memory.buffer.byteLength;
|
|
66
|
-
if (needed > currentSize) {
|
|
67
|
-
const pagesNeeded = Math.ceil(needed / 65536);
|
|
68
|
-
const currentPages = currentSize / 65536;
|
|
69
|
-
wasmExports.memory.grow(pagesNeeded - currentPages);
|
|
70
|
-
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
function getF32View() {
|
|
74
|
-
if (!f32View) {
|
|
75
|
-
throw new Error("SIMD memory is not initialized");
|
|
76
|
-
}
|
|
77
|
-
return f32View;
|
|
78
|
-
}
|
|
79
|
-
function copyOneVector(a) {
|
|
80
|
-
getF32View().set(a, 0);
|
|
81
|
-
}
|
|
82
|
-
function copyTwoVectors(a, b) {
|
|
83
|
-
const view = getF32View();
|
|
84
|
-
view.set(a, 0);
|
|
85
|
-
view.set(b, OFFSET_B_BYTES / 4);
|
|
86
|
-
}
|
|
87
|
-
function simdDotProduct(a, b) {
|
|
88
|
-
if (!wasmExports || !f32View) return null;
|
|
89
|
-
ensureMemory(a.length);
|
|
90
|
-
copyTwoVectors(a, b);
|
|
91
|
-
return wasmExports.dot_product(0, OFFSET_B_BYTES, a.length);
|
|
92
|
-
}
|
|
93
|
-
function simdMagnitude(a) {
|
|
94
|
-
if (!wasmExports || !f32View) return null;
|
|
95
|
-
ensureMemory(a.length);
|
|
96
|
-
copyOneVector(a);
|
|
97
|
-
return wasmExports.magnitude(0, a.length);
|
|
98
|
-
}
|
|
99
|
-
function simdSquaredEuclideanDistance(a, b) {
|
|
100
|
-
if (!wasmExports || !f32View) return null;
|
|
101
|
-
ensureMemory(a.length);
|
|
102
|
-
copyTwoVectors(a, b);
|
|
103
|
-
return wasmExports.squared_euclidean_distance(0, OFFSET_B_BYTES, a.length);
|
|
104
|
-
}
|
|
105
|
-
function isSimdAvailable() {
|
|
106
|
-
return wasmExports !== null;
|
|
107
|
-
}
|
|
108
|
-
var arenaModule;
|
|
109
|
-
function getArenaModule() {
|
|
110
|
-
if (arenaModule !== void 0) return arenaModule;
|
|
111
|
-
try {
|
|
112
|
-
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
113
|
-
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
114
|
-
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
|
|
115
|
-
arenaModule = null;
|
|
116
|
-
return null;
|
|
117
|
-
}
|
|
118
|
-
arenaModule = new WebAssembly.Module(wasmBuffer);
|
|
119
|
-
} catch {
|
|
120
|
-
arenaModule = null;
|
|
121
|
-
}
|
|
122
|
-
return arenaModule;
|
|
123
|
-
}
|
|
124
|
-
function createArenaSimd() {
|
|
125
|
-
try {
|
|
126
|
-
const module = getArenaModule();
|
|
127
|
-
if (module === null) return null;
|
|
128
|
-
const instance = new WebAssembly.Instance(module);
|
|
129
|
-
const exports$1 = instance.exports;
|
|
130
|
-
if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.osq_dot_nibbles_4x4 !== "function" || typeof exports$1.osq_dot_bits !== "function" || typeof exports$1.osq_dot_bits_pair !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function" || !(exports$1.memory instanceof WebAssembly.Memory)) {
|
|
131
|
-
return null;
|
|
132
|
-
}
|
|
133
|
-
return exports$1;
|
|
134
|
-
} catch {
|
|
135
|
-
return null;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
var sharedMemoryModule;
|
|
139
|
-
function getSharedMemoryModule() {
|
|
140
|
-
if (sharedMemoryModule !== void 0) return sharedMemoryModule;
|
|
141
|
-
try {
|
|
142
|
-
const bytes = decodeBase64(SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64);
|
|
143
|
-
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
144
|
-
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
|
|
145
|
-
sharedMemoryModule = null;
|
|
146
|
-
return null;
|
|
147
|
-
}
|
|
148
|
-
sharedMemoryModule = new WebAssembly.Module(wasmBuffer);
|
|
149
|
-
} catch {
|
|
150
|
-
sharedMemoryModule = null;
|
|
151
|
-
}
|
|
152
|
-
return sharedMemoryModule;
|
|
153
|
-
}
|
|
154
|
-
function createSharedArenaSimd(memory) {
|
|
155
|
-
try {
|
|
156
|
-
const module = getSharedMemoryModule();
|
|
157
|
-
if (module === null) return null;
|
|
158
|
-
const instance = new WebAssembly.Instance(module, { env: { memory } });
|
|
159
|
-
const exports$1 = instance.exports;
|
|
160
|
-
if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.osq_dot_nibbles_4x4 !== "function" || typeof exports$1.osq_dot_bits !== "function" || typeof exports$1.osq_dot_bits_pair !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function") {
|
|
161
|
-
return null;
|
|
162
|
-
}
|
|
163
|
-
return {
|
|
164
|
-
memory,
|
|
165
|
-
dot_product: exports$1.dot_product,
|
|
166
|
-
magnitude: exports$1.magnitude,
|
|
167
|
-
squared_euclidean_distance: exports$1.squared_euclidean_distance,
|
|
168
|
-
dot_u8: exports$1.dot_u8,
|
|
169
|
-
sqdist_u8: exports$1.sqdist_u8,
|
|
170
|
-
osq_dot_nibbles_4x4: exports$1.osq_dot_nibbles_4x4,
|
|
171
|
-
osq_dot_bits: exports$1.osq_dot_bits,
|
|
172
|
-
osq_dot_bits_pair: exports$1.osq_dot_bits_pair
|
|
173
|
-
};
|
|
174
|
-
} catch {
|
|
175
|
-
return null;
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
function arenaFloat32Distance(simd, byteA, byteB, dimension, metric, magnitudeA, magnitudeB) {
|
|
179
|
-
if (metric === "euclidean") {
|
|
180
|
-
return Math.sqrt(simd.squared_euclidean_distance(byteA, byteB, dimension));
|
|
181
|
-
}
|
|
182
|
-
const dot = simd.dot_product(byteA, byteB, dimension);
|
|
183
|
-
if (metric === "dotProduct") return -dot;
|
|
184
|
-
if (magnitudeA === 0 || magnitudeB === 0) return 1;
|
|
185
|
-
return 1 - dot / (magnitudeA * magnitudeB);
|
|
186
|
-
}
|
|
187
9
|
|
|
188
10
|
// src/vector/shared-buffers/growable.ts
|
|
189
11
|
function sharedMemoryAvailable() {
|
|
@@ -317,1107 +139,1378 @@ function graphShapeOf(header) {
|
|
|
317
139
|
};
|
|
318
140
|
}
|
|
319
141
|
|
|
320
|
-
// src/vector/
|
|
321
|
-
var
|
|
322
|
-
var
|
|
323
|
-
function
|
|
324
|
-
|
|
325
|
-
}
|
|
326
|
-
function computeVectorBlockLayout(dimension, blockBytes2 = BLOCK_MAX_BYTES) {
|
|
327
|
-
const scratchStride = alignUp(dimension * 4);
|
|
328
|
-
const slotsOffset = alignUp(VECTOR_SCRATCH_SLOTS * scratchStride);
|
|
329
|
-
const slotStride = alignUp(dimension * 4);
|
|
142
|
+
// src/vector/hnsw/locks.ts
|
|
143
|
+
var FREE = 0;
|
|
144
|
+
var EXCLUSIVE = -1;
|
|
145
|
+
function openGraphLocks(handles, threadSlot) {
|
|
146
|
+
let words = fixedView(handles.locks, Int32Array);
|
|
330
147
|
return {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
148
|
+
threadSlot,
|
|
149
|
+
header: handles.header,
|
|
150
|
+
held: handles.heldLocks,
|
|
151
|
+
words(ord) {
|
|
152
|
+
if (ord < words.length) return words;
|
|
153
|
+
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
154
|
+
words = fixedView(handles.locks, Int32Array);
|
|
155
|
+
return words;
|
|
156
|
+
}
|
|
336
157
|
};
|
|
337
158
|
}
|
|
338
|
-
|
|
339
|
-
|
|
159
|
+
var blockingAllowed = typeof document === "undefined";
|
|
160
|
+
function canWait(words) {
|
|
161
|
+
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
340
162
|
}
|
|
341
|
-
function
|
|
342
|
-
|
|
163
|
+
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
164
|
+
if (!canWait(words)) return;
|
|
165
|
+
Atomics.wait(words, index, seen, sliceMs);
|
|
343
166
|
}
|
|
344
|
-
function
|
|
345
|
-
|
|
346
|
-
try {
|
|
347
|
-
const memory = new WebAssembly.Memory({ initial: pages, maximum: MAX_WASM_PAGES, shared: true });
|
|
348
|
-
return createSharedArenaSimd(memory) === null ? null : memory;
|
|
349
|
-
} catch {
|
|
350
|
-
return null;
|
|
351
|
-
}
|
|
167
|
+
function heldIndex(locks, kind) {
|
|
168
|
+
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
352
169
|
}
|
|
353
|
-
function
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
170
|
+
function acquireExclusive(words, index) {
|
|
171
|
+
let spins = 0;
|
|
172
|
+
for (; ; ) {
|
|
173
|
+
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
174
|
+
if (seen === FREE) return;
|
|
175
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
176
|
+
spins += 1;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
362
180
|
}
|
|
363
|
-
return { layout, storage: { kind: "bytes", buffer: createGrowableBuffer(initialBytes, fullBlockBytes(layout)) } };
|
|
364
|
-
}
|
|
365
|
-
function fullBlockBytes(layout) {
|
|
366
|
-
return Math.min(BLOCK_MAX_BYTES, layout.slotsOffset + layout.capacity * layout.slotStride);
|
|
367
181
|
}
|
|
368
|
-
function
|
|
369
|
-
|
|
370
|
-
if (
|
|
371
|
-
memory.grow(pagesFor(nextGrowthTarget(have, bytes, fullBlockBytes(layout))) - have / WASM_PAGE_BYTES);
|
|
182
|
+
function releaseExclusive(words, index) {
|
|
183
|
+
Atomics.store(words, index, FREE);
|
|
184
|
+
if (canWait(words)) Atomics.notify(words, index);
|
|
372
185
|
}
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
186
|
+
var WRITE_HELD = 1;
|
|
187
|
+
var WRITER_WAITING = 2;
|
|
188
|
+
var VERSION_STEP = 4;
|
|
189
|
+
function beginNodeRead(locks, ord) {
|
|
190
|
+
const words = locks.words(ord);
|
|
191
|
+
for (; ; ) {
|
|
192
|
+
const seen = Atomics.load(words, ord);
|
|
193
|
+
if ((seen & WRITE_HELD) === 0) return seen;
|
|
379
194
|
}
|
|
380
|
-
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
381
|
-
}
|
|
382
|
-
function blockBuffer(handle) {
|
|
383
|
-
const storage = handle.storage;
|
|
384
|
-
if (storage.kind === "bytes") return storage.buffer;
|
|
385
|
-
return storage.kind === "shared-memory" ? storage.memory.buffer : storage.simd.memory.buffer;
|
|
386
195
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
const { layout, storage } = handle;
|
|
391
|
-
const simd = storage.kind === "shared-memory" ? createSharedArenaSimd(storage.memory) : storage.kind === "private-memory" ? storage.simd : null;
|
|
392
|
-
const hasScratch = simd !== null && threadSlot >= 0 && threadSlot < VECTOR_SCRATCH_SLOTS;
|
|
393
|
-
const scratchSlot = hasScratch ? threadSlot : 0;
|
|
394
|
-
let float32View = fixedView(blockBuffer(handle), Float32Array);
|
|
395
|
-
let byteView = fixedView(blockBuffer(handle), Uint8Array);
|
|
396
|
-
let dataView = new DataView(blockBuffer(handle));
|
|
397
|
-
return {
|
|
398
|
-
handle,
|
|
399
|
-
simd,
|
|
400
|
-
hasScratch,
|
|
401
|
-
scratchByteOffset: scratchSlot * layout.scratchStride,
|
|
402
|
-
stagedOrdinal: NOTHING_STAGED,
|
|
403
|
-
float32(neededLength) {
|
|
404
|
-
if (neededLength > float32View.length) float32View = fixedView(blockBuffer(handle), Float32Array);
|
|
405
|
-
return float32View;
|
|
406
|
-
},
|
|
407
|
-
bytes(neededLength) {
|
|
408
|
-
if (neededLength > byteView.length) byteView = fixedView(blockBuffer(handle), Uint8Array);
|
|
409
|
-
return byteView;
|
|
410
|
-
},
|
|
411
|
-
data(neededLength) {
|
|
412
|
-
if (neededLength > dataView.byteLength) dataView = new DataView(blockBuffer(handle));
|
|
413
|
-
return dataView;
|
|
414
|
-
}
|
|
415
|
-
};
|
|
416
|
-
}
|
|
417
|
-
function slotByteOffset(layout, localOrdinal) {
|
|
418
|
-
return layout.slotsOffset + localOrdinal * layout.slotStride;
|
|
196
|
+
function nodeReadHeld(locks, ord, version) {
|
|
197
|
+
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
198
|
+
return Atomics.load(locks.words(ord), ord) === version;
|
|
419
199
|
}
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
for (
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
200
|
+
function lockNodeWrite(locks, ord) {
|
|
201
|
+
const words = locks.words(ord);
|
|
202
|
+
let spins = 0;
|
|
203
|
+
for (; ; ) {
|
|
204
|
+
const seen = Atomics.load(words, ord);
|
|
205
|
+
if ((seen & WRITE_HELD) === 0) {
|
|
206
|
+
const claimed = seen | WRITE_HELD;
|
|
207
|
+
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
208
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
209
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
215
|
+
spins += 1;
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
const waiting = seen | WRITER_WAITING;
|
|
219
|
+
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
220
|
+
waitOn(words, ord, waiting);
|
|
221
|
+
}
|
|
429
222
|
}
|
|
430
|
-
POPCOUNT[value] = count;
|
|
431
223
|
}
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
for (let n = 0; n < bytes; n++) {
|
|
437
|
-
const left = a[aOffset + n];
|
|
438
|
-
const right = b[bOffset + n];
|
|
439
|
-
total += (left & 15) * (right & 15) + (left >> 4) * (right >> 4);
|
|
440
|
-
}
|
|
441
|
-
return total;
|
|
224
|
+
function releaseWrite(words, ord) {
|
|
225
|
+
const seen = Atomics.load(words, ord);
|
|
226
|
+
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
227
|
+
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
442
228
|
}
|
|
443
|
-
function
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
229
|
+
function unlockNodeWrite(locks, ord) {
|
|
230
|
+
releaseWrite(locks.words(ord), ord);
|
|
231
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
232
|
+
}
|
|
233
|
+
function lockGraphShared(locks) {
|
|
234
|
+
const { header } = locks;
|
|
235
|
+
let spins = 0;
|
|
236
|
+
for (; ; ) {
|
|
237
|
+
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
238
|
+
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
239
|
+
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
240
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
continue;
|
|
449
244
|
}
|
|
245
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
246
|
+
spins += 1;
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
450
250
|
}
|
|
451
|
-
return total;
|
|
452
251
|
}
|
|
453
|
-
function
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
const plane = stagedOffset + p * codeBytes;
|
|
458
|
-
let bitsSet = 0;
|
|
459
|
-
for (let n = 0; n < codeBytes; n++) bitsSet += POPCOUNT[document2[documentOffset + n] & staged[plane + n]];
|
|
460
|
-
total += 2 ** p * bitsSet;
|
|
461
|
-
}
|
|
462
|
-
return total;
|
|
252
|
+
function unlockGraphShared(locks) {
|
|
253
|
+
const { header } = locks;
|
|
254
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
255
|
+
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
463
256
|
}
|
|
464
|
-
function
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
}
|
|
257
|
+
function lockGraphExclusive(locks) {
|
|
258
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
259
|
+
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
260
|
+
try {
|
|
261
|
+
acquireExclusive(locks.header, GRAPH_LOCK);
|
|
262
|
+
} finally {
|
|
263
|
+
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
264
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
473
265
|
}
|
|
474
|
-
return total;
|
|
475
|
-
}
|
|
476
|
-
function osqByteLevelProducts(document2, documentOffset, query, queryOffset, dimension) {
|
|
477
|
-
let total = 0;
|
|
478
|
-
for (let i = 0; i < dimension; i++) total += document2[documentOffset + i] * query[queryOffset + i];
|
|
479
|
-
return total;
|
|
480
266
|
}
|
|
481
|
-
function
|
|
482
|
-
|
|
483
|
-
const queryStep = (query.upper - query.lower) / OSQ_STEPS[queryBits];
|
|
484
|
-
const centred = document2.lower * query.lower * dimension + query.lower * documentStep * document2.sum + document2.lower * queryStep * query.sum + documentStep * queryStep * products;
|
|
485
|
-
if (metric === "euclidean") return Math.sqrt(Math.max(0, document2.correction + query.correction - 2 * centred));
|
|
486
|
-
const similarity = centred + document2.correction + query.correction - centroidDot;
|
|
487
|
-
if (metric === "cosine") return Math.min(Math.max(similarity, -1), 1);
|
|
488
|
-
return similarity;
|
|
267
|
+
function unlockGraphExclusive(locks) {
|
|
268
|
+
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
489
269
|
}
|
|
490
|
-
function
|
|
491
|
-
|
|
492
|
-
case "cosine":
|
|
493
|
-
return 1 - estimate;
|
|
494
|
-
case "dotProduct":
|
|
495
|
-
return -estimate;
|
|
496
|
-
case "euclidean":
|
|
497
|
-
return estimate;
|
|
498
|
-
}
|
|
270
|
+
function lockEntry(locks) {
|
|
271
|
+
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
499
272
|
}
|
|
500
|
-
function
|
|
501
|
-
|
|
273
|
+
function unlockEntry(locks) {
|
|
274
|
+
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
502
275
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
var TRAILER_SUM = 12;
|
|
510
|
-
var QUERY_PLANES_FOR_NARROW_CODES = 4;
|
|
511
|
-
var LEVELS_IN_A_TWO_BIT_BYTE = 4;
|
|
512
|
-
function osqCodeBytes(dimension, bits) {
|
|
513
|
-
return Math.ceil(dimension * bits / 8);
|
|
514
|
-
}
|
|
515
|
-
function osqRecordBytes(dimension, bits) {
|
|
516
|
-
return osqCodeBytes(dimension, bits) + OSQ_RECORD_TRAILER_BYTES;
|
|
517
|
-
}
|
|
518
|
-
function packLevels(levels, bits, target, offset) {
|
|
519
|
-
const dimension = levels.length;
|
|
520
|
-
if (bits === 8) {
|
|
521
|
-
target.set(levels, offset);
|
|
522
|
-
return;
|
|
276
|
+
function releaseLocksHeldBy(handles, threadSlot) {
|
|
277
|
+
const held = handles.heldLocks;
|
|
278
|
+
const words = new Int32Array(handles.locks);
|
|
279
|
+
const base = threadSlot * HELD_WORDS_PER_THREAD;
|
|
280
|
+
if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
|
|
281
|
+
Atomics.sub(handles.header, GRAPH_WRITERS_WAITING, 1);
|
|
523
282
|
}
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
const
|
|
528
|
-
if (
|
|
529
|
-
|
|
283
|
+
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
284
|
+
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
285
|
+
if (writeOrd >= 0 && writeOrd < words.length) {
|
|
286
|
+
const seen = Atomics.load(words, writeOrd);
|
|
287
|
+
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
288
|
+
}
|
|
289
|
+
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
290
|
+
for (; ; ) {
|
|
291
|
+
const readers = Atomics.load(handles.header, GRAPH_LOCK);
|
|
292
|
+
if (readers <= FREE) return;
|
|
293
|
+
if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
294
|
+
if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
530
297
|
}
|
|
531
298
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
299
|
+
|
|
300
|
+
// src/vector/simd-wasm-binary.ts
|
|
301
|
+
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8DDw4AAQAAAgIDAgIEAgQCAgUDAQABB6QBCgZtZW1vcnkCAAtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
302
|
+
var SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64 = "AGFzbQEAAAABIgVgA39/fwF9YAJ/fwF9YAN/f38Bf2ABewF/YAR/f39/AX8CEgEDZW52Bm1lbW9yeQIDAYCABAMPDgABAAACAgMCAgQCBAICB5sBCQtkb3RfcHJvZHVjdAAACW1hZ25pdHVkZQABGnNxdWFyZWRfZXVjbGlkZWFuX2Rpc3RhbmNlAAISZXVjbGlkZWFuX2Rpc3RhbmNlAAMGZG90X3U4AAQJc3FkaXN0X3U4AAUMb3NxX2RvdF9iaXRzAAkRb3NxX2RvdF9iaXRzX3BhaXIACxNvc3FfZG90X25pYmJsZXNfNHg0AA0K2RQOygIEAX8EewF9BH8gAkFwcSEJIAJBfHEhCv0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAJTw0BIAAgA0ECdGohCyABIANBAnRqIQwgBCAL/QAEACAM/QAEAP3mAf3kASEEIAUgC/0ABBAgDP0ABBD95gH95AEhBSAGIAv9AAQgIAz9AAQg/eYB/eQBIQYgByAL/QAEMCAM/QAEMP3mAf3kASEHIANBEGohAwwACwsCQANAIAMgCk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAEIAX95AEgBiAH/eQB/eQBIQQgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEIAkADQCADIAJPDQEgCCAAIANBAnRqKgIAIAEgA0ECdGoqAgCUkiEIIANBAWohAwwACwsgCAupAQQBfwJ7AX0BfyABQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhA0EAIQICQANAIAIgBk8NASAAIAJBAnRq/QAEACEEIAMgBCAE/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQUCQANAIAIgAU8NASAFIAAgAkECdGoqAgAgACACQQJ0aioCAJSSIQUgAkEBaiECDAALCyAFkQvAAQUBfwJ7AX0BfwF9IAJBfHEhB/0MAAAAAAAAAAAAAAAAAAAAACEEQQAhAwJAA0AgAyAHTw0BIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eUBIQUgBCAFIAX95gH95AEhBCADQQRqIQMMAAsLIAT9HwAgBP0fAZIgBP0fAiAE/R8DkpIhBgJAA0AgAyACTw0BIAAgA0ECdGoqAgAgASADQQJ0aioCAJMhCCAGIAggCJSSIQYgA0EBaiEDDAALCyAGCwsAIAAgASACEAKRC7QBAwF/A3sCfyACQXBxIQhBAP0RIQRBACEDAkADQCADIAhPDQEgACADav0ABAAhBSABIANq/QAEACEGIAQgBf2JASAG/YkB/boB/a4BIQQgBCAF/YoBIAb9igH9ugH9rgEhBCADQRBqIQMMAAsLIAT9GwAgBP0bAWogBP0bAiAE/RsDamohBwJAA0AgAyACTw0BIAcgACADai0AACABIANqLQAAbGohByADQQFqIQMMAAsLIAcLzQEDAX8FewN/IAJBcHEhC0EA/REhBEEAIQMCQANAIAMgC08NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBf2JASAG/YkB/ZEBIQcgBf2KASAG/YoB/ZEBIQggBCAHIAf9ugH9rgEhBCAEIAggCP26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEJAkADQCADIAJPDQEgACADai0AACABIANqLQAAayEKIAkgCiAKbGohCSADQQFqIQMMAAsLIAkLIQEBeyAA/X8hASAB/RsAIAH9GwFqIAH9GwIgAf0bA2pqC68CAwh/A3sBfyABIAJqIQggCCACaiEJIAkgAmohCiACQXBxIQUCQANAIAMgBU8NASADQYAQaiEGIAYgBUsEQCAFIQYLQQD9ECENA0AgACADav0ABAAhCyALIAogA2r9AAQA/U79YiEMIAwgDP1uIAsgCSADav0ABAD9Tv1i/W4hDCAMIAz9biALIAggA2r9AAQA/U79Yv1uIQwgDCAM/W4gCyABIANq/QAEAP1O/WL9biEMIA0gDP19/Y4BIQ0gA0EQaiEDIAMgBkkNAAsgByANEAZqIQcMAAsLAkADQCADIAJPDQEgACADai0AACEOQQAhBAJAA0AgBEEETw0BIAcgDiABIAIgBGxqIANqLQAAcWkgBHRqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLhwMDCH8IewF/IAEgAmohCCAIIAJqIQkgCSACaiEKQQP9DyERIAJBcHEhBQJAA0AgAyAFTw0BIANBgBBqIQYgBiAFSwRAIAUhBgtBAP0QIRIDQCAAIANq/QAEACELIAsgEf1OIQwgC0EC/W0gEf1OIQ0gC0EE/W0gEf1OIQ4gC0EG/W0hDyABIANq/QAEACEQIBIgDCAQ/Z4BIAwgEP2fAf2OAf2OASESIAggA2r9AAQAIRAgEiANIBD9ngEgDSAQ/Z8B/Y4B/Y4BIRIgCSADav0ABAAhECASIA4gEP2eASAOIBD9nwH9jgH9jgEhEiAKIANq/QAEACEQIBIgDyAQ/Z4BIA8gEP2fAf2OAf2OASESIANBEGohAyADIAZJDQALIAcgEhAGaiEHDAALCwJAA0AgAyACTw0BIAAgA2otAAAhE0EAIQQCQANAIARBBE8NASAHIBMgBEEBdHZBA3EgASACIARsaiADai0AAGxqIQcgBEEBaiEEDAALCyADQQFqIQMMAAsLIAcLGwAgA0ECRgR/IAAgASACEAgFIAAgASACEAcLC54BAgR/AXsgAkFwcSEEAkADQCADIARPDQEgA0GAEGohBSAFIARLBEAgBCEFC0EA/RAhBwNAIAcgACADav0ABAAgASADav0ABAD9Tv1i/X39jgEhByADQRBqIQMgAyAFSQ0ACyAGIAcQBmohBgwACwsCQANAIAMgAk8NASAGIAAgA2otAAAgASADai0AAHFpaiEGIANBAWohAwwACwsgBgsbACADQQJGBH8gACABIAIQDAUgACABIAIQCgsL2gIDBH8IewR/QdUAIRFB1QAhEkHVAP0PIQdBqgH9DyEIIAJBcHEhBAJAA0AgAyAETw0BIANB4ANqIQUgBSAESwRAIAQhBQtBAP0PIQ5BAP0PIQxBAP0PIQ0DQCAAIANq/QAEACEJIAEgA2r9AAQAIQogCSAK/U4hCyAOIAv9Yv1uIQ4gDCALIAj9Tv1i/W4hDCANIAkgCkEB/W0gB/1OIApBAf1rIAj9Tv1Q/U79Yv1uIQ0gA0EQaiEDIAMgBUkNAAsgBiAO/X0QBiAM/X0QBkEDbCAN/X0QBkEBdGpqaiEGDAALCwJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIRAgBiAPIBFxIBBxaSAPIBFxIBBBAXYgEnFxaUEBdGogD0EBdiAScSAQIBFxcWlBAXQgD0EBdiAScSAQQQF2IBJxcWlBAnRqamohBiADQQFqIQMMAAsLIAYL/AEDA38IewJ/IAJBcHEhBEEP/Q8hDEEA/REhDUEAIQMCQANAIAMgBE8NASAAIANq/QAEACEGIAEgA2r9AAQAIQcgBiAM/U4hCCAGQQT9bSEJIAcgDP1OIQogB0EE/W0hCyANIAggCv2eASAJIAv9ngH9jgH9fyAIIAr9nwEgCSAL/Z8B/Y4B/X/9rgH9rgEhDSADQRBqIQMMAAsLIA39GwAgDf0bAWogDf0bAiAN/RsDamohBQJAA0AgAyACTw0BIAAgA2otAAAhDyABIANqLQAAIQ4gBSAPQQ9xIA5BD3FsIA9BBHYgDkEEdmxqaiEFIANBAWohAwwACwsgBQs=";
|
|
303
|
+
|
|
304
|
+
// src/vector/simd.ts
|
|
305
|
+
var wasmExports = null;
|
|
306
|
+
var f32View = null;
|
|
307
|
+
function decodeBase64(base64) {
|
|
308
|
+
if (typeof Buffer !== "undefined") {
|
|
309
|
+
return new Uint8Array(Buffer.from(base64, "base64"));
|
|
537
310
|
}
|
|
538
|
-
const
|
|
539
|
-
const
|
|
540
|
-
for (let i = 0; i <
|
|
541
|
-
|
|
311
|
+
const binary = atob(base64);
|
|
312
|
+
const bytes = new Uint8Array(binary.length);
|
|
313
|
+
for (let i = 0; i < binary.length; i++) {
|
|
314
|
+
bytes[i] = binary.charCodeAt(i);
|
|
542
315
|
}
|
|
543
|
-
return
|
|
544
|
-
}
|
|
545
|
-
function osqStagedQueryBytes(dimension, documentBits) {
|
|
546
|
-
if (documentBits === 8) return dimension;
|
|
547
|
-
if (documentBits === 4) return osqCodeBytes(dimension, 4);
|
|
548
|
-
return QUERY_PLANES_FOR_NARROW_CODES * osqCodeBytes(dimension, documentBits);
|
|
316
|
+
return bytes;
|
|
549
317
|
}
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
318
|
+
try {
|
|
319
|
+
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
320
|
+
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
321
|
+
if (typeof WebAssembly !== "undefined" && typeof WebAssembly.validate === "function" && WebAssembly.validate(wasmBuffer)) {
|
|
322
|
+
const module = new WebAssembly.Module(wasmBuffer);
|
|
323
|
+
const instance = new WebAssembly.Instance(module);
|
|
324
|
+
wasmExports = instance.exports;
|
|
325
|
+
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
554
326
|
}
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
327
|
+
} catch {
|
|
328
|
+
wasmExports = null;
|
|
329
|
+
f32View = null;
|
|
330
|
+
}
|
|
331
|
+
if (wasmExports === null) markSimdUnavailable();
|
|
332
|
+
var OFFSET_B_BYTES = 16384;
|
|
333
|
+
function ensureMemory(dim) {
|
|
334
|
+
if (!wasmExports || !f32View) return;
|
|
335
|
+
const needed = OFFSET_B_BYTES + dim * 4;
|
|
336
|
+
const currentSize = wasmExports.memory.buffer.byteLength;
|
|
337
|
+
if (needed > currentSize) {
|
|
338
|
+
const pagesNeeded = Math.ceil(needed / 65536);
|
|
339
|
+
const currentPages = currentSize / 65536;
|
|
340
|
+
wasmExports.memory.grow(pagesNeeded - currentPages);
|
|
341
|
+
f32View = new Float32Array(wasmExports.memory.buffer);
|
|
562
342
|
}
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
const bit = 1 << (i & 7);
|
|
568
|
-
for (let plane = 0; plane < QUERY_PLANES_FOR_NARROW_CODES; plane++) {
|
|
569
|
-
if (value >> plane & 1) target[plane * planeBytes + byte] |= bit;
|
|
570
|
-
}
|
|
343
|
+
}
|
|
344
|
+
function getF32View() {
|
|
345
|
+
if (!f32View) {
|
|
346
|
+
throw new Error("SIMD memory is not initialized");
|
|
571
347
|
}
|
|
348
|
+
return f32View;
|
|
572
349
|
}
|
|
573
|
-
function
|
|
574
|
-
|
|
575
|
-
const at = offset + osqCodeBytes(code.levels.length, bits);
|
|
576
|
-
trailer.setFloat32(at + TRAILER_LOWER, code.lower, true);
|
|
577
|
-
trailer.setFloat32(at + TRAILER_UPPER, code.upper, true);
|
|
578
|
-
trailer.setFloat32(at + TRAILER_CORRECTION, code.correction, true);
|
|
579
|
-
trailer.setUint32(at + TRAILER_SUM, code.sum, true);
|
|
350
|
+
function copyOneVector(a) {
|
|
351
|
+
getF32View().set(a, 0);
|
|
580
352
|
}
|
|
581
|
-
function
|
|
582
|
-
const
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
into.correction = trailer.getFloat32(at + TRAILER_CORRECTION, true);
|
|
586
|
-
into.sum = trailer.getUint32(at + TRAILER_SUM, true);
|
|
587
|
-
return into;
|
|
353
|
+
function copyTwoVectors(a, b) {
|
|
354
|
+
const view = getF32View();
|
|
355
|
+
view.set(a, 0);
|
|
356
|
+
view.set(b, OFFSET_B_BYTES / 4);
|
|
588
357
|
}
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
const queryBytes = osqStagedQueryBytes(dimension, bits);
|
|
595
|
-
const scratchStride = alignUp(queryBytes);
|
|
596
|
-
const slotsOffset = alignUp(VECTOR_SCRATCH_SLOTS * scratchStride);
|
|
597
|
-
const slotStride = osqRecordBytes(dimension, bits);
|
|
598
|
-
return {
|
|
599
|
-
dimension,
|
|
600
|
-
bits,
|
|
601
|
-
queryBits,
|
|
602
|
-
codeBytes,
|
|
603
|
-
queryBytes,
|
|
604
|
-
scratchStride,
|
|
605
|
-
slotsOffset,
|
|
606
|
-
slotStride,
|
|
607
|
-
capacity: Math.floor((BLOCK_MAX_BYTES - slotsOffset) / slotStride)
|
|
608
|
-
};
|
|
358
|
+
function simdDotProduct(a, b) {
|
|
359
|
+
if (!wasmExports || !f32View) return null;
|
|
360
|
+
ensureMemory(a.length);
|
|
361
|
+
copyTwoVectors(a, b);
|
|
362
|
+
return wasmExports.dot_product(0, OFFSET_B_BYTES, a.length);
|
|
609
363
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
var STORE_CALIBRATED = 3;
|
|
616
|
-
var STORE_CODE_COUNT = 4;
|
|
617
|
-
var STORE_DOC_ID_BYTES = 5;
|
|
618
|
-
var STORE_CODE_BLOCK_COUNT = 6;
|
|
619
|
-
var STORE_CALIBRATION_GENERATION = 7;
|
|
620
|
-
var STORE_RELEASED_VECTORS_TO_DISK = 8;
|
|
621
|
-
var STORE_UNPLACED_COUNT = 9;
|
|
622
|
-
var STORE_REMOVED_COUNT = 10;
|
|
623
|
-
var STORE_HEADER_WORDS = 16;
|
|
624
|
-
var IN_MEMORY = -1;
|
|
625
|
-
function fixedInt32(words) {
|
|
626
|
-
return new Int32Array(createFixedBuffer(words * 4), 0, words);
|
|
364
|
+
function simdMagnitude(a) {
|
|
365
|
+
if (!wasmExports || !f32View) return null;
|
|
366
|
+
ensureMemory(a.length);
|
|
367
|
+
copyOneVector(a);
|
|
368
|
+
return wasmExports.magnitude(0, a.length);
|
|
627
369
|
}
|
|
628
|
-
function
|
|
629
|
-
|
|
370
|
+
function simdSquaredEuclideanDistance(a, b) {
|
|
371
|
+
if (!wasmExports || !f32View) return null;
|
|
372
|
+
ensureMemory(a.length);
|
|
373
|
+
copyTwoVectors(a, b);
|
|
374
|
+
return wasmExports.squared_euclidean_distance(0, OFFSET_B_BYTES, a.length);
|
|
630
375
|
}
|
|
631
|
-
|
|
632
|
-
|
|
376
|
+
var arenaModule;
|
|
377
|
+
function getArenaModule() {
|
|
378
|
+
if (arenaModule !== void 0) return arenaModule;
|
|
379
|
+
try {
|
|
380
|
+
const bytes = decodeBase64(SIMD_DISTANCE_WASM_BASE64);
|
|
381
|
+
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
382
|
+
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
|
|
383
|
+
arenaModule = null;
|
|
384
|
+
return null;
|
|
385
|
+
}
|
|
386
|
+
arenaModule = new WebAssembly.Module(wasmBuffer);
|
|
387
|
+
} catch {
|
|
388
|
+
arenaModule = null;
|
|
389
|
+
}
|
|
390
|
+
return arenaModule;
|
|
633
391
|
}
|
|
634
|
-
function
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
(
|
|
638
|
-
|
|
392
|
+
function createArenaSimd() {
|
|
393
|
+
try {
|
|
394
|
+
const module = getArenaModule();
|
|
395
|
+
if (module === null) return null;
|
|
396
|
+
const instance = new WebAssembly.Instance(module);
|
|
397
|
+
const exports$1 = instance.exports;
|
|
398
|
+
if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.osq_dot_nibbles_4x4 !== "function" || typeof exports$1.osq_dot_bits !== "function" || typeof exports$1.osq_dot_bits_pair !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function" || !(exports$1.memory instanceof WebAssembly.Memory)) {
|
|
399
|
+
return null;
|
|
400
|
+
}
|
|
401
|
+
return exports$1;
|
|
402
|
+
} catch {
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
639
405
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
406
|
+
var sharedMemoryModule;
|
|
407
|
+
function getSharedMemoryModule() {
|
|
408
|
+
if (sharedMemoryModule !== void 0) return sharedMemoryModule;
|
|
409
|
+
try {
|
|
410
|
+
const bytes = decodeBase64(SIMD_DISTANCE_SHARED_MEMORY_WASM_BASE64);
|
|
411
|
+
const wasmBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
412
|
+
if (typeof WebAssembly === "undefined" || typeof WebAssembly.validate !== "function" || !WebAssembly.validate(wasmBuffer)) {
|
|
413
|
+
sharedMemoryModule = null;
|
|
414
|
+
return null;
|
|
415
|
+
}
|
|
416
|
+
sharedMemoryModule = new WebAssembly.Module(wasmBuffer);
|
|
417
|
+
} catch {
|
|
418
|
+
sharedMemoryModule = null;
|
|
419
|
+
}
|
|
420
|
+
return sharedMemoryModule;
|
|
644
421
|
}
|
|
645
|
-
function
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
};
|
|
422
|
+
function createSharedArenaSimd(memory) {
|
|
423
|
+
try {
|
|
424
|
+
const module = getSharedMemoryModule();
|
|
425
|
+
if (module === null) return null;
|
|
426
|
+
const instance = new WebAssembly.Instance(module, { env: { memory } });
|
|
427
|
+
const exports$1 = instance.exports;
|
|
428
|
+
if (typeof exports$1.dot_u8 !== "function" || typeof exports$1.sqdist_u8 !== "function" || typeof exports$1.osq_dot_nibbles_4x4 !== "function" || typeof exports$1.osq_dot_bits !== "function" || typeof exports$1.osq_dot_bits_pair !== "function" || typeof exports$1.dot_product !== "function" || typeof exports$1.magnitude !== "function" || typeof exports$1.squared_euclidean_distance !== "function") {
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
return {
|
|
432
|
+
memory,
|
|
433
|
+
dot_product: exports$1.dot_product,
|
|
434
|
+
magnitude: exports$1.magnitude,
|
|
435
|
+
squared_euclidean_distance: exports$1.squared_euclidean_distance,
|
|
436
|
+
dot_u8: exports$1.dot_u8,
|
|
437
|
+
sqdist_u8: exports$1.sqdist_u8,
|
|
438
|
+
osq_dot_nibbles_4x4: exports$1.osq_dot_nibbles_4x4,
|
|
439
|
+
osq_dot_bits: exports$1.osq_dot_bits,
|
|
440
|
+
osq_dot_bits_pair: exports$1.osq_dot_bits_pair
|
|
441
|
+
};
|
|
442
|
+
} catch {
|
|
443
|
+
return null;
|
|
444
|
+
}
|
|
669
445
|
}
|
|
670
|
-
function
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
if (block !== null) bytes += blockBuffer(block).byteLength;
|
|
446
|
+
function arenaFloat32Distance(simd, byteA, byteB, dimension, metric, magnitudeA, magnitudeB) {
|
|
447
|
+
if (metric === "euclidean") {
|
|
448
|
+
return Math.sqrt(simd.squared_euclidean_distance(byteA, byteB, dimension));
|
|
674
449
|
}
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
bytes += handles.unplacedChunks.byteLength;
|
|
680
|
-
bytes += handles.removed.byteLength;
|
|
681
|
-
bytes += handles.partitions.byteLength;
|
|
682
|
-
bytes += handles.docIdBytes.byteLength;
|
|
683
|
-
bytes += handles.docIdOffsets.byteLength;
|
|
684
|
-
bytes += handles.codePresent.byteLength;
|
|
685
|
-
bytes += handles.diskFile.byteLength;
|
|
686
|
-
bytes += handles.diskOffset.byteLength;
|
|
687
|
-
return bytes;
|
|
450
|
+
const dot = simd.dot_product(byteA, byteB, dimension);
|
|
451
|
+
if (metric === "dotProduct") return -dot;
|
|
452
|
+
if (magnitudeA === 0 || magnitudeB === 0) return 1;
|
|
453
|
+
return 1 - dot / (magnitudeA * magnitudeB);
|
|
688
454
|
}
|
|
689
455
|
|
|
690
|
-
// src/vector/
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
return
|
|
456
|
+
// src/vector/similarity.ts
|
|
457
|
+
function magnitude(v) {
|
|
458
|
+
const result = simdMagnitude(v);
|
|
459
|
+
if (result !== null) return result;
|
|
460
|
+
let sum = 0;
|
|
461
|
+
for (let i = 0; i < v.length; i++) {
|
|
462
|
+
sum += v[i] * v[i];
|
|
463
|
+
}
|
|
464
|
+
return Math.sqrt(sum);
|
|
699
465
|
}
|
|
700
|
-
function
|
|
701
|
-
|
|
466
|
+
function cosineSimilarityWithMagnitudes(a, b, magA, magB) {
|
|
467
|
+
if (magA === 0 || magB === 0) return 0;
|
|
468
|
+
return dotProduct(a, b) / (magA * magB);
|
|
702
469
|
}
|
|
703
|
-
function
|
|
704
|
-
const
|
|
705
|
-
if (
|
|
706
|
-
|
|
707
|
-
|
|
470
|
+
function dotProduct(a, b) {
|
|
471
|
+
const result = simdDotProduct(a, b);
|
|
472
|
+
if (result !== null) return result;
|
|
473
|
+
let sum = 0;
|
|
474
|
+
for (let i = 0; i < a.length; i++) {
|
|
475
|
+
sum += a[i] * b[i];
|
|
708
476
|
}
|
|
709
|
-
return
|
|
477
|
+
return sum;
|
|
710
478
|
}
|
|
711
|
-
function
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
if (!regions.every(isSharedMemory)) return null;
|
|
722
|
-
const codeBlocks = [];
|
|
723
|
-
for (const block of store.codeBlocks) {
|
|
724
|
-
const bytes = blockBytes(block);
|
|
725
|
-
if (bytes === null) return null;
|
|
726
|
-
codeBlocks.push(bytes);
|
|
727
|
-
}
|
|
728
|
-
const vectorBlocks = [];
|
|
729
|
-
for (const block of store.blocks) {
|
|
730
|
-
const bytes = block === null ? null : blockBytes(block);
|
|
731
|
-
if (block !== null && bytes === null) return null;
|
|
732
|
-
vectorBlocks.push(bytes);
|
|
479
|
+
function euclideanDistance(a, b) {
|
|
480
|
+
return Math.sqrt(squaredEuclideanDistance(a, b));
|
|
481
|
+
}
|
|
482
|
+
function squaredEuclideanDistance(a, b) {
|
|
483
|
+
const result = simdSquaredEuclideanDistance(a, b);
|
|
484
|
+
if (result !== null) return result;
|
|
485
|
+
let sum = 0;
|
|
486
|
+
for (let i = 0; i < a.length; i++) {
|
|
487
|
+
const diff = a[i] - b[i];
|
|
488
|
+
sum += diff * diff;
|
|
733
489
|
}
|
|
490
|
+
return sum;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// src/vector/hnsw/adjacency/views.ts
|
|
494
|
+
var ABSENT = -1;
|
|
495
|
+
function openAdjacency(handles, m, mMax0) {
|
|
734
496
|
return {
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
vectorOffset: fixedView(store.diskOffset, Uint32Array),
|
|
747
|
-
vectorFiles: [...store.vectorFiles],
|
|
748
|
-
codeBlocks,
|
|
749
|
-
vectorBlocks
|
|
497
|
+
m,
|
|
498
|
+
mMax0,
|
|
499
|
+
level0Stride: mMax0 + 2,
|
|
500
|
+
upperStride: m + 2,
|
|
501
|
+
handles,
|
|
502
|
+
header: handles.header,
|
|
503
|
+
nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
|
|
504
|
+
level0: fixedView(handles.level0, Int32Array),
|
|
505
|
+
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
506
|
+
upper: fixedView(handles.upper, Int32Array),
|
|
507
|
+
incoming: fixedView(handles.incoming, Int32Array)
|
|
750
508
|
};
|
|
751
509
|
}
|
|
752
|
-
function
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
510
|
+
function rebindNodes(adj) {
|
|
511
|
+
adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
|
|
512
|
+
adj.incoming = fixedView(adj.handles.incoming, Int32Array);
|
|
513
|
+
adj.level0 = fixedView(adj.handles.level0, Int32Array);
|
|
514
|
+
adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
|
|
515
|
+
}
|
|
516
|
+
function rebindUpper(adj) {
|
|
517
|
+
adj.upper = fixedView(adj.handles.upper, Int32Array);
|
|
518
|
+
}
|
|
519
|
+
function reachNode(adj, ord) {
|
|
520
|
+
if (ord < adj.nodeLevels.length) return true;
|
|
521
|
+
if (ord >= adj.handles.nodeLevels.byteLength) return false;
|
|
522
|
+
rebindNodes(adj);
|
|
523
|
+
return ord < adj.nodeLevels.length;
|
|
524
|
+
}
|
|
525
|
+
function reachUpper(adj, end) {
|
|
526
|
+
if (end <= adj.upper.length) return true;
|
|
527
|
+
if (end * 4 > adj.handles.upper.byteLength) return false;
|
|
528
|
+
rebindUpper(adj);
|
|
529
|
+
return end <= adj.upper.length;
|
|
530
|
+
}
|
|
531
|
+
function adjacencySlots(adj) {
|
|
532
|
+
return Atomics.load(adj.header, GRAPH_SLOTS);
|
|
533
|
+
}
|
|
534
|
+
function upperUsed(adj) {
|
|
535
|
+
return Atomics.load(adj.header, GRAPH_UPPER_USED);
|
|
536
|
+
}
|
|
537
|
+
function ensureAdjacencyCapacity(adj, needed) {
|
|
538
|
+
if (needed <= adj.nodeLevels.length) return;
|
|
539
|
+
const { handles } = adj;
|
|
540
|
+
if (needed <= handles.nodeLevels.byteLength) {
|
|
541
|
+
rebindNodes(adj);
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
const capacity = Math.max(needed, Math.min(handles.nodeLevels.byteLength * 2, handles.nodeLevels.maxByteLength));
|
|
545
|
+
growBufferToExactly(handles.level0, capacity * adj.level0Stride * 4);
|
|
546
|
+
growBufferToExactly(handles.upperBase, capacity * 4);
|
|
547
|
+
growBufferToExactly(handles.locks, capacity * 4);
|
|
548
|
+
growBufferToExactly(handles.tombstones, capacity);
|
|
549
|
+
growBufferToExactly(handles.incoming, capacity * 4);
|
|
550
|
+
growBufferToExactly(handles.nodeLevels, capacity);
|
|
551
|
+
publishGraphCapacity(handles);
|
|
552
|
+
rebindNodes(adj);
|
|
553
|
+
}
|
|
554
|
+
function growUpperTo(adj, entries) {
|
|
555
|
+
const held = adj.handles.upper.byteLength;
|
|
556
|
+
growBufferTo(adj.handles.upper, entries * 4);
|
|
557
|
+
if (adj.handles.upper.byteLength !== held) publishGraphCapacity(adj.handles);
|
|
558
|
+
}
|
|
559
|
+
function ensureUpperCapacity(adj, entries) {
|
|
560
|
+
growUpperTo(adj, entries);
|
|
561
|
+
rebindUpper(adj);
|
|
562
|
+
}
|
|
563
|
+
function ensureUpperRoom(adj, levels) {
|
|
564
|
+
if (levels < 1) return;
|
|
565
|
+
growUpperTo(adj, Atomics.load(adj.header, GRAPH_UPPER_USED) + levels * adj.upperStride);
|
|
566
|
+
}
|
|
567
|
+
function allocateUpperBlock(adj, levels) {
|
|
568
|
+
const size = levels * adj.upperStride;
|
|
569
|
+
for (; ; ) {
|
|
570
|
+
const offset = Atomics.load(adj.header, GRAPH_UPPER_USED);
|
|
571
|
+
growUpperTo(adj, offset + size);
|
|
572
|
+
if (Atomics.compareExchange(adj.header, GRAPH_UPPER_USED, offset, offset + size) !== offset) continue;
|
|
573
|
+
rebindUpper(adj);
|
|
574
|
+
return offset;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
function raiseSlots(adj, slots) {
|
|
578
|
+
for (; ; ) {
|
|
579
|
+
const seen = Atomics.load(adj.header, GRAPH_SLOTS);
|
|
580
|
+
if (seen >= slots) return;
|
|
581
|
+
if (Atomics.compareExchange(adj.header, GRAPH_SLOTS, seen, slots) === seen) return;
|
|
761
582
|
}
|
|
762
|
-
return {
|
|
763
|
-
core,
|
|
764
|
-
handle,
|
|
765
|
-
layoutRevision: store.layoutRevision,
|
|
766
|
-
vectorBlockCount: store.blocks.length,
|
|
767
|
-
codeBlockCount: store.codeBlocks.length,
|
|
768
|
-
retryAt: handle === null ? Date.now() + NATIVE_ATTACH_RETRY_MS : Number.POSITIVE_INFINITY,
|
|
769
|
-
ordinals: new Int32Array(0),
|
|
770
|
-
distances: new Float64Array(0)
|
|
771
|
-
};
|
|
772
583
|
}
|
|
773
|
-
function
|
|
774
|
-
|
|
584
|
+
function createNode(adj, ord, maxLayer) {
|
|
585
|
+
ensureAdjacencyCapacity(adj, ord + 1);
|
|
586
|
+
adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) + 1 : 0;
|
|
587
|
+
adj.level0[ord * adj.level0Stride] = 0;
|
|
588
|
+
raiseSlots(adj, ord + 1);
|
|
589
|
+
Atomics.store(adj.nodeLevels, ord, maxLayer + 1);
|
|
775
590
|
}
|
|
776
|
-
function
|
|
777
|
-
|
|
591
|
+
function deleteNode(adj, ord) {
|
|
592
|
+
if (ord < 0 || !reachNode(adj, ord)) return;
|
|
593
|
+
Atomics.store(adj.nodeLevels, ord, 0);
|
|
594
|
+
adj.upperBase[ord] = 0;
|
|
595
|
+
adj.level0[ord * adj.level0Stride] = 0;
|
|
596
|
+
Atomics.store(adj.incoming, ord, 0);
|
|
778
597
|
}
|
|
779
|
-
function
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
598
|
+
function resetAdjacency(adj) {
|
|
599
|
+
rebindNodes(adj);
|
|
600
|
+
rebindUpper(adj);
|
|
601
|
+
adj.incoming.fill(0);
|
|
602
|
+
adj.nodeLevels.fill(0);
|
|
603
|
+
adj.level0.fill(0);
|
|
604
|
+
adj.upperBase.fill(0);
|
|
605
|
+
adj.upper.fill(0);
|
|
606
|
+
Atomics.store(adj.header, GRAPH_UPPER_USED, 0);
|
|
607
|
+
Atomics.store(adj.header, GRAPH_SLOTS, 0);
|
|
789
608
|
}
|
|
790
|
-
function
|
|
791
|
-
|
|
792
|
-
return
|
|
609
|
+
function nodeLevel(adj, ord) {
|
|
610
|
+
if (ord < 0 || !reachNode(adj, ord)) return ABSENT;
|
|
611
|
+
return adj.nodeLevels[ord] - 1;
|
|
793
612
|
}
|
|
794
|
-
function
|
|
795
|
-
|
|
796
|
-
return attached === null ? 0 : attached.ordinals.byteLength + attached.distances.byteLength;
|
|
613
|
+
function hasNode(adj, ord) {
|
|
614
|
+
return nodeLevel(adj, ord) !== ABSENT;
|
|
797
615
|
}
|
|
798
|
-
function
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
616
|
+
function graphBytes(adj) {
|
|
617
|
+
const handles = adj.handles;
|
|
618
|
+
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// src/vector/hnsw/adjacency/incoming.ts
|
|
622
|
+
function incomingCount(adj, ord) {
|
|
623
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
624
|
+
return Atomics.load(adj.incoming, ord);
|
|
625
|
+
}
|
|
626
|
+
function noteLinked(adj, ord) {
|
|
627
|
+
if (ord < 0 || !reachNode(adj, ord)) return;
|
|
628
|
+
Atomics.add(adj.incoming, ord, 1);
|
|
629
|
+
}
|
|
630
|
+
function noteUnlinked(adj, ord) {
|
|
631
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
632
|
+
for (; ; ) {
|
|
633
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
634
|
+
if (seen <= 0) return 0;
|
|
635
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return seen - 1;
|
|
806
636
|
}
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
637
|
+
}
|
|
638
|
+
function releaseUnlessLast(adj, ord) {
|
|
639
|
+
if (ord < 0 || !reachNode(adj, ord)) return true;
|
|
640
|
+
for (; ; ) {
|
|
641
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
642
|
+
if (seen <= 1) return false;
|
|
643
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return true;
|
|
810
644
|
}
|
|
811
|
-
return isUsable(attached) ? attached : null;
|
|
812
645
|
}
|
|
813
|
-
function
|
|
814
|
-
|
|
815
|
-
store.ordinals = new Int32Array(count);
|
|
816
|
-
store.distances = new Float64Array(count);
|
|
646
|
+
function holdsNodeAt(adj, ord) {
|
|
647
|
+
return ord >= 0 && ord < adj.nodeLevels.length && adj.nodeLevels[ord] !== 0;
|
|
817
648
|
}
|
|
818
|
-
function
|
|
819
|
-
if (
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
649
|
+
function countLinksOutOf(adj, ord, step) {
|
|
650
|
+
if (!holdsNodeAt(adj, ord)) return;
|
|
651
|
+
const incoming = adj.incoming;
|
|
652
|
+
const level0 = adj.level0;
|
|
653
|
+
const base = ord * adj.level0Stride;
|
|
654
|
+
const count = level0[base];
|
|
655
|
+
for (let i = 1; i <= count; i++) {
|
|
656
|
+
const neighbour = level0[base + i];
|
|
657
|
+
if (neighbour === ord || !holdsNodeAt(adj, neighbour)) continue;
|
|
658
|
+
if (incoming[neighbour] + step >= 0) incoming[neighbour] += step;
|
|
826
659
|
}
|
|
827
|
-
return status === NATIVE_OK ? store.distances : null;
|
|
828
660
|
}
|
|
829
|
-
function
|
|
830
|
-
|
|
831
|
-
if (store === null) return null;
|
|
832
|
-
reserveOrdinals(store, ordinals.length);
|
|
833
|
-
store.ordinals.set(ordinals);
|
|
834
|
-
return nativeScores(store, query, metric, ordinals.length);
|
|
661
|
+
function countEveryLink(adj, slots) {
|
|
662
|
+
for (let ord = 0; ord < slots; ord++) countLinksOutOf(adj, ord, 1);
|
|
835
663
|
}
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
if (store === null) return false;
|
|
841
|
-
let status;
|
|
842
|
-
try {
|
|
843
|
-
status = store.core[work](store.handle, nativeMetricCode(metric), ordinals, ordinals.length);
|
|
844
|
-
} catch (error) {
|
|
845
|
-
stopUsingTheNativeSearchCore(error);
|
|
846
|
-
return false;
|
|
847
|
-
}
|
|
848
|
-
if (status === NATIVE_OK) return true;
|
|
849
|
-
if (status !== NATIVE_NEEDS_ROOM) return false;
|
|
850
|
-
detachNativeStore(handles);
|
|
851
|
-
}
|
|
852
|
-
return false;
|
|
664
|
+
|
|
665
|
+
// src/vector/hnsw/adjacency/neighbors.ts
|
|
666
|
+
function layerArray(adj, layer) {
|
|
667
|
+
return layer === 0 ? adj.level0 : adj.upper;
|
|
853
668
|
}
|
|
854
|
-
function
|
|
855
|
-
|
|
669
|
+
function layerBase(adj, ord, layer) {
|
|
670
|
+
const level2 = nodeLevel(adj, ord);
|
|
671
|
+
if (level2 === ABSENT || layer > level2) return ABSENT;
|
|
672
|
+
if (layer === 0) return ord * adj.level0Stride;
|
|
673
|
+
const base = adj.upperBase[ord];
|
|
674
|
+
if (base === 0) return ABSENT;
|
|
675
|
+
const start = base - 1 + (layer - 1) * adj.upperStride;
|
|
676
|
+
return reachUpper(adj, start + adj.upperStride) ? start : ABSENT;
|
|
856
677
|
}
|
|
857
|
-
function
|
|
858
|
-
|
|
859
|
-
|
|
678
|
+
function neighborCount(adj, ord, layer) {
|
|
679
|
+
const base = layerBase(adj, ord, layer);
|
|
680
|
+
if (base === ABSENT) return 0;
|
|
681
|
+
return layerArray(adj, layer)[base];
|
|
860
682
|
}
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
function graphMemory(graph) {
|
|
869
|
-
const regions = [
|
|
870
|
-
graph.header.buffer,
|
|
871
|
-
graph.nodeLevels,
|
|
872
|
-
graph.level0,
|
|
873
|
-
graph.upperBase,
|
|
874
|
-
graph.upper,
|
|
875
|
-
graph.locks,
|
|
876
|
-
graph.tombstones,
|
|
877
|
-
graph.incoming,
|
|
878
|
-
graph.heldLocks.buffer
|
|
879
|
-
];
|
|
880
|
-
if (!regions.every(isSharedMemory)) return null;
|
|
881
|
-
return {
|
|
882
|
-
graphHeader: graph.header,
|
|
883
|
-
nodeLevels: fixedView(graph.nodeLevels, Uint8Array),
|
|
884
|
-
level0: fixedView(graph.level0, Int32Array),
|
|
885
|
-
upperBase: fixedView(graph.upperBase, Int32Array),
|
|
886
|
-
upper: fixedView(graph.upper, Int32Array),
|
|
887
|
-
locks: fixedView(graph.locks, Int32Array),
|
|
888
|
-
tombstones: fixedView(graph.tombstones, Uint8Array),
|
|
889
|
-
incoming: fixedView(graph.incoming, Int32Array),
|
|
890
|
-
heldLocks: graph.heldLocks
|
|
891
|
-
};
|
|
683
|
+
function readNeighbors(adj, ord, layer, out) {
|
|
684
|
+
const base = layerBase(adj, ord, layer);
|
|
685
|
+
if (base === ABSENT) return 0;
|
|
686
|
+
const array = layerArray(adj, layer);
|
|
687
|
+
const count = Math.min(array[base], out.length);
|
|
688
|
+
for (let i = 0; i < count; i++) out[i] = array[base + i + 1];
|
|
689
|
+
return count;
|
|
892
690
|
}
|
|
893
|
-
function
|
|
894
|
-
const
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
691
|
+
function collectNeighbors(adj, ord, layer) {
|
|
692
|
+
const base = layerBase(adj, ord, layer);
|
|
693
|
+
if (base === ABSENT) return [];
|
|
694
|
+
const array = layerArray(adj, layer);
|
|
695
|
+
const count = array[base];
|
|
696
|
+
const collected = new Array(count);
|
|
697
|
+
for (let i = 0; i < count; i++) collected[i] = array[base + i + 1];
|
|
698
|
+
return collected;
|
|
699
|
+
}
|
|
700
|
+
function addNeighbor(adj, ord, layer, neighborOrd) {
|
|
701
|
+
const base = layerBase(adj, ord, layer);
|
|
702
|
+
if (base === ABSENT) return;
|
|
703
|
+
const array = layerArray(adj, layer);
|
|
704
|
+
const count = array[base];
|
|
705
|
+
for (let i = 1; i <= count; i++) {
|
|
706
|
+
if (array[base + i] === neighborOrd) return;
|
|
707
|
+
}
|
|
708
|
+
const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
|
|
709
|
+
if (count + 1 > stride - 1) return;
|
|
710
|
+
array[base + count + 1] = neighborOrd;
|
|
711
|
+
array[base] = count + 1;
|
|
712
|
+
if (layer === 0) noteLinked(adj, neighborOrd);
|
|
713
|
+
}
|
|
714
|
+
function removeNeighbor(adj, ord, layer, neighborOrd) {
|
|
715
|
+
const base = layerBase(adj, ord, layer);
|
|
716
|
+
if (base === ABSENT) return;
|
|
717
|
+
const array = layerArray(adj, layer);
|
|
718
|
+
const count = array[base];
|
|
719
|
+
for (let i = 1; i <= count; i++) {
|
|
720
|
+
if (array[base + i] !== neighborOrd) continue;
|
|
721
|
+
for (let j = i; j < count; j++) {
|
|
722
|
+
array[base + j] = array[base + j + 1];
|
|
901
723
|
}
|
|
724
|
+
array[base] = count - 1;
|
|
725
|
+
if (layer === 0) noteUnlinked(adj, neighborOrd);
|
|
726
|
+
return;
|
|
902
727
|
}
|
|
903
|
-
return { handle, field: null };
|
|
904
|
-
}
|
|
905
|
-
var workspaceOwner = null;
|
|
906
|
-
function nativeScratchBytes(state, field) {
|
|
907
|
-
const perField = field.store.ordinals.byteLength + field.store.distances.byteLength + field.wakes.byteLength;
|
|
908
|
-
const owner = workspaceOwner?.deref();
|
|
909
|
-
if (owner !== void 0 && owner !== state) return perField;
|
|
910
|
-
workspaceOwner = new WeakRef(state);
|
|
911
|
-
return perField + field.core.workspaceBytes();
|
|
912
|
-
}
|
|
913
|
-
function attachedNativeField(state) {
|
|
914
|
-
if (!nativeSearchCoreIsEnabled()) return null;
|
|
915
|
-
return attachedGraphs.get(state)?.field ?? null;
|
|
916
728
|
}
|
|
917
|
-
function
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
const
|
|
922
|
-
|
|
923
|
-
let
|
|
924
|
-
|
|
925
|
-
attached = attach2(core, state);
|
|
926
|
-
attachedGraphs.set(state, attached);
|
|
927
|
-
}
|
|
928
|
-
if (attached.handle === null) return null;
|
|
929
|
-
if (attached.field === null) {
|
|
930
|
-
attached.field = { core, graph: attached.handle, store, wakes: new Int32Array(WAKE_LIST_WORDS) };
|
|
729
|
+
function writeNeighbors(adj, ord, layer, neighbors, count) {
|
|
730
|
+
const base = layerBase(adj, ord, layer);
|
|
731
|
+
if (base === ABSENT) return;
|
|
732
|
+
const array = layerArray(adj, layer);
|
|
733
|
+
const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
|
|
734
|
+
const limit = Math.min(count, stride - 1);
|
|
735
|
+
for (let i = 0; i < limit; i++) {
|
|
736
|
+
array[base + i + 1] = neighbors[i];
|
|
931
737
|
}
|
|
932
|
-
|
|
933
|
-
return attached.field;
|
|
738
|
+
array[base] = limit;
|
|
934
739
|
}
|
|
935
|
-
function
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
if (ordinal < 0 || ordinal >= Atomics.load(state.header, GRAPH_SLOTS)) continue;
|
|
946
|
-
Atomics.notify(state.locks.words(ordinal), ordinal);
|
|
740
|
+
function holds(neighbors, count, ord) {
|
|
741
|
+
for (let i = 0; i < count; i++) if (neighbors[i] === ord) return true;
|
|
742
|
+
return false;
|
|
743
|
+
}
|
|
744
|
+
function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
745
|
+
const base = layerBase(adj, ord, layer);
|
|
746
|
+
if (base === ABSENT) return;
|
|
747
|
+
if (layer > 0) {
|
|
748
|
+
writeNeighbors(adj, ord, layer, neighbors, count);
|
|
749
|
+
return;
|
|
947
750
|
}
|
|
751
|
+
const array = adj.level0;
|
|
752
|
+
const held = array.slice(base + 1, base + 1 + array[base]);
|
|
753
|
+
const limit = Math.min(count, adj.level0Stride - 1);
|
|
754
|
+
for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
|
|
755
|
+
for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
|
|
756
|
+
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
948
757
|
}
|
|
949
758
|
|
|
950
|
-
// src/vector/
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
759
|
+
// src/vector/hnsw/shared.ts
|
|
760
|
+
function entryPointOf(state) {
|
|
761
|
+
return Atomics.load(state.header, GRAPH_ENTRY_POINT);
|
|
762
|
+
}
|
|
763
|
+
function topLayerOf(state) {
|
|
764
|
+
return Atomics.load(state.header, GRAPH_TOP_LAYER);
|
|
765
|
+
}
|
|
766
|
+
function nodeCountOf(state) {
|
|
767
|
+
return Atomics.load(state.header, GRAPH_NODE_COUNT);
|
|
768
|
+
}
|
|
769
|
+
function tombstoneCountOf(state) {
|
|
770
|
+
return Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT);
|
|
771
|
+
}
|
|
772
|
+
function ensureCapacity(state, needed) {
|
|
773
|
+
ensureAdjacencyCapacity(state.adjacency, needed);
|
|
774
|
+
ensureVisited(state, needed);
|
|
775
|
+
}
|
|
776
|
+
function ensureVisited(state, needed) {
|
|
777
|
+
if (needed <= state.visited.length) return;
|
|
778
|
+
let capacity = state.visited.length === 0 ? 16 : state.visited.length;
|
|
779
|
+
while (capacity < needed) capacity *= 2;
|
|
780
|
+
const next = new Uint32Array(capacity);
|
|
781
|
+
next.set(state.visited);
|
|
782
|
+
state.visited = next;
|
|
783
|
+
}
|
|
784
|
+
function nextVisitStamp(state) {
|
|
785
|
+
state.visitStamp++;
|
|
786
|
+
if (state.visitStamp === 4294967295) {
|
|
787
|
+
state.visited.fill(0);
|
|
788
|
+
state.visitStamp = 1;
|
|
959
789
|
}
|
|
960
|
-
|
|
961
|
-
return status;
|
|
790
|
+
return state.visitStamp;
|
|
962
791
|
}
|
|
963
|
-
function
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
field.graph,
|
|
969
|
-
field.store.handle,
|
|
970
|
-
nativeMetricCode(metric),
|
|
971
|
-
ordinal,
|
|
972
|
-
topLayer,
|
|
973
|
-
graphHeldAlone ? 1 : 0,
|
|
974
|
-
state.locks.threadSlot,
|
|
975
|
-
field.wakes
|
|
976
|
-
)
|
|
977
|
-
);
|
|
978
|
-
if (status === NATIVE_OK) return "placed";
|
|
979
|
-
return status === NATIVE_NEEDS_ROOM ? "needsRoom" : "failed";
|
|
792
|
+
function reachTombstone(state, ord) {
|
|
793
|
+
if (ord < state.tombstones.length) return true;
|
|
794
|
+
if (ord >= state.adjacency.handles.tombstones.byteLength) return false;
|
|
795
|
+
state.tombstones = fixedView(state.adjacency.handles.tombstones, Uint8Array);
|
|
796
|
+
return ord < state.tombstones.length;
|
|
980
797
|
}
|
|
981
|
-
function
|
|
982
|
-
|
|
983
|
-
return writeToTheGraph(state, field, remove) === NATIVE_OK;
|
|
798
|
+
function isTombstoned(state, ord) {
|
|
799
|
+
return ord >= 0 && reachTombstone(state, ord) && state.tombstones[ord] === 1;
|
|
984
800
|
}
|
|
985
|
-
function
|
|
986
|
-
|
|
987
|
-
return writeToTheGraph(state, field, compact) === NATIVE_OK;
|
|
801
|
+
function nodeExists(state, ord) {
|
|
802
|
+
return hasNode(state.adjacency, ord);
|
|
988
803
|
}
|
|
989
|
-
function
|
|
990
|
-
|
|
991
|
-
field.graph,
|
|
992
|
-
field.store.handle,
|
|
993
|
-
nativeMetricCode(metric),
|
|
994
|
-
nearby,
|
|
995
|
-
state.locks.threadSlot,
|
|
996
|
-
field.wakes
|
|
997
|
-
);
|
|
998
|
-
return writeToTheGraph(state, field, connect) === NATIVE_OK;
|
|
804
|
+
function nodeMaxLayer(state, ord) {
|
|
805
|
+
return nodeLevel(state.adjacency, ord);
|
|
999
806
|
}
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
handles,
|
|
1010
|
-
header: handles.header,
|
|
1011
|
-
nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
|
|
1012
|
-
level0: fixedView(handles.level0, Int32Array),
|
|
1013
|
-
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
1014
|
-
upper: fixedView(handles.upper, Int32Array),
|
|
1015
|
-
incoming: fixedView(handles.incoming, Int32Array)
|
|
1016
|
-
};
|
|
807
|
+
function toDistance(a, b, magA, magB, metric) {
|
|
808
|
+
switch (metric) {
|
|
809
|
+
case "cosine":
|
|
810
|
+
return 1 - cosineSimilarityWithMagnitudes(a, b, magA, magB);
|
|
811
|
+
case "dotProduct":
|
|
812
|
+
return -dotProduct(a, b);
|
|
813
|
+
case "euclidean":
|
|
814
|
+
return euclideanDistance(a, b);
|
|
815
|
+
}
|
|
1017
816
|
}
|
|
1018
|
-
function
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
817
|
+
function toScore(distance, metric) {
|
|
818
|
+
switch (metric) {
|
|
819
|
+
case "cosine":
|
|
820
|
+
return 1 - distance;
|
|
821
|
+
case "dotProduct":
|
|
822
|
+
return -distance;
|
|
823
|
+
case "euclidean":
|
|
824
|
+
return 1 / (1 + distance);
|
|
825
|
+
}
|
|
1023
826
|
}
|
|
1024
|
-
function
|
|
1025
|
-
|
|
827
|
+
function randomLevel(mL) {
|
|
828
|
+
let u = Math.random();
|
|
829
|
+
if (u === 0) u = Number.MIN_VALUE;
|
|
830
|
+
return Math.min(Math.floor(-Math.log(u) * mL), MAX_LAYER_CAP);
|
|
1026
831
|
}
|
|
1027
|
-
function
|
|
1028
|
-
|
|
1029
|
-
if (ord >= adj.handles.nodeLevels.byteLength) return false;
|
|
1030
|
-
rebindNodes(adj);
|
|
1031
|
-
return ord < adj.nodeLevels.length;
|
|
832
|
+
function entryForOrd(state, ord) {
|
|
833
|
+
return state.store.entryForOrdinal(ord);
|
|
1032
834
|
}
|
|
1033
|
-
function
|
|
1034
|
-
|
|
1035
|
-
if (end * 4 > adj.handles.upper.byteLength) return false;
|
|
1036
|
-
rebindUpper(adj);
|
|
1037
|
-
return end <= adj.upper.length;
|
|
835
|
+
function buildsFromCodes(state) {
|
|
836
|
+
return activeQuantizer(state) !== void 0;
|
|
1038
837
|
}
|
|
1039
|
-
function
|
|
1040
|
-
|
|
838
|
+
function activeQuantizer(state) {
|
|
839
|
+
const quantizer = state.quantizer;
|
|
840
|
+
if (quantizer === void 0 || !quantizer.isCalibrated()) return void 0;
|
|
841
|
+
return quantizer;
|
|
1041
842
|
}
|
|
1042
|
-
function
|
|
1043
|
-
|
|
843
|
+
function nodeDistanceByOrd(state, aOrd, bOrd, metric) {
|
|
844
|
+
if (buildsFromCodes(state) && state.quantizer !== void 0) {
|
|
845
|
+
const estimated = state.quantizer.distanceBetweenOrdinals(aOrd, bOrd);
|
|
846
|
+
if (estimated !== Number.POSITIVE_INFINITY) return estimated;
|
|
847
|
+
}
|
|
848
|
+
return state.store.distanceByOrdinal(aOrd, bOrd, metric);
|
|
849
|
+
}
|
|
850
|
+
function nodeDistanceFunction(state, metric) {
|
|
851
|
+
const storeDistance = state.store.pairDistance(metric);
|
|
852
|
+
const quantizer = state.quantizer;
|
|
853
|
+
if (quantizer === void 0) return storeDistance;
|
|
854
|
+
const codeDistance = quantizer.pairDistance();
|
|
855
|
+
return (aOrd, bOrd) => {
|
|
856
|
+
if (buildsFromCodes(state)) {
|
|
857
|
+
const estimated = codeDistance(aOrd, bOrd);
|
|
858
|
+
if (estimated !== Number.POSITIVE_INFINITY) return estimated;
|
|
859
|
+
}
|
|
860
|
+
return storeDistance(aOrd, bOrd);
|
|
861
|
+
};
|
|
862
|
+
}
|
|
863
|
+
function queryDistanceByOrd(state, qVec, qMag, ord, metric) {
|
|
864
|
+
const entry = state.store.entryForOrdinal(ord);
|
|
865
|
+
if (!entry) return Number.POSITIVE_INFINITY;
|
|
866
|
+
return toDistance(qVec, entry.vector, qMag, entry.magnitude, metric);
|
|
867
|
+
}
|
|
868
|
+
function maxConns(state, layer) {
|
|
869
|
+
return layer === 0 ? state.Mmax0 : state.M;
|
|
1044
870
|
}
|
|
1045
|
-
function
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
if (needed <= handles.nodeLevels.byteLength) {
|
|
1049
|
-
rebindNodes(adj);
|
|
1050
|
-
return;
|
|
871
|
+
function addConnection(connections, ord) {
|
|
872
|
+
for (let i = 0; i < connections.length; i++) {
|
|
873
|
+
if (connections[i] === ord) return;
|
|
1051
874
|
}
|
|
1052
|
-
|
|
1053
|
-
growBufferToExactly(handles.level0, capacity * adj.level0Stride * 4);
|
|
1054
|
-
growBufferToExactly(handles.upperBase, capacity * 4);
|
|
1055
|
-
growBufferToExactly(handles.locks, capacity * 4);
|
|
1056
|
-
growBufferToExactly(handles.tombstones, capacity);
|
|
1057
|
-
growBufferToExactly(handles.incoming, capacity * 4);
|
|
1058
|
-
growBufferToExactly(handles.nodeLevels, capacity);
|
|
1059
|
-
publishGraphCapacity(handles);
|
|
1060
|
-
rebindNodes(adj);
|
|
875
|
+
connections.push(ord);
|
|
1061
876
|
}
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
877
|
+
|
|
878
|
+
// src/vector/vector-store/blocks.ts
|
|
879
|
+
var SCRATCH_ALIGNMENT_BYTES = 16;
|
|
880
|
+
var BLOCK_MAX_BYTES = MAX_WASM_PAGES * WASM_PAGE_BYTES;
|
|
881
|
+
function alignUp(bytes) {
|
|
882
|
+
return Math.ceil(bytes / SCRATCH_ALIGNMENT_BYTES) * SCRATCH_ALIGNMENT_BYTES;
|
|
1066
883
|
}
|
|
1067
|
-
function
|
|
1068
|
-
|
|
1069
|
-
|
|
884
|
+
function computeVectorBlockLayout(dimension, blockBytes2 = BLOCK_MAX_BYTES) {
|
|
885
|
+
const scratchStride = alignUp(dimension * 4);
|
|
886
|
+
const slotsOffset = alignUp(VECTOR_SCRATCH_SLOTS * scratchStride);
|
|
887
|
+
const slotStride = alignUp(dimension * 4);
|
|
888
|
+
return {
|
|
889
|
+
dimension,
|
|
890
|
+
scratchStride,
|
|
891
|
+
slotsOffset,
|
|
892
|
+
slotStride,
|
|
893
|
+
capacity: Math.max(1, Math.floor((Math.min(blockBytes2, BLOCK_MAX_BYTES) - slotsOffset) / slotStride))
|
|
894
|
+
};
|
|
1070
895
|
}
|
|
1071
|
-
function
|
|
1072
|
-
|
|
1073
|
-
growUpperTo(adj, Atomics.load(adj.header, GRAPH_UPPER_USED) + levels * adj.upperStride);
|
|
896
|
+
function initialBlockBytes(layout) {
|
|
897
|
+
return layout.slotsOffset + VECTOR_BLOCK_INITIAL_SLOTS * layout.slotStride;
|
|
1074
898
|
}
|
|
1075
|
-
function
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
return
|
|
899
|
+
function pagesFor(bytes) {
|
|
900
|
+
return Math.ceil(bytes / WASM_PAGE_BYTES);
|
|
901
|
+
}
|
|
902
|
+
function createSharedMemory(pages) {
|
|
903
|
+
if (!sharedMemoryAvailable() || typeof WebAssembly === "undefined") return null;
|
|
904
|
+
try {
|
|
905
|
+
const memory = new WebAssembly.Memory({ initial: pages, maximum: MAX_WASM_PAGES, shared: true });
|
|
906
|
+
return createSharedArenaSimd(memory) === null ? null : memory;
|
|
907
|
+
} catch {
|
|
908
|
+
return null;
|
|
1083
909
|
}
|
|
1084
910
|
}
|
|
1085
|
-
function
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
911
|
+
function createVectorBlock(layout) {
|
|
912
|
+
const initialBytes = initialBlockBytes(layout);
|
|
913
|
+
const shared = createSharedMemory(pagesFor(initialBytes));
|
|
914
|
+
if (shared !== null) return { layout, storage: { kind: "shared-memory", memory: shared } };
|
|
915
|
+
const simd = createArenaSimd();
|
|
916
|
+
if (simd !== null) {
|
|
917
|
+
const missing = pagesFor(initialBytes) - simd.memory.buffer.byteLength / WASM_PAGE_BYTES;
|
|
918
|
+
if (missing > 0) simd.memory.grow(missing);
|
|
919
|
+
return { layout, storage: { kind: "private-memory", simd } };
|
|
1090
920
|
}
|
|
921
|
+
return { layout, storage: { kind: "bytes", buffer: createGrowableBuffer(initialBytes, fullBlockBytes(layout)) } };
|
|
1091
922
|
}
|
|
1092
|
-
function
|
|
1093
|
-
|
|
1094
|
-
adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) + 1 : 0;
|
|
1095
|
-
adj.level0[ord * adj.level0Stride] = 0;
|
|
1096
|
-
raiseSlots(adj, ord + 1);
|
|
1097
|
-
Atomics.store(adj.nodeLevels, ord, maxLayer + 1);
|
|
923
|
+
function fullBlockBytes(layout) {
|
|
924
|
+
return Math.min(BLOCK_MAX_BYTES, layout.slotsOffset + layout.capacity * layout.slotStride);
|
|
1098
925
|
}
|
|
1099
|
-
function
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
adj.level0[ord * adj.level0Stride] = 0;
|
|
1104
|
-
Atomics.store(adj.incoming, ord, 0);
|
|
926
|
+
function growMemoryTo(memory, bytes, layout) {
|
|
927
|
+
const have = memory.buffer.byteLength;
|
|
928
|
+
if (have >= bytes) return;
|
|
929
|
+
memory.grow(pagesFor(nextGrowthTarget(have, bytes, fullBlockBytes(layout))) - have / WASM_PAGE_BYTES);
|
|
1105
930
|
}
|
|
1106
|
-
function
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
Atomics.store(adj.header, GRAPH_UPPER_USED, 0);
|
|
1115
|
-
Atomics.store(adj.header, GRAPH_SLOTS, 0);
|
|
931
|
+
function ensureBlockSlots(handle, slots) {
|
|
932
|
+
const bytes = handle.layout.slotsOffset + Math.min(slots, handle.layout.capacity) * handle.layout.slotStride;
|
|
933
|
+
const storage = handle.storage;
|
|
934
|
+
if (storage.kind === "bytes") {
|
|
935
|
+
growBufferTo(storage.buffer, bytes);
|
|
936
|
+
return;
|
|
937
|
+
}
|
|
938
|
+
growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
|
|
1116
939
|
}
|
|
1117
|
-
function
|
|
1118
|
-
|
|
1119
|
-
|
|
940
|
+
function blockBuffer(handle) {
|
|
941
|
+
const storage = handle.storage;
|
|
942
|
+
if (storage.kind === "bytes") return storage.buffer;
|
|
943
|
+
return storage.kind === "shared-memory" ? storage.memory.buffer : storage.simd.memory.buffer;
|
|
1120
944
|
}
|
|
1121
|
-
|
|
1122
|
-
|
|
945
|
+
var NOTHING_STAGED = -1;
|
|
946
|
+
var QUERY_STAGED = -2;
|
|
947
|
+
function openVectorBlock(handle, threadSlot) {
|
|
948
|
+
const { layout, storage } = handle;
|
|
949
|
+
const simd = storage.kind === "shared-memory" ? createSharedArenaSimd(storage.memory) : storage.kind === "private-memory" ? storage.simd : null;
|
|
950
|
+
const hasScratch = simd !== null && threadSlot >= 0 && threadSlot < VECTOR_SCRATCH_SLOTS;
|
|
951
|
+
const scratchSlot = hasScratch ? threadSlot : 0;
|
|
952
|
+
let float32View;
|
|
953
|
+
let byteView;
|
|
954
|
+
let dataView;
|
|
955
|
+
function rebindViews() {
|
|
956
|
+
const buffer = blockBuffer(handle);
|
|
957
|
+
float32View = fixedView(buffer, Float32Array);
|
|
958
|
+
byteView = fixedView(buffer, Uint8Array);
|
|
959
|
+
dataView = new DataView(buffer, 0, byteView.length);
|
|
960
|
+
}
|
|
961
|
+
rebindViews();
|
|
962
|
+
return {
|
|
963
|
+
handle,
|
|
964
|
+
simd,
|
|
965
|
+
hasScratch,
|
|
966
|
+
scratchByteOffset: scratchSlot * layout.scratchStride,
|
|
967
|
+
stagedOrdinal: NOTHING_STAGED,
|
|
968
|
+
float32(neededLength) {
|
|
969
|
+
if (neededLength > float32View.length) rebindViews();
|
|
970
|
+
return float32View;
|
|
971
|
+
},
|
|
972
|
+
bytes(neededLength) {
|
|
973
|
+
if (neededLength > byteView.length) rebindViews();
|
|
974
|
+
return byteView;
|
|
975
|
+
},
|
|
976
|
+
data(neededLength) {
|
|
977
|
+
if (neededLength > byteView.length) rebindViews();
|
|
978
|
+
return dataView;
|
|
979
|
+
}
|
|
980
|
+
};
|
|
1123
981
|
}
|
|
1124
|
-
function
|
|
1125
|
-
|
|
1126
|
-
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
982
|
+
function slotByteOffset(layout, localOrdinal) {
|
|
983
|
+
return layout.slotsOffset + localOrdinal * layout.slotStride;
|
|
1127
984
|
}
|
|
1128
985
|
|
|
1129
|
-
// src/vector/
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
986
|
+
// src/vector/osq/estimate.ts
|
|
987
|
+
var POPCOUNT = new Uint8Array(256);
|
|
988
|
+
for (let value = 0; value < 256; value++) {
|
|
989
|
+
let count = 0;
|
|
990
|
+
let rest = value;
|
|
991
|
+
while (rest !== 0) {
|
|
992
|
+
count += rest & 1;
|
|
993
|
+
rest >>= 1;
|
|
994
|
+
}
|
|
995
|
+
POPCOUNT[value] = count;
|
|
1133
996
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
997
|
+
var LEVEL_BIT_MASK = { 1: 255, 2: 85 };
|
|
998
|
+
var STAGED_QUERY_PLANES = 4;
|
|
999
|
+
function osqNibbleProducts(a, aOffset, b, bOffset, bytes) {
|
|
1000
|
+
let total = 0;
|
|
1001
|
+
for (let n = 0; n < bytes; n++) {
|
|
1002
|
+
const left = a[aOffset + n];
|
|
1003
|
+
const right = b[bOffset + n];
|
|
1004
|
+
total += (left & 15) * (right & 15) + (left >> 4) * (right >> 4);
|
|
1005
|
+
}
|
|
1006
|
+
return total;
|
|
1137
1007
|
}
|
|
1138
|
-
function
|
|
1139
|
-
|
|
1140
|
-
for (; ; ) {
|
|
1141
|
-
const
|
|
1142
|
-
|
|
1143
|
-
|
|
1008
|
+
function twoBitStagedProducts(document2, documentOffset, streams, streamsOffset, streamBytes) {
|
|
1009
|
+
let total = 0;
|
|
1010
|
+
for (let n = 0; n < streamBytes; n++) {
|
|
1011
|
+
const packed = document2[documentOffset + n];
|
|
1012
|
+
for (let k = 0; k < STAGED_QUERY_PLANES; k++) {
|
|
1013
|
+
total += (packed >> 2 * k & 3) * streams[streamsOffset + k * streamBytes + n];
|
|
1014
|
+
}
|
|
1144
1015
|
}
|
|
1016
|
+
return total;
|
|
1145
1017
|
}
|
|
1146
|
-
function
|
|
1147
|
-
if (
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1018
|
+
function osqStagedQueryProducts(document2, documentOffset, documentBits, staged, stagedOffset, codeBytes) {
|
|
1019
|
+
if (documentBits === 2) return twoBitStagedProducts(document2, documentOffset, staged, stagedOffset, codeBytes);
|
|
1020
|
+
let total = 0;
|
|
1021
|
+
for (let p = 0; p < STAGED_QUERY_PLANES; p++) {
|
|
1022
|
+
const plane = stagedOffset + p * codeBytes;
|
|
1023
|
+
let bitsSet = 0;
|
|
1024
|
+
for (let n = 0; n < codeBytes; n++) bitsSet += POPCOUNT[document2[documentOffset + n] & staged[plane + n]];
|
|
1025
|
+
total += 2 ** p * bitsSet;
|
|
1152
1026
|
}
|
|
1027
|
+
return total;
|
|
1153
1028
|
}
|
|
1154
|
-
function
|
|
1155
|
-
|
|
1029
|
+
function osqNarrowPairProducts(a, aOffset, b, bOffset, bits, bytes) {
|
|
1030
|
+
const mask = LEVEL_BIT_MASK[bits];
|
|
1031
|
+
let total = 0;
|
|
1032
|
+
for (let j = 0; j < bits; j++) {
|
|
1033
|
+
for (let l = 0; l < bits; l++) {
|
|
1034
|
+
let bitsSet = 0;
|
|
1035
|
+
for (let n = 0; n < bytes; n++) bitsSet += POPCOUNT[a[aOffset + n] >> j & b[bOffset + n] >> l & mask];
|
|
1036
|
+
total += 2 ** (j + l) * bitsSet;
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
return total;
|
|
1156
1040
|
}
|
|
1157
|
-
function
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1041
|
+
function osqByteLevelProducts(document2, documentOffset, query, queryOffset, dimension) {
|
|
1042
|
+
let total = 0;
|
|
1043
|
+
for (let i = 0; i < dimension; i++) total += document2[documentOffset + i] * query[queryOffset + i];
|
|
1044
|
+
return total;
|
|
1045
|
+
}
|
|
1046
|
+
function osqEstimate(products, document2, documentBits, query, queryBits, dimension, centroidDot, metric) {
|
|
1047
|
+
const documentStep = (document2.upper - document2.lower) / OSQ_STEPS[documentBits];
|
|
1048
|
+
const queryStep = (query.upper - query.lower) / OSQ_STEPS[queryBits];
|
|
1049
|
+
const centred = document2.lower * query.lower * dimension + query.lower * documentStep * document2.sum + document2.lower * queryStep * query.sum + documentStep * queryStep * products;
|
|
1050
|
+
if (metric === "euclidean") return Math.sqrt(Math.max(0, document2.correction + query.correction - 2 * centred));
|
|
1051
|
+
const similarity = centred + document2.correction + query.correction - centroidDot;
|
|
1052
|
+
if (metric === "cosine") return Math.min(Math.max(similarity, -1), 1);
|
|
1053
|
+
return similarity;
|
|
1054
|
+
}
|
|
1055
|
+
function osqDistance(estimate, metric) {
|
|
1056
|
+
switch (metric) {
|
|
1057
|
+
case "cosine":
|
|
1058
|
+
return 1 - estimate;
|
|
1059
|
+
case "dotProduct":
|
|
1060
|
+
return -estimate;
|
|
1061
|
+
case "euclidean":
|
|
1062
|
+
return estimate;
|
|
1167
1063
|
}
|
|
1168
1064
|
}
|
|
1169
|
-
function
|
|
1170
|
-
|
|
1065
|
+
function osqQueryBits(bits) {
|
|
1066
|
+
return OSQ_QUERY_BITS[bits];
|
|
1171
1067
|
}
|
|
1172
1068
|
|
|
1173
|
-
// src/vector/
|
|
1174
|
-
function
|
|
1175
|
-
|
|
1069
|
+
// src/vector/vector-store/code-blocks.ts
|
|
1070
|
+
function computeCodeBlockLayout(dimension, bits) {
|
|
1071
|
+
const queryBits = osqQueryBits(bits);
|
|
1072
|
+
const codeBytes = osqCodeBytes(dimension, bits);
|
|
1073
|
+
const queryBytes = osqStagedQueryBytes(dimension, bits);
|
|
1074
|
+
const scratchStride = alignUp(queryBytes);
|
|
1075
|
+
const slotsOffset = alignUp(VECTOR_SCRATCH_SLOTS * scratchStride);
|
|
1076
|
+
const slotStride = osqRecordBytes(dimension, bits);
|
|
1077
|
+
return {
|
|
1078
|
+
dimension,
|
|
1079
|
+
bits,
|
|
1080
|
+
queryBits,
|
|
1081
|
+
codeBytes,
|
|
1082
|
+
queryBytes,
|
|
1083
|
+
scratchStride,
|
|
1084
|
+
slotsOffset,
|
|
1085
|
+
slotStride,
|
|
1086
|
+
capacity: Math.floor((BLOCK_MAX_BYTES - slotsOffset) / slotStride)
|
|
1087
|
+
};
|
|
1176
1088
|
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1089
|
+
|
|
1090
|
+
// src/vector/vector-store/handles.ts
|
|
1091
|
+
var STORE_SLOTS = 0;
|
|
1092
|
+
var STORE_LIVE_COUNT = 1;
|
|
1093
|
+
var STORE_BLOCK_COUNT = 2;
|
|
1094
|
+
var STORE_CALIBRATED = 3;
|
|
1095
|
+
var STORE_CODE_COUNT = 4;
|
|
1096
|
+
var STORE_DOC_ID_BYTES = 5;
|
|
1097
|
+
var STORE_CODE_BLOCK_COUNT = 6;
|
|
1098
|
+
var STORE_CALIBRATION_GENERATION = 7;
|
|
1099
|
+
var STORE_RELEASED_VECTORS_TO_DISK = 8;
|
|
1100
|
+
var STORE_UNPLACED_COUNT = 9;
|
|
1101
|
+
var STORE_REMOVED_COUNT = 10;
|
|
1102
|
+
var STORE_HEADER_WORDS = 16;
|
|
1103
|
+
var IN_MEMORY = -1;
|
|
1104
|
+
function fixedInt32(words) {
|
|
1105
|
+
return new Int32Array(createFixedBuffer(words * 4), 0, words);
|
|
1185
1106
|
}
|
|
1186
|
-
function
|
|
1187
|
-
|
|
1188
|
-
if (base === ABSENT) return 0;
|
|
1189
|
-
return layerArray(adj, layer)[base];
|
|
1107
|
+
function fixedFloat32(length) {
|
|
1108
|
+
return new Float32Array(createFixedBuffer(length * 4), 0, length);
|
|
1190
1109
|
}
|
|
1191
|
-
function
|
|
1192
|
-
|
|
1193
|
-
if (base === ABSENT) return 0;
|
|
1194
|
-
const array = layerArray(adj, layer);
|
|
1195
|
-
const count = Math.min(array[base], out.length);
|
|
1196
|
-
for (let i = 0; i < count; i++) out[i] = array[base + i + 1];
|
|
1197
|
-
return count;
|
|
1110
|
+
function unplacedChunkCount(ordinals) {
|
|
1111
|
+
return Math.floor(ordinals / UNPLACED_FLAG_CHUNK_ORDINALS) + 1;
|
|
1198
1112
|
}
|
|
1199
|
-
function
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
const collected = new Array(count);
|
|
1205
|
-
for (let i = 0; i < count; i++) collected[i] = array[base + i + 1];
|
|
1206
|
-
return collected;
|
|
1113
|
+
function perOrdinal2(bytesPerOrdinal) {
|
|
1114
|
+
return createGrowableBuffer(
|
|
1115
|
+
VECTOR_STORE_INITIAL_CAPACITY * bytesPerOrdinal,
|
|
1116
|
+
(MAX_VECTOR_ORDINALS + 1) * bytesPerOrdinal
|
|
1117
|
+
);
|
|
1207
1118
|
}
|
|
1208
|
-
function
|
|
1209
|
-
const
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
const count = array[base];
|
|
1213
|
-
for (let i = 1; i <= count; i++) {
|
|
1214
|
-
if (array[base + i] === neighborOrd) return;
|
|
1215
|
-
}
|
|
1216
|
-
const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
|
|
1217
|
-
if (count + 1 > stride - 1) return;
|
|
1218
|
-
array[base + count + 1] = neighborOrd;
|
|
1219
|
-
array[base] = count + 1;
|
|
1220
|
-
if (layer === 0) noteLinked(adj, neighborOrd);
|
|
1119
|
+
function inMemoryTable() {
|
|
1120
|
+
const buffer = perOrdinal2(4);
|
|
1121
|
+
new Int32Array(buffer).fill(IN_MEMORY);
|
|
1122
|
+
return buffer;
|
|
1221
1123
|
}
|
|
1222
|
-
function
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1124
|
+
function createSharedVectorStoreHandles(dimension, codeBits, blockBytes2 = BLOCK_MAX_BYTES) {
|
|
1125
|
+
return {
|
|
1126
|
+
dimension,
|
|
1127
|
+
codeBits,
|
|
1128
|
+
layout: computeVectorBlockLayout(dimension, blockBytes2),
|
|
1129
|
+
codeLayout: codeBits === null ? null : computeCodeBlockLayout(dimension, codeBits),
|
|
1130
|
+
layoutRevision: 0,
|
|
1131
|
+
header: fixedInt32(STORE_HEADER_WORDS),
|
|
1132
|
+
blocks: [],
|
|
1133
|
+
codeBlocks: [],
|
|
1134
|
+
vectorFiles: [],
|
|
1135
|
+
magnitudes: perOrdinal2(8),
|
|
1136
|
+
present: perOrdinal2(1),
|
|
1137
|
+
unplaced: perOrdinal2(1),
|
|
1138
|
+
unplacedChunks: createGrowableBuffer(4, unplacedChunkCount(MAX_VECTOR_ORDINALS) * 4),
|
|
1139
|
+
removed: perOrdinal2(1),
|
|
1140
|
+
partitions: perOrdinal2(4),
|
|
1141
|
+
docIdBytes: createGrowableBuffer(VECTOR_STORE_INITIAL_CAPACITY * 16, MAX_DOC_ID_TABLE_BYTES),
|
|
1142
|
+
docIdOffsets: perOrdinal2(4),
|
|
1143
|
+
codePresent: perOrdinal2(1),
|
|
1144
|
+
diskFile: inMemoryTable(),
|
|
1145
|
+
diskOffset: perOrdinal2(4),
|
|
1146
|
+
centroid: fixedFloat32(dimension)
|
|
1147
|
+
};
|
|
1236
1148
|
}
|
|
1237
|
-
function
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
|
|
1242
|
-
const limit = Math.min(count, stride - 1);
|
|
1243
|
-
for (let i = 0; i < limit; i++) {
|
|
1244
|
-
array[base + i + 1] = neighbors[i];
|
|
1149
|
+
function sharedVectorStoreBytes(handles) {
|
|
1150
|
+
let bytes = handles.header.byteLength + handles.centroid.byteLength;
|
|
1151
|
+
for (const block of handles.blocks) {
|
|
1152
|
+
if (block !== null) bytes += blockBuffer(block).byteLength;
|
|
1245
1153
|
}
|
|
1246
|
-
|
|
1154
|
+
for (const block of handles.codeBlocks) bytes += blockBuffer(block).byteLength;
|
|
1155
|
+
bytes += handles.magnitudes.byteLength;
|
|
1156
|
+
bytes += handles.present.byteLength;
|
|
1157
|
+
bytes += handles.unplaced.byteLength;
|
|
1158
|
+
bytes += handles.unplacedChunks.byteLength;
|
|
1159
|
+
bytes += handles.removed.byteLength;
|
|
1160
|
+
bytes += handles.partitions.byteLength;
|
|
1161
|
+
bytes += handles.docIdBytes.byteLength;
|
|
1162
|
+
bytes += handles.docIdOffsets.byteLength;
|
|
1163
|
+
bytes += handles.codePresent.byteLength;
|
|
1164
|
+
bytes += handles.diskFile.byteLength;
|
|
1165
|
+
bytes += handles.diskOffset.byteLength;
|
|
1166
|
+
return bytes;
|
|
1247
1167
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1168
|
+
|
|
1169
|
+
// src/vector/native/types.ts
|
|
1170
|
+
var NATIVE_OK = 0;
|
|
1171
|
+
var NATIVE_NEEDS_ROOM = 3;
|
|
1172
|
+
|
|
1173
|
+
// src/vector/native/store.ts
|
|
1174
|
+
var METRIC_CODES2 = { cosine: 0, dotProduct: 1, euclidean: 2 };
|
|
1175
|
+
var attachedStores = /* @__PURE__ */ new WeakMap();
|
|
1176
|
+
function nativeMetricCode(metric) {
|
|
1177
|
+
return METRIC_CODES2[metric];
|
|
1251
1178
|
}
|
|
1252
|
-
function
|
|
1253
|
-
|
|
1254
|
-
if (base === ABSENT) return;
|
|
1255
|
-
if (layer > 0) {
|
|
1256
|
-
writeNeighbors(adj, ord, layer, neighbors, count);
|
|
1257
|
-
return;
|
|
1258
|
-
}
|
|
1259
|
-
const array = adj.level0;
|
|
1260
|
-
const held = array.slice(base + 1, base + 1 + array[base]);
|
|
1261
|
-
const limit = Math.min(count, adj.level0Stride - 1);
|
|
1262
|
-
for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
|
|
1263
|
-
for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
|
|
1264
|
-
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
1179
|
+
function isSharedMemory(buffer) {
|
|
1180
|
+
return typeof SharedArrayBuffer === "function" && buffer instanceof SharedArrayBuffer;
|
|
1265
1181
|
}
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
let sum = 0;
|
|
1272
|
-
for (let i = 0; i < v.length; i++) {
|
|
1273
|
-
sum += v[i] * v[i];
|
|
1182
|
+
function blockBytes(block) {
|
|
1183
|
+
const storage = block.storage;
|
|
1184
|
+
if (storage.kind === "shared-memory") return new Uint8Array(storage.memory.buffer, block.layout.slotsOffset);
|
|
1185
|
+
if (storage.kind === "bytes" && isSharedMemory(storage.buffer)) {
|
|
1186
|
+
return new Uint8Array(storage.buffer, block.layout.slotsOffset);
|
|
1274
1187
|
}
|
|
1275
|
-
return
|
|
1188
|
+
return null;
|
|
1276
1189
|
}
|
|
1277
|
-
function
|
|
1278
|
-
|
|
1279
|
-
|
|
1190
|
+
function storeMemory(store) {
|
|
1191
|
+
const regions = [
|
|
1192
|
+
store.header.buffer,
|
|
1193
|
+
store.codePresent,
|
|
1194
|
+
store.centroid.buffer,
|
|
1195
|
+
store.magnitudes,
|
|
1196
|
+
store.present,
|
|
1197
|
+
store.diskFile,
|
|
1198
|
+
store.diskOffset
|
|
1199
|
+
];
|
|
1200
|
+
if (!regions.every(isSharedMemory)) return null;
|
|
1201
|
+
const codeBlocks = [];
|
|
1202
|
+
for (const block of store.codeBlocks) {
|
|
1203
|
+
const bytes = blockBytes(block);
|
|
1204
|
+
if (bytes === null) return null;
|
|
1205
|
+
codeBlocks.push(bytes);
|
|
1206
|
+
}
|
|
1207
|
+
const vectorBlocks = [];
|
|
1208
|
+
for (const block of store.blocks) {
|
|
1209
|
+
const bytes = block === null ? null : blockBytes(block);
|
|
1210
|
+
if (block !== null && bytes === null) return null;
|
|
1211
|
+
vectorBlocks.push(bytes);
|
|
1212
|
+
}
|
|
1213
|
+
return {
|
|
1214
|
+
storeHeader: store.header,
|
|
1215
|
+
dimension: store.dimension,
|
|
1216
|
+
bits: store.codeBits ?? 0,
|
|
1217
|
+
recordsPerBlock: store.codeLayout?.capacity ?? 0,
|
|
1218
|
+
vectorsPerBlock: store.layout.capacity,
|
|
1219
|
+
vectorStrideBytes: store.layout.slotStride,
|
|
1220
|
+
codePresent: fixedView(store.codePresent, Uint8Array),
|
|
1221
|
+
centroid: store.centroid,
|
|
1222
|
+
magnitudes: fixedView(store.magnitudes, Float64Array),
|
|
1223
|
+
present: fixedView(store.present, Uint8Array),
|
|
1224
|
+
vectorFile: fixedView(store.diskFile, Int32Array),
|
|
1225
|
+
vectorOffset: fixedView(store.diskOffset, Uint32Array),
|
|
1226
|
+
vectorFiles: [...store.vectorFiles],
|
|
1227
|
+
codeBlocks,
|
|
1228
|
+
vectorBlocks
|
|
1229
|
+
};
|
|
1280
1230
|
}
|
|
1281
|
-
function
|
|
1282
|
-
const
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1231
|
+
function attach(core, store) {
|
|
1232
|
+
const memory = storeMemory(store);
|
|
1233
|
+
let handle = null;
|
|
1234
|
+
if (memory !== null) {
|
|
1235
|
+
try {
|
|
1236
|
+
handle = core.attachStore(memory);
|
|
1237
|
+
} catch {
|
|
1238
|
+
handle = null;
|
|
1239
|
+
}
|
|
1287
1240
|
}
|
|
1288
|
-
return
|
|
1241
|
+
return {
|
|
1242
|
+
core,
|
|
1243
|
+
handle,
|
|
1244
|
+
layoutRevision: store.layoutRevision,
|
|
1245
|
+
vectorBlockCount: store.blocks.length,
|
|
1246
|
+
codeBlockCount: store.codeBlocks.length,
|
|
1247
|
+
retryAt: handle === null ? Date.now() + NATIVE_ATTACH_RETRY_MS : Number.POSITIVE_INFINITY,
|
|
1248
|
+
ordinals: new Int32Array(0),
|
|
1249
|
+
distances: new Float64Array(0)
|
|
1250
|
+
};
|
|
1289
1251
|
}
|
|
1290
|
-
function
|
|
1291
|
-
return
|
|
1252
|
+
function stillCurrent(attached, store) {
|
|
1253
|
+
return Date.now() < attached.retryAt && attached.layoutRevision === store.layoutRevision && attached.vectorBlockCount === store.blocks.length && attached.codeBlockCount === store.codeBlocks.length;
|
|
1292
1254
|
}
|
|
1293
|
-
function
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1255
|
+
function isUsable(attached) {
|
|
1256
|
+
return attached.handle !== null;
|
|
1257
|
+
}
|
|
1258
|
+
function detachNativeStore(store) {
|
|
1259
|
+
const attached = attachedStores.get(store);
|
|
1260
|
+
if (attached === void 0) return;
|
|
1261
|
+
attachedStores.delete(store);
|
|
1262
|
+
if (attached.handle === null) return;
|
|
1263
|
+
try {
|
|
1264
|
+
attached.core.detachStore(attached.handle);
|
|
1265
|
+
} catch (error) {
|
|
1266
|
+
stopUsingTheNativeSearchCore(error);
|
|
1300
1267
|
}
|
|
1301
|
-
return sum;
|
|
1302
1268
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
return Atomics.load(state.header, GRAPH_ENTRY_POINT);
|
|
1269
|
+
function attachedNativeStore(store) {
|
|
1270
|
+
const attached = attachedStores.get(store);
|
|
1271
|
+
return attached !== void 0 && isUsable(attached) ? attached : null;
|
|
1307
1272
|
}
|
|
1308
|
-
function
|
|
1309
|
-
|
|
1273
|
+
function nativeStoreScratchBytes(store) {
|
|
1274
|
+
const attached = attachedNativeStore(store);
|
|
1275
|
+
return attached === null ? 0 : attached.ordinals.byteLength + attached.distances.byteLength;
|
|
1310
1276
|
}
|
|
1311
|
-
function
|
|
1312
|
-
|
|
1277
|
+
function nativeStoreFor(store) {
|
|
1278
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
1279
|
+
const core = loadNativeCore();
|
|
1280
|
+
if (core === null) return null;
|
|
1281
|
+
let attached = attachedStores.get(store);
|
|
1282
|
+
if (attached !== void 0 && !stillCurrent(attached, store)) {
|
|
1283
|
+
detachNativeStore(store);
|
|
1284
|
+
attached = void 0;
|
|
1285
|
+
}
|
|
1286
|
+
if (attached === void 0) {
|
|
1287
|
+
attached = attach(core, store);
|
|
1288
|
+
attachedStores.set(store, attached);
|
|
1289
|
+
}
|
|
1290
|
+
return isUsable(attached) ? attached : null;
|
|
1313
1291
|
}
|
|
1314
|
-
function
|
|
1315
|
-
|
|
1292
|
+
function reserveOrdinals(store, count) {
|
|
1293
|
+
if (store.ordinals.length >= count) return;
|
|
1294
|
+
store.ordinals = new Int32Array(count);
|
|
1295
|
+
store.distances = new Float64Array(count);
|
|
1316
1296
|
}
|
|
1317
|
-
function
|
|
1318
|
-
|
|
1319
|
-
|
|
1297
|
+
function nativeScores(store, query, metric, count) {
|
|
1298
|
+
if (count === 0) return store.distances;
|
|
1299
|
+
let status;
|
|
1300
|
+
try {
|
|
1301
|
+
status = store.core.score(store.handle, query, nativeMetricCode(metric), store.ordinals, count, store.distances);
|
|
1302
|
+
} catch (error) {
|
|
1303
|
+
stopUsingTheNativeSearchCore(error);
|
|
1304
|
+
return null;
|
|
1305
|
+
}
|
|
1306
|
+
return status === NATIVE_OK ? store.distances : null;
|
|
1320
1307
|
}
|
|
1321
|
-
function
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
state.visited = next;
|
|
1308
|
+
function nativeScoresOf(handles, query, metric, ordinals) {
|
|
1309
|
+
const store = nativeStoreFor(handles);
|
|
1310
|
+
if (store === null) return null;
|
|
1311
|
+
reserveOrdinals(store, ordinals.length);
|
|
1312
|
+
store.ordinals.set(ordinals);
|
|
1313
|
+
return nativeScores(store, query, metric, ordinals.length);
|
|
1328
1314
|
}
|
|
1329
|
-
function
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1315
|
+
function workOnRecords(handles, metric, ordinals, work) {
|
|
1316
|
+
if (ordinals.length === 0) return true;
|
|
1317
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1318
|
+
const store = nativeStoreFor(handles);
|
|
1319
|
+
if (store === null) return false;
|
|
1320
|
+
let status;
|
|
1321
|
+
try {
|
|
1322
|
+
status = store.core[work](store.handle, nativeMetricCode(metric), ordinals, ordinals.length);
|
|
1323
|
+
} catch (error) {
|
|
1324
|
+
stopUsingTheNativeSearchCore(error);
|
|
1325
|
+
return false;
|
|
1326
|
+
}
|
|
1327
|
+
if (status === NATIVE_OK) return true;
|
|
1328
|
+
if (status !== NATIVE_NEEDS_ROOM) return false;
|
|
1329
|
+
detachNativeStore(handles);
|
|
1334
1330
|
}
|
|
1335
|
-
return
|
|
1331
|
+
return false;
|
|
1336
1332
|
}
|
|
1337
|
-
function
|
|
1338
|
-
|
|
1339
|
-
if (ord >= state.adjacency.handles.tombstones.byteLength) return false;
|
|
1340
|
-
state.tombstones = fixedView(state.adjacency.handles.tombstones, Uint8Array);
|
|
1341
|
-
return ord < state.tombstones.length;
|
|
1333
|
+
function nativeCalibrate(handles, metric, ordinals) {
|
|
1334
|
+
return workOnRecords(handles, metric, ordinals, "calibrate");
|
|
1342
1335
|
}
|
|
1343
|
-
function
|
|
1344
|
-
|
|
1336
|
+
function nativeQuantise(handles, metric, ordinals) {
|
|
1337
|
+
if (Atomics.load(handles.header, STORE_CALIBRATED) !== 1) return false;
|
|
1338
|
+
return workOnRecords(handles, metric, ordinals, "quantise");
|
|
1345
1339
|
}
|
|
1346
|
-
|
|
1347
|
-
|
|
1340
|
+
|
|
1341
|
+
// src/vector/ordinal-filter.ts
|
|
1342
|
+
function createOrdinalFilter(slots) {
|
|
1343
|
+
return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
|
|
1348
1344
|
}
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1345
|
+
function addToOrdinalFilter(filter, ordinal) {
|
|
1346
|
+
const index = ordinal >>> 3;
|
|
1347
|
+
if (index >= filter.bits.length) return;
|
|
1348
|
+
const bit = 1 << (ordinal & 7);
|
|
1349
|
+
if ((filter.bits[index] & bit) !== 0) return;
|
|
1350
|
+
filter.bits[index] |= bit;
|
|
1351
|
+
filter.count += 1;
|
|
1351
1352
|
}
|
|
1352
|
-
function
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1353
|
+
function ordinalFilterHas(filter, ordinal) {
|
|
1354
|
+
const index = ordinal >>> 3;
|
|
1355
|
+
if (index >= filter.bits.length) return false;
|
|
1356
|
+
return (filter.bits[index] & 1 << (ordinal & 7)) !== 0;
|
|
1357
|
+
}
|
|
1358
|
+
function removeFromOrdinalFilter(filter, ordinal) {
|
|
1359
|
+
const index = ordinal >>> 3;
|
|
1360
|
+
if (index >= filter.bits.length) return;
|
|
1361
|
+
const bit = 1 << (ordinal & 7);
|
|
1362
|
+
if ((filter.bits[index] & bit) === 0) return;
|
|
1363
|
+
filter.bits[index] &= ~bit;
|
|
1364
|
+
filter.count -= 1;
|
|
1365
|
+
}
|
|
1366
|
+
function* ordinalFilterValues(filter) {
|
|
1367
|
+
const bits = filter.bits;
|
|
1368
|
+
for (let index = 0; index < bits.length; index++) {
|
|
1369
|
+
const byte = bits[index];
|
|
1370
|
+
if (byte === 0) continue;
|
|
1371
|
+
for (let offset = 0; offset < 8; offset++) {
|
|
1372
|
+
if ((byte & 1 << offset) !== 0) yield index << 3 | offset;
|
|
1373
|
+
}
|
|
1360
1374
|
}
|
|
1361
1375
|
}
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1376
|
+
|
|
1377
|
+
// src/vector/native/field.ts
|
|
1378
|
+
var WAKE_COUNT_WORDS = 1;
|
|
1379
|
+
var WAKES_THE_CORE_KEEPS_PER_CALL = 256;
|
|
1380
|
+
var WAKE_LIST_WORDS = WAKE_COUNT_WORDS + WAKES_THE_CORE_KEEPS_PER_CALL;
|
|
1381
|
+
var WAKE_ENTRY_LOCK = -1;
|
|
1382
|
+
var attachedGraphs = /* @__PURE__ */ new WeakMap();
|
|
1383
|
+
function graphMemory(graph) {
|
|
1384
|
+
const regions = [
|
|
1385
|
+
graph.header.buffer,
|
|
1386
|
+
graph.nodeLevels,
|
|
1387
|
+
graph.level0,
|
|
1388
|
+
graph.upperBase,
|
|
1389
|
+
graph.upper,
|
|
1390
|
+
graph.locks,
|
|
1391
|
+
graph.tombstones,
|
|
1392
|
+
graph.incoming,
|
|
1393
|
+
graph.heldLocks.buffer
|
|
1394
|
+
];
|
|
1395
|
+
if (!regions.every(isSharedMemory)) return null;
|
|
1396
|
+
return {
|
|
1397
|
+
graphHeader: graph.header,
|
|
1398
|
+
nodeLevels: fixedView(graph.nodeLevels, Uint8Array),
|
|
1399
|
+
level0: fixedView(graph.level0, Int32Array),
|
|
1400
|
+
upperBase: fixedView(graph.upperBase, Int32Array),
|
|
1401
|
+
upper: fixedView(graph.upper, Int32Array),
|
|
1402
|
+
locks: fixedView(graph.locks, Int32Array),
|
|
1403
|
+
tombstones: fixedView(graph.tombstones, Uint8Array),
|
|
1404
|
+
incoming: fixedView(graph.incoming, Int32Array),
|
|
1405
|
+
heldLocks: graph.heldLocks
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
function attach2(core, state) {
|
|
1409
|
+
const memory = graphMemory(state.adjacency.handles);
|
|
1410
|
+
let handle = null;
|
|
1411
|
+
if (memory !== null) {
|
|
1412
|
+
try {
|
|
1413
|
+
handle = core.attachGraph(memory);
|
|
1414
|
+
} catch {
|
|
1415
|
+
handle = null;
|
|
1416
|
+
}
|
|
1370
1417
|
}
|
|
1418
|
+
return { handle, field: null };
|
|
1371
1419
|
}
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1420
|
+
var workspaceOwner = null;
|
|
1421
|
+
function nativeScratchBytes(state, field) {
|
|
1422
|
+
const perField = field.store.ordinals.byteLength + field.store.distances.byteLength + field.wakes.byteLength;
|
|
1423
|
+
const owner = workspaceOwner?.deref();
|
|
1424
|
+
if (owner !== void 0 && owner !== state) return perField;
|
|
1425
|
+
workspaceOwner = new WeakRef(state);
|
|
1426
|
+
return perField + field.core.workspaceBytes();
|
|
1376
1427
|
}
|
|
1377
|
-
function
|
|
1378
|
-
|
|
1428
|
+
function attachedNativeField(state) {
|
|
1429
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
1430
|
+
return attachedGraphs.get(state)?.field ?? null;
|
|
1379
1431
|
}
|
|
1380
|
-
function
|
|
1381
|
-
|
|
1432
|
+
function nativeFieldFor(state) {
|
|
1433
|
+
if (!nativeSearchCoreIsEnabled()) return null;
|
|
1434
|
+
const core = loadNativeCore();
|
|
1435
|
+
if (core === null) return null;
|
|
1436
|
+
const store = nativeStoreFor(state.store.handles);
|
|
1437
|
+
if (store === null) return null;
|
|
1438
|
+
let attached = attachedGraphs.get(state);
|
|
1439
|
+
if (attached === void 0) {
|
|
1440
|
+
attached = attach2(core, state);
|
|
1441
|
+
attachedGraphs.set(state, attached);
|
|
1442
|
+
}
|
|
1443
|
+
if (attached.handle === null) return null;
|
|
1444
|
+
if (attached.field === null) {
|
|
1445
|
+
attached.field = { core, graph: attached.handle, store, wakes: new Int32Array(WAKE_LIST_WORDS) };
|
|
1446
|
+
}
|
|
1447
|
+
attached.field.store = store;
|
|
1448
|
+
return attached.field;
|
|
1382
1449
|
}
|
|
1383
|
-
function
|
|
1384
|
-
const
|
|
1385
|
-
if (
|
|
1386
|
-
|
|
1450
|
+
function wakeSleepingThreads(state, field) {
|
|
1451
|
+
const count = field.wakes[0];
|
|
1452
|
+
if (count === 0) return;
|
|
1453
|
+
field.wakes[0] = 0;
|
|
1454
|
+
for (let i = 1; i <= count; i++) {
|
|
1455
|
+
const ordinal = field.wakes[i];
|
|
1456
|
+
if (ordinal === WAKE_ENTRY_LOCK) {
|
|
1457
|
+
Atomics.notify(state.header, GRAPH_ENTRY_LOCK);
|
|
1458
|
+
continue;
|
|
1459
|
+
}
|
|
1460
|
+
if (ordinal < 0 || ordinal >= Atomics.load(state.header, GRAPH_SLOTS)) continue;
|
|
1461
|
+
Atomics.notify(state.locks.words(ordinal), ordinal);
|
|
1462
|
+
}
|
|
1387
1463
|
}
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1464
|
+
|
|
1465
|
+
// src/vector/native/writes.ts
|
|
1466
|
+
var CALL_RAISED = -1;
|
|
1467
|
+
function writeToTheGraph(state, field, write) {
|
|
1468
|
+
let status;
|
|
1469
|
+
try {
|
|
1470
|
+
status = write();
|
|
1471
|
+
} catch (error) {
|
|
1472
|
+
stopUsingTheNativeSearchCore(error);
|
|
1473
|
+
return CALL_RAISED;
|
|
1392
1474
|
}
|
|
1393
|
-
|
|
1475
|
+
wakeSleepingThreads(state, field);
|
|
1476
|
+
return status;
|
|
1394
1477
|
}
|
|
1395
|
-
function
|
|
1396
|
-
const
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1478
|
+
function nativePlace(state, field, metric, ordinal, topLayer, graphHeldAlone) {
|
|
1479
|
+
const status = writeToTheGraph(
|
|
1480
|
+
state,
|
|
1481
|
+
field,
|
|
1482
|
+
() => field.core.place(
|
|
1483
|
+
field.graph,
|
|
1484
|
+
field.store.handle,
|
|
1485
|
+
nativeMetricCode(metric),
|
|
1486
|
+
ordinal,
|
|
1487
|
+
topLayer,
|
|
1488
|
+
graphHeldAlone ? 1 : 0,
|
|
1489
|
+
state.locks.threadSlot,
|
|
1490
|
+
field.wakes
|
|
1491
|
+
)
|
|
1492
|
+
);
|
|
1493
|
+
if (status === NATIVE_OK) return "placed";
|
|
1494
|
+
return status === NATIVE_NEEDS_ROOM ? "needsRoom" : "failed";
|
|
1407
1495
|
}
|
|
1408
|
-
function
|
|
1409
|
-
const
|
|
1410
|
-
|
|
1411
|
-
return toDistance(qVec, entry.vector, qMag, entry.magnitude, metric);
|
|
1496
|
+
function nativeRemove(state, field, ordinal) {
|
|
1497
|
+
const remove = () => field.core.remove(field.graph, ordinal, state.locks.threadSlot, field.wakes);
|
|
1498
|
+
return writeToTheGraph(state, field, remove) === NATIVE_OK;
|
|
1412
1499
|
}
|
|
1413
|
-
function
|
|
1414
|
-
|
|
1500
|
+
function nativeCompact(state, field, metric) {
|
|
1501
|
+
const compact = () => field.core.compact(field.graph, field.store.handle, nativeMetricCode(metric), state.locks.threadSlot, field.wakes);
|
|
1502
|
+
return writeToTheGraph(state, field, compact) === NATIVE_OK;
|
|
1415
1503
|
}
|
|
1416
|
-
function
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1504
|
+
function nativeConnect(state, field, metric, nearby) {
|
|
1505
|
+
const connect = () => field.core.connectUnreached(
|
|
1506
|
+
field.graph,
|
|
1507
|
+
field.store.handle,
|
|
1508
|
+
nativeMetricCode(metric),
|
|
1509
|
+
nearby,
|
|
1510
|
+
state.locks.threadSlot,
|
|
1511
|
+
field.wakes
|
|
1512
|
+
);
|
|
1513
|
+
return writeToTheGraph(state, field, connect) === NATIVE_OK;
|
|
1421
1514
|
}
|
|
1422
1515
|
|
|
1423
1516
|
// src/vector/hnsw/workspace.ts
|
|
@@ -1662,165 +1755,7 @@ function writeKeepingLastLinks(state, ord, sorted, kept, held) {
|
|
|
1662
1755
|
noteNodeLeftUnlinked(state);
|
|
1663
1756
|
}
|
|
1664
1757
|
for (let k = 0; k < kept.size; k++) if (!held(kept.ords[k])) noteLinked(adjacency, kept.ords[k]);
|
|
1665
|
-
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1666
|
-
}
|
|
1667
|
-
|
|
1668
|
-
// src/vector/hnsw/locks.ts
|
|
1669
|
-
var FREE = 0;
|
|
1670
|
-
var EXCLUSIVE = -1;
|
|
1671
|
-
function openGraphLocks(handles, threadSlot) {
|
|
1672
|
-
let words = fixedView(handles.locks, Int32Array);
|
|
1673
|
-
return {
|
|
1674
|
-
threadSlot,
|
|
1675
|
-
header: handles.header,
|
|
1676
|
-
held: handles.heldLocks,
|
|
1677
|
-
words(ord) {
|
|
1678
|
-
if (ord < words.length) return words;
|
|
1679
|
-
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
1680
|
-
words = fixedView(handles.locks, Int32Array);
|
|
1681
|
-
return words;
|
|
1682
|
-
}
|
|
1683
|
-
};
|
|
1684
|
-
}
|
|
1685
|
-
var blockingAllowed = typeof document === "undefined";
|
|
1686
|
-
function canWait(words) {
|
|
1687
|
-
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
1688
|
-
}
|
|
1689
|
-
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
1690
|
-
if (!canWait(words)) return;
|
|
1691
|
-
Atomics.wait(words, index, seen, sliceMs);
|
|
1692
|
-
}
|
|
1693
|
-
function heldIndex(locks, kind) {
|
|
1694
|
-
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
1695
|
-
}
|
|
1696
|
-
function acquireExclusive(words, index) {
|
|
1697
|
-
let spins = 0;
|
|
1698
|
-
for (; ; ) {
|
|
1699
|
-
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
1700
|
-
if (seen === FREE) return;
|
|
1701
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1702
|
-
spins += 1;
|
|
1703
|
-
continue;
|
|
1704
|
-
}
|
|
1705
|
-
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1706
|
-
}
|
|
1707
|
-
}
|
|
1708
|
-
function releaseExclusive(words, index) {
|
|
1709
|
-
Atomics.store(words, index, FREE);
|
|
1710
|
-
if (canWait(words)) Atomics.notify(words, index);
|
|
1711
|
-
}
|
|
1712
|
-
var WRITE_HELD = 1;
|
|
1713
|
-
var WRITER_WAITING = 2;
|
|
1714
|
-
var VERSION_STEP = 4;
|
|
1715
|
-
function beginNodeRead(locks, ord) {
|
|
1716
|
-
const words = locks.words(ord);
|
|
1717
|
-
for (; ; ) {
|
|
1718
|
-
const seen = Atomics.load(words, ord);
|
|
1719
|
-
if ((seen & WRITE_HELD) === 0) return seen;
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
1722
|
-
function nodeReadHeld(locks, ord, version) {
|
|
1723
|
-
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
1724
|
-
return Atomics.load(locks.words(ord), ord) === version;
|
|
1725
|
-
}
|
|
1726
|
-
function lockNodeWrite(locks, ord) {
|
|
1727
|
-
const words = locks.words(ord);
|
|
1728
|
-
let spins = 0;
|
|
1729
|
-
for (; ; ) {
|
|
1730
|
-
const seen = Atomics.load(words, ord);
|
|
1731
|
-
if ((seen & WRITE_HELD) === 0) {
|
|
1732
|
-
const claimed = seen | WRITE_HELD;
|
|
1733
|
-
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
1734
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
1735
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
1736
|
-
return;
|
|
1737
|
-
}
|
|
1738
|
-
continue;
|
|
1739
|
-
}
|
|
1740
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1741
|
-
spins += 1;
|
|
1742
|
-
continue;
|
|
1743
|
-
}
|
|
1744
|
-
const waiting = seen | WRITER_WAITING;
|
|
1745
|
-
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
1746
|
-
waitOn(words, ord, waiting);
|
|
1747
|
-
}
|
|
1748
|
-
}
|
|
1749
|
-
}
|
|
1750
|
-
function releaseWrite(words, ord) {
|
|
1751
|
-
const seen = Atomics.load(words, ord);
|
|
1752
|
-
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
1753
|
-
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
1754
|
-
}
|
|
1755
|
-
function unlockNodeWrite(locks, ord) {
|
|
1756
|
-
releaseWrite(locks.words(ord), ord);
|
|
1757
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
1758
|
-
}
|
|
1759
|
-
function lockGraphShared(locks) {
|
|
1760
|
-
const { header } = locks;
|
|
1761
|
-
let spins = 0;
|
|
1762
|
-
for (; ; ) {
|
|
1763
|
-
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
1764
|
-
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
1765
|
-
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
1766
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
1767
|
-
return;
|
|
1768
|
-
}
|
|
1769
|
-
continue;
|
|
1770
|
-
}
|
|
1771
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1772
|
-
spins += 1;
|
|
1773
|
-
continue;
|
|
1774
|
-
}
|
|
1775
|
-
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1776
|
-
}
|
|
1777
|
-
}
|
|
1778
|
-
function unlockGraphShared(locks) {
|
|
1779
|
-
const { header } = locks;
|
|
1780
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
1781
|
-
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
1782
|
-
}
|
|
1783
|
-
function lockGraphExclusive(locks) {
|
|
1784
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
1785
|
-
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1786
|
-
try {
|
|
1787
|
-
acquireExclusive(locks.header, GRAPH_LOCK);
|
|
1788
|
-
} finally {
|
|
1789
|
-
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1790
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
1791
|
-
}
|
|
1792
|
-
}
|
|
1793
|
-
function unlockGraphExclusive(locks) {
|
|
1794
|
-
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
1795
|
-
}
|
|
1796
|
-
function lockEntry(locks) {
|
|
1797
|
-
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1798
|
-
}
|
|
1799
|
-
function unlockEntry(locks) {
|
|
1800
|
-
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1801
|
-
}
|
|
1802
|
-
function releaseLocksHeldBy(handles, threadSlot) {
|
|
1803
|
-
const held = handles.heldLocks;
|
|
1804
|
-
const words = new Int32Array(handles.locks);
|
|
1805
|
-
const base = threadSlot * HELD_WORDS_PER_THREAD;
|
|
1806
|
-
if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
|
|
1807
|
-
Atomics.sub(handles.header, GRAPH_WRITERS_WAITING, 1);
|
|
1808
|
-
}
|
|
1809
|
-
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
1810
|
-
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
1811
|
-
if (writeOrd >= 0 && writeOrd < words.length) {
|
|
1812
|
-
const seen = Atomics.load(words, writeOrd);
|
|
1813
|
-
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
1814
|
-
}
|
|
1815
|
-
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
1816
|
-
for (; ; ) {
|
|
1817
|
-
const readers = Atomics.load(handles.header, GRAPH_LOCK);
|
|
1818
|
-
if (readers <= FREE) return;
|
|
1819
|
-
if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
1820
|
-
if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
|
|
1821
|
-
return;
|
|
1822
|
-
}
|
|
1823
|
-
}
|
|
1758
|
+
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1824
1759
|
}
|
|
1825
1760
|
|
|
1826
1761
|
// src/vector/hnsw/connect-unreached.ts
|
|
@@ -2413,42 +2348,6 @@ function rebuild(state) {
|
|
|
2413
2348
|
}
|
|
2414
2349
|
}
|
|
2415
2350
|
|
|
2416
|
-
// src/vector/ordinal-filter.ts
|
|
2417
|
-
function createOrdinalFilter(slots) {
|
|
2418
|
-
return { bits: new Uint8Array(slots + 7 >>> 3), count: 0 };
|
|
2419
|
-
}
|
|
2420
|
-
function addToOrdinalFilter(filter, ordinal) {
|
|
2421
|
-
const index = ordinal >>> 3;
|
|
2422
|
-
if (index >= filter.bits.length) return;
|
|
2423
|
-
const bit = 1 << (ordinal & 7);
|
|
2424
|
-
if ((filter.bits[index] & bit) !== 0) return;
|
|
2425
|
-
filter.bits[index] |= bit;
|
|
2426
|
-
filter.count += 1;
|
|
2427
|
-
}
|
|
2428
|
-
function ordinalFilterHas(filter, ordinal) {
|
|
2429
|
-
const index = ordinal >>> 3;
|
|
2430
|
-
if (index >= filter.bits.length) return false;
|
|
2431
|
-
return (filter.bits[index] & 1 << (ordinal & 7)) !== 0;
|
|
2432
|
-
}
|
|
2433
|
-
function removeFromOrdinalFilter(filter, ordinal) {
|
|
2434
|
-
const index = ordinal >>> 3;
|
|
2435
|
-
if (index >= filter.bits.length) return;
|
|
2436
|
-
const bit = 1 << (ordinal & 7);
|
|
2437
|
-
if ((filter.bits[index] & bit) === 0) return;
|
|
2438
|
-
filter.bits[index] &= ~bit;
|
|
2439
|
-
filter.count -= 1;
|
|
2440
|
-
}
|
|
2441
|
-
function* ordinalFilterValues(filter) {
|
|
2442
|
-
const bits = filter.bits;
|
|
2443
|
-
for (let index = 0; index < bits.length; index++) {
|
|
2444
|
-
const byte = bits[index];
|
|
2445
|
-
if (byte === 0) continue;
|
|
2446
|
-
for (let offset = 0; offset < 8; offset++) {
|
|
2447
|
-
if ((byte & 1 << offset) !== 0) yield index << 3 | offset;
|
|
2448
|
-
}
|
|
2449
|
-
}
|
|
2450
|
-
}
|
|
2451
|
-
|
|
2452
2351
|
// src/vector/native/walk.ts
|
|
2453
2352
|
function copyCandidates(store, count, target) {
|
|
2454
2353
|
ensureListCapacity(target, count);
|
|
@@ -3069,10 +2968,10 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
3069
2968
|
if (simd === null || !block.hasScratch) return generic;
|
|
3070
2969
|
const presentView = codePresent;
|
|
3071
2970
|
const limit = Math.min(presentView.length, capacity);
|
|
3072
|
-
const
|
|
2971
|
+
const { slotsOffset, slotStride } = layout;
|
|
2972
|
+
const data = block.data(slotsOffset + slotStride);
|
|
3073
2973
|
const dataLength = data.byteLength;
|
|
3074
2974
|
const queryOffset = block.scratchByteOffset;
|
|
3075
|
-
const { slotsOffset, slotStride } = layout;
|
|
3076
2975
|
return (ordinal) => {
|
|
3077
2976
|
if (ordinal < 0 || ordinal >= limit || block.stagedOrdinal !== QUERY_STAGED) return generic(ordinal);
|
|
3078
2977
|
if (presentView[ordinal] !== 1) return Number.POSITIVE_INFINITY;
|
|
@@ -3101,9 +3000,9 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
3101
3000
|
if (simd === null) return distanceBetween;
|
|
3102
3001
|
const presentView = codePresent;
|
|
3103
3002
|
const limit = Math.min(presentView.length, capacity);
|
|
3104
|
-
const data = block.data(0);
|
|
3105
|
-
const dataLength = data.byteLength;
|
|
3106
3003
|
const { slotsOffset, slotStride } = layout;
|
|
3004
|
+
const data = block.data(slotsOffset + slotStride);
|
|
3005
|
+
const dataLength = data.byteLength;
|
|
3107
3006
|
return (ordA, ordB) => {
|
|
3108
3007
|
if (ordA < 0 || ordB < 0 || ordA >= limit || ordB >= limit) return distanceBetween(ordA, ordB);
|
|
3109
3008
|
if (presentView[ordA] !== 1 || presentView[ordB] !== 1) return Number.POSITIVE_INFINITY;
|
|
@@ -3128,475 +3027,6 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
3128
3027
|
}
|
|
3129
3028
|
};
|
|
3130
3029
|
}
|
|
3131
|
-
function createVectorFileReader() {
|
|
3132
|
-
const descriptors = /* @__PURE__ */ new Map();
|
|
3133
|
-
function descriptorOf(path) {
|
|
3134
|
-
const open = descriptors.get(path);
|
|
3135
|
-
if (open !== void 0) return open;
|
|
3136
|
-
try {
|
|
3137
|
-
const descriptor = openSync(path, "r");
|
|
3138
|
-
descriptors.set(path, descriptor);
|
|
3139
|
-
return descriptor;
|
|
3140
|
-
} catch {
|
|
3141
|
-
return null;
|
|
3142
|
-
}
|
|
3143
|
-
}
|
|
3144
|
-
function closeDescriptor(path, descriptor) {
|
|
3145
|
-
descriptors.delete(path);
|
|
3146
|
-
try {
|
|
3147
|
-
closeSync(descriptor);
|
|
3148
|
-
} catch {
|
|
3149
|
-
}
|
|
3150
|
-
}
|
|
3151
|
-
return {
|
|
3152
|
-
readInto(path, offset, target) {
|
|
3153
|
-
const descriptor = descriptorOf(path);
|
|
3154
|
-
if (descriptor === null) return false;
|
|
3155
|
-
let read = 0;
|
|
3156
|
-
while (read < target.byteLength) {
|
|
3157
|
-
let chunk;
|
|
3158
|
-
try {
|
|
3159
|
-
chunk = readSync(descriptor, target, read, target.byteLength - read, offset + read);
|
|
3160
|
-
} catch {
|
|
3161
|
-
closeDescriptor(path, descriptor);
|
|
3162
|
-
return false;
|
|
3163
|
-
}
|
|
3164
|
-
if (chunk <= 0) return false;
|
|
3165
|
-
read += chunk;
|
|
3166
|
-
}
|
|
3167
|
-
return true;
|
|
3168
|
-
},
|
|
3169
|
-
retainOnly(paths) {
|
|
3170
|
-
for (const [path, descriptor] of descriptors) {
|
|
3171
|
-
if (!paths.has(path)) closeDescriptor(path, descriptor);
|
|
3172
|
-
}
|
|
3173
|
-
},
|
|
3174
|
-
close() {
|
|
3175
|
-
for (const [path, descriptor] of descriptors) closeDescriptor(path, descriptor);
|
|
3176
|
-
}
|
|
3177
|
-
};
|
|
3178
|
-
}
|
|
3179
|
-
|
|
3180
|
-
// src/vector/vector-store/hot-block-distance.ts
|
|
3181
|
-
function jsDistance(a, b, magA, magB, metric) {
|
|
3182
|
-
switch (metric) {
|
|
3183
|
-
case "cosine":
|
|
3184
|
-
return 1 - cosineSimilarityWithMagnitudes(a, b, magA, magB);
|
|
3185
|
-
case "dotProduct":
|
|
3186
|
-
return -dotProduct(a, b);
|
|
3187
|
-
case "euclidean":
|
|
3188
|
-
return euclideanDistance(a, b);
|
|
3189
|
-
}
|
|
3190
|
-
}
|
|
3191
|
-
function inMemory(reads, ordinal) {
|
|
3192
|
-
return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
|
|
3193
|
-
}
|
|
3194
|
-
function queryDistanceOf(source, prepared, metric) {
|
|
3195
|
-
const generic = (ordinal) => source.distanceFromArena(prepared, ordinal, metric);
|
|
3196
|
-
const reads = source.hotBlockReads();
|
|
3197
|
-
const simd = reads?.block.simd ?? null;
|
|
3198
|
-
if (reads === null || simd === null || !reads.block.hasScratch) return generic;
|
|
3199
|
-
const { block, present, magnitudes, slotsOffset, slotStride, dimension } = reads;
|
|
3200
|
-
const scratchOffset = block.scratchByteOffset;
|
|
3201
|
-
const queryMagnitude = prepared.magnitude;
|
|
3202
|
-
return (ordinal) => {
|
|
3203
|
-
if (!inMemory(reads, ordinal) || block.stagedOrdinal !== QUERY_STAGED) return generic(ordinal);
|
|
3204
|
-
if (present[ordinal] !== 1) return Number.POSITIVE_INFINITY;
|
|
3205
|
-
const slotOffset = slotsOffset + ordinal * slotStride;
|
|
3206
|
-
return arenaFloat32Distance(simd, scratchOffset, slotOffset, dimension, metric, queryMagnitude, magnitudes[ordinal]);
|
|
3207
|
-
};
|
|
3208
|
-
}
|
|
3209
|
-
function pairDistanceOf(source, metric) {
|
|
3210
|
-
const generic = (ordA, ordB) => source.distanceByOrdinal(ordA, ordB, metric);
|
|
3211
|
-
const reads = source.hotBlockReads();
|
|
3212
|
-
const simd = reads?.block.simd ?? null;
|
|
3213
|
-
if (reads === null || simd === null) return generic;
|
|
3214
|
-
const { present, magnitudes, slotsOffset, slotStride, dimension } = reads;
|
|
3215
|
-
return (ordA, ordB) => {
|
|
3216
|
-
if (!inMemory(reads, ordA) || !inMemory(reads, ordB) || present[ordA] !== 1) return generic(ordA, ordB);
|
|
3217
|
-
if (present[ordB] !== 1) return Number.POSITIVE_INFINITY;
|
|
3218
|
-
const offsetA = slotsOffset + ordA * slotStride;
|
|
3219
|
-
const offsetB = slotsOffset + ordB * slotStride;
|
|
3220
|
-
return arenaFloat32Distance(simd, offsetA, offsetB, dimension, metric, magnitudes[ordA], magnitudes[ordB]);
|
|
3221
|
-
};
|
|
3222
|
-
}
|
|
3223
|
-
function ordinalDistanceOf(source, from, metric) {
|
|
3224
|
-
const pair = pairDistanceOf(source, metric);
|
|
3225
|
-
return (ordinal) => pair(from, ordinal);
|
|
3226
|
-
}
|
|
3227
|
-
|
|
3228
|
-
// src/vector/vector-store/view.ts
|
|
3229
|
-
var decoder = new TextDecoder();
|
|
3230
|
-
var NOT_HELD = 0;
|
|
3231
|
-
var HELD_COLD = 1;
|
|
3232
|
-
var HELD_HOT = 2;
|
|
3233
|
-
function openSharedVectorStore(initial, threadSlot) {
|
|
3234
|
-
let handles = initial;
|
|
3235
|
-
const { dimension, layout } = initial;
|
|
3236
|
-
const capacity = layout.capacity;
|
|
3237
|
-
const codeCapacity = initial.codeLayout?.capacity ?? 1;
|
|
3238
|
-
const blocks = [];
|
|
3239
|
-
const codeBlocks = [];
|
|
3240
|
-
let present = fixedView(handles.present, Uint8Array);
|
|
3241
|
-
let magnitudes = fixedView(handles.magnitudes, Float64Array);
|
|
3242
|
-
let partitions = fixedView(handles.partitions, Int32Array);
|
|
3243
|
-
let docIdBytes = fixedView(handles.docIdBytes, Uint8Array);
|
|
3244
|
-
let docIdOffsets = fixedView(handles.docIdOffsets, Uint32Array);
|
|
3245
|
-
let diskFile = fixedView(handles.diskFile, Int32Array);
|
|
3246
|
-
let diskOffset = fixedView(handles.diskOffset, Uint32Array);
|
|
3247
|
-
let currentQuery = null;
|
|
3248
|
-
let reader = null;
|
|
3249
|
-
const coldScratchA = new Float32Array(dimension);
|
|
3250
|
-
const coldScratchB = new Float32Array(dimension);
|
|
3251
|
-
function rebind() {
|
|
3252
|
-
present = fixedView(handles.present, Uint8Array);
|
|
3253
|
-
magnitudes = fixedView(handles.magnitudes, Float64Array);
|
|
3254
|
-
partitions = fixedView(handles.partitions, Int32Array);
|
|
3255
|
-
docIdBytes = fixedView(handles.docIdBytes, Uint8Array);
|
|
3256
|
-
docIdOffsets = fixedView(handles.docIdOffsets, Uint32Array);
|
|
3257
|
-
diskFile = fixedView(handles.diskFile, Int32Array);
|
|
3258
|
-
diskOffset = fixedView(handles.diskOffset, Uint32Array);
|
|
3259
|
-
}
|
|
3260
|
-
function blockAt(index) {
|
|
3261
|
-
let block = blocks[index];
|
|
3262
|
-
if (block === void 0) {
|
|
3263
|
-
const handle = handles.blocks[index];
|
|
3264
|
-
if (handle === null || handle === void 0) {
|
|
3265
|
-
throw new Error(`Vector block ${index} holds no vectors on this thread`);
|
|
3266
|
-
}
|
|
3267
|
-
block = openVectorBlock(handle, threadSlot);
|
|
3268
|
-
blocks[index] = block;
|
|
3269
|
-
}
|
|
3270
|
-
return block;
|
|
3271
|
-
}
|
|
3272
|
-
function codeBlockAt(index) {
|
|
3273
|
-
let block = codeBlocks[index];
|
|
3274
|
-
if (block === void 0) {
|
|
3275
|
-
block = openVectorBlock(handles.codeBlocks[index], threadSlot);
|
|
3276
|
-
codeBlocks[index] = block;
|
|
3277
|
-
}
|
|
3278
|
-
return block;
|
|
3279
|
-
}
|
|
3280
|
-
function fileOf(ordinal) {
|
|
3281
|
-
if (ordinal >= diskFile.length) {
|
|
3282
|
-
if (ordinal * 4 >= handles.diskFile.byteLength) return IN_MEMORY;
|
|
3283
|
-
rebind();
|
|
3284
|
-
}
|
|
3285
|
-
return diskFile[ordinal];
|
|
3286
|
-
}
|
|
3287
|
-
function isCold(ordinal) {
|
|
3288
|
-
return fileOf(ordinal) !== IN_MEMORY;
|
|
3289
|
-
}
|
|
3290
|
-
function holdState(ordinal) {
|
|
3291
|
-
if (ordinal < 0) return NOT_HELD;
|
|
3292
|
-
if (ordinal >= present.length) {
|
|
3293
|
-
if (ordinal >= handles.present.byteLength) return NOT_HELD;
|
|
3294
|
-
rebind();
|
|
3295
|
-
}
|
|
3296
|
-
if (present[ordinal] !== 1) return NOT_HELD;
|
|
3297
|
-
if (isCold(ordinal)) return HELD_COLD;
|
|
3298
|
-
const index = Math.floor(ordinal / capacity);
|
|
3299
|
-
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
3300
|
-
}
|
|
3301
|
-
function holds2(ordinal) {
|
|
3302
|
-
return holdState(ordinal) !== NOT_HELD;
|
|
3303
|
-
}
|
|
3304
|
-
function readFromFile(file, offset, target) {
|
|
3305
|
-
const path = handles.vectorFiles[file];
|
|
3306
|
-
if (path === void 0 || path.length === 0) return false;
|
|
3307
|
-
if (reader === null) reader = createVectorFileReader();
|
|
3308
|
-
const bytes = new Uint8Array(target.buffer, target.byteOffset, target.byteLength);
|
|
3309
|
-
return reader.readInto(path, offset, bytes);
|
|
3310
|
-
}
|
|
3311
|
-
function readColdInto(ordinal, target) {
|
|
3312
|
-
const file = fileOf(ordinal);
|
|
3313
|
-
if (file === IN_MEMORY) return false;
|
|
3314
|
-
if (ordinal >= diskOffset.length) rebind();
|
|
3315
|
-
return readFromFile(file, diskOffset[ordinal], target);
|
|
3316
|
-
}
|
|
3317
|
-
function vectorIn(block, local) {
|
|
3318
|
-
const base = slotByteOffset(layout, local) / 4;
|
|
3319
|
-
return block.float32(base + dimension).subarray(base, base + dimension);
|
|
3320
|
-
}
|
|
3321
|
-
function hotVector(ordinal) {
|
|
3322
|
-
return vectorIn(blockAt(ordinal / capacity | 0), ordinal % capacity);
|
|
3323
|
-
}
|
|
3324
|
-
function vectorFor(ordinal, scratch) {
|
|
3325
|
-
if (!isCold(ordinal)) return hotVector(ordinal);
|
|
3326
|
-
return readColdInto(ordinal, scratch) ? scratch : null;
|
|
3327
|
-
}
|
|
3328
|
-
function stageVector(block, ordinal) {
|
|
3329
|
-
if (block.stagedOrdinal === ordinal) return true;
|
|
3330
|
-
const source = vectorFor(ordinal, coldScratchA);
|
|
3331
|
-
if (source === null) return false;
|
|
3332
|
-
block.float32(0).set(source, block.scratchByteOffset / 4);
|
|
3333
|
-
block.stagedOrdinal = ordinal;
|
|
3334
|
-
return true;
|
|
3335
|
-
}
|
|
3336
|
-
function hotBlockReads() {
|
|
3337
|
-
const handle = handles.blocks[0];
|
|
3338
|
-
if (handle === null || handle === void 0) return null;
|
|
3339
|
-
const { slotsOffset, slotStride } = layout;
|
|
3340
|
-
const limit = Math.min(present.length, diskFile.length, magnitudes.length, capacity);
|
|
3341
|
-
return { block: blockAt(0), present, diskFile, magnitudes, limit, slotsOffset, slotStride, dimension };
|
|
3342
|
-
}
|
|
3343
|
-
function stageQuery(block) {
|
|
3344
|
-
if (currentQuery === null) return false;
|
|
3345
|
-
if (block.stagedOrdinal !== QUERY_STAGED) {
|
|
3346
|
-
block.float32(0).set(currentQuery, block.scratchByteOffset / 4);
|
|
3347
|
-
block.stagedOrdinal = QUERY_STAGED;
|
|
3348
|
-
}
|
|
3349
|
-
return true;
|
|
3350
|
-
}
|
|
3351
|
-
const view = {
|
|
3352
|
-
dimension,
|
|
3353
|
-
threadSlot,
|
|
3354
|
-
get handles() {
|
|
3355
|
-
return handles;
|
|
3356
|
-
},
|
|
3357
|
-
get slots() {
|
|
3358
|
-
return Atomics.load(handles.header, STORE_SLOTS);
|
|
3359
|
-
},
|
|
3360
|
-
get simdAvailable() {
|
|
3361
|
-
const first = handles.blocks.find((block2) => block2 !== null);
|
|
3362
|
-
if (first === void 0) return false;
|
|
3363
|
-
const block = blockAt(handles.blocks.indexOf(first));
|
|
3364
|
-
return block.simd !== null && block.hasScratch;
|
|
3365
|
-
},
|
|
3366
|
-
get holdsEveryBlock() {
|
|
3367
|
-
return Atomics.load(handles.header, STORE_BLOCK_COUNT) <= handles.blocks.length && Atomics.load(handles.header, STORE_CODE_BLOCK_COUNT) <= handles.codeBlocks.length;
|
|
3368
|
-
},
|
|
3369
|
-
adoptHandles(next) {
|
|
3370
|
-
detachNativeStore(handles);
|
|
3371
|
-
handles = next;
|
|
3372
|
-
rebind();
|
|
3373
|
-
for (let index = 0; index < next.blocks.length; index++) {
|
|
3374
|
-
if (next.blocks[index] === null) blocks[index] = void 0;
|
|
3375
|
-
else if (blocks[index] === void 0) blockAt(index);
|
|
3376
|
-
}
|
|
3377
|
-
for (let index = codeBlocks.length; index < next.codeBlocks.length; index++) codeBlockAt(index);
|
|
3378
|
-
reader?.retainOnly(new Set(next.vectorFiles.filter((path) => path.length > 0)));
|
|
3379
|
-
},
|
|
3380
|
-
rebind,
|
|
3381
|
-
holdsOrdinal: holds2,
|
|
3382
|
-
isCold,
|
|
3383
|
-
readColdInto,
|
|
3384
|
-
readFromFile,
|
|
3385
|
-
close() {
|
|
3386
|
-
detachNativeStore(handles);
|
|
3387
|
-
reader?.close();
|
|
3388
|
-
reader = null;
|
|
3389
|
-
},
|
|
3390
|
-
blockOf(ordinal) {
|
|
3391
|
-
return blockAt(ordinal / capacity | 0);
|
|
3392
|
-
},
|
|
3393
|
-
localOrdinal(ordinal) {
|
|
3394
|
-
return ordinal % capacity;
|
|
3395
|
-
},
|
|
3396
|
-
codeBlockOf(ordinal) {
|
|
3397
|
-
return codeBlockAt(ordinal / codeCapacity | 0);
|
|
3398
|
-
},
|
|
3399
|
-
codeLocalOrdinal(ordinal) {
|
|
3400
|
-
return ordinal % codeCapacity;
|
|
3401
|
-
},
|
|
3402
|
-
openCodeBlocks() {
|
|
3403
|
-
return codeBlocks;
|
|
3404
|
-
},
|
|
3405
|
-
vectorAt(ordinal) {
|
|
3406
|
-
if (!isCold(ordinal)) return hotVector(ordinal);
|
|
3407
|
-
const copy = new Float32Array(dimension);
|
|
3408
|
-
if (!readColdInto(ordinal, copy)) {
|
|
3409
|
-
throw new NarsilError(
|
|
3410
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3411
|
-
`The vector at ordinal ${ordinal} is missing from its checkpoint file`,
|
|
3412
|
-
{ ordinal, path: handles.vectorFiles[fileOf(ordinal)] }
|
|
3413
|
-
);
|
|
3414
|
-
}
|
|
3415
|
-
return copy;
|
|
3416
|
-
},
|
|
3417
|
-
magnitudeAt(ordinal) {
|
|
3418
|
-
if (ordinal >= magnitudes.length) rebind();
|
|
3419
|
-
return magnitudes[ordinal];
|
|
3420
|
-
},
|
|
3421
|
-
entryForOrdinal(ordinal) {
|
|
3422
|
-
if (!holds2(ordinal)) return void 0;
|
|
3423
|
-
return { vector: view.vectorAt(ordinal), magnitude: magnitudes[ordinal] };
|
|
3424
|
-
},
|
|
3425
|
-
docIdAt(ordinal) {
|
|
3426
|
-
if (ordinal < 0 || ordinal >= view.slots) return void 0;
|
|
3427
|
-
if (ordinal + 1 >= docIdOffsets.length) rebind();
|
|
3428
|
-
const start = docIdOffsets[ordinal];
|
|
3429
|
-
const end = docIdOffsets[ordinal + 1];
|
|
3430
|
-
if (end <= start) return void 0;
|
|
3431
|
-
if (end > docIdBytes.length) rebind();
|
|
3432
|
-
return decoder.decode(docIdBytes.subarray(start, end));
|
|
3433
|
-
},
|
|
3434
|
-
partitionAt(ordinal) {
|
|
3435
|
-
if (ordinal < 0) return void 0;
|
|
3436
|
-
if (ordinal >= partitions.length) {
|
|
3437
|
-
if (ordinal * 4 >= handles.partitions.byteLength) return void 0;
|
|
3438
|
-
rebind();
|
|
3439
|
-
}
|
|
3440
|
-
const partition = partitions[ordinal];
|
|
3441
|
-
return partition < 0 ? void 0 : partition;
|
|
3442
|
-
},
|
|
3443
|
-
distanceByOrdinal(ordA, ordB, metric) {
|
|
3444
|
-
if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
|
|
3445
|
-
if (!isCold(ordB)) {
|
|
3446
|
-
const indexA = ordA / capacity | 0;
|
|
3447
|
-
const indexB = ordB / capacity | 0;
|
|
3448
|
-
const blockB = blockAt(indexB);
|
|
3449
|
-
const localB = ordB % capacity;
|
|
3450
|
-
if (blockB.simd !== null) {
|
|
3451
|
-
if (indexA === indexB && !isCold(ordA)) {
|
|
3452
|
-
return arenaFloat32Distance(
|
|
3453
|
-
blockB.simd,
|
|
3454
|
-
slotByteOffset(layout, ordA % capacity),
|
|
3455
|
-
slotByteOffset(layout, localB),
|
|
3456
|
-
dimension,
|
|
3457
|
-
metric,
|
|
3458
|
-
magnitudes[ordA],
|
|
3459
|
-
magnitudes[ordB]
|
|
3460
|
-
);
|
|
3461
|
-
}
|
|
3462
|
-
if (blockB.hasScratch && stageVector(blockB, ordA)) {
|
|
3463
|
-
return arenaFloat32Distance(
|
|
3464
|
-
blockB.simd,
|
|
3465
|
-
blockB.scratchByteOffset,
|
|
3466
|
-
slotByteOffset(layout, localB),
|
|
3467
|
-
dimension,
|
|
3468
|
-
metric,
|
|
3469
|
-
magnitudes[ordA],
|
|
3470
|
-
magnitudes[ordB]
|
|
3471
|
-
);
|
|
3472
|
-
}
|
|
3473
|
-
}
|
|
3474
|
-
}
|
|
3475
|
-
const a = vectorFor(ordA, coldScratchA);
|
|
3476
|
-
const b = vectorFor(ordB, coldScratchB);
|
|
3477
|
-
if (a === null || b === null) return Number.POSITIVE_INFINITY;
|
|
3478
|
-
return jsDistance(a, b, magnitudes[ordA], magnitudes[ordB], metric);
|
|
3479
|
-
},
|
|
3480
|
-
prepareQueryArena(query) {
|
|
3481
|
-
if (query.length !== dimension || !view.simdAvailable) return null;
|
|
3482
|
-
currentQuery = query;
|
|
3483
|
-
for (const block of blocks) {
|
|
3484
|
-
if (block !== void 0) block.stagedOrdinal = NOTHING_STAGED;
|
|
3485
|
-
}
|
|
3486
|
-
return { magnitude: magnitude(query) };
|
|
3487
|
-
},
|
|
3488
|
-
distanceFromArena(prepared, ordinal, metric) {
|
|
3489
|
-
const state = holdState(ordinal);
|
|
3490
|
-
if (state === NOT_HELD) return Number.POSITIVE_INFINITY;
|
|
3491
|
-
if (currentQuery === null) return Number.POSITIVE_INFINITY;
|
|
3492
|
-
if (state === HELD_COLD) {
|
|
3493
|
-
if (!readColdInto(ordinal, coldScratchA)) return Number.POSITIVE_INFINITY;
|
|
3494
|
-
return jsDistance(currentQuery, coldScratchA, prepared.magnitude, magnitudes[ordinal], metric);
|
|
3495
|
-
}
|
|
3496
|
-
const block = blockAt(ordinal / capacity | 0);
|
|
3497
|
-
const local = ordinal % capacity;
|
|
3498
|
-
if (block.simd !== null && block.hasScratch && stageQuery(block)) {
|
|
3499
|
-
return arenaFloat32Distance(
|
|
3500
|
-
block.simd,
|
|
3501
|
-
block.scratchByteOffset,
|
|
3502
|
-
slotByteOffset(layout, local),
|
|
3503
|
-
dimension,
|
|
3504
|
-
metric,
|
|
3505
|
-
prepared.magnitude,
|
|
3506
|
-
magnitudes[ordinal]
|
|
3507
|
-
);
|
|
3508
|
-
}
|
|
3509
|
-
return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
|
|
3510
|
-
},
|
|
3511
|
-
hotBlockReads,
|
|
3512
|
-
queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
|
|
3513
|
-
ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
|
|
3514
|
-
pairDistance: (metric) => pairDistanceOf(view, metric)
|
|
3515
|
-
};
|
|
3516
|
-
for (let index = 0; index < handles.blocks.length; index++) {
|
|
3517
|
-
if (handles.blocks[index] !== null) blockAt(index);
|
|
3518
|
-
}
|
|
3519
|
-
for (let index = 0; index < handles.codeBlocks.length; index++) codeBlockAt(index);
|
|
3520
|
-
return view;
|
|
3521
|
-
}
|
|
3522
|
-
|
|
3523
|
-
// src/vector/shared-field/view.ts
|
|
3524
|
-
function openSharedVectorField(initial, threadSlot) {
|
|
3525
|
-
let handles = initial;
|
|
3526
|
-
const store = openSharedVectorStore(initial.store, threadSlot);
|
|
3527
|
-
const codeLayout = initial.store.codeLayout;
|
|
3528
|
-
const quantizer = initial.quantization !== "none" && codeLayout !== null ? openSharedQuantizer(store, codeLayout, initial.metric) : void 0;
|
|
3529
|
-
let graph = null;
|
|
3530
|
-
let ordinals = null;
|
|
3531
|
-
let scanned = 0;
|
|
3532
|
-
let placed = 0;
|
|
3533
|
-
function openGraph(next) {
|
|
3534
|
-
graph = next.graph === null ? null : openGraphState(next.graph, next.dimension, store, quantizer, threadSlot);
|
|
3535
|
-
}
|
|
3536
|
-
openGraph(initial);
|
|
3537
|
-
function docIdOf(ordinal) {
|
|
3538
|
-
if (!store.holdsOrdinal(ordinal)) return void 0;
|
|
3539
|
-
return store.docIdAt(ordinal);
|
|
3540
|
-
}
|
|
3541
|
-
function ordinalOf(docId) {
|
|
3542
|
-
const slots = store.slots;
|
|
3543
|
-
if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
|
|
3544
|
-
for (; scanned < slots; scanned++) {
|
|
3545
|
-
const id = store.docIdAt(scanned);
|
|
3546
|
-
if (id !== void 0) ordinals.set(id, scanned);
|
|
3547
|
-
}
|
|
3548
|
-
const ordinal = ordinals.get(docId);
|
|
3549
|
-
return ordinal !== void 0 && store.holdsOrdinal(ordinal) ? ordinal : void 0;
|
|
3550
|
-
}
|
|
3551
|
-
return {
|
|
3552
|
-
get handles() {
|
|
3553
|
-
return handles;
|
|
3554
|
-
},
|
|
3555
|
-
store,
|
|
3556
|
-
quantizer,
|
|
3557
|
-
get graph() {
|
|
3558
|
-
return graph;
|
|
3559
|
-
},
|
|
3560
|
-
get liveSize() {
|
|
3561
|
-
return graph === null ? 0 : nodeCountOf(graph) - tombstoneCountOf(graph);
|
|
3562
|
-
},
|
|
3563
|
-
get readsEveryVector() {
|
|
3564
|
-
return store.holdsEveryBlock;
|
|
3565
|
-
},
|
|
3566
|
-
get graphRetired() {
|
|
3567
|
-
return graph !== null && Atomics.load(graph.header, GRAPH_RETIRED) === 1;
|
|
3568
|
-
},
|
|
3569
|
-
docIdOf,
|
|
3570
|
-
ordinalOf,
|
|
3571
|
-
vectorOf(docId) {
|
|
3572
|
-
const ordinal = ordinalOf(docId);
|
|
3573
|
-
return ordinal === void 0 ? void 0 : store.vectorAt(ordinal);
|
|
3574
|
-
},
|
|
3575
|
-
adopt(next) {
|
|
3576
|
-
store.adoptHandles(next.store);
|
|
3577
|
-
if (next.graph?.header !== handles.graph?.header) openGraph(next);
|
|
3578
|
-
handles = next;
|
|
3579
|
-
},
|
|
3580
|
-
close() {
|
|
3581
|
-
store.close();
|
|
3582
|
-
},
|
|
3583
|
-
insertOrdinal(ordinal) {
|
|
3584
|
-
if (graph === null) return false;
|
|
3585
|
-
const fresh = insertNode(graph, ordinal);
|
|
3586
|
-
if (fresh) placed += 1;
|
|
3587
|
-
return fresh;
|
|
3588
|
-
},
|
|
3589
|
-
takeOutcome() {
|
|
3590
|
-
const outcome = { placed };
|
|
3591
|
-
placed = 0;
|
|
3592
|
-
return outcome;
|
|
3593
|
-
},
|
|
3594
|
-
searchOrdinals(query, k, metric, minSimilarity, options) {
|
|
3595
|
-
if (graph === null) return [];
|
|
3596
|
-
return searchOrdinals(graph, docIdOf, query, k, metric, minSimilarity, options);
|
|
3597
|
-
}
|
|
3598
|
-
};
|
|
3599
|
-
}
|
|
3600
3030
|
|
|
3601
3031
|
// src/vector/osq/index.ts
|
|
3602
3032
|
function createOsqQuantizer(dimension, bits, metric, store) {
|
|
@@ -3705,527 +3135,6 @@ function createOsqQuantizer(dimension, bits, metric, store) {
|
|
|
3705
3135
|
};
|
|
3706
3136
|
}
|
|
3707
3137
|
|
|
3708
|
-
// src/vector/vector-store/doc-ids.ts
|
|
3709
|
-
var encoder = new TextEncoder();
|
|
3710
|
-
function appendDocId(handles, ordinal, docId) {
|
|
3711
|
-
const bytes = encoder.encode(docId);
|
|
3712
|
-
const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
|
|
3713
|
-
const end = start + bytes.length;
|
|
3714
|
-
growBufferTo(handles.docIdBytes, end);
|
|
3715
|
-
growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
|
|
3716
|
-
new Uint8Array(handles.docIdBytes).set(bytes, start);
|
|
3717
|
-
const offsets = new Uint32Array(handles.docIdOffsets);
|
|
3718
|
-
offsets[ordinal] = start;
|
|
3719
|
-
offsets[ordinal + 1] = end;
|
|
3720
|
-
Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
|
|
3721
|
-
}
|
|
3722
|
-
function resetDocIds(handles) {
|
|
3723
|
-
new Uint32Array(handles.docIdOffsets).fill(0);
|
|
3724
|
-
Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
|
|
3725
|
-
}
|
|
3726
|
-
|
|
3727
|
-
// src/vector/vector-store/ordinal-reads.ts
|
|
3728
|
-
function entryAt(store, ordinal) {
|
|
3729
|
-
return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
|
|
3730
|
-
}
|
|
3731
|
-
function createOrdinalReads(openStore, readable) {
|
|
3732
|
-
function pairDistance(metric) {
|
|
3733
|
-
const open = openStore();
|
|
3734
|
-
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
3735
|
-
const inner = open.view.pairDistance(metric);
|
|
3736
|
-
return (ordA, ordB) => readable(ordA) && readable(ordB) ? inner(ordA, ordB) : Number.POSITIVE_INFINITY;
|
|
3737
|
-
}
|
|
3738
|
-
return {
|
|
3739
|
-
entryForOrdinal(ordinal) {
|
|
3740
|
-
const open = openStore();
|
|
3741
|
-
if (open === null || !readable(ordinal)) return void 0;
|
|
3742
|
-
return entryAt(open, ordinal);
|
|
3743
|
-
},
|
|
3744
|
-
distanceByOrdinal(ordA, ordB, metric) {
|
|
3745
|
-
const open = openStore();
|
|
3746
|
-
if (open === null || !readable(ordA) || !readable(ordB)) return Number.POSITIVE_INFINITY;
|
|
3747
|
-
return open.view.distanceByOrdinal(ordA, ordB, metric);
|
|
3748
|
-
},
|
|
3749
|
-
distanceFromArena(prepared, ordinal, metric) {
|
|
3750
|
-
const open = openStore();
|
|
3751
|
-
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
3752
|
-
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
3753
|
-
},
|
|
3754
|
-
queryDistance(prepared, metric) {
|
|
3755
|
-
const open = openStore();
|
|
3756
|
-
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
3757
|
-
const inner = open.view.queryDistance(prepared, metric);
|
|
3758
|
-
return (ordinal) => readable(ordinal) ? inner(ordinal) : Number.POSITIVE_INFINITY;
|
|
3759
|
-
},
|
|
3760
|
-
ordinalDistance(from, metric) {
|
|
3761
|
-
const pair = pairDistance(metric);
|
|
3762
|
-
return (ordinal) => pair(from, ordinal);
|
|
3763
|
-
},
|
|
3764
|
-
pairDistance
|
|
3765
|
-
};
|
|
3766
|
-
}
|
|
3767
|
-
|
|
3768
|
-
// src/vector/vector-store/listing.ts
|
|
3769
|
-
function* entriesOf(open, documents) {
|
|
3770
|
-
if (open === null) return;
|
|
3771
|
-
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3772
|
-
const docId = documents[ordinal];
|
|
3773
|
-
if (docId === void 0) continue;
|
|
3774
|
-
yield [docId, entryAt(open, ordinal)];
|
|
3775
|
-
}
|
|
3776
|
-
}
|
|
3777
|
-
function* docIdsOf(open, documents) {
|
|
3778
|
-
if (open === null) return;
|
|
3779
|
-
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3780
|
-
const docId = documents[ordinal];
|
|
3781
|
-
if (docId !== void 0) yield docId;
|
|
3782
|
-
}
|
|
3783
|
-
}
|
|
3784
|
-
function snapshotOf(open, documents, replacedOrdinals) {
|
|
3785
|
-
const slots = documents.length;
|
|
3786
|
-
const dimension = open === null ? 0 : open.handles.dimension;
|
|
3787
|
-
const vectors = new Float32Array(slots * dimension);
|
|
3788
|
-
const magnitudes = new Float64Array(slots);
|
|
3789
|
-
const docIds = new Array(slots);
|
|
3790
|
-
for (let ordinal = 0; ordinal < slots; ordinal++) {
|
|
3791
|
-
docIds[ordinal] = documents[ordinal] ?? null;
|
|
3792
|
-
if (open === null || documents[ordinal] === void 0 && !replacedOrdinals.has(ordinal)) continue;
|
|
3793
|
-
vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
|
|
3794
|
-
magnitudes[ordinal] = open.magnitudes[ordinal];
|
|
3795
|
-
}
|
|
3796
|
-
return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
|
|
3797
|
-
}
|
|
3798
|
-
|
|
3799
|
-
// src/vector/vector-store/open-store.ts
|
|
3800
|
-
function openHandles(dimension, codeBits, blockBytes2) {
|
|
3801
|
-
const handles = createSharedVectorStoreHandles(dimension, codeBits, blockBytes2);
|
|
3802
|
-
return {
|
|
3803
|
-
handles,
|
|
3804
|
-
view: openSharedVectorStore(handles, 0),
|
|
3805
|
-
magnitudes: new Float64Array(handles.magnitudes),
|
|
3806
|
-
present: new Uint8Array(handles.present),
|
|
3807
|
-
partitions: new Int32Array(handles.partitions),
|
|
3808
|
-
diskFile: new Int32Array(handles.diskFile),
|
|
3809
|
-
diskOffset: new Uint32Array(handles.diskOffset)
|
|
3810
|
-
};
|
|
3811
|
-
}
|
|
3812
|
-
function relayout(store) {
|
|
3813
|
-
store.handles.layoutRevision += 1;
|
|
3814
|
-
store.view.adoptHandles(store.handles);
|
|
3815
|
-
}
|
|
3816
|
-
function ensureCodeSlot(store, ordinal) {
|
|
3817
|
-
const { handles } = store;
|
|
3818
|
-
if (handles.codeLayout === null) return;
|
|
3819
|
-
const blockIndex = Math.floor(ordinal / handles.codeLayout.capacity);
|
|
3820
|
-
while (handles.codeBlocks.length <= blockIndex) {
|
|
3821
|
-
handles.codeBlocks.push(createVectorBlock(handles.codeLayout));
|
|
3822
|
-
Atomics.store(handles.header, STORE_CODE_BLOCK_COUNT, handles.codeBlocks.length);
|
|
3823
|
-
relayout(store);
|
|
3824
|
-
}
|
|
3825
|
-
ensureBlockSlots(handles.codeBlocks[blockIndex], ordinal % handles.codeLayout.capacity + 1);
|
|
3826
|
-
}
|
|
3827
|
-
function ensureFloatSlot(store, ordinal) {
|
|
3828
|
-
const { handles } = store;
|
|
3829
|
-
const blockIndex = Math.floor(ordinal / handles.layout.capacity);
|
|
3830
|
-
let changed = false;
|
|
3831
|
-
while (handles.blocks.length <= blockIndex) {
|
|
3832
|
-
handles.blocks.push(null);
|
|
3833
|
-
changed = true;
|
|
3834
|
-
}
|
|
3835
|
-
if (handles.blocks[blockIndex] === null) {
|
|
3836
|
-
handles.blocks[blockIndex] = createVectorBlock(handles.layout);
|
|
3837
|
-
changed = true;
|
|
3838
|
-
}
|
|
3839
|
-
if (changed) {
|
|
3840
|
-
Atomics.store(handles.header, STORE_BLOCK_COUNT, handles.blocks.length);
|
|
3841
|
-
relayout(store);
|
|
3842
|
-
}
|
|
3843
|
-
const block = handles.blocks[blockIndex];
|
|
3844
|
-
if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
|
|
3845
|
-
}
|
|
3846
|
-
function ensureOrdinal(store, ordinal, cold) {
|
|
3847
|
-
if (ordinal >= MAX_VECTOR_ORDINALS) {
|
|
3848
|
-
throw new NarsilError(
|
|
3849
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
3850
|
-
`A vector field holds at most ${MAX_VECTOR_ORDINALS} vectors, deleted ones included, until it is rebuilt`,
|
|
3851
|
-
{ ordinal, maxOrdinals: MAX_VECTOR_ORDINALS }
|
|
3852
|
-
);
|
|
3853
|
-
}
|
|
3854
|
-
const { handles } = store;
|
|
3855
|
-
growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
|
|
3856
|
-
growBufferTo(handles.present, ordinal + 1);
|
|
3857
|
-
growBufferTo(handles.unplaced, ordinal + 1);
|
|
3858
|
-
growBufferTo(handles.unplacedChunks, unplacedChunkCount(ordinal) * 4);
|
|
3859
|
-
growBufferTo(handles.removed, ordinal + 1);
|
|
3860
|
-
growBufferTo(handles.partitions, (ordinal + 1) * 4);
|
|
3861
|
-
growBufferTo(handles.codePresent, ordinal + 1);
|
|
3862
|
-
const diskFileBytes = handles.diskFile.byteLength;
|
|
3863
|
-
growBufferTo(handles.diskFile, (ordinal + 1) * 4);
|
|
3864
|
-
if (handles.diskFile.byteLength > diskFileBytes) {
|
|
3865
|
-
new Int32Array(handles.diskFile).fill(IN_MEMORY, diskFileBytes / 4);
|
|
3866
|
-
}
|
|
3867
|
-
growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
|
|
3868
|
-
if (!cold) ensureFloatSlot(store, ordinal);
|
|
3869
|
-
ensureCodeSlot(store, ordinal);
|
|
3870
|
-
}
|
|
3871
|
-
function writeVector(store, ordinal, vector) {
|
|
3872
|
-
const block = store.view.blockOf(ordinal);
|
|
3873
|
-
const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
|
|
3874
|
-
block.float32(base + vector.length).set(vector, base);
|
|
3875
|
-
store.magnitudes[ordinal] = magnitude(vector);
|
|
3876
|
-
}
|
|
3877
|
-
function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
3878
|
-
const capacity = store.handles.layout.capacity;
|
|
3879
|
-
const slots = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
3880
|
-
const end = Math.min(slots, (blockIndex + 1) * capacity);
|
|
3881
|
-
if (end < (blockIndex + 1) * capacity) return false;
|
|
3882
|
-
for (let ordinal = blockIndex * capacity; ordinal < end; ordinal++) {
|
|
3883
|
-
const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
3884
|
-
if (held && store.diskFile[ordinal] === IN_MEMORY) return false;
|
|
3885
|
-
}
|
|
3886
|
-
return true;
|
|
3887
|
-
}
|
|
3888
|
-
function fileHoldsSameBytes(store, ordinal, location) {
|
|
3889
|
-
const fromFile = new Float32Array(store.handles.dimension);
|
|
3890
|
-
if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
|
|
3891
|
-
const held = store.view.vectorAt(ordinal);
|
|
3892
|
-
const a = new Uint8Array(fromFile.buffer);
|
|
3893
|
-
const b = new Uint8Array(held.buffer, held.byteOffset, held.byteLength);
|
|
3894
|
-
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
3895
|
-
return true;
|
|
3896
|
-
}
|
|
3897
|
-
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
3898
|
-
const referenced = /* @__PURE__ */ new Set();
|
|
3899
|
-
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3900
|
-
const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
3901
|
-
if (held && store.diskFile[ordinal] !== IN_MEMORY) {
|
|
3902
|
-
referenced.add(store.diskFile[ordinal]);
|
|
3903
|
-
}
|
|
3904
|
-
}
|
|
3905
|
-
let forgotten = 0;
|
|
3906
|
-
for (let file = 0; file < store.handles.vectorFiles.length; file++) {
|
|
3907
|
-
if (referenced.has(file) || store.handles.vectorFiles[file] === "") continue;
|
|
3908
|
-
store.handles.vectorFiles[file] = "";
|
|
3909
|
-
forgotten += 1;
|
|
3910
|
-
}
|
|
3911
|
-
return forgotten;
|
|
3912
|
-
}
|
|
3913
|
-
function partitionFilterOf(store, documents, partitionIds) {
|
|
3914
|
-
const filter = createOrdinalFilter(documents.length);
|
|
3915
|
-
if (store === null) return filter;
|
|
3916
|
-
let highest = -1;
|
|
3917
|
-
for (const partitionId of partitionIds) {
|
|
3918
|
-
if (partitionId > highest) highest = partitionId;
|
|
3919
|
-
}
|
|
3920
|
-
if (highest < 0) return filter;
|
|
3921
|
-
const wanted = new Uint8Array(highest + 1);
|
|
3922
|
-
for (const partitionId of partitionIds) {
|
|
3923
|
-
if (partitionId >= 0) wanted[partitionId] = 1;
|
|
3924
|
-
}
|
|
3925
|
-
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
3926
|
-
if (documents[ordinal] === void 0) continue;
|
|
3927
|
-
const partitionId = store.partitions[ordinal];
|
|
3928
|
-
if (partitionId < 0 || partitionId > highest || wanted[partitionId] === 0) continue;
|
|
3929
|
-
addToOrdinalFilter(filter, ordinal);
|
|
3930
|
-
}
|
|
3931
|
-
return filter;
|
|
3932
|
-
}
|
|
3933
|
-
|
|
3934
|
-
// src/vector/vector-store/index.ts
|
|
3935
|
-
var UNKNOWN_PARTITION = -1;
|
|
3936
|
-
var NO_PARTITION = -2;
|
|
3937
|
-
function createVectorStore(options) {
|
|
3938
|
-
const codeBits = options?.codeBits ?? null;
|
|
3939
|
-
const blockBytes2 = options?.blockBytes ?? BLOCK_MAX_BYTES;
|
|
3940
|
-
const docToOrd = /* @__PURE__ */ new Map();
|
|
3941
|
-
const ordToDoc = [];
|
|
3942
|
-
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
3943
|
-
let unknownPartitions = 0;
|
|
3944
|
-
let liveCount = 0;
|
|
3945
|
-
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes2);
|
|
3946
|
-
function forgetDocuments() {
|
|
3947
|
-
docToOrd.clear();
|
|
3948
|
-
ordToDoc.length = 0;
|
|
3949
|
-
replacedOrdinals.clear();
|
|
3950
|
-
unknownPartitions = 0;
|
|
3951
|
-
liveCount = 0;
|
|
3952
|
-
}
|
|
3953
|
-
function readable(ordinal) {
|
|
3954
|
-
return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
|
|
3955
|
-
}
|
|
3956
|
-
function requireOpen() {
|
|
3957
|
-
if (open === null) {
|
|
3958
|
-
throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
|
|
3959
|
-
}
|
|
3960
|
-
return open;
|
|
3961
|
-
}
|
|
3962
|
-
function recordPartition(store, ordinal, partitionId) {
|
|
3963
|
-
const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
|
|
3964
|
-
store.partitions[ordinal] = given;
|
|
3965
|
-
if (given === UNKNOWN_PARTITION) unknownPartitions += 1;
|
|
3966
|
-
}
|
|
3967
|
-
function retireOrdinal(store, ordinal) {
|
|
3968
|
-
ordToDoc[ordinal] = void 0;
|
|
3969
|
-
Atomics.store(store.present, ordinal, 0);
|
|
3970
|
-
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
3971
|
-
}
|
|
3972
|
-
function keepReplacedOrdinal(store, ordinal) {
|
|
3973
|
-
ordToDoc[ordinal] = void 0;
|
|
3974
|
-
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
3975
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
3976
|
-
replacedOrdinals.add(ordinal);
|
|
3977
|
-
}
|
|
3978
|
-
function restoreReplacedVector(store, ordinal, vector) {
|
|
3979
|
-
store.diskFile[ordinal] = IN_MEMORY;
|
|
3980
|
-
writeVector(store, ordinal, vector);
|
|
3981
|
-
Atomics.store(store.present, ordinal, 1);
|
|
3982
|
-
replacedOrdinals.add(ordinal);
|
|
3983
|
-
}
|
|
3984
|
-
function claimSlot(store, docId, cold) {
|
|
3985
|
-
const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
|
|
3986
|
-
ensureOrdinal(store, ordinal, cold);
|
|
3987
|
-
store.diskFile[ordinal] = IN_MEMORY;
|
|
3988
|
-
appendDocId(store.handles, ordinal, docId);
|
|
3989
|
-
return ordinal;
|
|
3990
|
-
}
|
|
3991
|
-
function claimOrdinal(store, docId, cold, partitionId) {
|
|
3992
|
-
const ordinal = claimSlot(store, docId, cold);
|
|
3993
|
-
recordPartition(store, ordinal, partitionId);
|
|
3994
|
-
ordToDoc[ordinal] = docId;
|
|
3995
|
-
docToOrd.set(docId, ordinal);
|
|
3996
|
-
return ordinal;
|
|
3997
|
-
}
|
|
3998
|
-
function publishOrdinal(store, ordinal) {
|
|
3999
|
-
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
4000
|
-
Atomics.store(store.present, ordinal, 1);
|
|
4001
|
-
}
|
|
4002
|
-
function publishOnDisk(store, ordinal, vectorMagnitude, location) {
|
|
4003
|
-
store.magnitudes[ordinal] = vectorMagnitude;
|
|
4004
|
-
store.diskOffset[ordinal] = location.offset;
|
|
4005
|
-
store.diskFile[ordinal] = location.fileIndex;
|
|
4006
|
-
publishOrdinal(store, ordinal);
|
|
4007
|
-
}
|
|
4008
|
-
function appendOrdinal(store, docId, vector, partitionId) {
|
|
4009
|
-
const ordinal = claimOrdinal(store, docId, false, partitionId);
|
|
4010
|
-
writeVector(store, ordinal, vector);
|
|
4011
|
-
publishOrdinal(store, ordinal);
|
|
4012
|
-
return ordinal;
|
|
4013
|
-
}
|
|
4014
|
-
function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
|
|
4015
|
-
if (previous !== void 0) {
|
|
4016
|
-
if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
|
|
4017
|
-
else retireOrdinal(store, previous);
|
|
4018
|
-
return ordinal;
|
|
4019
|
-
}
|
|
4020
|
-
liveCount++;
|
|
4021
|
-
Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
|
|
4022
|
-
return ordinal;
|
|
4023
|
-
}
|
|
4024
|
-
return {
|
|
4025
|
-
...createOrdinalReads(() => open, readable),
|
|
4026
|
-
get size() {
|
|
4027
|
-
return liveCount;
|
|
4028
|
-
},
|
|
4029
|
-
get slots() {
|
|
4030
|
-
return ordToDoc.length;
|
|
4031
|
-
},
|
|
4032
|
-
get dimension() {
|
|
4033
|
-
return open === null ? 0 : open.handles.dimension;
|
|
4034
|
-
},
|
|
4035
|
-
get codeBits() {
|
|
4036
|
-
return codeBits;
|
|
4037
|
-
},
|
|
4038
|
-
get handles() {
|
|
4039
|
-
return requireOpen().handles;
|
|
4040
|
-
},
|
|
4041
|
-
get view() {
|
|
4042
|
-
return requireOpen().view;
|
|
4043
|
-
},
|
|
4044
|
-
get replacedVectorCount() {
|
|
4045
|
-
return replacedOrdinals.size;
|
|
4046
|
-
},
|
|
4047
|
-
insert(docId, vector, partitionId, keepsReplacedVector = false) {
|
|
4048
|
-
if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
|
|
4049
|
-
const store = open;
|
|
4050
|
-
if (vector.length !== store.handles.dimension) {
|
|
4051
|
-
throw new NarsilError(
|
|
4052
|
-
ErrorCodes.VECTOR_DIMENSION_MISMATCH,
|
|
4053
|
-
`Vector dimension mismatch: expected ${store.handles.dimension}, got ${vector.length}`,
|
|
4054
|
-
{ expected: store.handles.dimension, received: vector.length }
|
|
4055
|
-
);
|
|
4056
|
-
}
|
|
4057
|
-
const previous = docToOrd.get(docId);
|
|
4058
|
-
const ordinal = appendOrdinal(store, docId, vector, partitionId);
|
|
4059
|
-
return settleReplacement(store, ordinal, previous, keepsReplacedVector);
|
|
4060
|
-
},
|
|
4061
|
-
releaseReplacedVectors() {
|
|
4062
|
-
const released = [...replacedOrdinals];
|
|
4063
|
-
replacedOrdinals.clear();
|
|
4064
|
-
if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
|
|
4065
|
-
return released;
|
|
4066
|
-
},
|
|
4067
|
-
insertCold(docId, vectorMagnitude, location, partitionId) {
|
|
4068
|
-
const store = requireOpen();
|
|
4069
|
-
const previous = docToOrd.get(docId);
|
|
4070
|
-
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
4071
|
-
const ordinal = claimOrdinal(store, docId, true, partitionId);
|
|
4072
|
-
publishOnDisk(store, ordinal, vectorMagnitude, location);
|
|
4073
|
-
return settleReplacement(store, ordinal, previous, false);
|
|
4074
|
-
},
|
|
4075
|
-
insertReplaced(docId, vector, location) {
|
|
4076
|
-
if (open === null) open = openHandles(vector.length, codeBits, blockBytes2);
|
|
4077
|
-
const store = open;
|
|
4078
|
-
if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
4079
|
-
const ordinal = claimSlot(store, docId, location !== void 0);
|
|
4080
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
4081
|
-
ordToDoc[ordinal] = void 0;
|
|
4082
|
-
replacedOrdinals.add(ordinal);
|
|
4083
|
-
if (location !== void 0) {
|
|
4084
|
-
publishOnDisk(store, ordinal, magnitude(vector), location);
|
|
4085
|
-
return ordinal;
|
|
4086
|
-
}
|
|
4087
|
-
writeVector(store, ordinal, vector);
|
|
4088
|
-
publishOrdinal(store, ordinal);
|
|
4089
|
-
return ordinal;
|
|
4090
|
-
},
|
|
4091
|
-
addVectorFile(path) {
|
|
4092
|
-
const store = requireOpen();
|
|
4093
|
-
const existing = store.handles.vectorFiles.indexOf(path);
|
|
4094
|
-
if (existing !== -1) return existing;
|
|
4095
|
-
store.handles.vectorFiles.push(path);
|
|
4096
|
-
relayout(store);
|
|
4097
|
-
return store.handles.vectorFiles.length - 1;
|
|
4098
|
-
},
|
|
4099
|
-
isCold(ordinal) {
|
|
4100
|
-
if (open === null) return false;
|
|
4101
|
-
return open.view.isCold(ordinal);
|
|
4102
|
-
},
|
|
4103
|
-
releaseToFile(ordinal, location) {
|
|
4104
|
-
const store = requireOpen();
|
|
4105
|
-
if (ordToDoc[ordinal] === void 0) return false;
|
|
4106
|
-
if (store.diskFile[ordinal] === IN_MEMORY && !fileHoldsSameBytes(store, ordinal, location)) return false;
|
|
4107
|
-
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
4108
|
-
store.diskOffset[ordinal] = location.offset;
|
|
4109
|
-
store.diskFile[ordinal] = location.fileIndex;
|
|
4110
|
-
return true;
|
|
4111
|
-
},
|
|
4112
|
-
releaseColdBlocks() {
|
|
4113
|
-
const store = requireOpen();
|
|
4114
|
-
let released = 0;
|
|
4115
|
-
for (let index = 0; index < store.handles.blocks.length; index++) {
|
|
4116
|
-
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
|
|
4117
|
-
store.handles.blocks[index] = null;
|
|
4118
|
-
released += 1;
|
|
4119
|
-
}
|
|
4120
|
-
if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
|
|
4121
|
-
return released;
|
|
4122
|
-
},
|
|
4123
|
-
setPartition(docId, partitionId) {
|
|
4124
|
-
const ordinal = docToOrd.get(docId);
|
|
4125
|
-
if (ordinal === void 0 || open === null || partitionId < 0) return;
|
|
4126
|
-
if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
4127
|
-
open.partitions[ordinal] = partitionId;
|
|
4128
|
-
},
|
|
4129
|
-
forgetPartition(docId) {
|
|
4130
|
-
const ordinal = docToOrd.get(docId);
|
|
4131
|
-
if (ordinal === void 0 || open === null) return;
|
|
4132
|
-
if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
4133
|
-
open.partitions[ordinal] = NO_PARTITION;
|
|
4134
|
-
},
|
|
4135
|
-
partitionOfOrdinal(ordinal) {
|
|
4136
|
-
if (open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
|
|
4137
|
-
const partitionId = open.partitions[ordinal];
|
|
4138
|
-
return partitionId < 0 ? void 0 : partitionId;
|
|
4139
|
-
},
|
|
4140
|
-
get partitionsKnown() {
|
|
4141
|
-
return unknownPartitions === 0;
|
|
4142
|
-
},
|
|
4143
|
-
partitionFilter(partitionIds) {
|
|
4144
|
-
return partitionFilterOf(open, ordToDoc, partitionIds);
|
|
4145
|
-
},
|
|
4146
|
-
remove(docId) {
|
|
4147
|
-
const ordinal = docToOrd.get(docId);
|
|
4148
|
-
if (ordinal === void 0 || open === null) return;
|
|
4149
|
-
docToOrd.delete(docId);
|
|
4150
|
-
retireOrdinal(open, ordinal);
|
|
4151
|
-
liveCount--;
|
|
4152
|
-
Atomics.sub(open.handles.header, STORE_LIVE_COUNT, 1);
|
|
4153
|
-
},
|
|
4154
|
-
get(docId) {
|
|
4155
|
-
const ordinal = docToOrd.get(docId);
|
|
4156
|
-
return ordinal === void 0 || open === null ? void 0 : entryAt(open, ordinal);
|
|
4157
|
-
},
|
|
4158
|
-
has(docId) {
|
|
4159
|
-
return docToOrd.has(docId);
|
|
4160
|
-
},
|
|
4161
|
-
holdsOrdinal(ordinal) {
|
|
4162
|
-
return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
|
|
4163
|
-
},
|
|
4164
|
-
entries: () => entriesOf(open, ordToDoc),
|
|
4165
|
-
docIds: () => docIdsOf(open, ordToDoc),
|
|
4166
|
-
clear() {
|
|
4167
|
-
forgetDocuments();
|
|
4168
|
-
if (open === null) return;
|
|
4169
|
-
open.present.fill(0);
|
|
4170
|
-
new Uint8Array(open.handles.codePresent).fill(0);
|
|
4171
|
-
new Int32Array(open.handles.diskFile).fill(IN_MEMORY);
|
|
4172
|
-
open.handles.centroid.fill(0);
|
|
4173
|
-
if (open.handles.vectorFiles.length > 0) {
|
|
4174
|
-
open.handles.vectorFiles.length = 0;
|
|
4175
|
-
relayout(open);
|
|
4176
|
-
}
|
|
4177
|
-
resetDocIds(open.handles);
|
|
4178
|
-
Atomics.store(open.handles.header, STORE_SLOTS, 0);
|
|
4179
|
-
Atomics.store(open.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 0);
|
|
4180
|
-
Atomics.store(open.handles.header, STORE_LIVE_COUNT, 0);
|
|
4181
|
-
Atomics.store(open.handles.header, STORE_CODE_COUNT, 0);
|
|
4182
|
-
Atomics.store(open.handles.header, STORE_CALIBRATED, 0);
|
|
4183
|
-
Atomics.add(open.handles.header, STORE_CALIBRATION_GENERATION, 1);
|
|
4184
|
-
},
|
|
4185
|
-
release() {
|
|
4186
|
-
forgetDocuments();
|
|
4187
|
-
open?.view.close();
|
|
4188
|
-
open = null;
|
|
4189
|
-
},
|
|
4190
|
-
closeFiles: () => open?.view.close(),
|
|
4191
|
-
getOrdinal(docId) {
|
|
4192
|
-
return docToOrd.get(docId);
|
|
4193
|
-
},
|
|
4194
|
-
docIdForOrdinal(ordinal) {
|
|
4195
|
-
return ordToDoc[ordinal];
|
|
4196
|
-
},
|
|
4197
|
-
prepareQueryArena(query) {
|
|
4198
|
-
return open === null ? null : open.view.prepareQueryArena(query);
|
|
4199
|
-
},
|
|
4200
|
-
exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
|
|
4201
|
-
restoreSnapshot(snapshot) {
|
|
4202
|
-
this.clear();
|
|
4203
|
-
if (snapshot.dimension === 0 || snapshot.slots === 0) return;
|
|
4204
|
-
if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes2);
|
|
4205
|
-
const store = open;
|
|
4206
|
-
const replaced = new Set(snapshot.replacedOrdinals);
|
|
4207
|
-
for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
|
|
4208
|
-
const docId = snapshot.docIds[ordinal];
|
|
4209
|
-
const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
|
|
4210
|
-
if (docId === null || docId === void 0) {
|
|
4211
|
-
ensureOrdinal(store, ordinal, false);
|
|
4212
|
-
ordToDoc[ordinal] = void 0;
|
|
4213
|
-
store.partitions[ordinal] = NO_PARTITION;
|
|
4214
|
-
if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
|
|
4215
|
-
Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
|
|
4216
|
-
continue;
|
|
4217
|
-
}
|
|
4218
|
-
appendOrdinal(store, docId, vector);
|
|
4219
|
-
liveCount++;
|
|
4220
|
-
}
|
|
4221
|
-
Atomics.store(store.handles.header, STORE_LIVE_COUNT, liveCount);
|
|
4222
|
-
},
|
|
4223
|
-
memoryBytes() {
|
|
4224
|
-
return open === null ? 0 : sharedVectorStoreBytes(open.handles);
|
|
4225
|
-
}
|
|
4226
|
-
};
|
|
4227
|
-
}
|
|
4228
|
-
|
|
4229
3138
|
// src/vector/hnsw/numbered-graph.ts
|
|
4230
3139
|
var NO_NUMBER = -1;
|
|
4231
3140
|
var NO_ORDINAL = -1;
|
|
@@ -4681,4 +3590,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4681
3590
|
};
|
|
4682
3591
|
}
|
|
4683
3592
|
|
|
4684
|
-
export {
|
|
3593
|
+
export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, insertNode, magnitude, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, slotByteOffset, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
|