@delali/narsil 0.3.1-next.84 → 0.3.1-next.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RZXNWQSM.mjs → chunk-B3F7LZXM.mjs} +610 -205
- package/dist/{chunk-XIQZKKTK.mjs → chunk-IDLP54FH.mjs} +39 -10
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-IA67SOJO.mjs → chunk-XPK3GB2P.mjs} +271 -93
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6WWFY5EI.mjs → direct-executor-SI2QV4YI.mjs} +3 -3
- package/dist/distribution.mjs +5 -5
- package/dist/index.browser.mjs +782 -173
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
|
|
2
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-
|
|
2
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-NFF5TDRG.mjs';
|
|
3
3
|
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
|
|
4
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -12,6 +12,7 @@ var VECTOR_STORE_INITIAL_CAPACITY = 16;
|
|
|
12
12
|
var VECTOR_BLOCK_INITIAL_SLOTS = 256;
|
|
13
13
|
var DISK_STORAGE_BLOCK_BYTES = 67108864;
|
|
14
14
|
var MAX_VECTOR_ORDINALS = 16777216;
|
|
15
|
+
var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
|
|
15
16
|
var VECTOR_SCRATCH_SLOTS = 64;
|
|
16
17
|
var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
|
|
17
18
|
var MAX_DOC_ID_TABLE_BYTES = 1073741824;
|
|
@@ -239,8 +240,11 @@ var GRAPH_M = 2;
|
|
|
239
240
|
var GRAPH_MMAX0 = 3;
|
|
240
241
|
var GRAPH_EF_CONSTRUCTION = 4;
|
|
241
242
|
var GRAPH_METRIC = 5;
|
|
243
|
+
var GRAPH_RETIRED = 6;
|
|
242
244
|
var GRAPH_NODE_COUNT = 32;
|
|
243
245
|
var GRAPH_TOMBSTONE_COUNT = 33;
|
|
246
|
+
var GRAPH_UNLINKED_NODES = 34;
|
|
247
|
+
var GRAPH_INCOMING_UNCOUNTED = 35;
|
|
244
248
|
var GRAPH_UPPER_USED = 64;
|
|
245
249
|
var GRAPH_SLOTS = 65;
|
|
246
250
|
var GRAPH_UPPER_CAPACITY = 66;
|
|
@@ -278,6 +282,7 @@ function createSharedGraphHandles(shape) {
|
|
|
278
282
|
upper: perOrdinal(upperStride * 4, INITIAL_UPPER_SLOTS),
|
|
279
283
|
locks: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
280
284
|
tombstones: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
285
|
+
incoming: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
281
286
|
heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
|
|
282
287
|
};
|
|
283
288
|
publishGraphCapacity(handles);
|
|
@@ -297,7 +302,8 @@ function publishGraphCapacity(handles) {
|
|
|
297
302
|
Math.floor(handles.level0.byteLength / (level0Stride * 4)),
|
|
298
303
|
Math.floor(handles.upperBase.byteLength / 4),
|
|
299
304
|
Math.floor(handles.locks.byteLength / 4),
|
|
300
|
-
handles.tombstones.byteLength
|
|
305
|
+
handles.tombstones.byteLength,
|
|
306
|
+
Math.floor(handles.incoming.byteLength / 4)
|
|
301
307
|
);
|
|
302
308
|
raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
|
|
303
309
|
raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
|
|
@@ -385,9 +391,16 @@ function openVectorBlock(handle, threadSlot) {
|
|
|
385
391
|
const simd = storage.kind === "shared-memory" ? createSharedArenaSimd(storage.memory) : storage.kind === "private-memory" ? storage.simd : null;
|
|
386
392
|
const hasScratch = simd !== null && threadSlot >= 0 && threadSlot < VECTOR_SCRATCH_SLOTS;
|
|
387
393
|
const scratchSlot = hasScratch ? threadSlot : 0;
|
|
388
|
-
let float32View
|
|
389
|
-
let byteView
|
|
390
|
-
let dataView
|
|
394
|
+
let float32View;
|
|
395
|
+
let byteView;
|
|
396
|
+
let dataView;
|
|
397
|
+
function rebindViews() {
|
|
398
|
+
const buffer = blockBuffer(handle);
|
|
399
|
+
float32View = fixedView(buffer, Float32Array);
|
|
400
|
+
byteView = fixedView(buffer, Uint8Array);
|
|
401
|
+
dataView = new DataView(buffer, 0, byteView.length);
|
|
402
|
+
}
|
|
403
|
+
rebindViews();
|
|
391
404
|
return {
|
|
392
405
|
handle,
|
|
393
406
|
simd,
|
|
@@ -395,15 +408,15 @@ function openVectorBlock(handle, threadSlot) {
|
|
|
395
408
|
scratchByteOffset: scratchSlot * layout.scratchStride,
|
|
396
409
|
stagedOrdinal: NOTHING_STAGED,
|
|
397
410
|
float32(neededLength) {
|
|
398
|
-
if (neededLength > float32View.length)
|
|
411
|
+
if (neededLength > float32View.length) rebindViews();
|
|
399
412
|
return float32View;
|
|
400
413
|
},
|
|
401
414
|
bytes(neededLength) {
|
|
402
|
-
if (neededLength > byteView.length)
|
|
415
|
+
if (neededLength > byteView.length) rebindViews();
|
|
403
416
|
return byteView;
|
|
404
417
|
},
|
|
405
418
|
data(neededLength) {
|
|
406
|
-
if (neededLength >
|
|
419
|
+
if (neededLength > byteView.length) rebindViews();
|
|
407
420
|
return dataView;
|
|
408
421
|
}
|
|
409
422
|
};
|
|
@@ -612,6 +625,8 @@ var STORE_DOC_ID_BYTES = 5;
|
|
|
612
625
|
var STORE_CODE_BLOCK_COUNT = 6;
|
|
613
626
|
var STORE_CALIBRATION_GENERATION = 7;
|
|
614
627
|
var STORE_RELEASED_VECTORS_TO_DISK = 8;
|
|
628
|
+
var STORE_UNPLACED_COUNT = 9;
|
|
629
|
+
var STORE_REMOVED_COUNT = 10;
|
|
615
630
|
var STORE_HEADER_WORDS = 16;
|
|
616
631
|
var IN_MEMORY = -1;
|
|
617
632
|
function fixedInt32(words) {
|
|
@@ -620,6 +635,9 @@ function fixedInt32(words) {
|
|
|
620
635
|
function fixedFloat32(length) {
|
|
621
636
|
return new Float32Array(createFixedBuffer(length * 4), 0, length);
|
|
622
637
|
}
|
|
638
|
+
function unplacedChunkCount(ordinals) {
|
|
639
|
+
return Math.floor(ordinals / UNPLACED_FLAG_CHUNK_ORDINALS) + 1;
|
|
640
|
+
}
|
|
623
641
|
function perOrdinal2(bytesPerOrdinal) {
|
|
624
642
|
return createGrowableBuffer(
|
|
625
643
|
VECTOR_STORE_INITIAL_CAPACITY * bytesPerOrdinal,
|
|
@@ -644,6 +662,9 @@ function createSharedVectorStoreHandles(dimension, codeBits, blockBytes2 = BLOCK
|
|
|
644
662
|
vectorFiles: [],
|
|
645
663
|
magnitudes: perOrdinal2(8),
|
|
646
664
|
present: perOrdinal2(1),
|
|
665
|
+
unplaced: perOrdinal2(1),
|
|
666
|
+
unplacedChunks: createGrowableBuffer(4, unplacedChunkCount(MAX_VECTOR_ORDINALS) * 4),
|
|
667
|
+
removed: perOrdinal2(1),
|
|
647
668
|
partitions: perOrdinal2(4),
|
|
648
669
|
docIdBytes: createGrowableBuffer(VECTOR_STORE_INITIAL_CAPACITY * 16, MAX_DOC_ID_TABLE_BYTES),
|
|
649
670
|
docIdOffsets: perOrdinal2(4),
|
|
@@ -661,6 +682,9 @@ function sharedVectorStoreBytes(handles) {
|
|
|
661
682
|
for (const block of handles.codeBlocks) bytes += blockBuffer(block).byteLength;
|
|
662
683
|
bytes += handles.magnitudes.byteLength;
|
|
663
684
|
bytes += handles.present.byteLength;
|
|
685
|
+
bytes += handles.unplaced.byteLength;
|
|
686
|
+
bytes += handles.unplacedChunks.byteLength;
|
|
687
|
+
bytes += handles.removed.byteLength;
|
|
664
688
|
bytes += handles.partitions.byteLength;
|
|
665
689
|
bytes += handles.docIdBytes.byteLength;
|
|
666
690
|
bytes += handles.docIdOffsets.byteLength;
|
|
@@ -857,6 +881,7 @@ function graphMemory(graph) {
|
|
|
857
881
|
graph.upper,
|
|
858
882
|
graph.locks,
|
|
859
883
|
graph.tombstones,
|
|
884
|
+
graph.incoming,
|
|
860
885
|
graph.heldLocks.buffer
|
|
861
886
|
];
|
|
862
887
|
if (!regions.every(isSharedMemory)) return null;
|
|
@@ -868,6 +893,7 @@ function graphMemory(graph) {
|
|
|
868
893
|
upper: fixedView(graph.upper, Int32Array),
|
|
869
894
|
locks: fixedView(graph.locks, Int32Array),
|
|
870
895
|
tombstones: fixedView(graph.tombstones, Uint8Array),
|
|
896
|
+
incoming: fixedView(graph.incoming, Int32Array),
|
|
871
897
|
heldLocks: graph.heldLocks
|
|
872
898
|
};
|
|
873
899
|
}
|
|
@@ -967,6 +993,17 @@ function nativeCompact(state, field, metric) {
|
|
|
967
993
|
const compact = () => field.core.compact(field.graph, field.store.handle, nativeMetricCode(metric), state.locks.threadSlot, field.wakes);
|
|
968
994
|
return writeToTheGraph(state, field, compact) === NATIVE_OK;
|
|
969
995
|
}
|
|
996
|
+
function nativeConnect(state, field, metric, nearby) {
|
|
997
|
+
const connect = () => field.core.connectUnreached(
|
|
998
|
+
field.graph,
|
|
999
|
+
field.store.handle,
|
|
1000
|
+
nativeMetricCode(metric),
|
|
1001
|
+
nearby,
|
|
1002
|
+
state.locks.threadSlot,
|
|
1003
|
+
field.wakes
|
|
1004
|
+
);
|
|
1005
|
+
return writeToTheGraph(state, field, connect) === NATIVE_OK;
|
|
1006
|
+
}
|
|
970
1007
|
|
|
971
1008
|
// src/vector/hnsw/adjacency/views.ts
|
|
972
1009
|
var ABSENT = -1;
|
|
@@ -981,11 +1018,13 @@ function openAdjacency(handles, m, mMax0) {
|
|
|
981
1018
|
nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
|
|
982
1019
|
level0: fixedView(handles.level0, Int32Array),
|
|
983
1020
|
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
984
|
-
upper: fixedView(handles.upper, Int32Array)
|
|
1021
|
+
upper: fixedView(handles.upper, Int32Array),
|
|
1022
|
+
incoming: fixedView(handles.incoming, Int32Array)
|
|
985
1023
|
};
|
|
986
1024
|
}
|
|
987
1025
|
function rebindNodes(adj) {
|
|
988
1026
|
adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
|
|
1027
|
+
adj.incoming = fixedView(adj.handles.incoming, Int32Array);
|
|
989
1028
|
adj.level0 = fixedView(adj.handles.level0, Int32Array);
|
|
990
1029
|
adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
|
|
991
1030
|
}
|
|
@@ -1022,6 +1061,7 @@ function ensureAdjacencyCapacity(adj, needed) {
|
|
|
1022
1061
|
growBufferToExactly(handles.upperBase, capacity * 4);
|
|
1023
1062
|
growBufferToExactly(handles.locks, capacity * 4);
|
|
1024
1063
|
growBufferToExactly(handles.tombstones, capacity);
|
|
1064
|
+
growBufferToExactly(handles.incoming, capacity * 4);
|
|
1025
1065
|
growBufferToExactly(handles.nodeLevels, capacity);
|
|
1026
1066
|
publishGraphCapacity(handles);
|
|
1027
1067
|
rebindNodes(adj);
|
|
@@ -1068,10 +1108,12 @@ function deleteNode(adj, ord) {
|
|
|
1068
1108
|
Atomics.store(adj.nodeLevels, ord, 0);
|
|
1069
1109
|
adj.upperBase[ord] = 0;
|
|
1070
1110
|
adj.level0[ord * adj.level0Stride] = 0;
|
|
1111
|
+
Atomics.store(adj.incoming, ord, 0);
|
|
1071
1112
|
}
|
|
1072
1113
|
function resetAdjacency(adj) {
|
|
1073
1114
|
rebindNodes(adj);
|
|
1074
1115
|
rebindUpper(adj);
|
|
1116
|
+
adj.incoming.fill(0);
|
|
1075
1117
|
adj.nodeLevels.fill(0);
|
|
1076
1118
|
adj.level0.fill(0);
|
|
1077
1119
|
adj.upperBase.fill(0);
|
|
@@ -1088,7 +1130,51 @@ function hasNode(adj, ord) {
|
|
|
1088
1130
|
}
|
|
1089
1131
|
function graphBytes(adj) {
|
|
1090
1132
|
const handles = adj.handles;
|
|
1091
|
-
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.heldLocks.byteLength;
|
|
1133
|
+
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
// src/vector/hnsw/adjacency/incoming.ts
|
|
1137
|
+
function incomingCount(adj, ord) {
|
|
1138
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
1139
|
+
return Atomics.load(adj.incoming, ord);
|
|
1140
|
+
}
|
|
1141
|
+
function noteLinked(adj, ord) {
|
|
1142
|
+
if (ord < 0 || !reachNode(adj, ord)) return;
|
|
1143
|
+
Atomics.add(adj.incoming, ord, 1);
|
|
1144
|
+
}
|
|
1145
|
+
function noteUnlinked(adj, ord) {
|
|
1146
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
1147
|
+
for (; ; ) {
|
|
1148
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
1149
|
+
if (seen <= 0) return 0;
|
|
1150
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return seen - 1;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
function releaseUnlessLast(adj, ord) {
|
|
1154
|
+
if (ord < 0 || !reachNode(adj, ord)) return true;
|
|
1155
|
+
for (; ; ) {
|
|
1156
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
1157
|
+
if (seen <= 1) return false;
|
|
1158
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return true;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
function holdsNodeAt(adj, ord) {
|
|
1162
|
+
return ord >= 0 && ord < adj.nodeLevels.length && adj.nodeLevels[ord] !== 0;
|
|
1163
|
+
}
|
|
1164
|
+
function countLinksOutOf(adj, ord, step) {
|
|
1165
|
+
if (!holdsNodeAt(adj, ord)) return;
|
|
1166
|
+
const incoming = adj.incoming;
|
|
1167
|
+
const level0 = adj.level0;
|
|
1168
|
+
const base = ord * adj.level0Stride;
|
|
1169
|
+
const count = level0[base];
|
|
1170
|
+
for (let i = 1; i <= count; i++) {
|
|
1171
|
+
const neighbour = level0[base + i];
|
|
1172
|
+
if (neighbour === ord || !holdsNodeAt(adj, neighbour)) continue;
|
|
1173
|
+
if (incoming[neighbour] + step >= 0) incoming[neighbour] += step;
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
function countEveryLink(adj, slots) {
|
|
1177
|
+
for (let ord = 0; ord < slots; ord++) countLinksOutOf(adj, ord, 1);
|
|
1092
1178
|
}
|
|
1093
1179
|
|
|
1094
1180
|
// src/vector/hnsw/adjacency/neighbors.ts
|
|
@@ -1138,6 +1224,7 @@ function addNeighbor(adj, ord, layer, neighborOrd) {
|
|
|
1138
1224
|
if (count + 1 > stride - 1) return;
|
|
1139
1225
|
array[base + count + 1] = neighborOrd;
|
|
1140
1226
|
array[base] = count + 1;
|
|
1227
|
+
if (layer === 0) noteLinked(adj, neighborOrd);
|
|
1141
1228
|
}
|
|
1142
1229
|
function removeNeighbor(adj, ord, layer, neighborOrd) {
|
|
1143
1230
|
const base = layerBase(adj, ord, layer);
|
|
@@ -1150,10 +1237,11 @@ function removeNeighbor(adj, ord, layer, neighborOrd) {
|
|
|
1150
1237
|
array[base + j] = array[base + j + 1];
|
|
1151
1238
|
}
|
|
1152
1239
|
array[base] = count - 1;
|
|
1240
|
+
if (layer === 0) noteUnlinked(adj, neighborOrd);
|
|
1153
1241
|
return;
|
|
1154
1242
|
}
|
|
1155
1243
|
}
|
|
1156
|
-
function
|
|
1244
|
+
function writeNeighbors(adj, ord, layer, neighbors, count) {
|
|
1157
1245
|
const base = layerBase(adj, ord, layer);
|
|
1158
1246
|
if (base === ABSENT) return;
|
|
1159
1247
|
const array = layerArray(adj, layer);
|
|
@@ -1164,163 +1252,23 @@ function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
|
1164
1252
|
}
|
|
1165
1253
|
array[base] = limit;
|
|
1166
1254
|
}
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
var EXCLUSIVE = -1;
|
|
1171
|
-
function openGraphLocks(handles, threadSlot) {
|
|
1172
|
-
let words = fixedView(handles.locks, Int32Array);
|
|
1173
|
-
return {
|
|
1174
|
-
threadSlot,
|
|
1175
|
-
header: handles.header,
|
|
1176
|
-
held: handles.heldLocks,
|
|
1177
|
-
words(ord) {
|
|
1178
|
-
if (ord < words.length) return words;
|
|
1179
|
-
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
1180
|
-
words = fixedView(handles.locks, Int32Array);
|
|
1181
|
-
return words;
|
|
1182
|
-
}
|
|
1183
|
-
};
|
|
1184
|
-
}
|
|
1185
|
-
var blockingAllowed = typeof document === "undefined";
|
|
1186
|
-
function canWait(words) {
|
|
1187
|
-
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
1188
|
-
}
|
|
1189
|
-
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
1190
|
-
if (!canWait(words)) return;
|
|
1191
|
-
Atomics.wait(words, index, seen, sliceMs);
|
|
1192
|
-
}
|
|
1193
|
-
function heldIndex(locks, kind) {
|
|
1194
|
-
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
1195
|
-
}
|
|
1196
|
-
function acquireExclusive(words, index) {
|
|
1197
|
-
let spins = 0;
|
|
1198
|
-
for (; ; ) {
|
|
1199
|
-
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
1200
|
-
if (seen === FREE) return;
|
|
1201
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1202
|
-
spins += 1;
|
|
1203
|
-
continue;
|
|
1204
|
-
}
|
|
1205
|
-
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
function releaseExclusive(words, index) {
|
|
1209
|
-
Atomics.store(words, index, FREE);
|
|
1210
|
-
if (canWait(words)) Atomics.notify(words, index);
|
|
1211
|
-
}
|
|
1212
|
-
var WRITE_HELD = 1;
|
|
1213
|
-
var WRITER_WAITING = 2;
|
|
1214
|
-
var VERSION_STEP = 4;
|
|
1215
|
-
function beginNodeRead(locks, ord) {
|
|
1216
|
-
const words = locks.words(ord);
|
|
1217
|
-
for (; ; ) {
|
|
1218
|
-
const seen = Atomics.load(words, ord);
|
|
1219
|
-
if ((seen & WRITE_HELD) === 0) return seen;
|
|
1220
|
-
}
|
|
1221
|
-
}
|
|
1222
|
-
function nodeReadHeld(locks, ord, version) {
|
|
1223
|
-
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
1224
|
-
return Atomics.load(locks.words(ord), ord) === version;
|
|
1225
|
-
}
|
|
1226
|
-
function lockNodeWrite(locks, ord) {
|
|
1227
|
-
const words = locks.words(ord);
|
|
1228
|
-
let spins = 0;
|
|
1229
|
-
for (; ; ) {
|
|
1230
|
-
const seen = Atomics.load(words, ord);
|
|
1231
|
-
if ((seen & WRITE_HELD) === 0) {
|
|
1232
|
-
const claimed = seen | WRITE_HELD;
|
|
1233
|
-
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
1234
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
1235
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
1236
|
-
return;
|
|
1237
|
-
}
|
|
1238
|
-
continue;
|
|
1239
|
-
}
|
|
1240
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1241
|
-
spins += 1;
|
|
1242
|
-
continue;
|
|
1243
|
-
}
|
|
1244
|
-
const waiting = seen | WRITER_WAITING;
|
|
1245
|
-
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
1246
|
-
waitOn(words, ord, waiting);
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
}
|
|
1250
|
-
function releaseWrite(words, ord) {
|
|
1251
|
-
const seen = Atomics.load(words, ord);
|
|
1252
|
-
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
1253
|
-
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
1254
|
-
}
|
|
1255
|
-
function unlockNodeWrite(locks, ord) {
|
|
1256
|
-
releaseWrite(locks.words(ord), ord);
|
|
1257
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
1258
|
-
}
|
|
1259
|
-
function lockGraphShared(locks) {
|
|
1260
|
-
const { header } = locks;
|
|
1261
|
-
let spins = 0;
|
|
1262
|
-
for (; ; ) {
|
|
1263
|
-
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
1264
|
-
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
1265
|
-
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
1266
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
1267
|
-
return;
|
|
1268
|
-
}
|
|
1269
|
-
continue;
|
|
1270
|
-
}
|
|
1271
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1272
|
-
spins += 1;
|
|
1273
|
-
continue;
|
|
1274
|
-
}
|
|
1275
|
-
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
function unlockGraphShared(locks) {
|
|
1279
|
-
const { header } = locks;
|
|
1280
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
1281
|
-
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
1282
|
-
}
|
|
1283
|
-
function lockGraphExclusive(locks) {
|
|
1284
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
1285
|
-
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1286
|
-
try {
|
|
1287
|
-
acquireExclusive(locks.header, GRAPH_LOCK);
|
|
1288
|
-
} finally {
|
|
1289
|
-
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1290
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
1291
|
-
}
|
|
1292
|
-
}
|
|
1293
|
-
function unlockGraphExclusive(locks) {
|
|
1294
|
-
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
1295
|
-
}
|
|
1296
|
-
function lockEntry(locks) {
|
|
1297
|
-
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1298
|
-
}
|
|
1299
|
-
function unlockEntry(locks) {
|
|
1300
|
-
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1255
|
+
function holds(neighbors, count, ord) {
|
|
1256
|
+
for (let i = 0; i < count; i++) if (neighbors[i] === ord) return true;
|
|
1257
|
+
return false;
|
|
1301
1258
|
}
|
|
1302
|
-
function
|
|
1303
|
-
const
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
}
|
|
1309
|
-
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
1310
|
-
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
1311
|
-
if (writeOrd >= 0 && writeOrd < words.length) {
|
|
1312
|
-
const seen = Atomics.load(words, writeOrd);
|
|
1313
|
-
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
1314
|
-
}
|
|
1315
|
-
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
1316
|
-
for (; ; ) {
|
|
1317
|
-
const readers = Atomics.load(handles.header, GRAPH_LOCK);
|
|
1318
|
-
if (readers <= FREE) return;
|
|
1319
|
-
if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
1320
|
-
if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
|
|
1321
|
-
return;
|
|
1322
|
-
}
|
|
1259
|
+
function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
1260
|
+
const base = layerBase(adj, ord, layer);
|
|
1261
|
+
if (base === ABSENT) return;
|
|
1262
|
+
if (layer > 0) {
|
|
1263
|
+
writeNeighbors(adj, ord, layer, neighbors, count);
|
|
1264
|
+
return;
|
|
1323
1265
|
}
|
|
1266
|
+
const array = adj.level0;
|
|
1267
|
+
const held = array.slice(base + 1, base + 1 + array[base]);
|
|
1268
|
+
const limit = Math.min(count, adj.level0Stride - 1);
|
|
1269
|
+
for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
|
|
1270
|
+
for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
|
|
1271
|
+
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
1324
1272
|
}
|
|
1325
1273
|
|
|
1326
1274
|
// src/vector/similarity.ts
|
|
@@ -1498,7 +1446,8 @@ function createHNSWWorkspace() {
|
|
|
1498
1446
|
pruneCandidates: createList(INITIAL_LIST_CAPACITY),
|
|
1499
1447
|
pruneSelection: createList(INITIAL_LIST_CAPACITY),
|
|
1500
1448
|
repairCandidates: createList(INITIAL_LIST_CAPACITY),
|
|
1501
|
-
repairSelection: createList(INITIAL_LIST_CAPACITY)
|
|
1449
|
+
repairSelection: createList(INITIAL_LIST_CAPACITY),
|
|
1450
|
+
droppedLinks: createList(INITIAL_LIST_CAPACITY)
|
|
1502
1451
|
};
|
|
1503
1452
|
}
|
|
1504
1453
|
function linkSelectionsFor(workspace, layers) {
|
|
@@ -1656,6 +1605,419 @@ function setEntryPointsFromList(workspace, list) {
|
|
|
1656
1605
|
workspace.entryPointCount = list.size;
|
|
1657
1606
|
}
|
|
1658
1607
|
|
|
1608
|
+
// src/vector/hnsw/last-links.ts
|
|
1609
|
+
function isLive(state, ord) {
|
|
1610
|
+
return nodeExists(state, ord) && !isTombstoned(state, ord);
|
|
1611
|
+
}
|
|
1612
|
+
function noteNodeLeftUnlinked(state) {
|
|
1613
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 1);
|
|
1614
|
+
}
|
|
1615
|
+
function listHolds(list, ord) {
|
|
1616
|
+
for (let i = 0; i < list.size; i++) if (list.ords[i] === ord) return true;
|
|
1617
|
+
return false;
|
|
1618
|
+
}
|
|
1619
|
+
function collectDropped(sorted, kept, dropped) {
|
|
1620
|
+
dropped.size = 0;
|
|
1621
|
+
let next = 0;
|
|
1622
|
+
for (let i = 0; i < sorted.size; i++) {
|
|
1623
|
+
const ord = sorted.ords[i];
|
|
1624
|
+
if (next < kept.size && kept.ords[next] === ord) {
|
|
1625
|
+
next += 1;
|
|
1626
|
+
continue;
|
|
1627
|
+
}
|
|
1628
|
+
if (listHolds(kept, ord)) continue;
|
|
1629
|
+
appendToList(dropped, ord, sorted.distances[i]);
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
function replaceFarthestShared(state, kept, held, ord, distance) {
|
|
1633
|
+
const tried = /* @__PURE__ */ new Set();
|
|
1634
|
+
for (; ; ) {
|
|
1635
|
+
let farthest = -1;
|
|
1636
|
+
for (let k = 0; k < kept.size; k++) {
|
|
1637
|
+
if (tried.has(k) || incomingCount(state.adjacency, kept.ords[k]) < 2) continue;
|
|
1638
|
+
if (farthest === -1 || kept.distances[k] > kept.distances[farthest]) farthest = k;
|
|
1639
|
+
}
|
|
1640
|
+
if (farthest === -1) return false;
|
|
1641
|
+
tried.add(farthest);
|
|
1642
|
+
const replaced = kept.ords[farthest];
|
|
1643
|
+
if (held(replaced) && !releaseUnlessLast(state.adjacency, replaced)) continue;
|
|
1644
|
+
kept.ords[farthest] = ord;
|
|
1645
|
+
kept.distances[farthest] = distance;
|
|
1646
|
+
return true;
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
function writeKeepingLastLinks(state, ord, sorted, kept, held) {
|
|
1650
|
+
const adjacency = state.adjacency;
|
|
1651
|
+
const limit = state.Mmax0;
|
|
1652
|
+
const dropped = state.workspace.droppedLinks;
|
|
1653
|
+
collectDropped(sorted, kept, dropped);
|
|
1654
|
+
ensureListCapacity(kept, limit);
|
|
1655
|
+
for (let i = 0; i < dropped.size; i++) {
|
|
1656
|
+
const candidate = dropped.ords[i];
|
|
1657
|
+
if (!held(candidate)) continue;
|
|
1658
|
+
if (!isLive(state, candidate)) {
|
|
1659
|
+
noteUnlinked(adjacency, candidate);
|
|
1660
|
+
continue;
|
|
1661
|
+
}
|
|
1662
|
+
if (releaseUnlessLast(adjacency, candidate)) continue;
|
|
1663
|
+
if (kept.size < limit) {
|
|
1664
|
+
appendToList(kept, candidate, dropped.distances[i]);
|
|
1665
|
+
continue;
|
|
1666
|
+
}
|
|
1667
|
+
if (replaceFarthestShared(state, kept, held, candidate, dropped.distances[i])) continue;
|
|
1668
|
+
noteUnlinked(adjacency, candidate);
|
|
1669
|
+
noteNodeLeftUnlinked(state);
|
|
1670
|
+
}
|
|
1671
|
+
for (let k = 0; k < kept.size; k++) if (!held(kept.ords[k])) noteLinked(adjacency, kept.ords[k]);
|
|
1672
|
+
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
// src/vector/hnsw/locks.ts
|
|
1676
|
+
var FREE = 0;
|
|
1677
|
+
var EXCLUSIVE = -1;
|
|
1678
|
+
function openGraphLocks(handles, threadSlot) {
|
|
1679
|
+
let words = fixedView(handles.locks, Int32Array);
|
|
1680
|
+
return {
|
|
1681
|
+
threadSlot,
|
|
1682
|
+
header: handles.header,
|
|
1683
|
+
held: handles.heldLocks,
|
|
1684
|
+
words(ord) {
|
|
1685
|
+
if (ord < words.length) return words;
|
|
1686
|
+
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
1687
|
+
words = fixedView(handles.locks, Int32Array);
|
|
1688
|
+
return words;
|
|
1689
|
+
}
|
|
1690
|
+
};
|
|
1691
|
+
}
|
|
1692
|
+
var blockingAllowed = typeof document === "undefined";
|
|
1693
|
+
function canWait(words) {
|
|
1694
|
+
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
1695
|
+
}
|
|
1696
|
+
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
1697
|
+
if (!canWait(words)) return;
|
|
1698
|
+
Atomics.wait(words, index, seen, sliceMs);
|
|
1699
|
+
}
|
|
1700
|
+
function heldIndex(locks, kind) {
|
|
1701
|
+
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
1702
|
+
}
|
|
1703
|
+
function acquireExclusive(words, index) {
|
|
1704
|
+
let spins = 0;
|
|
1705
|
+
for (; ; ) {
|
|
1706
|
+
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
1707
|
+
if (seen === FREE) return;
|
|
1708
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1709
|
+
spins += 1;
|
|
1710
|
+
continue;
|
|
1711
|
+
}
|
|
1712
|
+
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
function releaseExclusive(words, index) {
|
|
1716
|
+
Atomics.store(words, index, FREE);
|
|
1717
|
+
if (canWait(words)) Atomics.notify(words, index);
|
|
1718
|
+
}
|
|
1719
|
+
var WRITE_HELD = 1;
|
|
1720
|
+
var WRITER_WAITING = 2;
|
|
1721
|
+
var VERSION_STEP = 4;
|
|
1722
|
+
function beginNodeRead(locks, ord) {
|
|
1723
|
+
const words = locks.words(ord);
|
|
1724
|
+
for (; ; ) {
|
|
1725
|
+
const seen = Atomics.load(words, ord);
|
|
1726
|
+
if ((seen & WRITE_HELD) === 0) return seen;
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
function nodeReadHeld(locks, ord, version) {
|
|
1730
|
+
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
1731
|
+
return Atomics.load(locks.words(ord), ord) === version;
|
|
1732
|
+
}
|
|
1733
|
+
function lockNodeWrite(locks, ord) {
|
|
1734
|
+
const words = locks.words(ord);
|
|
1735
|
+
let spins = 0;
|
|
1736
|
+
for (; ; ) {
|
|
1737
|
+
const seen = Atomics.load(words, ord);
|
|
1738
|
+
if ((seen & WRITE_HELD) === 0) {
|
|
1739
|
+
const claimed = seen | WRITE_HELD;
|
|
1740
|
+
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
1741
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
1742
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
1743
|
+
return;
|
|
1744
|
+
}
|
|
1745
|
+
continue;
|
|
1746
|
+
}
|
|
1747
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1748
|
+
spins += 1;
|
|
1749
|
+
continue;
|
|
1750
|
+
}
|
|
1751
|
+
const waiting = seen | WRITER_WAITING;
|
|
1752
|
+
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
1753
|
+
waitOn(words, ord, waiting);
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
function releaseWrite(words, ord) {
|
|
1758
|
+
const seen = Atomics.load(words, ord);
|
|
1759
|
+
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
1760
|
+
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
1761
|
+
}
|
|
1762
|
+
function unlockNodeWrite(locks, ord) {
|
|
1763
|
+
releaseWrite(locks.words(ord), ord);
|
|
1764
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
1765
|
+
}
|
|
1766
|
+
function lockGraphShared(locks) {
|
|
1767
|
+
const { header } = locks;
|
|
1768
|
+
let spins = 0;
|
|
1769
|
+
for (; ; ) {
|
|
1770
|
+
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
1771
|
+
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
1772
|
+
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
1773
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
1774
|
+
return;
|
|
1775
|
+
}
|
|
1776
|
+
continue;
|
|
1777
|
+
}
|
|
1778
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1779
|
+
spins += 1;
|
|
1780
|
+
continue;
|
|
1781
|
+
}
|
|
1782
|
+
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
function unlockGraphShared(locks) {
|
|
1786
|
+
const { header } = locks;
|
|
1787
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
1788
|
+
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
1789
|
+
}
|
|
1790
|
+
function lockGraphExclusive(locks) {
|
|
1791
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
1792
|
+
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1793
|
+
try {
|
|
1794
|
+
acquireExclusive(locks.header, GRAPH_LOCK);
|
|
1795
|
+
} finally {
|
|
1796
|
+
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1797
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
function unlockGraphExclusive(locks) {
|
|
1801
|
+
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
1802
|
+
}
|
|
1803
|
+
function lockEntry(locks) {
|
|
1804
|
+
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1805
|
+
}
|
|
1806
|
+
function unlockEntry(locks) {
|
|
1807
|
+
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1808
|
+
}
|
|
1809
|
+
function releaseLocksHeldBy(handles, threadSlot) {
|
|
1810
|
+
const held = handles.heldLocks;
|
|
1811
|
+
const words = new Int32Array(handles.locks);
|
|
1812
|
+
const base = threadSlot * HELD_WORDS_PER_THREAD;
|
|
1813
|
+
if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
|
|
1814
|
+
Atomics.sub(handles.header, GRAPH_WRITERS_WAITING, 1);
|
|
1815
|
+
}
|
|
1816
|
+
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
1817
|
+
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
1818
|
+
if (writeOrd >= 0 && writeOrd < words.length) {
|
|
1819
|
+
const seen = Atomics.load(words, writeOrd);
|
|
1820
|
+
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
1821
|
+
}
|
|
1822
|
+
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
1823
|
+
for (; ; ) {
|
|
1824
|
+
const readers = Atomics.load(handles.header, GRAPH_LOCK);
|
|
1825
|
+
if (readers <= FREE) return;
|
|
1826
|
+
if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
1827
|
+
if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
|
|
1828
|
+
return;
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
// src/vector/hnsw/connect-unreached.ts
|
|
1834
|
+
var NOT_REACHED_THROUGH_A_NODE = -1;
|
|
1835
|
+
function reach(walk, node, through) {
|
|
1836
|
+
walk.reached[node] = 1;
|
|
1837
|
+
walk.firstReachedThrough[node] = through;
|
|
1838
|
+
walk.queue[walk.tail++] = node;
|
|
1839
|
+
}
|
|
1840
|
+
function walkFrom(state, walk, start, through) {
|
|
1841
|
+
const adjacency = state.adjacency;
|
|
1842
|
+
if (walk.reached[start] === 1) return;
|
|
1843
|
+
reach(walk, start, through);
|
|
1844
|
+
while (walk.head < walk.tail) {
|
|
1845
|
+
const ord = walk.queue[walk.head++];
|
|
1846
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1847
|
+
if (base === ABSENT) continue;
|
|
1848
|
+
const count = adjacency.level0[base];
|
|
1849
|
+
for (let i = 1; i <= count; i++) {
|
|
1850
|
+
const next = adjacency.level0[base + i];
|
|
1851
|
+
if (next < 0 || next >= walk.reached.length || walk.reached[next] === 1 || !nodeExists(state, next)) continue;
|
|
1852
|
+
reach(walk, next, ord);
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
function hasRoom(state, _walk, host) {
|
|
1857
|
+
const base = layerBase(state.adjacency, host, 0);
|
|
1858
|
+
return base !== ABSENT && state.adjacency.level0[base] < state.Mmax0;
|
|
1859
|
+
}
|
|
1860
|
+
function reachedThroughAnotherNode(walk, host, neighbour) {
|
|
1861
|
+
if (neighbour < 0 || neighbour >= walk.reached.length || walk.reached[neighbour] !== 1) return false;
|
|
1862
|
+
const through = walk.firstReachedThrough[neighbour];
|
|
1863
|
+
return through !== NOT_REACHED_THROUGH_A_NODE && through !== host;
|
|
1864
|
+
}
|
|
1865
|
+
function holdsLinkOutsideTheWalk(state, walk, host) {
|
|
1866
|
+
const adjacency = state.adjacency;
|
|
1867
|
+
const base = layerBase(adjacency, host, 0);
|
|
1868
|
+
if (base === ABSENT) return false;
|
|
1869
|
+
const count = adjacency.level0[base];
|
|
1870
|
+
for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
|
|
1871
|
+
return false;
|
|
1872
|
+
}
|
|
1873
|
+
function listOf(state, ord) {
|
|
1874
|
+
const adjacency = state.adjacency;
|
|
1875
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1876
|
+
if (base === ABSENT) return new Int32Array(0);
|
|
1877
|
+
return adjacency.level0.slice(base + 1, base + 1 + adjacency.level0[base]);
|
|
1878
|
+
}
|
|
1879
|
+
function nearestHost(state, walk, orphan, nearby, test) {
|
|
1880
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
1881
|
+
let best = -1;
|
|
1882
|
+
let bestDistance = Number.POSITIVE_INFINITY;
|
|
1883
|
+
const consider = (host) => {
|
|
1884
|
+
if (host === orphan || walk.reached[host] !== 1 || !isLive(state, host) || !test(state, walk, host)) return;
|
|
1885
|
+
const away = distance(orphan, host);
|
|
1886
|
+
if (best === -1 || away < bestDistance) {
|
|
1887
|
+
best = host;
|
|
1888
|
+
bestDistance = away;
|
|
1889
|
+
}
|
|
1890
|
+
};
|
|
1891
|
+
for (const host of nearby.list) consider(host);
|
|
1892
|
+
for (const host of listOf(state, orphan))
|
|
1893
|
+
if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
|
|
1894
|
+
if (best !== -1) return best;
|
|
1895
|
+
for (let host = 0; host < walk.reached.length; host++) consider(host);
|
|
1896
|
+
return best;
|
|
1897
|
+
}
|
|
1898
|
+
function takeOutFarthestLinkOutsideTheWalk(state, walk, host) {
|
|
1899
|
+
const adjacency = state.adjacency;
|
|
1900
|
+
const base = layerBase(adjacency, host, 0);
|
|
1901
|
+
if (base === ABSENT) return;
|
|
1902
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
1903
|
+
const count = adjacency.level0[base];
|
|
1904
|
+
let farthest = -1;
|
|
1905
|
+
let farthestDistance = Number.NEGATIVE_INFINITY;
|
|
1906
|
+
for (let i = 1; i <= count; i++) {
|
|
1907
|
+
const neighbour = adjacency.level0[base + i];
|
|
1908
|
+
if (!reachedThroughAnotherNode(walk, host, neighbour)) continue;
|
|
1909
|
+
const away = distance(host, neighbour);
|
|
1910
|
+
if (farthest === -1 || away > farthestDistance) {
|
|
1911
|
+
farthest = i;
|
|
1912
|
+
farthestDistance = away;
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
if (farthest === -1) return;
|
|
1916
|
+
const taken = adjacency.level0[base + farthest];
|
|
1917
|
+
for (let i = farthest; i < count; i++) adjacency.level0[base + i] = adjacency.level0[base + i + 1];
|
|
1918
|
+
adjacency.level0[base] = count - 1;
|
|
1919
|
+
noteUnlinked(adjacency, taken);
|
|
1920
|
+
}
|
|
1921
|
+
function appendTo(state, host, orphan) {
|
|
1922
|
+
const adjacency = state.adjacency;
|
|
1923
|
+
const base = layerBase(adjacency, host, 0);
|
|
1924
|
+
if (base === ABSENT) return;
|
|
1925
|
+
const count = adjacency.level0[base];
|
|
1926
|
+
adjacency.level0[base + count + 1] = orphan;
|
|
1927
|
+
adjacency.level0[base] = count + 1;
|
|
1928
|
+
noteLinked(adjacency, orphan);
|
|
1929
|
+
}
|
|
1930
|
+
function connectUnreached(state, nearby) {
|
|
1931
|
+
const slots = adjacencySlots(state.adjacency);
|
|
1932
|
+
const entry = entryPointOf(state);
|
|
1933
|
+
if (entry < 0 || entry >= slots || !nodeExists(state, entry)) return;
|
|
1934
|
+
const walk = {
|
|
1935
|
+
reached: new Uint8Array(slots),
|
|
1936
|
+
firstReachedThrough: new Int32Array(slots),
|
|
1937
|
+
queue: new Int32Array(slots),
|
|
1938
|
+
head: 0,
|
|
1939
|
+
tail: 0
|
|
1940
|
+
};
|
|
1941
|
+
walkFrom(state, walk, entry, NOT_REACHED_THROUGH_A_NODE);
|
|
1942
|
+
const held = { list: [], holds: new Uint8Array(slots) };
|
|
1943
|
+
for (let i = 0; i < nearby.length; i++) {
|
|
1944
|
+
const ord = nearby[i];
|
|
1945
|
+
if (ord < 0 || ord >= slots || held.holds[ord] === 1) continue;
|
|
1946
|
+
held.holds[ord] = 1;
|
|
1947
|
+
held.list.push(ord);
|
|
1948
|
+
}
|
|
1949
|
+
for (let orphan = 0; orphan < slots; orphan++) {
|
|
1950
|
+
if (walk.reached[orphan] === 1 || !isLive(state, orphan)) continue;
|
|
1951
|
+
let host = nearestHost(state, walk, orphan, held, hasRoom);
|
|
1952
|
+
if (host === -1) {
|
|
1953
|
+
host = nearestHost(state, walk, orphan, held, holdsLinkOutsideTheWalk);
|
|
1954
|
+
if (host === -1) continue;
|
|
1955
|
+
takeOutFarthestLinkOutsideTheWalk(state, walk, host);
|
|
1956
|
+
}
|
|
1957
|
+
appendTo(state, host, orphan);
|
|
1958
|
+
walkFrom(state, walk, orphan, host);
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
function holdsUnlinkedNode(state) {
|
|
1962
|
+
const slots = adjacencySlots(state.adjacency);
|
|
1963
|
+
const entry = entryPointOf(state);
|
|
1964
|
+
const incoming = state.adjacency.incoming;
|
|
1965
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
1966
|
+
if (incoming[ord] === 0 && ord !== entry && isLive(state, ord)) return true;
|
|
1967
|
+
}
|
|
1968
|
+
return false;
|
|
1969
|
+
}
|
|
1970
|
+
function connectThroughEitherCore(state, nearby) {
|
|
1971
|
+
const field = nativeFieldFor(state);
|
|
1972
|
+
if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
|
|
1973
|
+
noteFallback("connect");
|
|
1974
|
+
connectUnreached(state, nearby);
|
|
1975
|
+
}
|
|
1976
|
+
function connectAfterRemoval(state, nearby) {
|
|
1977
|
+
connectThroughEitherCore(state, nearby);
|
|
1978
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1979
|
+
}
|
|
1980
|
+
function connectAfterPlacement(state, holdsGraphLock) {
|
|
1981
|
+
if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
|
|
1982
|
+
if (!holdsGraphLock) lockGraphExclusive(state.locks);
|
|
1983
|
+
try {
|
|
1984
|
+
if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
|
|
1985
|
+
connectThroughEitherCore(state, new Int32Array(0));
|
|
1986
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1987
|
+
} finally {
|
|
1988
|
+
if (!holdsGraphLock) unlockGraphExclusive(state.locks);
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
function baseNeighboursOfRemoved(state, removed) {
|
|
1992
|
+
const leaving = new Set(removed);
|
|
1993
|
+
const nearby = [];
|
|
1994
|
+
for (const ord of removed) {
|
|
1995
|
+
for (const neighbour of listOf(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
|
|
1996
|
+
}
|
|
1997
|
+
return Int32Array.from(nearby);
|
|
1998
|
+
}
|
|
1999
|
+
function leaveIncomingUncounted(state) {
|
|
2000
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
2001
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 1);
|
|
2002
|
+
}
|
|
2003
|
+
function countIncomingBeforeWriting(state, holdsGraphLock) {
|
|
2004
|
+
if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
|
|
2005
|
+
if (!holdsGraphLock) lockGraphExclusive(state.locks);
|
|
2006
|
+
try {
|
|
2007
|
+
if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
|
|
2008
|
+
const adjacency = state.adjacency;
|
|
2009
|
+
const slots = adjacencySlots(adjacency);
|
|
2010
|
+
if (slots > 0) reachNode(adjacency, slots - 1);
|
|
2011
|
+
adjacency.incoming.fill(0);
|
|
2012
|
+
countEveryLink(adjacency, slots);
|
|
2013
|
+
if (holdsUnlinkedNode(state)) connectThroughEitherCore(state, new Int32Array(0));
|
|
2014
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
2015
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
|
|
2016
|
+
} finally {
|
|
2017
|
+
if (!holdsGraphLock) unlockGraphExclusive(state.locks);
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
|
|
1659
2021
|
// src/vector/hnsw/graph-ops.ts
|
|
1660
2022
|
function readNeighborsLocked(state, ord, layer) {
|
|
1661
2023
|
for (; ; ) {
|
|
@@ -1752,14 +2114,24 @@ function pruneConnections(state, ord, layer, distance) {
|
|
|
1752
2114
|
for (let i = 1; i <= count; i++) {
|
|
1753
2115
|
const connOrd = neighbors[base + i];
|
|
1754
2116
|
const dist = distance(ord, connOrd);
|
|
1755
|
-
if (dist === Number.POSITIVE_INFINITY)
|
|
2117
|
+
if (dist === Number.POSITIVE_INFINITY) {
|
|
2118
|
+
if (layer === 0) noteUnlinked(adjacency, connOrd);
|
|
2119
|
+
continue;
|
|
2120
|
+
}
|
|
1756
2121
|
candidates.ords[candidates.size] = connOrd;
|
|
1757
2122
|
candidates.distances[candidates.size] = dist;
|
|
1758
2123
|
candidates.size += 1;
|
|
1759
2124
|
}
|
|
1760
2125
|
const kept = state.workspace.pruneSelection;
|
|
1761
2126
|
selectNeighborsHeuristic(state, candidates, mc, distance, kept);
|
|
1762
|
-
|
|
2127
|
+
if (layer === 0) {
|
|
2128
|
+
writeKeepingLastLinks(state, ord, state.workspace.working, kept, everyCandidateWasHeld);
|
|
2129
|
+
return;
|
|
2130
|
+
}
|
|
2131
|
+
writeNeighbors(adjacency, ord, layer, kept.ords, kept.size);
|
|
2132
|
+
}
|
|
2133
|
+
function everyCandidateWasHeld() {
|
|
2134
|
+
return true;
|
|
1763
2135
|
}
|
|
1764
2136
|
|
|
1765
2137
|
// src/vector/hnsw/mutation.ts
|
|
@@ -1873,11 +2245,15 @@ function linkNeighborsBack(state, ord, linkTop) {
|
|
|
1873
2245
|
}
|
|
1874
2246
|
function linkNode(state, ord, level2) {
|
|
1875
2247
|
const topLayer = topLayerOf(state);
|
|
2248
|
+
const formerEntry = entryPointOf(state);
|
|
1876
2249
|
const linkTop = selectNeighborsPerLayer(state, ord, level2);
|
|
1877
2250
|
if (linkTop < 0) return;
|
|
1878
2251
|
writeOwnLists(state, ord, linkTop);
|
|
1879
2252
|
linkNeighborsBack(state, ord, linkTop);
|
|
1880
|
-
if (level2
|
|
2253
|
+
if (level2 <= topLayer) return;
|
|
2254
|
+
raiseEntry(state, ord, level2);
|
|
2255
|
+
const demoted = entryPointOf(state) !== formerEntry && isLive(state, formerEntry);
|
|
2256
|
+
if (demoted && incomingCount(state.adjacency, formerEntry) === 0) noteNodeLeftUnlinked(state);
|
|
1881
2257
|
}
|
|
1882
2258
|
function writeRecordBeforePlacement(state, ord) {
|
|
1883
2259
|
const quantizer = state.quantizer;
|
|
@@ -1903,8 +2279,12 @@ function insertNode(state, ord, holdsGraphLock = false) {
|
|
|
1903
2279
|
if (!state.store.holdsOrdinal(ord)) return false;
|
|
1904
2280
|
ensureCapacity(state, ord + 1);
|
|
1905
2281
|
if (nodeExists(state, ord)) return false;
|
|
2282
|
+
countIncomingBeforeWriting(state, holdsGraphLock);
|
|
1906
2283
|
writeRecordBeforePlacement(state, ord);
|
|
1907
|
-
if (placeThroughTheCore(state, ord, holdsGraphLock))
|
|
2284
|
+
if (placeThroughTheCore(state, ord, holdsGraphLock)) {
|
|
2285
|
+
connectAfterPlacement(state, holdsGraphLock);
|
|
2286
|
+
return true;
|
|
2287
|
+
}
|
|
1908
2288
|
if (nodeExists(state, ord)) return false;
|
|
1909
2289
|
noteFallback("place");
|
|
1910
2290
|
if (!holdsGraphLock) lockGraphShared(state.locks);
|
|
@@ -1913,11 +2293,11 @@ function insertNode(state, ord, holdsGraphLock = false) {
|
|
|
1913
2293
|
const level2 = randomLevel(state.mL);
|
|
1914
2294
|
createNode(state.adjacency, ord, level2);
|
|
1915
2295
|
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
1916
|
-
if (claimEntryIfEmpty(state, ord, level2))
|
|
1917
|
-
linkNode(state, ord, level2);
|
|
2296
|
+
if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
|
|
1918
2297
|
} finally {
|
|
1919
2298
|
if (!holdsGraphLock) unlockGraphShared(state.locks);
|
|
1920
2299
|
}
|
|
2300
|
+
connectAfterPlacement(state, holdsGraphLock);
|
|
1921
2301
|
return true;
|
|
1922
2302
|
}
|
|
1923
2303
|
function removeNodeEager(state, ord, excludeOrds) {
|
|
@@ -1935,7 +2315,8 @@ function removeNodeEager(state, ord, excludeOrds) {
|
|
|
1935
2315
|
if (layer > nodeMaxLayer(state, neighborOrd)) continue;
|
|
1936
2316
|
const mc = maxConns(state, layer);
|
|
1937
2317
|
if (neighborCount(state.adjacency, neighborOrd, layer) >= mc) continue;
|
|
1938
|
-
const
|
|
2318
|
+
const heldNow = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
|
|
2319
|
+
const candidateOrds = new Set(heldNow);
|
|
1939
2320
|
for (const otherOrd of formerNeighbors) {
|
|
1940
2321
|
if (otherOrd !== neighborOrd && otherOrd !== ord) {
|
|
1941
2322
|
if (excludeOrds?.has(otherOrd)) continue;
|
|
@@ -1946,12 +2327,19 @@ function removeNodeEager(state, ord, excludeOrds) {
|
|
|
1946
2327
|
candidates.size = 0;
|
|
1947
2328
|
for (const candOrd of candidateOrds) {
|
|
1948
2329
|
const dist = distance(neighborOrd, candOrd);
|
|
1949
|
-
if (dist === Number.POSITIVE_INFINITY)
|
|
2330
|
+
if (dist === Number.POSITIVE_INFINITY) {
|
|
2331
|
+
if (layer === 0 && heldNow.has(candOrd)) noteUnlinked(state.adjacency, candOrd);
|
|
2332
|
+
continue;
|
|
2333
|
+
}
|
|
1950
2334
|
appendToList(candidates, candOrd, dist);
|
|
1951
2335
|
}
|
|
1952
2336
|
const selected = state.workspace.repairSelection;
|
|
1953
2337
|
selectNeighborsHeuristic(state, candidates, mc, distance, selected);
|
|
1954
|
-
|
|
2338
|
+
if (layer === 0) {
|
|
2339
|
+
writeKeepingLastLinks(state, neighborOrd, state.workspace.working, selected, (held) => heldNow.has(held));
|
|
2340
|
+
} else {
|
|
2341
|
+
replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
|
|
2342
|
+
}
|
|
1955
2343
|
for (let i = 0; i < selected.size; i++) {
|
|
1956
2344
|
const newConnOrd = selected.ords[i];
|
|
1957
2345
|
if (layer <= nodeMaxLayer(state, newConnOrd)) {
|
|
@@ -1961,6 +2349,8 @@ function removeNodeEager(state, ord, excludeOrds) {
|
|
|
1961
2349
|
}
|
|
1962
2350
|
}
|
|
1963
2351
|
}
|
|
2352
|
+
const formerBaseNeighbors = collectNeighbors(state.adjacency, ord, 0);
|
|
2353
|
+
for (const neighborOrd of formerBaseNeighbors) noteUnlinked(state.adjacency, neighborOrd);
|
|
1964
2354
|
deleteNode(state.adjacency, ord);
|
|
1965
2355
|
Atomics.sub(state.header, GRAPH_NODE_COUNT, 1);
|
|
1966
2356
|
clearTombstone(state, ord);
|
|
@@ -1984,9 +2374,7 @@ function markTombstone(state, ord) {
|
|
|
1984
2374
|
}
|
|
1985
2375
|
function compactTombstones(state) {
|
|
1986
2376
|
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
1987
|
-
|
|
1988
|
-
if (field !== null && nativeCompact(state, field, state.buildMetric)) return;
|
|
1989
|
-
noteFallback("compact");
|
|
2377
|
+
countIncomingBeforeWriting(state, true);
|
|
1990
2378
|
const tombstonedOrds = [];
|
|
1991
2379
|
const slots = adjacencySlots(state.adjacency);
|
|
1992
2380
|
for (let ord = 0; ord < slots; ord++) {
|
|
@@ -1994,10 +2382,16 @@ function compactTombstones(state) {
|
|
|
1994
2382
|
tombstonedOrds.push(ord);
|
|
1995
2383
|
}
|
|
1996
2384
|
}
|
|
1997
|
-
const
|
|
1998
|
-
|
|
1999
|
-
|
|
2385
|
+
const nearby = baseNeighboursOfRemoved(state, tombstonedOrds);
|
|
2386
|
+
const field = nativeFieldFor(state);
|
|
2387
|
+
if (field === null || !nativeCompact(state, field, state.buildMetric)) {
|
|
2388
|
+
noteFallback("compact");
|
|
2389
|
+
const excl = new Set(tombstonedOrds);
|
|
2390
|
+
for (const ord of tombstonedOrds) {
|
|
2391
|
+
removeNodeEager(state, ord, excl);
|
|
2392
|
+
}
|
|
2000
2393
|
}
|
|
2394
|
+
connectAfterRemoval(state, nearby);
|
|
2001
2395
|
}
|
|
2002
2396
|
function resetGraph(state) {
|
|
2003
2397
|
resetAdjacency(state.adjacency);
|
|
@@ -2007,6 +2401,7 @@ function resetGraph(state) {
|
|
|
2007
2401
|
Atomics.store(state.header, GRAPH_NODE_COUNT, 0);
|
|
2008
2402
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, -1);
|
|
2009
2403
|
Atomics.store(state.header, GRAPH_TOP_LAYER, -1);
|
|
2404
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
|
|
2010
2405
|
}
|
|
2011
2406
|
function rebuild(state) {
|
|
2012
2407
|
const nodeCount = Atomics.load(state.header, GRAPH_NODE_COUNT);
|
|
@@ -2485,7 +2880,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2485
2880
|
let centroidGeneration = -1;
|
|
2486
2881
|
let currentQuery = null;
|
|
2487
2882
|
const singleOrdinal = new Int32Array(1);
|
|
2488
|
-
function
|
|
2883
|
+
function reach2(ordinal) {
|
|
2489
2884
|
if (ordinal < codePresent.length) return true;
|
|
2490
2885
|
if (ordinal >= store.handles.codePresent.byteLength) return false;
|
|
2491
2886
|
codePresent = fixedView(store.handles.codePresent, Uint8Array);
|
|
@@ -2506,8 +2901,8 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2506
2901
|
if (generation !== centroidGeneration) recomputeCentroidDot(generation);
|
|
2507
2902
|
return centroidDot;
|
|
2508
2903
|
}
|
|
2509
|
-
function
|
|
2510
|
-
return ordinal >= 0 &&
|
|
2904
|
+
function holds2(ordinal) {
|
|
2905
|
+
return ordinal >= 0 && reach2(ordinal) && codePresent[ordinal] === 1;
|
|
2511
2906
|
}
|
|
2512
2907
|
function recordOffset(ordinal) {
|
|
2513
2908
|
return slotByteOffset(layout, ordinal % capacity);
|
|
@@ -2528,7 +2923,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2528
2923
|
}
|
|
2529
2924
|
}
|
|
2530
2925
|
function writeRecordOf(ordinal) {
|
|
2531
|
-
if (!
|
|
2926
|
+
if (!reach2(ordinal) || !store.holdsOrdinal(ordinal)) return;
|
|
2532
2927
|
const code = osqQuantize(store.vectorAt(ordinal), store.handles.centroid, bits, metric, scratch);
|
|
2533
2928
|
const block = store.codeBlockOf(ordinal);
|
|
2534
2929
|
const offset = recordOffset(ordinal);
|
|
@@ -2599,7 +2994,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2599
2994
|
return currentQuery;
|
|
2600
2995
|
}
|
|
2601
2996
|
function distanceBetween(ordA, ordB) {
|
|
2602
|
-
if (!
|
|
2997
|
+
if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
|
|
2603
2998
|
const blockA = store.codeBlockOf(ordA);
|
|
2604
2999
|
const blockB = store.codeBlockOf(ordB);
|
|
2605
3000
|
const offsetA = recordOffset(ordA);
|
|
@@ -2627,7 +3022,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2627
3022
|
return isCalibrated() ? store.handles.centroid : null;
|
|
2628
3023
|
},
|
|
2629
3024
|
isCalibrated,
|
|
2630
|
-
holdsOrdinal:
|
|
3025
|
+
holdsOrdinal: holds2,
|
|
2631
3026
|
writeCodes(ordinal) {
|
|
2632
3027
|
singleOrdinal[0] = ordinal;
|
|
2633
3028
|
writeCodesOf(singleOrdinal);
|
|
@@ -2642,7 +3037,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2642
3037
|
return true;
|
|
2643
3038
|
},
|
|
2644
3039
|
restoreRecord(ordinal, record) {
|
|
2645
|
-
if (!
|
|
3040
|
+
if (!reach2(ordinal) || record.length !== layout.slotStride) return;
|
|
2646
3041
|
const block = store.codeBlockOf(ordinal);
|
|
2647
3042
|
const offset = recordOffset(ordinal);
|
|
2648
3043
|
block.bytes(offset + layout.slotStride).set(record, offset);
|
|
@@ -2654,7 +3049,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2654
3049
|
return block.bytes(offset + layout.slotStride).subarray(offset, offset + layout.slotStride);
|
|
2655
3050
|
},
|
|
2656
3051
|
clearCodes(ordinal) {
|
|
2657
|
-
if (ordinal < 0 || !
|
|
3052
|
+
if (ordinal < 0 || !reach2(ordinal)) return;
|
|
2658
3053
|
if (Atomics.exchange(codePresent, ordinal, 0) === 1) Atomics.sub(store.handles.header, STORE_CODE_COUNT, 1);
|
|
2659
3054
|
},
|
|
2660
3055
|
writeCentroid,
|
|
@@ -2670,21 +3065,21 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2670
3065
|
},
|
|
2671
3066
|
prepareQuery,
|
|
2672
3067
|
distanceFromPreparedByOrdinal(prepared, ordinal) {
|
|
2673
|
-
if (!
|
|
3068
|
+
if (!holds2(ordinal)) return Number.POSITIVE_INFINITY;
|
|
2674
3069
|
return estimateDistance(ordinal, prepared);
|
|
2675
3070
|
},
|
|
2676
3071
|
preparedDistance(prepared) {
|
|
2677
|
-
const generic = (ordinal) =>
|
|
3072
|
+
const generic = (ordinal) => holds2(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
|
|
2678
3073
|
if (store.handles.codeBlocks.length === 0) return generic;
|
|
2679
3074
|
const block = store.codeBlockOf(0);
|
|
2680
3075
|
const simd = block.simd;
|
|
2681
3076
|
if (simd === null || !block.hasScratch) return generic;
|
|
2682
3077
|
const presentView = codePresent;
|
|
2683
3078
|
const limit = Math.min(presentView.length, capacity);
|
|
2684
|
-
const
|
|
3079
|
+
const { slotsOffset, slotStride } = layout;
|
|
3080
|
+
const data = block.data(slotsOffset + slotStride);
|
|
2685
3081
|
const dataLength = data.byteLength;
|
|
2686
3082
|
const queryOffset = block.scratchByteOffset;
|
|
2687
|
-
const { slotsOffset, slotStride } = layout;
|
|
2688
3083
|
return (ordinal) => {
|
|
2689
3084
|
if (ordinal < 0 || ordinal >= limit || block.stagedOrdinal !== QUERY_STAGED) return generic(ordinal);
|
|
2690
3085
|
if (presentView[ordinal] !== 1) return Number.POSITIVE_INFINITY;
|
|
@@ -2713,9 +3108,9 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2713
3108
|
if (simd === null) return distanceBetween;
|
|
2714
3109
|
const presentView = codePresent;
|
|
2715
3110
|
const limit = Math.min(presentView.length, capacity);
|
|
2716
|
-
const data = block.data(0);
|
|
2717
|
-
const dataLength = data.byteLength;
|
|
2718
3111
|
const { slotsOffset, slotStride } = layout;
|
|
3112
|
+
const data = block.data(slotsOffset + slotStride);
|
|
3113
|
+
const dataLength = data.byteLength;
|
|
2719
3114
|
return (ordA, ordB) => {
|
|
2720
3115
|
if (ordA < 0 || ordB < 0 || ordA >= limit || ordB >= limit) return distanceBetween(ordA, ordB);
|
|
2721
3116
|
if (presentView[ordA] !== 1 || presentView[ordB] !== 1) return Number.POSITIVE_INFINITY;
|
|
@@ -2910,7 +3305,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
2910
3305
|
const index = Math.floor(ordinal / capacity);
|
|
2911
3306
|
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
2912
3307
|
}
|
|
2913
|
-
function
|
|
3308
|
+
function holds2(ordinal) {
|
|
2914
3309
|
return holdState(ordinal) !== NOT_HELD;
|
|
2915
3310
|
}
|
|
2916
3311
|
function readFromFile(file, offset, target) {
|
|
@@ -2941,7 +3336,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
2941
3336
|
if (block.stagedOrdinal === ordinal) return true;
|
|
2942
3337
|
const source = vectorFor(ordinal, coldScratchA);
|
|
2943
3338
|
if (source === null) return false;
|
|
2944
|
-
block.float32(
|
|
3339
|
+
block.float32(block.scratchByteOffset / 4 + dimension).set(source, block.scratchByteOffset / 4);
|
|
2945
3340
|
block.stagedOrdinal = ordinal;
|
|
2946
3341
|
return true;
|
|
2947
3342
|
}
|
|
@@ -2955,7 +3350,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
2955
3350
|
function stageQuery(block) {
|
|
2956
3351
|
if (currentQuery === null) return false;
|
|
2957
3352
|
if (block.stagedOrdinal !== QUERY_STAGED) {
|
|
2958
|
-
block.float32(
|
|
3353
|
+
block.float32(block.scratchByteOffset / 4 + dimension).set(currentQuery, block.scratchByteOffset / 4);
|
|
2959
3354
|
block.stagedOrdinal = QUERY_STAGED;
|
|
2960
3355
|
}
|
|
2961
3356
|
return true;
|
|
@@ -2990,7 +3385,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
2990
3385
|
reader?.retainOnly(new Set(next.vectorFiles.filter((path) => path.length > 0)));
|
|
2991
3386
|
},
|
|
2992
3387
|
rebind,
|
|
2993
|
-
holdsOrdinal:
|
|
3388
|
+
holdsOrdinal: holds2,
|
|
2994
3389
|
isCold,
|
|
2995
3390
|
readColdInto,
|
|
2996
3391
|
readFromFile,
|
|
@@ -3031,7 +3426,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
3031
3426
|
return magnitudes[ordinal];
|
|
3032
3427
|
},
|
|
3033
3428
|
entryForOrdinal(ordinal) {
|
|
3034
|
-
if (!
|
|
3429
|
+
if (!holds2(ordinal)) return void 0;
|
|
3035
3430
|
return { vector: view.vectorAt(ordinal), magnitude: magnitudes[ordinal] };
|
|
3036
3431
|
},
|
|
3037
3432
|
docIdAt(ordinal) {
|
|
@@ -3053,7 +3448,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
3053
3448
|
return partition < 0 ? void 0 : partition;
|
|
3054
3449
|
},
|
|
3055
3450
|
distanceByOrdinal(ordA, ordB, metric) {
|
|
3056
|
-
if (!
|
|
3451
|
+
if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
|
|
3057
3452
|
if (!isCold(ordB)) {
|
|
3058
3453
|
const indexA = ordA / capacity | 0;
|
|
3059
3454
|
const indexB = ordB / capacity | 0;
|
|
@@ -3175,6 +3570,9 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
3175
3570
|
get readsEveryVector() {
|
|
3176
3571
|
return store.holdsEveryBlock;
|
|
3177
3572
|
},
|
|
3573
|
+
get graphRetired() {
|
|
3574
|
+
return graph !== null && Atomics.load(graph.header, GRAPH_RETIRED) === 1;
|
|
3575
|
+
},
|
|
3178
3576
|
docIdOf,
|
|
3179
3577
|
ordinalOf,
|
|
3180
3578
|
vectorOf(docId) {
|
|
@@ -3463,6 +3861,9 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
3463
3861
|
const { handles } = store;
|
|
3464
3862
|
growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
|
|
3465
3863
|
growBufferTo(handles.present, ordinal + 1);
|
|
3864
|
+
growBufferTo(handles.unplaced, ordinal + 1);
|
|
3865
|
+
growBufferTo(handles.unplacedChunks, unplacedChunkCount(ordinal) * 4);
|
|
3866
|
+
growBufferTo(handles.removed, ordinal + 1);
|
|
3466
3867
|
growBufferTo(handles.partitions, (ordinal + 1) * 4);
|
|
3467
3868
|
growBufferTo(handles.codePresent, ordinal + 1);
|
|
3468
3869
|
const diskFileBytes = handles.diskFile.byteLength;
|
|
@@ -4023,7 +4424,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
|
|
|
4023
4424
|
}
|
|
4024
4425
|
cursor += count + 1;
|
|
4025
4426
|
if (ordinal !== NO_ORDINAL && layer <= keptTop) {
|
|
4026
|
-
|
|
4427
|
+
writeNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
|
|
4027
4428
|
}
|
|
4028
4429
|
}
|
|
4029
4430
|
}
|
|
@@ -4035,6 +4436,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
|
|
|
4035
4436
|
const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
|
|
4036
4437
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
|
|
4037
4438
|
Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
|
|
4439
|
+
leaveIncomingUncounted(state);
|
|
4038
4440
|
}
|
|
4039
4441
|
|
|
4040
4442
|
// src/vector/hnsw/persistence.ts
|
|
@@ -4073,7 +4475,7 @@ function deserializeGraph(state, store, data) {
|
|
|
4073
4475
|
for (const node of nodes) {
|
|
4074
4476
|
createNode(state.adjacency, node.ord, node.maxLayer);
|
|
4075
4477
|
for (const [layer, neighbors] of node.layers) {
|
|
4076
|
-
|
|
4478
|
+
writeNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
|
|
4077
4479
|
}
|
|
4078
4480
|
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
4079
4481
|
}
|
|
@@ -4099,6 +4501,7 @@ function deserializeGraph(state, store, data) {
|
|
|
4099
4501
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, bestOrd);
|
|
4100
4502
|
Atomics.store(state.header, GRAPH_TOP_LAYER, bestLayer);
|
|
4101
4503
|
}
|
|
4504
|
+
leaveIncomingUncounted(state);
|
|
4102
4505
|
}
|
|
4103
4506
|
|
|
4104
4507
|
// src/vector/hnsw/scratch-bytes.ts
|
|
@@ -4159,6 +4562,7 @@ function restoreSnapshot(state, snapshot) {
|
|
|
4159
4562
|
Atomics.store(state.header, GRAPH_NODE_COUNT, snapshot.nodeCount);
|
|
4160
4563
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, snapshot.entryPointOrd);
|
|
4161
4564
|
Atomics.store(state.header, GRAPH_TOP_LAYER, snapshot.topLayer);
|
|
4565
|
+
leaveIncomingUncounted(state);
|
|
4162
4566
|
}
|
|
4163
4567
|
|
|
4164
4568
|
// src/vector/hnsw/index.ts
|
|
@@ -4270,6 +4674,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4270
4674
|
},
|
|
4271
4675
|
search: (query, k, searchMetric, minSimilarity, options) => search(state, docIdOf, query, k, searchMetric, minSimilarity, options),
|
|
4272
4676
|
clear: () => exclusively(() => resetGraph(state)),
|
|
4677
|
+
retire: () => Atomics.store(shared.header, GRAPH_RETIRED, 1),
|
|
4273
4678
|
entries: entriesIterator,
|
|
4274
4679
|
compactionNeeded,
|
|
4275
4680
|
compactTombstones: () => exclusively(() => compactTombstones(state)),
|
|
@@ -4283,4 +4688,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4283
4688
|
};
|
|
4284
4689
|
}
|
|
4285
4690
|
|
|
4286
|
-
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct,
|
|
4691
|
+
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, STORE_UNPLACED_COUNT, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, euclideanDistance, graphWithoutNodes, isSimdAvailable, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
|