@delali/narsil 0.3.1-next.84 → 0.3.1-next.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XIQZKKTK.mjs → chunk-67MQF5CV.mjs} +39 -10
- package/dist/{chunk-B3UOQJD3.mjs → chunk-NFF5TDRG.mjs} +1 -1
- package/dist/{chunk-RZXNWQSM.mjs → chunk-NIWDQTZB.mjs} +591 -193
- package/dist/{chunk-IA67SOJO.mjs → chunk-RCSXJCI5.mjs} +271 -93
- package/dist/{chunk-XZF4LZ3X.mjs → chunk-RM6FRYEC.mjs} +31 -6
- package/dist/{chunk-POWOTUKJ.mjs → chunk-YRX6XMOQ.mjs} +1 -0
- package/dist/{direct-executor-6WWFY5EI.mjs → direct-executor-NY7ZTZIH.mjs} +3 -3
- package/dist/distribution.mjs +5 -5
- package/dist/index.browser.mjs +763 -161
- package/dist/index.mjs +5 -5
- package/dist/persistence/durability/checkpoint-worker.mjs +5 -5
- package/dist/server.mjs +31 -16
- package/dist/{thread-33M5AOEI.mjs → thread-EUUNJNKU.mjs} +226 -41
- package/dist/vector/search-worker.d.ts +8 -0
- package/dist/vector/search-worker.mjs +2 -2
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
|
|
2
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-
|
|
2
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-NFF5TDRG.mjs';
|
|
3
3
|
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
|
|
4
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -12,6 +12,7 @@ var VECTOR_STORE_INITIAL_CAPACITY = 16;
|
|
|
12
12
|
var VECTOR_BLOCK_INITIAL_SLOTS = 256;
|
|
13
13
|
var DISK_STORAGE_BLOCK_BYTES = 67108864;
|
|
14
14
|
var MAX_VECTOR_ORDINALS = 16777216;
|
|
15
|
+
var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
|
|
15
16
|
var VECTOR_SCRATCH_SLOTS = 64;
|
|
16
17
|
var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
|
|
17
18
|
var MAX_DOC_ID_TABLE_BYTES = 1073741824;
|
|
@@ -239,8 +240,11 @@ var GRAPH_M = 2;
|
|
|
239
240
|
var GRAPH_MMAX0 = 3;
|
|
240
241
|
var GRAPH_EF_CONSTRUCTION = 4;
|
|
241
242
|
var GRAPH_METRIC = 5;
|
|
243
|
+
var GRAPH_RETIRED = 6;
|
|
242
244
|
var GRAPH_NODE_COUNT = 32;
|
|
243
245
|
var GRAPH_TOMBSTONE_COUNT = 33;
|
|
246
|
+
var GRAPH_UNLINKED_NODES = 34;
|
|
247
|
+
var GRAPH_INCOMING_UNCOUNTED = 35;
|
|
244
248
|
var GRAPH_UPPER_USED = 64;
|
|
245
249
|
var GRAPH_SLOTS = 65;
|
|
246
250
|
var GRAPH_UPPER_CAPACITY = 66;
|
|
@@ -278,6 +282,7 @@ function createSharedGraphHandles(shape) {
|
|
|
278
282
|
upper: perOrdinal(upperStride * 4, INITIAL_UPPER_SLOTS),
|
|
279
283
|
locks: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
280
284
|
tombstones: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
285
|
+
incoming: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
281
286
|
heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
|
|
282
287
|
};
|
|
283
288
|
publishGraphCapacity(handles);
|
|
@@ -297,7 +302,8 @@ function publishGraphCapacity(handles) {
|
|
|
297
302
|
Math.floor(handles.level0.byteLength / (level0Stride * 4)),
|
|
298
303
|
Math.floor(handles.upperBase.byteLength / 4),
|
|
299
304
|
Math.floor(handles.locks.byteLength / 4),
|
|
300
|
-
handles.tombstones.byteLength
|
|
305
|
+
handles.tombstones.byteLength,
|
|
306
|
+
Math.floor(handles.incoming.byteLength / 4)
|
|
301
307
|
);
|
|
302
308
|
raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
|
|
303
309
|
raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
|
|
@@ -612,6 +618,8 @@ var STORE_DOC_ID_BYTES = 5;
|
|
|
612
618
|
var STORE_CODE_BLOCK_COUNT = 6;
|
|
613
619
|
var STORE_CALIBRATION_GENERATION = 7;
|
|
614
620
|
var STORE_RELEASED_VECTORS_TO_DISK = 8;
|
|
621
|
+
var STORE_UNPLACED_COUNT = 9;
|
|
622
|
+
var STORE_REMOVED_COUNT = 10;
|
|
615
623
|
var STORE_HEADER_WORDS = 16;
|
|
616
624
|
var IN_MEMORY = -1;
|
|
617
625
|
function fixedInt32(words) {
|
|
@@ -620,6 +628,9 @@ function fixedInt32(words) {
|
|
|
620
628
|
function fixedFloat32(length) {
|
|
621
629
|
return new Float32Array(createFixedBuffer(length * 4), 0, length);
|
|
622
630
|
}
|
|
631
|
+
function unplacedChunkCount(ordinals) {
|
|
632
|
+
return Math.floor(ordinals / UNPLACED_FLAG_CHUNK_ORDINALS) + 1;
|
|
633
|
+
}
|
|
623
634
|
function perOrdinal2(bytesPerOrdinal) {
|
|
624
635
|
return createGrowableBuffer(
|
|
625
636
|
VECTOR_STORE_INITIAL_CAPACITY * bytesPerOrdinal,
|
|
@@ -644,6 +655,9 @@ function createSharedVectorStoreHandles(dimension, codeBits, blockBytes2 = BLOCK
|
|
|
644
655
|
vectorFiles: [],
|
|
645
656
|
magnitudes: perOrdinal2(8),
|
|
646
657
|
present: perOrdinal2(1),
|
|
658
|
+
unplaced: perOrdinal2(1),
|
|
659
|
+
unplacedChunks: createGrowableBuffer(4, unplacedChunkCount(MAX_VECTOR_ORDINALS) * 4),
|
|
660
|
+
removed: perOrdinal2(1),
|
|
647
661
|
partitions: perOrdinal2(4),
|
|
648
662
|
docIdBytes: createGrowableBuffer(VECTOR_STORE_INITIAL_CAPACITY * 16, MAX_DOC_ID_TABLE_BYTES),
|
|
649
663
|
docIdOffsets: perOrdinal2(4),
|
|
@@ -661,6 +675,9 @@ function sharedVectorStoreBytes(handles) {
|
|
|
661
675
|
for (const block of handles.codeBlocks) bytes += blockBuffer(block).byteLength;
|
|
662
676
|
bytes += handles.magnitudes.byteLength;
|
|
663
677
|
bytes += handles.present.byteLength;
|
|
678
|
+
bytes += handles.unplaced.byteLength;
|
|
679
|
+
bytes += handles.unplacedChunks.byteLength;
|
|
680
|
+
bytes += handles.removed.byteLength;
|
|
664
681
|
bytes += handles.partitions.byteLength;
|
|
665
682
|
bytes += handles.docIdBytes.byteLength;
|
|
666
683
|
bytes += handles.docIdOffsets.byteLength;
|
|
@@ -857,6 +874,7 @@ function graphMemory(graph) {
|
|
|
857
874
|
graph.upper,
|
|
858
875
|
graph.locks,
|
|
859
876
|
graph.tombstones,
|
|
877
|
+
graph.incoming,
|
|
860
878
|
graph.heldLocks.buffer
|
|
861
879
|
];
|
|
862
880
|
if (!regions.every(isSharedMemory)) return null;
|
|
@@ -868,6 +886,7 @@ function graphMemory(graph) {
|
|
|
868
886
|
upper: fixedView(graph.upper, Int32Array),
|
|
869
887
|
locks: fixedView(graph.locks, Int32Array),
|
|
870
888
|
tombstones: fixedView(graph.tombstones, Uint8Array),
|
|
889
|
+
incoming: fixedView(graph.incoming, Int32Array),
|
|
871
890
|
heldLocks: graph.heldLocks
|
|
872
891
|
};
|
|
873
892
|
}
|
|
@@ -967,6 +986,17 @@ function nativeCompact(state, field, metric) {
|
|
|
967
986
|
const compact = () => field.core.compact(field.graph, field.store.handle, nativeMetricCode(metric), state.locks.threadSlot, field.wakes);
|
|
968
987
|
return writeToTheGraph(state, field, compact) === NATIVE_OK;
|
|
969
988
|
}
|
|
989
|
+
function nativeConnect(state, field, metric, nearby) {
|
|
990
|
+
const connect = () => field.core.connectUnreached(
|
|
991
|
+
field.graph,
|
|
992
|
+
field.store.handle,
|
|
993
|
+
nativeMetricCode(metric),
|
|
994
|
+
nearby,
|
|
995
|
+
state.locks.threadSlot,
|
|
996
|
+
field.wakes
|
|
997
|
+
);
|
|
998
|
+
return writeToTheGraph(state, field, connect) === NATIVE_OK;
|
|
999
|
+
}
|
|
970
1000
|
|
|
971
1001
|
// src/vector/hnsw/adjacency/views.ts
|
|
972
1002
|
var ABSENT = -1;
|
|
@@ -981,11 +1011,13 @@ function openAdjacency(handles, m, mMax0) {
|
|
|
981
1011
|
nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
|
|
982
1012
|
level0: fixedView(handles.level0, Int32Array),
|
|
983
1013
|
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
984
|
-
upper: fixedView(handles.upper, Int32Array)
|
|
1014
|
+
upper: fixedView(handles.upper, Int32Array),
|
|
1015
|
+
incoming: fixedView(handles.incoming, Int32Array)
|
|
985
1016
|
};
|
|
986
1017
|
}
|
|
987
1018
|
function rebindNodes(adj) {
|
|
988
1019
|
adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
|
|
1020
|
+
adj.incoming = fixedView(adj.handles.incoming, Int32Array);
|
|
989
1021
|
adj.level0 = fixedView(adj.handles.level0, Int32Array);
|
|
990
1022
|
adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
|
|
991
1023
|
}
|
|
@@ -1022,6 +1054,7 @@ function ensureAdjacencyCapacity(adj, needed) {
|
|
|
1022
1054
|
growBufferToExactly(handles.upperBase, capacity * 4);
|
|
1023
1055
|
growBufferToExactly(handles.locks, capacity * 4);
|
|
1024
1056
|
growBufferToExactly(handles.tombstones, capacity);
|
|
1057
|
+
growBufferToExactly(handles.incoming, capacity * 4);
|
|
1025
1058
|
growBufferToExactly(handles.nodeLevels, capacity);
|
|
1026
1059
|
publishGraphCapacity(handles);
|
|
1027
1060
|
rebindNodes(adj);
|
|
@@ -1068,10 +1101,12 @@ function deleteNode(adj, ord) {
|
|
|
1068
1101
|
Atomics.store(adj.nodeLevels, ord, 0);
|
|
1069
1102
|
adj.upperBase[ord] = 0;
|
|
1070
1103
|
adj.level0[ord * adj.level0Stride] = 0;
|
|
1104
|
+
Atomics.store(adj.incoming, ord, 0);
|
|
1071
1105
|
}
|
|
1072
1106
|
function resetAdjacency(adj) {
|
|
1073
1107
|
rebindNodes(adj);
|
|
1074
1108
|
rebindUpper(adj);
|
|
1109
|
+
adj.incoming.fill(0);
|
|
1075
1110
|
adj.nodeLevels.fill(0);
|
|
1076
1111
|
adj.level0.fill(0);
|
|
1077
1112
|
adj.upperBase.fill(0);
|
|
@@ -1088,7 +1123,51 @@ function hasNode(adj, ord) {
|
|
|
1088
1123
|
}
|
|
1089
1124
|
function graphBytes(adj) {
|
|
1090
1125
|
const handles = adj.handles;
|
|
1091
|
-
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.heldLocks.byteLength;
|
|
1126
|
+
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
// src/vector/hnsw/adjacency/incoming.ts
|
|
1130
|
+
function incomingCount(adj, ord) {
|
|
1131
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
1132
|
+
return Atomics.load(adj.incoming, ord);
|
|
1133
|
+
}
|
|
1134
|
+
function noteLinked(adj, ord) {
|
|
1135
|
+
if (ord < 0 || !reachNode(adj, ord)) return;
|
|
1136
|
+
Atomics.add(adj.incoming, ord, 1);
|
|
1137
|
+
}
|
|
1138
|
+
function noteUnlinked(adj, ord) {
|
|
1139
|
+
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
1140
|
+
for (; ; ) {
|
|
1141
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
1142
|
+
if (seen <= 0) return 0;
|
|
1143
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return seen - 1;
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
function releaseUnlessLast(adj, ord) {
|
|
1147
|
+
if (ord < 0 || !reachNode(adj, ord)) return true;
|
|
1148
|
+
for (; ; ) {
|
|
1149
|
+
const seen = Atomics.load(adj.incoming, ord);
|
|
1150
|
+
if (seen <= 1) return false;
|
|
1151
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return true;
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
function holdsNodeAt(adj, ord) {
|
|
1155
|
+
return ord >= 0 && ord < adj.nodeLevels.length && adj.nodeLevels[ord] !== 0;
|
|
1156
|
+
}
|
|
1157
|
+
function countLinksOutOf(adj, ord, step) {
|
|
1158
|
+
if (!holdsNodeAt(adj, ord)) return;
|
|
1159
|
+
const incoming = adj.incoming;
|
|
1160
|
+
const level0 = adj.level0;
|
|
1161
|
+
const base = ord * adj.level0Stride;
|
|
1162
|
+
const count = level0[base];
|
|
1163
|
+
for (let i = 1; i <= count; i++) {
|
|
1164
|
+
const neighbour = level0[base + i];
|
|
1165
|
+
if (neighbour === ord || !holdsNodeAt(adj, neighbour)) continue;
|
|
1166
|
+
if (incoming[neighbour] + step >= 0) incoming[neighbour] += step;
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
function countEveryLink(adj, slots) {
|
|
1170
|
+
for (let ord = 0; ord < slots; ord++) countLinksOutOf(adj, ord, 1);
|
|
1092
1171
|
}
|
|
1093
1172
|
|
|
1094
1173
|
// src/vector/hnsw/adjacency/neighbors.ts
|
|
@@ -1138,6 +1217,7 @@ function addNeighbor(adj, ord, layer, neighborOrd) {
|
|
|
1138
1217
|
if (count + 1 > stride - 1) return;
|
|
1139
1218
|
array[base + count + 1] = neighborOrd;
|
|
1140
1219
|
array[base] = count + 1;
|
|
1220
|
+
if (layer === 0) noteLinked(adj, neighborOrd);
|
|
1141
1221
|
}
|
|
1142
1222
|
function removeNeighbor(adj, ord, layer, neighborOrd) {
|
|
1143
1223
|
const base = layerBase(adj, ord, layer);
|
|
@@ -1150,10 +1230,11 @@ function removeNeighbor(adj, ord, layer, neighborOrd) {
|
|
|
1150
1230
|
array[base + j] = array[base + j + 1];
|
|
1151
1231
|
}
|
|
1152
1232
|
array[base] = count - 1;
|
|
1233
|
+
if (layer === 0) noteUnlinked(adj, neighborOrd);
|
|
1153
1234
|
return;
|
|
1154
1235
|
}
|
|
1155
1236
|
}
|
|
1156
|
-
function
|
|
1237
|
+
function writeNeighbors(adj, ord, layer, neighbors, count) {
|
|
1157
1238
|
const base = layerBase(adj, ord, layer);
|
|
1158
1239
|
if (base === ABSENT) return;
|
|
1159
1240
|
const array = layerArray(adj, layer);
|
|
@@ -1164,163 +1245,23 @@ function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
|
1164
1245
|
}
|
|
1165
1246
|
array[base] = limit;
|
|
1166
1247
|
}
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
var EXCLUSIVE = -1;
|
|
1171
|
-
function openGraphLocks(handles, threadSlot) {
|
|
1172
|
-
let words = fixedView(handles.locks, Int32Array);
|
|
1173
|
-
return {
|
|
1174
|
-
threadSlot,
|
|
1175
|
-
header: handles.header,
|
|
1176
|
-
held: handles.heldLocks,
|
|
1177
|
-
words(ord) {
|
|
1178
|
-
if (ord < words.length) return words;
|
|
1179
|
-
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
1180
|
-
words = fixedView(handles.locks, Int32Array);
|
|
1181
|
-
return words;
|
|
1182
|
-
}
|
|
1183
|
-
};
|
|
1184
|
-
}
|
|
1185
|
-
var blockingAllowed = typeof document === "undefined";
|
|
1186
|
-
function canWait(words) {
|
|
1187
|
-
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
1188
|
-
}
|
|
1189
|
-
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
1190
|
-
if (!canWait(words)) return;
|
|
1191
|
-
Atomics.wait(words, index, seen, sliceMs);
|
|
1192
|
-
}
|
|
1193
|
-
function heldIndex(locks, kind) {
|
|
1194
|
-
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
1195
|
-
}
|
|
1196
|
-
function acquireExclusive(words, index) {
|
|
1197
|
-
let spins = 0;
|
|
1198
|
-
for (; ; ) {
|
|
1199
|
-
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
1200
|
-
if (seen === FREE) return;
|
|
1201
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1202
|
-
spins += 1;
|
|
1203
|
-
continue;
|
|
1204
|
-
}
|
|
1205
|
-
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
function releaseExclusive(words, index) {
|
|
1209
|
-
Atomics.store(words, index, FREE);
|
|
1210
|
-
if (canWait(words)) Atomics.notify(words, index);
|
|
1211
|
-
}
|
|
1212
|
-
var WRITE_HELD = 1;
|
|
1213
|
-
var WRITER_WAITING = 2;
|
|
1214
|
-
var VERSION_STEP = 4;
|
|
1215
|
-
function beginNodeRead(locks, ord) {
|
|
1216
|
-
const words = locks.words(ord);
|
|
1217
|
-
for (; ; ) {
|
|
1218
|
-
const seen = Atomics.load(words, ord);
|
|
1219
|
-
if ((seen & WRITE_HELD) === 0) return seen;
|
|
1220
|
-
}
|
|
1221
|
-
}
|
|
1222
|
-
function nodeReadHeld(locks, ord, version) {
|
|
1223
|
-
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
1224
|
-
return Atomics.load(locks.words(ord), ord) === version;
|
|
1225
|
-
}
|
|
1226
|
-
function lockNodeWrite(locks, ord) {
|
|
1227
|
-
const words = locks.words(ord);
|
|
1228
|
-
let spins = 0;
|
|
1229
|
-
for (; ; ) {
|
|
1230
|
-
const seen = Atomics.load(words, ord);
|
|
1231
|
-
if ((seen & WRITE_HELD) === 0) {
|
|
1232
|
-
const claimed = seen | WRITE_HELD;
|
|
1233
|
-
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
1234
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
1235
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
1236
|
-
return;
|
|
1237
|
-
}
|
|
1238
|
-
continue;
|
|
1239
|
-
}
|
|
1240
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1241
|
-
spins += 1;
|
|
1242
|
-
continue;
|
|
1243
|
-
}
|
|
1244
|
-
const waiting = seen | WRITER_WAITING;
|
|
1245
|
-
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
1246
|
-
waitOn(words, ord, waiting);
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
}
|
|
1250
|
-
function releaseWrite(words, ord) {
|
|
1251
|
-
const seen = Atomics.load(words, ord);
|
|
1252
|
-
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
1253
|
-
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
1254
|
-
}
|
|
1255
|
-
function unlockNodeWrite(locks, ord) {
|
|
1256
|
-
releaseWrite(locks.words(ord), ord);
|
|
1257
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
1258
|
-
}
|
|
1259
|
-
function lockGraphShared(locks) {
|
|
1260
|
-
const { header } = locks;
|
|
1261
|
-
let spins = 0;
|
|
1262
|
-
for (; ; ) {
|
|
1263
|
-
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
1264
|
-
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
1265
|
-
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
1266
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
1267
|
-
return;
|
|
1268
|
-
}
|
|
1269
|
-
continue;
|
|
1270
|
-
}
|
|
1271
|
-
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1272
|
-
spins += 1;
|
|
1273
|
-
continue;
|
|
1274
|
-
}
|
|
1275
|
-
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
function unlockGraphShared(locks) {
|
|
1279
|
-
const { header } = locks;
|
|
1280
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
1281
|
-
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
1282
|
-
}
|
|
1283
|
-
function lockGraphExclusive(locks) {
|
|
1284
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
1285
|
-
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1286
|
-
try {
|
|
1287
|
-
acquireExclusive(locks.header, GRAPH_LOCK);
|
|
1288
|
-
} finally {
|
|
1289
|
-
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1290
|
-
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
1291
|
-
}
|
|
1292
|
-
}
|
|
1293
|
-
function unlockGraphExclusive(locks) {
|
|
1294
|
-
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
1295
|
-
}
|
|
1296
|
-
function lockEntry(locks) {
|
|
1297
|
-
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1298
|
-
}
|
|
1299
|
-
function unlockEntry(locks) {
|
|
1300
|
-
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1248
|
+
function holds(neighbors, count, ord) {
|
|
1249
|
+
for (let i = 0; i < count; i++) if (neighbors[i] === ord) return true;
|
|
1250
|
+
return false;
|
|
1301
1251
|
}
|
|
1302
|
-
function
|
|
1303
|
-
const
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
}
|
|
1309
|
-
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
1310
|
-
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
1311
|
-
if (writeOrd >= 0 && writeOrd < words.length) {
|
|
1312
|
-
const seen = Atomics.load(words, writeOrd);
|
|
1313
|
-
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
1314
|
-
}
|
|
1315
|
-
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
1316
|
-
for (; ; ) {
|
|
1317
|
-
const readers = Atomics.load(handles.header, GRAPH_LOCK);
|
|
1318
|
-
if (readers <= FREE) return;
|
|
1319
|
-
if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
1320
|
-
if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
|
|
1321
|
-
return;
|
|
1322
|
-
}
|
|
1252
|
+
function replaceNeighbors(adj, ord, layer, neighbors, count) {
|
|
1253
|
+
const base = layerBase(adj, ord, layer);
|
|
1254
|
+
if (base === ABSENT) return;
|
|
1255
|
+
if (layer > 0) {
|
|
1256
|
+
writeNeighbors(adj, ord, layer, neighbors, count);
|
|
1257
|
+
return;
|
|
1323
1258
|
}
|
|
1259
|
+
const array = adj.level0;
|
|
1260
|
+
const held = array.slice(base + 1, base + 1 + array[base]);
|
|
1261
|
+
const limit = Math.min(count, adj.level0Stride - 1);
|
|
1262
|
+
for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
|
|
1263
|
+
for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
|
|
1264
|
+
writeNeighbors(adj, ord, 0, neighbors, count);
|
|
1324
1265
|
}
|
|
1325
1266
|
|
|
1326
1267
|
// src/vector/similarity.ts
|
|
@@ -1498,7 +1439,8 @@ function createHNSWWorkspace() {
|
|
|
1498
1439
|
pruneCandidates: createList(INITIAL_LIST_CAPACITY),
|
|
1499
1440
|
pruneSelection: createList(INITIAL_LIST_CAPACITY),
|
|
1500
1441
|
repairCandidates: createList(INITIAL_LIST_CAPACITY),
|
|
1501
|
-
repairSelection: createList(INITIAL_LIST_CAPACITY)
|
|
1442
|
+
repairSelection: createList(INITIAL_LIST_CAPACITY),
|
|
1443
|
+
droppedLinks: createList(INITIAL_LIST_CAPACITY)
|
|
1502
1444
|
};
|
|
1503
1445
|
}
|
|
1504
1446
|
function linkSelectionsFor(workspace, layers) {
|
|
@@ -1656,6 +1598,419 @@ function setEntryPointsFromList(workspace, list) {
|
|
|
1656
1598
|
workspace.entryPointCount = list.size;
|
|
1657
1599
|
}
|
|
1658
1600
|
|
|
1601
|
+
// src/vector/hnsw/last-links.ts
|
|
1602
|
+
function isLive(state, ord) {
|
|
1603
|
+
return nodeExists(state, ord) && !isTombstoned(state, ord);
|
|
1604
|
+
}
|
|
1605
|
+
function noteNodeLeftUnlinked(state) {
|
|
1606
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 1);
|
|
1607
|
+
}
|
|
1608
|
+
function listHolds(list, ord) {
|
|
1609
|
+
for (let i = 0; i < list.size; i++) if (list.ords[i] === ord) return true;
|
|
1610
|
+
return false;
|
|
1611
|
+
}
|
|
1612
|
+
function collectDropped(sorted, kept, dropped) {
|
|
1613
|
+
dropped.size = 0;
|
|
1614
|
+
let next = 0;
|
|
1615
|
+
for (let i = 0; i < sorted.size; i++) {
|
|
1616
|
+
const ord = sorted.ords[i];
|
|
1617
|
+
if (next < kept.size && kept.ords[next] === ord) {
|
|
1618
|
+
next += 1;
|
|
1619
|
+
continue;
|
|
1620
|
+
}
|
|
1621
|
+
if (listHolds(kept, ord)) continue;
|
|
1622
|
+
appendToList(dropped, ord, sorted.distances[i]);
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
function replaceFarthestShared(state, kept, held, ord, distance) {
|
|
1626
|
+
const tried = /* @__PURE__ */ new Set();
|
|
1627
|
+
for (; ; ) {
|
|
1628
|
+
let farthest = -1;
|
|
1629
|
+
for (let k = 0; k < kept.size; k++) {
|
|
1630
|
+
if (tried.has(k) || incomingCount(state.adjacency, kept.ords[k]) < 2) continue;
|
|
1631
|
+
if (farthest === -1 || kept.distances[k] > kept.distances[farthest]) farthest = k;
|
|
1632
|
+
}
|
|
1633
|
+
if (farthest === -1) return false;
|
|
1634
|
+
tried.add(farthest);
|
|
1635
|
+
const replaced = kept.ords[farthest];
|
|
1636
|
+
if (held(replaced) && !releaseUnlessLast(state.adjacency, replaced)) continue;
|
|
1637
|
+
kept.ords[farthest] = ord;
|
|
1638
|
+
kept.distances[farthest] = distance;
|
|
1639
|
+
return true;
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
function writeKeepingLastLinks(state, ord, sorted, kept, held) {
|
|
1643
|
+
const adjacency = state.adjacency;
|
|
1644
|
+
const limit = state.Mmax0;
|
|
1645
|
+
const dropped = state.workspace.droppedLinks;
|
|
1646
|
+
collectDropped(sorted, kept, dropped);
|
|
1647
|
+
ensureListCapacity(kept, limit);
|
|
1648
|
+
for (let i = 0; i < dropped.size; i++) {
|
|
1649
|
+
const candidate = dropped.ords[i];
|
|
1650
|
+
if (!held(candidate)) continue;
|
|
1651
|
+
if (!isLive(state, candidate)) {
|
|
1652
|
+
noteUnlinked(adjacency, candidate);
|
|
1653
|
+
continue;
|
|
1654
|
+
}
|
|
1655
|
+
if (releaseUnlessLast(adjacency, candidate)) continue;
|
|
1656
|
+
if (kept.size < limit) {
|
|
1657
|
+
appendToList(kept, candidate, dropped.distances[i]);
|
|
1658
|
+
continue;
|
|
1659
|
+
}
|
|
1660
|
+
if (replaceFarthestShared(state, kept, held, candidate, dropped.distances[i])) continue;
|
|
1661
|
+
noteUnlinked(adjacency, candidate);
|
|
1662
|
+
noteNodeLeftUnlinked(state);
|
|
1663
|
+
}
|
|
1664
|
+
for (let k = 0; k < kept.size; k++) if (!held(kept.ords[k])) noteLinked(adjacency, kept.ords[k]);
|
|
1665
|
+
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
// src/vector/hnsw/locks.ts
|
|
1669
|
+
var FREE = 0;
|
|
1670
|
+
var EXCLUSIVE = -1;
|
|
1671
|
+
function openGraphLocks(handles, threadSlot) {
|
|
1672
|
+
let words = fixedView(handles.locks, Int32Array);
|
|
1673
|
+
return {
|
|
1674
|
+
threadSlot,
|
|
1675
|
+
header: handles.header,
|
|
1676
|
+
held: handles.heldLocks,
|
|
1677
|
+
words(ord) {
|
|
1678
|
+
if (ord < words.length) return words;
|
|
1679
|
+
if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
|
|
1680
|
+
words = fixedView(handles.locks, Int32Array);
|
|
1681
|
+
return words;
|
|
1682
|
+
}
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
var blockingAllowed = typeof document === "undefined";
|
|
1686
|
+
function canWait(words) {
|
|
1687
|
+
return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
|
|
1688
|
+
}
|
|
1689
|
+
function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
|
|
1690
|
+
if (!canWait(words)) return;
|
|
1691
|
+
Atomics.wait(words, index, seen, sliceMs);
|
|
1692
|
+
}
|
|
1693
|
+
function heldIndex(locks, kind) {
|
|
1694
|
+
return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
|
|
1695
|
+
}
|
|
1696
|
+
function acquireExclusive(words, index) {
|
|
1697
|
+
let spins = 0;
|
|
1698
|
+
for (; ; ) {
|
|
1699
|
+
const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
|
|
1700
|
+
if (seen === FREE) return;
|
|
1701
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1702
|
+
spins += 1;
|
|
1703
|
+
continue;
|
|
1704
|
+
}
|
|
1705
|
+
waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
function releaseExclusive(words, index) {
|
|
1709
|
+
Atomics.store(words, index, FREE);
|
|
1710
|
+
if (canWait(words)) Atomics.notify(words, index);
|
|
1711
|
+
}
|
|
1712
|
+
var WRITE_HELD = 1;
|
|
1713
|
+
var WRITER_WAITING = 2;
|
|
1714
|
+
var VERSION_STEP = 4;
|
|
1715
|
+
function beginNodeRead(locks, ord) {
|
|
1716
|
+
const words = locks.words(ord);
|
|
1717
|
+
for (; ; ) {
|
|
1718
|
+
const seen = Atomics.load(words, ord);
|
|
1719
|
+
if ((seen & WRITE_HELD) === 0) return seen;
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
function nodeReadHeld(locks, ord, version) {
|
|
1723
|
+
Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
|
|
1724
|
+
return Atomics.load(locks.words(ord), ord) === version;
|
|
1725
|
+
}
|
|
1726
|
+
function lockNodeWrite(locks, ord) {
|
|
1727
|
+
const words = locks.words(ord);
|
|
1728
|
+
let spins = 0;
|
|
1729
|
+
for (; ; ) {
|
|
1730
|
+
const seen = Atomics.load(words, ord);
|
|
1731
|
+
if ((seen & WRITE_HELD) === 0) {
|
|
1732
|
+
const claimed = seen | WRITE_HELD;
|
|
1733
|
+
if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
|
|
1734
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
|
|
1735
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
|
|
1736
|
+
return;
|
|
1737
|
+
}
|
|
1738
|
+
continue;
|
|
1739
|
+
}
|
|
1740
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1741
|
+
spins += 1;
|
|
1742
|
+
continue;
|
|
1743
|
+
}
|
|
1744
|
+
const waiting = seen | WRITER_WAITING;
|
|
1745
|
+
if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
|
|
1746
|
+
waitOn(words, ord, waiting);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
function releaseWrite(words, ord) {
|
|
1751
|
+
const seen = Atomics.load(words, ord);
|
|
1752
|
+
const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
|
|
1753
|
+
if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
|
|
1754
|
+
}
|
|
1755
|
+
function unlockNodeWrite(locks, ord) {
|
|
1756
|
+
releaseWrite(locks.words(ord), ord);
|
|
1757
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
|
|
1758
|
+
}
|
|
1759
|
+
function lockGraphShared(locks) {
|
|
1760
|
+
const { header } = locks;
|
|
1761
|
+
let spins = 0;
|
|
1762
|
+
for (; ; ) {
|
|
1763
|
+
const seen = Atomics.load(header, GRAPH_LOCK);
|
|
1764
|
+
if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
|
|
1765
|
+
if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
|
|
1766
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
|
|
1767
|
+
return;
|
|
1768
|
+
}
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
if (spins < LOCK_SPIN_ITERATIONS) {
|
|
1772
|
+
spins += 1;
|
|
1773
|
+
continue;
|
|
1774
|
+
}
|
|
1775
|
+
waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
function unlockGraphShared(locks) {
|
|
1779
|
+
const { header } = locks;
|
|
1780
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
|
|
1781
|
+
if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
|
|
1782
|
+
}
|
|
1783
|
+
function lockGraphExclusive(locks) {
|
|
1784
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
|
|
1785
|
+
Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1786
|
+
try {
|
|
1787
|
+
acquireExclusive(locks.header, GRAPH_LOCK);
|
|
1788
|
+
} finally {
|
|
1789
|
+
Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
|
|
1790
|
+
Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
function unlockGraphExclusive(locks) {
|
|
1794
|
+
releaseExclusive(locks.header, GRAPH_LOCK);
|
|
1795
|
+
}
|
|
1796
|
+
function lockEntry(locks) {
|
|
1797
|
+
acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1798
|
+
}
|
|
1799
|
+
function unlockEntry(locks) {
|
|
1800
|
+
releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
|
|
1801
|
+
}
|
|
1802
|
+
function releaseLocksHeldBy(handles, threadSlot) {
|
|
1803
|
+
const held = handles.heldLocks;
|
|
1804
|
+
const words = new Int32Array(handles.locks);
|
|
1805
|
+
const base = threadSlot * HELD_WORDS_PER_THREAD;
|
|
1806
|
+
if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
|
|
1807
|
+
Atomics.sub(handles.header, GRAPH_WRITERS_WAITING, 1);
|
|
1808
|
+
}
|
|
1809
|
+
const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
|
|
1810
|
+
const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
|
|
1811
|
+
if (writeOrd >= 0 && writeOrd < words.length) {
|
|
1812
|
+
const seen = Atomics.load(words, writeOrd);
|
|
1813
|
+
if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
|
|
1814
|
+
}
|
|
1815
|
+
if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
|
|
1816
|
+
for (; ; ) {
|
|
1817
|
+
const readers = Atomics.load(handles.header, GRAPH_LOCK);
|
|
1818
|
+
if (readers <= FREE) return;
|
|
1819
|
+
if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
|
|
1820
|
+
if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
|
|
1821
|
+
return;
|
|
1822
|
+
}
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
// src/vector/hnsw/connect-unreached.ts
|
|
1827
|
+
var NOT_REACHED_THROUGH_A_NODE = -1;
|
|
1828
|
+
function reach(walk, node, through) {
|
|
1829
|
+
walk.reached[node] = 1;
|
|
1830
|
+
walk.firstReachedThrough[node] = through;
|
|
1831
|
+
walk.queue[walk.tail++] = node;
|
|
1832
|
+
}
|
|
1833
|
+
function walkFrom(state, walk, start, through) {
|
|
1834
|
+
const adjacency = state.adjacency;
|
|
1835
|
+
if (walk.reached[start] === 1) return;
|
|
1836
|
+
reach(walk, start, through);
|
|
1837
|
+
while (walk.head < walk.tail) {
|
|
1838
|
+
const ord = walk.queue[walk.head++];
|
|
1839
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1840
|
+
if (base === ABSENT) continue;
|
|
1841
|
+
const count = adjacency.level0[base];
|
|
1842
|
+
for (let i = 1; i <= count; i++) {
|
|
1843
|
+
const next = adjacency.level0[base + i];
|
|
1844
|
+
if (next < 0 || next >= walk.reached.length || walk.reached[next] === 1 || !nodeExists(state, next)) continue;
|
|
1845
|
+
reach(walk, next, ord);
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
function hasRoom(state, _walk, host) {
|
|
1850
|
+
const base = layerBase(state.adjacency, host, 0);
|
|
1851
|
+
return base !== ABSENT && state.adjacency.level0[base] < state.Mmax0;
|
|
1852
|
+
}
|
|
1853
|
+
function reachedThroughAnotherNode(walk, host, neighbour) {
|
|
1854
|
+
if (neighbour < 0 || neighbour >= walk.reached.length || walk.reached[neighbour] !== 1) return false;
|
|
1855
|
+
const through = walk.firstReachedThrough[neighbour];
|
|
1856
|
+
return through !== NOT_REACHED_THROUGH_A_NODE && through !== host;
|
|
1857
|
+
}
|
|
1858
|
+
function holdsLinkOutsideTheWalk(state, walk, host) {
|
|
1859
|
+
const adjacency = state.adjacency;
|
|
1860
|
+
const base = layerBase(adjacency, host, 0);
|
|
1861
|
+
if (base === ABSENT) return false;
|
|
1862
|
+
const count = adjacency.level0[base];
|
|
1863
|
+
for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
|
|
1864
|
+
return false;
|
|
1865
|
+
}
|
|
1866
|
+
function listOf(state, ord) {
|
|
1867
|
+
const adjacency = state.adjacency;
|
|
1868
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1869
|
+
if (base === ABSENT) return new Int32Array(0);
|
|
1870
|
+
return adjacency.level0.slice(base + 1, base + 1 + adjacency.level0[base]);
|
|
1871
|
+
}
|
|
1872
|
+
function nearestHost(state, walk, orphan, nearby, test) {
|
|
1873
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
1874
|
+
let best = -1;
|
|
1875
|
+
let bestDistance = Number.POSITIVE_INFINITY;
|
|
1876
|
+
const consider = (host) => {
|
|
1877
|
+
if (host === orphan || walk.reached[host] !== 1 || !isLive(state, host) || !test(state, walk, host)) return;
|
|
1878
|
+
const away = distance(orphan, host);
|
|
1879
|
+
if (best === -1 || away < bestDistance) {
|
|
1880
|
+
best = host;
|
|
1881
|
+
bestDistance = away;
|
|
1882
|
+
}
|
|
1883
|
+
};
|
|
1884
|
+
for (const host of nearby.list) consider(host);
|
|
1885
|
+
for (const host of listOf(state, orphan))
|
|
1886
|
+
if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
|
|
1887
|
+
if (best !== -1) return best;
|
|
1888
|
+
for (let host = 0; host < walk.reached.length; host++) consider(host);
|
|
1889
|
+
return best;
|
|
1890
|
+
}
|
|
1891
|
+
function takeOutFarthestLinkOutsideTheWalk(state, walk, host) {
|
|
1892
|
+
const adjacency = state.adjacency;
|
|
1893
|
+
const base = layerBase(adjacency, host, 0);
|
|
1894
|
+
if (base === ABSENT) return;
|
|
1895
|
+
const distance = nodeDistanceFunction(state, state.buildMetric);
|
|
1896
|
+
const count = adjacency.level0[base];
|
|
1897
|
+
let farthest = -1;
|
|
1898
|
+
let farthestDistance = Number.NEGATIVE_INFINITY;
|
|
1899
|
+
for (let i = 1; i <= count; i++) {
|
|
1900
|
+
const neighbour = adjacency.level0[base + i];
|
|
1901
|
+
if (!reachedThroughAnotherNode(walk, host, neighbour)) continue;
|
|
1902
|
+
const away = distance(host, neighbour);
|
|
1903
|
+
if (farthest === -1 || away > farthestDistance) {
|
|
1904
|
+
farthest = i;
|
|
1905
|
+
farthestDistance = away;
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
if (farthest === -1) return;
|
|
1909
|
+
const taken = adjacency.level0[base + farthest];
|
|
1910
|
+
for (let i = farthest; i < count; i++) adjacency.level0[base + i] = adjacency.level0[base + i + 1];
|
|
1911
|
+
adjacency.level0[base] = count - 1;
|
|
1912
|
+
noteUnlinked(adjacency, taken);
|
|
1913
|
+
}
|
|
1914
|
+
function appendTo(state, host, orphan) {
|
|
1915
|
+
const adjacency = state.adjacency;
|
|
1916
|
+
const base = layerBase(adjacency, host, 0);
|
|
1917
|
+
if (base === ABSENT) return;
|
|
1918
|
+
const count = adjacency.level0[base];
|
|
1919
|
+
adjacency.level0[base + count + 1] = orphan;
|
|
1920
|
+
adjacency.level0[base] = count + 1;
|
|
1921
|
+
noteLinked(adjacency, orphan);
|
|
1922
|
+
}
|
|
1923
|
+
function connectUnreached(state, nearby) {
|
|
1924
|
+
const slots = adjacencySlots(state.adjacency);
|
|
1925
|
+
const entry = entryPointOf(state);
|
|
1926
|
+
if (entry < 0 || entry >= slots || !nodeExists(state, entry)) return;
|
|
1927
|
+
const walk = {
|
|
1928
|
+
reached: new Uint8Array(slots),
|
|
1929
|
+
firstReachedThrough: new Int32Array(slots),
|
|
1930
|
+
queue: new Int32Array(slots),
|
|
1931
|
+
head: 0,
|
|
1932
|
+
tail: 0
|
|
1933
|
+
};
|
|
1934
|
+
walkFrom(state, walk, entry, NOT_REACHED_THROUGH_A_NODE);
|
|
1935
|
+
const held = { list: [], holds: new Uint8Array(slots) };
|
|
1936
|
+
for (let i = 0; i < nearby.length; i++) {
|
|
1937
|
+
const ord = nearby[i];
|
|
1938
|
+
if (ord < 0 || ord >= slots || held.holds[ord] === 1) continue;
|
|
1939
|
+
held.holds[ord] = 1;
|
|
1940
|
+
held.list.push(ord);
|
|
1941
|
+
}
|
|
1942
|
+
for (let orphan = 0; orphan < slots; orphan++) {
|
|
1943
|
+
if (walk.reached[orphan] === 1 || !isLive(state, orphan)) continue;
|
|
1944
|
+
let host = nearestHost(state, walk, orphan, held, hasRoom);
|
|
1945
|
+
if (host === -1) {
|
|
1946
|
+
host = nearestHost(state, walk, orphan, held, holdsLinkOutsideTheWalk);
|
|
1947
|
+
if (host === -1) continue;
|
|
1948
|
+
takeOutFarthestLinkOutsideTheWalk(state, walk, host);
|
|
1949
|
+
}
|
|
1950
|
+
appendTo(state, host, orphan);
|
|
1951
|
+
walkFrom(state, walk, orphan, host);
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
function holdsUnlinkedNode(state) {
|
|
1955
|
+
const slots = adjacencySlots(state.adjacency);
|
|
1956
|
+
const entry = entryPointOf(state);
|
|
1957
|
+
const incoming = state.adjacency.incoming;
|
|
1958
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
1959
|
+
if (incoming[ord] === 0 && ord !== entry && isLive(state, ord)) return true;
|
|
1960
|
+
}
|
|
1961
|
+
return false;
|
|
1962
|
+
}
|
|
1963
|
+
function connectThroughEitherCore(state, nearby) {
|
|
1964
|
+
const field = nativeFieldFor(state);
|
|
1965
|
+
if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
|
|
1966
|
+
noteFallback("connect");
|
|
1967
|
+
connectUnreached(state, nearby);
|
|
1968
|
+
}
|
|
1969
|
+
function connectAfterRemoval(state, nearby) {
|
|
1970
|
+
connectThroughEitherCore(state, nearby);
|
|
1971
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1972
|
+
}
|
|
1973
|
+
function connectAfterPlacement(state, holdsGraphLock) {
|
|
1974
|
+
if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
|
|
1975
|
+
if (!holdsGraphLock) lockGraphExclusive(state.locks);
|
|
1976
|
+
try {
|
|
1977
|
+
if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
|
|
1978
|
+
connectThroughEitherCore(state, new Int32Array(0));
|
|
1979
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1980
|
+
} finally {
|
|
1981
|
+
if (!holdsGraphLock) unlockGraphExclusive(state.locks);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
function baseNeighboursOfRemoved(state, removed) {
|
|
1985
|
+
const leaving = new Set(removed);
|
|
1986
|
+
const nearby = [];
|
|
1987
|
+
for (const ord of removed) {
|
|
1988
|
+
for (const neighbour of listOf(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
|
|
1989
|
+
}
|
|
1990
|
+
return Int32Array.from(nearby);
|
|
1991
|
+
}
|
|
1992
|
+
function leaveIncomingUncounted(state) {
|
|
1993
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
1994
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 1);
|
|
1995
|
+
}
|
|
1996
|
+
function countIncomingBeforeWriting(state, holdsGraphLock) {
|
|
1997
|
+
if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
|
|
1998
|
+
if (!holdsGraphLock) lockGraphExclusive(state.locks);
|
|
1999
|
+
try {
|
|
2000
|
+
if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
|
|
2001
|
+
const adjacency = state.adjacency;
|
|
2002
|
+
const slots = adjacencySlots(adjacency);
|
|
2003
|
+
if (slots > 0) reachNode(adjacency, slots - 1);
|
|
2004
|
+
adjacency.incoming.fill(0);
|
|
2005
|
+
countEveryLink(adjacency, slots);
|
|
2006
|
+
if (holdsUnlinkedNode(state)) connectThroughEitherCore(state, new Int32Array(0));
|
|
2007
|
+
Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
|
|
2008
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
|
|
2009
|
+
} finally {
|
|
2010
|
+
if (!holdsGraphLock) unlockGraphExclusive(state.locks);
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
|
|
1659
2014
|
// src/vector/hnsw/graph-ops.ts
|
|
1660
2015
|
function readNeighborsLocked(state, ord, layer) {
|
|
1661
2016
|
for (; ; ) {
|
|
@@ -1752,14 +2107,24 @@ function pruneConnections(state, ord, layer, distance) {
|
|
|
1752
2107
|
for (let i = 1; i <= count; i++) {
|
|
1753
2108
|
const connOrd = neighbors[base + i];
|
|
1754
2109
|
const dist = distance(ord, connOrd);
|
|
1755
|
-
if (dist === Number.POSITIVE_INFINITY)
|
|
2110
|
+
if (dist === Number.POSITIVE_INFINITY) {
|
|
2111
|
+
if (layer === 0) noteUnlinked(adjacency, connOrd);
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
1756
2114
|
candidates.ords[candidates.size] = connOrd;
|
|
1757
2115
|
candidates.distances[candidates.size] = dist;
|
|
1758
2116
|
candidates.size += 1;
|
|
1759
2117
|
}
|
|
1760
2118
|
const kept = state.workspace.pruneSelection;
|
|
1761
2119
|
selectNeighborsHeuristic(state, candidates, mc, distance, kept);
|
|
1762
|
-
|
|
2120
|
+
if (layer === 0) {
|
|
2121
|
+
writeKeepingLastLinks(state, ord, state.workspace.working, kept, everyCandidateWasHeld);
|
|
2122
|
+
return;
|
|
2123
|
+
}
|
|
2124
|
+
writeNeighbors(adjacency, ord, layer, kept.ords, kept.size);
|
|
2125
|
+
}
|
|
2126
|
+
function everyCandidateWasHeld() {
|
|
2127
|
+
return true;
|
|
1763
2128
|
}
|
|
1764
2129
|
|
|
1765
2130
|
// src/vector/hnsw/mutation.ts
|
|
@@ -1873,11 +2238,15 @@ function linkNeighborsBack(state, ord, linkTop) {
|
|
|
1873
2238
|
}
|
|
1874
2239
|
function linkNode(state, ord, level2) {
|
|
1875
2240
|
const topLayer = topLayerOf(state);
|
|
2241
|
+
const formerEntry = entryPointOf(state);
|
|
1876
2242
|
const linkTop = selectNeighborsPerLayer(state, ord, level2);
|
|
1877
2243
|
if (linkTop < 0) return;
|
|
1878
2244
|
writeOwnLists(state, ord, linkTop);
|
|
1879
2245
|
linkNeighborsBack(state, ord, linkTop);
|
|
1880
|
-
if (level2
|
|
2246
|
+
if (level2 <= topLayer) return;
|
|
2247
|
+
raiseEntry(state, ord, level2);
|
|
2248
|
+
const demoted = entryPointOf(state) !== formerEntry && isLive(state, formerEntry);
|
|
2249
|
+
if (demoted && incomingCount(state.adjacency, formerEntry) === 0) noteNodeLeftUnlinked(state);
|
|
1881
2250
|
}
|
|
1882
2251
|
function writeRecordBeforePlacement(state, ord) {
|
|
1883
2252
|
const quantizer = state.quantizer;
|
|
@@ -1903,8 +2272,12 @@ function insertNode(state, ord, holdsGraphLock = false) {
|
|
|
1903
2272
|
if (!state.store.holdsOrdinal(ord)) return false;
|
|
1904
2273
|
ensureCapacity(state, ord + 1);
|
|
1905
2274
|
if (nodeExists(state, ord)) return false;
|
|
2275
|
+
countIncomingBeforeWriting(state, holdsGraphLock);
|
|
1906
2276
|
writeRecordBeforePlacement(state, ord);
|
|
1907
|
-
if (placeThroughTheCore(state, ord, holdsGraphLock))
|
|
2277
|
+
if (placeThroughTheCore(state, ord, holdsGraphLock)) {
|
|
2278
|
+
connectAfterPlacement(state, holdsGraphLock);
|
|
2279
|
+
return true;
|
|
2280
|
+
}
|
|
1908
2281
|
if (nodeExists(state, ord)) return false;
|
|
1909
2282
|
noteFallback("place");
|
|
1910
2283
|
if (!holdsGraphLock) lockGraphShared(state.locks);
|
|
@@ -1913,11 +2286,11 @@ function insertNode(state, ord, holdsGraphLock = false) {
|
|
|
1913
2286
|
const level2 = randomLevel(state.mL);
|
|
1914
2287
|
createNode(state.adjacency, ord, level2);
|
|
1915
2288
|
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
1916
|
-
if (claimEntryIfEmpty(state, ord, level2))
|
|
1917
|
-
linkNode(state, ord, level2);
|
|
2289
|
+
if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
|
|
1918
2290
|
} finally {
|
|
1919
2291
|
if (!holdsGraphLock) unlockGraphShared(state.locks);
|
|
1920
2292
|
}
|
|
2293
|
+
connectAfterPlacement(state, holdsGraphLock);
|
|
1921
2294
|
return true;
|
|
1922
2295
|
}
|
|
1923
2296
|
function removeNodeEager(state, ord, excludeOrds) {
|
|
@@ -1935,7 +2308,8 @@ function removeNodeEager(state, ord, excludeOrds) {
|
|
|
1935
2308
|
if (layer > nodeMaxLayer(state, neighborOrd)) continue;
|
|
1936
2309
|
const mc = maxConns(state, layer);
|
|
1937
2310
|
if (neighborCount(state.adjacency, neighborOrd, layer) >= mc) continue;
|
|
1938
|
-
const
|
|
2311
|
+
const heldNow = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
|
|
2312
|
+
const candidateOrds = new Set(heldNow);
|
|
1939
2313
|
for (const otherOrd of formerNeighbors) {
|
|
1940
2314
|
if (otherOrd !== neighborOrd && otherOrd !== ord) {
|
|
1941
2315
|
if (excludeOrds?.has(otherOrd)) continue;
|
|
@@ -1946,12 +2320,19 @@ function removeNodeEager(state, ord, excludeOrds) {
|
|
|
1946
2320
|
candidates.size = 0;
|
|
1947
2321
|
for (const candOrd of candidateOrds) {
|
|
1948
2322
|
const dist = distance(neighborOrd, candOrd);
|
|
1949
|
-
if (dist === Number.POSITIVE_INFINITY)
|
|
2323
|
+
if (dist === Number.POSITIVE_INFINITY) {
|
|
2324
|
+
if (layer === 0 && heldNow.has(candOrd)) noteUnlinked(state.adjacency, candOrd);
|
|
2325
|
+
continue;
|
|
2326
|
+
}
|
|
1950
2327
|
appendToList(candidates, candOrd, dist);
|
|
1951
2328
|
}
|
|
1952
2329
|
const selected = state.workspace.repairSelection;
|
|
1953
2330
|
selectNeighborsHeuristic(state, candidates, mc, distance, selected);
|
|
1954
|
-
|
|
2331
|
+
if (layer === 0) {
|
|
2332
|
+
writeKeepingLastLinks(state, neighborOrd, state.workspace.working, selected, (held) => heldNow.has(held));
|
|
2333
|
+
} else {
|
|
2334
|
+
replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
|
|
2335
|
+
}
|
|
1955
2336
|
for (let i = 0; i < selected.size; i++) {
|
|
1956
2337
|
const newConnOrd = selected.ords[i];
|
|
1957
2338
|
if (layer <= nodeMaxLayer(state, newConnOrd)) {
|
|
@@ -1961,6 +2342,8 @@ function removeNodeEager(state, ord, excludeOrds) {
|
|
|
1961
2342
|
}
|
|
1962
2343
|
}
|
|
1963
2344
|
}
|
|
2345
|
+
const formerBaseNeighbors = collectNeighbors(state.adjacency, ord, 0);
|
|
2346
|
+
for (const neighborOrd of formerBaseNeighbors) noteUnlinked(state.adjacency, neighborOrd);
|
|
1964
2347
|
deleteNode(state.adjacency, ord);
|
|
1965
2348
|
Atomics.sub(state.header, GRAPH_NODE_COUNT, 1);
|
|
1966
2349
|
clearTombstone(state, ord);
|
|
@@ -1984,9 +2367,7 @@ function markTombstone(state, ord) {
|
|
|
1984
2367
|
}
|
|
1985
2368
|
function compactTombstones(state) {
|
|
1986
2369
|
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
1987
|
-
|
|
1988
|
-
if (field !== null && nativeCompact(state, field, state.buildMetric)) return;
|
|
1989
|
-
noteFallback("compact");
|
|
2370
|
+
countIncomingBeforeWriting(state, true);
|
|
1990
2371
|
const tombstonedOrds = [];
|
|
1991
2372
|
const slots = adjacencySlots(state.adjacency);
|
|
1992
2373
|
for (let ord = 0; ord < slots; ord++) {
|
|
@@ -1994,10 +2375,16 @@ function compactTombstones(state) {
|
|
|
1994
2375
|
tombstonedOrds.push(ord);
|
|
1995
2376
|
}
|
|
1996
2377
|
}
|
|
1997
|
-
const
|
|
1998
|
-
|
|
1999
|
-
|
|
2378
|
+
const nearby = baseNeighboursOfRemoved(state, tombstonedOrds);
|
|
2379
|
+
const field = nativeFieldFor(state);
|
|
2380
|
+
if (field === null || !nativeCompact(state, field, state.buildMetric)) {
|
|
2381
|
+
noteFallback("compact");
|
|
2382
|
+
const excl = new Set(tombstonedOrds);
|
|
2383
|
+
for (const ord of tombstonedOrds) {
|
|
2384
|
+
removeNodeEager(state, ord, excl);
|
|
2385
|
+
}
|
|
2000
2386
|
}
|
|
2387
|
+
connectAfterRemoval(state, nearby);
|
|
2001
2388
|
}
|
|
2002
2389
|
function resetGraph(state) {
|
|
2003
2390
|
resetAdjacency(state.adjacency);
|
|
@@ -2007,6 +2394,7 @@ function resetGraph(state) {
|
|
|
2007
2394
|
Atomics.store(state.header, GRAPH_NODE_COUNT, 0);
|
|
2008
2395
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, -1);
|
|
2009
2396
|
Atomics.store(state.header, GRAPH_TOP_LAYER, -1);
|
|
2397
|
+
Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
|
|
2010
2398
|
}
|
|
2011
2399
|
function rebuild(state) {
|
|
2012
2400
|
const nodeCount = Atomics.load(state.header, GRAPH_NODE_COUNT);
|
|
@@ -2485,7 +2873,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2485
2873
|
let centroidGeneration = -1;
|
|
2486
2874
|
let currentQuery = null;
|
|
2487
2875
|
const singleOrdinal = new Int32Array(1);
|
|
2488
|
-
function
|
|
2876
|
+
function reach2(ordinal) {
|
|
2489
2877
|
if (ordinal < codePresent.length) return true;
|
|
2490
2878
|
if (ordinal >= store.handles.codePresent.byteLength) return false;
|
|
2491
2879
|
codePresent = fixedView(store.handles.codePresent, Uint8Array);
|
|
@@ -2506,8 +2894,8 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2506
2894
|
if (generation !== centroidGeneration) recomputeCentroidDot(generation);
|
|
2507
2895
|
return centroidDot;
|
|
2508
2896
|
}
|
|
2509
|
-
function
|
|
2510
|
-
return ordinal >= 0 &&
|
|
2897
|
+
function holds2(ordinal) {
|
|
2898
|
+
return ordinal >= 0 && reach2(ordinal) && codePresent[ordinal] === 1;
|
|
2511
2899
|
}
|
|
2512
2900
|
function recordOffset(ordinal) {
|
|
2513
2901
|
return slotByteOffset(layout, ordinal % capacity);
|
|
@@ -2528,7 +2916,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2528
2916
|
}
|
|
2529
2917
|
}
|
|
2530
2918
|
function writeRecordOf(ordinal) {
|
|
2531
|
-
if (!
|
|
2919
|
+
if (!reach2(ordinal) || !store.holdsOrdinal(ordinal)) return;
|
|
2532
2920
|
const code = osqQuantize(store.vectorAt(ordinal), store.handles.centroid, bits, metric, scratch);
|
|
2533
2921
|
const block = store.codeBlockOf(ordinal);
|
|
2534
2922
|
const offset = recordOffset(ordinal);
|
|
@@ -2599,7 +2987,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2599
2987
|
return currentQuery;
|
|
2600
2988
|
}
|
|
2601
2989
|
function distanceBetween(ordA, ordB) {
|
|
2602
|
-
if (!
|
|
2990
|
+
if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
|
|
2603
2991
|
const blockA = store.codeBlockOf(ordA);
|
|
2604
2992
|
const blockB = store.codeBlockOf(ordB);
|
|
2605
2993
|
const offsetA = recordOffset(ordA);
|
|
@@ -2627,7 +3015,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2627
3015
|
return isCalibrated() ? store.handles.centroid : null;
|
|
2628
3016
|
},
|
|
2629
3017
|
isCalibrated,
|
|
2630
|
-
holdsOrdinal:
|
|
3018
|
+
holdsOrdinal: holds2,
|
|
2631
3019
|
writeCodes(ordinal) {
|
|
2632
3020
|
singleOrdinal[0] = ordinal;
|
|
2633
3021
|
writeCodesOf(singleOrdinal);
|
|
@@ -2642,7 +3030,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2642
3030
|
return true;
|
|
2643
3031
|
},
|
|
2644
3032
|
restoreRecord(ordinal, record) {
|
|
2645
|
-
if (!
|
|
3033
|
+
if (!reach2(ordinal) || record.length !== layout.slotStride) return;
|
|
2646
3034
|
const block = store.codeBlockOf(ordinal);
|
|
2647
3035
|
const offset = recordOffset(ordinal);
|
|
2648
3036
|
block.bytes(offset + layout.slotStride).set(record, offset);
|
|
@@ -2654,7 +3042,7 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2654
3042
|
return block.bytes(offset + layout.slotStride).subarray(offset, offset + layout.slotStride);
|
|
2655
3043
|
},
|
|
2656
3044
|
clearCodes(ordinal) {
|
|
2657
|
-
if (ordinal < 0 || !
|
|
3045
|
+
if (ordinal < 0 || !reach2(ordinal)) return;
|
|
2658
3046
|
if (Atomics.exchange(codePresent, ordinal, 0) === 1) Atomics.sub(store.handles.header, STORE_CODE_COUNT, 1);
|
|
2659
3047
|
},
|
|
2660
3048
|
writeCentroid,
|
|
@@ -2670,11 +3058,11 @@ function openSharedQuantizer(store, layout, metric) {
|
|
|
2670
3058
|
},
|
|
2671
3059
|
prepareQuery,
|
|
2672
3060
|
distanceFromPreparedByOrdinal(prepared, ordinal) {
|
|
2673
|
-
if (!
|
|
3061
|
+
if (!holds2(ordinal)) return Number.POSITIVE_INFINITY;
|
|
2674
3062
|
return estimateDistance(ordinal, prepared);
|
|
2675
3063
|
},
|
|
2676
3064
|
preparedDistance(prepared) {
|
|
2677
|
-
const generic = (ordinal) =>
|
|
3065
|
+
const generic = (ordinal) => holds2(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
|
|
2678
3066
|
if (store.handles.codeBlocks.length === 0) return generic;
|
|
2679
3067
|
const block = store.codeBlockOf(0);
|
|
2680
3068
|
const simd = block.simd;
|
|
@@ -2910,7 +3298,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
2910
3298
|
const index = Math.floor(ordinal / capacity);
|
|
2911
3299
|
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
2912
3300
|
}
|
|
2913
|
-
function
|
|
3301
|
+
function holds2(ordinal) {
|
|
2914
3302
|
return holdState(ordinal) !== NOT_HELD;
|
|
2915
3303
|
}
|
|
2916
3304
|
function readFromFile(file, offset, target) {
|
|
@@ -2990,7 +3378,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
2990
3378
|
reader?.retainOnly(new Set(next.vectorFiles.filter((path) => path.length > 0)));
|
|
2991
3379
|
},
|
|
2992
3380
|
rebind,
|
|
2993
|
-
holdsOrdinal:
|
|
3381
|
+
holdsOrdinal: holds2,
|
|
2994
3382
|
isCold,
|
|
2995
3383
|
readColdInto,
|
|
2996
3384
|
readFromFile,
|
|
@@ -3031,7 +3419,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
3031
3419
|
return magnitudes[ordinal];
|
|
3032
3420
|
},
|
|
3033
3421
|
entryForOrdinal(ordinal) {
|
|
3034
|
-
if (!
|
|
3422
|
+
if (!holds2(ordinal)) return void 0;
|
|
3035
3423
|
return { vector: view.vectorAt(ordinal), magnitude: magnitudes[ordinal] };
|
|
3036
3424
|
},
|
|
3037
3425
|
docIdAt(ordinal) {
|
|
@@ -3053,7 +3441,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
3053
3441
|
return partition < 0 ? void 0 : partition;
|
|
3054
3442
|
},
|
|
3055
3443
|
distanceByOrdinal(ordA, ordB, metric) {
|
|
3056
|
-
if (!
|
|
3444
|
+
if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
|
|
3057
3445
|
if (!isCold(ordB)) {
|
|
3058
3446
|
const indexA = ordA / capacity | 0;
|
|
3059
3447
|
const indexB = ordB / capacity | 0;
|
|
@@ -3175,6 +3563,9 @@ function openSharedVectorField(initial, threadSlot) {
|
|
|
3175
3563
|
get readsEveryVector() {
|
|
3176
3564
|
return store.holdsEveryBlock;
|
|
3177
3565
|
},
|
|
3566
|
+
get graphRetired() {
|
|
3567
|
+
return graph !== null && Atomics.load(graph.header, GRAPH_RETIRED) === 1;
|
|
3568
|
+
},
|
|
3178
3569
|
docIdOf,
|
|
3179
3570
|
ordinalOf,
|
|
3180
3571
|
vectorOf(docId) {
|
|
@@ -3463,6 +3854,9 @@ function ensureOrdinal(store, ordinal, cold) {
|
|
|
3463
3854
|
const { handles } = store;
|
|
3464
3855
|
growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
|
|
3465
3856
|
growBufferTo(handles.present, ordinal + 1);
|
|
3857
|
+
growBufferTo(handles.unplaced, ordinal + 1);
|
|
3858
|
+
growBufferTo(handles.unplacedChunks, unplacedChunkCount(ordinal) * 4);
|
|
3859
|
+
growBufferTo(handles.removed, ordinal + 1);
|
|
3466
3860
|
growBufferTo(handles.partitions, (ordinal + 1) * 4);
|
|
3467
3861
|
growBufferTo(handles.codePresent, ordinal + 1);
|
|
3468
3862
|
const diskFileBytes = handles.diskFile.byteLength;
|
|
@@ -4023,7 +4417,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
|
|
|
4023
4417
|
}
|
|
4024
4418
|
cursor += count + 1;
|
|
4025
4419
|
if (ordinal !== NO_ORDINAL && layer <= keptTop) {
|
|
4026
|
-
|
|
4420
|
+
writeNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
|
|
4027
4421
|
}
|
|
4028
4422
|
}
|
|
4029
4423
|
}
|
|
@@ -4035,6 +4429,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
|
|
|
4035
4429
|
const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
|
|
4036
4430
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
|
|
4037
4431
|
Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
|
|
4432
|
+
leaveIncomingUncounted(state);
|
|
4038
4433
|
}
|
|
4039
4434
|
|
|
4040
4435
|
// src/vector/hnsw/persistence.ts
|
|
@@ -4073,7 +4468,7 @@ function deserializeGraph(state, store, data) {
|
|
|
4073
4468
|
for (const node of nodes) {
|
|
4074
4469
|
createNode(state.adjacency, node.ord, node.maxLayer);
|
|
4075
4470
|
for (const [layer, neighbors] of node.layers) {
|
|
4076
|
-
|
|
4471
|
+
writeNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
|
|
4077
4472
|
}
|
|
4078
4473
|
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
4079
4474
|
}
|
|
@@ -4099,6 +4494,7 @@ function deserializeGraph(state, store, data) {
|
|
|
4099
4494
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, bestOrd);
|
|
4100
4495
|
Atomics.store(state.header, GRAPH_TOP_LAYER, bestLayer);
|
|
4101
4496
|
}
|
|
4497
|
+
leaveIncomingUncounted(state);
|
|
4102
4498
|
}
|
|
4103
4499
|
|
|
4104
4500
|
// src/vector/hnsw/scratch-bytes.ts
|
|
@@ -4159,6 +4555,7 @@ function restoreSnapshot(state, snapshot) {
|
|
|
4159
4555
|
Atomics.store(state.header, GRAPH_NODE_COUNT, snapshot.nodeCount);
|
|
4160
4556
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, snapshot.entryPointOrd);
|
|
4161
4557
|
Atomics.store(state.header, GRAPH_TOP_LAYER, snapshot.topLayer);
|
|
4558
|
+
leaveIncomingUncounted(state);
|
|
4162
4559
|
}
|
|
4163
4560
|
|
|
4164
4561
|
// src/vector/hnsw/index.ts
|
|
@@ -4270,6 +4667,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4270
4667
|
},
|
|
4271
4668
|
search: (query, k, searchMetric, minSimilarity, options) => search(state, docIdOf, query, k, searchMetric, minSimilarity, options),
|
|
4272
4669
|
clear: () => exclusively(() => resetGraph(state)),
|
|
4670
|
+
retire: () => Atomics.store(shared.header, GRAPH_RETIRED, 1),
|
|
4273
4671
|
entries: entriesIterator,
|
|
4274
4672
|
compactionNeeded,
|
|
4275
4673
|
compactTombstones: () => exclusively(() => compactTombstones(state)),
|
|
@@ -4283,4 +4681,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4283
4681
|
};
|
|
4284
4682
|
}
|
|
4285
4683
|
|
|
4286
|
-
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct,
|
|
4684
|
+
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, STORE_UNPLACED_COUNT, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, euclideanDistance, graphWithoutNodes, isSimdAvailable, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
|