@delali/narsil 0.3.1-next.84 → 0.3.1-next.85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
2
- import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-B3UOQJD3.mjs';
2
+ import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-NFF5TDRG.mjs';
3
3
  import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
4
4
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
@@ -12,6 +12,7 @@ var VECTOR_STORE_INITIAL_CAPACITY = 16;
12
12
  var VECTOR_BLOCK_INITIAL_SLOTS = 256;
13
13
  var DISK_STORAGE_BLOCK_BYTES = 67108864;
14
14
  var MAX_VECTOR_ORDINALS = 16777216;
15
+ var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
15
16
  var VECTOR_SCRATCH_SLOTS = 64;
16
17
  var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
17
18
  var MAX_DOC_ID_TABLE_BYTES = 1073741824;
@@ -239,8 +240,11 @@ var GRAPH_M = 2;
239
240
  var GRAPH_MMAX0 = 3;
240
241
  var GRAPH_EF_CONSTRUCTION = 4;
241
242
  var GRAPH_METRIC = 5;
243
+ var GRAPH_RETIRED = 6;
242
244
  var GRAPH_NODE_COUNT = 32;
243
245
  var GRAPH_TOMBSTONE_COUNT = 33;
246
+ var GRAPH_UNLINKED_NODES = 34;
247
+ var GRAPH_INCOMING_UNCOUNTED = 35;
244
248
  var GRAPH_UPPER_USED = 64;
245
249
  var GRAPH_SLOTS = 65;
246
250
  var GRAPH_UPPER_CAPACITY = 66;
@@ -278,6 +282,7 @@ function createSharedGraphHandles(shape) {
278
282
  upper: perOrdinal(upperStride * 4, INITIAL_UPPER_SLOTS),
279
283
  locks: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
280
284
  tombstones: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
285
+ incoming: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
281
286
  heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
282
287
  };
283
288
  publishGraphCapacity(handles);
@@ -297,7 +302,8 @@ function publishGraphCapacity(handles) {
297
302
  Math.floor(handles.level0.byteLength / (level0Stride * 4)),
298
303
  Math.floor(handles.upperBase.byteLength / 4),
299
304
  Math.floor(handles.locks.byteLength / 4),
300
- handles.tombstones.byteLength
305
+ handles.tombstones.byteLength,
306
+ Math.floor(handles.incoming.byteLength / 4)
301
307
  );
302
308
  raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
303
309
  raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
@@ -612,6 +618,8 @@ var STORE_DOC_ID_BYTES = 5;
612
618
  var STORE_CODE_BLOCK_COUNT = 6;
613
619
  var STORE_CALIBRATION_GENERATION = 7;
614
620
  var STORE_RELEASED_VECTORS_TO_DISK = 8;
621
+ var STORE_UNPLACED_COUNT = 9;
622
+ var STORE_REMOVED_COUNT = 10;
615
623
  var STORE_HEADER_WORDS = 16;
616
624
  var IN_MEMORY = -1;
617
625
  function fixedInt32(words) {
@@ -620,6 +628,9 @@ function fixedInt32(words) {
620
628
  function fixedFloat32(length) {
621
629
  return new Float32Array(createFixedBuffer(length * 4), 0, length);
622
630
  }
631
+ function unplacedChunkCount(ordinals) {
632
+ return Math.floor(ordinals / UNPLACED_FLAG_CHUNK_ORDINALS) + 1;
633
+ }
623
634
  function perOrdinal2(bytesPerOrdinal) {
624
635
  return createGrowableBuffer(
625
636
  VECTOR_STORE_INITIAL_CAPACITY * bytesPerOrdinal,
@@ -644,6 +655,9 @@ function createSharedVectorStoreHandles(dimension, codeBits, blockBytes2 = BLOCK
644
655
  vectorFiles: [],
645
656
  magnitudes: perOrdinal2(8),
646
657
  present: perOrdinal2(1),
658
+ unplaced: perOrdinal2(1),
659
+ unplacedChunks: createGrowableBuffer(4, unplacedChunkCount(MAX_VECTOR_ORDINALS) * 4),
660
+ removed: perOrdinal2(1),
647
661
  partitions: perOrdinal2(4),
648
662
  docIdBytes: createGrowableBuffer(VECTOR_STORE_INITIAL_CAPACITY * 16, MAX_DOC_ID_TABLE_BYTES),
649
663
  docIdOffsets: perOrdinal2(4),
@@ -661,6 +675,9 @@ function sharedVectorStoreBytes(handles) {
661
675
  for (const block of handles.codeBlocks) bytes += blockBuffer(block).byteLength;
662
676
  bytes += handles.magnitudes.byteLength;
663
677
  bytes += handles.present.byteLength;
678
+ bytes += handles.unplaced.byteLength;
679
+ bytes += handles.unplacedChunks.byteLength;
680
+ bytes += handles.removed.byteLength;
664
681
  bytes += handles.partitions.byteLength;
665
682
  bytes += handles.docIdBytes.byteLength;
666
683
  bytes += handles.docIdOffsets.byteLength;
@@ -857,6 +874,7 @@ function graphMemory(graph) {
857
874
  graph.upper,
858
875
  graph.locks,
859
876
  graph.tombstones,
877
+ graph.incoming,
860
878
  graph.heldLocks.buffer
861
879
  ];
862
880
  if (!regions.every(isSharedMemory)) return null;
@@ -868,6 +886,7 @@ function graphMemory(graph) {
868
886
  upper: fixedView(graph.upper, Int32Array),
869
887
  locks: fixedView(graph.locks, Int32Array),
870
888
  tombstones: fixedView(graph.tombstones, Uint8Array),
889
+ incoming: fixedView(graph.incoming, Int32Array),
871
890
  heldLocks: graph.heldLocks
872
891
  };
873
892
  }
@@ -967,6 +986,17 @@ function nativeCompact(state, field, metric) {
967
986
  const compact = () => field.core.compact(field.graph, field.store.handle, nativeMetricCode(metric), state.locks.threadSlot, field.wakes);
968
987
  return writeToTheGraph(state, field, compact) === NATIVE_OK;
969
988
  }
989
+ function nativeConnect(state, field, metric, nearby) {
990
+ const connect = () => field.core.connectUnreached(
991
+ field.graph,
992
+ field.store.handle,
993
+ nativeMetricCode(metric),
994
+ nearby,
995
+ state.locks.threadSlot,
996
+ field.wakes
997
+ );
998
+ return writeToTheGraph(state, field, connect) === NATIVE_OK;
999
+ }
970
1000
 
971
1001
  // src/vector/hnsw/adjacency/views.ts
972
1002
  var ABSENT = -1;
@@ -981,11 +1011,13 @@ function openAdjacency(handles, m, mMax0) {
981
1011
  nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
982
1012
  level0: fixedView(handles.level0, Int32Array),
983
1013
  upperBase: fixedView(handles.upperBase, Int32Array),
984
- upper: fixedView(handles.upper, Int32Array)
1014
+ upper: fixedView(handles.upper, Int32Array),
1015
+ incoming: fixedView(handles.incoming, Int32Array)
985
1016
  };
986
1017
  }
987
1018
  function rebindNodes(adj) {
988
1019
  adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
1020
+ adj.incoming = fixedView(adj.handles.incoming, Int32Array);
989
1021
  adj.level0 = fixedView(adj.handles.level0, Int32Array);
990
1022
  adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
991
1023
  }
@@ -1022,6 +1054,7 @@ function ensureAdjacencyCapacity(adj, needed) {
1022
1054
  growBufferToExactly(handles.upperBase, capacity * 4);
1023
1055
  growBufferToExactly(handles.locks, capacity * 4);
1024
1056
  growBufferToExactly(handles.tombstones, capacity);
1057
+ growBufferToExactly(handles.incoming, capacity * 4);
1025
1058
  growBufferToExactly(handles.nodeLevels, capacity);
1026
1059
  publishGraphCapacity(handles);
1027
1060
  rebindNodes(adj);
@@ -1068,10 +1101,12 @@ function deleteNode(adj, ord) {
1068
1101
  Atomics.store(adj.nodeLevels, ord, 0);
1069
1102
  adj.upperBase[ord] = 0;
1070
1103
  adj.level0[ord * adj.level0Stride] = 0;
1104
+ Atomics.store(adj.incoming, ord, 0);
1071
1105
  }
1072
1106
  function resetAdjacency(adj) {
1073
1107
  rebindNodes(adj);
1074
1108
  rebindUpper(adj);
1109
+ adj.incoming.fill(0);
1075
1110
  adj.nodeLevels.fill(0);
1076
1111
  adj.level0.fill(0);
1077
1112
  adj.upperBase.fill(0);
@@ -1088,7 +1123,51 @@ function hasNode(adj, ord) {
1088
1123
  }
1089
1124
  function graphBytes(adj) {
1090
1125
  const handles = adj.handles;
1091
- return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.heldLocks.byteLength;
1126
+ return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
1127
+ }
1128
+
1129
+ // src/vector/hnsw/adjacency/incoming.ts
1130
+ function incomingCount(adj, ord) {
1131
+ if (ord < 0 || !reachNode(adj, ord)) return 0;
1132
+ return Atomics.load(adj.incoming, ord);
1133
+ }
1134
+ function noteLinked(adj, ord) {
1135
+ if (ord < 0 || !reachNode(adj, ord)) return;
1136
+ Atomics.add(adj.incoming, ord, 1);
1137
+ }
1138
+ function noteUnlinked(adj, ord) {
1139
+ if (ord < 0 || !reachNode(adj, ord)) return 0;
1140
+ for (; ; ) {
1141
+ const seen = Atomics.load(adj.incoming, ord);
1142
+ if (seen <= 0) return 0;
1143
+ if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return seen - 1;
1144
+ }
1145
+ }
1146
+ function releaseUnlessLast(adj, ord) {
1147
+ if (ord < 0 || !reachNode(adj, ord)) return true;
1148
+ for (; ; ) {
1149
+ const seen = Atomics.load(adj.incoming, ord);
1150
+ if (seen <= 1) return false;
1151
+ if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return true;
1152
+ }
1153
+ }
1154
+ function holdsNodeAt(adj, ord) {
1155
+ return ord >= 0 && ord < adj.nodeLevels.length && adj.nodeLevels[ord] !== 0;
1156
+ }
1157
+ function countLinksOutOf(adj, ord, step) {
1158
+ if (!holdsNodeAt(adj, ord)) return;
1159
+ const incoming = adj.incoming;
1160
+ const level0 = adj.level0;
1161
+ const base = ord * adj.level0Stride;
1162
+ const count = level0[base];
1163
+ for (let i = 1; i <= count; i++) {
1164
+ const neighbour = level0[base + i];
1165
+ if (neighbour === ord || !holdsNodeAt(adj, neighbour)) continue;
1166
+ if (incoming[neighbour] + step >= 0) incoming[neighbour] += step;
1167
+ }
1168
+ }
1169
+ function countEveryLink(adj, slots) {
1170
+ for (let ord = 0; ord < slots; ord++) countLinksOutOf(adj, ord, 1);
1092
1171
  }
1093
1172
 
1094
1173
  // src/vector/hnsw/adjacency/neighbors.ts
@@ -1138,6 +1217,7 @@ function addNeighbor(adj, ord, layer, neighborOrd) {
1138
1217
  if (count + 1 > stride - 1) return;
1139
1218
  array[base + count + 1] = neighborOrd;
1140
1219
  array[base] = count + 1;
1220
+ if (layer === 0) noteLinked(adj, neighborOrd);
1141
1221
  }
1142
1222
  function removeNeighbor(adj, ord, layer, neighborOrd) {
1143
1223
  const base = layerBase(adj, ord, layer);
@@ -1150,10 +1230,11 @@ function removeNeighbor(adj, ord, layer, neighborOrd) {
1150
1230
  array[base + j] = array[base + j + 1];
1151
1231
  }
1152
1232
  array[base] = count - 1;
1233
+ if (layer === 0) noteUnlinked(adj, neighborOrd);
1153
1234
  return;
1154
1235
  }
1155
1236
  }
1156
- function replaceNeighbors(adj, ord, layer, neighbors, count) {
1237
+ function writeNeighbors(adj, ord, layer, neighbors, count) {
1157
1238
  const base = layerBase(adj, ord, layer);
1158
1239
  if (base === ABSENT) return;
1159
1240
  const array = layerArray(adj, layer);
@@ -1164,163 +1245,23 @@ function replaceNeighbors(adj, ord, layer, neighbors, count) {
1164
1245
  }
1165
1246
  array[base] = limit;
1166
1247
  }
1167
-
1168
- // src/vector/hnsw/locks.ts
1169
- var FREE = 0;
1170
- var EXCLUSIVE = -1;
1171
- function openGraphLocks(handles, threadSlot) {
1172
- let words = fixedView(handles.locks, Int32Array);
1173
- return {
1174
- threadSlot,
1175
- header: handles.header,
1176
- held: handles.heldLocks,
1177
- words(ord) {
1178
- if (ord < words.length) return words;
1179
- if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
1180
- words = fixedView(handles.locks, Int32Array);
1181
- return words;
1182
- }
1183
- };
1184
- }
1185
- var blockingAllowed = typeof document === "undefined";
1186
- function canWait(words) {
1187
- return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
1188
- }
1189
- function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
1190
- if (!canWait(words)) return;
1191
- Atomics.wait(words, index, seen, sliceMs);
1192
- }
1193
- function heldIndex(locks, kind) {
1194
- return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
1195
- }
1196
- function acquireExclusive(words, index) {
1197
- let spins = 0;
1198
- for (; ; ) {
1199
- const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
1200
- if (seen === FREE) return;
1201
- if (spins < LOCK_SPIN_ITERATIONS) {
1202
- spins += 1;
1203
- continue;
1204
- }
1205
- waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
1206
- }
1207
- }
1208
- function releaseExclusive(words, index) {
1209
- Atomics.store(words, index, FREE);
1210
- if (canWait(words)) Atomics.notify(words, index);
1211
- }
1212
- var WRITE_HELD = 1;
1213
- var WRITER_WAITING = 2;
1214
- var VERSION_STEP = 4;
1215
- function beginNodeRead(locks, ord) {
1216
- const words = locks.words(ord);
1217
- for (; ; ) {
1218
- const seen = Atomics.load(words, ord);
1219
- if ((seen & WRITE_HELD) === 0) return seen;
1220
- }
1221
- }
1222
- function nodeReadHeld(locks, ord, version) {
1223
- Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
1224
- return Atomics.load(locks.words(ord), ord) === version;
1225
- }
1226
- function lockNodeWrite(locks, ord) {
1227
- const words = locks.words(ord);
1228
- let spins = 0;
1229
- for (; ; ) {
1230
- const seen = Atomics.load(words, ord);
1231
- if ((seen & WRITE_HELD) === 0) {
1232
- const claimed = seen | WRITE_HELD;
1233
- if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
1234
- Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
1235
- Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
1236
- return;
1237
- }
1238
- continue;
1239
- }
1240
- if (spins < LOCK_SPIN_ITERATIONS) {
1241
- spins += 1;
1242
- continue;
1243
- }
1244
- const waiting = seen | WRITER_WAITING;
1245
- if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
1246
- waitOn(words, ord, waiting);
1247
- }
1248
- }
1249
- }
1250
- function releaseWrite(words, ord) {
1251
- const seen = Atomics.load(words, ord);
1252
- const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
1253
- if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
1254
- }
1255
- function unlockNodeWrite(locks, ord) {
1256
- releaseWrite(locks.words(ord), ord);
1257
- Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
1258
- }
1259
- function lockGraphShared(locks) {
1260
- const { header } = locks;
1261
- let spins = 0;
1262
- for (; ; ) {
1263
- const seen = Atomics.load(header, GRAPH_LOCK);
1264
- if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
1265
- if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
1266
- Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
1267
- return;
1268
- }
1269
- continue;
1270
- }
1271
- if (spins < LOCK_SPIN_ITERATIONS) {
1272
- spins += 1;
1273
- continue;
1274
- }
1275
- waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
1276
- }
1277
- }
1278
- function unlockGraphShared(locks) {
1279
- const { header } = locks;
1280
- Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
1281
- if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
1282
- }
1283
- function lockGraphExclusive(locks) {
1284
- Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
1285
- Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
1286
- try {
1287
- acquireExclusive(locks.header, GRAPH_LOCK);
1288
- } finally {
1289
- Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
1290
- Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
1291
- }
1292
- }
1293
- function unlockGraphExclusive(locks) {
1294
- releaseExclusive(locks.header, GRAPH_LOCK);
1295
- }
1296
- function lockEntry(locks) {
1297
- acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
1298
- }
1299
- function unlockEntry(locks) {
1300
- releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
1248
+ function holds(neighbors, count, ord) {
1249
+ for (let i = 0; i < count; i++) if (neighbors[i] === ord) return true;
1250
+ return false;
1301
1251
  }
1302
- function releaseLocksHeldBy(handles, threadSlot) {
1303
- const held = handles.heldLocks;
1304
- const words = new Int32Array(handles.locks);
1305
- const base = threadSlot * HELD_WORDS_PER_THREAD;
1306
- if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
1307
- Atomics.sub(handles.header, GRAPH_WRITERS_WAITING, 1);
1308
- }
1309
- const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
1310
- const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
1311
- if (writeOrd >= 0 && writeOrd < words.length) {
1312
- const seen = Atomics.load(words, writeOrd);
1313
- if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
1314
- }
1315
- if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
1316
- for (; ; ) {
1317
- const readers = Atomics.load(handles.header, GRAPH_LOCK);
1318
- if (readers <= FREE) return;
1319
- if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
1320
- if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
1321
- return;
1322
- }
1252
+ function replaceNeighbors(adj, ord, layer, neighbors, count) {
1253
+ const base = layerBase(adj, ord, layer);
1254
+ if (base === ABSENT) return;
1255
+ if (layer > 0) {
1256
+ writeNeighbors(adj, ord, layer, neighbors, count);
1257
+ return;
1323
1258
  }
1259
+ const array = adj.level0;
1260
+ const held = array.slice(base + 1, base + 1 + array[base]);
1261
+ const limit = Math.min(count, adj.level0Stride - 1);
1262
+ for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
1263
+ for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
1264
+ writeNeighbors(adj, ord, 0, neighbors, count);
1324
1265
  }
1325
1266
 
1326
1267
  // src/vector/similarity.ts
@@ -1498,7 +1439,8 @@ function createHNSWWorkspace() {
1498
1439
  pruneCandidates: createList(INITIAL_LIST_CAPACITY),
1499
1440
  pruneSelection: createList(INITIAL_LIST_CAPACITY),
1500
1441
  repairCandidates: createList(INITIAL_LIST_CAPACITY),
1501
- repairSelection: createList(INITIAL_LIST_CAPACITY)
1442
+ repairSelection: createList(INITIAL_LIST_CAPACITY),
1443
+ droppedLinks: createList(INITIAL_LIST_CAPACITY)
1502
1444
  };
1503
1445
  }
1504
1446
  function linkSelectionsFor(workspace, layers) {
@@ -1656,6 +1598,419 @@ function setEntryPointsFromList(workspace, list) {
1656
1598
  workspace.entryPointCount = list.size;
1657
1599
  }
1658
1600
 
1601
+ // src/vector/hnsw/last-links.ts
1602
+ function isLive(state, ord) {
1603
+ return nodeExists(state, ord) && !isTombstoned(state, ord);
1604
+ }
1605
+ function noteNodeLeftUnlinked(state) {
1606
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 1);
1607
+ }
1608
+ function listHolds(list, ord) {
1609
+ for (let i = 0; i < list.size; i++) if (list.ords[i] === ord) return true;
1610
+ return false;
1611
+ }
1612
+ function collectDropped(sorted, kept, dropped) {
1613
+ dropped.size = 0;
1614
+ let next = 0;
1615
+ for (let i = 0; i < sorted.size; i++) {
1616
+ const ord = sorted.ords[i];
1617
+ if (next < kept.size && kept.ords[next] === ord) {
1618
+ next += 1;
1619
+ continue;
1620
+ }
1621
+ if (listHolds(kept, ord)) continue;
1622
+ appendToList(dropped, ord, sorted.distances[i]);
1623
+ }
1624
+ }
1625
+ function replaceFarthestShared(state, kept, held, ord, distance) {
1626
+ const tried = /* @__PURE__ */ new Set();
1627
+ for (; ; ) {
1628
+ let farthest = -1;
1629
+ for (let k = 0; k < kept.size; k++) {
1630
+ if (tried.has(k) || incomingCount(state.adjacency, kept.ords[k]) < 2) continue;
1631
+ if (farthest === -1 || kept.distances[k] > kept.distances[farthest]) farthest = k;
1632
+ }
1633
+ if (farthest === -1) return false;
1634
+ tried.add(farthest);
1635
+ const replaced = kept.ords[farthest];
1636
+ if (held(replaced) && !releaseUnlessLast(state.adjacency, replaced)) continue;
1637
+ kept.ords[farthest] = ord;
1638
+ kept.distances[farthest] = distance;
1639
+ return true;
1640
+ }
1641
+ }
1642
+ function writeKeepingLastLinks(state, ord, sorted, kept, held) {
1643
+ const adjacency = state.adjacency;
1644
+ const limit = state.Mmax0;
1645
+ const dropped = state.workspace.droppedLinks;
1646
+ collectDropped(sorted, kept, dropped);
1647
+ ensureListCapacity(kept, limit);
1648
+ for (let i = 0; i < dropped.size; i++) {
1649
+ const candidate = dropped.ords[i];
1650
+ if (!held(candidate)) continue;
1651
+ if (!isLive(state, candidate)) {
1652
+ noteUnlinked(adjacency, candidate);
1653
+ continue;
1654
+ }
1655
+ if (releaseUnlessLast(adjacency, candidate)) continue;
1656
+ if (kept.size < limit) {
1657
+ appendToList(kept, candidate, dropped.distances[i]);
1658
+ continue;
1659
+ }
1660
+ if (replaceFarthestShared(state, kept, held, candidate, dropped.distances[i])) continue;
1661
+ noteUnlinked(adjacency, candidate);
1662
+ noteNodeLeftUnlinked(state);
1663
+ }
1664
+ for (let k = 0; k < kept.size; k++) if (!held(kept.ords[k])) noteLinked(adjacency, kept.ords[k]);
1665
+ writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
1666
+ }
1667
+
1668
+ // src/vector/hnsw/locks.ts
1669
+ var FREE = 0;
1670
+ var EXCLUSIVE = -1;
1671
+ function openGraphLocks(handles, threadSlot) {
1672
+ let words = fixedView(handles.locks, Int32Array);
1673
+ return {
1674
+ threadSlot,
1675
+ header: handles.header,
1676
+ held: handles.heldLocks,
1677
+ words(ord) {
1678
+ if (ord < words.length) return words;
1679
+ if (ord * 4 >= handles.locks.byteLength) growBufferTo(handles.locks, (ord + 1) * 4);
1680
+ words = fixedView(handles.locks, Int32Array);
1681
+ return words;
1682
+ }
1683
+ };
1684
+ }
1685
+ var blockingAllowed = typeof document === "undefined";
1686
+ function canWait(words) {
1687
+ return blockingAllowed && sharedMemoryAvailable() && words.buffer instanceof SharedArrayBuffer;
1688
+ }
1689
+ function waitOn(words, index, seen, sliceMs = LOCK_WAIT_SLICE_MS) {
1690
+ if (!canWait(words)) return;
1691
+ Atomics.wait(words, index, seen, sliceMs);
1692
+ }
1693
+ function heldIndex(locks, kind) {
1694
+ return locks.threadSlot * HELD_WORDS_PER_THREAD + kind;
1695
+ }
1696
+ function acquireExclusive(words, index) {
1697
+ let spins = 0;
1698
+ for (; ; ) {
1699
+ const seen = Atomics.compareExchange(words, index, FREE, EXCLUSIVE);
1700
+ if (seen === FREE) return;
1701
+ if (spins < LOCK_SPIN_ITERATIONS) {
1702
+ spins += 1;
1703
+ continue;
1704
+ }
1705
+ waitOn(words, index, seen, GRAPH_LOCK_WAIT_SLICE_MS);
1706
+ }
1707
+ }
1708
+ function releaseExclusive(words, index) {
1709
+ Atomics.store(words, index, FREE);
1710
+ if (canWait(words)) Atomics.notify(words, index);
1711
+ }
1712
+ var WRITE_HELD = 1;
1713
+ var WRITER_WAITING = 2;
1714
+ var VERSION_STEP = 4;
1715
+ function beginNodeRead(locks, ord) {
1716
+ const words = locks.words(ord);
1717
+ for (; ; ) {
1718
+ const seen = Atomics.load(words, ord);
1719
+ if ((seen & WRITE_HELD) === 0) return seen;
1720
+ }
1721
+ }
1722
+ function nodeReadHeld(locks, ord, version) {
1723
+ Atomics.add(locks.held, heldIndex(locks, HELD_FENCE), 0);
1724
+ return Atomics.load(locks.words(ord), ord) === version;
1725
+ }
1726
+ function lockNodeWrite(locks, ord) {
1727
+ const words = locks.words(ord);
1728
+ let spins = 0;
1729
+ for (; ; ) {
1730
+ const seen = Atomics.load(words, ord);
1731
+ if ((seen & WRITE_HELD) === 0) {
1732
+ const claimed = seen | WRITE_HELD;
1733
+ if (Atomics.compareExchange(words, ord, seen, claimed) === seen) {
1734
+ Atomics.store(locks.held, heldIndex(locks, HELD_WRITE_VERSION), claimed);
1735
+ Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), ord + 1);
1736
+ return;
1737
+ }
1738
+ continue;
1739
+ }
1740
+ if (spins < LOCK_SPIN_ITERATIONS) {
1741
+ spins += 1;
1742
+ continue;
1743
+ }
1744
+ const waiting = seen | WRITER_WAITING;
1745
+ if (waiting === seen || Atomics.compareExchange(words, ord, seen, waiting) === seen) {
1746
+ waitOn(words, ord, waiting);
1747
+ }
1748
+ }
1749
+ }
1750
+ function releaseWrite(words, ord) {
1751
+ const seen = Atomics.load(words, ord);
1752
+ const previous = Atomics.exchange(words, ord, (seen & -4) + VERSION_STEP);
1753
+ if ((previous & WRITER_WAITING) !== 0 && canWait(words)) Atomics.notify(words, ord);
1754
+ }
1755
+ function unlockNodeWrite(locks, ord) {
1756
+ releaseWrite(locks.words(ord), ord);
1757
+ Atomics.store(locks.held, heldIndex(locks, HELD_WRITE), 0);
1758
+ }
1759
+ function lockGraphShared(locks) {
1760
+ const { header } = locks;
1761
+ let spins = 0;
1762
+ for (; ; ) {
1763
+ const seen = Atomics.load(header, GRAPH_LOCK);
1764
+ if (seen >= FREE && Atomics.load(header, GRAPH_WRITERS_WAITING) === 0) {
1765
+ if (Atomics.compareExchange(header, GRAPH_LOCK, seen, seen + 1) === seen) {
1766
+ Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 1);
1767
+ return;
1768
+ }
1769
+ continue;
1770
+ }
1771
+ if (spins < LOCK_SPIN_ITERATIONS) {
1772
+ spins += 1;
1773
+ continue;
1774
+ }
1775
+ waitOn(header, GRAPH_LOCK, seen, GRAPH_LOCK_WAIT_SLICE_MS);
1776
+ }
1777
+ }
1778
+ function unlockGraphShared(locks) {
1779
+ const { header } = locks;
1780
+ Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH), 0);
1781
+ if (Atomics.sub(header, GRAPH_LOCK, 1) === 1 && canWait(header)) Atomics.notify(header, GRAPH_LOCK);
1782
+ }
1783
+ function lockGraphExclusive(locks) {
1784
+ Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 1);
1785
+ Atomics.add(locks.header, GRAPH_WRITERS_WAITING, 1);
1786
+ try {
1787
+ acquireExclusive(locks.header, GRAPH_LOCK);
1788
+ } finally {
1789
+ Atomics.sub(locks.header, GRAPH_WRITERS_WAITING, 1);
1790
+ Atomics.store(locks.held, heldIndex(locks, HELD_GRAPH_WAITING), 0);
1791
+ }
1792
+ }
1793
+ function unlockGraphExclusive(locks) {
1794
+ releaseExclusive(locks.header, GRAPH_LOCK);
1795
+ }
1796
+ function lockEntry(locks) {
1797
+ acquireExclusive(locks.header, GRAPH_ENTRY_LOCK);
1798
+ }
1799
+ function unlockEntry(locks) {
1800
+ releaseExclusive(locks.header, GRAPH_ENTRY_LOCK);
1801
+ }
1802
+ function releaseLocksHeldBy(handles, threadSlot) {
1803
+ const held = handles.heldLocks;
1804
+ const words = new Int32Array(handles.locks);
1805
+ const base = threadSlot * HELD_WORDS_PER_THREAD;
1806
+ if (Atomics.exchange(held, base + HELD_GRAPH_WAITING, 0) === 1) {
1807
+ Atomics.sub(handles.header, GRAPH_WRITERS_WAITING, 1);
1808
+ }
1809
+ const writeOrd = Atomics.exchange(held, base + HELD_WRITE, 0) - 1;
1810
+ const writeVersion = Atomics.load(held, base + HELD_WRITE_VERSION);
1811
+ if (writeOrd >= 0 && writeOrd < words.length) {
1812
+ const seen = Atomics.load(words, writeOrd);
1813
+ if ((seen & ~WRITER_WAITING) === writeVersion) releaseWrite(words, writeOrd);
1814
+ }
1815
+ if (Atomics.exchange(held, base + HELD_GRAPH, 0) !== 1) return;
1816
+ for (; ; ) {
1817
+ const readers = Atomics.load(handles.header, GRAPH_LOCK);
1818
+ if (readers <= FREE) return;
1819
+ if (Atomics.compareExchange(handles.header, GRAPH_LOCK, readers, readers - 1) === readers) {
1820
+ if (canWait(handles.header)) Atomics.notify(handles.header, GRAPH_LOCK);
1821
+ return;
1822
+ }
1823
+ }
1824
+ }
1825
+
1826
+ // src/vector/hnsw/connect-unreached.ts
1827
+ var NOT_REACHED_THROUGH_A_NODE = -1;
1828
+ function reach(walk, node, through) {
1829
+ walk.reached[node] = 1;
1830
+ walk.firstReachedThrough[node] = through;
1831
+ walk.queue[walk.tail++] = node;
1832
+ }
1833
+ function walkFrom(state, walk, start, through) {
1834
+ const adjacency = state.adjacency;
1835
+ if (walk.reached[start] === 1) return;
1836
+ reach(walk, start, through);
1837
+ while (walk.head < walk.tail) {
1838
+ const ord = walk.queue[walk.head++];
1839
+ const base = layerBase(adjacency, ord, 0);
1840
+ if (base === ABSENT) continue;
1841
+ const count = adjacency.level0[base];
1842
+ for (let i = 1; i <= count; i++) {
1843
+ const next = adjacency.level0[base + i];
1844
+ if (next < 0 || next >= walk.reached.length || walk.reached[next] === 1 || !nodeExists(state, next)) continue;
1845
+ reach(walk, next, ord);
1846
+ }
1847
+ }
1848
+ }
1849
+ function hasRoom(state, _walk, host) {
1850
+ const base = layerBase(state.adjacency, host, 0);
1851
+ return base !== ABSENT && state.adjacency.level0[base] < state.Mmax0;
1852
+ }
1853
+ function reachedThroughAnotherNode(walk, host, neighbour) {
1854
+ if (neighbour < 0 || neighbour >= walk.reached.length || walk.reached[neighbour] !== 1) return false;
1855
+ const through = walk.firstReachedThrough[neighbour];
1856
+ return through !== NOT_REACHED_THROUGH_A_NODE && through !== host;
1857
+ }
1858
+ function holdsLinkOutsideTheWalk(state, walk, host) {
1859
+ const adjacency = state.adjacency;
1860
+ const base = layerBase(adjacency, host, 0);
1861
+ if (base === ABSENT) return false;
1862
+ const count = adjacency.level0[base];
1863
+ for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
1864
+ return false;
1865
+ }
1866
+ function listOf(state, ord) {
1867
+ const adjacency = state.adjacency;
1868
+ const base = layerBase(adjacency, ord, 0);
1869
+ if (base === ABSENT) return new Int32Array(0);
1870
+ return adjacency.level0.slice(base + 1, base + 1 + adjacency.level0[base]);
1871
+ }
1872
+ function nearestHost(state, walk, orphan, nearby, test) {
1873
+ const distance = nodeDistanceFunction(state, state.buildMetric);
1874
+ let best = -1;
1875
+ let bestDistance = Number.POSITIVE_INFINITY;
1876
+ const consider = (host) => {
1877
+ if (host === orphan || walk.reached[host] !== 1 || !isLive(state, host) || !test(state, walk, host)) return;
1878
+ const away = distance(orphan, host);
1879
+ if (best === -1 || away < bestDistance) {
1880
+ best = host;
1881
+ bestDistance = away;
1882
+ }
1883
+ };
1884
+ for (const host of nearby.list) consider(host);
1885
+ for (const host of listOf(state, orphan))
1886
+ if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
1887
+ if (best !== -1) return best;
1888
+ for (let host = 0; host < walk.reached.length; host++) consider(host);
1889
+ return best;
1890
+ }
1891
+ function takeOutFarthestLinkOutsideTheWalk(state, walk, host) {
1892
+ const adjacency = state.adjacency;
1893
+ const base = layerBase(adjacency, host, 0);
1894
+ if (base === ABSENT) return;
1895
+ const distance = nodeDistanceFunction(state, state.buildMetric);
1896
+ const count = adjacency.level0[base];
1897
+ let farthest = -1;
1898
+ let farthestDistance = Number.NEGATIVE_INFINITY;
1899
+ for (let i = 1; i <= count; i++) {
1900
+ const neighbour = adjacency.level0[base + i];
1901
+ if (!reachedThroughAnotherNode(walk, host, neighbour)) continue;
1902
+ const away = distance(host, neighbour);
1903
+ if (farthest === -1 || away > farthestDistance) {
1904
+ farthest = i;
1905
+ farthestDistance = away;
1906
+ }
1907
+ }
1908
+ if (farthest === -1) return;
1909
+ const taken = adjacency.level0[base + farthest];
1910
+ for (let i = farthest; i < count; i++) adjacency.level0[base + i] = adjacency.level0[base + i + 1];
1911
+ adjacency.level0[base] = count - 1;
1912
+ noteUnlinked(adjacency, taken);
1913
+ }
1914
+ function appendTo(state, host, orphan) {
1915
+ const adjacency = state.adjacency;
1916
+ const base = layerBase(adjacency, host, 0);
1917
+ if (base === ABSENT) return;
1918
+ const count = adjacency.level0[base];
1919
+ adjacency.level0[base + count + 1] = orphan;
1920
+ adjacency.level0[base] = count + 1;
1921
+ noteLinked(adjacency, orphan);
1922
+ }
1923
+ function connectUnreached(state, nearby) {
1924
+ const slots = adjacencySlots(state.adjacency);
1925
+ const entry = entryPointOf(state);
1926
+ if (entry < 0 || entry >= slots || !nodeExists(state, entry)) return;
1927
+ const walk = {
1928
+ reached: new Uint8Array(slots),
1929
+ firstReachedThrough: new Int32Array(slots),
1930
+ queue: new Int32Array(slots),
1931
+ head: 0,
1932
+ tail: 0
1933
+ };
1934
+ walkFrom(state, walk, entry, NOT_REACHED_THROUGH_A_NODE);
1935
+ const held = { list: [], holds: new Uint8Array(slots) };
1936
+ for (let i = 0; i < nearby.length; i++) {
1937
+ const ord = nearby[i];
1938
+ if (ord < 0 || ord >= slots || held.holds[ord] === 1) continue;
1939
+ held.holds[ord] = 1;
1940
+ held.list.push(ord);
1941
+ }
1942
+ for (let orphan = 0; orphan < slots; orphan++) {
1943
+ if (walk.reached[orphan] === 1 || !isLive(state, orphan)) continue;
1944
+ let host = nearestHost(state, walk, orphan, held, hasRoom);
1945
+ if (host === -1) {
1946
+ host = nearestHost(state, walk, orphan, held, holdsLinkOutsideTheWalk);
1947
+ if (host === -1) continue;
1948
+ takeOutFarthestLinkOutsideTheWalk(state, walk, host);
1949
+ }
1950
+ appendTo(state, host, orphan);
1951
+ walkFrom(state, walk, orphan, host);
1952
+ }
1953
+ }
1954
+ function holdsUnlinkedNode(state) {
1955
+ const slots = adjacencySlots(state.adjacency);
1956
+ const entry = entryPointOf(state);
1957
+ const incoming = state.adjacency.incoming;
1958
+ for (let ord = 0; ord < slots; ord++) {
1959
+ if (incoming[ord] === 0 && ord !== entry && isLive(state, ord)) return true;
1960
+ }
1961
+ return false;
1962
+ }
1963
+ function connectThroughEitherCore(state, nearby) {
1964
+ const field = nativeFieldFor(state);
1965
+ if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
1966
+ noteFallback("connect");
1967
+ connectUnreached(state, nearby);
1968
+ }
1969
+ function connectAfterRemoval(state, nearby) {
1970
+ connectThroughEitherCore(state, nearby);
1971
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
1972
+ }
1973
+ function connectAfterPlacement(state, holdsGraphLock) {
1974
+ if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
1975
+ if (!holdsGraphLock) lockGraphExclusive(state.locks);
1976
+ try {
1977
+ if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
1978
+ connectThroughEitherCore(state, new Int32Array(0));
1979
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
1980
+ } finally {
1981
+ if (!holdsGraphLock) unlockGraphExclusive(state.locks);
1982
+ }
1983
+ }
1984
+ function baseNeighboursOfRemoved(state, removed) {
1985
+ const leaving = new Set(removed);
1986
+ const nearby = [];
1987
+ for (const ord of removed) {
1988
+ for (const neighbour of listOf(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
1989
+ }
1990
+ return Int32Array.from(nearby);
1991
+ }
1992
+ function leaveIncomingUncounted(state) {
1993
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
1994
+ Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 1);
1995
+ }
1996
+ function countIncomingBeforeWriting(state, holdsGraphLock) {
1997
+ if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
1998
+ if (!holdsGraphLock) lockGraphExclusive(state.locks);
1999
+ try {
2000
+ if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
2001
+ const adjacency = state.adjacency;
2002
+ const slots = adjacencySlots(adjacency);
2003
+ if (slots > 0) reachNode(adjacency, slots - 1);
2004
+ adjacency.incoming.fill(0);
2005
+ countEveryLink(adjacency, slots);
2006
+ if (holdsUnlinkedNode(state)) connectThroughEitherCore(state, new Int32Array(0));
2007
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
2008
+ Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
2009
+ } finally {
2010
+ if (!holdsGraphLock) unlockGraphExclusive(state.locks);
2011
+ }
2012
+ }
2013
+
1659
2014
  // src/vector/hnsw/graph-ops.ts
1660
2015
  function readNeighborsLocked(state, ord, layer) {
1661
2016
  for (; ; ) {
@@ -1752,14 +2107,24 @@ function pruneConnections(state, ord, layer, distance) {
1752
2107
  for (let i = 1; i <= count; i++) {
1753
2108
  const connOrd = neighbors[base + i];
1754
2109
  const dist = distance(ord, connOrd);
1755
- if (dist === Number.POSITIVE_INFINITY) continue;
2110
+ if (dist === Number.POSITIVE_INFINITY) {
2111
+ if (layer === 0) noteUnlinked(adjacency, connOrd);
2112
+ continue;
2113
+ }
1756
2114
  candidates.ords[candidates.size] = connOrd;
1757
2115
  candidates.distances[candidates.size] = dist;
1758
2116
  candidates.size += 1;
1759
2117
  }
1760
2118
  const kept = state.workspace.pruneSelection;
1761
2119
  selectNeighborsHeuristic(state, candidates, mc, distance, kept);
1762
- replaceNeighbors(adjacency, ord, layer, kept.ords, kept.size);
2120
+ if (layer === 0) {
2121
+ writeKeepingLastLinks(state, ord, state.workspace.working, kept, everyCandidateWasHeld);
2122
+ return;
2123
+ }
2124
+ writeNeighbors(adjacency, ord, layer, kept.ords, kept.size);
2125
+ }
2126
+ function everyCandidateWasHeld() {
2127
+ return true;
1763
2128
  }
1764
2129
 
1765
2130
  // src/vector/hnsw/mutation.ts
@@ -1873,11 +2238,15 @@ function linkNeighborsBack(state, ord, linkTop) {
1873
2238
  }
1874
2239
  function linkNode(state, ord, level2) {
1875
2240
  const topLayer = topLayerOf(state);
2241
+ const formerEntry = entryPointOf(state);
1876
2242
  const linkTop = selectNeighborsPerLayer(state, ord, level2);
1877
2243
  if (linkTop < 0) return;
1878
2244
  writeOwnLists(state, ord, linkTop);
1879
2245
  linkNeighborsBack(state, ord, linkTop);
1880
- if (level2 > topLayer) raiseEntry(state, ord, level2);
2246
+ if (level2 <= topLayer) return;
2247
+ raiseEntry(state, ord, level2);
2248
+ const demoted = entryPointOf(state) !== formerEntry && isLive(state, formerEntry);
2249
+ if (demoted && incomingCount(state.adjacency, formerEntry) === 0) noteNodeLeftUnlinked(state);
1881
2250
  }
1882
2251
  function writeRecordBeforePlacement(state, ord) {
1883
2252
  const quantizer = state.quantizer;
@@ -1903,8 +2272,12 @@ function insertNode(state, ord, holdsGraphLock = false) {
1903
2272
  if (!state.store.holdsOrdinal(ord)) return false;
1904
2273
  ensureCapacity(state, ord + 1);
1905
2274
  if (nodeExists(state, ord)) return false;
2275
+ countIncomingBeforeWriting(state, holdsGraphLock);
1906
2276
  writeRecordBeforePlacement(state, ord);
1907
- if (placeThroughTheCore(state, ord, holdsGraphLock)) return true;
2277
+ if (placeThroughTheCore(state, ord, holdsGraphLock)) {
2278
+ connectAfterPlacement(state, holdsGraphLock);
2279
+ return true;
2280
+ }
1908
2281
  if (nodeExists(state, ord)) return false;
1909
2282
  noteFallback("place");
1910
2283
  if (!holdsGraphLock) lockGraphShared(state.locks);
@@ -1913,11 +2286,11 @@ function insertNode(state, ord, holdsGraphLock = false) {
1913
2286
  const level2 = randomLevel(state.mL);
1914
2287
  createNode(state.adjacency, ord, level2);
1915
2288
  Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
1916
- if (claimEntryIfEmpty(state, ord, level2)) return true;
1917
- linkNode(state, ord, level2);
2289
+ if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
1918
2290
  } finally {
1919
2291
  if (!holdsGraphLock) unlockGraphShared(state.locks);
1920
2292
  }
2293
+ connectAfterPlacement(state, holdsGraphLock);
1921
2294
  return true;
1922
2295
  }
1923
2296
  function removeNodeEager(state, ord, excludeOrds) {
@@ -1935,7 +2308,8 @@ function removeNodeEager(state, ord, excludeOrds) {
1935
2308
  if (layer > nodeMaxLayer(state, neighborOrd)) continue;
1936
2309
  const mc = maxConns(state, layer);
1937
2310
  if (neighborCount(state.adjacency, neighborOrd, layer) >= mc) continue;
1938
- const candidateOrds = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
2311
+ const heldNow = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
2312
+ const candidateOrds = new Set(heldNow);
1939
2313
  for (const otherOrd of formerNeighbors) {
1940
2314
  if (otherOrd !== neighborOrd && otherOrd !== ord) {
1941
2315
  if (excludeOrds?.has(otherOrd)) continue;
@@ -1946,12 +2320,19 @@ function removeNodeEager(state, ord, excludeOrds) {
1946
2320
  candidates.size = 0;
1947
2321
  for (const candOrd of candidateOrds) {
1948
2322
  const dist = distance(neighborOrd, candOrd);
1949
- if (dist === Number.POSITIVE_INFINITY) continue;
2323
+ if (dist === Number.POSITIVE_INFINITY) {
2324
+ if (layer === 0 && heldNow.has(candOrd)) noteUnlinked(state.adjacency, candOrd);
2325
+ continue;
2326
+ }
1950
2327
  appendToList(candidates, candOrd, dist);
1951
2328
  }
1952
2329
  const selected = state.workspace.repairSelection;
1953
2330
  selectNeighborsHeuristic(state, candidates, mc, distance, selected);
1954
- replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
2331
+ if (layer === 0) {
2332
+ writeKeepingLastLinks(state, neighborOrd, state.workspace.working, selected, (held) => heldNow.has(held));
2333
+ } else {
2334
+ replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
2335
+ }
1955
2336
  for (let i = 0; i < selected.size; i++) {
1956
2337
  const newConnOrd = selected.ords[i];
1957
2338
  if (layer <= nodeMaxLayer(state, newConnOrd)) {
@@ -1961,6 +2342,8 @@ function removeNodeEager(state, ord, excludeOrds) {
1961
2342
  }
1962
2343
  }
1963
2344
  }
2345
+ const formerBaseNeighbors = collectNeighbors(state.adjacency, ord, 0);
2346
+ for (const neighborOrd of formerBaseNeighbors) noteUnlinked(state.adjacency, neighborOrd);
1964
2347
  deleteNode(state.adjacency, ord);
1965
2348
  Atomics.sub(state.header, GRAPH_NODE_COUNT, 1);
1966
2349
  clearTombstone(state, ord);
@@ -1984,9 +2367,7 @@ function markTombstone(state, ord) {
1984
2367
  }
1985
2368
  function compactTombstones(state) {
1986
2369
  if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
1987
- const field = nativeFieldFor(state);
1988
- if (field !== null && nativeCompact(state, field, state.buildMetric)) return;
1989
- noteFallback("compact");
2370
+ countIncomingBeforeWriting(state, true);
1990
2371
  const tombstonedOrds = [];
1991
2372
  const slots = adjacencySlots(state.adjacency);
1992
2373
  for (let ord = 0; ord < slots; ord++) {
@@ -1994,10 +2375,16 @@ function compactTombstones(state) {
1994
2375
  tombstonedOrds.push(ord);
1995
2376
  }
1996
2377
  }
1997
- const excl = new Set(tombstonedOrds);
1998
- for (const ord of tombstonedOrds) {
1999
- removeNodeEager(state, ord, excl);
2378
+ const nearby = baseNeighboursOfRemoved(state, tombstonedOrds);
2379
+ const field = nativeFieldFor(state);
2380
+ if (field === null || !nativeCompact(state, field, state.buildMetric)) {
2381
+ noteFallback("compact");
2382
+ const excl = new Set(tombstonedOrds);
2383
+ for (const ord of tombstonedOrds) {
2384
+ removeNodeEager(state, ord, excl);
2385
+ }
2000
2386
  }
2387
+ connectAfterRemoval(state, nearby);
2001
2388
  }
2002
2389
  function resetGraph(state) {
2003
2390
  resetAdjacency(state.adjacency);
@@ -2007,6 +2394,7 @@ function resetGraph(state) {
2007
2394
  Atomics.store(state.header, GRAPH_NODE_COUNT, 0);
2008
2395
  Atomics.store(state.header, GRAPH_ENTRY_POINT, -1);
2009
2396
  Atomics.store(state.header, GRAPH_TOP_LAYER, -1);
2397
+ Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
2010
2398
  }
2011
2399
  function rebuild(state) {
2012
2400
  const nodeCount = Atomics.load(state.header, GRAPH_NODE_COUNT);
@@ -2485,7 +2873,7 @@ function openSharedQuantizer(store, layout, metric) {
2485
2873
  let centroidGeneration = -1;
2486
2874
  let currentQuery = null;
2487
2875
  const singleOrdinal = new Int32Array(1);
2488
- function reach(ordinal) {
2876
+ function reach2(ordinal) {
2489
2877
  if (ordinal < codePresent.length) return true;
2490
2878
  if (ordinal >= store.handles.codePresent.byteLength) return false;
2491
2879
  codePresent = fixedView(store.handles.codePresent, Uint8Array);
@@ -2506,8 +2894,8 @@ function openSharedQuantizer(store, layout, metric) {
2506
2894
  if (generation !== centroidGeneration) recomputeCentroidDot(generation);
2507
2895
  return centroidDot;
2508
2896
  }
2509
- function holds(ordinal) {
2510
- return ordinal >= 0 && reach(ordinal) && codePresent[ordinal] === 1;
2897
+ function holds2(ordinal) {
2898
+ return ordinal >= 0 && reach2(ordinal) && codePresent[ordinal] === 1;
2511
2899
  }
2512
2900
  function recordOffset(ordinal) {
2513
2901
  return slotByteOffset(layout, ordinal % capacity);
@@ -2528,7 +2916,7 @@ function openSharedQuantizer(store, layout, metric) {
2528
2916
  }
2529
2917
  }
2530
2918
  function writeRecordOf(ordinal) {
2531
- if (!reach(ordinal) || !store.holdsOrdinal(ordinal)) return;
2919
+ if (!reach2(ordinal) || !store.holdsOrdinal(ordinal)) return;
2532
2920
  const code = osqQuantize(store.vectorAt(ordinal), store.handles.centroid, bits, metric, scratch);
2533
2921
  const block = store.codeBlockOf(ordinal);
2534
2922
  const offset = recordOffset(ordinal);
@@ -2599,7 +2987,7 @@ function openSharedQuantizer(store, layout, metric) {
2599
2987
  return currentQuery;
2600
2988
  }
2601
2989
  function distanceBetween(ordA, ordB) {
2602
- if (!holds(ordA) || !holds(ordB)) return Number.POSITIVE_INFINITY;
2990
+ if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
2603
2991
  const blockA = store.codeBlockOf(ordA);
2604
2992
  const blockB = store.codeBlockOf(ordB);
2605
2993
  const offsetA = recordOffset(ordA);
@@ -2627,7 +3015,7 @@ function openSharedQuantizer(store, layout, metric) {
2627
3015
  return isCalibrated() ? store.handles.centroid : null;
2628
3016
  },
2629
3017
  isCalibrated,
2630
- holdsOrdinal: holds,
3018
+ holdsOrdinal: holds2,
2631
3019
  writeCodes(ordinal) {
2632
3020
  singleOrdinal[0] = ordinal;
2633
3021
  writeCodesOf(singleOrdinal);
@@ -2642,7 +3030,7 @@ function openSharedQuantizer(store, layout, metric) {
2642
3030
  return true;
2643
3031
  },
2644
3032
  restoreRecord(ordinal, record) {
2645
- if (!reach(ordinal) || record.length !== layout.slotStride) return;
3033
+ if (!reach2(ordinal) || record.length !== layout.slotStride) return;
2646
3034
  const block = store.codeBlockOf(ordinal);
2647
3035
  const offset = recordOffset(ordinal);
2648
3036
  block.bytes(offset + layout.slotStride).set(record, offset);
@@ -2654,7 +3042,7 @@ function openSharedQuantizer(store, layout, metric) {
2654
3042
  return block.bytes(offset + layout.slotStride).subarray(offset, offset + layout.slotStride);
2655
3043
  },
2656
3044
  clearCodes(ordinal) {
2657
- if (ordinal < 0 || !reach(ordinal)) return;
3045
+ if (ordinal < 0 || !reach2(ordinal)) return;
2658
3046
  if (Atomics.exchange(codePresent, ordinal, 0) === 1) Atomics.sub(store.handles.header, STORE_CODE_COUNT, 1);
2659
3047
  },
2660
3048
  writeCentroid,
@@ -2670,11 +3058,11 @@ function openSharedQuantizer(store, layout, metric) {
2670
3058
  },
2671
3059
  prepareQuery,
2672
3060
  distanceFromPreparedByOrdinal(prepared, ordinal) {
2673
- if (!holds(ordinal)) return Number.POSITIVE_INFINITY;
3061
+ if (!holds2(ordinal)) return Number.POSITIVE_INFINITY;
2674
3062
  return estimateDistance(ordinal, prepared);
2675
3063
  },
2676
3064
  preparedDistance(prepared) {
2677
- const generic = (ordinal) => holds(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
3065
+ const generic = (ordinal) => holds2(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
2678
3066
  if (store.handles.codeBlocks.length === 0) return generic;
2679
3067
  const block = store.codeBlockOf(0);
2680
3068
  const simd = block.simd;
@@ -2910,7 +3298,7 @@ function openSharedVectorStore(initial, threadSlot) {
2910
3298
  const index = Math.floor(ordinal / capacity);
2911
3299
  return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
2912
3300
  }
2913
- function holds(ordinal) {
3301
+ function holds2(ordinal) {
2914
3302
  return holdState(ordinal) !== NOT_HELD;
2915
3303
  }
2916
3304
  function readFromFile(file, offset, target) {
@@ -2990,7 +3378,7 @@ function openSharedVectorStore(initial, threadSlot) {
2990
3378
  reader?.retainOnly(new Set(next.vectorFiles.filter((path) => path.length > 0)));
2991
3379
  },
2992
3380
  rebind,
2993
- holdsOrdinal: holds,
3381
+ holdsOrdinal: holds2,
2994
3382
  isCold,
2995
3383
  readColdInto,
2996
3384
  readFromFile,
@@ -3031,7 +3419,7 @@ function openSharedVectorStore(initial, threadSlot) {
3031
3419
  return magnitudes[ordinal];
3032
3420
  },
3033
3421
  entryForOrdinal(ordinal) {
3034
- if (!holds(ordinal)) return void 0;
3422
+ if (!holds2(ordinal)) return void 0;
3035
3423
  return { vector: view.vectorAt(ordinal), magnitude: magnitudes[ordinal] };
3036
3424
  },
3037
3425
  docIdAt(ordinal) {
@@ -3053,7 +3441,7 @@ function openSharedVectorStore(initial, threadSlot) {
3053
3441
  return partition < 0 ? void 0 : partition;
3054
3442
  },
3055
3443
  distanceByOrdinal(ordA, ordB, metric) {
3056
- if (!holds(ordA) || !holds(ordB)) return Number.POSITIVE_INFINITY;
3444
+ if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
3057
3445
  if (!isCold(ordB)) {
3058
3446
  const indexA = ordA / capacity | 0;
3059
3447
  const indexB = ordB / capacity | 0;
@@ -3175,6 +3563,9 @@ function openSharedVectorField(initial, threadSlot) {
3175
3563
  get readsEveryVector() {
3176
3564
  return store.holdsEveryBlock;
3177
3565
  },
3566
+ get graphRetired() {
3567
+ return graph !== null && Atomics.load(graph.header, GRAPH_RETIRED) === 1;
3568
+ },
3178
3569
  docIdOf,
3179
3570
  ordinalOf,
3180
3571
  vectorOf(docId) {
@@ -3463,6 +3854,9 @@ function ensureOrdinal(store, ordinal, cold) {
3463
3854
  const { handles } = store;
3464
3855
  growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
3465
3856
  growBufferTo(handles.present, ordinal + 1);
3857
+ growBufferTo(handles.unplaced, ordinal + 1);
3858
+ growBufferTo(handles.unplacedChunks, unplacedChunkCount(ordinal) * 4);
3859
+ growBufferTo(handles.removed, ordinal + 1);
3466
3860
  growBufferTo(handles.partitions, (ordinal + 1) * 4);
3467
3861
  growBufferTo(handles.codePresent, ordinal + 1);
3468
3862
  const diskFileBytes = handles.diskFile.byteLength;
@@ -4023,7 +4417,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
4023
4417
  }
4024
4418
  cursor += count + 1;
4025
4419
  if (ordinal !== NO_ORDINAL && layer <= keptTop) {
4026
- replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
4420
+ writeNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
4027
4421
  }
4028
4422
  }
4029
4423
  }
@@ -4035,6 +4429,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
4035
4429
  const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
4036
4430
  Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
4037
4431
  Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
4432
+ leaveIncomingUncounted(state);
4038
4433
  }
4039
4434
 
4040
4435
  // src/vector/hnsw/persistence.ts
@@ -4073,7 +4468,7 @@ function deserializeGraph(state, store, data) {
4073
4468
  for (const node of nodes) {
4074
4469
  createNode(state.adjacency, node.ord, node.maxLayer);
4075
4470
  for (const [layer, neighbors] of node.layers) {
4076
- replaceNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
4471
+ writeNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
4077
4472
  }
4078
4473
  Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
4079
4474
  }
@@ -4099,6 +4494,7 @@ function deserializeGraph(state, store, data) {
4099
4494
  Atomics.store(state.header, GRAPH_ENTRY_POINT, bestOrd);
4100
4495
  Atomics.store(state.header, GRAPH_TOP_LAYER, bestLayer);
4101
4496
  }
4497
+ leaveIncomingUncounted(state);
4102
4498
  }
4103
4499
 
4104
4500
  // src/vector/hnsw/scratch-bytes.ts
@@ -4159,6 +4555,7 @@ function restoreSnapshot(state, snapshot) {
4159
4555
  Atomics.store(state.header, GRAPH_NODE_COUNT, snapshot.nodeCount);
4160
4556
  Atomics.store(state.header, GRAPH_ENTRY_POINT, snapshot.entryPointOrd);
4161
4557
  Atomics.store(state.header, GRAPH_TOP_LAYER, snapshot.topLayer);
4558
+ leaveIncomingUncounted(state);
4162
4559
  }
4163
4560
 
4164
4561
  // src/vector/hnsw/index.ts
@@ -4270,6 +4667,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
4270
4667
  },
4271
4668
  search: (query, k, searchMetric, minSimilarity, options) => search(state, docIdOf, query, k, searchMetric, minSimilarity, options),
4272
4669
  clear: () => exclusively(() => resetGraph(state)),
4670
+ retire: () => Atomics.store(shared.header, GRAPH_RETIRED, 1),
4273
4671
  entries: entriesIterator,
4274
4672
  compactionNeeded,
4275
4673
  compactTombstones: () => exclusively(() => compactTombstones(state)),
@@ -4283,4 +4681,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
4283
4681
  };
4284
4682
  }
4285
4683
 
4286
- export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, graphWithoutNodes, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
4684
+ export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, STORE_UNPLACED_COUNT, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, euclideanDistance, graphWithoutNodes, isSimdAvailable, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };