@delali/narsil 0.3.1-next.84 → 0.3.1-next.85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4875,10 +4875,10 @@ function firstAtOrAfter2(sorted, key) {
4875
4875
  }
4876
4876
  return lo;
4877
4877
  }
4878
- function settledColumn(column, holds) {
4878
+ function settledColumn(column, holds2) {
4879
4879
  const { keys, values } = column;
4880
4880
  const order = [];
4881
- for (let at = 0; at < keys.length; at++) if (holds(values[at])) order.push(at);
4881
+ for (let at = 0; at < keys.length; at++) if (holds2(values[at])) order.push(at);
4882
4882
  order.sort((left, right) => compareUnits(keys[left], keys[right]) || compareUnits(values[left], values[right]));
4883
4883
  const settled2 = { keys: [], values: [] };
4884
4884
  for (const at of order) {
@@ -4890,8 +4890,8 @@ function settledColumn(column, holds) {
4890
4890
  return settled2;
4891
4891
  }
4892
4892
  var ValueAffixes = class {
4893
- constructor(holds, values) {
4894
- this.holds = holds;
4893
+ constructor(holds2, values) {
4894
+ this.holds = holds2;
4895
4895
  for (const value of values) this.add(value);
4896
4896
  }
4897
4897
  forward = { keys: [], values: [] };
@@ -6792,7 +6792,7 @@ function withinBounds(value, bounds) {
6792
6792
  }
6793
6793
  return true;
6794
6794
  }
6795
- function selectTextRange(column, requested, capacity, isLive2) {
6795
+ function selectTextRange(column, requested, capacity, isLive3) {
6796
6796
  const bounds = {
6797
6797
  ...requested,
6798
6798
  lower: requested.lower === null ? null : truncateSortString(requested.lower),
@@ -6813,7 +6813,7 @@ function selectTextRange(column, requested, capacity, isLive2) {
6813
6813
  }
6814
6814
  for (let position = start; position < end; position++) {
6815
6815
  const internalId = ordered[position];
6816
- if (!column.isDirty(internalId) && isLive2(internalId)) bitsetSet(result, internalId);
6816
+ if (!column.isDirty(internalId) && isLive3(internalId)) bitsetSet(result, internalId);
6817
6817
  }
6818
6818
  for (const internalId of column.dirtyStream().present) {
6819
6819
  if (withinBounds(column.valueOf(internalId), bounds)) bitsetSet(result, internalId);
@@ -11753,6 +11753,7 @@ var VECTOR_STORE_INITIAL_CAPACITY = 16;
11753
11753
  var VECTOR_BLOCK_INITIAL_SLOTS = 256;
11754
11754
  var DISK_STORAGE_BLOCK_BYTES = 67108864;
11755
11755
  var MAX_VECTOR_ORDINALS = 16777216;
11756
+ var UNPLACED_FLAG_CHUNK_ORDINALS = 1024;
11756
11757
  var VECTOR_SCRATCH_SLOTS = 64;
11757
11758
  var SCRATCH_SLOTS_PER_THREAD_POOL = 31;
11758
11759
  var MAX_DOC_ID_TABLE_BYTES = 1073741824;
@@ -12298,6 +12299,8 @@ var STORE_DOC_ID_BYTES = 5;
12298
12299
  var STORE_CODE_BLOCK_COUNT = 6;
12299
12300
  var STORE_CALIBRATION_GENERATION = 7;
12300
12301
  var STORE_RELEASED_VECTORS_TO_DISK = 8;
12302
+ var STORE_UNPLACED_COUNT = 9;
12303
+ var STORE_REMOVED_COUNT = 10;
12301
12304
  var STORE_HEADER_WORDS = 16;
12302
12305
  var IN_MEMORY = -1;
12303
12306
  function fixedInt32(words) {
@@ -12306,6 +12309,9 @@ function fixedInt32(words) {
12306
12309
  function fixedFloat32(length) {
12307
12310
  return new Float32Array(createFixedBuffer(length * 4), 0, length);
12308
12311
  }
12312
+ function unplacedChunkCount(ordinals) {
12313
+ return Math.floor(ordinals / UNPLACED_FLAG_CHUNK_ORDINALS) + 1;
12314
+ }
12309
12315
  function perOrdinal(bytesPerOrdinal) {
12310
12316
  return createGrowableBuffer(
12311
12317
  VECTOR_STORE_INITIAL_CAPACITY * bytesPerOrdinal,
@@ -12330,6 +12336,9 @@ function createSharedVectorStoreHandles(dimension, codeBits, blockBytes2 = BLOCK
12330
12336
  vectorFiles: [],
12331
12337
  magnitudes: perOrdinal(8),
12332
12338
  present: perOrdinal(1),
12339
+ unplaced: perOrdinal(1),
12340
+ unplacedChunks: createGrowableBuffer(4, unplacedChunkCount(MAX_VECTOR_ORDINALS) * 4),
12341
+ removed: perOrdinal(1),
12333
12342
  partitions: perOrdinal(4),
12334
12343
  docIdBytes: createGrowableBuffer(VECTOR_STORE_INITIAL_CAPACITY * 16, MAX_DOC_ID_TABLE_BYTES),
12335
12344
  docIdOffsets: perOrdinal(4),
@@ -12347,6 +12356,9 @@ function sharedVectorStoreBytes(handles) {
12347
12356
  for (const block of handles.codeBlocks) bytes += blockBuffer(block).byteLength;
12348
12357
  bytes += handles.magnitudes.byteLength;
12349
12358
  bytes += handles.present.byteLength;
12359
+ bytes += handles.unplaced.byteLength;
12360
+ bytes += handles.unplacedChunks.byteLength;
12361
+ bytes += handles.removed.byteLength;
12350
12362
  bytes += handles.partitions.byteLength;
12351
12363
  bytes += handles.docIdBytes.byteLength;
12352
12364
  bytes += handles.docIdOffsets.byteLength;
@@ -12600,7 +12612,7 @@ function openSharedQuantizer(store, layout, metric) {
12600
12612
  let centroidGeneration = -1;
12601
12613
  let currentQuery = null;
12602
12614
  const singleOrdinal = new Int32Array(1);
12603
- function reach(ordinal) {
12615
+ function reach2(ordinal) {
12604
12616
  if (ordinal < codePresent.length) return true;
12605
12617
  if (ordinal >= store.handles.codePresent.byteLength) return false;
12606
12618
  codePresent = fixedView(store.handles.codePresent, Uint8Array);
@@ -12621,8 +12633,8 @@ function openSharedQuantizer(store, layout, metric) {
12621
12633
  if (generation !== centroidGeneration) recomputeCentroidDot(generation);
12622
12634
  return centroidDot;
12623
12635
  }
12624
- function holds(ordinal) {
12625
- return ordinal >= 0 && reach(ordinal) && codePresent[ordinal] === 1;
12636
+ function holds2(ordinal) {
12637
+ return ordinal >= 0 && reach2(ordinal) && codePresent[ordinal] === 1;
12626
12638
  }
12627
12639
  function recordOffset(ordinal) {
12628
12640
  return slotByteOffset(layout, ordinal % capacity);
@@ -12643,7 +12655,7 @@ function openSharedQuantizer(store, layout, metric) {
12643
12655
  }
12644
12656
  }
12645
12657
  function writeRecordOf(ordinal) {
12646
- if (!reach(ordinal) || !store.holdsOrdinal(ordinal)) return;
12658
+ if (!reach2(ordinal) || !store.holdsOrdinal(ordinal)) return;
12647
12659
  const code = osqQuantize(store.vectorAt(ordinal), store.handles.centroid, bits, metric, scratch);
12648
12660
  const block = store.codeBlockOf(ordinal);
12649
12661
  const offset = recordOffset(ordinal);
@@ -12713,7 +12725,7 @@ function openSharedQuantizer(store, layout, metric) {
12713
12725
  return currentQuery;
12714
12726
  }
12715
12727
  function distanceBetween(ordA, ordB) {
12716
- if (!holds(ordA) || !holds(ordB)) return Number.POSITIVE_INFINITY;
12728
+ if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
12717
12729
  const blockA = store.codeBlockOf(ordA);
12718
12730
  const blockB = store.codeBlockOf(ordB);
12719
12731
  const offsetA = recordOffset(ordA);
@@ -12741,7 +12753,7 @@ function openSharedQuantizer(store, layout, metric) {
12741
12753
  return isCalibrated() ? store.handles.centroid : null;
12742
12754
  },
12743
12755
  isCalibrated,
12744
- holdsOrdinal: holds,
12756
+ holdsOrdinal: holds2,
12745
12757
  writeCodes(ordinal) {
12746
12758
  singleOrdinal[0] = ordinal;
12747
12759
  writeCodesOf(singleOrdinal);
@@ -12755,7 +12767,7 @@ function openSharedQuantizer(store, layout, metric) {
12755
12767
  return true;
12756
12768
  },
12757
12769
  restoreRecord(ordinal, record) {
12758
- if (!reach(ordinal) || record.length !== layout.slotStride) return;
12770
+ if (!reach2(ordinal) || record.length !== layout.slotStride) return;
12759
12771
  const block = store.codeBlockOf(ordinal);
12760
12772
  const offset = recordOffset(ordinal);
12761
12773
  block.bytes(offset + layout.slotStride).set(record, offset);
@@ -12767,7 +12779,7 @@ function openSharedQuantizer(store, layout, metric) {
12767
12779
  return block.bytes(offset + layout.slotStride).subarray(offset, offset + layout.slotStride);
12768
12780
  },
12769
12781
  clearCodes(ordinal) {
12770
- if (ordinal < 0 || !reach(ordinal)) return;
12782
+ if (ordinal < 0 || !reach2(ordinal)) return;
12771
12783
  if (Atomics.exchange(codePresent, ordinal, 0) === 1) Atomics.sub(store.handles.header, STORE_CODE_COUNT, 1);
12772
12784
  },
12773
12785
  writeCentroid,
@@ -12783,11 +12795,11 @@ function openSharedQuantizer(store, layout, metric) {
12783
12795
  },
12784
12796
  prepareQuery,
12785
12797
  distanceFromPreparedByOrdinal(prepared, ordinal) {
12786
- if (!holds(ordinal)) return Number.POSITIVE_INFINITY;
12798
+ if (!holds2(ordinal)) return Number.POSITIVE_INFINITY;
12787
12799
  return estimateDistance(ordinal, prepared);
12788
12800
  },
12789
12801
  preparedDistance(prepared) {
12790
- const generic = (ordinal) => holds(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
12802
+ const generic = (ordinal) => holds2(ordinal) ? estimateDistance(ordinal, prepared) : Number.POSITIVE_INFINITY;
12791
12803
  if (store.handles.codeBlocks.length === 0) return generic;
12792
12804
  const block = store.codeBlockOf(0);
12793
12805
  const simd = block.simd;
@@ -13261,7 +13273,7 @@ function openSharedVectorStore(initial, threadSlot) {
13261
13273
  const index = Math.floor(ordinal / capacity);
13262
13274
  return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
13263
13275
  }
13264
- function holds(ordinal) {
13276
+ function holds2(ordinal) {
13265
13277
  return holdState(ordinal) !== NOT_HELD;
13266
13278
  }
13267
13279
  function readFromFile(file, offset, target) {
@@ -13341,7 +13353,7 @@ function openSharedVectorStore(initial, threadSlot) {
13341
13353
  reader?.retainOnly(new Set(next.vectorFiles.filter((path) => path.length > 0)));
13342
13354
  },
13343
13355
  rebind,
13344
- holdsOrdinal: holds,
13356
+ holdsOrdinal: holds2,
13345
13357
  isCold,
13346
13358
  readColdInto,
13347
13359
  readFromFile,
@@ -13382,7 +13394,7 @@ function openSharedVectorStore(initial, threadSlot) {
13382
13394
  return magnitudes[ordinal];
13383
13395
  },
13384
13396
  entryForOrdinal(ordinal) {
13385
- if (!holds(ordinal)) return void 0;
13397
+ if (!holds2(ordinal)) return void 0;
13386
13398
  return { vector: view.vectorAt(ordinal), magnitude: magnitudes[ordinal] };
13387
13399
  },
13388
13400
  docIdAt(ordinal) {
@@ -13404,7 +13416,7 @@ function openSharedVectorStore(initial, threadSlot) {
13404
13416
  return partition < 0 ? void 0 : partition;
13405
13417
  },
13406
13418
  distanceByOrdinal(ordA, ordB, metric) {
13407
- if (!holds(ordA) || !holds(ordB)) return Number.POSITIVE_INFINITY;
13419
+ if (!holds2(ordA) || !holds2(ordB)) return Number.POSITIVE_INFINITY;
13408
13420
  if (!isCold(ordB)) {
13409
13421
  const indexA = ordA / capacity | 0;
13410
13422
  const indexB = ordB / capacity | 0;
@@ -13541,6 +13553,9 @@ function ensureOrdinal(store, ordinal, cold) {
13541
13553
  const { handles } = store;
13542
13554
  growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
13543
13555
  growBufferTo(handles.present, ordinal + 1);
13556
+ growBufferTo(handles.unplaced, ordinal + 1);
13557
+ growBufferTo(handles.unplacedChunks, unplacedChunkCount(ordinal) * 4);
13558
+ growBufferTo(handles.removed, ordinal + 1);
13544
13559
  growBufferTo(handles.partitions, (ordinal + 1) * 4);
13545
13560
  growBufferTo(handles.codePresent, ordinal + 1);
13546
13561
  const diskFileBytes = handles.diskFile.byteLength;
@@ -13917,8 +13932,11 @@ var GRAPH_M = 2;
13917
13932
  var GRAPH_MMAX0 = 3;
13918
13933
  var GRAPH_EF_CONSTRUCTION = 4;
13919
13934
  var GRAPH_METRIC = 5;
13935
+ var GRAPH_RETIRED = 6;
13920
13936
  var GRAPH_NODE_COUNT = 32;
13921
13937
  var GRAPH_TOMBSTONE_COUNT = 33;
13938
+ var GRAPH_UNLINKED_NODES = 34;
13939
+ var GRAPH_INCOMING_UNCOUNTED = 35;
13922
13940
  var GRAPH_UPPER_USED = 64;
13923
13941
  var GRAPH_SLOTS = 65;
13924
13942
  var GRAPH_UPPER_CAPACITY = 66;
@@ -13956,6 +13974,7 @@ function createSharedGraphHandles(shape) {
13956
13974
  upper: perOrdinal2(upperStride * 4, INITIAL_UPPER_SLOTS),
13957
13975
  locks: perOrdinal2(4, VECTOR_STORE_INITIAL_CAPACITY),
13958
13976
  tombstones: perOrdinal2(1, VECTOR_STORE_INITIAL_CAPACITY),
13977
+ incoming: perOrdinal2(4, VECTOR_STORE_INITIAL_CAPACITY),
13959
13978
  heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
13960
13979
  };
13961
13980
  publishGraphCapacity(handles);
@@ -13975,7 +13994,8 @@ function publishGraphCapacity(handles) {
13975
13994
  Math.floor(handles.level0.byteLength / (level0Stride * 4)),
13976
13995
  Math.floor(handles.upperBase.byteLength / 4),
13977
13996
  Math.floor(handles.locks.byteLength / 4),
13978
- handles.tombstones.byteLength
13997
+ handles.tombstones.byteLength,
13998
+ Math.floor(handles.incoming.byteLength / 4)
13979
13999
  );
13980
14000
  raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
13981
14001
  raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
@@ -14002,11 +14022,13 @@ function openAdjacency(handles, m, mMax0) {
14002
14022
  nodeLevels: fixedView(handles.nodeLevels, Uint8Array),
14003
14023
  level0: fixedView(handles.level0, Int32Array),
14004
14024
  upperBase: fixedView(handles.upperBase, Int32Array),
14005
- upper: fixedView(handles.upper, Int32Array)
14025
+ upper: fixedView(handles.upper, Int32Array),
14026
+ incoming: fixedView(handles.incoming, Int32Array)
14006
14027
  };
14007
14028
  }
14008
14029
  function rebindNodes(adj) {
14009
14030
  adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
14031
+ adj.incoming = fixedView(adj.handles.incoming, Int32Array);
14010
14032
  adj.level0 = fixedView(adj.handles.level0, Int32Array);
14011
14033
  adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
14012
14034
  }
@@ -14043,6 +14065,7 @@ function ensureAdjacencyCapacity(adj, needed) {
14043
14065
  growBufferToExactly(handles.upperBase, capacity * 4);
14044
14066
  growBufferToExactly(handles.locks, capacity * 4);
14045
14067
  growBufferToExactly(handles.tombstones, capacity);
14068
+ growBufferToExactly(handles.incoming, capacity * 4);
14046
14069
  growBufferToExactly(handles.nodeLevels, capacity);
14047
14070
  publishGraphCapacity(handles);
14048
14071
  rebindNodes(adj);
@@ -14089,10 +14112,12 @@ function deleteNode(adj, ord) {
14089
14112
  Atomics.store(adj.nodeLevels, ord, 0);
14090
14113
  adj.upperBase[ord] = 0;
14091
14114
  adj.level0[ord * adj.level0Stride] = 0;
14115
+ Atomics.store(adj.incoming, ord, 0);
14092
14116
  }
14093
14117
  function resetAdjacency(adj) {
14094
14118
  rebindNodes(adj);
14095
14119
  rebindUpper(adj);
14120
+ adj.incoming.fill(0);
14096
14121
  adj.nodeLevels.fill(0);
14097
14122
  adj.level0.fill(0);
14098
14123
  adj.upperBase.fill(0);
@@ -14109,7 +14134,51 @@ function hasNode(adj, ord) {
14109
14134
  }
14110
14135
  function graphBytes(adj) {
14111
14136
  const handles = adj.handles;
14112
- return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.heldLocks.byteLength;
14137
+ return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
14138
+ }
14139
+
14140
+ // src/vector/hnsw/adjacency/incoming.ts
14141
+ function incomingCount(adj, ord) {
14142
+ if (ord < 0 || !reachNode(adj, ord)) return 0;
14143
+ return Atomics.load(adj.incoming, ord);
14144
+ }
14145
+ function noteLinked(adj, ord) {
14146
+ if (ord < 0 || !reachNode(adj, ord)) return;
14147
+ Atomics.add(adj.incoming, ord, 1);
14148
+ }
14149
+ function noteUnlinked(adj, ord) {
14150
+ if (ord < 0 || !reachNode(adj, ord)) return 0;
14151
+ for (; ; ) {
14152
+ const seen = Atomics.load(adj.incoming, ord);
14153
+ if (seen <= 0) return 0;
14154
+ if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return seen - 1;
14155
+ }
14156
+ }
14157
+ function releaseUnlessLast(adj, ord) {
14158
+ if (ord < 0 || !reachNode(adj, ord)) return true;
14159
+ for (; ; ) {
14160
+ const seen = Atomics.load(adj.incoming, ord);
14161
+ if (seen <= 1) return false;
14162
+ if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) === seen) return true;
14163
+ }
14164
+ }
14165
+ function holdsNodeAt(adj, ord) {
14166
+ return ord >= 0 && ord < adj.nodeLevels.length && adj.nodeLevels[ord] !== 0;
14167
+ }
14168
+ function countLinksOutOf(adj, ord, step) {
14169
+ if (!holdsNodeAt(adj, ord)) return;
14170
+ const incoming = adj.incoming;
14171
+ const level0 = adj.level0;
14172
+ const base = ord * adj.level0Stride;
14173
+ const count = level0[base];
14174
+ for (let i = 1; i <= count; i++) {
14175
+ const neighbour = level0[base + i];
14176
+ if (neighbour === ord || !holdsNodeAt(adj, neighbour)) continue;
14177
+ if (incoming[neighbour] + step >= 0) incoming[neighbour] += step;
14178
+ }
14179
+ }
14180
+ function countEveryLink(adj, slots) {
14181
+ for (let ord = 0; ord < slots; ord++) countLinksOutOf(adj, ord, 1);
14113
14182
  }
14114
14183
 
14115
14184
  // src/vector/hnsw/adjacency/neighbors.ts
@@ -14159,6 +14228,7 @@ function addNeighbor(adj, ord, layer, neighborOrd) {
14159
14228
  if (count + 1 > stride - 1) return;
14160
14229
  array[base + count + 1] = neighborOrd;
14161
14230
  array[base] = count + 1;
14231
+ if (layer === 0) noteLinked(adj, neighborOrd);
14162
14232
  }
14163
14233
  function removeNeighbor(adj, ord, layer, neighborOrd) {
14164
14234
  const base = layerBase(adj, ord, layer);
@@ -14171,10 +14241,11 @@ function removeNeighbor(adj, ord, layer, neighborOrd) {
14171
14241
  array[base + j] = array[base + j + 1];
14172
14242
  }
14173
14243
  array[base] = count - 1;
14244
+ if (layer === 0) noteUnlinked(adj, neighborOrd);
14174
14245
  return;
14175
14246
  }
14176
14247
  }
14177
- function replaceNeighbors(adj, ord, layer, neighbors, count) {
14248
+ function writeNeighbors(adj, ord, layer, neighbors, count) {
14178
14249
  const base = layerBase(adj, ord, layer);
14179
14250
  if (base === ABSENT) return;
14180
14251
  const array = layerArray(adj, layer);
@@ -14185,6 +14256,24 @@ function replaceNeighbors(adj, ord, layer, neighbors, count) {
14185
14256
  }
14186
14257
  array[base] = limit;
14187
14258
  }
14259
+ function holds(neighbors, count, ord) {
14260
+ for (let i = 0; i < count; i++) if (neighbors[i] === ord) return true;
14261
+ return false;
14262
+ }
14263
+ function replaceNeighbors(adj, ord, layer, neighbors, count) {
14264
+ const base = layerBase(adj, ord, layer);
14265
+ if (base === ABSENT) return;
14266
+ if (layer > 0) {
14267
+ writeNeighbors(adj, ord, layer, neighbors, count);
14268
+ return;
14269
+ }
14270
+ const array = adj.level0;
14271
+ const held = array.slice(base + 1, base + 1 + array[base]);
14272
+ const limit = Math.min(count, adj.level0Stride - 1);
14273
+ for (const former of held) if (!holds(neighbors, limit, former)) noteUnlinked(adj, former);
14274
+ for (let i = 0; i < limit; i++) if (!holds(held, held.length, neighbors[i])) noteLinked(adj, neighbors[i]);
14275
+ writeNeighbors(adj, ord, 0, neighbors, count);
14276
+ }
14188
14277
 
14189
14278
  // src/vector/hnsw/locks.ts
14190
14279
  var FREE = 0;
@@ -14415,6 +14504,17 @@ function nativeCompact(state, field, metric) {
14415
14504
  const compact2 = () => field.core.compact(field.graph, field.store.handle, nativeMetricCode(metric), state.locks.threadSlot, field.wakes);
14416
14505
  return writeToTheGraph(state, field, compact2) === NATIVE_OK;
14417
14506
  }
14507
+ function nativeConnect(state, field, metric, nearby) {
14508
+ const connect = () => field.core.connectUnreached(
14509
+ field.graph,
14510
+ field.store.handle,
14511
+ nativeMetricCode(metric),
14512
+ nearby,
14513
+ state.locks.threadSlot,
14514
+ field.wakes
14515
+ );
14516
+ return writeToTheGraph(state, field, connect) === NATIVE_OK;
14517
+ }
14418
14518
 
14419
14519
  // src/vector/hnsw/shared.ts
14420
14520
  function entryPointOf(state) {
@@ -14554,7 +14654,8 @@ function createHNSWWorkspace() {
14554
14654
  pruneCandidates: createList(INITIAL_LIST_CAPACITY),
14555
14655
  pruneSelection: createList(INITIAL_LIST_CAPACITY),
14556
14656
  repairCandidates: createList(INITIAL_LIST_CAPACITY),
14557
- repairSelection: createList(INITIAL_LIST_CAPACITY)
14657
+ repairSelection: createList(INITIAL_LIST_CAPACITY),
14658
+ droppedLinks: createList(INITIAL_LIST_CAPACITY)
14558
14659
  };
14559
14660
  }
14560
14661
  function linkSelectionsFor(workspace, layers) {
@@ -14712,6 +14813,260 @@ function setEntryPointsFromList(workspace, list) {
14712
14813
  workspace.entryPointCount = list.size;
14713
14814
  }
14714
14815
 
14816
+ // src/vector/hnsw/last-links.ts
14817
+ function isLive(state, ord) {
14818
+ return nodeExists(state, ord) && !isTombstoned(state, ord);
14819
+ }
14820
+ function noteNodeLeftUnlinked(state) {
14821
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 1);
14822
+ }
14823
+ function listHolds(list, ord) {
14824
+ for (let i = 0; i < list.size; i++) if (list.ords[i] === ord) return true;
14825
+ return false;
14826
+ }
14827
+ function collectDropped(sorted, kept, dropped) {
14828
+ dropped.size = 0;
14829
+ let next = 0;
14830
+ for (let i = 0; i < sorted.size; i++) {
14831
+ const ord = sorted.ords[i];
14832
+ if (next < kept.size && kept.ords[next] === ord) {
14833
+ next += 1;
14834
+ continue;
14835
+ }
14836
+ if (listHolds(kept, ord)) continue;
14837
+ appendToList(dropped, ord, sorted.distances[i]);
14838
+ }
14839
+ }
14840
+ function replaceFarthestShared(state, kept, held, ord, distance) {
14841
+ const tried = /* @__PURE__ */ new Set();
14842
+ for (; ; ) {
14843
+ let farthest = -1;
14844
+ for (let k = 0; k < kept.size; k++) {
14845
+ if (tried.has(k) || incomingCount(state.adjacency, kept.ords[k]) < 2) continue;
14846
+ if (farthest === -1 || kept.distances[k] > kept.distances[farthest]) farthest = k;
14847
+ }
14848
+ if (farthest === -1) return false;
14849
+ tried.add(farthest);
14850
+ const replaced = kept.ords[farthest];
14851
+ if (held(replaced) && !releaseUnlessLast(state.adjacency, replaced)) continue;
14852
+ kept.ords[farthest] = ord;
14853
+ kept.distances[farthest] = distance;
14854
+ return true;
14855
+ }
14856
+ }
14857
+ function writeKeepingLastLinks(state, ord, sorted, kept, held) {
14858
+ const adjacency = state.adjacency;
14859
+ const limit = state.Mmax0;
14860
+ const dropped = state.workspace.droppedLinks;
14861
+ collectDropped(sorted, kept, dropped);
14862
+ ensureListCapacity(kept, limit);
14863
+ for (let i = 0; i < dropped.size; i++) {
14864
+ const candidate = dropped.ords[i];
14865
+ if (!held(candidate)) continue;
14866
+ if (!isLive(state, candidate)) {
14867
+ noteUnlinked(adjacency, candidate);
14868
+ continue;
14869
+ }
14870
+ if (releaseUnlessLast(adjacency, candidate)) continue;
14871
+ if (kept.size < limit) {
14872
+ appendToList(kept, candidate, dropped.distances[i]);
14873
+ continue;
14874
+ }
14875
+ if (replaceFarthestShared(state, kept, held, candidate, dropped.distances[i])) continue;
14876
+ noteUnlinked(adjacency, candidate);
14877
+ noteNodeLeftUnlinked(state);
14878
+ }
14879
+ for (let k = 0; k < kept.size; k++) if (!held(kept.ords[k])) noteLinked(adjacency, kept.ords[k]);
14880
+ writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
14881
+ }
14882
+
14883
+ // src/vector/hnsw/connect-unreached.ts
14884
+ var NOT_REACHED_THROUGH_A_NODE = -1;
14885
+ function reach(walk, node, through) {
14886
+ walk.reached[node] = 1;
14887
+ walk.firstReachedThrough[node] = through;
14888
+ walk.queue[walk.tail++] = node;
14889
+ }
14890
+ function walkFrom(state, walk, start, through) {
14891
+ const adjacency = state.adjacency;
14892
+ if (walk.reached[start] === 1) return;
14893
+ reach(walk, start, through);
14894
+ while (walk.head < walk.tail) {
14895
+ const ord = walk.queue[walk.head++];
14896
+ const base = layerBase(adjacency, ord, 0);
14897
+ if (base === ABSENT) continue;
14898
+ const count = adjacency.level0[base];
14899
+ for (let i = 1; i <= count; i++) {
14900
+ const next = adjacency.level0[base + i];
14901
+ if (next < 0 || next >= walk.reached.length || walk.reached[next] === 1 || !nodeExists(state, next)) continue;
14902
+ reach(walk, next, ord);
14903
+ }
14904
+ }
14905
+ }
14906
+ function hasRoom(state, _walk, host) {
14907
+ const base = layerBase(state.adjacency, host, 0);
14908
+ return base !== ABSENT && state.adjacency.level0[base] < state.Mmax0;
14909
+ }
14910
+ function reachedThroughAnotherNode(walk, host, neighbour) {
14911
+ if (neighbour < 0 || neighbour >= walk.reached.length || walk.reached[neighbour] !== 1) return false;
14912
+ const through = walk.firstReachedThrough[neighbour];
14913
+ return through !== NOT_REACHED_THROUGH_A_NODE && through !== host;
14914
+ }
14915
+ function holdsLinkOutsideTheWalk(state, walk, host) {
14916
+ const adjacency = state.adjacency;
14917
+ const base = layerBase(adjacency, host, 0);
14918
+ if (base === ABSENT) return false;
14919
+ const count = adjacency.level0[base];
14920
+ for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
14921
+ return false;
14922
+ }
14923
+ function listOf(state, ord) {
14924
+ const adjacency = state.adjacency;
14925
+ const base = layerBase(adjacency, ord, 0);
14926
+ if (base === ABSENT) return new Int32Array(0);
14927
+ return adjacency.level0.slice(base + 1, base + 1 + adjacency.level0[base]);
14928
+ }
14929
+ function nearestHost(state, walk, orphan, nearby, test) {
14930
+ const distance = nodeDistanceFunction(state, state.buildMetric);
14931
+ let best = -1;
14932
+ let bestDistance = Number.POSITIVE_INFINITY;
14933
+ const consider = (host) => {
14934
+ if (host === orphan || walk.reached[host] !== 1 || !isLive(state, host) || !test(state, walk, host)) return;
14935
+ const away = distance(orphan, host);
14936
+ if (best === -1 || away < bestDistance) {
14937
+ best = host;
14938
+ bestDistance = away;
14939
+ }
14940
+ };
14941
+ for (const host of nearby.list) consider(host);
14942
+ for (const host of listOf(state, orphan))
14943
+ if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
14944
+ if (best !== -1) return best;
14945
+ for (let host = 0; host < walk.reached.length; host++) consider(host);
14946
+ return best;
14947
+ }
14948
+ function takeOutFarthestLinkOutsideTheWalk(state, walk, host) {
14949
+ const adjacency = state.adjacency;
14950
+ const base = layerBase(adjacency, host, 0);
14951
+ if (base === ABSENT) return;
14952
+ const distance = nodeDistanceFunction(state, state.buildMetric);
14953
+ const count = adjacency.level0[base];
14954
+ let farthest = -1;
14955
+ let farthestDistance = Number.NEGATIVE_INFINITY;
14956
+ for (let i = 1; i <= count; i++) {
14957
+ const neighbour = adjacency.level0[base + i];
14958
+ if (!reachedThroughAnotherNode(walk, host, neighbour)) continue;
14959
+ const away = distance(host, neighbour);
14960
+ if (farthest === -1 || away > farthestDistance) {
14961
+ farthest = i;
14962
+ farthestDistance = away;
14963
+ }
14964
+ }
14965
+ if (farthest === -1) return;
14966
+ const taken = adjacency.level0[base + farthest];
14967
+ for (let i = farthest; i < count; i++) adjacency.level0[base + i] = adjacency.level0[base + i + 1];
14968
+ adjacency.level0[base] = count - 1;
14969
+ noteUnlinked(adjacency, taken);
14970
+ }
14971
+ function appendTo(state, host, orphan) {
14972
+ const adjacency = state.adjacency;
14973
+ const base = layerBase(adjacency, host, 0);
14974
+ if (base === ABSENT) return;
14975
+ const count = adjacency.level0[base];
14976
+ adjacency.level0[base + count + 1] = orphan;
14977
+ adjacency.level0[base] = count + 1;
14978
+ noteLinked(adjacency, orphan);
14979
+ }
14980
+ function connectUnreached(state, nearby) {
14981
+ const slots = adjacencySlots(state.adjacency);
14982
+ const entry = entryPointOf(state);
14983
+ if (entry < 0 || entry >= slots || !nodeExists(state, entry)) return;
14984
+ const walk = {
14985
+ reached: new Uint8Array(slots),
14986
+ firstReachedThrough: new Int32Array(slots),
14987
+ queue: new Int32Array(slots),
14988
+ head: 0,
14989
+ tail: 0
14990
+ };
14991
+ walkFrom(state, walk, entry, NOT_REACHED_THROUGH_A_NODE);
14992
+ const held = { list: [], holds: new Uint8Array(slots) };
14993
+ for (let i = 0; i < nearby.length; i++) {
14994
+ const ord = nearby[i];
14995
+ if (ord < 0 || ord >= slots || held.holds[ord] === 1) continue;
14996
+ held.holds[ord] = 1;
14997
+ held.list.push(ord);
14998
+ }
14999
+ for (let orphan = 0; orphan < slots; orphan++) {
15000
+ if (walk.reached[orphan] === 1 || !isLive(state, orphan)) continue;
15001
+ let host = nearestHost(state, walk, orphan, held, hasRoom);
15002
+ if (host === -1) {
15003
+ host = nearestHost(state, walk, orphan, held, holdsLinkOutsideTheWalk);
15004
+ if (host === -1) continue;
15005
+ takeOutFarthestLinkOutsideTheWalk(state, walk, host);
15006
+ }
15007
+ appendTo(state, host, orphan);
15008
+ walkFrom(state, walk, orphan, host);
15009
+ }
15010
+ }
15011
+ function holdsUnlinkedNode(state) {
15012
+ const slots = adjacencySlots(state.adjacency);
15013
+ const entry = entryPointOf(state);
15014
+ const incoming = state.adjacency.incoming;
15015
+ for (let ord = 0; ord < slots; ord++) {
15016
+ if (incoming[ord] === 0 && ord !== entry && isLive(state, ord)) return true;
15017
+ }
15018
+ return false;
15019
+ }
15020
+ function connectThroughEitherCore(state, nearby) {
15021
+ const field = nativeFieldFor();
15022
+ if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
15023
+ connectUnreached(state, nearby);
15024
+ }
15025
+ function connectAfterRemoval(state, nearby) {
15026
+ connectThroughEitherCore(state, nearby);
15027
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
15028
+ }
15029
+ function connectAfterPlacement(state, holdsGraphLock) {
15030
+ if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
15031
+ if (!holdsGraphLock) lockGraphExclusive(state.locks);
15032
+ try {
15033
+ if (Atomics.load(state.header, GRAPH_UNLINKED_NODES) === 0) return;
15034
+ connectThroughEitherCore(state, new Int32Array(0));
15035
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
15036
+ } finally {
15037
+ if (!holdsGraphLock) unlockGraphExclusive(state.locks);
15038
+ }
15039
+ }
15040
+ function baseNeighboursOfRemoved(state, removed) {
15041
+ const leaving = new Set(removed);
15042
+ const nearby = [];
15043
+ for (const ord of removed) {
15044
+ for (const neighbour of listOf(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
15045
+ }
15046
+ return Int32Array.from(nearby);
15047
+ }
15048
+ function leaveIncomingUncounted(state) {
15049
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
15050
+ Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 1);
15051
+ }
15052
+ function countIncomingBeforeWriting(state, holdsGraphLock) {
15053
+ if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
15054
+ if (!holdsGraphLock) lockGraphExclusive(state.locks);
15055
+ try {
15056
+ if (Atomics.load(state.header, GRAPH_INCOMING_UNCOUNTED) === 0) return;
15057
+ const adjacency = state.adjacency;
15058
+ const slots = adjacencySlots(adjacency);
15059
+ if (slots > 0) reachNode(adjacency, slots - 1);
15060
+ adjacency.incoming.fill(0);
15061
+ countEveryLink(adjacency, slots);
15062
+ if (holdsUnlinkedNode(state)) connectThroughEitherCore(state, new Int32Array(0));
15063
+ Atomics.store(state.header, GRAPH_UNLINKED_NODES, 0);
15064
+ Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
15065
+ } finally {
15066
+ if (!holdsGraphLock) unlockGraphExclusive(state.locks);
15067
+ }
15068
+ }
15069
+
14715
15070
  // src/vector/hnsw/graph-ops.ts
14716
15071
  function readNeighborsLocked(state, ord, layer) {
14717
15072
  for (; ; ) {
@@ -14808,14 +15163,24 @@ function pruneConnections(state, ord, layer, distance) {
14808
15163
  for (let i = 1; i <= count; i++) {
14809
15164
  const connOrd = neighbors[base + i];
14810
15165
  const dist = distance(ord, connOrd);
14811
- if (dist === Number.POSITIVE_INFINITY) continue;
15166
+ if (dist === Number.POSITIVE_INFINITY) {
15167
+ if (layer === 0) noteUnlinked(adjacency, connOrd);
15168
+ continue;
15169
+ }
14812
15170
  candidates.ords[candidates.size] = connOrd;
14813
15171
  candidates.distances[candidates.size] = dist;
14814
15172
  candidates.size += 1;
14815
15173
  }
14816
15174
  const kept = state.workspace.pruneSelection;
14817
15175
  selectNeighborsHeuristic(state, candidates, mc, distance, kept);
14818
- replaceNeighbors(adjacency, ord, layer, kept.ords, kept.size);
15176
+ if (layer === 0) {
15177
+ writeKeepingLastLinks(state, ord, state.workspace.working, kept, everyCandidateWasHeld);
15178
+ return;
15179
+ }
15180
+ writeNeighbors(adjacency, ord, layer, kept.ords, kept.size);
15181
+ }
15182
+ function everyCandidateWasHeld() {
15183
+ return true;
14819
15184
  }
14820
15185
 
14821
15186
  // src/vector/hnsw/mutation.ts
@@ -14929,11 +15294,15 @@ function linkNeighborsBack(state, ord, linkTop) {
14929
15294
  }
14930
15295
  function linkNode(state, ord, level2) {
14931
15296
  const topLayer = topLayerOf(state);
15297
+ const formerEntry = entryPointOf(state);
14932
15298
  const linkTop = selectNeighborsPerLayer(state, ord, level2);
14933
15299
  if (linkTop < 0) return;
14934
15300
  writeOwnLists(state, ord, linkTop);
14935
15301
  linkNeighborsBack(state, ord, linkTop);
14936
- if (level2 > topLayer) raiseEntry(state, ord, level2);
15302
+ if (level2 <= topLayer) return;
15303
+ raiseEntry(state, ord, level2);
15304
+ const demoted = entryPointOf(state) !== formerEntry && isLive(state, formerEntry);
15305
+ if (demoted && incomingCount(state.adjacency, formerEntry) === 0) noteNodeLeftUnlinked(state);
14937
15306
  }
14938
15307
  function writeRecordBeforePlacement(state, ord) {
14939
15308
  const quantizer = state.quantizer;
@@ -14959,8 +15328,12 @@ function insertNode(state, ord, holdsGraphLock = false) {
14959
15328
  if (!state.store.holdsOrdinal(ord)) return false;
14960
15329
  ensureCapacity(state, ord + 1);
14961
15330
  if (nodeExists(state, ord)) return false;
15331
+ countIncomingBeforeWriting(state, holdsGraphLock);
14962
15332
  writeRecordBeforePlacement(state, ord);
14963
- if (placeThroughTheCore(state, ord, holdsGraphLock)) return true;
15333
+ if (placeThroughTheCore(state, ord, holdsGraphLock)) {
15334
+ connectAfterPlacement(state, holdsGraphLock);
15335
+ return true;
15336
+ }
14964
15337
  if (nodeExists(state, ord)) return false;
14965
15338
  if (!holdsGraphLock) lockGraphShared(state.locks);
14966
15339
  try {
@@ -14968,11 +15341,11 @@ function insertNode(state, ord, holdsGraphLock = false) {
14968
15341
  const level2 = randomLevel(state.mL);
14969
15342
  createNode(state.adjacency, ord, level2);
14970
15343
  Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
14971
- if (claimEntryIfEmpty(state, ord, level2)) return true;
14972
- linkNode(state, ord, level2);
15344
+ if (!claimEntryIfEmpty(state, ord, level2)) linkNode(state, ord, level2);
14973
15345
  } finally {
14974
15346
  if (!holdsGraphLock) unlockGraphShared(state.locks);
14975
15347
  }
15348
+ connectAfterPlacement(state, holdsGraphLock);
14976
15349
  return true;
14977
15350
  }
14978
15351
  function removeNodeEager(state, ord, excludeOrds) {
@@ -14990,7 +15363,8 @@ function removeNodeEager(state, ord, excludeOrds) {
14990
15363
  if (layer > nodeMaxLayer(state, neighborOrd)) continue;
14991
15364
  const mc = maxConns(state, layer);
14992
15365
  if (neighborCount(state.adjacency, neighborOrd, layer) >= mc) continue;
14993
- const candidateOrds = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
15366
+ const heldNow = new Set(collectNeighbors(state.adjacency, neighborOrd, layer));
15367
+ const candidateOrds = new Set(heldNow);
14994
15368
  for (const otherOrd of formerNeighbors) {
14995
15369
  if (otherOrd !== neighborOrd && otherOrd !== ord) {
14996
15370
  if (excludeOrds?.has(otherOrd)) continue;
@@ -15001,12 +15375,19 @@ function removeNodeEager(state, ord, excludeOrds) {
15001
15375
  candidates.size = 0;
15002
15376
  for (const candOrd of candidateOrds) {
15003
15377
  const dist = distance(neighborOrd, candOrd);
15004
- if (dist === Number.POSITIVE_INFINITY) continue;
15378
+ if (dist === Number.POSITIVE_INFINITY) {
15379
+ if (layer === 0 && heldNow.has(candOrd)) noteUnlinked(state.adjacency, candOrd);
15380
+ continue;
15381
+ }
15005
15382
  appendToList(candidates, candOrd, dist);
15006
15383
  }
15007
15384
  const selected = state.workspace.repairSelection;
15008
15385
  selectNeighborsHeuristic(state, candidates, mc, distance, selected);
15009
- replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
15386
+ if (layer === 0) {
15387
+ writeKeepingLastLinks(state, neighborOrd, state.workspace.working, selected, (held) => heldNow.has(held));
15388
+ } else {
15389
+ replaceNeighbors(state.adjacency, neighborOrd, layer, selected.ords, selected.size);
15390
+ }
15010
15391
  for (let i = 0; i < selected.size; i++) {
15011
15392
  const newConnOrd = selected.ords[i];
15012
15393
  if (layer <= nodeMaxLayer(state, newConnOrd)) {
@@ -15016,6 +15397,8 @@ function removeNodeEager(state, ord, excludeOrds) {
15016
15397
  }
15017
15398
  }
15018
15399
  }
15400
+ const formerBaseNeighbors = collectNeighbors(state.adjacency, ord, 0);
15401
+ for (const neighborOrd of formerBaseNeighbors) noteUnlinked(state.adjacency, neighborOrd);
15019
15402
  deleteNode(state.adjacency, ord);
15020
15403
  Atomics.sub(state.header, GRAPH_NODE_COUNT, 1);
15021
15404
  clearTombstone(state, ord);
@@ -15038,8 +15421,7 @@ function markTombstone(state, ord) {
15038
15421
  }
15039
15422
  function compactTombstones(state) {
15040
15423
  if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
15041
- const field = nativeFieldFor();
15042
- if (field !== null && nativeCompact(state, field, state.buildMetric)) return;
15424
+ countIncomingBeforeWriting(state, true);
15043
15425
  const tombstonedOrds = [];
15044
15426
  const slots = adjacencySlots(state.adjacency);
15045
15427
  for (let ord = 0; ord < slots; ord++) {
@@ -15047,10 +15429,15 @@ function compactTombstones(state) {
15047
15429
  tombstonedOrds.push(ord);
15048
15430
  }
15049
15431
  }
15050
- const excl = new Set(tombstonedOrds);
15051
- for (const ord of tombstonedOrds) {
15052
- removeNodeEager(state, ord, excl);
15432
+ const nearby = baseNeighboursOfRemoved(state, tombstonedOrds);
15433
+ const field = nativeFieldFor();
15434
+ if (field === null || !nativeCompact(state, field, state.buildMetric)) {
15435
+ const excl = new Set(tombstonedOrds);
15436
+ for (const ord of tombstonedOrds) {
15437
+ removeNodeEager(state, ord, excl);
15438
+ }
15053
15439
  }
15440
+ connectAfterRemoval(state, nearby);
15054
15441
  }
15055
15442
  function resetGraph(state) {
15056
15443
  resetAdjacency(state.adjacency);
@@ -15060,6 +15447,7 @@ function resetGraph(state) {
15060
15447
  Atomics.store(state.header, GRAPH_NODE_COUNT, 0);
15061
15448
  Atomics.store(state.header, GRAPH_ENTRY_POINT, -1);
15062
15449
  Atomics.store(state.header, GRAPH_TOP_LAYER, -1);
15450
+ Atomics.store(state.header, GRAPH_INCOMING_UNCOUNTED, 0);
15063
15451
  }
15064
15452
  function rebuild(state) {
15065
15453
  const nodeCount = Atomics.load(state.header, GRAPH_NODE_COUNT);
@@ -15269,7 +15657,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
15269
15657
  }
15270
15658
  cursor += count + 1;
15271
15659
  if (ordinal !== NO_ORDINAL && layer <= keptTop) {
15272
- replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
15660
+ writeNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
15273
15661
  }
15274
15662
  }
15275
15663
  }
@@ -15281,6 +15669,7 @@ function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
15281
15669
  const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
15282
15670
  Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
15283
15671
  Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
15672
+ leaveIncomingUncounted(state);
15284
15673
  }
15285
15674
 
15286
15675
  // src/vector/hnsw/persistence.ts
@@ -15319,7 +15708,7 @@ function deserializeGraph(state, store, data) {
15319
15708
  for (const node of nodes) {
15320
15709
  createNode(state.adjacency, node.ord, node.maxLayer);
15321
15710
  for (const [layer, neighbors] of node.layers) {
15322
- replaceNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
15711
+ writeNeighbors(state.adjacency, node.ord, layer, neighbors, neighbors.length);
15323
15712
  }
15324
15713
  Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
15325
15714
  }
@@ -15345,6 +15734,7 @@ function deserializeGraph(state, store, data) {
15345
15734
  Atomics.store(state.header, GRAPH_ENTRY_POINT, bestOrd);
15346
15735
  Atomics.store(state.header, GRAPH_TOP_LAYER, bestLayer);
15347
15736
  }
15737
+ leaveIncomingUncounted(state);
15348
15738
  }
15349
15739
 
15350
15740
  // src/vector/hnsw/scratch-bytes.ts
@@ -15644,6 +16034,7 @@ function restoreSnapshot(state, snapshot) {
15644
16034
  Atomics.store(state.header, GRAPH_NODE_COUNT, snapshot.nodeCount);
15645
16035
  Atomics.store(state.header, GRAPH_ENTRY_POINT, snapshot.entryPointOrd);
15646
16036
  Atomics.store(state.header, GRAPH_TOP_LAYER, snapshot.topLayer);
16037
+ leaveIncomingUncounted(state);
15647
16038
  }
15648
16039
 
15649
16040
  // src/vector/hnsw/state.ts
@@ -15779,6 +16170,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
15779
16170
  },
15780
16171
  search: (query, k, searchMetric, minSimilarity, options) => search(state, docIdOf, query, k, searchMetric, minSimilarity, options),
15781
16172
  clear: () => exclusively(() => resetGraph(state)),
16173
+ retire: () => Atomics.store(shared2.header, GRAPH_RETIRED, 1),
15782
16174
  entries: entriesIterator,
15783
16175
  compactionNeeded,
15784
16176
  compactTombstones: () => exclusively(() => compactTombstones(state)),
@@ -15842,6 +16234,7 @@ function releaseReplacedVectors(state) {
15842
16234
  for (const ordinal of state.store.releaseReplacedVectors()) state.osq?.removeOrdinal(ordinal);
15843
16235
  }
15844
16236
  function adoptGraph(state, graph) {
16237
+ if (state.hnsw !== null && state.hnsw !== graph) state.hnsw.retire();
15845
16238
  if (graph === null && state.hnsw !== null) {
15846
16239
  state.hnsw.clear();
15847
16240
  }
@@ -16607,7 +17000,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
16607
17000
  continue;
16608
17001
  }
16609
17002
  if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
16610
- state.buffer.delete(docId);
17003
+ if (graph === state.hnsw) state.buffer.delete(docId);
16611
17004
  }
16612
17005
  if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
16613
17006
  fallback.shortAnswerWarned = true;
@@ -16624,7 +17017,7 @@ async function insertLocally(state, graph, docIds, admit) {
16624
17017
  const ordinal = state.store.getOrdinal(docId);
16625
17018
  if (ordinal === void 0) continue;
16626
17019
  graph.insertOrdinal(ordinal);
16627
- state.buffer.delete(docId);
17020
+ if (graph === state.hnsw) state.buffer.delete(docId);
16628
17021
  count += 1;
16629
17022
  if (count % BUILD_CHUNK_SIZE === 0) {
16630
17023
  noteWrite(state);
@@ -16741,6 +17134,10 @@ async function buildGraphFromStore(state) {
16741
17134
  }
16742
17135
  const previous = state.hnsw;
16743
17136
  adoptGraph(state, graph);
17137
+ for (const docId of [...state.buffer]) {
17138
+ const ordinal = state.store.getOrdinal(docId);
17139
+ if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) state.buffer.delete(docId);
17140
+ }
16744
17141
  if (previous !== null && previous !== graph) dropSharedGraph(state, previous);
16745
17142
  await releasePendingLocations(state);
16746
17143
  }
@@ -17249,7 +17646,7 @@ function registryMatches(state, listedKeys) {
17249
17646
  if (listed.length !== state.savedFiles.length) return false;
17250
17647
  return state.savedFiles.every((file, index) => file.key === listed[index]);
17251
17648
  }
17252
- function isLive(state, ordinal) {
17649
+ function isLive2(state, ordinal) {
17253
17650
  if (ordinal === NO_LIVE_VECTOR_AT_POSITION) return false;
17254
17651
  const docId = state.store.docIdForOrdinal(ordinal);
17255
17652
  return docId !== void 0 && !state.tombstones.has(docId);
@@ -17258,7 +17655,7 @@ function isGraphTombstone(state, ordinal) {
17258
17655
  return ordinal !== NO_LIVE_VECTOR_AT_POSITION && state.hnsw !== null && state.hnsw.holdsTombstone(ordinal);
17259
17656
  }
17260
17657
  function isCarried(state, ordinal) {
17261
- return isLive(state, ordinal) || isGraphTombstone(state, ordinal);
17658
+ return isLive2(state, ordinal) || isGraphTombstone(state, ordinal);
17262
17659
  }
17263
17660
  function carriedCountOf(state, file) {
17264
17661
  let carried = 0;
@@ -17266,7 +17663,7 @@ function carriedCountOf(state, file) {
17266
17663
  return carried;
17267
17664
  }
17268
17665
  function deadBitsOf(state, file) {
17269
- return deadBitsWhere(file.ordinals.length, (position) => !isLive(state, file.ordinals[position]));
17666
+ return deadBitsWhere(file.ordinals.length, (position) => !isLive2(state, file.ordinals[position]));
17270
17667
  }
17271
17668
  function filesWorthKeeping(state, listed, calibration, carriedCounts) {
17272
17669
  const kept = [];
@@ -17348,7 +17745,7 @@ function graphPartsWithoutNodes(parts, numbers) {
17348
17745
  }
17349
17746
  function plannedVectorOf(state, ordinal) {
17350
17747
  const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
17351
- return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
17748
+ return { ordinal, docId, tombstoned: !isLive2(state, ordinal) };
17352
17749
  }
17353
17750
  function readFile(state, planned, centroid) {
17354
17751
  const dimension = state.dimension;
@@ -17590,6 +17987,87 @@ function beginCheckpointRestore(state, shape) {
17590
17987
  return { addFile, finish };
17591
17988
  }
17592
17989
 
17990
+ // src/vector/vector-index/flagged-documents.ts
17991
+ var FlaggedDocuments = class extends Set {
17992
+ constructor(store, layout) {
17993
+ super();
17994
+ this.store = store;
17995
+ this.layout = layout;
17996
+ }
17997
+ ordinals = /* @__PURE__ */ new Map();
17998
+ handles = null;
17999
+ flags = new Uint8Array(0);
18000
+ chunks = null;
18001
+ add(docId) {
18002
+ super.add(docId);
18003
+ const handles = this.openHandles();
18004
+ if (handles === null) return this;
18005
+ this.flagCurrentOrdinal(docId);
18006
+ Atomics.store(handles.header, this.layout.countWord, this.size);
18007
+ return this;
18008
+ }
18009
+ delete(docId) {
18010
+ const held = super.delete(docId);
18011
+ const handles = this.openHandles();
18012
+ if (handles === null) return held;
18013
+ const ordinal = this.ordinals.get(docId);
18014
+ if (ordinal !== void 0) {
18015
+ this.ordinals.delete(docId);
18016
+ this.unflag(ordinal);
18017
+ }
18018
+ Atomics.store(handles.header, this.layout.countWord, this.size);
18019
+ return held;
18020
+ }
18021
+ clear() {
18022
+ super.clear();
18023
+ const handles = this.openHandles();
18024
+ if (handles !== null) {
18025
+ for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
18026
+ Atomics.store(handles.header, this.layout.countWord, 0);
18027
+ }
18028
+ this.ordinals.clear();
18029
+ }
18030
+ flagCurrentOrdinal(docId) {
18031
+ const ordinal = this.store.getOrdinal(docId);
18032
+ const previous = this.ordinals.get(docId);
18033
+ if (ordinal === void 0 || ordinal === previous) return;
18034
+ if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
18035
+ Atomics.store(this.flags, ordinal, 1);
18036
+ this.ordinals.set(docId, ordinal);
18037
+ if (previous !== void 0) this.unflag(previous);
18038
+ }
18039
+ unflag(ordinal) {
18040
+ Atomics.store(this.flags, ordinal, 0);
18041
+ if (this.chunks !== null) Atomics.sub(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
18042
+ }
18043
+ openHandles() {
18044
+ if (this.store.dimension === 0) return null;
18045
+ const handles = this.store.handles;
18046
+ if (handles === this.handles) return handles;
18047
+ this.handles = handles;
18048
+ this.flags = new Uint8Array(this.layout.flagsOf(handles));
18049
+ const chunksOf2 = this.layout.chunksOf;
18050
+ this.chunks = chunksOf2 === null ? null : new Int32Array(chunksOf2(handles));
18051
+ this.ordinals.clear();
18052
+ for (const docId of this) this.flagCurrentOrdinal(docId);
18053
+ return handles;
18054
+ }
18055
+ };
18056
+ function createUnplacedVectors(store) {
18057
+ return new FlaggedDocuments(store, {
18058
+ flagsOf: (handles) => handles.unplaced,
18059
+ chunksOf: (handles) => handles.unplacedChunks,
18060
+ countWord: STORE_UNPLACED_COUNT
18061
+ });
18062
+ }
18063
+ function createRemovedDocuments(store) {
18064
+ return new FlaggedDocuments(store, {
18065
+ flagsOf: (handles) => handles.removed,
18066
+ chunksOf: null,
18067
+ countWord: STORE_REMOVED_COUNT
18068
+ });
18069
+ }
18070
+
17593
18071
  // src/vector/vector-index/maintenance.ts
17594
18072
  function compact(state) {
17595
18073
  if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
@@ -17787,6 +18265,43 @@ function createHeap2(compare) {
17787
18265
  };
17788
18266
  }
17789
18267
 
18268
+ // src/vector/vector-index/merge-results.ts
18269
+ function highScoreFirst(a, b) {
18270
+ return b.score - a.score || compareCodePoints(a.docId, b.docId);
18271
+ }
18272
+ function mergeResults(hnswResults, bufferResults, k) {
18273
+ const seen = /* @__PURE__ */ new Set();
18274
+ const merged = [];
18275
+ let hi = 0;
18276
+ let bi = 0;
18277
+ while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
18278
+ const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
18279
+ const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
18280
+ let pick;
18281
+ if (h && b) {
18282
+ if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
18283
+ pick = h;
18284
+ hi++;
18285
+ } else {
18286
+ pick = b;
18287
+ bi++;
18288
+ }
18289
+ } else if (h) {
18290
+ pick = h;
18291
+ hi++;
18292
+ } else if (b) {
18293
+ pick = b;
18294
+ bi++;
18295
+ } else {
18296
+ break;
18297
+ }
18298
+ if (seen.has(pick.docId)) continue;
18299
+ seen.add(pick.docId);
18300
+ merged.push(pick);
18301
+ }
18302
+ return merged;
18303
+ }
18304
+
17790
18305
  // src/vector/vector-index/search.ts
17791
18306
  function* bufferCandidates(state, filter) {
17792
18307
  for (const docId of state.buffer) {
@@ -17816,9 +18331,6 @@ function fetchedOutcome(state, results, minSimilarity, filter) {
17816
18331
  function scannedOutcome(scan2) {
17817
18332
  return { results: scan2.results, matched: scan2.matched, matchedExact: true };
17818
18333
  }
17819
- function highScoreFirst(a, b) {
17820
- return b.score - a.score || compareCodePoints(a.docId, b.docId);
17821
- }
17822
18334
  function liveCandidates(state, candidates) {
17823
18335
  const docIds = [];
17824
18336
  const ordinals = [];
@@ -17883,38 +18395,6 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
17883
18395
  }
17884
18396
  return { results: heap.toSortedArray().reverse(), matched };
17885
18397
  }
17886
- function mergeResults(hnswResults, bufferResults, k) {
17887
- const seen = /* @__PURE__ */ new Set();
17888
- const merged = [];
17889
- let hi = 0;
17890
- let bi = 0;
17891
- while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
17892
- const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
17893
- const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
17894
- let pick;
17895
- if (h && b) {
17896
- if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
17897
- pick = h;
17898
- hi++;
17899
- } else {
17900
- pick = b;
17901
- bi++;
17902
- }
17903
- } else if (h) {
17904
- pick = h;
17905
- hi++;
17906
- } else if (b) {
17907
- pick = b;
17908
- bi++;
17909
- } else {
17910
- break;
17911
- }
17912
- if (seen.has(pick.docId)) continue;
17913
- seen.add(pick.docId);
17914
- merged.push(pick);
17915
- }
17916
- return merged;
17917
- }
17918
18398
  function search2(state, query, k, options) {
17919
18399
  return searchWithFilter(state, query, k, options, filterForOptions(state, options));
17920
18400
  }
@@ -18000,8 +18480,8 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
18000
18480
  hnswConfig,
18001
18481
  workerCopies,
18002
18482
  store,
18003
- tombstones: /* @__PURE__ */ new Set(),
18004
- buffer: /* @__PURE__ */ new Set(),
18483
+ tombstones: createRemovedDocuments(store),
18484
+ buffer: createUnplacedVectors(store),
18005
18485
  pendingLocations: /* @__PURE__ */ new Map(),
18006
18486
  savedFiles: [],
18007
18487
  savedSignature: null,
@@ -20477,55 +20957,93 @@ function runSegmentAction(entry, action) {
20477
20957
  }
20478
20958
  }
20479
20959
 
20480
- // src/vector/shared-field/searcher.ts
20481
- function bestFirst2(a, b) {
20482
- return b.score - a.score || (a.docId < b.docId ? -1 : a.docId > b.docId ? 1 : 0);
20960
+ // src/vector/shared-field/store-flags.ts
20961
+ function bytesOf(flags, source) {
20962
+ if (flags.source !== source) {
20963
+ flags.source = source;
20964
+ flags.view = new Uint8Array(source);
20965
+ }
20966
+ return flags.view;
20483
20967
  }
20484
- function distancesInTypeScript(state, query, metric, ordinals) {
20485
- const arenaQuery = state.store.prepareQueryArena(query);
20968
+ function countsOf(chunks, source) {
20969
+ if (chunks.source !== source) {
20970
+ chunks.source = source;
20971
+ chunks.view = new Int32Array(source);
20972
+ }
20973
+ return chunks.view;
20974
+ }
20975
+ function readStoreFlags(store) {
20976
+ const unplaced = { source: null, view: new Uint8Array(0) };
20977
+ const unplacedChunks = { source: null, view: new Int32Array(0) };
20978
+ const removed = { source: null, view: new Uint8Array(0) };
20979
+ return {
20980
+ liveDocuments() {
20981
+ const header2 = store.handles.header;
20982
+ return Atomics.load(header2, STORE_LIVE_COUNT) - Atomics.load(header2, STORE_REMOVED_COUNT);
20983
+ },
20984
+ isRemoved(ordinal) {
20985
+ const bytes = bytesOf(removed, store.handles.removed);
20986
+ return ordinal < bytes.length && Atomics.load(bytes, ordinal) === 1;
20987
+ },
20988
+ unplacedOrdinals() {
20989
+ const handles = store.handles;
20990
+ if (Atomics.load(handles.header, STORE_UNPLACED_COUNT) === 0) return [];
20991
+ const bytes = bytesOf(unplaced, handles.unplaced);
20992
+ const counts = countsOf(unplacedChunks, handles.unplacedChunks);
20993
+ const end = Math.min(store.slots, bytes.length);
20994
+ const ordinals = [];
20995
+ for (let chunk = 0; chunk < counts.length; chunk++) {
20996
+ const from = chunk * UNPLACED_FLAG_CHUNK_ORDINALS;
20997
+ if (from >= end) break;
20998
+ if (Atomics.load(counts, chunk) === 0) continue;
20999
+ const to = Math.min(end, from + UNPLACED_FLAG_CHUNK_ORDINALS);
21000
+ for (let ordinal = from; ordinal < to; ordinal++) {
21001
+ if (Atomics.load(bytes, ordinal) === 1) ordinals.push(ordinal);
21002
+ }
21003
+ }
21004
+ return ordinals;
21005
+ }
21006
+ };
21007
+ }
21008
+
21009
+ // src/vector/shared-field/searcher.ts
21010
+ function distancesInTypeScript(store, query, metric, ordinals) {
21011
+ const arenaQuery = store.prepareQueryArena(query);
20486
21012
  const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
20487
21013
  const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
20488
21014
  for (let i = 0; i < ordinals.length; i++) {
20489
21015
  if (arenaQuery) {
20490
- distances[i] = state.store.distanceFromArena(arenaQuery, ordinals[i], metric);
21016
+ distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
20491
21017
  continue;
20492
21018
  }
20493
- const entry = entryForOrd(state, ordinals[i]);
21019
+ const entry = store.entryForOrdinal(ordinals[i]);
20494
21020
  if (entry) distances[i] = toDistance(query, entry.vector, queryMagnitude, entry.magnitude, metric);
20495
21021
  }
20496
21022
  return distances;
20497
21023
  }
20498
- function bruteForceOrdinals(state, docIdOf, query, k, metric, minSimilarity, filter) {
20499
- const ordinals = [];
20500
- const docIds = [];
20501
- for (const ord of ordinalFilterValues(filter)) {
20502
- if (isTombstoned(state, ord)) continue;
20503
- const docId = docIdOf(ord);
20504
- if (docId === void 0) continue;
20505
- ordinals.push(ord);
20506
- docIds.push(docId);
20507
- }
20508
- const distances = nativeScoresOf(state.store.handles, query, metric, ordinals) ?? distancesInTypeScript(state, query, metric, ordinals);
20509
- const heap = createBoundedMaxHeap(bestFirst2, k);
21024
+ function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
21025
+ const { ordinals, docIds } = candidates;
21026
+ if (ordinals.length === 0) return { results: [], matched: 0 };
21027
+ const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals);
21028
+ const heap = createBoundedMaxHeap(highScoreFirst, k);
20510
21029
  let matched = 0;
20511
21030
  for (let i = 0; i < ordinals.length; i++) {
20512
21031
  if (distances[i] === Number.POSITIVE_INFINITY) continue;
20513
21032
  const score = toScore(distances[i], metric);
20514
21033
  if (score < minSimilarity) continue;
20515
21034
  matched++;
20516
- heap.push({ ord: ordinals[i], score, docId: docIds[i] });
21035
+ heap.push({ docId: docIds[i], score });
20517
21036
  }
20518
- return { hits: heap.toSortedArray().reverse(), matched, matchedExact: true };
21037
+ return { results: heap.toSortedArray().reverse(), matched };
20519
21038
  }
20520
21039
  function createSharedVectorSearcher(options) {
20521
21040
  const { fieldName, view, holdsDocument } = options;
21041
+ const flags = readStoreFlags(view.store);
20522
21042
  function filterForDocIds(allowed) {
20523
21043
  const filter = createOrdinalFilter(view.store.slots);
20524
- const graph = view.graph;
20525
21044
  for (const docId of allowed) {
20526
21045
  const ordinal = view.ordinalOf(docId);
20527
- if (ordinal === void 0) continue;
20528
- if (graph !== null && isTombstoned(graph, ordinal)) continue;
21046
+ if (ordinal === void 0 || flags.isRemoved(ordinal)) continue;
20529
21047
  addToOrdinalFilter(filter, ordinal);
20530
21048
  }
20531
21049
  return filter;
@@ -20535,9 +21053,9 @@ function createSharedVectorSearcher(options) {
20535
21053
  const filter = createOrdinalFilter(slots);
20536
21054
  for (let ordinal = 0; ordinal < slots; ordinal++) {
20537
21055
  const partition = view.store.partitionAt(ordinal);
20538
- if (partition !== void 0 && allowed.has(partition) && view.store.holdsOrdinal(ordinal)) {
20539
- addToOrdinalFilter(filter, ordinal);
20540
- }
21056
+ if (partition === void 0 || !allowed.has(partition)) continue;
21057
+ if (!view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal)) continue;
21058
+ addToOrdinalFilter(filter, ordinal);
20541
21059
  }
20542
21060
  return filter;
20543
21061
  }
@@ -20546,18 +21064,62 @@ function createSharedVectorSearcher(options) {
20546
21064
  if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
20547
21065
  return void 0;
20548
21066
  }
20549
- function hitsFor(query, k, searchOptions, filter) {
21067
+ function candidatesOf(ordinals, filter) {
21068
+ const candidates = { ordinals: [], docIds: [] };
21069
+ for (const ordinal of ordinals) {
21070
+ if (filter !== void 0 && !ordinalFilterHas(filter, ordinal)) continue;
21071
+ if (flags.isRemoved(ordinal)) continue;
21072
+ const docId = view.docIdOf(ordinal);
21073
+ if (docId === void 0 || view.ordinalOf(docId) !== ordinal) continue;
21074
+ candidates.ordinals.push(ordinal);
21075
+ candidates.docIds.push(docId);
21076
+ }
21077
+ return candidates;
21078
+ }
21079
+ function scanned(scan2) {
21080
+ return { results: scan2.results, matched: scan2.matched, matchedExact: true, resolvedEveryHit: true };
21081
+ }
21082
+ function search3(query, k, searchOptions, filter) {
20550
21083
  const { metric, minSimilarity, efSearch, oversample } = searchOptions;
20551
- const liveSize2 = view.liveSize;
21084
+ const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
21085
+ const liveDocuments = flags.liveDocuments();
21086
+ if (liveDocuments <= 0) return empty;
21087
+ if (filter !== void 0 && filter.count === 0) return empty;
21088
+ function fetched(results, resolvedEveryHit2) {
21089
+ if (minSimilarity === Number.NEGATIVE_INFINITY) {
21090
+ const matched = filter === void 0 ? liveDocuments : filter.count;
21091
+ return { results, matched, matchedExact: true, resolvedEveryHit: resolvedEveryHit2 };
21092
+ }
21093
+ return { results, matched: results.length, matchedExact: false, resolvedEveryHit: resolvedEveryHit2 };
21094
+ }
21095
+ if (k <= 0) return fetched([], true);
20552
21096
  const graph = view.graph;
20553
- if (graph !== null && filter !== void 0 && liveSize2 > 0 && filter.count / liveSize2 < view.handles.filterThreshold) {
20554
- return bruteForceOrdinals(graph, view.docIdOf, query, k, metric, minSimilarity, filter);
21097
+ if (graph === null) {
21098
+ const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
21099
+ return scanned(scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity));
20555
21100
  }
20556
- const hits = view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample });
20557
- if (minSimilarity === Number.NEGATIVE_INFINITY) {
20558
- return { hits, matched: filter === void 0 ? liveSize2 : filter.count, matchedExact: true };
21101
+ if (filter !== void 0) {
21102
+ const graphLiveSize = view.liveSize;
21103
+ const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
21104
+ if (selectivity < view.handles.filterThreshold) {
21105
+ const candidates = candidatesOf(ordinalFilterValues(filter), filter);
21106
+ return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
21107
+ }
21108
+ }
21109
+ let resolvedEveryHit = true;
21110
+ const graphResults = [];
21111
+ for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, { filter, efSearch, oversample })) {
21112
+ const docId = view.docIdOf(hit.ord);
21113
+ if (docId === void 0) {
21114
+ resolvedEveryHit = false;
21115
+ continue;
21116
+ }
21117
+ graphResults.push({ docId, score: hit.score });
20559
21118
  }
20560
- return { hits, matched: hits.length, matchedExact: false };
21119
+ const unplaced = flags.unplacedOrdinals();
21120
+ if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
21121
+ const scan2 = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
21122
+ return fetched(mergeResults(graphResults, scan2.results, k), resolvedEveryHit);
20561
21123
  }
20562
21124
  return {
20563
21125
  fieldName,
@@ -20572,22 +21134,20 @@ function createSharedVectorSearcher(options) {
20572
21134
  { expected: view.handles.dimension, received: query.length }
20573
21135
  );
20574
21136
  }
20575
- const filter = filterFor(searchOptions);
20576
- if (filter !== void 0 && filter.count === 0) {
20577
- return Promise.resolve({ results: [], matched: 0, matchedExact: true });
20578
- }
20579
- const scanned = hitsFor(query, k, searchOptions, filter);
20580
- const results = [];
20581
- for (const hit of scanned.hits) {
20582
- const docId = view.docIdOf(hit.ord);
20583
- if (docId === void 0 || !holdsDocument(docId)) continue;
20584
- results.push({ docId, score: hit.score });
21137
+ const outcome = search3(query, k, searchOptions, filterFor(searchOptions));
21138
+ if (view.graphRetired) {
21139
+ throw new NarsilError(
21140
+ ErrorCodes.INDEX_NOT_FOUND,
21141
+ `The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
21142
+ { fieldName }
21143
+ );
20585
21144
  }
20586
- const holdsEveryHit = results.length === scanned.hits.length;
21145
+ const results = outcome.results.filter((result) => holdsDocument(result.docId));
21146
+ const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
20587
21147
  return Promise.resolve({
20588
21148
  results,
20589
- matched: scanned.matched,
20590
- matchedExact: scanned.matchedExact && holdsEveryHit
21149
+ matched: outcome.matched,
21150
+ matchedExact: outcome.matchedExact && holdsEveryHit
20591
21151
  });
20592
21152
  }
20593
21153
  };
@@ -20636,6 +21196,9 @@ function openSharedVectorField(initial, threadSlot) {
20636
21196
  get readsEveryVector() {
20637
21197
  return store.holdsEveryBlock;
20638
21198
  },
21199
+ get graphRetired() {
21200
+ return graph !== null && Atomics.load(graph.header, GRAPH_RETIRED) === 1;
21201
+ },
20639
21202
  docIdOf,
20640
21203
  ordinalOf,
20641
21204
  vectorOf(docId) {
@@ -20697,7 +21260,8 @@ function loadHeldVectorCopy(held, manager, fieldName, handle, handles, threadSlo
20697
21260
  );
20698
21261
  }
20699
21262
  function holdsVectorField(held, fieldName) {
20700
- return held.fields.get(fieldName)?.view.readsEveryVector === true;
21263
+ const view = held.fields.get(fieldName)?.view;
21264
+ return view?.readsEveryVector === true && !view.graphRetired;
20701
21265
  }
20702
21266
  function dropHeldVectorCopy(held, fieldName, handle) {
20703
21267
  held.builds.get(handle)?.view.close();
@@ -20768,6 +21332,9 @@ function createDirectExecutor(options) {
20768
21332
  return vectorIndexes;
20769
21333
  }
20770
21334
  function createIndex(indexName, config, language, analysisStale = false) {
21335
+ addIndex(indexName, config, language, analysisStale, null);
21336
+ }
21337
+ function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
20771
21338
  if (indexes.has(indexName)) {
20772
21339
  throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${indexName}" already exists`, {
20773
21340
  indexName
@@ -20788,7 +21355,8 @@ function createDirectExecutor(options) {
20788
21355
  },
20789
21356
  vectorIndexes,
20790
21357
  vectorCopies: createHeldVectorCopies(),
20791
- analysisStale
21358
+ analysisStale,
21359
+ awaitedCopyLoad
20792
21360
  });
20793
21361
  }
20794
21362
  function dropIndex(indexName) {
@@ -20810,7 +21378,7 @@ function createDirectExecutor(options) {
20810
21378
  }
20811
21379
  function queryContextOf(indexName) {
20812
21380
  const entry = indexes.get(indexName);
20813
- if (entry === void 0) return void 0;
21381
+ if (entry === void 0 || entry.awaitedCopyLoad !== null) return void 0;
20814
21382
  return {
20815
21383
  manager: entry.manager,
20816
21384
  config: entry.config,
@@ -20840,7 +21408,12 @@ function createDirectExecutor(options) {
20840
21408
  }
20841
21409
  case "createIndex": {
20842
21410
  const language = getLanguage(action.config.language ?? "english");
20843
- createIndex(action.indexName, action.config, language, action.analysisStale ?? false);
21411
+ addIndex(action.indexName, action.config, language, action.analysisStale ?? false, action.copyLoad ?? null);
21412
+ return void 0;
21413
+ }
21414
+ case "copyLoaded": {
21415
+ const entry = indexes.get(action.indexName);
21416
+ if (entry?.awaitedCopyLoad === action.copyLoad) entry.awaitedCopyLoad = null;
20844
21417
  return void 0;
20845
21418
  }
20846
21419
  case "dropIndex": {
@@ -21488,12 +22061,12 @@ async function lockDirectory(root) {
21488
22061
  const resolvedRoot = pathMod.resolve(root);
21489
22062
  const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
21490
22063
  const self = await describeSelf();
21491
- async function holds() {
22064
+ async function holds2() {
21492
22065
  const holder = await readHolder(fs, lockPath);
21493
22066
  return typeof holder === "object" && holder.nonce === self.nonce;
21494
22067
  }
21495
22068
  async function removeOwnLockFile() {
21496
- if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
22069
+ if (await holds2().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
21497
22070
  }
21498
22071
  let directoryKey = null;
21499
22072
  let closeDirectory = () => void 0;
@@ -21519,7 +22092,7 @@ async function lockDirectory(root) {
21519
22092
  const heldKey = directoryKey;
21520
22093
  let released = false;
21521
22094
  return {
21522
- holds,
22095
+ holds: holds2,
21523
22096
  async release() {
21524
22097
  if (released) return;
21525
22098
  released = true;
@@ -29320,6 +29893,7 @@ function shareFrozenSegment(segment, analysis) {
29320
29893
  }
29321
29894
 
29322
29895
  // src/engine/worker-resync.ts
29896
+ var copyLoadSequence = 0;
29323
29897
  function captureComposite(manager, partitionId) {
29324
29898
  const partition = manager.getPartition(partitionId);
29325
29899
  if (!isCompositePartition(partition)) return null;
@@ -29380,8 +29954,8 @@ async function sendPartition(indexName, executors, captured) {
29380
29954
  )
29381
29955
  );
29382
29956
  }
29383
- async function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
29384
- await loadIndexOntoWorkers(
29957
+ function transferIndexToPool(indexName, pool, config, manager, analysisStale = false) {
29958
+ return loadIndexOntoWorkers(
29385
29959
  indexName,
29386
29960
  pool.getAllExecutors(),
29387
29961
  config,
@@ -29391,6 +29965,8 @@ async function transferIndexToPool(indexName, pool, config, manager, analysisSta
29391
29965
  );
29392
29966
  }
29393
29967
  async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped, analysisStale = false) {
29968
+ copyLoadSequence += 1;
29969
+ const copyLoad = `${indexName}#${copyLoadSequence}`;
29394
29970
  const captured = capturePartitions(manager);
29395
29971
  await Promise.allSettled(
29396
29972
  executors.map(
@@ -29405,13 +29981,15 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
29405
29981
  indexName,
29406
29982
  config,
29407
29983
  requestId: `resync-create-${indexName}`,
29408
- analysisStale
29984
+ analysisStale,
29985
+ copyLoad
29409
29986
  })
29410
29987
  )
29411
29988
  );
29412
29989
  for (const partition of captured) {
29413
29990
  await sendPartition(indexName, executors, partition);
29414
29991
  }
29992
+ return copyLoad;
29415
29993
  }
29416
29994
 
29417
29995
  // src/workers/protocol.ts
@@ -29659,6 +30237,14 @@ function enqueueReplication(state, indexName, action) {
29659
30237
  queue.inFlight.push(run);
29660
30238
  queue.tail = run;
29661
30239
  }
30240
+ function markCopiesLoaded(state, indexName, copyLoad) {
30241
+ enqueueReplication(state, indexName, {
30242
+ type: "copyLoaded",
30243
+ indexName,
30244
+ copyLoad,
30245
+ requestId: `copy-loaded-${copyLoad}`
30246
+ });
30247
+ }
29662
30248
  async function replicateToWorkers(state, action) {
29663
30249
  if (!("indexName" in action)) {
29664
30250
  if (state.workerPool === null) return;
@@ -30608,8 +31194,9 @@ async function loadOntoReplacement(state, replacement, indexName) {
30608
31194
  if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
30609
31195
  const buffered = [];
30610
31196
  state.copyLoadBuffers.set(indexName, buffered);
31197
+ let copyLoad;
30611
31198
  try {
30612
- await loadIndexOntoWorkers(
31199
+ copyLoad = await loadIndexOntoWorkers(
30613
31200
  indexName,
30614
31201
  [replacement.executor],
30615
31202
  entry.config,
@@ -30622,6 +31209,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
30622
31209
  state.copyLoadBuffers.delete(indexName);
30623
31210
  for (const action of buffered) enqueueReplication(state, indexName, action);
30624
31211
  }
31212
+ markCopiesLoaded(state, indexName, copyLoad);
30625
31213
  refreshVectorCopies(state, indexName);
30626
31214
  }
30627
31215
  async function replaceWorker(state, pool, workerId) {
@@ -30751,21 +31339,21 @@ async function ensurePool(state) {
30751
31339
  function documentCountOf(state, indexName) {
30752
31340
  return state.executor.getManager(indexName)?.countDocuments() ?? 0;
30753
31341
  }
30754
- function copyThresholdReason(state, indexName, incomingCount = 0) {
31342
+ function copyThresholdReason(state, indexName, incomingCount2 = 0) {
30755
31343
  const held = documentCountOf(state, indexName);
30756
- if (incomingCount <= 0) {
31344
+ if (incomingCount2 <= 0) {
30757
31345
  return `Index "${indexName}" holds ${held} documents, reaching the copy threshold of ${state.copyThreshold}`;
30758
31346
  }
30759
- return `Index "${indexName}" holds ${held} documents, and the batch that the engine is writing adds ${incomingCount}, for ${held + incomingCount} against the copy threshold of ${state.copyThreshold}`;
31347
+ return `Index "${indexName}" holds ${held} documents, and the batch that the engine is writing adds ${incomingCount2}, for ${held + incomingCount2} against the copy threshold of ${state.copyThreshold}`;
30760
31348
  }
30761
- function indexReadyForCopies(state, indexName, incomingCount = 0) {
31349
+ function indexReadyForCopies(state, indexName, incomingCount2 = 0) {
30762
31350
  if (!copiesAllowed(state)) return false;
30763
31351
  if (state.scaledOutIndexes.has(indexName) || state.desyncedIndexes.has(indexName)) return false;
30764
31352
  if (state.copyTransitions.has(indexName)) return false;
30765
31353
  if (state.callbacks?.shouldDeferCopies?.()) return false;
30766
31354
  const entry = state.indexRegistry.get(indexName);
30767
31355
  if (entry === void 0 || state.executor.getManager(indexName) === void 0) return false;
30768
- if (documentCountOf(state, indexName) + incomingCount < state.copyThreshold) return false;
31356
+ if (documentCountOf(state, indexName) + incomingCount2 < state.copyThreshold) return false;
30769
31357
  const ineligibility = workerIneligibility(indexName, entry.config, state.bootstrapModule);
30770
31358
  if (ineligibility !== null) {
30771
31359
  reportIneligible(state, indexName, ineligibility);
@@ -30798,7 +31386,13 @@ async function loadCopies(state, indexName, reason) {
30798
31386
  state.copyLoadBuffers.set(indexName, buffered);
30799
31387
  try {
30800
31388
  freezeLiveTailsBeforeCopiesLoad(manager);
30801
- await transferIndexToPool(indexName, pool, entry.config, manager, state.callbacks?.isAnalysisStale?.(indexName));
31389
+ const copyLoad = await transferIndexToPool(
31390
+ indexName,
31391
+ pool,
31392
+ entry.config,
31393
+ manager,
31394
+ state.callbacks?.isAnalysisStale?.(indexName)
31395
+ );
30802
31396
  if (state.desyncedIndexes.has(indexName)) {
30803
31397
  discardLoadedCopies(state, pool, indexName);
30804
31398
  return;
@@ -30807,6 +31401,7 @@ async function loadCopies(state, indexName, reason) {
30807
31401
  state.lastAccessAt.set(indexName, Date.now());
30808
31402
  state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
30809
31403
  for (const action of buffered) enqueueReplication(state, indexName, action);
31404
+ markCopiesLoaded(state, indexName, copyLoad);
30810
31405
  } catch (err) {
30811
31406
  if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
30812
31407
  throw err;
@@ -30849,15 +31444,15 @@ async function scaleOutReadyIndexes(state) {
30849
31444
  }, 0);
30850
31445
  }
30851
31446
  }
30852
- async function scaleOutBeforeBatch(state, indexName, incomingCount) {
30853
- if (incomingCount <= 0) return;
31447
+ async function scaleOutBeforeBatch(state, indexName, incomingCount2) {
31448
+ if (incomingCount2 <= 0) return;
30854
31449
  const pending = state.copyTransitions.get(indexName);
30855
31450
  if (pending !== void 0) {
30856
31451
  if (pending.kind === "load") await pending.done;
30857
31452
  return;
30858
31453
  }
30859
- if (!indexReadyForCopies(state, indexName, incomingCount)) return;
30860
- await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName, incomingCount));
31454
+ if (!indexReadyForCopies(state, indexName, incomingCount2)) return;
31455
+ await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName, incomingCount2));
30861
31456
  }
30862
31457
 
30863
31458
  // src/engine/orchestration/idle.ts
@@ -31214,8 +31809,15 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
31214
31809
  const manager = executor.getManager(indexName);
31215
31810
  if (pool !== null && entry !== void 0 && manager !== void 0) {
31216
31811
  try {
31217
- await transferIndexToPool(indexName, pool, entry.config, manager, callbacks?.isAnalysisStale?.(indexName));
31812
+ const copyLoad = await transferIndexToPool(
31813
+ indexName,
31814
+ pool,
31815
+ entry.config,
31816
+ manager,
31817
+ callbacks?.isAnalysisStale?.(indexName)
31818
+ );
31218
31819
  state.scaledOutIndexes.add(indexName);
31820
+ markCopiesLoaded(state, indexName, copyLoad);
31219
31821
  refreshVectorCopies(state, indexName);
31220
31822
  } catch (repairError) {
31221
31823
  throw new NarsilError(
@@ -31280,7 +31882,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
31280
31882
  }
31281
31883
  return {
31282
31884
  scaleOutReadyIndexes: () => scaleOutReadyIndexes(state),
31283
- scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
31885
+ scaleOutBeforeBatch: (indexName, incomingCount2) => scaleOutBeforeBatch(state, indexName, incomingCount2),
31284
31886
  replicateToWorkers: replicate,
31285
31887
  awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
31286
31888
  awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
@@ -34229,7 +34831,7 @@ function dateOf(value, path) {
34229
34831
  refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
34230
34832
  return new Date(milliseconds);
34231
34833
  }
34232
- function bytesOf(value, path) {
34834
+ function bytesOf2(value, path) {
34233
34835
  const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
34234
34836
  if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
34235
34837
  return bytes;
@@ -34259,7 +34861,7 @@ function untagTag(tag, value, path, depth) {
34259
34861
  case DATE_TAG:
34260
34862
  return dateOf(value, path);
34261
34863
  case BYTES_TAG:
34262
- return bytesOf(value, path);
34864
+ return bytesOf2(value, path);
34263
34865
  case NUMBER_TAG:
34264
34866
  return numberOf(value, path);
34265
34867
  case OBJECT_TAG:
@@ -36602,15 +37204,15 @@ function oversampleOf(vectorConfig) {
36602
37204
  return oversample;
36603
37205
  }
36604
37206
  function vectorFetchDepth(limit, offset, cursorDepth) {
36605
- const reach = cursorDepth + offset + limit;
36606
- if (reach > RESULT_WINDOW) {
37207
+ const reach2 = cursorDepth + offset + limit;
37208
+ if (reach2 > RESULT_WINDOW) {
36607
37209
  throw new NarsilError(
36608
37210
  ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
36609
- `The engine pages a vector search through the first ${RESULT_WINDOW} results, and this page ends at ${reach}. Narrow the search with a filter or a similarity floor`,
37211
+ `The engine pages a vector search through the first ${RESULT_WINDOW} results, and this page ends at ${reach2}. Narrow the search with a filter or a similarity floor`,
36610
37212
  { limit, offset, cursorDepth, window: RESULT_WINDOW }
36611
37213
  );
36612
37214
  }
36613
- return Math.min(reach + 1, RESULT_WINDOW);
37215
+ return Math.min(reach2 + 1, RESULT_WINDOW);
36614
37216
  }
36615
37217
  async function executeVectorSearch(params, context, limit, offset, cursorDepth = 0) {
36616
37218
  const { manager, config, partitionIds } = context;