faiss 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/lib/faiss/version.rb +1 -1
  4. data/vendor/faiss/faiss/AutoTune.cpp +3 -1
  5. data/vendor/faiss/faiss/Clustering.cpp +9 -1
  6. data/vendor/faiss/faiss/IVFlib.cpp +14 -3
  7. data/vendor/faiss/faiss/Index.h +2 -2
  8. data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
  9. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
  10. data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
  11. data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
  12. data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
  13. data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
  14. data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
  15. data/vendor/faiss/faiss/IndexEDEN.h +57 -0
  16. data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
  17. data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
  18. data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
  19. data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
  20. data/vendor/faiss/faiss/IndexIDMap.h +2 -0
  21. data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
  22. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
  23. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
  24. data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
  25. data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
  26. data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
  27. data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
  28. data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
  29. data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
  30. data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
  31. data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
  32. data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
  33. data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
  34. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
  35. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
  36. data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
  37. data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
  38. data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
  39. data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
  40. data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
  41. data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
  42. data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
  43. data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
  44. data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
  45. data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
  46. data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
  47. data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
  48. data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
  49. data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
  50. data/vendor/faiss/faiss/IndexShards.cpp +2 -2
  51. data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
  52. data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
  53. data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
  54. data/vendor/faiss/faiss/SuperKMeans.h +30 -0
  55. data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
  56. data/vendor/faiss/faiss/clone_index.cpp +5 -0
  57. data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
  58. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
  59. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
  60. data/vendor/faiss/faiss/factory_tools.cpp +51 -4
  61. data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
  62. data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
  63. data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
  64. data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
  65. data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
  66. data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
  67. data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
  68. data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
  69. data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
  70. data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
  71. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
  72. data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
  73. data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
  74. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
  75. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
  76. data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
  77. data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
  78. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
  79. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
  80. data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
  81. data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
  82. data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
  83. data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
  84. data/vendor/faiss/faiss/impl/HNSW.h +51 -13
  85. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
  86. data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
  87. data/vendor/faiss/faiss/impl/Panorama.h +20 -7
  88. data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
  89. data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
  90. data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
  91. data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
  92. data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
  93. data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
  94. data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
  95. data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
  96. data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
  97. data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
  98. data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
  99. data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
  100. data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
  101. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
  102. data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
  103. data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
  104. data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
  105. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
  106. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
  107. data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
  108. data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
  109. data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
  110. data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
  111. data/vendor/faiss/faiss/impl/io_macros.h +25 -0
  112. data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
  113. data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
  114. data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
  115. data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
  116. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
  117. data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
  118. data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
  119. data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
  120. data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
  121. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
  122. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
  123. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
  124. data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
  125. data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
  126. data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
  127. data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
  128. data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
  129. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
  130. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
  131. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
  132. data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
  133. data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
  134. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
  135. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
  136. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
  137. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
  138. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
  139. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
  140. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
  141. data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
  142. data/vendor/faiss/faiss/index_factory.cpp +37 -7
  143. data/vendor/faiss/faiss/index_io.h +16 -0
  144. data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
  145. data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
  146. data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
  147. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
  148. data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
  149. data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
  150. data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
  151. data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
  152. data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
  153. data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
  154. data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
  155. data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
  156. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
  157. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
  158. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
  159. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
  160. data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
  161. data/vendor/faiss/faiss/utils/bf16.h +34 -0
  162. data/vendor/faiss/faiss/utils/distances.cpp +14 -2
  163. data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
  164. data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
  165. data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
  166. data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
  167. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
  168. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
  169. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
  170. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
  171. data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
  172. data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
  173. data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
  174. data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
  175. data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
  176. data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
  177. data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
  178. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
  179. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
  180. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
  181. data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
  182. data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
  183. data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
  184. data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
  185. data/vendor/faiss/faiss/utils/utils.cpp +9 -27
  186. metadata +27 -2
@@ -47,12 +47,17 @@ HNSWStats hnsw_stats;
47
47
 
48
48
  namespace {
49
49
 
50
+ // Returns the storage's native distance computer. For similarity metrics
51
+ // (e.g. METRIC_INNER_PRODUCT), distance values are real similarity scores
52
+ // (larger = better); HNSW handles the ordering via `hnsw.is_similarity`.
53
+ //
54
+ // NOTE: callers that drive the legacy max-heap-only code paths (notably
55
+ // `search_from_candidates_2` in the IndexHNSW2Level mixed search) cannot
56
+ // consume similarity scores directly; they assume smaller-is-better.
57
+ // Those paths only fire for the (default) L2 IndexHNSW2Level + Index2Layer
58
+ // configuration today, so passing the native DC is safe in practice.
50
59
  DistanceComputer* storage_distance_computer(const Index* storage) {
51
- if (is_similarity_metric(storage->metric_type)) {
52
- return new NegativeDistanceComputer(storage->get_distance_computer());
53
- } else {
54
- return storage->get_distance_computer();
55
- }
60
+ return storage->get_distance_computer();
56
61
  }
57
62
 
58
63
  void hnsw_add_vertices(
@@ -145,7 +150,8 @@ void hnsw_add_vertices(
145
150
 
146
151
  #pragma omp parallel if (i1 > i0 + 100)
147
152
  {
148
- VisitedTable vt(ntotal, hnsw.use_visited_hashset);
153
+ std::unique_ptr<VisitedTable> vt =
154
+ VisitedTable::create(ntotal, hnsw.use_visited_hashset);
149
155
 
150
156
  std::unique_ptr<DistanceComputer> dis(
151
157
  storage_distance_computer(index_hnsw.storage));
@@ -171,7 +177,7 @@ void hnsw_add_vertices(
171
177
  pt_level,
172
178
  pt_id,
173
179
  locks,
174
- vt,
180
+ *vt,
175
181
  index_hnsw.keep_max_size_level0 && (pt_level == 0));
176
182
 
177
183
  if (do_display && i - i0 > prev_display + 10000) {
@@ -213,13 +219,16 @@ void hnsw_add_vertices(
213
219
  **************************************************************/
214
220
 
215
221
  IndexHNSW::IndexHNSW(int d_in, int M, MetricType metric)
216
- : Index(d_in, metric), hnsw(M) {}
222
+ : Index(d_in, metric), hnsw(M) {
223
+ hnsw.is_similarity = is_similarity_metric(metric);
224
+ }
217
225
 
218
226
  IndexHNSW::IndexHNSW(Index* storage_in, int M)
219
227
  : Index(storage_in->d, storage_in->metric_type),
220
228
  hnsw(M),
221
229
  storage(storage_in) {
222
230
  metric_arg = storage->metric_arg;
231
+ hnsw.is_similarity = is_similarity_metric(metric_type);
223
232
  }
224
233
 
225
234
  IndexHNSW::~IndexHNSW() {
@@ -271,12 +280,12 @@ void hnsw_search(
271
280
 
272
281
  #pragma omp parallel if (i1 - i0 > 1)
273
282
  {
274
- std::unique_ptr<VisitedTable> vt;
283
+ VisitedTable* vt = nullptr;
275
284
  std::unique_ptr<typename BlockResultHandler::SingleResultHandler>
276
285
  res;
277
286
  std::unique_ptr<DistanceComputer> dis;
278
287
  try {
279
- vt = std::make_unique<VisitedTable>(
288
+ vt = &VisitedTable::get_reusable(
280
289
  index->ntotal, hnsw.use_visited_hashset);
281
290
  res = std::make_unique<
282
291
  typename BlockResultHandler::SingleResultHandler>(bres);
@@ -325,16 +334,14 @@ void IndexHNSW::search(
325
334
  const SearchParameters* params) const {
326
335
  FAISS_THROW_IF_NOT(k > 0);
327
336
 
328
- using RH = HeapBlockResultHandler<HNSW::C>;
329
- RH bres(n, distances, labels, k);
330
-
331
- hnsw_search(this, n, x, bres, params);
332
-
333
337
  if (is_similarity_metric(this->metric_type)) {
334
- // we need to revert the negated distances
335
- for (idx_t i = 0; i < k * n; i++) {
336
- distances[i] = -distances[i];
337
- }
338
+ using RH = HeapBlockResultHandler<HNSW::C_similarity>;
339
+ RH bres(n, distances, labels, k);
340
+ hnsw_search(this, n, x, bres, params);
341
+ } else {
342
+ using RH = HeapBlockResultHandler<HNSW::C_distance>;
343
+ RH bres(n, distances, labels, k);
344
+ hnsw_search(this, n, x, bres, params);
338
345
  }
339
346
  }
340
347
 
@@ -344,16 +351,14 @@ void IndexHNSW::range_search(
344
351
  float radius,
345
352
  RangeSearchResult* result,
346
353
  const SearchParameters* params) const {
347
- using RH = RangeSearchBlockResultHandler<HNSW::C>;
348
- RH bres(result, is_similarity_metric(metric_type) ? -radius : radius);
349
-
350
- hnsw_search(this, n, x, bres, params);
351
-
352
- if (is_similarity_metric(this->metric_type)) {
353
- // we need to revert the negated distances
354
- for (size_t i = 0; i < result->lims[result->nq]; i++) {
355
- result->distances[i] = -result->distances[i];
356
- }
354
+ if (is_similarity_metric(metric_type)) {
355
+ using RH = RangeSearchBlockResultHandler<HNSW::C_similarity>;
356
+ RH bres(result, radius);
357
+ hnsw_search(this, n, x, bres, params);
358
+ } else {
359
+ using RH = RangeSearchBlockResultHandler<HNSW::C_distance>;
360
+ RH bres(result, radius);
361
+ hnsw_search(this, n, x, bres, params);
357
362
  }
358
363
  }
359
364
 
@@ -361,8 +366,13 @@ void IndexHNSW::search1(
361
366
  const float* x,
362
367
  ResultHandler& handler,
363
368
  SearchParameters* params) const {
364
- SingleQueryBlockResultHandler<HNSW::C, false> bres(handler);
365
- hnsw_search(this, 1, x, bres, params);
369
+ if (is_similarity_metric(metric_type)) {
370
+ SingleQueryBlockResultHandler<HNSW::C_similarity, false> bres(handler);
371
+ hnsw_search(this, 1, x, bres, params);
372
+ } else {
373
+ SingleQueryBlockResultHandler<HNSW::C_distance, false> bres(handler);
374
+ hnsw_search(this, 1, x, bres, params);
375
+ }
366
376
  }
367
377
 
368
378
  void IndexHNSW::add(idx_t n, const float* x) {
@@ -459,63 +469,64 @@ void IndexHNSW::search_level_0(
459
469
 
460
470
  size_t hnsw_ntotal = hnsw.levels.size();
461
471
 
462
- using RH = HeapBlockResultHandler<HNSW::C>;
463
- RH bres(n, distances, labels, k);
472
+ auto run = [&]<class C>() {
473
+ using RH = HeapBlockResultHandler<C>;
474
+ RH bres(n, distances, labels, k);
464
475
 
465
- std::exception_ptr ex;
466
- std::atomic<bool> interrupt{false};
476
+ std::exception_ptr ex;
477
+ std::atomic<bool> interrupt{false};
467
478
  #pragma omp parallel
468
- {
469
- std::unique_ptr<DistanceComputer> qdis;
470
- HNSWStats search_stats;
471
- std::unique_ptr<VisitedTable> vt;
472
- std::unique_ptr<RH::SingleResultHandler> res;
473
- try {
474
- qdis.reset(storage_distance_computer(storage));
475
- vt = std::make_unique<VisitedTable>(
476
- hnsw_ntotal, hnsw.use_visited_hashset);
477
- res = std::make_unique<RH::SingleResultHandler>(bres);
478
- } catch (...) {
479
- omp_capture_exception(ex, [&] { interrupt = true; });
480
- }
481
-
482
- #pragma omp for
483
- for (idx_t i = 0; i < n; i++) {
484
- if (interrupt.load(std::memory_order_relaxed)) {
485
- continue;
486
- }
479
+ {
480
+ std::unique_ptr<DistanceComputer> qdis;
481
+ HNSWStats search_stats;
482
+ VisitedTable* vt = nullptr;
483
+ std::unique_ptr<typename RH::SingleResultHandler> res;
487
484
  try {
488
- res->begin(i);
489
- qdis->set_query(x + i * d);
490
-
491
- hnsw.search_level_0(
492
- *qdis.get(),
493
- *res,
494
- nprobe,
495
- nearest + i * nprobe,
496
- nearest_d + i * nprobe,
497
- search_type,
498
- search_stats,
499
- *vt,
500
- params);
501
- res->end();
502
- vt->advance();
485
+ qdis.reset(storage_distance_computer(storage));
486
+ vt = &VisitedTable::get_reusable(
487
+ hnsw_ntotal, hnsw.use_visited_hashset);
488
+ res = std::make_unique<typename RH::SingleResultHandler>(bres);
503
489
  } catch (...) {
504
490
  omp_capture_exception(ex, [&] { interrupt = true; });
505
491
  }
506
- }
492
+
493
+ #pragma omp for
494
+ for (idx_t i = 0; i < n; i++) {
495
+ if (interrupt.load(std::memory_order_relaxed)) {
496
+ continue;
497
+ }
498
+ try {
499
+ res->begin(i);
500
+ qdis->set_query(x + i * d);
501
+
502
+ hnsw.search_level_0(
503
+ *qdis.get(),
504
+ *res,
505
+ nprobe,
506
+ nearest + i * nprobe,
507
+ nearest_d + i * nprobe,
508
+ search_type,
509
+ search_stats,
510
+ *vt,
511
+ params);
512
+ res->end();
513
+ vt->advance();
514
+ } catch (...) {
515
+ omp_capture_exception(ex, [&] { interrupt = true; });
516
+ }
517
+ }
507
518
  #pragma omp critical
508
- {
509
- hnsw_stats.combine(search_stats);
519
+ {
520
+ hnsw_stats.combine(search_stats);
521
+ }
510
522
  }
511
- }
512
- omp_rethrow_if_exception(ex);
523
+ omp_rethrow_if_exception(ex);
524
+ };
525
+
513
526
  if (is_similarity_metric(this->metric_type)) {
514
- // we need to revert the negated distances
515
- #pragma omp parallel for
516
- for (int64_t i = 0; i < k * n; i++) {
517
- distances[i] = -distances[i];
518
- }
527
+ run.template operator()<HNSW::C_similarity>();
528
+ } else {
529
+ run.template operator()<HNSW::C_distance>();
519
530
  }
520
531
  }
521
532
 
@@ -569,7 +580,8 @@ void IndexHNSW::init_level_0_from_entry_points(
569
580
 
570
581
  #pragma omp parallel
571
582
  {
572
- VisitedTable vt(ntotal, hnsw.use_visited_hashset);
583
+ std::unique_ptr<VisitedTable> vt =
584
+ VisitedTable::create(ntotal, hnsw.use_visited_hashset);
573
585
 
574
586
  std::unique_ptr<DistanceComputer> dis(
575
587
  storage_distance_computer(storage));
@@ -583,7 +595,7 @@ void IndexHNSW::init_level_0_from_entry_points(
583
595
  dis->set_query(vec.data());
584
596
 
585
597
  hnsw.add_links_starting_from(
586
- *dis, pt_id, nearest, (*dis)(nearest), 0, locks, vt);
598
+ *dis, pt_id, nearest, (*dis)(nearest), 0, locks, *vt);
587
599
 
588
600
  if (verbose && i % 10000 == 0) {
589
601
  printf(" %d / %d\r", i, n);
@@ -824,7 +836,7 @@ int search_from_candidates_2(
824
836
  idx_t* I,
825
837
  float* D,
826
838
  MinimaxHeap& candidates,
827
- VisitedTable& vt,
839
+ VisitedTableVector& vt,
828
840
  HNSWStats& stats,
829
841
  int level,
830
842
  int nres_in = 0) {
@@ -891,8 +903,7 @@ void IndexHNSW2Level::search(
891
903
  idx_t* labels,
892
904
  const SearchParameters* params) const {
893
905
  FAISS_THROW_IF_NOT(k > 0);
894
- FAISS_THROW_IF_NOT_MSG(
895
- !params, "search params not supported for this index");
906
+ FAISS_THROW_IF_MSG(params, "search params not supported for this index");
896
907
 
897
908
  if (dynamic_cast<const Index2Layer*>(storage)) {
898
909
  IndexHNSW::search(n, x, k, distances, labels);
@@ -934,8 +945,7 @@ void IndexHNSW2Level::search(
934
945
  constexpr int candidates_size = 1;
935
946
  std::unique_ptr<MinimaxHeap> candidates;
936
947
  try {
937
- vt = std::make_unique<VisitedTable>(
938
- ntotal, /*use_hashset=*/false);
948
+ vt = VisitedTable::create(ntotal, /*use_hashset=*/false);
939
949
  dis.reset(storage_distance_computer(storage));
940
950
  candidates = std::make_unique<MinimaxHeap>(candidates_size);
941
951
  } catch (...) {
@@ -987,7 +997,7 @@ void IndexHNSW2Level::search(
987
997
  idxi,
988
998
  simi,
989
999
  *candidates,
990
- *vt,
1000
+ static_cast<VisitedTableVector&>(*vt),
991
1001
  search_stats,
992
1002
  0,
993
1003
  k);
@@ -1073,8 +1083,8 @@ IndexHNSWCagra::IndexHNSWCagra(
1073
1083
  }
1074
1084
 
1075
1085
  void IndexHNSWCagra::add(idx_t n, const float* x) {
1076
- FAISS_THROW_IF_NOT_MSG(
1077
- !base_level_only,
1086
+ FAISS_THROW_IF_MSG(
1087
+ base_level_only,
1078
1088
  "Cannot add vectors when base_level_only is set to True");
1079
1089
 
1080
1090
  IndexHNSW::add(n, x);
@@ -1101,28 +1111,40 @@ void IndexHNSWCagra::search(
1101
1111
  std::vector<storage_idx_t> nearest(n);
1102
1112
  std::vector<float> nearest_d(n);
1103
1113
 
1114
+ auto pick_entrypoints = [&]<class C>() {
1104
1115
  #pragma omp parallel for
1105
- for (idx_t i = 0; i < n; i++) {
1106
- std::unique_ptr<DistanceComputer> dis(
1107
- storage_distance_computer(this->storage));
1108
- dis->set_query(x + i * d);
1109
- nearest[i] = -1;
1110
- nearest_d[i] = std::numeric_limits<float>::max();
1111
-
1112
- std::random_device rd;
1113
- std::mt19937 gen(rd());
1114
- std::uniform_int_distribution<idx_t> distrib(0, this->ntotal - 1);
1115
-
1116
- for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
1117
- auto idx = distrib(gen);
1118
- auto distance = (*dis)(idx);
1119
- if (distance < nearest_d[i]) {
1120
- nearest[i] = static_cast<storage_idx_t>(idx);
1121
- nearest_d[i] = distance;
1116
+ for (idx_t i = 0; i < n; i++) {
1117
+ std::unique_ptr<DistanceComputer> dis(
1118
+ storage_distance_computer(this->storage));
1119
+ dis->set_query(x + i * d);
1120
+ nearest[i] = -1;
1121
+ // C::neutral() is the "worst possible" value: +inf for
1122
+ // CMax (distance) and -inf for CMin (similarity). The
1123
+ // first real candidate will always be strictly better.
1124
+ nearest_d[i] = C::neutral();
1125
+
1126
+ std::random_device rd;
1127
+ std::mt19937 gen(rd());
1128
+ std::uniform_int_distribution<idx_t> distrib(
1129
+ 0, this->ntotal - 1);
1130
+
1131
+ for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
1132
+ auto idx = distrib(gen);
1133
+ auto distance = (*dis)(idx);
1134
+ if (C::cmp(nearest_d[i], distance)) {
1135
+ nearest[i] = static_cast<storage_idx_t>(idx);
1136
+ nearest_d[i] = distance;
1137
+ }
1122
1138
  }
1139
+ FAISS_THROW_IF_NOT_MSG(
1140
+ nearest[i] >= 0, "Could not find a valid entrypoint.");
1123
1141
  }
1124
- FAISS_THROW_IF_NOT_MSG(
1125
- nearest[i] >= 0, "Could not find a valid entrypoint.");
1142
+ };
1143
+
1144
+ if (is_similarity_metric(metric_type)) {
1145
+ pick_entrypoints.template operator()<HNSW::C_similarity>();
1146
+ } else {
1147
+ pick_entrypoints.template operator()<HNSW::C_distance>();
1126
1148
  }
1127
1149
 
1128
1150
  search_level_0(
@@ -1150,56 +1172,63 @@ void IndexHNSWCagra::range_search(
1150
1172
  return;
1151
1173
  }
1152
1174
 
1153
- const HNSW& hnsw = this->hnsw;
1154
- size_t n1 = 0, n2 = 0, ndis = 0, nhops = 0;
1155
- float threshold = is_similarity_metric(metric_type) ? -radius : radius;
1156
- RangeSearchPartialResult pres(result);
1175
+ auto run = [&]<class C>() {
1176
+ const HNSW& hnsw = this->hnsw;
1177
+ size_t n1 = 0, n2 = 0, ndis = 0, nhops = 0;
1178
+ RangeSearchPartialResult pres(result);
1157
1179
 
1158
- for (idx_t i = 0; i < n; i++) {
1159
- std::unique_ptr<DistanceComputer> dis(
1160
- storage_distance_computer(storage));
1161
- dis->set_query(x + i * d);
1180
+ for (idx_t i = 0; i < n; i++) {
1181
+ std::unique_ptr<DistanceComputer> dis(
1182
+ storage_distance_computer(storage));
1183
+ dis->set_query(x + i * d);
1162
1184
 
1163
- storage_idx_t nearest = -1;
1164
- float nearest_d = std::numeric_limits<float>::max();
1185
+ storage_idx_t nearest = -1;
1186
+ // C::neutral() is the "worst possible" value under C: +inf for
1187
+ // CMax (distance) and -inf for CMin (similarity). The first
1188
+ // real candidate will always be strictly better.
1189
+ float nearest_d = C::neutral();
1165
1190
 
1166
- std::random_device rd;
1167
- std::mt19937 gen(rd());
1168
- std::uniform_int_distribution<idx_t> distrib(0, ntotal - 1);
1191
+ std::random_device rd;
1192
+ std::mt19937 gen(rd());
1193
+ std::uniform_int_distribution<idx_t> distrib(0, ntotal - 1);
1169
1194
 
1170
- for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
1171
- auto idx = distrib(gen);
1172
- auto distance = (*dis)(idx);
1173
- if (distance < nearest_d) {
1174
- nearest = idx;
1175
- nearest_d = distance;
1195
+ for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
1196
+ auto idx = distrib(gen);
1197
+ auto distance = (*dis)(idx);
1198
+ // C::cmp(nearest_d, distance) is true iff distance is
1199
+ // strictly better than the current nearest_d.
1200
+ if (C::cmp(nearest_d, distance)) {
1201
+ nearest = idx;
1202
+ nearest_d = distance;
1203
+ }
1176
1204
  }
1205
+ FAISS_THROW_IF_NOT_MSG(
1206
+ nearest >= 0, "Could not find a valid entrypoint.");
1207
+
1208
+ RangeQueryResult& qres = pres.new_result(i);
1209
+ RangeResultHandler<C> res(&qres, radius);
1210
+ VisitedTable& vt = VisitedTable::get_reusable(
1211
+ ntotal, hnsw.use_visited_hashset);
1212
+ HNSWStats stats;
1213
+ hnsw.search_level_0(
1214
+ *dis, res, 1, &nearest, &nearest_d, 1, stats, vt, params);
1215
+ n1 += stats.n1;
1216
+ n2 += stats.n2;
1217
+ ndis += stats.ndis;
1218
+ nhops += stats.nhops;
1177
1219
  }
1178
- FAISS_THROW_IF_NOT_MSG(
1179
- nearest >= 0, "Could not find a valid entrypoint.");
1180
-
1181
- RangeQueryResult& qres = pres.new_result(i);
1182
- RangeResultHandler<HNSW::C> res(&qres, threshold);
1183
- VisitedTable vt(ntotal, hnsw.use_visited_hashset);
1184
- HNSWStats stats;
1185
- hnsw.search_level_0(
1186
- *dis, res, 1, &nearest, &nearest_d, 1, stats, vt, params);
1187
- n1 += stats.n1;
1188
- n2 += stats.n2;
1189
- ndis += stats.ndis;
1190
- nhops += stats.nhops;
1191
- }
1192
1220
 
1193
- pres.set_lims();
1194
- result->do_allocation();
1195
- pres.copy_result();
1221
+ pres.set_lims();
1222
+ result->do_allocation();
1223
+ pres.copy_result();
1196
1224
 
1197
- hnsw_stats.combine({n1, n2, ndis, nhops});
1225
+ hnsw_stats.combine({n1, n2, ndis, nhops});
1226
+ };
1198
1227
 
1199
1228
  if (is_similarity_metric(metric_type)) {
1200
- for (size_t i = 0; i < result->lims[result->nq]; i++) {
1201
- result->distances[i] = -result->distances[i];
1202
- }
1229
+ run.template operator()<HNSW::C_similarity>();
1230
+ } else {
1231
+ run.template operator()<HNSW::C_distance>();
1203
1232
  }
1204
1233
  }
1205
1234
 
@@ -138,7 +138,8 @@ void IndexIDMapTemplate<IndexT>::add_sa_codes(
138
138
  idx_t n,
139
139
  const uint8_t* codes,
140
140
  const idx_t* xids) {
141
- index->add_sa_codes(n, codes, xids);
141
+ // don't pass the ids to the sub-index, they are kept in id_map
142
+ index->add_sa_codes(n, codes, nullptr);
142
143
  for (idx_t i = 0; i < n; i++) {
143
144
  id_map.push_back(xids[i]);
144
145
  }
@@ -228,14 +229,14 @@ void IndexIDMapTemplate<IndexT>::range_search(
228
229
  typename IndexT::distance_t radius,
229
230
  RangeSearchResult* result,
230
231
  const SearchParameters* params) const {
231
- if (params) {
232
+ if (params && params->sel) {
232
233
  SearchParameters internal_search_parameters;
233
234
  IDSelectorTranslated id_selector_translated(id_map, params->sel);
234
235
  internal_search_parameters.sel = &id_selector_translated;
235
236
 
236
237
  index->range_search(n, x, radius, result, &internal_search_parameters);
237
238
  } else {
238
- index->range_search(n, x, radius, result);
239
+ index->range_search(n, x, radius, result, params);
239
240
  }
240
241
 
241
242
  const idx_t id_map_size = static_cast<idx_t>(id_map.size());
@@ -329,6 +330,18 @@ void IndexIDMap2Template<IndexT>::add_with_ids(
329
330
  xids);
330
331
  }
331
332
 
333
+ template <typename IndexT>
334
+ void IndexIDMap2Template<IndexT>::add_sa_codes(
335
+ idx_t n,
336
+ const uint8_t* codes,
337
+ const idx_t* xids) {
338
+ idx_t prev_ntotal = this->ntotal;
339
+ IndexIDMapTemplate<IndexT>::add_sa_codes(n, codes, xids);
340
+ for (idx_t i = prev_ntotal; i < this->ntotal; i++) {
341
+ rev_map[this->id_map[i]] = i;
342
+ }
343
+ }
344
+
332
345
  template <typename IndexT>
333
346
  void IndexIDMap2Template<IndexT>::check_consistency() const {
334
347
  FAISS_THROW_IF_NOT(rev_map.size() == this->id_map.size());
@@ -110,6 +110,8 @@ struct IndexIDMap2Template : IndexIDMapTemplate<IndexT> {
110
110
  NumericType numeric_type,
111
111
  const idx_t* xids) override;
112
112
 
113
+ void add_sa_codes(idx_t n, const uint8_t* x, const idx_t* xids) override;
114
+
113
115
  size_t remove_ids(const IDSelector& sel) override;
114
116
 
115
117
  void reconstruct(idx_t key, component_t* recons) const override;
@@ -17,7 +17,6 @@
17
17
  #include <cstdio>
18
18
  #include <limits>
19
19
 
20
- #include <faiss/utils/hamming.h>
21
20
  #include <faiss/utils/utils.h>
22
21
 
23
22
  #include <faiss/IndexFlat.h>
@@ -501,7 +500,7 @@ void IndexIVF::search_preassigned(
501
500
  if (!do_heap_init) {
502
501
  return;
503
502
  }
504
- if (metric_type == METRIC_INNER_PRODUCT) {
503
+ if (is_similarity_metric(metric_type)) {
505
504
  heap_heapify<HeapForIP>(k, simi, idxi);
506
505
  } else {
507
506
  heap_heapify<HeapForL2>(k, simi, idxi);
@@ -512,7 +511,7 @@ void IndexIVF::search_preassigned(
512
511
  const idx_t* local_idx,
513
512
  float* simi,
514
513
  idx_t* idxi) {
515
- if (metric_type == METRIC_INNER_PRODUCT) {
514
+ if (is_similarity_metric(metric_type)) {
516
515
  heap_addn<HeapForIP>(
517
516
  k, simi, idxi, local_dis, local_idx, k);
518
517
  } else {
@@ -525,7 +524,7 @@ void IndexIVF::search_preassigned(
525
524
  if (!do_heap_init) {
526
525
  return;
527
526
  }
528
- if (metric_type == METRIC_INNER_PRODUCT) {
527
+ if (is_similarity_metric(metric_type)) {
529
528
  heap_reorder<HeapForIP>(k, simi, idxi);
530
529
  } else {
531
530
  heap_reorder<HeapForL2>(k, simi, idxi);
@@ -599,7 +598,7 @@ void IndexIVF::search_preassigned(
599
598
 
600
599
  size_t old_scan_cnt = 0;
601
600
  size_t old_heap_updates = 0;
602
- if (metric_type == METRIC_INNER_PRODUCT) {
601
+ if (is_similarity_metric(metric_type)) {
603
602
  HeapResultHandler<HeapForIP, false> handler(
604
603
  k, simi, idxi);
605
604
  old_scan_cnt = handler.stats.scan_cnt;
@@ -1056,7 +1055,19 @@ InvertedListScanner* IndexIVF::get_InvertedListScanner(
1056
1055
 
1057
1056
  void IndexIVF::reconstruct(idx_t key, float* recons) const {
1058
1057
  idx_t lo = direct_map.get(key);
1059
- reconstruct_from_offset(lo_listno(lo), lo_offset(lo), recons);
1058
+ const size_t list_no = lo_listno(lo);
1059
+ const size_t offset = lo_offset(lo);
1060
+ FAISS_THROW_IF_NOT_FMT(
1061
+ list_no < nlist,
1062
+ "IndexIVF::reconstruct: list_no %zd out of range (nlist=%zd)",
1063
+ list_no,
1064
+ nlist);
1065
+ FAISS_THROW_IF_NOT_FMT(
1066
+ offset < invlists->list_size(list_no),
1067
+ "IndexIVF::reconstruct: offset %zd out of range (list_size=%zd)",
1068
+ offset,
1069
+ invlists->list_size(list_no));
1070
+ reconstruct_from_offset(list_no, offset, recons);
1060
1071
  }
1061
1072
 
1062
1073
  void IndexIVF::reconstruct_n(idx_t i0, idx_t ni, float* recons) const {
@@ -1223,7 +1234,7 @@ void IndexIVF::search_and_return_codes(
1223
1234
  } else {
1224
1235
  size_t list_no = lo_listno(key);
1225
1236
  size_t offset = lo_offset(key);
1226
- const uint8_t* cc = invlists->get_single_code(list_no, offset);
1237
+ InvertedLists::ScopedCodes cc(invlists, list_no, offset);
1227
1238
 
1228
1239
  labels[ij] = invlists->get_single_id(list_no, offset);
1229
1240
 
@@ -1231,7 +1242,7 @@ void IndexIVF::search_and_return_codes(
1231
1242
  encode_listno(list_no, code1);
1232
1243
  code1 += code_size_1 - code_size;
1233
1244
  }
1234
- memcpy(code1, cc, code_size);
1245
+ memcpy(code1, cc.get(), code_size);
1235
1246
  }
1236
1247
  }
1237
1248
  }
@@ -145,8 +145,8 @@ void IndexIVFAdditiveQuantizer::reconstruct_from_offset(
145
145
  int64_t list_no,
146
146
  int64_t offset,
147
147
  float* recons) const {
148
- const uint8_t* code = invlists->get_single_code(list_no, offset);
149
- aq->decode(code, recons, 1);
148
+ InvertedLists::ScopedCodes sc(invlists, list_no, offset);
149
+ aq->decode(sc.get(), recons, 1);
150
150
  if (by_residual) {
151
151
  std::vector<float> centroid(d);
152
152
  quantizer->reconstruct(list_no, centroid.data());
@@ -283,7 +283,7 @@ InvertedListScanner* IndexIVFAdditiveQuantizer::get_InvertedListScanner(
283
283
  bool store_pairs,
284
284
  const IDSelector* sel,
285
285
  const IVFSearchParameters*) const {
286
- FAISS_THROW_IF_NOT(!sel);
286
+ FAISS_THROW_IF_MSG(sel, "id selector not supported for this index");
287
287
  if (metric_type == METRIC_INNER_PRODUCT) {
288
288
  if (aq->search_type == AdditiveQuantizer::ST_decompress) {
289
289
  return new AQInvertedListScannerDecompress<true>(
@@ -48,8 +48,8 @@ void IndexIVFAdditiveQuantizerFastScan::init(
48
48
  MetricType metric,
49
49
  int bbs_,
50
50
  bool own_invlists_) {
51
- FAISS_THROW_IF_NOT(aq_ != nullptr);
52
- FAISS_THROW_IF_NOT(!aq_->nbits.empty());
51
+ FAISS_THROW_IF_NOT(aq_);
52
+ FAISS_THROW_IF_MSG(aq_->nbits.empty(), "quantizer nbits must not be empty");
53
53
  FAISS_THROW_IF_NOT(aq_->nbits[0] == 4);
54
54
  if (metric == METRIC_INNER_PRODUCT) {
55
55
  FAISS_THROW_IF_NOT_MSG(
@@ -310,8 +310,7 @@ void IndexIVFAdditiveQuantizerFastScan::search(
310
310
  float* distances,
311
311
  idx_t* labels,
312
312
  const SearchParameters* params) const {
313
- FAISS_THROW_IF_NOT_MSG(
314
- !params, "search params not supported for this index");
313
+ FAISS_THROW_IF_MSG(params, "search params not supported for this index");
315
314
 
316
315
  FAISS_THROW_IF_NOT(k > 0);
317
316
  bool rescale = (rescale_norm && norm_scale > 1 && metric_type == METRIC_L2);