faiss 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/lib/faiss/version.rb +1 -1
  4. data/vendor/faiss/faiss/AutoTune.cpp +3 -1
  5. data/vendor/faiss/faiss/Clustering.cpp +9 -1
  6. data/vendor/faiss/faiss/IVFlib.cpp +14 -3
  7. data/vendor/faiss/faiss/Index.h +2 -2
  8. data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
  9. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
  10. data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
  11. data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
  12. data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
  13. data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
  14. data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
  15. data/vendor/faiss/faiss/IndexEDEN.h +57 -0
  16. data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
  17. data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
  18. data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
  19. data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
  20. data/vendor/faiss/faiss/IndexIDMap.h +2 -0
  21. data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
  22. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
  23. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
  24. data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
  25. data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
  26. data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
  27. data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
  28. data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
  29. data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
  30. data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
  31. data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
  32. data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
  33. data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
  34. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
  35. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
  36. data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
  37. data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
  38. data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
  39. data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
  40. data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
  41. data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
  42. data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
  43. data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
  44. data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
  45. data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
  46. data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
  47. data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
  48. data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
  49. data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
  50. data/vendor/faiss/faiss/IndexShards.cpp +2 -2
  51. data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
  52. data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
  53. data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
  54. data/vendor/faiss/faiss/SuperKMeans.h +30 -0
  55. data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
  56. data/vendor/faiss/faiss/clone_index.cpp +5 -0
  57. data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
  58. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
  59. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
  60. data/vendor/faiss/faiss/factory_tools.cpp +51 -4
  61. data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
  62. data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
  63. data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
  64. data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
  65. data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
  66. data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
  67. data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
  68. data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
  69. data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
  70. data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
  71. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
  72. data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
  73. data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
  74. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
  75. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
  76. data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
  77. data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
  78. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
  79. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
  80. data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
  81. data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
  82. data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
  83. data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
  84. data/vendor/faiss/faiss/impl/HNSW.h +51 -13
  85. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
  86. data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
  87. data/vendor/faiss/faiss/impl/Panorama.h +20 -7
  88. data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
  89. data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
  90. data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
  91. data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
  92. data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
  93. data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
  94. data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
  95. data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
  96. data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
  97. data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
  98. data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
  99. data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
  100. data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
  101. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
  102. data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
  103. data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
  104. data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
  105. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
  106. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
  107. data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
  108. data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
  109. data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
  110. data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
  111. data/vendor/faiss/faiss/impl/io_macros.h +25 -0
  112. data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
  113. data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
  114. data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
  115. data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
  116. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
  117. data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
  118. data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
  119. data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
  120. data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
  121. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
  122. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
  123. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
  124. data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
  125. data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
  126. data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
  127. data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
  128. data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
  129. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
  130. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
  131. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
  132. data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
  133. data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
  134. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
  135. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
  136. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
  137. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
  138. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
  139. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
  140. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
  141. data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
  142. data/vendor/faiss/faiss/index_factory.cpp +37 -7
  143. data/vendor/faiss/faiss/index_io.h +16 -0
  144. data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
  145. data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
  146. data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
  147. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
  148. data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
  149. data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
  150. data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
  151. data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
  152. data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
  153. data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
  154. data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
  155. data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
  156. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
  157. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
  158. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
  159. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
  160. data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
  161. data/vendor/faiss/faiss/utils/bf16.h +34 -0
  162. data/vendor/faiss/faiss/utils/distances.cpp +14 -2
  163. data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
  164. data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
  165. data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
  166. data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
  167. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
  168. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
  169. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
  170. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
  171. data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
  172. data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
  173. data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
  174. data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
  175. data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
  176. data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
  177. data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
  178. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
  179. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
  180. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
  181. data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
  182. data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
  183. data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
  184. data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
  185. data/vendor/faiss/faiss/utils/utils.cpp +9 -27
  186. metadata +27 -2
@@ -20,23 +20,39 @@ namespace faiss {
20
20
  namespace pq_code_distance {
21
21
 
22
22
  template <class PQCodeDist>
23
- struct PQDistanceComputer : FlatCodesDistanceComputer {
23
+ struct PQDistanceComputer final : FlatCodesDistanceComputer {
24
24
  using PQDecoder = typename PQCodeDist::PQDecoder;
25
- size_t d;
26
25
  MetricType metric;
27
- idx_t nb;
28
26
  const ProductQuantizer& pq;
29
27
  const float* sdc;
30
28
  std::vector<float> precomputed_table;
31
- size_t ndis;
32
- const float* q;
33
29
 
34
30
  float distance_to_code(const uint8_t* code) final {
35
- ndis++;
36
-
37
- float dis = PQCodeDist::distance_single_code(
31
+ return PQCodeDist::distance_single_code(
38
32
  pq.M, pq.nbits, precomputed_table.data(), code);
39
- return dis;
33
+ }
34
+
35
+ void distance_to_code_batch_4(
36
+ const uint8_t* c1,
37
+ const uint8_t* c2,
38
+ const uint8_t* c3,
39
+ const uint8_t* c4,
40
+ float& d1,
41
+ float& d2,
42
+ float& d3,
43
+ float& d4) override {
44
+ PQCodeDist::distance_four_codes(
45
+ pq.M,
46
+ pq.nbits,
47
+ precomputed_table.data(),
48
+ c1,
49
+ c2,
50
+ c3,
51
+ c4,
52
+ d1,
53
+ d2,
54
+ d3,
55
+ d4);
40
56
  }
41
57
 
42
58
  float symmetric_dis(idx_t i, idx_t j) override {
@@ -50,7 +66,6 @@ struct PQDistanceComputer : FlatCodesDistanceComputer {
50
66
  accu += sdci[codei.decode() + (codej.decode() << codei.nbits)];
51
67
  sdci += uint64_t(1) << (2 * codei.nbits);
52
68
  }
53
- ndis++;
54
69
  return accu;
55
70
  }
56
71
 
@@ -58,18 +73,14 @@ struct PQDistanceComputer : FlatCodesDistanceComputer {
58
73
  : FlatCodesDistanceComputer(
59
74
  storage.codes.data(),
60
75
  storage.code_size),
61
- pq(storage.pq),
62
- q(nullptr) {
76
+ pq(storage.pq) {
63
77
  precomputed_table.resize(pq.M * pq.ksub);
64
- nb = storage.ntotal;
65
- d = storage.d;
66
78
  metric = storage.metric_type;
67
79
  if (pq.sdc_table.size() == pq.ksub * pq.ksub * pq.M) {
68
80
  sdc = pq.sdc_table.data();
69
81
  } else {
70
82
  sdc = nullptr;
71
83
  }
72
- ndis = 0;
73
84
  }
74
85
 
75
86
  void set_query(const float* x) override {
@@ -9,13 +9,15 @@
9
9
 
10
10
  #define THE_SIMD_LEVEL SIMDLevel::AVX2
11
11
 
12
- // NOLINTNEXTLINE(facebook-hte-InlineHeader)
12
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
13
13
  #include <faiss/impl/pq_code_distance/pq_code_distance-avx2.h>
14
- // NOLINTNEXTLINE(facebook-hte-InlineHeader)
14
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
15
15
  #include <faiss/utils/hamming_distance/hamming_computer-avx2.h>
16
- // NOLINTNEXTLINE(facebook-hte-InlineHeader)
16
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
17
+ #include <faiss/impl/pq_code_distance/pq_scan_impl.h>
18
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
17
19
  #include <faiss/impl/pq_code_distance/PQDistanceComputer_impl.h>
18
- // NOLINTNEXTLINE(facebook-hte-InlineHeader)
20
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
19
21
  #include <faiss/impl/pq_code_distance/IVFPQScanner_impl.h>
20
22
 
21
23
  #endif // COMPILE_SIMD_AVX2
@@ -14,6 +14,8 @@
14
14
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
15
15
  #include <faiss/utils/hamming_distance/hamming_computer-avx512.h>
16
16
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
17
+ #include <faiss/impl/pq_code_distance/pq_scan_impl.h>
18
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader)
17
19
  #include <faiss/impl/pq_code_distance/PQDistanceComputer_impl.h>
18
20
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
19
21
  #include <faiss/impl/pq_code_distance/IVFPQScanner_impl.h>
@@ -14,6 +14,8 @@
14
14
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
15
15
  #include <faiss/utils/hamming_distance/hamming_computer-neon.h>
16
16
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
17
+ #include <faiss/impl/pq_code_distance/pq_scan_impl.h>
18
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader)
17
19
  #include <faiss/impl/pq_code_distance/PQDistanceComputer_impl.h>
18
20
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
19
21
  #include <faiss/impl/pq_code_distance/IVFPQScanner_impl.h>
@@ -17,9 +17,29 @@
17
17
 
18
18
  #include <faiss/impl/pq_code_distance/pq_code_distance-generic.h>
19
19
 
20
+ #define THE_SIMD_LEVEL SIMDLevel::NONE
21
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader)
22
+ #include <faiss/impl/pq_code_distance/pq_scan_impl.h>
23
+ #undef THE_SIMD_LEVEL
24
+
20
25
  namespace faiss {
21
26
  namespace pq_code_distance {
22
27
 
28
+ void pq_scan_8bit(
29
+ size_t M,
30
+ const float* dis_table,
31
+ const uint8_t* codes,
32
+ size_t ncodes,
33
+ size_t k,
34
+ float* heap_dis,
35
+ int64_t* heap_ids,
36
+ bool max_heap) {
37
+ with_simd_level([&]<SIMDLevel SL>() {
38
+ pq_scan_8bit_impl<SL>(
39
+ M, dis_table, codes, ncodes, k, heap_dis, heap_ids, max_heap);
40
+ });
41
+ }
42
+
23
43
  float pq_code_distance_8bit_single(
24
44
  size_t M,
25
45
  const float* sim_table,
@@ -245,11 +245,47 @@ FAISS_API void pq_code_distance_8bit_four(
245
245
  float& result2,
246
246
  float& result3);
247
247
 
248
+ /*********************************************************************
249
+ * Standalone PQ scan — SIMD-dispatched full-index scan.
250
+ *
251
+ * Scans all ncodes PQ codes against a precomputed distance table,
252
+ * maintaining a k-nearest-neighbor heap. Uses the SIMD PQ distance
253
+ * kernels (AVX2 gathers, etc.) for the inner loop, with the SIMD
254
+ * gathers inlined into the scan loop in each per-SIMD TU.
255
+ *
256
+ * Definitions are in pq_scan_impl.h (per-SIMD TUs) and
257
+ * pq_code_distance-generic.cpp (dispatch wrapper).
258
+ *********************************************************************/
259
+
260
+ template <SIMDLevel SL>
261
+ void pq_scan_8bit_impl(
262
+ size_t M,
263
+ const float* dis_table,
264
+ const uint8_t* codes,
265
+ size_t ncodes,
266
+ size_t k,
267
+ float* heap_dis,
268
+ int64_t* heap_ids,
269
+ bool max_heap);
270
+
271
+ /// Scan all ncodes 8-bit PQ codes, dispatching to the best SIMD level.
272
+ /// max_heap=true for L2 (CMax), false for IP (CMin).
273
+ FAISS_API void pq_scan_8bit(
274
+ size_t M,
275
+ const float* dis_table,
276
+ const uint8_t* codes,
277
+ size_t ncodes,
278
+ size_t k,
279
+ float* heap_dis,
280
+ int64_t* heap_ids,
281
+ bool max_heap);
282
+
248
283
  } // namespace pq_code_distance
249
284
 
250
285
  // Re-export public API into namespace faiss for convenience
251
286
  using pq_code_distance::pq_code_distance_8bit_four;
252
287
  using pq_code_distance::pq_code_distance_8bit_single;
288
+ using pq_code_distance::pq_scan_8bit;
253
289
  using pq_code_distance::PQCodeDistance;
254
290
  using pq_code_distance::PQCodeDistanceScalar;
255
291
 
@@ -347,4 +347,9 @@ void pq_code_distance_8bit_four_impl<SIMDLevel::ARM_SVE>(
347
347
  } // namespace pq_code_distance
348
348
  } // namespace faiss
349
349
 
350
+ #define THE_SIMD_LEVEL SIMDLevel::ARM_SVE
351
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader)
352
+ #include <faiss/impl/pq_code_distance/pq_scan_impl.h>
353
+ #undef THE_SIMD_LEVEL
354
+
350
355
  #endif // COMPILE_SIMD_ARM_SVE
@@ -0,0 +1,105 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ #pragma once
9
+
10
+ #ifndef THE_SIMD_LEVEL
11
+ #error "THE_SIMD_LEVEL must be defined before including pq_scan_impl.h"
12
+ #endif
13
+
14
+ #include <faiss/impl/pq_code_distance/pq_code_distance-inl.h>
15
+ #include <faiss/utils/Heap.h>
16
+
17
+ namespace faiss {
18
+ namespace pq_code_distance {
19
+
20
+ namespace {
21
+
22
+ template <class C>
23
+ void pq_scan_8bit_at_level(
24
+ size_t M,
25
+ const float* dis_table,
26
+ const uint8_t* codes,
27
+ size_t ncodes,
28
+ size_t k,
29
+ float* heap_dis,
30
+ int64_t* heap_ids) {
31
+ const size_t code_size = M;
32
+
33
+ size_t j = 0;
34
+ for (; j + 3 < ncodes; j += 4) {
35
+ float d0, d1, d2, d3;
36
+ pq_code_distance_8bit_four_impl<THE_SIMD_LEVEL>( // NOLINT(facebook-modularize-issue-check)
37
+ M,
38
+ dis_table,
39
+ codes + j * code_size,
40
+ codes + (j + 1) * code_size,
41
+ codes + (j + 2) * code_size,
42
+ codes + (j + 3) * code_size,
43
+ d0,
44
+ d1,
45
+ d2,
46
+ d3);
47
+
48
+ if (C::cmp(heap_dis[0], d0)) {
49
+ heap_replace_top<C>(k, heap_dis, heap_ids, d0, j);
50
+ }
51
+ if (C::cmp(heap_dis[0], d1)) {
52
+ heap_replace_top<C>(k, heap_dis, heap_ids, d1, j + 1);
53
+ }
54
+ if (C::cmp(heap_dis[0], d2)) {
55
+ heap_replace_top<C>(k, heap_dis, heap_ids, d2, j + 2);
56
+ }
57
+ if (C::cmp(heap_dis[0], d3)) {
58
+ heap_replace_top<C>(k, heap_dis, heap_ids, d3, j + 3);
59
+ }
60
+ }
61
+
62
+ for (; j < ncodes; j++) {
63
+ float dis = pq_code_distance_8bit_single_impl<THE_SIMD_LEVEL>( // NOLINT(facebook-modularize-issue-check)
64
+ M, dis_table, codes + j * code_size);
65
+ if (C::cmp(heap_dis[0], dis)) {
66
+ heap_replace_top<C>(k, heap_dis, heap_ids, dis, j);
67
+ }
68
+ }
69
+ }
70
+
71
+ } // anonymous namespace
72
+
73
+ template <SIMDLevel SL>
74
+ void pq_scan_8bit_impl(
75
+ size_t M,
76
+ const float* dis_table,
77
+ const uint8_t* codes,
78
+ size_t ncodes,
79
+ size_t k,
80
+ float* heap_dis,
81
+ int64_t* heap_ids,
82
+ bool max_heap);
83
+
84
+ // NOLINTNEXTLINE(facebook-hte-MisplacedTemplateSpecialization,facebook-modularize-issue-check)
85
+ template <>
86
+ void pq_scan_8bit_impl<THE_SIMD_LEVEL>(
87
+ size_t M,
88
+ const float* dis_table,
89
+ const uint8_t* codes,
90
+ size_t ncodes,
91
+ size_t k,
92
+ float* heap_dis,
93
+ int64_t* heap_ids,
94
+ bool max_heap) {
95
+ if (max_heap) {
96
+ pq_scan_8bit_at_level<CMax<float, int64_t>>(
97
+ M, dis_table, codes, ncodes, k, heap_dis, heap_ids);
98
+ } else {
99
+ pq_scan_8bit_at_level<CMin<float, int64_t>>(
100
+ M, dis_table, codes, ncodes, k, heap_dis, heap_ids);
101
+ }
102
+ }
103
+
104
+ } // namespace pq_code_distance
105
+ } // namespace faiss
@@ -56,6 +56,8 @@ void pq_code_distance_8bit_four_impl<SIMDLevel::RISCV_RVV>(
56
56
 
57
57
  #define THE_SIMD_LEVEL SIMDLevel::RISCV_RVV
58
58
 
59
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader)
60
+ #include <faiss/impl/pq_code_distance/pq_scan_impl.h>
59
61
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
60
62
  #include <faiss/utils/hamming_distance/hamming_computer-rvv.h>
61
63
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
@@ -0,0 +1,195 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ // Generic (NONE) implementations of Top1 and Reservoir add_results, plus the
9
+ // runtime-dispatch method bodies. SIMD specialisations live in
10
+ // result_handler_avx2.cpp and result_handler_avx512.cpp.
11
+
12
+ #include <faiss/impl/ResultHandler.h>
13
+ #include <faiss/impl/simd_dispatch.h>
14
+
15
+ namespace faiss {
16
+
17
+ // ----------------------------------------------------------------
18
+ // SIMD-level masks
19
+ // ----------------------------------------------------------------
20
+
21
+ // Top-1: scalar fallback + AVX2 (8-wide) + AVX512 (16-wide).
22
+ constexpr int TOP1_SIMD_LEVELS = (1 << int(SIMDLevel::NONE)) |
23
+ (1 << int(SIMDLevel::AVX2)) | (1 << int(SIMDLevel::AVX512));
24
+
25
+ // Reservoir: scalar fallback + AVX512 compress path.
26
+ // VPCOMPRESSPS/VPCOMPRESSD require AVX512F so there is no AVX2 path.
27
+ // On non-AVX512 hosts the dispatch falls back to NONE automatically.
28
+ constexpr int RESERVOIR_SIMD_LEVELS =
29
+ (1 << int(SIMDLevel::NONE)) | (1 << int(SIMDLevel::AVX512));
30
+
31
+ // ----------------------------------------------------------------
32
+ // Scalar (NONE) helper implementations
33
+ // ----------------------------------------------------------------
34
+
35
+ namespace {
36
+
37
+ template <class C, bool use_sel>
38
+ void top1_add_results_none(
39
+ Top1BlockResultHandler<C, use_sel>* self,
40
+ size_t j0,
41
+ size_t j1,
42
+ const float* dis_tab_in) {
43
+ using T = typename C::T;
44
+ using TI = typename C::TI;
45
+
46
+ for (size_t qi = self->i0; qi < self->i1; qi++) {
47
+ const T* dis_tab_i = dis_tab_in + (j1 - j0) * (qi - self->i0) - j0;
48
+
49
+ // Hoist best_dis / best_idx into locals so the compiler keeps them in
50
+ // registers across the inner loop (no aliasing with dis_tab reads).
51
+ T best_dis = self->dis_tab[qi];
52
+ TI best_idx = self->ids_tab[qi];
53
+
54
+ for (size_t j = j0; j < j1; j++) {
55
+ if (C::cmp(best_dis, dis_tab_i[j])) {
56
+ best_dis = dis_tab_i[j];
57
+ best_idx = (TI)j;
58
+ }
59
+ }
60
+
61
+ self->dis_tab[qi] = best_dis;
62
+ self->ids_tab[qi] = best_idx;
63
+ }
64
+ }
65
+
66
+ template <class C, bool use_sel>
67
+ void reservoir_add_results_none(
68
+ ReservoirBlockResultHandler<C, use_sel>* self,
69
+ size_t j0,
70
+ size_t j1,
71
+ const float* dis_in) {
72
+ using T = typename C::T;
73
+ using TI = typename C::TI;
74
+
75
+ #pragma omp parallel for
76
+ for (int64_t qi = (int64_t)self->i0; qi < (int64_t)self->i1; qi++) {
77
+ ReservoirTopN<C>& res = self->reservoirs[qi - (int64_t)self->i0];
78
+ const T* dis_tab_i = dis_in + (j1 - j0) * (qi - (int64_t)self->i0) - j0;
79
+
80
+ // Hoist res.i and res.threshold into locals so the compiler keeps
81
+ // them in registers.
82
+ size_t ri = res.i;
83
+ T thresh = res.threshold;
84
+
85
+ for (size_t j = j0; j < j1; j++) {
86
+ T dis = dis_tab_i[j];
87
+ if (C::cmp(thresh, dis)) {
88
+ res.vals[ri] = dis;
89
+ res.ids[ri] = (TI)j;
90
+ ri++;
91
+ if (ri >= res.capacity) {
92
+ res.i = ri;
93
+ res.shrink_fuzzy();
94
+ ri = res.i;
95
+ thresh = res.threshold;
96
+ }
97
+ }
98
+ }
99
+ res.i = ri;
100
+ }
101
+ }
102
+
103
+ } // namespace
104
+
105
+ // ----------------------------------------------------------------
106
+ // SIMDLevel::NONE explicit specialisations
107
+ // ----------------------------------------------------------------
108
+
109
+ // Instantiate top1_add_results_tpl<C, use_sel, SIMDLevel::NONE> and
110
+ // reservoir_add_results_tpl<C, use_sel, SIMDLevel::NONE> for all
111
+ // (C, use_sel) combinations that the rest of FAISS uses.
112
+ #define INSTANTIATE_NONE(C, use_sel) \
113
+ template <> \
114
+ void top1_add_results_tpl<C, use_sel, SIMDLevel::NONE>( \
115
+ Top1BlockResultHandler<C, use_sel> * self, \
116
+ size_t j0, \
117
+ size_t j1, \
118
+ const float* dis_tab) { \
119
+ top1_add_results_none<C, use_sel>(self, j0, j1, dis_tab); \
120
+ } \
121
+ template <> \
122
+ void reservoir_add_results_tpl<C, use_sel, SIMDLevel::NONE>( \
123
+ ReservoirBlockResultHandler<C, use_sel> * self, \
124
+ size_t j0, \
125
+ size_t j1, \
126
+ const float* dis_in) { \
127
+ reservoir_add_results_none<C, use_sel>(self, j0, j1, dis_in); \
128
+ }
129
+
130
+ // Type aliases so the comma in CMax<float, int64_t> doesn't split macro args.
131
+ using CMaxFI = CMax<float, int64_t>;
132
+ using CMinFI = CMin<float, int64_t>;
133
+
134
+ INSTANTIATE_NONE(CMaxFI, false)
135
+ INSTANTIATE_NONE(CMaxFI, true)
136
+ INSTANTIATE_NONE(CMinFI, false)
137
+ INSTANTIATE_NONE(CMinFI, true)
138
+
139
+ #undef INSTANTIATE_NONE
140
+
141
+ // ----------------------------------------------------------------
142
+ // add_results method definitions — dispatch to the right SL kernel
143
+ // ----------------------------------------------------------------
144
+
145
+ template <class C, bool use_sel>
146
+ void Top1BlockResultHandler<C, use_sel>::add_results(
147
+ size_t j0,
148
+ size_t j1,
149
+ const T* dis_tab_2) {
150
+ with_selected_simd_levels<TOP1_SIMD_LEVELS>([&]<SIMDLevel SL>() {
151
+ top1_add_results_tpl<C, use_sel, SL>(this, j0, j1, dis_tab_2);
152
+ });
153
+ }
154
+
155
+ template <class C, bool use_sel>
156
+ void ReservoirBlockResultHandler<C, use_sel>::add_results(
157
+ size_t j0,
158
+ size_t j1,
159
+ const T* dis_in) {
160
+ with_selected_simd_levels<RESERVOIR_SIMD_LEVELS>([&]<SIMDLevel SL>() {
161
+ reservoir_add_results_tpl<C, use_sel, SL>(this, j0, j1, dis_in);
162
+ });
163
+ }
164
+
165
+ // ----------------------------------------------------------------
166
+ // Explicit class-template instantiations (force linkage)
167
+ // ----------------------------------------------------------------
168
+
169
+ template void Top1BlockResultHandler<CMax<float, int64_t>, false>::add_results(
170
+ size_t,
171
+ size_t,
172
+ const float*);
173
+ template void Top1BlockResultHandler<CMax<float, int64_t>, true>::add_results(
174
+ size_t,
175
+ size_t,
176
+ const float*);
177
+ template void Top1BlockResultHandler<CMin<float, int64_t>, false>::add_results(
178
+ size_t,
179
+ size_t,
180
+ const float*);
181
+ template void Top1BlockResultHandler<CMin<float, int64_t>, true>::add_results(
182
+ size_t,
183
+ size_t,
184
+ const float*);
185
+
186
+ template void ReservoirBlockResultHandler<CMax<float, int64_t>, false>::
187
+ add_results(size_t, size_t, const float*);
188
+ template void ReservoirBlockResultHandler<CMax<float, int64_t>, true>::
189
+ add_results(size_t, size_t, const float*);
190
+ template void ReservoirBlockResultHandler<CMin<float, int64_t>, false>::
191
+ add_results(size_t, size_t, const float*);
192
+ template void ReservoirBlockResultHandler<CMin<float, int64_t>, true>::
193
+ add_results(size_t, size_t, const float*);
194
+
195
+ } // namespace faiss
@@ -0,0 +1,133 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ // AVX2 specialisation of Top1 add_results (8-wide branchless argmin/argmax).
9
+ // Reservoir stays on the NONE path — VPCOMPRESSPS requires AVX512F.
10
+
11
+ #ifdef COMPILE_SIMD_AVX2
12
+
13
+ #include <faiss/impl/ResultHandler.h>
14
+
15
+ #include <immintrin.h>
16
+ #include <type_traits>
17
+
18
+ namespace faiss {
19
+
20
+ namespace {
21
+
22
+ /// Templated AVX2 implementation of Top1 add_results for both CMax (keeps the
23
+ /// smallest distance) and CMin (keeps the largest similarity).
24
+ template <class C, bool use_sel>
25
+ void top1_add_results_avx2(
26
+ Top1BlockResultHandler<C, use_sel>* self,
27
+ size_t j0,
28
+ size_t j1,
29
+ const float* dis_tab_in) {
30
+ static_assert(
31
+ std::is_same<typename C::T, float>::value,
32
+ "This code expects float distances");
33
+ using TI = typename C::TI;
34
+ const __m256i vstep = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
35
+
36
+ for (size_t qi = self->i0; qi < self->i1; qi++) {
37
+ const float* dis_tab_i = dis_tab_in + (j1 - j0) * (qi - self->i0) - j0;
38
+
39
+ // Hoist best_dis / best_idx into locals so the compiler keeps them in
40
+ // registers across the inner loop (no aliasing with dis_tab reads).
41
+ float best_dis = self->dis_tab[qi];
42
+ TI best_idx = self->ids_tab[qi];
43
+ size_t j = j0;
44
+
45
+ __m256 vbest = _mm256_set1_ps(best_dis);
46
+ __m256i vbest_idx = _mm256_set1_epi32((int32_t)best_idx);
47
+
48
+ for (; j + 8 <= j1; j += 8) {
49
+ __m256 vdis = _mm256_loadu_ps(dis_tab_i + j);
50
+ __m256i vidx =
51
+ _mm256_add_epi32(_mm256_set1_epi32((int32_t)j), vstep);
52
+
53
+ // CMax (L2 nearest neighbour): keep lane if dis < best.
54
+ // CMin (inner product): keep lane if dis > best.
55
+ __m256 mask;
56
+ if constexpr (C::is_max) {
57
+ mask = _mm256_cmp_ps(vdis, vbest, _CMP_LT_OS);
58
+ } else {
59
+ mask = _mm256_cmp_ps(vdis, vbest, _CMP_GT_OS);
60
+ }
61
+ vbest = _mm256_blendv_ps(vbest, vdis, mask);
62
+ vbest_idx = _mm256_blendv_epi8(
63
+ vbest_idx, vidx, _mm256_castps_si256(mask));
64
+ }
65
+
66
+ // Horizontal reduction across 8 lanes.
67
+ alignas(32) float best_arr[8];
68
+ alignas(32) int32_t idx_arr[8];
69
+ _mm256_store_ps(best_arr, vbest);
70
+ _mm256_store_si256((__m256i*)idx_arr, vbest_idx);
71
+ for (int k = 0; k < 8; k++) {
72
+ if (C::cmp(best_dis, best_arr[k])) {
73
+ best_dis = best_arr[k];
74
+ best_idx = (TI)idx_arr[k];
75
+ }
76
+ }
77
+
78
+ // Scalar tail.
79
+ for (; j < j1; j++) {
80
+ if (C::cmp(best_dis, dis_tab_i[j])) {
81
+ best_dis = dis_tab_i[j];
82
+ best_idx = (TI)j;
83
+ }
84
+ }
85
+
86
+ self->dis_tab[qi] = best_dis;
87
+ self->ids_tab[qi] = best_idx;
88
+ }
89
+ }
90
+
91
+ } // namespace
92
+
93
+ // Explicit specialisations for AVX2
94
+
95
+ template <>
96
+ void top1_add_results_tpl<CMax<float, int64_t>, false, SIMDLevel::AVX2>(
97
+ Top1BlockResultHandler<CMax<float, int64_t>, false>* self,
98
+ size_t j0,
99
+ size_t j1,
100
+ const float* dis_tab) {
101
+ top1_add_results_avx2<CMax<float, int64_t>, false>(self, j0, j1, dis_tab);
102
+ }
103
+
104
+ template <>
105
+ void top1_add_results_tpl<CMax<float, int64_t>, true, SIMDLevel::AVX2>(
106
+ Top1BlockResultHandler<CMax<float, int64_t>, true>* self,
107
+ size_t j0,
108
+ size_t j1,
109
+ const float* dis_tab) {
110
+ top1_add_results_avx2<CMax<float, int64_t>, true>(self, j0, j1, dis_tab);
111
+ }
112
+
113
+ template <>
114
+ void top1_add_results_tpl<CMin<float, int64_t>, false, SIMDLevel::AVX2>(
115
+ Top1BlockResultHandler<CMin<float, int64_t>, false>* self,
116
+ size_t j0,
117
+ size_t j1,
118
+ const float* dis_tab) {
119
+ top1_add_results_avx2<CMin<float, int64_t>, false>(self, j0, j1, dis_tab);
120
+ }
121
+
122
+ template <>
123
+ void top1_add_results_tpl<CMin<float, int64_t>, true, SIMDLevel::AVX2>(
124
+ Top1BlockResultHandler<CMin<float, int64_t>, true>* self,
125
+ size_t j0,
126
+ size_t j1,
127
+ const float* dis_tab) {
128
+ top1_add_results_avx2<CMin<float, int64_t>, true>(self, j0, j1, dis_tab);
129
+ }
130
+
131
+ } // namespace faiss
132
+
133
+ #endif // COMPILE_SIMD_AVX2