faiss 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/lib/faiss/version.rb +1 -1
  4. data/vendor/faiss/faiss/AutoTune.cpp +3 -1
  5. data/vendor/faiss/faiss/Clustering.cpp +9 -1
  6. data/vendor/faiss/faiss/IVFlib.cpp +14 -3
  7. data/vendor/faiss/faiss/Index.h +2 -2
  8. data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
  9. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
  10. data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
  11. data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
  12. data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
  13. data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
  14. data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
  15. data/vendor/faiss/faiss/IndexEDEN.h +57 -0
  16. data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
  17. data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
  18. data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
  19. data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
  20. data/vendor/faiss/faiss/IndexIDMap.h +2 -0
  21. data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
  22. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
  23. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
  24. data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
  25. data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
  26. data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
  27. data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
  28. data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
  29. data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
  30. data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
  31. data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
  32. data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
  33. data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
  34. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
  35. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
  36. data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
  37. data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
  38. data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
  39. data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
  40. data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
  41. data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
  42. data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
  43. data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
  44. data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
  45. data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
  46. data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
  47. data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
  48. data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
  49. data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
  50. data/vendor/faiss/faiss/IndexShards.cpp +2 -2
  51. data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
  52. data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
  53. data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
  54. data/vendor/faiss/faiss/SuperKMeans.h +30 -0
  55. data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
  56. data/vendor/faiss/faiss/clone_index.cpp +5 -0
  57. data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
  58. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
  59. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
  60. data/vendor/faiss/faiss/factory_tools.cpp +51 -4
  61. data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
  62. data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
  63. data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
  64. data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
  65. data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
  66. data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
  67. data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
  68. data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
  69. data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
  70. data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
  71. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
  72. data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
  73. data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
  74. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
  75. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
  76. data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
  77. data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
  78. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
  79. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
  80. data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
  81. data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
  82. data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
  83. data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
  84. data/vendor/faiss/faiss/impl/HNSW.h +51 -13
  85. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
  86. data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
  87. data/vendor/faiss/faiss/impl/Panorama.h +20 -7
  88. data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
  89. data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
  90. data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
  91. data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
  92. data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
  93. data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
  94. data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
  95. data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
  96. data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
  97. data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
  98. data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
  99. data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
  100. data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
  101. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
  102. data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
  103. data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
  104. data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
  105. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
  106. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
  107. data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
  108. data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
  109. data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
  110. data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
  111. data/vendor/faiss/faiss/impl/io_macros.h +25 -0
  112. data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
  113. data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
  114. data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
  115. data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
  116. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
  117. data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
  118. data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
  119. data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
  120. data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
  121. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
  122. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
  123. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
  124. data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
  125. data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
  126. data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
  127. data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
  128. data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
  129. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
  130. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
  131. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
  132. data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
  133. data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
  134. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
  135. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
  136. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
  137. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
  138. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
  139. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
  140. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
  141. data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
  142. data/vendor/faiss/faiss/index_factory.cpp +37 -7
  143. data/vendor/faiss/faiss/index_io.h +16 -0
  144. data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
  145. data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
  146. data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
  147. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
  148. data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
  149. data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
  150. data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
  151. data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
  152. data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
  153. data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
  154. data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
  155. data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
  156. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
  157. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
  158. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
  159. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
  160. data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
  161. data/vendor/faiss/faiss/utils/bf16.h +34 -0
  162. data/vendor/faiss/faiss/utils/distances.cpp +14 -2
  163. data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
  164. data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
  165. data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
  166. data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
  167. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
  168. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
  169. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
  170. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
  171. data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
  172. data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
  173. data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
  174. data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
  175. data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
  176. data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
  177. data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
  178. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
  179. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
  180. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
  181. data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
  182. data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
  183. data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
  184. data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
  185. data/vendor/faiss/faiss/utils/utils.cpp +9 -27
  186. metadata +27 -2
@@ -29,7 +29,6 @@
29
29
  #include <svs/runtime/vamana_index.h>
30
30
 
31
31
  #include <memory>
32
- #include <span>
33
32
  #include "faiss/svs/IndexSVSVamana.h"
34
33
 
35
34
  namespace faiss {
@@ -44,8 +43,9 @@ IndexSVSVamanaLeanVec::IndexSVSVamanaLeanVec(
44
43
  size_t degree,
45
44
  MetricType metric,
46
45
  size_t leanvec_dims,
47
- SVSStorageKind storage_kind)
48
- : IndexSVSVamana(d, degree, metric, storage_kind) {
46
+ SVSStorageKind storage_kind,
47
+ bool is_static)
48
+ : IndexSVSVamana(d, degree, metric, storage_kind, is_static) {
49
49
  is_trained = false;
50
50
  leanvec_d = leanvec_dims == 0 ? d / 2 : leanvec_dims;
51
51
  }
@@ -120,7 +120,8 @@ void IndexSVSVamanaLeanVec::deserialize_training_data(std::istream& in) {
120
120
  training_data = tdata;
121
121
  }
122
122
 
123
- void IndexSVSVamanaLeanVec::create_impl() {
123
+ void IndexSVSVamanaLeanVec::create_impl(idx_t n, const float* x) {
124
+ FAISS_THROW_IF_MSG(impl, "impl already created");
124
125
  ntotal = 0;
125
126
  auto svs_metric = to_svs_metric(metric_type);
126
127
  auto svs_storage_kind = to_svs_storage_kind(storage_kind);
@@ -136,29 +137,68 @@ void IndexSVSVamanaLeanVec::create_impl() {
136
137
  .search_window_size = search_window_size,
137
138
  .search_buffer_capacity = search_buffer_capacity,
138
139
  };
140
+
139
141
  auto status = svs_runtime::Status_Ok;
140
- if (training_data) {
141
- status = svs_runtime::DynamicVamanaIndexLeanVec::build(
142
- &impl,
143
- d,
144
- svs_metric,
145
- svs_storage_kind,
146
- training_data,
147
- build_params,
148
- search_params);
142
+ if (is_static) {
143
+ FAISS_THROW_IF_NOT_MSG(
144
+ n > 0 && x != nullptr,
145
+ "Static Vamana LeanVec index requires data at build time.");
146
+ if (training_data) {
147
+ status = svs_runtime::VamanaIndexLeanVec::build(
148
+ &impl,
149
+ d,
150
+ svs_metric,
151
+ svs_storage_kind,
152
+ training_data,
153
+ build_params,
154
+ search_params);
155
+ } else {
156
+ status = svs_runtime::VamanaIndexLeanVec::build(
157
+ &impl,
158
+ d,
159
+ svs_metric,
160
+ svs_storage_kind,
161
+ leanvec_d,
162
+ build_params,
163
+ search_params);
164
+ }
165
+ if (!status.ok()) {
166
+ FAISS_THROW_MSG(status.message());
167
+ }
168
+ FAISS_THROW_IF_NOT(impl);
169
+ // Populate the static index with the full dataset (one-shot add).
170
+ status = impl->add(static_cast<size_t>(n), x);
171
+ if (!status.ok()) {
172
+ auto destroy_status = svs_runtime::VamanaIndex::destroy(impl);
173
+ FAISS_ASSERT(destroy_status.ok());
174
+ impl = nullptr;
175
+ FAISS_THROW_MSG(status.message());
176
+ }
149
177
  } else {
150
- status = svs_runtime::DynamicVamanaIndexLeanVec::build(
151
- &impl,
152
- d,
153
- svs_metric,
154
- svs_storage_kind,
155
- leanvec_d,
156
- build_params,
157
- search_params);
158
- }
159
-
160
- if (!status.ok()) {
161
- FAISS_THROW_MSG(status.message());
178
+ svs_runtime::DynamicVamanaIndex* dyn_impl = nullptr;
179
+ if (training_data) {
180
+ status = svs_runtime::DynamicVamanaIndexLeanVec::build(
181
+ &dyn_impl,
182
+ d,
183
+ svs_metric,
184
+ svs_storage_kind,
185
+ training_data,
186
+ build_params,
187
+ search_params);
188
+ } else {
189
+ status = svs_runtime::DynamicVamanaIndexLeanVec::build(
190
+ &dyn_impl,
191
+ d,
192
+ svs_metric,
193
+ svs_storage_kind,
194
+ leanvec_d,
195
+ build_params,
196
+ search_params);
197
+ }
198
+ if (!status.ok()) {
199
+ FAISS_THROW_MSG(status.message());
200
+ }
201
+ impl = dyn_impl;
162
202
  }
163
203
  FAISS_THROW_IF_NOT(impl);
164
204
  }
@@ -35,7 +35,8 @@ struct IndexSVSVamanaLeanVec : IndexSVSVamana {
35
35
  size_t degree,
36
36
  MetricType metric = METRIC_L2,
37
37
  size_t leanvec_dims = 0,
38
- SVSStorageKind storage = SVSStorageKind::SVS_LeanVec4x4);
38
+ SVSStorageKind storage = SVSStorageKind::SVS_LeanVec4x4,
39
+ bool is_static = false);
39
40
 
40
41
  ~IndexSVSVamanaLeanVec() override;
41
42
 
@@ -66,7 +67,7 @@ struct IndexSVSVamanaLeanVec : IndexSVSVamana {
66
67
  svs_runtime::LeanVecTrainingData* training_data{nullptr};
67
68
 
68
69
  protected:
69
- void create_impl() override;
70
+ void create_impl(idx_t n, const float* x) override;
70
71
  };
71
72
 
72
73
  } // namespace faiss
@@ -118,7 +118,7 @@ struct HeapWithBucketsForHamming32<
118
118
  }
119
119
  }
120
120
 
121
- if (valid_counter == 8) {
121
+ if (valid_counter == 0) {
122
122
  continue; // Skip if all vectors are filtered out
123
123
  }
124
124
 
@@ -7,8 +7,13 @@
7
7
 
8
8
  #pragma once
9
9
 
10
+ #include <cstddef>
10
11
  #include <cstdint>
11
12
 
13
+ #if defined(__AVX512F__) || defined(__AVX512BF16__)
14
+ #include <immintrin.h>
15
+ #endif
16
+
12
17
  namespace faiss {
13
18
 
14
19
  namespace {
@@ -33,4 +38,33 @@ inline float decode_bf16(const uint16_t v) {
33
38
  return fp.as_f32;
34
39
  }
35
40
 
41
+ inline void encode_bf16_simd(const float* src, uint16_t* dst, size_t n) {
42
+ size_t i = 0;
43
+ #ifdef __AVX512BF16__
44
+ for (; i + 16 <= n; i += 16) {
45
+ __m512 v = _mm512_loadu_ps(src + i);
46
+ __m256bh encoded = _mm512_cvtneps_pbh(v);
47
+ _mm256_storeu_epi16(dst + i, (__m256i)encoded);
48
+ }
49
+ #endif
50
+ for (; i < n; i++) {
51
+ dst[i] = encode_bf16(src[i]);
52
+ }
53
+ }
54
+
55
+ inline void decode_bf16_simd(const uint16_t* src, float* dst, size_t n) {
56
+ size_t i = 0;
57
+ #if defined(__AVX512F__)
58
+ for (; i + 16 <= n; i += 16) {
59
+ __m256i v = _mm256_loadu_si256((const __m256i*)(src + i));
60
+ __m512i w = _mm512_cvtepu16_epi32(v);
61
+ w = _mm512_slli_epi32(w, 16);
62
+ _mm512_storeu_ps(dst + i, _mm512_castsi512_ps(w));
63
+ }
64
+ #endif
65
+ for (; i < n; i++) {
66
+ dst[i] = decode_bf16(src[i]);
67
+ }
68
+ }
69
+
36
70
  } // namespace faiss
@@ -291,6 +291,11 @@ void exhaustive_inner_product_seq(
291
291
  BlockResultHandler& res) {
292
292
  using SingleResultHandler =
293
293
  typename BlockResultHandler::SingleResultHandler;
294
+
295
+ if (nx == 0) {
296
+ return;
297
+ }
298
+
294
299
  [[maybe_unused]] int nt = std::min(int(nx), omp_get_max_threads());
295
300
 
296
301
  #pragma omp parallel num_threads(nt)
@@ -327,6 +332,11 @@ void exhaustive_L2sqr_seq(
327
332
  BlockResultHandler& res) {
328
333
  using SingleResultHandler =
329
334
  typename BlockResultHandler::SingleResultHandler;
335
+
336
+ if (nx == 0) {
337
+ return;
338
+ }
339
+
330
340
  [[maybe_unused]] int nt = std::min(int(nx), omp_get_max_threads());
331
341
 
332
342
  #pragma omp parallel num_threads(nt)
@@ -531,8 +541,10 @@ void exhaustive_L2sqr_blas<Top1BlockResultHandler<CMax<float, int64_t>>>(
531
541
  return;
532
542
  }
533
543
 
534
- with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A2>([&]<SIMDLevel SL>() {
535
- if constexpr (SL == SIMDLevel::AVX2 || SL == SIMDLevel::ARM_SVE) {
544
+ with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A1>([&]<SIMDLevel SL>() {
545
+ if constexpr (
546
+ SL == SIMDLevel::AVX2 || SL == SIMDLevel::AVX512 ||
547
+ SL == SIMDLevel::ARM_SVE) {
536
548
  exhaustive_L2sqr_blas_cmax<SL>(x, y, d, nx, ny, res, y_norms);
537
549
  } else {
538
550
  exhaustive_L2sqr_blas_default_impl<
@@ -9,20 +9,20 @@
9
9
 
10
10
  #include <faiss/utils/distances.h>
11
11
 
12
- #include <algorithm>
13
12
  #include <cmath>
14
- #include <cstdio>
15
13
  #include <cstring>
16
14
 
17
- #include <faiss/impl/FaissAssert.h>
18
15
  #include <faiss/impl/simdlib/simdlib_dispatch.h>
19
16
 
20
17
  #define THE_SIMD_LEVEL SIMDLevel::NONE
21
18
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
22
19
  #include <faiss/utils/simd_impl/distances_autovec-inl.h>
23
20
 
21
+ // This TU is the SIMDLevel::NONE build of the specializations that
22
+ // distances_simdlib256.h emits, so the include is required despite no symbol
23
+ // being named directly here.
24
24
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
25
- #include <faiss/utils/simd_impl/distances_simdlib256.h>
25
+ #include <faiss/utils/simd_impl/distances_simdlib256.h> // IWYU pragma: keep
26
26
 
27
27
  namespace faiss {
28
28
 
@@ -29,11 +29,9 @@ namespace {
29
29
  template <class VD>
30
30
  struct ExtraDistanceComputer : FlatCodesDistanceComputer {
31
31
  VD vd;
32
- idx_t nb;
33
- const float* q;
34
- const float* b;
35
32
 
36
33
  float symmetric_dis(idx_t i, idx_t j) final {
34
+ const float* b = (const float*)codes;
37
35
  return vd(b + j * vd.d, b + i * vd.d);
38
36
  }
39
37
 
@@ -41,16 +39,9 @@ struct ExtraDistanceComputer : FlatCodesDistanceComputer {
41
39
  return vd(q, (float*)code);
42
40
  }
43
41
 
44
- ExtraDistanceComputer(
45
- const VD& vd_in,
46
- const float* xb,
47
- size_t nb_in,
48
- const float* q_in = nullptr)
42
+ ExtraDistanceComputer(const VD& vd_in, const float* xb)
49
43
  : FlatCodesDistanceComputer((uint8_t*)xb, vd_in.d * sizeof(float)),
50
- vd(vd_in),
51
- nb(nb_in),
52
- q(q_in),
53
- b(xb) {}
44
+ vd(vd_in) {}
54
45
 
55
46
  void set_query(const float* x) override {
56
47
  q = x;
@@ -149,11 +140,10 @@ FlatCodesDistanceComputer* get_extra_distance_computer(
149
140
  size_t d,
150
141
  MetricType mt,
151
142
  float metric_arg,
152
- size_t nb,
153
143
  const float* xb) {
154
144
  return with_VectorDistance(
155
145
  d, mt, metric_arg, [&](auto vd) -> FlatCodesDistanceComputer* {
156
- return new ExtraDistanceComputer<decltype(vd)>(vd, xb, nb);
146
+ return new ExtraDistanceComputer<decltype(vd)>(vd, xb);
157
147
  });
158
148
  }
159
149
 
@@ -49,12 +49,11 @@ void knn_extra_metrics(
49
49
  const IDSelector* sel = nullptr);
50
50
 
51
51
  /** get a DistanceComputer that refers to this type of distance and
52
- * indexes a flat array of size nb */
52
+ * indexes a flat array */
53
53
  FlatCodesDistanceComputer* get_extra_distance_computer(
54
54
  size_t d,
55
55
  MetricType mt,
56
56
  float metric_arg,
57
- size_t nb,
58
57
  const float* xb);
59
58
 
60
59
  /// Dispatch to a lambda with MetricType as a compile-time constant.
@@ -33,7 +33,7 @@
33
33
  // Scalar (NONE) fallback — only needs the generic specializations.
34
34
  #define THE_SIMD_LEVEL SIMDLevel::NONE
35
35
  // NOLINTNEXTLINE(facebook-hte-InlineHeader)
36
- #include <faiss/utils/hamming_distance/hamming_computer-generic.h>
36
+ #include <faiss/utils/hamming_distance/hamming_computer-generic.h> // IWYU pragma: keep
37
37
  #include <faiss/utils/hamming_distance/hamming_impl.h>
38
38
  #undef THE_SIMD_LEVEL
39
39
 
@@ -146,7 +146,7 @@ void hammings(
146
146
  size_t nb,
147
147
  size_t ncodes,
148
148
  hamdis_t* __restrict dis) {
149
- with_simd_level([&]<SIMDLevel SL>() {
149
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
150
150
  hammings_fixSL<SL>(a, b, na, nb, ncodes, dis);
151
151
  });
152
152
  }
@@ -170,7 +170,7 @@ void hammings_knn_hc(
170
170
  int order,
171
171
  ApproxTopK_mode_t approx_topk_mode,
172
172
  const faiss::IDSelector* sel) {
173
- with_simd_level([&]<SIMDLevel SL>() {
173
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
174
174
  hammings_knn_hc_fixSL<SL>(
175
175
  ha, a, b, nb, ncodes, order, approx_topk_mode, sel);
176
176
  });
@@ -186,7 +186,7 @@ void hammings_knn_mc(
186
186
  int32_t* __restrict distances,
187
187
  int64_t* __restrict labels,
188
188
  const faiss::IDSelector* sel) {
189
- with_simd_level([&]<SIMDLevel SL>() {
189
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
190
190
  hammings_knn_mc_fixSL<SL>(
191
191
  a, b, na, nb, k, ncodes, distances, labels, sel);
192
192
  });
@@ -201,7 +201,7 @@ void hamming_range_search(
201
201
  size_t code_size,
202
202
  RangeSearchResult* result,
203
203
  const faiss::IDSelector* sel) {
204
- with_simd_level([&]<SIMDLevel SL>() {
204
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
205
205
  hamming_range_search_fixSL<SL>(
206
206
  a, b, na, nb, radius, code_size, result, sel);
207
207
  });
@@ -215,7 +215,7 @@ void hamming_count_thres(
215
215
  hamdis_t ht,
216
216
  size_t ncodes,
217
217
  size_t* nptr) {
218
- with_simd_level([&]<SIMDLevel SL>() {
218
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
219
219
  hamming_count_thres_fixSL<SL>(bs1, bs2, n1, n2, ht, ncodes, nptr);
220
220
  });
221
221
  }
@@ -226,7 +226,7 @@ void crosshamming_count_thres(
226
226
  hamdis_t ht,
227
227
  size_t ncodes,
228
228
  size_t* nptr) {
229
- with_simd_level([&]<SIMDLevel SL>() {
229
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
230
230
  crosshamming_count_thres_fixSL<SL>(dbs, n, ht, ncodes, nptr);
231
231
  });
232
232
  }
@@ -240,7 +240,7 @@ size_t match_hamming_thres(
240
240
  size_t ncodes,
241
241
  int64_t* idx,
242
242
  hamdis_t* dis) {
243
- return with_simd_level([&]<SIMDLevel SL>() -> size_t {
243
+ return with_simd_level_a0_spr([&]<SIMDLevel SL>() -> size_t {
244
244
  return match_hamming_thres_fixSL<SL>(
245
245
  bs1, bs2, n1, n2, ht, ncodes, idx, dis);
246
246
  });
@@ -253,7 +253,7 @@ void generalized_hammings_knn_hc(
253
253
  size_t nb,
254
254
  size_t code_size,
255
255
  int ordered) {
256
- with_simd_level([&]<SIMDLevel SL>() {
256
+ with_simd_level_a0_spr([&]<SIMDLevel SL>() {
257
257
  generalized_hammings_knn_hc_fixSL<SL>(ha, a, b, nb, code_size, ordered);
258
258
  });
259
259
  }
@@ -8,8 +8,9 @@
8
8
  #ifdef COMPILE_SIMD_AVX2
9
9
 
10
10
  #define THE_SIMD_LEVEL SIMDLevel::AVX2
11
- // NOLINTNEXTLINE(facebook-hte-InlineHeader)
11
+ // NOLINTBEGIN(facebook-hte-InlineHeader,facebook-unused-include-check)
12
12
  #include <faiss/utils/hamming_distance/hamming_computer-avx2.h>
13
13
  #include <faiss/utils/hamming_distance/hamming_impl.h>
14
+ // NOLINTEND(facebook-hte-InlineHeader,facebook-unused-include-check)
14
15
 
15
16
  #endif // COMPILE_SIMD_AVX2
@@ -0,0 +1,15 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ #ifdef COMPILE_SIMD_AVX512_SPR
9
+
10
+ #define THE_SIMD_LEVEL SIMDLevel::AVX512_SPR
11
+ // NOLINTNEXTLINE(facebook-hte-InlineHeader)
12
+ #include <faiss/utils/hamming_distance/hamming_computer-avx512_spr.h>
13
+ #include <faiss/utils/hamming_distance/hamming_impl.h>
14
+
15
+ #endif // COMPILE_SIMD_AVX512_SPR
@@ -10,10 +10,10 @@
10
10
 
11
11
  // AVX512 HammingComputer and GenHammingComputer specializations.
12
12
  // Types without custom AVX512 code inherit from the NONE specializations
13
- // in hamming_computer-generic.h. Custom specializations for
14
- // HammingComputer64 and HammingComputerDefault use _mm512_popcnt_epi64
15
- // when __AVX512VPOPCNTDQ__ is available. GenHammingComputer classes
16
- // leverage SSE/AVX2 intrinsics.
13
+ // in hamming_computer-generic.h. HammingComputer64 and
14
+ // HammingComputerDefault use scalar popcount here; the VPOPCNTDQ fast
15
+ // path lives in hamming_computer-avx512_spr.h (AVX512_SPR level).
16
+ // GenHammingComputer classes leverage SSE/AVX2 intrinsics.
17
17
 
18
18
  #include <cassert>
19
19
  #include <cstdint>
@@ -74,18 +74,10 @@ struct HammingComputer64_tpl<SIMDLevel::AVX512> {
74
74
 
75
75
  inline int hamming(const uint8_t* b8) const {
76
76
  const uint64_t* b = reinterpret_cast<const uint64_t*>(b8);
77
- #ifdef __AVX512VPOPCNTDQ__
78
- __m512i vxor =
79
- _mm512_xor_si512(_mm512_loadu_si512(a), _mm512_loadu_si512(b));
80
- __m512i vpcnt = _mm512_popcnt_epi64(vxor);
81
- // reduce performs better than adding the lower and higher parts
82
- return _mm512_reduce_add_epi32(vpcnt);
83
- #else
84
77
  return popcount64(b[0] ^ a0) + popcount64(b[1] ^ a1) +
85
78
  popcount64(b[2] ^ a2) + popcount64(b[3] ^ a3) +
86
79
  popcount64(b[4] ^ a4) + popcount64(b[5] ^ a5) +
87
80
  popcount64(b[6] ^ a6) + popcount64(b[7] ^ a7);
88
- #endif
89
81
  }
90
82
 
91
83
  inline static constexpr int get_code_size() {
@@ -112,27 +104,11 @@ struct HammingComputerDefault_tpl<SIMDLevel::AVX512> {
112
104
  }
113
105
 
114
106
  int hamming(const uint8_t* b8) const {
115
- int accu = 0;
116
-
117
107
  const uint64_t* a64 = reinterpret_cast<const uint64_t*>(a8);
118
108
  const uint64_t* b64 = reinterpret_cast<const uint64_t*>(b8);
119
109
 
120
- int i = 0;
121
- #ifdef __AVX512VPOPCNTDQ__
122
- int quotient64 = quotient8 / 8;
123
- for (; i < quotient64; ++i) {
124
- __m512i vxor = _mm512_xor_si512(
125
- _mm512_loadu_si512(&a64[i * 8]),
126
- _mm512_loadu_si512(&b64[i * 8]));
127
- __m512i vpcnt = _mm512_popcnt_epi64(vxor);
128
- // reduce performs better than adding the lower and higher parts
129
- accu += _mm512_reduce_add_epi32(vpcnt);
130
- }
131
- i *= 8;
132
- #endif
133
- accu += hamming_popcount_tail(
134
- a64, b64, i, quotient8, a8, b8, remainder8);
135
- return accu;
110
+ return hamming_popcount_tail(
111
+ a64, b64, 0, quotient8, a8, b8, remainder8);
136
112
  }
137
113
 
138
114
  inline int get_code_size() const {
@@ -0,0 +1,171 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ #ifndef HAMMING_COMPUTER_AVX512_SPR_H
9
+ #define HAMMING_COMPUTER_AVX512_SPR_H
10
+
11
+ // AVX512_SPR HammingComputer specializations using VPOPCNTDQ.
12
+ // On Sapphire Rapids+, _mm512_popcnt_epi64 (and _mm256_popcnt_epi64 with VL)
13
+ // are unconditionally available. This gives a faster path than the scalar
14
+ // popcount fallback used in the base AVX512 specializations when compiled
15
+ // without -mavx512vpopcntdq.
16
+
17
+ #include <cassert>
18
+ #include <cstdint>
19
+
20
+ #include <faiss/impl/platform_macros.h>
21
+ #include <faiss/utils/hamming_distance/hamming_computer-avx512.h>
22
+
23
+ #include <immintrin.h>
24
+
25
+ namespace faiss {
26
+
27
+ /***************************************************************************
28
+ * AVX512_SPR inheriting specializations for types without custom SPR code.
29
+ ***************************************************************************/
30
+
31
+ #define FAISS_INHERIT_HAMMING_SPR(Class) \
32
+ template <> \
33
+ struct Class## \
34
+ _tpl<SIMDLevel::AVX512_SPR> : Class##_tpl<SIMDLevel::AVX512> { \
35
+ using Class##_tpl<SIMDLevel::AVX512>::Class##_tpl; \
36
+ }
37
+
38
+ FAISS_INHERIT_HAMMING_SPR(HammingComputer16);
39
+ FAISS_INHERIT_HAMMING_SPR(HammingComputer20);
40
+ FAISS_INHERIT_HAMMING_SPR(GenHammingComputer8);
41
+ FAISS_INHERIT_HAMMING_SPR(GenHammingComputer16);
42
+ FAISS_INHERIT_HAMMING_SPR(GenHammingComputer32);
43
+ FAISS_INHERIT_HAMMING_SPR(GenHammingComputerM8);
44
+
45
+ #undef FAISS_INHERIT_HAMMING_SPR
46
+
47
+ /***************************************************************************
48
+ * Custom AVX512_SPR specializations using VPOPCNTDQ.
49
+ ***************************************************************************/
50
+
51
+ template <>
52
+ struct HammingComputer32_tpl<SIMDLevel::AVX512_SPR> {
53
+ const uint8_t* a8;
54
+
55
+ HammingComputer32_tpl() {}
56
+
57
+ HammingComputer32_tpl(const uint8_t* a8_in, int code_size) {
58
+ set(a8_in, code_size);
59
+ }
60
+
61
+ void set(const uint8_t* a8_in, FAISS_MAYBE_UNUSED int code_size) {
62
+ assert(code_size == 32);
63
+ a8 = a8_in;
64
+ }
65
+
66
+ inline int hamming(const uint8_t* b8) const {
67
+ __m256i va = _mm256_loadu_si256((const __m256i*)a8);
68
+ __m256i vb = _mm256_loadu_si256((const __m256i*)b8);
69
+ __m256i vxor = _mm256_xor_si256(va, vb);
70
+ __m256i vpcnt = _mm256_popcnt_epi64(vxor);
71
+ __m128i lo = _mm256_castsi256_si128(vpcnt);
72
+ __m128i hi = _mm256_extracti128_si256(vpcnt, 1);
73
+ __m128i sum = _mm_add_epi64(lo, hi);
74
+ return static_cast<int>(
75
+ _mm_extract_epi64(sum, 0) + _mm_extract_epi64(sum, 1));
76
+ }
77
+
78
+ inline static constexpr int get_code_size() {
79
+ return 32;
80
+ }
81
+ };
82
+
83
+ template <>
84
+ struct HammingComputer64_tpl<SIMDLevel::AVX512_SPR> {
85
+ const uint8_t* a8;
86
+
87
+ HammingComputer64_tpl() {}
88
+
89
+ HammingComputer64_tpl(const uint8_t* a8_in, int code_size) {
90
+ set(a8_in, code_size);
91
+ }
92
+
93
+ void set(const uint8_t* a8_in, FAISS_MAYBE_UNUSED int code_size) {
94
+ assert(code_size == 64);
95
+ a8 = a8_in;
96
+ }
97
+
98
+ inline int hamming(const uint8_t* b8) const {
99
+ __m512i vxor = _mm512_xor_si512(
100
+ _mm512_loadu_si512(a8), _mm512_loadu_si512(b8));
101
+ __m512i vpcnt = _mm512_popcnt_epi64(vxor);
102
+ return _mm512_reduce_add_epi32(vpcnt);
103
+ }
104
+
105
+ inline static constexpr int get_code_size() {
106
+ return 64;
107
+ }
108
+ };
109
+
110
+ template <>
111
+ struct HammingComputerDefault_tpl<SIMDLevel::AVX512_SPR> {
112
+ const uint8_t* a8;
113
+ int quotient8;
114
+ int remainder8;
115
+
116
+ HammingComputerDefault_tpl() {}
117
+
118
+ HammingComputerDefault_tpl(const uint8_t* a8_in, int code_size) {
119
+ set(a8_in, code_size);
120
+ }
121
+
122
+ void set(const uint8_t* a8_2, int code_size) {
123
+ this->a8 = a8_2;
124
+ quotient8 = code_size / 8;
125
+ remainder8 = code_size % 8;
126
+ }
127
+
128
+ int hamming(const uint8_t* b8) const {
129
+ int accu = 0;
130
+
131
+ const uint64_t* a64 = reinterpret_cast<const uint64_t*>(a8);
132
+ const uint64_t* b64 = reinterpret_cast<const uint64_t*>(b8);
133
+
134
+ int i = 0;
135
+ int quotient64 = quotient8 / 8;
136
+ for (; i < quotient64; ++i) {
137
+ __m512i vxor = _mm512_xor_si512(
138
+ _mm512_loadu_si512(&a64[i * 8]),
139
+ _mm512_loadu_si512(&b64[i * 8]));
140
+ __m512i vpcnt = _mm512_popcnt_epi64(vxor);
141
+ accu += _mm512_reduce_add_epi32(vpcnt);
142
+ }
143
+ i *= 8;
144
+
145
+ // Handle 4-word (256-bit) remainder with VPOPCNTDQ VL
146
+ if (i + 4 <= quotient8) {
147
+ __m256i vxor = _mm256_xor_si256(
148
+ _mm256_loadu_si256((const __m256i*)&a64[i]),
149
+ _mm256_loadu_si256((const __m256i*)&b64[i]));
150
+ __m256i vpcnt = _mm256_popcnt_epi64(vxor);
151
+ __m128i lo = _mm256_castsi256_si128(vpcnt);
152
+ __m128i hi = _mm256_extracti128_si256(vpcnt, 1);
153
+ __m128i sum = _mm_add_epi64(lo, hi);
154
+ accu += static_cast<int>(
155
+ _mm_extract_epi64(sum, 0) + _mm_extract_epi64(sum, 1));
156
+ i += 4;
157
+ }
158
+
159
+ accu += hamming_popcount_tail(
160
+ a64, b64, i, quotient8, a8, b8, remainder8);
161
+ return accu;
162
+ }
163
+
164
+ inline int get_code_size() const {
165
+ return quotient8 * 8 + remainder8;
166
+ }
167
+ };
168
+
169
+ } // namespace faiss
170
+
171
+ #endif
@@ -18,8 +18,6 @@
18
18
  #include <faiss/utils/AlignedTable.h>
19
19
  #include <faiss/utils/ordered_key_value.h>
20
20
 
21
- #include <faiss/impl/platform_macros.h>
22
-
23
21
  namespace faiss {
24
22
 
25
23
  /******************************************************************