faiss 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +3 -1
- data/vendor/faiss/faiss/Clustering.cpp +9 -1
- data/vendor/faiss/faiss/IVFlib.cpp +14 -3
- data/vendor/faiss/faiss/Index.h +2 -2
- data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
- data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
- data/vendor/faiss/faiss/IndexEDEN.h +57 -0
- data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
- data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
- data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
- data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
- data/vendor/faiss/faiss/IndexIDMap.h +2 -0
- data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
- data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
- data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
- data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
- data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
- data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
- data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
- data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
- data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
- data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
- data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
- data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
- data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
- data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
- data/vendor/faiss/faiss/IndexShards.cpp +2 -2
- data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
- data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
- data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
- data/vendor/faiss/faiss/SuperKMeans.h +30 -0
- data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
- data/vendor/faiss/faiss/clone_index.cpp +5 -0
- data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
- data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
- data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
- data/vendor/faiss/faiss/factory_tools.cpp +51 -4
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
- data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
- data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
- data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
- data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
- data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
- data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
- data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
- data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
- data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
- data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
- data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
- data/vendor/faiss/faiss/impl/HNSW.h +51 -13
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
- data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
- data/vendor/faiss/faiss/impl/Panorama.h +20 -7
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
- data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
- data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
- data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
- data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
- data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
- data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
- data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
- data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
- data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
- data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
- data/vendor/faiss/faiss/impl/io_macros.h +25 -0
- data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
- data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
- data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
- data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
- data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
- data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
- data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
- data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
- data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
- data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
- data/vendor/faiss/faiss/index_factory.cpp +37 -7
- data/vendor/faiss/faiss/index_io.h +16 -0
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
- data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
- data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
- data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
- data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
- data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
- data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
- data/vendor/faiss/faiss/utils/bf16.h +34 -0
- data/vendor/faiss/faiss/utils/distances.cpp +14 -2
- data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
- data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
- data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
- data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
- data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
- data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
- data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
- data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
- data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
- data/vendor/faiss/faiss/utils/utils.cpp +9 -27
- metadata +27 -2
|
@@ -29,7 +29,6 @@
|
|
|
29
29
|
#include <svs/runtime/vamana_index.h>
|
|
30
30
|
|
|
31
31
|
#include <memory>
|
|
32
|
-
#include <span>
|
|
33
32
|
#include "faiss/svs/IndexSVSVamana.h"
|
|
34
33
|
|
|
35
34
|
namespace faiss {
|
|
@@ -44,8 +43,9 @@ IndexSVSVamanaLeanVec::IndexSVSVamanaLeanVec(
|
|
|
44
43
|
size_t degree,
|
|
45
44
|
MetricType metric,
|
|
46
45
|
size_t leanvec_dims,
|
|
47
|
-
SVSStorageKind storage_kind
|
|
48
|
-
|
|
46
|
+
SVSStorageKind storage_kind,
|
|
47
|
+
bool is_static)
|
|
48
|
+
: IndexSVSVamana(d, degree, metric, storage_kind, is_static) {
|
|
49
49
|
is_trained = false;
|
|
50
50
|
leanvec_d = leanvec_dims == 0 ? d / 2 : leanvec_dims;
|
|
51
51
|
}
|
|
@@ -120,7 +120,8 @@ void IndexSVSVamanaLeanVec::deserialize_training_data(std::istream& in) {
|
|
|
120
120
|
training_data = tdata;
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
-
void IndexSVSVamanaLeanVec::create_impl() {
|
|
123
|
+
void IndexSVSVamanaLeanVec::create_impl(idx_t n, const float* x) {
|
|
124
|
+
FAISS_THROW_IF_MSG(impl, "impl already created");
|
|
124
125
|
ntotal = 0;
|
|
125
126
|
auto svs_metric = to_svs_metric(metric_type);
|
|
126
127
|
auto svs_storage_kind = to_svs_storage_kind(storage_kind);
|
|
@@ -136,29 +137,68 @@ void IndexSVSVamanaLeanVec::create_impl() {
|
|
|
136
137
|
.search_window_size = search_window_size,
|
|
137
138
|
.search_buffer_capacity = search_buffer_capacity,
|
|
138
139
|
};
|
|
140
|
+
|
|
139
141
|
auto status = svs_runtime::Status_Ok;
|
|
140
|
-
if (
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
142
|
+
if (is_static) {
|
|
143
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
144
|
+
n > 0 && x != nullptr,
|
|
145
|
+
"Static Vamana LeanVec index requires data at build time.");
|
|
146
|
+
if (training_data) {
|
|
147
|
+
status = svs_runtime::VamanaIndexLeanVec::build(
|
|
148
|
+
&impl,
|
|
149
|
+
d,
|
|
150
|
+
svs_metric,
|
|
151
|
+
svs_storage_kind,
|
|
152
|
+
training_data,
|
|
153
|
+
build_params,
|
|
154
|
+
search_params);
|
|
155
|
+
} else {
|
|
156
|
+
status = svs_runtime::VamanaIndexLeanVec::build(
|
|
157
|
+
&impl,
|
|
158
|
+
d,
|
|
159
|
+
svs_metric,
|
|
160
|
+
svs_storage_kind,
|
|
161
|
+
leanvec_d,
|
|
162
|
+
build_params,
|
|
163
|
+
search_params);
|
|
164
|
+
}
|
|
165
|
+
if (!status.ok()) {
|
|
166
|
+
FAISS_THROW_MSG(status.message());
|
|
167
|
+
}
|
|
168
|
+
FAISS_THROW_IF_NOT(impl);
|
|
169
|
+
// Populate the static index with the full dataset (one-shot add).
|
|
170
|
+
status = impl->add(static_cast<size_t>(n), x);
|
|
171
|
+
if (!status.ok()) {
|
|
172
|
+
auto destroy_status = svs_runtime::VamanaIndex::destroy(impl);
|
|
173
|
+
FAISS_ASSERT(destroy_status.ok());
|
|
174
|
+
impl = nullptr;
|
|
175
|
+
FAISS_THROW_MSG(status.message());
|
|
176
|
+
}
|
|
149
177
|
} else {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
178
|
+
svs_runtime::DynamicVamanaIndex* dyn_impl = nullptr;
|
|
179
|
+
if (training_data) {
|
|
180
|
+
status = svs_runtime::DynamicVamanaIndexLeanVec::build(
|
|
181
|
+
&dyn_impl,
|
|
182
|
+
d,
|
|
183
|
+
svs_metric,
|
|
184
|
+
svs_storage_kind,
|
|
185
|
+
training_data,
|
|
186
|
+
build_params,
|
|
187
|
+
search_params);
|
|
188
|
+
} else {
|
|
189
|
+
status = svs_runtime::DynamicVamanaIndexLeanVec::build(
|
|
190
|
+
&dyn_impl,
|
|
191
|
+
d,
|
|
192
|
+
svs_metric,
|
|
193
|
+
svs_storage_kind,
|
|
194
|
+
leanvec_d,
|
|
195
|
+
build_params,
|
|
196
|
+
search_params);
|
|
197
|
+
}
|
|
198
|
+
if (!status.ok()) {
|
|
199
|
+
FAISS_THROW_MSG(status.message());
|
|
200
|
+
}
|
|
201
|
+
impl = dyn_impl;
|
|
162
202
|
}
|
|
163
203
|
FAISS_THROW_IF_NOT(impl);
|
|
164
204
|
}
|
|
@@ -35,7 +35,8 @@ struct IndexSVSVamanaLeanVec : IndexSVSVamana {
|
|
|
35
35
|
size_t degree,
|
|
36
36
|
MetricType metric = METRIC_L2,
|
|
37
37
|
size_t leanvec_dims = 0,
|
|
38
|
-
SVSStorageKind storage = SVSStorageKind::SVS_LeanVec4x4
|
|
38
|
+
SVSStorageKind storage = SVSStorageKind::SVS_LeanVec4x4,
|
|
39
|
+
bool is_static = false);
|
|
39
40
|
|
|
40
41
|
~IndexSVSVamanaLeanVec() override;
|
|
41
42
|
|
|
@@ -66,7 +67,7 @@ struct IndexSVSVamanaLeanVec : IndexSVSVamana {
|
|
|
66
67
|
svs_runtime::LeanVecTrainingData* training_data{nullptr};
|
|
67
68
|
|
|
68
69
|
protected:
|
|
69
|
-
void create_impl() override;
|
|
70
|
+
void create_impl(idx_t n, const float* x) override;
|
|
70
71
|
};
|
|
71
72
|
|
|
72
73
|
} // namespace faiss
|
|
@@ -7,8 +7,13 @@
|
|
|
7
7
|
|
|
8
8
|
#pragma once
|
|
9
9
|
|
|
10
|
+
#include <cstddef>
|
|
10
11
|
#include <cstdint>
|
|
11
12
|
|
|
13
|
+
#if defined(__AVX512F__) || defined(__AVX512BF16__)
|
|
14
|
+
#include <immintrin.h>
|
|
15
|
+
#endif
|
|
16
|
+
|
|
12
17
|
namespace faiss {
|
|
13
18
|
|
|
14
19
|
namespace {
|
|
@@ -33,4 +38,33 @@ inline float decode_bf16(const uint16_t v) {
|
|
|
33
38
|
return fp.as_f32;
|
|
34
39
|
}
|
|
35
40
|
|
|
41
|
+
inline void encode_bf16_simd(const float* src, uint16_t* dst, size_t n) {
|
|
42
|
+
size_t i = 0;
|
|
43
|
+
#ifdef __AVX512BF16__
|
|
44
|
+
for (; i + 16 <= n; i += 16) {
|
|
45
|
+
__m512 v = _mm512_loadu_ps(src + i);
|
|
46
|
+
__m256bh encoded = _mm512_cvtneps_pbh(v);
|
|
47
|
+
_mm256_storeu_epi16(dst + i, (__m256i)encoded);
|
|
48
|
+
}
|
|
49
|
+
#endif
|
|
50
|
+
for (; i < n; i++) {
|
|
51
|
+
dst[i] = encode_bf16(src[i]);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
inline void decode_bf16_simd(const uint16_t* src, float* dst, size_t n) {
|
|
56
|
+
size_t i = 0;
|
|
57
|
+
#if defined(__AVX512F__)
|
|
58
|
+
for (; i + 16 <= n; i += 16) {
|
|
59
|
+
__m256i v = _mm256_loadu_si256((const __m256i*)(src + i));
|
|
60
|
+
__m512i w = _mm512_cvtepu16_epi32(v);
|
|
61
|
+
w = _mm512_slli_epi32(w, 16);
|
|
62
|
+
_mm512_storeu_ps(dst + i, _mm512_castsi512_ps(w));
|
|
63
|
+
}
|
|
64
|
+
#endif
|
|
65
|
+
for (; i < n; i++) {
|
|
66
|
+
dst[i] = decode_bf16(src[i]);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
36
70
|
} // namespace faiss
|
|
@@ -291,6 +291,11 @@ void exhaustive_inner_product_seq(
|
|
|
291
291
|
BlockResultHandler& res) {
|
|
292
292
|
using SingleResultHandler =
|
|
293
293
|
typename BlockResultHandler::SingleResultHandler;
|
|
294
|
+
|
|
295
|
+
if (nx == 0) {
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
|
|
294
299
|
[[maybe_unused]] int nt = std::min(int(nx), omp_get_max_threads());
|
|
295
300
|
|
|
296
301
|
#pragma omp parallel num_threads(nt)
|
|
@@ -327,6 +332,11 @@ void exhaustive_L2sqr_seq(
|
|
|
327
332
|
BlockResultHandler& res) {
|
|
328
333
|
using SingleResultHandler =
|
|
329
334
|
typename BlockResultHandler::SingleResultHandler;
|
|
335
|
+
|
|
336
|
+
if (nx == 0) {
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
|
|
330
340
|
[[maybe_unused]] int nt = std::min(int(nx), omp_get_max_threads());
|
|
331
341
|
|
|
332
342
|
#pragma omp parallel num_threads(nt)
|
|
@@ -531,8 +541,10 @@ void exhaustive_L2sqr_blas<Top1BlockResultHandler<CMax<float, int64_t>>>(
|
|
|
531
541
|
return;
|
|
532
542
|
}
|
|
533
543
|
|
|
534
|
-
with_selected_simd_levels<
|
|
535
|
-
if constexpr (
|
|
544
|
+
with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A1>([&]<SIMDLevel SL>() {
|
|
545
|
+
if constexpr (
|
|
546
|
+
SL == SIMDLevel::AVX2 || SL == SIMDLevel::AVX512 ||
|
|
547
|
+
SL == SIMDLevel::ARM_SVE) {
|
|
536
548
|
exhaustive_L2sqr_blas_cmax<SL>(x, y, d, nx, ny, res, y_norms);
|
|
537
549
|
} else {
|
|
538
550
|
exhaustive_L2sqr_blas_default_impl<
|
|
@@ -9,20 +9,20 @@
|
|
|
9
9
|
|
|
10
10
|
#include <faiss/utils/distances.h>
|
|
11
11
|
|
|
12
|
-
#include <algorithm>
|
|
13
12
|
#include <cmath>
|
|
14
|
-
#include <cstdio>
|
|
15
13
|
#include <cstring>
|
|
16
14
|
|
|
17
|
-
#include <faiss/impl/FaissAssert.h>
|
|
18
15
|
#include <faiss/impl/simdlib/simdlib_dispatch.h>
|
|
19
16
|
|
|
20
17
|
#define THE_SIMD_LEVEL SIMDLevel::NONE
|
|
21
18
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
22
19
|
#include <faiss/utils/simd_impl/distances_autovec-inl.h>
|
|
23
20
|
|
|
21
|
+
// This TU is the SIMDLevel::NONE build of the specializations that
|
|
22
|
+
// distances_simdlib256.h emits, so the include is required despite no symbol
|
|
23
|
+
// being named directly here.
|
|
24
24
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
25
|
-
#include <faiss/utils/simd_impl/distances_simdlib256.h>
|
|
25
|
+
#include <faiss/utils/simd_impl/distances_simdlib256.h> // IWYU pragma: keep
|
|
26
26
|
|
|
27
27
|
namespace faiss {
|
|
28
28
|
|
|
@@ -29,11 +29,9 @@ namespace {
|
|
|
29
29
|
template <class VD>
|
|
30
30
|
struct ExtraDistanceComputer : FlatCodesDistanceComputer {
|
|
31
31
|
VD vd;
|
|
32
|
-
idx_t nb;
|
|
33
|
-
const float* q;
|
|
34
|
-
const float* b;
|
|
35
32
|
|
|
36
33
|
float symmetric_dis(idx_t i, idx_t j) final {
|
|
34
|
+
const float* b = (const float*)codes;
|
|
37
35
|
return vd(b + j * vd.d, b + i * vd.d);
|
|
38
36
|
}
|
|
39
37
|
|
|
@@ -41,16 +39,9 @@ struct ExtraDistanceComputer : FlatCodesDistanceComputer {
|
|
|
41
39
|
return vd(q, (float*)code);
|
|
42
40
|
}
|
|
43
41
|
|
|
44
|
-
ExtraDistanceComputer(
|
|
45
|
-
const VD& vd_in,
|
|
46
|
-
const float* xb,
|
|
47
|
-
size_t nb_in,
|
|
48
|
-
const float* q_in = nullptr)
|
|
42
|
+
ExtraDistanceComputer(const VD& vd_in, const float* xb)
|
|
49
43
|
: FlatCodesDistanceComputer((uint8_t*)xb, vd_in.d * sizeof(float)),
|
|
50
|
-
vd(vd_in)
|
|
51
|
-
nb(nb_in),
|
|
52
|
-
q(q_in),
|
|
53
|
-
b(xb) {}
|
|
44
|
+
vd(vd_in) {}
|
|
54
45
|
|
|
55
46
|
void set_query(const float* x) override {
|
|
56
47
|
q = x;
|
|
@@ -149,11 +140,10 @@ FlatCodesDistanceComputer* get_extra_distance_computer(
|
|
|
149
140
|
size_t d,
|
|
150
141
|
MetricType mt,
|
|
151
142
|
float metric_arg,
|
|
152
|
-
size_t nb,
|
|
153
143
|
const float* xb) {
|
|
154
144
|
return with_VectorDistance(
|
|
155
145
|
d, mt, metric_arg, [&](auto vd) -> FlatCodesDistanceComputer* {
|
|
156
|
-
return new ExtraDistanceComputer<decltype(vd)>(vd, xb
|
|
146
|
+
return new ExtraDistanceComputer<decltype(vd)>(vd, xb);
|
|
157
147
|
});
|
|
158
148
|
}
|
|
159
149
|
|
|
@@ -49,12 +49,11 @@ void knn_extra_metrics(
|
|
|
49
49
|
const IDSelector* sel = nullptr);
|
|
50
50
|
|
|
51
51
|
/** get a DistanceComputer that refers to this type of distance and
|
|
52
|
-
* indexes a flat array
|
|
52
|
+
* indexes a flat array */
|
|
53
53
|
FlatCodesDistanceComputer* get_extra_distance_computer(
|
|
54
54
|
size_t d,
|
|
55
55
|
MetricType mt,
|
|
56
56
|
float metric_arg,
|
|
57
|
-
size_t nb,
|
|
58
57
|
const float* xb);
|
|
59
58
|
|
|
60
59
|
/// Dispatch to a lambda with MetricType as a compile-time constant.
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
// Scalar (NONE) fallback — only needs the generic specializations.
|
|
34
34
|
#define THE_SIMD_LEVEL SIMDLevel::NONE
|
|
35
35
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
36
|
-
#include <faiss/utils/hamming_distance/hamming_computer-generic.h>
|
|
36
|
+
#include <faiss/utils/hamming_distance/hamming_computer-generic.h> // IWYU pragma: keep
|
|
37
37
|
#include <faiss/utils/hamming_distance/hamming_impl.h>
|
|
38
38
|
#undef THE_SIMD_LEVEL
|
|
39
39
|
|
|
@@ -146,7 +146,7 @@ void hammings(
|
|
|
146
146
|
size_t nb,
|
|
147
147
|
size_t ncodes,
|
|
148
148
|
hamdis_t* __restrict dis) {
|
|
149
|
-
|
|
149
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
150
150
|
hammings_fixSL<SL>(a, b, na, nb, ncodes, dis);
|
|
151
151
|
});
|
|
152
152
|
}
|
|
@@ -170,7 +170,7 @@ void hammings_knn_hc(
|
|
|
170
170
|
int order,
|
|
171
171
|
ApproxTopK_mode_t approx_topk_mode,
|
|
172
172
|
const faiss::IDSelector* sel) {
|
|
173
|
-
|
|
173
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
174
174
|
hammings_knn_hc_fixSL<SL>(
|
|
175
175
|
ha, a, b, nb, ncodes, order, approx_topk_mode, sel);
|
|
176
176
|
});
|
|
@@ -186,7 +186,7 @@ void hammings_knn_mc(
|
|
|
186
186
|
int32_t* __restrict distances,
|
|
187
187
|
int64_t* __restrict labels,
|
|
188
188
|
const faiss::IDSelector* sel) {
|
|
189
|
-
|
|
189
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
190
190
|
hammings_knn_mc_fixSL<SL>(
|
|
191
191
|
a, b, na, nb, k, ncodes, distances, labels, sel);
|
|
192
192
|
});
|
|
@@ -201,7 +201,7 @@ void hamming_range_search(
|
|
|
201
201
|
size_t code_size,
|
|
202
202
|
RangeSearchResult* result,
|
|
203
203
|
const faiss::IDSelector* sel) {
|
|
204
|
-
|
|
204
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
205
205
|
hamming_range_search_fixSL<SL>(
|
|
206
206
|
a, b, na, nb, radius, code_size, result, sel);
|
|
207
207
|
});
|
|
@@ -215,7 +215,7 @@ void hamming_count_thres(
|
|
|
215
215
|
hamdis_t ht,
|
|
216
216
|
size_t ncodes,
|
|
217
217
|
size_t* nptr) {
|
|
218
|
-
|
|
218
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
219
219
|
hamming_count_thres_fixSL<SL>(bs1, bs2, n1, n2, ht, ncodes, nptr);
|
|
220
220
|
});
|
|
221
221
|
}
|
|
@@ -226,7 +226,7 @@ void crosshamming_count_thres(
|
|
|
226
226
|
hamdis_t ht,
|
|
227
227
|
size_t ncodes,
|
|
228
228
|
size_t* nptr) {
|
|
229
|
-
|
|
229
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
230
230
|
crosshamming_count_thres_fixSL<SL>(dbs, n, ht, ncodes, nptr);
|
|
231
231
|
});
|
|
232
232
|
}
|
|
@@ -240,7 +240,7 @@ size_t match_hamming_thres(
|
|
|
240
240
|
size_t ncodes,
|
|
241
241
|
int64_t* idx,
|
|
242
242
|
hamdis_t* dis) {
|
|
243
|
-
return
|
|
243
|
+
return with_simd_level_a0_spr([&]<SIMDLevel SL>() -> size_t {
|
|
244
244
|
return match_hamming_thres_fixSL<SL>(
|
|
245
245
|
bs1, bs2, n1, n2, ht, ncodes, idx, dis);
|
|
246
246
|
});
|
|
@@ -253,7 +253,7 @@ void generalized_hammings_knn_hc(
|
|
|
253
253
|
size_t nb,
|
|
254
254
|
size_t code_size,
|
|
255
255
|
int ordered) {
|
|
256
|
-
|
|
256
|
+
with_simd_level_a0_spr([&]<SIMDLevel SL>() {
|
|
257
257
|
generalized_hammings_knn_hc_fixSL<SL>(ha, a, b, nb, code_size, ordered);
|
|
258
258
|
});
|
|
259
259
|
}
|
|
@@ -8,8 +8,9 @@
|
|
|
8
8
|
#ifdef COMPILE_SIMD_AVX2
|
|
9
9
|
|
|
10
10
|
#define THE_SIMD_LEVEL SIMDLevel::AVX2
|
|
11
|
-
//
|
|
11
|
+
// NOLINTBEGIN(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
12
12
|
#include <faiss/utils/hamming_distance/hamming_computer-avx2.h>
|
|
13
13
|
#include <faiss/utils/hamming_distance/hamming_impl.h>
|
|
14
|
+
// NOLINTEND(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
14
15
|
|
|
15
16
|
#endif // COMPILE_SIMD_AVX2
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#ifdef COMPILE_SIMD_AVX512_SPR
|
|
9
|
+
|
|
10
|
+
#define THE_SIMD_LEVEL SIMDLevel::AVX512_SPR
|
|
11
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
12
|
+
#include <faiss/utils/hamming_distance/hamming_computer-avx512_spr.h>
|
|
13
|
+
#include <faiss/utils/hamming_distance/hamming_impl.h>
|
|
14
|
+
|
|
15
|
+
#endif // COMPILE_SIMD_AVX512_SPR
|
|
@@ -10,10 +10,10 @@
|
|
|
10
10
|
|
|
11
11
|
// AVX512 HammingComputer and GenHammingComputer specializations.
|
|
12
12
|
// Types without custom AVX512 code inherit from the NONE specializations
|
|
13
|
-
// in hamming_computer-generic.h.
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// leverage SSE/AVX2 intrinsics.
|
|
13
|
+
// in hamming_computer-generic.h. HammingComputer64 and
|
|
14
|
+
// HammingComputerDefault use scalar popcount here; the VPOPCNTDQ fast
|
|
15
|
+
// path lives in hamming_computer-avx512_spr.h (AVX512_SPR level).
|
|
16
|
+
// GenHammingComputer classes leverage SSE/AVX2 intrinsics.
|
|
17
17
|
|
|
18
18
|
#include <cassert>
|
|
19
19
|
#include <cstdint>
|
|
@@ -74,18 +74,10 @@ struct HammingComputer64_tpl<SIMDLevel::AVX512> {
|
|
|
74
74
|
|
|
75
75
|
inline int hamming(const uint8_t* b8) const {
|
|
76
76
|
const uint64_t* b = reinterpret_cast<const uint64_t*>(b8);
|
|
77
|
-
#ifdef __AVX512VPOPCNTDQ__
|
|
78
|
-
__m512i vxor =
|
|
79
|
-
_mm512_xor_si512(_mm512_loadu_si512(a), _mm512_loadu_si512(b));
|
|
80
|
-
__m512i vpcnt = _mm512_popcnt_epi64(vxor);
|
|
81
|
-
// reduce performs better than adding the lower and higher parts
|
|
82
|
-
return _mm512_reduce_add_epi32(vpcnt);
|
|
83
|
-
#else
|
|
84
77
|
return popcount64(b[0] ^ a0) + popcount64(b[1] ^ a1) +
|
|
85
78
|
popcount64(b[2] ^ a2) + popcount64(b[3] ^ a3) +
|
|
86
79
|
popcount64(b[4] ^ a4) + popcount64(b[5] ^ a5) +
|
|
87
80
|
popcount64(b[6] ^ a6) + popcount64(b[7] ^ a7);
|
|
88
|
-
#endif
|
|
89
81
|
}
|
|
90
82
|
|
|
91
83
|
inline static constexpr int get_code_size() {
|
|
@@ -112,27 +104,11 @@ struct HammingComputerDefault_tpl<SIMDLevel::AVX512> {
|
|
|
112
104
|
}
|
|
113
105
|
|
|
114
106
|
int hamming(const uint8_t* b8) const {
|
|
115
|
-
int accu = 0;
|
|
116
|
-
|
|
117
107
|
const uint64_t* a64 = reinterpret_cast<const uint64_t*>(a8);
|
|
118
108
|
const uint64_t* b64 = reinterpret_cast<const uint64_t*>(b8);
|
|
119
109
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
int quotient64 = quotient8 / 8;
|
|
123
|
-
for (; i < quotient64; ++i) {
|
|
124
|
-
__m512i vxor = _mm512_xor_si512(
|
|
125
|
-
_mm512_loadu_si512(&a64[i * 8]),
|
|
126
|
-
_mm512_loadu_si512(&b64[i * 8]));
|
|
127
|
-
__m512i vpcnt = _mm512_popcnt_epi64(vxor);
|
|
128
|
-
// reduce performs better than adding the lower and higher parts
|
|
129
|
-
accu += _mm512_reduce_add_epi32(vpcnt);
|
|
130
|
-
}
|
|
131
|
-
i *= 8;
|
|
132
|
-
#endif
|
|
133
|
-
accu += hamming_popcount_tail(
|
|
134
|
-
a64, b64, i, quotient8, a8, b8, remainder8);
|
|
135
|
-
return accu;
|
|
110
|
+
return hamming_popcount_tail(
|
|
111
|
+
a64, b64, 0, quotient8, a8, b8, remainder8);
|
|
136
112
|
}
|
|
137
113
|
|
|
138
114
|
inline int get_code_size() const {
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#ifndef HAMMING_COMPUTER_AVX512_SPR_H
|
|
9
|
+
#define HAMMING_COMPUTER_AVX512_SPR_H
|
|
10
|
+
|
|
11
|
+
// AVX512_SPR HammingComputer specializations using VPOPCNTDQ.
|
|
12
|
+
// On Sapphire Rapids+, _mm512_popcnt_epi64 (and _mm256_popcnt_epi64 with VL)
|
|
13
|
+
// are unconditionally available. This gives a faster path than the scalar
|
|
14
|
+
// popcount fallback used in the base AVX512 specializations when compiled
|
|
15
|
+
// without -mavx512vpopcntdq.
|
|
16
|
+
|
|
17
|
+
#include <cassert>
|
|
18
|
+
#include <cstdint>
|
|
19
|
+
|
|
20
|
+
#include <faiss/impl/platform_macros.h>
|
|
21
|
+
#include <faiss/utils/hamming_distance/hamming_computer-avx512.h>
|
|
22
|
+
|
|
23
|
+
#include <immintrin.h>
|
|
24
|
+
|
|
25
|
+
namespace faiss {
|
|
26
|
+
|
|
27
|
+
/***************************************************************************
|
|
28
|
+
* AVX512_SPR inheriting specializations for types without custom SPR code.
|
|
29
|
+
***************************************************************************/
|
|
30
|
+
|
|
31
|
+
#define FAISS_INHERIT_HAMMING_SPR(Class) \
|
|
32
|
+
template <> \
|
|
33
|
+
struct Class## \
|
|
34
|
+
_tpl<SIMDLevel::AVX512_SPR> : Class##_tpl<SIMDLevel::AVX512> { \
|
|
35
|
+
using Class##_tpl<SIMDLevel::AVX512>::Class##_tpl; \
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
FAISS_INHERIT_HAMMING_SPR(HammingComputer16);
|
|
39
|
+
FAISS_INHERIT_HAMMING_SPR(HammingComputer20);
|
|
40
|
+
FAISS_INHERIT_HAMMING_SPR(GenHammingComputer8);
|
|
41
|
+
FAISS_INHERIT_HAMMING_SPR(GenHammingComputer16);
|
|
42
|
+
FAISS_INHERIT_HAMMING_SPR(GenHammingComputer32);
|
|
43
|
+
FAISS_INHERIT_HAMMING_SPR(GenHammingComputerM8);
|
|
44
|
+
|
|
45
|
+
#undef FAISS_INHERIT_HAMMING_SPR
|
|
46
|
+
|
|
47
|
+
/***************************************************************************
|
|
48
|
+
* Custom AVX512_SPR specializations using VPOPCNTDQ.
|
|
49
|
+
***************************************************************************/
|
|
50
|
+
|
|
51
|
+
template <>
|
|
52
|
+
struct HammingComputer32_tpl<SIMDLevel::AVX512_SPR> {
|
|
53
|
+
const uint8_t* a8;
|
|
54
|
+
|
|
55
|
+
HammingComputer32_tpl() {}
|
|
56
|
+
|
|
57
|
+
HammingComputer32_tpl(const uint8_t* a8_in, int code_size) {
|
|
58
|
+
set(a8_in, code_size);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
void set(const uint8_t* a8_in, FAISS_MAYBE_UNUSED int code_size) {
|
|
62
|
+
assert(code_size == 32);
|
|
63
|
+
a8 = a8_in;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
inline int hamming(const uint8_t* b8) const {
|
|
67
|
+
__m256i va = _mm256_loadu_si256((const __m256i*)a8);
|
|
68
|
+
__m256i vb = _mm256_loadu_si256((const __m256i*)b8);
|
|
69
|
+
__m256i vxor = _mm256_xor_si256(va, vb);
|
|
70
|
+
__m256i vpcnt = _mm256_popcnt_epi64(vxor);
|
|
71
|
+
__m128i lo = _mm256_castsi256_si128(vpcnt);
|
|
72
|
+
__m128i hi = _mm256_extracti128_si256(vpcnt, 1);
|
|
73
|
+
__m128i sum = _mm_add_epi64(lo, hi);
|
|
74
|
+
return static_cast<int>(
|
|
75
|
+
_mm_extract_epi64(sum, 0) + _mm_extract_epi64(sum, 1));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
inline static constexpr int get_code_size() {
|
|
79
|
+
return 32;
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
template <>
|
|
84
|
+
struct HammingComputer64_tpl<SIMDLevel::AVX512_SPR> {
|
|
85
|
+
const uint8_t* a8;
|
|
86
|
+
|
|
87
|
+
HammingComputer64_tpl() {}
|
|
88
|
+
|
|
89
|
+
HammingComputer64_tpl(const uint8_t* a8_in, int code_size) {
|
|
90
|
+
set(a8_in, code_size);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
void set(const uint8_t* a8_in, FAISS_MAYBE_UNUSED int code_size) {
|
|
94
|
+
assert(code_size == 64);
|
|
95
|
+
a8 = a8_in;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
inline int hamming(const uint8_t* b8) const {
|
|
99
|
+
__m512i vxor = _mm512_xor_si512(
|
|
100
|
+
_mm512_loadu_si512(a8), _mm512_loadu_si512(b8));
|
|
101
|
+
__m512i vpcnt = _mm512_popcnt_epi64(vxor);
|
|
102
|
+
return _mm512_reduce_add_epi32(vpcnt);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
inline static constexpr int get_code_size() {
|
|
106
|
+
return 64;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
template <>
|
|
111
|
+
struct HammingComputerDefault_tpl<SIMDLevel::AVX512_SPR> {
|
|
112
|
+
const uint8_t* a8;
|
|
113
|
+
int quotient8;
|
|
114
|
+
int remainder8;
|
|
115
|
+
|
|
116
|
+
HammingComputerDefault_tpl() {}
|
|
117
|
+
|
|
118
|
+
HammingComputerDefault_tpl(const uint8_t* a8_in, int code_size) {
|
|
119
|
+
set(a8_in, code_size);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
void set(const uint8_t* a8_2, int code_size) {
|
|
123
|
+
this->a8 = a8_2;
|
|
124
|
+
quotient8 = code_size / 8;
|
|
125
|
+
remainder8 = code_size % 8;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
int hamming(const uint8_t* b8) const {
|
|
129
|
+
int accu = 0;
|
|
130
|
+
|
|
131
|
+
const uint64_t* a64 = reinterpret_cast<const uint64_t*>(a8);
|
|
132
|
+
const uint64_t* b64 = reinterpret_cast<const uint64_t*>(b8);
|
|
133
|
+
|
|
134
|
+
int i = 0;
|
|
135
|
+
int quotient64 = quotient8 / 8;
|
|
136
|
+
for (; i < quotient64; ++i) {
|
|
137
|
+
__m512i vxor = _mm512_xor_si512(
|
|
138
|
+
_mm512_loadu_si512(&a64[i * 8]),
|
|
139
|
+
_mm512_loadu_si512(&b64[i * 8]));
|
|
140
|
+
__m512i vpcnt = _mm512_popcnt_epi64(vxor);
|
|
141
|
+
accu += _mm512_reduce_add_epi32(vpcnt);
|
|
142
|
+
}
|
|
143
|
+
i *= 8;
|
|
144
|
+
|
|
145
|
+
// Handle 4-word (256-bit) remainder with VPOPCNTDQ VL
|
|
146
|
+
if (i + 4 <= quotient8) {
|
|
147
|
+
__m256i vxor = _mm256_xor_si256(
|
|
148
|
+
_mm256_loadu_si256((const __m256i*)&a64[i]),
|
|
149
|
+
_mm256_loadu_si256((const __m256i*)&b64[i]));
|
|
150
|
+
__m256i vpcnt = _mm256_popcnt_epi64(vxor);
|
|
151
|
+
__m128i lo = _mm256_castsi256_si128(vpcnt);
|
|
152
|
+
__m128i hi = _mm256_extracti128_si256(vpcnt, 1);
|
|
153
|
+
__m128i sum = _mm_add_epi64(lo, hi);
|
|
154
|
+
accu += static_cast<int>(
|
|
155
|
+
_mm_extract_epi64(sum, 0) + _mm_extract_epi64(sum, 1));
|
|
156
|
+
i += 4;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
accu += hamming_popcount_tail(
|
|
160
|
+
a64, b64, i, quotient8, a8, b8, remainder8);
|
|
161
|
+
return accu;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
inline int get_code_size() const {
|
|
165
|
+
return quotient8 * 8 + remainder8;
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
} // namespace faiss
|
|
170
|
+
|
|
171
|
+
#endif
|