faiss 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/Clustering.h +8 -0
- data/vendor/faiss/faiss/Index.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +23 -2
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +12 -140
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +0 -6
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +6 -14
- data/vendor/faiss/faiss/IndexBinaryIVF.h +8 -1
- data/vendor/faiss/faiss/IndexFlat.cpp +8 -4
- data/vendor/faiss/faiss/IndexFlat.h +2 -2
- data/vendor/faiss/faiss/IndexHNSW.cpp +301 -91
- data/vendor/faiss/faiss/IndexHNSW.h +31 -7
- data/vendor/faiss/faiss/IndexIDMap.cpp +11 -6
- data/vendor/faiss/faiss/IndexIVF.cpp +19 -4
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +9 -1
- data/vendor/faiss/faiss/IndexIVFPQ.h +11 -0
- data/vendor/faiss/faiss/IndexShards.cpp +3 -3
- data/vendor/faiss/faiss/IndexShardsIVF.cpp +4 -3
- data/vendor/faiss/faiss/SuperKMeans.cpp +31 -8
- data/vendor/faiss/faiss/SuperKMeans.h +3 -2
- data/vendor/faiss/faiss/VectorTransform.cpp +38 -0
- data/vendor/faiss/faiss/VectorTransform.h +3 -0
- data/vendor/faiss/faiss/clone_index.cpp +3 -0
- data/vendor/faiss/faiss/gpu/GpuClonerOptions.h +1 -5
- data/vendor/faiss/faiss/gpu/GpuDistance.h +2 -5
- data/vendor/faiss/faiss/gpu/GpuIndex.h +4 -5
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +65 -42
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +5 -2
- data/vendor/faiss/faiss/impl/HNSW.cpp +419 -115
- data/vendor/faiss/faiss/impl/HNSW.h +46 -7
- data/vendor/faiss/faiss/impl/IDSelector.h +44 -0
- data/vendor/faiss/faiss/impl/NNDescent.cpp +10 -3
- data/vendor/faiss/faiss/impl/Panorama.h +11 -2
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +4 -4
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +1 -1
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +74 -8
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +47 -4
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -2
- data/vendor/faiss/faiss/impl/ResultHandler.h +9 -3
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +32 -30
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +6 -10
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +89 -13
- data/vendor/faiss/faiss/impl/expanded_scanners.h +5 -1
- data/vendor/faiss/faiss/impl/fast_scan/decompose_qbs.h +1 -0
- data/vendor/faiss/faiss/impl/index_read.cpp +130 -29
- data/vendor/faiss/faiss/impl/index_write.cpp +31 -26
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h +73 -39
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +4 -4
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +6 -1
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +2 -2
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +6 -2
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +37 -17
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +109 -10
- data/vendor/faiss/faiss/impl/simd_dispatch.h +40 -22
- data/vendor/faiss/faiss/index_factory.cpp +8 -1
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +7 -4
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +10 -4
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +16 -4
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +8 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +4 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +9 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +2 -1
- data/vendor/faiss/faiss/utils/distances.cpp +20 -13
- data/vendor/faiss/faiss/utils/distances_dispatch.h +30 -24
- data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp +1 -1
- data/vendor/faiss/faiss/utils/hamming.cpp +15 -9
- data/vendor/faiss/faiss/utils/hamming.h +10 -1
- data/vendor/faiss/faiss/utils/hamming_distance/common.h +14 -3
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_vpopcnt.cpp +24 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +1 -1
- data/vendor/faiss/faiss/utils/hamming_distance/{hamming_computer-avx512_spr.h → hamming_computer-avx512_vpopcnt.h} +85 -24
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_impl.h +141 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_arm_sve.cpp +194 -30
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +93 -55
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +57 -40
- data/vendor/faiss/faiss/utils/simd_impl/{rabitq_avx512_spr.cpp → rabitq_avx512_vpopcnt.cpp} +21 -24
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h +2 -7
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h +6 -1
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp +34 -0
- data/vendor/faiss/faiss/utils/simd_levels.cpp +172 -67
- data/vendor/faiss/faiss/utils/simd_levels.h +26 -15
- metadata +6 -5
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +0 -15
|
@@ -7,8 +7,15 @@
|
|
|
7
7
|
|
|
8
8
|
#include <faiss/utils/simd_levels.h>
|
|
9
9
|
|
|
10
|
+
#include <cstdint>
|
|
11
|
+
#include <cstdio>
|
|
10
12
|
#include <cstdlib>
|
|
11
13
|
|
|
14
|
+
#if defined(_MSC_VER)
|
|
15
|
+
// __cpuidex, _xgetbv
|
|
16
|
+
#include <intrin.h>
|
|
17
|
+
#endif
|
|
18
|
+
|
|
12
19
|
#include <faiss/impl/FaissAssert.h>
|
|
13
20
|
#include <faiss/impl/simd_dispatch.h>
|
|
14
21
|
|
|
@@ -24,6 +31,16 @@ uint64_t SIMDConfig::supported_simd_levels = 0;
|
|
|
24
31
|
// detect_x86_uarch_flags() at load time.
|
|
25
32
|
bool SIMDConfig::avx512_split = false;
|
|
26
33
|
|
|
34
|
+
// Resolved here rather than in the header so that dependents, which never see
|
|
35
|
+
// FAISS_ENABLE_DD, still get the answer for the faiss they link against.
|
|
36
|
+
bool SIMDConfig::has_dynamic_dispatch() {
|
|
37
|
+
#ifdef FAISS_ENABLE_DD
|
|
38
|
+
return true;
|
|
39
|
+
#else
|
|
40
|
+
return false;
|
|
41
|
+
#endif
|
|
42
|
+
}
|
|
43
|
+
|
|
27
44
|
// ARM SVE runtime detection
|
|
28
45
|
#if defined(__aarch64__) || defined(_M_ARM64)
|
|
29
46
|
|
|
@@ -57,49 +74,111 @@ static bool has_sve() {
|
|
|
57
74
|
}
|
|
58
75
|
#endif
|
|
59
76
|
|
|
77
|
+
namespace {
|
|
78
|
+
|
|
79
|
+
#if defined(__x86_64__) || defined(_M_X64)
|
|
80
|
+
|
|
81
|
+
// MSVC and clang-cl do not support GNU-style
|
|
82
|
+
// 64-bit inline assembly, MSVC defines _M_X64 instead of __x86_64__
|
|
83
|
+
|
|
84
|
+
#if defined(_MSC_VER)
|
|
85
|
+
|
|
86
|
+
[[maybe_unused]] void cpuid_count(
|
|
87
|
+
unsigned int leaf,
|
|
88
|
+
unsigned int subleaf,
|
|
89
|
+
unsigned int regs[4]) {
|
|
90
|
+
int r[4];
|
|
91
|
+
__cpuidex(r, static_cast<int>(leaf), static_cast<int>(subleaf));
|
|
92
|
+
for (int i = 0; i < 4; i++) {
|
|
93
|
+
regs[i] = static_cast<unsigned int>(r[i]);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
[[maybe_unused]] uint64_t xgetbv0() {
|
|
98
|
+
return static_cast<uint64_t>(_xgetbv(0));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
#else // GCC / Clang
|
|
102
|
+
|
|
103
|
+
[[maybe_unused]] void cpuid_count(
|
|
104
|
+
unsigned int leaf,
|
|
105
|
+
unsigned int subleaf,
|
|
106
|
+
unsigned int regs[4]) {
|
|
107
|
+
asm volatile("cpuid"
|
|
108
|
+
: "=a"(regs[0]), "=b"(regs[1]), "=c"(regs[2]), "=d"(regs[3])
|
|
109
|
+
: "a"(leaf), "c"(subleaf));
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
[[maybe_unused]] uint64_t xgetbv0() {
|
|
113
|
+
unsigned int eax, edx;
|
|
114
|
+
asm volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
|
|
115
|
+
return eax | (static_cast<uint64_t>(edx) << 32);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
#endif // _MSC_VER
|
|
119
|
+
|
|
60
120
|
// Detect x86 microarchitecture flags used for kernel routing. Uses raw
|
|
61
121
|
// cpuid so it is safe to run on any CPU regardless of compiled SIMD level.
|
|
62
|
-
#if defined(__x86_64__)
|
|
63
|
-
namespace {
|
|
64
122
|
void detect_x86_uarch_flags() {
|
|
65
|
-
unsigned int
|
|
123
|
+
unsigned int regs[4];
|
|
66
124
|
|
|
67
125
|
// Vendor string (CPUID.0): "AuthenticAMD" is EBX="Auth", EDX="enti",
|
|
68
126
|
// ECX="cAMD".
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
: "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
|
|
73
|
-
: "a"(eax), "c"(ecx));
|
|
74
|
-
const bool is_amd =
|
|
75
|
-
ebx == 0x68747541u && edx == 0x69746e65u && ecx == 0x444d4163u;
|
|
127
|
+
cpuid_count(0, 0, regs);
|
|
128
|
+
const bool is_amd = regs[1] == 0x68747541u && regs[3] == 0x69746e65u &&
|
|
129
|
+
regs[2] == 0x444d4163u;
|
|
76
130
|
|
|
77
131
|
// Family/model (CPUID.1 EAX).
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
: "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
|
|
82
|
-
: "a"(eax), "c"(ecx));
|
|
83
|
-
const unsigned int base_family = (eax >> 8) & 0xfu;
|
|
132
|
+
cpuid_count(1, 0, regs);
|
|
133
|
+
const unsigned int eax1 = regs[0];
|
|
134
|
+
const unsigned int base_family = (eax1 >> 8) & 0xfu;
|
|
84
135
|
const unsigned int display_family =
|
|
85
|
-
base_family + (base_family == 0xfu ? ((
|
|
136
|
+
base_family + (base_family == 0xfu ? ((eax1 >> 20) & 0xffu) : 0u);
|
|
86
137
|
// AMD Zen 4 / Zen 4c (Bergamo) is family 0x19 and splits AVX-512.
|
|
87
138
|
// (Zen 5, family 0x1A, has a native 512-bit datapath and is excluded.)
|
|
88
139
|
SIMDConfig::avx512_split = is_amd && display_family == 0x19u;
|
|
89
140
|
}
|
|
90
|
-
|
|
91
|
-
#else
|
|
92
|
-
|
|
141
|
+
|
|
142
|
+
#else // Not x86-64
|
|
143
|
+
|
|
93
144
|
void detect_x86_uarch_flags() {}
|
|
145
|
+
|
|
146
|
+
#endif // defined(__x86_64__) || defined(_M_X64)
|
|
147
|
+
|
|
94
148
|
} // namespace
|
|
149
|
+
|
|
150
|
+
/// Must mirror the case labels in with_selected_simd_levels. A static build
|
|
151
|
+
/// defines a COMPILE_SIMD_* macro for every level whose sources it compiles,
|
|
152
|
+
/// not only for SINGLE_SIMD_LEVEL, so this reports a correct set in both
|
|
153
|
+
/// modes and the result is not a single level.
|
|
154
|
+
uint64_t compiled_simd_levels() {
|
|
155
|
+
uint64_t mask = uint64_t(1) << static_cast<int>(SIMDLevel::NONE);
|
|
156
|
+
#ifdef COMPILE_SIMD_AVX2
|
|
157
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX2);
|
|
158
|
+
#endif
|
|
159
|
+
#ifdef COMPILE_SIMD_AVX512
|
|
160
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX512);
|
|
161
|
+
#endif
|
|
162
|
+
#ifdef COMPILE_SIMD_AVX512_VPOPCNT
|
|
163
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX512_VPOPCNT);
|
|
164
|
+
#endif
|
|
165
|
+
#ifdef COMPILE_SIMD_AVX512_SPR
|
|
166
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX512_SPR);
|
|
167
|
+
#endif
|
|
168
|
+
#ifdef COMPILE_SIMD_ARM_NEON
|
|
169
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::ARM_NEON);
|
|
170
|
+
#endif
|
|
171
|
+
#ifdef COMPILE_SIMD_ARM_SVE
|
|
172
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::ARM_SVE);
|
|
173
|
+
#endif
|
|
174
|
+
#ifdef COMPILE_SIMD_RISCV_RVV
|
|
175
|
+
mask |= uint64_t(1) << static_cast<int>(SIMDLevel::RISCV_RVV);
|
|
95
176
|
#endif
|
|
177
|
+
return mask;
|
|
178
|
+
}
|
|
96
179
|
|
|
97
180
|
#ifdef FAISS_ENABLE_DD
|
|
98
181
|
|
|
99
|
-
// =============================================================================
|
|
100
|
-
// Dynamic Dispatch (DD) mode implementation
|
|
101
|
-
// =============================================================================
|
|
102
|
-
|
|
103
182
|
// Static initializer to run constructor at load time
|
|
104
183
|
// NOLINTNEXTLINE(facebook-avoid-non-const-global-variables)
|
|
105
184
|
static SIMDConfig simd_config_initializer;
|
|
@@ -112,7 +191,22 @@ SIMDConfig::SIMDConfig(const char** faiss_simd_level_env) {
|
|
|
112
191
|
if (!env_var) {
|
|
113
192
|
level = auto_detect_simd_level();
|
|
114
193
|
} else {
|
|
115
|
-
level
|
|
194
|
+
// Forcing a level the CPU lacks is allowed. Forcing one the binary
|
|
195
|
+
// does not hold is not: dispatch would fall to NONE and skip every
|
|
196
|
+
// level between. Walk down to the nearest compiled level instead.
|
|
197
|
+
const uint64_t compiled = compiled_simd_levels();
|
|
198
|
+
const SIMDLevel requested = to_simd_level(env_var);
|
|
199
|
+
level = requested;
|
|
200
|
+
while (((compiled >> static_cast<int>(level)) & 1) == 0) {
|
|
201
|
+
level = get_simd_fallback(level);
|
|
202
|
+
}
|
|
203
|
+
if (level != requested) {
|
|
204
|
+
fprintf(stderr,
|
|
205
|
+
"faiss: FAISS_SIMD_LEVEL=%s is not compiled into this "
|
|
206
|
+
"build, using %s instead\n",
|
|
207
|
+
to_string(requested).c_str(),
|
|
208
|
+
to_string(level).c_str());
|
|
209
|
+
}
|
|
116
210
|
supported_simd_levels = (1 << static_cast<int>(level));
|
|
117
211
|
}
|
|
118
212
|
supported_simd_levels |= (1 << static_cast<int>(SIMDLevel::NONE));
|
|
@@ -144,80 +238,84 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
|
|
|
144
238
|
|
|
145
239
|
detect_x86_uarch_flags();
|
|
146
240
|
|
|
147
|
-
#if defined(__x86_64__) && \
|
|
241
|
+
#if (defined(__x86_64__) || defined(_M_X64)) && \
|
|
148
242
|
(defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512))
|
|
149
|
-
unsigned int
|
|
243
|
+
unsigned int regs[4];
|
|
150
244
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
asm volatile("cpuid"
|
|
154
|
-
: "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
|
|
155
|
-
: "a"(eax), "c"(ecx));
|
|
245
|
+
cpuid_count(1, 0, regs);
|
|
246
|
+
unsigned int ecx1 = regs[2];
|
|
156
247
|
|
|
157
|
-
bool has_avx = (
|
|
248
|
+
bool has_avx = (ecx1 & (1 << 28)) != 0;
|
|
158
249
|
|
|
159
250
|
bool has_xsave_osxsave =
|
|
160
|
-
(
|
|
251
|
+
(ecx1 & ((1 << 26) | (1 << 27))) == ((1 << 26) | (1 << 27));
|
|
161
252
|
|
|
162
253
|
bool avx_supported = false;
|
|
163
254
|
if (has_avx && has_xsave_osxsave) {
|
|
164
|
-
|
|
165
|
-
asm volatile("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0));
|
|
166
|
-
avx_supported = (xcr0 & 6) == 6;
|
|
255
|
+
avx_supported = (xgetbv0() & 6) == 6;
|
|
167
256
|
}
|
|
168
257
|
|
|
169
258
|
if (avx_supported) {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
//
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
259
|
+
cpuid_count(7, 0, regs);
|
|
260
|
+
unsigned int ebx7 = regs[1];
|
|
261
|
+
// EDX of CPUID leaf 7 subleaf 0 carries AVX512_FP16 (bit 23),
|
|
262
|
+
// needed for the SPR detection below. Kept in a local so a later
|
|
263
|
+
// xgetbv cannot clobber it.
|
|
264
|
+
unsigned int cpuid7_edx = regs[3];
|
|
265
|
+
// Leaf 7 subleaf 0, not leaf 1: leaf 1 ECX holds unrelated bits at
|
|
266
|
+
// these positions.
|
|
267
|
+
unsigned int ecx7 = regs[2];
|
|
268
|
+
[[maybe_unused]] bool has_avx512_vnni = (ecx7 & (1 << 11)) != 0;
|
|
269
|
+
[[maybe_unused]] bool has_avx512_vpopcntdq = (ecx7 & (1 << 14)) != 0;
|
|
270
|
+
// Bit 12 = AVX512_BITALG, needed for the byte-wise popcount kernels.
|
|
271
|
+
[[maybe_unused]] bool has_avx512_bitalg = (ecx7 & (1 << 12)) != 0;
|
|
272
|
+
|
|
273
|
+
uint64_t xcr0 = xgetbv0();
|
|
181
274
|
|
|
182
275
|
#if defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512)
|
|
183
|
-
bool has_avx2 = (
|
|
276
|
+
bool has_avx2 = (ebx7 & (1 << 5)) != 0;
|
|
184
277
|
if (has_avx2) {
|
|
185
278
|
supported_simd_levels |= (1 << static_cast<int>(SIMDLevel::AVX2));
|
|
186
279
|
detected_level = SIMDLevel::AVX2;
|
|
187
280
|
}
|
|
188
281
|
|
|
189
282
|
#if defined(COMPILE_SIMD_AVX512)
|
|
190
|
-
bool cpu_has_avx512f = (
|
|
283
|
+
bool cpu_has_avx512f = (ebx7 & (1 << 16)) != 0;
|
|
191
284
|
bool os_supports_avx512 = (xcr0 & 0xE0) == 0xE0;
|
|
192
285
|
bool has_avx512f = cpu_has_avx512f && os_supports_avx512;
|
|
193
286
|
if (has_avx512f) {
|
|
194
|
-
bool has_avx512cd = (
|
|
195
|
-
bool has_avx512vl = (
|
|
196
|
-
bool has_avx512dq = (
|
|
197
|
-
bool has_avx512bw = (
|
|
287
|
+
bool has_avx512cd = (ebx7 & (1 << 28)) != 0;
|
|
288
|
+
bool has_avx512vl = (ebx7 & (1 << 31)) != 0;
|
|
289
|
+
bool has_avx512dq = (ebx7 & (1 << 17)) != 0;
|
|
290
|
+
bool has_avx512bw = (ebx7 & (1 << 30)) != 0;
|
|
198
291
|
if (has_avx512bw && has_avx512cd && has_avx512vl && has_avx512dq) {
|
|
199
292
|
detected_level = SIMDLevel::AVX512;
|
|
200
293
|
supported_simd_levels |=
|
|
201
294
|
(1 << static_cast<int>(SIMDLevel::AVX512));
|
|
202
295
|
|
|
296
|
+
#if defined(COMPILE_SIMD_AVX512_VPOPCNT)
|
|
297
|
+
if (has_avx512_vpopcntdq && has_avx512_bitalg) {
|
|
298
|
+
detected_level = SIMDLevel::AVX512_VPOPCNT;
|
|
299
|
+
supported_simd_levels |=
|
|
300
|
+
(1 << static_cast<int>(SIMDLevel::AVX512_VPOPCNT));
|
|
301
|
+
}
|
|
302
|
+
#endif
|
|
303
|
+
|
|
203
304
|
#if defined(COMPILE_SIMD_AVX512_SPR)
|
|
204
305
|
// Check for Sapphire Rapids features.
|
|
205
|
-
// The SPR code path is compiled with
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
306
|
+
// The SPR code path is compiled with AVX512_VNNI, BF16,
|
|
307
|
+
// FP16 and VPOPCNTDQ, and falls back to the VPOPCNT kernels,
|
|
308
|
+
// which need BITALG. All five features are required.
|
|
309
|
+
// AMD Zen 4 has VPOPCNTDQ and BF16 but not FP16, and must
|
|
310
|
+
// remain on the AVX512_VPOPCNT level.
|
|
209
311
|
// CPUID EAX=7, ECX=1: EAX bit 5 = AVX512_BF16
|
|
210
312
|
// CPUID EAX=7, ECX=0: EDX bit 23 = AVX512_FP16
|
|
211
313
|
// (Linux: X86_FEATURE_AVX512_FP16 = 18*32+23)
|
|
212
314
|
bool has_avx512_fp16 = (cpuid7_edx & (1 << 23)) != 0;
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
: "=a"(eax1), "=b"(ebx1), "=c"(ecx1), "=d"(edx1)
|
|
218
|
-
: "a"(eax1), "c"(ecx1));
|
|
219
|
-
bool has_avx512_bf16 = (eax1 & (1 << 5)) != 0;
|
|
220
|
-
if (has_avx512_bf16 && has_avx512_fp16) {
|
|
315
|
+
cpuid_count(7, 1, regs);
|
|
316
|
+
const bool has_avx512_bf16 = (regs[0] & (1 << 5)) != 0;
|
|
317
|
+
if (has_avx512_vnni && has_avx512_vpopcntdq &&
|
|
318
|
+
has_avx512_bitalg && has_avx512_bf16 && has_avx512_fp16) {
|
|
221
319
|
detected_level = SIMDLevel::AVX512_SPR;
|
|
222
320
|
supported_simd_levels |=
|
|
223
321
|
(1 << static_cast<int>(SIMDLevel::AVX512_SPR));
|
|
@@ -228,7 +326,7 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
|
|
|
228
326
|
#endif // defined(COMPILE_SIMD_AVX512)
|
|
229
327
|
#endif // defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512)
|
|
230
328
|
}
|
|
231
|
-
#endif // defined(__x86_64__)
|
|
329
|
+
#endif // defined(__x86_64__) || defined(_M_X64)
|
|
232
330
|
|
|
233
331
|
#ifdef COMPILE_SIMD_ARM_NEON
|
|
234
332
|
// ARM NEON is standard on aarch64
|
|
@@ -311,6 +409,8 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
|
|
|
311
409
|
// In static mode, return the compiled-in level
|
|
312
410
|
#if defined(COMPILE_SIMD_AVX512_SPR)
|
|
313
411
|
return SIMDLevel::AVX512_SPR;
|
|
412
|
+
#elif defined(COMPILE_SIMD_AVX512_VPOPCNT)
|
|
413
|
+
return SIMDLevel::AVX512_VPOPCNT;
|
|
314
414
|
#elif defined(COMPILE_SIMD_AVX512)
|
|
315
415
|
return SIMDLevel::AVX512;
|
|
316
416
|
#elif defined(COMPILE_SIMD_AVX2)
|
|
@@ -345,6 +445,8 @@ std::string to_string(SIMDLevel level) {
|
|
|
345
445
|
return "AVX2";
|
|
346
446
|
case SIMDLevel::AVX512:
|
|
347
447
|
return "AVX512";
|
|
448
|
+
case SIMDLevel::AVX512_VPOPCNT:
|
|
449
|
+
return "AVX512_VPOPCNT";
|
|
348
450
|
case SIMDLevel::AVX512_SPR:
|
|
349
451
|
return "AVX512_SPR";
|
|
350
452
|
case SIMDLevel::ARM_NEON:
|
|
@@ -369,6 +471,9 @@ SIMDLevel to_simd_level(const std::string& level_str) {
|
|
|
369
471
|
if (level_str == "AVX512") {
|
|
370
472
|
return SIMDLevel::AVX512;
|
|
371
473
|
}
|
|
474
|
+
if (level_str == "AVX512_VPOPCNT") {
|
|
475
|
+
return SIMDLevel::AVX512_VPOPCNT;
|
|
476
|
+
}
|
|
372
477
|
if (level_str == "AVX512_SPR") {
|
|
373
478
|
return SIMDLevel::AVX512_SPR;
|
|
374
479
|
}
|
|
@@ -21,13 +21,18 @@ enum class SIMDLevel {
|
|
|
21
21
|
// x86
|
|
22
22
|
AVX2,
|
|
23
23
|
AVX512,
|
|
24
|
-
AVX512_SPR, // Sapphire Rapids:
|
|
24
|
+
AVX512_SPR, // Sapphire Rapids: AVX512_VPOPCNT + BF16 + FP16 + VNNI
|
|
25
25
|
// arm & aarch64
|
|
26
26
|
ARM_NEON,
|
|
27
27
|
ARM_SVE, // Scalable Vector Extension (ARMv8.2+)
|
|
28
28
|
// riscv
|
|
29
29
|
RISCV_RVV, // RISC-V Vector Extension (rv64gcv)
|
|
30
30
|
|
|
31
|
+
// Appended to preserve the numeric values of the existing public enum.
|
|
32
|
+
// AVX-512 core features plus AVX512_VPOPCNTDQ and AVX512_BITALG
|
|
33
|
+
// (Ice Lake, Zen 4, etc.).
|
|
34
|
+
AVX512_VPOPCNT,
|
|
35
|
+
|
|
31
36
|
COUNT
|
|
32
37
|
};
|
|
33
38
|
|
|
@@ -52,6 +57,8 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::NONE;
|
|
|
52
57
|
#else
|
|
53
58
|
#if defined(COMPILE_SIMD_AVX512_SPR)
|
|
54
59
|
inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::AVX512_SPR;
|
|
60
|
+
#elif defined(COMPILE_SIMD_AVX512_VPOPCNT)
|
|
61
|
+
inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::AVX512_VPOPCNT;
|
|
55
62
|
#elif defined(COMPILE_SIMD_AVX512)
|
|
56
63
|
inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::AVX512;
|
|
57
64
|
#elif defined(COMPILE_SIMD_AVX2)
|
|
@@ -71,7 +78,7 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::NONE;
|
|
|
71
78
|
* Helper to select the appropriate 256-bit SIMD level.
|
|
72
79
|
*
|
|
73
80
|
* For 256-bit SIMD types (simd16uint16, simd32uint8, etc.), maps:
|
|
74
|
-
* AVX512/AVX512_SPR → AVX2
|
|
81
|
+
* AVX512/AVX512_VPOPCNT/AVX512_SPR → AVX2
|
|
75
82
|
* AVX2 → AVX2
|
|
76
83
|
* ARM_NEON/ARM_SVE → ARM_NEON
|
|
77
84
|
* NONE → NONE
|
|
@@ -79,7 +86,8 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::NONE;
|
|
|
79
86
|
template <SIMDLevel SL>
|
|
80
87
|
struct simd256_level_selector {
|
|
81
88
|
static constexpr SIMDLevel value =
|
|
82
|
-
(SL == SIMDLevel::AVX512 || SL == SIMDLevel::
|
|
89
|
+
(SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_VPOPCNT ||
|
|
90
|
+
SL == SIMDLevel::AVX512_SPR)
|
|
83
91
|
? SIMDLevel::AVX2
|
|
84
92
|
: (SL == SIMDLevel::ARM_SVE ? SIMDLevel::ARM_NEON
|
|
85
93
|
: SL == SIMDLevel::RISCV_RVV ? SIMDLevel::NONE
|
|
@@ -96,13 +104,14 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL_256 =
|
|
|
96
104
|
* Helper to select the appropriate 512-bit SIMD level.
|
|
97
105
|
*
|
|
98
106
|
* For 512-bit SIMD types (simd32uint16, simd64uint8, etc.), maps:
|
|
99
|
-
* AVX512_SPR → AVX512
|
|
107
|
+
* AVX512_VPOPCNT/AVX512_SPR → AVX512 for generic 512-bit operations
|
|
100
108
|
* AVX512 → AVX512
|
|
101
109
|
* NONE → NONE
|
|
102
110
|
***************************************************************/
|
|
103
111
|
template <SIMDLevel SL>
|
|
104
112
|
struct simd512_level_selector {
|
|
105
|
-
static constexpr SIMDLevel value =
|
|
113
|
+
static constexpr SIMDLevel value =
|
|
114
|
+
(SL == SIMDLevel::AVX512_SPR || SL == SIMDLevel::AVX512_VPOPCNT)
|
|
106
115
|
? SIMDLevel::AVX512
|
|
107
116
|
: (SL == SIMDLevel::RISCV_RVV) ? SIMDLevel::NONE
|
|
108
117
|
: SL;
|
|
@@ -124,14 +133,21 @@ constexpr int simd_width() {
|
|
|
124
133
|
static_assert(
|
|
125
134
|
SL != SIMDLevel::RISCV_RVV,
|
|
126
135
|
"simd_width<RISCV_RVV> is not supported: RVV is variable-width");
|
|
127
|
-
if constexpr (
|
|
136
|
+
if constexpr (
|
|
137
|
+
SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_VPOPCNT ||
|
|
138
|
+
SL == SIMDLevel::AVX512_SPR) {
|
|
128
139
|
return 16;
|
|
129
|
-
else if constexpr (SL == SIMDLevel::AVX2 || SL == SIMDLevel::ARM_NEON)
|
|
140
|
+
} else if constexpr (SL == SIMDLevel::AVX2 || SL == SIMDLevel::ARM_NEON) {
|
|
130
141
|
return 8;
|
|
131
|
-
else
|
|
142
|
+
} else {
|
|
132
143
|
return 1;
|
|
144
|
+
}
|
|
133
145
|
}
|
|
134
146
|
|
|
147
|
+
/// Bitmask (1 << SIMDLevel) of the levels this binary holds code for,
|
|
148
|
+
/// whether or not the CPU supports them.
|
|
149
|
+
uint64_t compiled_simd_levels();
|
|
150
|
+
|
|
135
151
|
/// Convert SIMDLevel to string. Throws FaissException for invalid level.
|
|
136
152
|
std::string to_string(SIMDLevel level);
|
|
137
153
|
|
|
@@ -169,13 +185,8 @@ struct FAISS_API SIMDConfig {
|
|
|
169
185
|
|
|
170
186
|
static SIMDLevel auto_detect_simd_level();
|
|
171
187
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
return true;
|
|
175
|
-
#else
|
|
176
|
-
return false;
|
|
177
|
-
#endif
|
|
178
|
-
}
|
|
188
|
+
/// Whether this faiss build dispatches SIMD at runtime.
|
|
189
|
+
static bool has_dynamic_dispatch();
|
|
179
190
|
|
|
180
191
|
SIMDConfig(const char** faiss_simd_level_env = nullptr);
|
|
181
192
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: faiss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Kane
|
|
@@ -475,10 +475,10 @@ files:
|
|
|
475
475
|
- vendor/faiss/faiss/utils/hamming_distance/common.h
|
|
476
476
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp
|
|
477
477
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp
|
|
478
|
-
- vendor/faiss/faiss/utils/hamming_distance/
|
|
478
|
+
- vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_vpopcnt.cpp
|
|
479
479
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx2.h
|
|
480
480
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h
|
|
481
|
-
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-
|
|
481
|
+
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_vpopcnt.h
|
|
482
482
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-generic.h
|
|
483
483
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-neon.h
|
|
484
484
|
- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-rvv.h
|
|
@@ -513,13 +513,14 @@ files:
|
|
|
513
513
|
- vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h
|
|
514
514
|
- vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp
|
|
515
515
|
- vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp
|
|
516
|
-
- vendor/faiss/faiss/utils/simd_impl/
|
|
516
|
+
- vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_vpopcnt.cpp
|
|
517
517
|
- vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp
|
|
518
518
|
- vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp
|
|
519
519
|
- vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h
|
|
520
520
|
- vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h
|
|
521
521
|
- vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp
|
|
522
522
|
- vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp
|
|
523
|
+
- vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp
|
|
523
524
|
- vendor/faiss/faiss/utils/simd_levels.cpp
|
|
524
525
|
- vendor/faiss/faiss/utils/simd_levels.h
|
|
525
526
|
- vendor/faiss/faiss/utils/sorting.cpp
|
|
@@ -546,7 +547,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
546
547
|
- !ruby/object:Gem::Version
|
|
547
548
|
version: '0'
|
|
548
549
|
requirements: []
|
|
549
|
-
rubygems_version: 4.0.
|
|
550
|
+
rubygems_version: 4.0.20
|
|
550
551
|
specification_version: 4
|
|
551
552
|
summary: Efficient similarity search and clustering for Ruby
|
|
552
553
|
test_files: []
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
-
*
|
|
4
|
-
* This source code is licensed under the MIT license found in the
|
|
5
|
-
* LICENSE file in the root directory of this source tree.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
#ifdef COMPILE_SIMD_AVX512_SPR
|
|
9
|
-
|
|
10
|
-
#define THE_SIMD_LEVEL SIMDLevel::AVX512_SPR
|
|
11
|
-
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
12
|
-
#include <faiss/utils/hamming_distance/hamming_computer-avx512_spr.h>
|
|
13
|
-
#include <faiss/utils/hamming_distance/hamming_impl.h>
|
|
14
|
-
|
|
15
|
-
#endif // COMPILE_SIMD_AVX512_SPR
|