faiss 0.6.2 → 0.6.4

data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp

@@ -5,19 +5,107 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+// -*- c++ -*-
+
 #include <faiss/utils/rabitq_simd.h>
 
 #ifdef COMPILE_SIMD_RISCV_RVV
 
+#include <riscv_vector.h>
+
 namespace faiss::rabitq {
 
+// SWAR per-byte popcount over a u8m4 group: each output byte holds the
+// population count (0..8) of the corresponding input byte.
+static inline vuint8m4_t popcount_u8m4(vuint8m4_t v, size_t vl) {
+    vuint8m4_t t = __riscv_vsrl_vx_u8m4(v, 1, vl);
+    t = __riscv_vand_vx_u8m4(t, 0x55, vl);
+    v = __riscv_vsub_vv_u8m4(v, t, vl);
+    t = __riscv_vsrl_vx_u8m4(v, 2, vl);
+    t = __riscv_vand_vx_u8m4(t, 0x33, vl);
+    v = __riscv_vand_vx_u8m4(v, 0x33, vl);
+    v = __riscv_vadd_vv_u8m4(v, t, vl);
+    t = __riscv_vsrl_vx_u8m4(v, 4, vl);
+    v = __riscv_vadd_vv_u8m4(v, t, vl);
+    return __riscv_vand_vx_u8m4(v, 0x0F, vl);
+}
+
+// Shared body for bitwise_{and,xor}_dot_product. @p combine applies the
+// per-element bit op (AND or XOR) between the data and query bit-planes; the
+// popcount of the result for query bit-plane j is weighted by 2^j.
+template <typename Op>
+static inline uint64_t bitwise_dot_product_rvv(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb,
+        Op combine) {
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t vx = __riscv_vle8_v_u8m4(data + i, vl);
+        for (size_t j = 0; j < qb; j++) {
+            vuint8m4_t vq = __riscv_vle8_v_u8m4(query + j * size + i, vl);
+            vuint8m4_t vp = popcount_u8m4(combine(vx, vq, vl), vl);
+            vuint16m8_t vw = __riscv_vzext_vf2_u16m8(vp, vl);
+            vw = __riscv_vsll_vx_u16m8(vw, j, vl);
+            acc = __riscv_vadd_vv_u16m8_tu(acc, acc, vw, vl);
+        }
+        i += vl;
+    }
+    vuint32m1_t red = __riscv_vmv_v_x_u32m1(0, 1);
+    red = __riscv_vwredsumu_vs_u16m8_u32m1(acc, red, vlmax);
+    return __riscv_vmv_x_s_u32m1_u32(red);
+}
+
 template <>
 uint64_t bitwise_and_dot_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* query,
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_and_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    return bitwise_dot_product_rvv(
+            query, data, size, qb, [](vuint8m4_t a, vuint8m4_t b, size_t vl) {
+                return __riscv_vand_vv_u8m4(a, b, vl);
+            });
+}
+
+template <>
+BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
+        SIMDLevel::RISCV_RVV>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    // Fused single pass over @p data: reuse each loaded data group for both the
+    // doc-side popcount and the qb AND-dot bit-planes, mirroring the separate
+    // popcount<RVV> and bitwise_and_dot_product<RVV> kernels above.
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t dot_acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    vuint16m8_t pop_acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t vx = __riscv_vle8_v_u8m4(data + i, vl);
+        vuint16m8_t vxp = __riscv_vzext_vf2_u16m8(popcount_u8m4(vx, vl), vl);
+        pop_acc = __riscv_vadd_vv_u16m8_tu(pop_acc, pop_acc, vxp, vl);
+        for (size_t j = 0; j < qb; j++) {
+            vuint8m4_t vq = __riscv_vle8_v_u8m4(query + j * size + i, vl);
+            vuint8m4_t vp = popcount_u8m4(__riscv_vand_vv_u8m4(vx, vq, vl), vl);
+            vuint16m8_t vw = __riscv_vzext_vf2_u16m8(vp, vl);
+            vw = __riscv_vsll_vx_u16m8(vw, j, vl);
+            dot_acc = __riscv_vadd_vv_u16m8_tu(dot_acc, dot_acc, vw, vl);
+        }
+        i += vl;
+    }
+    vuint32m1_t dot_red = __riscv_vmv_v_x_u32m1(0, 1);
+    dot_red = __riscv_vwredsumu_vs_u16m8_u32m1(dot_acc, dot_red, vlmax);
+    vuint32m1_t pop_red = __riscv_vmv_v_x_u32m1(0, 1);
+    pop_red = __riscv_vwredsumu_vs_u16m8_u32m1(pop_acc, pop_red, vlmax);
+    return {__riscv_vmv_x_s_u32m1_u32(dot_red),
+            __riscv_vmv_x_s_u32m1_u32(pop_red)};
 }
 
 template <>
@@ -26,28 +114,77 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_xor_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    return bitwise_dot_product_rvv(
+            query, data, size, qb, [](vuint8m4_t a, vuint8m4_t b, size_t vl) {
+                return __riscv_vxor_vv_u8m4(a, b, vl);
+            });
 }
 
 template <>
 uint64_t popcount<SIMDLevel::RISCV_RVV>(const uint8_t* data, size_t size) {
-    return popcount<SIMDLevel::NONE>(data, size);
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t v = popcount_u8m4(__riscv_vle8_v_u8m4(data + i, vl), vl);
+        vuint16m8_t vw = __riscv_vzext_vf2_u16m8(v, vl);
+        acc = __riscv_vadd_vv_u16m8_tu(acc, acc, vw, vl);
+        i += vl;
+    }
+    vuint32m1_t red = __riscv_vmv_v_x_u32m1(0, 1);
+    red = __riscv_vwredsumu_vs_u16m8_u32m1(acc, red, vlmax);
+    return __riscv_vmv_x_s_u32m1_u32(red);
 }
 
 } // namespace faiss::rabitq
 
 namespace faiss::rabitq::multibit {
 
+static float ip_1exbit_rvv(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        float cb) {
+    size_t vlmax = __riscv_vsetvlmax_e32m8();
+    vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0.0f, vlmax);
+    size_t i = 0;
+    while (i < d) {
+        size_t vl = __riscv_vsetvl_e32m8(d - i);
+        vbool4_t sb = __riscv_vlm_v_b4(sign_bits + i / 8, vl);
+        vbool4_t eb = __riscv_vlm_v_b4(ex_code + i / 8, vl);
+        vfloat32m8_t recon = __riscv_vfmv_v_f_f32m8(cb, vl);
+        recon = __riscv_vfadd_vf_f32m8_mu(sb, recon, recon, 2.0f, vl);
+        recon = __riscv_vfadd_vf_f32m8_mu(eb, recon, recon, 1.0f, vl);
+        vfloat32m8_t rq = __riscv_vle32_v_f32m8(rotated_q + i, vl);
+        acc = __riscv_vfmacc_vv_f32m8_tu(acc, rq, recon, vl);
+        i += vl;
+    }
+    vfloat32m1_t sum = __riscv_vfmv_s_f_f32m1(0.0f, 1);
+    sum = __riscv_vfredusum_vs_f32m8_f32m1(acc, sum, vlmax);
+    return __riscv_vfmv_f_s_f32m1_f32(sum);
+}
+
+// no-tree-vectorize keeps the ex_bits >= 2 tail on the existing scalar
+// ip_scalar without GCC turning its 64-bit memcpy window into a vluxei64 gather
+// at unaligned addresses, which SIGBUSes on strict-align RVV cores (SpacemiT
+// X60).
 template <>
-float compute_inner_product<SIMDLevel::RISCV_RVV>(
+__attribute__((optimize("no-tree-vectorize", "no-tree-slp-vectorize"))) float
+compute_inner_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* __restrict sign_bits,
         const uint8_t* __restrict ex_code,
         const float* __restrict rotated_q,
         size_t d,
         size_t ex_bits,
         float cb) {
-    return compute_inner_product<SIMDLevel::NONE>(
-            sign_bits, ex_code, rotated_q, d, ex_bits, cb);
+    if (ex_bits == 1) {
+        return ip_1exbit_rvv(sign_bits, ex_code, rotated_q, d, cb);
+    }
+    // ex_bits >= 2 needs strided bit-plane extraction (PEXT on x86); RVV has no
+    // cheap equivalent without Zvbb, so reuse the scalar kernel.
+    return ip_scalar(sign_bits, ex_code, rotated_q, 0, d, ex_bits, cb);
 }
 
 } // namespace faiss::rabitq::multibit

data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h

@@ -9,13 +9,8 @@
 
 // Private dispatch wrapper for SuperKMeans's block_l2. Routes to the
 // highest available SIMD specialization at runtime (DD mode) or the
-// compiled-in level (static mode). aarch64 currently falls through to the
-// scalar primary template; adding NEON/SVE means just adding a new
-// specialization file alongside the AVX ones.
+// compiled-in level (static mode).
 //
-// Known perf gap: aarch64 (NEON/SVE) specializations are not implemented yet.
-// aarch64 falls through to the scalar primary template. Validating SVE requires
-// a Graviton-class host; deferred to a focused follow-up.
 
 #include <faiss/impl/simd_dispatch.h>
 #include <faiss/utils/simd_impl/super_kmeans_kernels.h>
@@ -24,7 +19,7 @@ namespace faiss {
 namespace detail {
 
 inline float block_l2_dispatch(const float* x, const float* y, int n) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0>(
+    return with_simd_level_with_sve(
             [&]<SIMDLevel SL>() { return block_l2<SL>(x, y, n); });
 }
 

data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h

@@ -16,7 +16,7 @@ namespace detail {
 
 // Squared L2 over `n` dimensions; n in [1, pdx_block_size].
 // Primary template is the scalar fallback; SIMDLevels without a dedicated
-// specialization (ARM_NEON, ARM_SVE, NONE, ...) use it directly.
+// specialization (ARM_NEON, NONE, ...) use it directly.
 template <SIMDLevel Level>
 inline float block_l2(const float* x, const float* y, int n) {
     float s = 0.0f;
@@ -39,5 +39,10 @@ template <>
 float block_l2<SIMDLevel::AVX512>(const float* x, const float* y, int n);
 #endif
 
+#ifdef COMPILE_SIMD_ARM_SVE
+template <>
+float block_l2<SIMDLevel::ARM_SVE>(const float* x, const float* y, int n);
+#endif
+
 } // namespace detail
 } // namespace faiss

data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifdef COMPILE_SIMD_ARM_SVE
+
+#include <faiss/utils/simd_impl/super_kmeans_kernels.h>
+
+#include <arm_sve.h>
+
+namespace faiss {
+namespace detail {
+
+template <>
+float block_l2<SIMDLevel::ARM_SVE>(const float* x, const float* y, int n) {
+    svfloat32_t acc = svdup_n_f32(0.0f);
+    const int lanes = static_cast<int>(svcntw());
+    for (int m = 0; m < n; m += lanes) {
+        const svbool_t pg = svwhilelt_b32(m, n);
+        const svfloat32_t xv = svld1_f32(pg, x + m);
+        const svfloat32_t yv = svld1_f32(pg, y + m);
+        const svfloat32_t diff = svsub_f32_x(pg, xv, yv);
+        acc = svmla_f32_m(pg, acc, diff, diff);
+    }
+    return svaddv_f32(svptrue_b32(), acc);
+}
+
+} // namespace detail
+} // namespace faiss
+
+#endif // COMPILE_SIMD_ARM_SVE

data/vendor/faiss/faiss/utils/simd_levels.cpp

@@ -7,8 +7,15 @@
 
 #include <faiss/utils/simd_levels.h>
 
+#include <cstdint>
+#include <cstdio>
 #include <cstdlib>
 
+#if defined(_MSC_VER)
+// __cpuidex, _xgetbv
+#include <intrin.h>
+#endif
+
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/simd_dispatch.h>
 
@@ -20,6 +27,20 @@ SIMDLevel SIMDConfig::level = SIMDLevel::NONE;
 // Bitmask of supported SIMD levels (1 << SIMDLevel)
 uint64_t SIMDConfig::supported_simd_levels = 0;
 
+// Microarchitecture flags (x86). Default false; set by
+// detect_x86_uarch_flags() at load time.
+bool SIMDConfig::avx512_split = false;
+
+// Resolved here rather than in the header so that dependents, which never see
+// FAISS_ENABLE_DD, still get the answer for the faiss they link against.
+bool SIMDConfig::has_dynamic_dispatch() {
+#ifdef FAISS_ENABLE_DD
+    return true;
+#else
+    return false;
+#endif
+}
+
 // ARM SVE runtime detection
 #if defined(__aarch64__) || defined(_M_ARM64)
 
@@ -53,11 +74,110 @@ static bool has_sve() {
 }
 #endif
 
-#ifdef FAISS_ENABLE_DD
+namespace {
 
-// =============================================================================
-// Dynamic Dispatch (DD) mode implementation
-// =============================================================================
+#if defined(__x86_64__) || defined(_M_X64)
+
+// MSVC and clang-cl do not support GNU-style
+// 64-bit inline assembly, MSVC defines _M_X64 instead of __x86_64__
+
+#if defined(_MSC_VER)
+
+[[maybe_unused]] void cpuid_count(
+        unsigned int leaf,
+        unsigned int subleaf,
+        unsigned int regs[4]) {
+    int r[4];
+    __cpuidex(r, static_cast<int>(leaf), static_cast<int>(subleaf));
+    for (int i = 0; i < 4; i++) {
+        regs[i] = static_cast<unsigned int>(r[i]);
+    }
+}
+
+[[maybe_unused]] uint64_t xgetbv0() {
+    return static_cast<uint64_t>(_xgetbv(0));
+}
+
+#else // GCC / Clang
+
+[[maybe_unused]] void cpuid_count(
+        unsigned int leaf,
+        unsigned int subleaf,
+        unsigned int regs[4]) {
+    asm volatile("cpuid"
+                 : "=a"(regs[0]), "=b"(regs[1]), "=c"(regs[2]), "=d"(regs[3])
+                 : "a"(leaf), "c"(subleaf));
+}
+
+[[maybe_unused]] uint64_t xgetbv0() {
+    unsigned int eax, edx;
+    asm volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
+    return eax | (static_cast<uint64_t>(edx) << 32);
+}
+
+#endif // _MSC_VER
+
+// Detect x86 microarchitecture flags used for kernel routing. Uses raw
+// cpuid so it is safe to run on any CPU regardless of compiled SIMD level.
+void detect_x86_uarch_flags() {
+    unsigned int regs[4];
+
+    // Vendor string (CPUID.0): "AuthenticAMD" is EBX="Auth", EDX="enti",
+    // ECX="cAMD".
+    cpuid_count(0, 0, regs);
+    const bool is_amd = regs[1] == 0x68747541u && regs[3] == 0x69746e65u &&
+            regs[2] == 0x444d4163u;
+
+    // Family/model (CPUID.1 EAX).
+    cpuid_count(1, 0, regs);
+    const unsigned int eax1 = regs[0];
+    const unsigned int base_family = (eax1 >> 8) & 0xfu;
+    const unsigned int display_family =
+            base_family + (base_family == 0xfu ? ((eax1 >> 20) & 0xffu) : 0u);
+    // AMD Zen 4 / Zen 4c (Bergamo) is family 0x19 and splits AVX-512.
+    // (Zen 5, family 0x1A, has a native 512-bit datapath and is excluded.)
+    SIMDConfig::avx512_split = is_amd && display_family == 0x19u;
+}
+
+#else // Not x86-64
+
+void detect_x86_uarch_flags() {}
+
+#endif // defined(__x86_64__) || defined(_M_X64)
+
+} // namespace
+
+/// Must mirror the case labels in with_selected_simd_levels. A static build
+/// defines a COMPILE_SIMD_* macro for every level whose sources it compiles,
+/// not only for SINGLE_SIMD_LEVEL, so this reports a correct set in both
+/// modes and the result is not a single level.
+uint64_t compiled_simd_levels() {
+    uint64_t mask = uint64_t(1) << static_cast<int>(SIMDLevel::NONE);
+#ifdef COMPILE_SIMD_AVX2
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX2);
+#endif
+#ifdef COMPILE_SIMD_AVX512
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX512);
+#endif
+#ifdef COMPILE_SIMD_AVX512_VPOPCNT
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX512_VPOPCNT);
+#endif
+#ifdef COMPILE_SIMD_AVX512_SPR
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::AVX512_SPR);
+#endif
+#ifdef COMPILE_SIMD_ARM_NEON
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::ARM_NEON);
+#endif
+#ifdef COMPILE_SIMD_ARM_SVE
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::ARM_SVE);
+#endif
+#ifdef COMPILE_SIMD_RISCV_RVV
+    mask |= uint64_t(1) << static_cast<int>(SIMDLevel::RISCV_RVV);
+#endif
+    return mask;
+}
+
+#ifdef FAISS_ENABLE_DD
 
 // Static initializer to run constructor at load time
 // NOLINTNEXTLINE(facebook-avoid-non-const-global-variables)
@@ -71,7 +191,22 @@ SIMDConfig::SIMDConfig(const char** faiss_simd_level_env) {
     if (!env_var) {
         level = auto_detect_simd_level();
     } else {
-        level = to_simd_level(env_var);
+        // Forcing a level the CPU lacks is allowed. Forcing one the binary
+        // does not hold is not: dispatch would fall to NONE and skip every
+        // level between. Walk down to the nearest compiled level instead.
+        const uint64_t compiled = compiled_simd_levels();
+        const SIMDLevel requested = to_simd_level(env_var);
+        level = requested;
+        while (((compiled >> static_cast<int>(level)) & 1) == 0) {
+            level = get_simd_fallback(level);
+        }
+        if (level != requested) {
+            fprintf(stderr,
+                    "faiss: FAISS_SIMD_LEVEL=%s is not compiled into this "
+                    "build, using %s instead\n",
+                    to_string(requested).c_str(),
+                    to_string(level).c_str());
+        }
         supported_simd_levels = (1 << static_cast<int>(level));
     }
     supported_simd_levels |= (1 << static_cast<int>(SIMDLevel::NONE));
@@ -101,80 +236,86 @@ bool SIMDConfig::is_simd_level_available(SIMDLevel l) {
 SIMDLevel SIMDConfig::auto_detect_simd_level() {
     SIMDLevel detected_level = SIMDLevel::NONE;
 
-#if defined(__x86_64__) && \
+    detect_x86_uarch_flags();
+
+#if (defined(__x86_64__) || defined(_M_X64)) && \
         (defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512))
-    unsigned int eax, ebx, ecx, edx;
+    unsigned int regs[4];
 
-    eax = 1;
-    ecx = 0;
-    asm volatile("cpuid"
-                 : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
-                 : "a"(eax), "c"(ecx));
+    cpuid_count(1, 0, regs);
+    unsigned int ecx1 = regs[2];
 
-    bool has_avx = (ecx & (1 << 28)) != 0;
+    bool has_avx = (ecx1 & (1 << 28)) != 0;
 
     bool has_xsave_osxsave =
-            (ecx & ((1 << 26) | (1 << 27))) == ((1 << 26) | (1 << 27));
+            (ecx1 & ((1 << 26) | (1 << 27))) == ((1 << 26) | (1 << 27));
 
     bool avx_supported = false;
     if (has_avx && has_xsave_osxsave) {
-        unsigned int xcr0;
-        asm volatile("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0));
-        avx_supported = (xcr0 & 6) == 6;
+        avx_supported = (xgetbv0() & 6) == 6;
     }
 
     if (avx_supported) {
-        eax = 7;
-        ecx = 0;
-        asm volatile("cpuid"
-                     : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
-                     : "a"(eax), "c"(ecx));
-        // Save EDX before xgetbv clobbers it — needed for
-        // AVX512_FP16 check (bit 23) in the SPR detection below.
-        unsigned int cpuid7_edx = edx;
-
-        unsigned int xcr0;
-        asm volatile("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0));
+        cpuid_count(7, 0, regs);
+        unsigned int ebx7 = regs[1];
+        // EDX of CPUID leaf 7 subleaf 0 carries AVX512_FP16 (bit 23),
+        // needed for the SPR detection below. Kept in a local so a later
+        // xgetbv cannot clobber it.
+        unsigned int cpuid7_edx = regs[3];
+        // Leaf 7 subleaf 0, not leaf 1: leaf 1 ECX holds unrelated bits at
+        // these positions.
+        unsigned int ecx7 = regs[2];
+        [[maybe_unused]] bool has_avx512_vnni = (ecx7 & (1 << 11)) != 0;
+        [[maybe_unused]] bool has_avx512_vpopcntdq = (ecx7 & (1 << 14)) != 0;
+        // Bit 12 = AVX512_BITALG, needed for the byte-wise popcount kernels.
+        [[maybe_unused]] bool has_avx512_bitalg = (ecx7 & (1 << 12)) != 0;
+
+        uint64_t xcr0 = xgetbv0();
 
 #if defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512)
-        bool has_avx2 = (ebx & (1 << 5)) != 0;
+        bool has_avx2 = (ebx7 & (1 << 5)) != 0;
         if (has_avx2) {
             supported_simd_levels |= (1 << static_cast<int>(SIMDLevel::AVX2));
             detected_level = SIMDLevel::AVX2;
         }
 
 #if defined(COMPILE_SIMD_AVX512)
-        bool cpu_has_avx512f = (ebx & (1 << 16)) != 0;
+        bool cpu_has_avx512f = (ebx7 & (1 << 16)) != 0;
         bool os_supports_avx512 = (xcr0 & 0xE0) == 0xE0;
         bool has_avx512f = cpu_has_avx512f && os_supports_avx512;
         if (has_avx512f) {
-            bool has_avx512cd = (ebx & (1 << 28)) != 0;
-            bool has_avx512vl = (ebx & (1 << 31)) != 0;
-            bool has_avx512dq = (ebx & (1 << 17)) != 0;
-            bool has_avx512bw = (ebx & (1 << 30)) != 0;
+            bool has_avx512cd = (ebx7 & (1 << 28)) != 0;
+            bool has_avx512vl = (ebx7 & (1 << 31)) != 0;
+            bool has_avx512dq = (ebx7 & (1 << 17)) != 0;
+            bool has_avx512bw = (ebx7 & (1 << 30)) != 0;
             if (has_avx512bw && has_avx512cd && has_avx512vl && has_avx512dq) {
                 detected_level = SIMDLevel::AVX512;
                 supported_simd_levels |=
                         (1 << static_cast<int>(SIMDLevel::AVX512));
 
+#if defined(COMPILE_SIMD_AVX512_VPOPCNT)
+                if (has_avx512_vpopcntdq && has_avx512_bitalg) {
+                    detected_level = SIMDLevel::AVX512_VPOPCNT;
+                    supported_simd_levels |=
+                            (1 << static_cast<int>(SIMDLevel::AVX512_VPOPCNT));
+                }
+#endif
+
 #if defined(COMPILE_SIMD_AVX512_SPR)
                 // Check for Sapphire Rapids features.
-                // The SPR code path is compiled with -mavx512fp16, so we
-                // must verify both AVX512_BF16 and AVX512_FP16 before
-                // dispatching to it. AMD Zen 4 (bergamo) has BF16 but
-                // not FP16 — using SPR code there causes SIGILL.
+                // The SPR code path is compiled with AVX512_VNNI, BF16,
+                // FP16 and VPOPCNTDQ, and falls back to the VPOPCNT kernels,
+                // which need BITALG. All five features are required.
+                // AMD Zen 4 has VPOPCNTDQ and BF16 but not FP16, and must
+                // remain on the AVX512_VPOPCNT level.
                 // CPUID EAX=7, ECX=1: EAX bit 5 = AVX512_BF16
                 // CPUID EAX=7, ECX=0: EDX bit 23 = AVX512_FP16
                 // (Linux: X86_FEATURE_AVX512_FP16 = 18*32+23)
                 bool has_avx512_fp16 = (cpuid7_edx & (1 << 23)) != 0;
-                unsigned int eax1, ebx1, ecx1, edx1;
-                eax1 = 7;
-                ecx1 = 1;
-                asm volatile("cpuid"
-                             : "=a"(eax1), "=b"(ebx1), "=c"(ecx1), "=d"(edx1)
-                             : "a"(eax1), "c"(ecx1));
-                bool has_avx512_bf16 = (eax1 & (1 << 5)) != 0;
-                if (has_avx512_bf16 && has_avx512_fp16) {
+                cpuid_count(7, 1, regs);
+                const bool has_avx512_bf16 = (regs[0] & (1 << 5)) != 0;
+                if (has_avx512_vnni && has_avx512_vpopcntdq &&
+                    has_avx512_bitalg && has_avx512_bf16 && has_avx512_fp16) {
                     detected_level = SIMDLevel::AVX512_SPR;
                     supported_simd_levels |=
                             (1 << static_cast<int>(SIMDLevel::AVX512_SPR));
@@ -185,7 +326,7 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
 #endif // defined(COMPILE_SIMD_AVX512)
 #endif // defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512)
     }
-#endif // defined(__x86_64__) && ...
+#endif // defined(__x86_64__) || defined(_M_X64)
 
 #ifdef COMPILE_SIMD_ARM_NEON
     // ARM NEON is standard on aarch64
@@ -264,9 +405,12 @@ bool SIMDConfig::is_simd_level_available(SIMDLevel l) {
 }
 
 SIMDLevel SIMDConfig::auto_detect_simd_level() {
+    detect_x86_uarch_flags();
     // In static mode, return the compiled-in level
 #if defined(COMPILE_SIMD_AVX512_SPR)
     return SIMDLevel::AVX512_SPR;
+#elif defined(COMPILE_SIMD_AVX512_VPOPCNT)
+    return SIMDLevel::AVX512_VPOPCNT;
 #elif defined(COMPILE_SIMD_AVX512)
     return SIMDLevel::AVX512;
 #elif defined(COMPILE_SIMD_AVX2)
@@ -301,6 +445,8 @@ std::string to_string(SIMDLevel level) {
             return "AVX2";
         case SIMDLevel::AVX512:
             return "AVX512";
+        case SIMDLevel::AVX512_VPOPCNT:
+            return "AVX512_VPOPCNT";
         case SIMDLevel::AVX512_SPR:
             return "AVX512_SPR";
         case SIMDLevel::ARM_NEON:
@@ -325,6 +471,9 @@ SIMDLevel to_simd_level(const std::string& level_str) {
     if (level_str == "AVX512") {
         return SIMDLevel::AVX512;
     }
+    if (level_str == "AVX512_VPOPCNT") {
+        return SIMDLevel::AVX512_VPOPCNT;
+    }
     if (level_str == "AVX512_SPR") {
         return SIMDLevel::AVX512_SPR;
     }