faiss 0.6.2 → 0.6.4

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h

@@ -28,6 +28,14 @@ namespace scalar_quantizer {
 // Define SL as alias for THE_LEVEL_TO_DISPATCH for use in this file
 constexpr SIMDLevel SL = THE_LEVEL_TO_DISPATCH;
 
+// The SPR scalar-quantizer implementation uses VPOPCNTDQ for its RaBitQ
+// popcount helpers, but those helpers belong to the narrower VPOPCNT
+// capability. Keep the SQ implementation at AVX512_SPR while explicitly
+// reusing the independently dispatched VPOPCNT kernels.
+template <SIMDLevel SL0>
+inline constexpr SIMDLevel rabitq_popcount_level =
+        SL0 == SIMDLevel::AVX512_SPR ? SIMDLevel::AVX512_VPOPCNT : SL0;
+
 /*******************************************************************
  * TurboQuant SIMD kernel: masked_sum
  * Compute sum of arr[j] where bit j of the bitmask is set.
@@ -259,9 +267,11 @@ struct DCTurboQuantFull : ScalarQuantizer::TurboQuantRefine::DistanceComputer {
             if (qb > 0) {
                 // Integer popcount path for 1-bit MSE
                 size_t byte_size = (d + 7) / 8;
-                uint64_t and_result = rabitq::bitwise_and_dot_product<SL2>(
+                uint64_t and_result = rabitq::bitwise_and_dot_product<
+                        rabitq_popcount_level<SL2>>(
                         rearranged_q.data(), code, byte_size, qb);
-                uint64_t pop = rabitq::popcount<SL2>(code, byte_size);
+                uint64_t pop = rabitq::popcount<rabitq_popcount_level<SL2>>(
+                        code, byte_size);
                 mse_dot = mse_base +
                         mse_int_scale * static_cast<float>(and_result) +
                         mse_popcnt_scale * static_cast<float>(pop);
@@ -316,9 +326,11 @@ struct DCTurboQuantFull : ScalarQuantizer::TurboQuantRefine::DistanceComputer {
         float qjl_dot;
         if (qb > 0 && int_qjl) {
             size_t byte_size = (d + 7) / 8;
-            uint64_t and_result = rabitq::bitwise_and_dot_product<SL2>(
-                    rearranged_qproj.data(), qjl_code, byte_size, qb);
-            uint64_t pop = rabitq::popcount<SL2>(qjl_code, byte_size);
+            uint64_t and_result =
+                    rabitq::bitwise_and_dot_product<rabitq_popcount_level<SL2>>(
+                            rearranged_qproj.data(), qjl_code, byte_size, qb);
+            uint64_t pop = rabitq::popcount<rabitq_popcount_level<SL2>>(
+                    qjl_code, byte_size);
             float pos_sum = qjl_popcnt_scale * static_cast<float>(pop) +
                     qjl_int_scale * static_cast<float>(and_result);
             qjl_dot = qjl_coeff * gamma * (2.0f * pos_sum - total_qproj_sum);
@@ -342,10 +354,17 @@ struct DCTurboQuantFull : ScalarQuantizer::TurboQuantRefine::DistanceComputer {
     }
 };
 
+// True for every level that runs the 512-bit kernels. AVX512_VPOPCNT is
+// unreachable here until an SQ translation unit is compiled at that level;
+// naming it keeps the 512-bit alignment rule correct when one is.
+template <SIMDLevel SL2>
+constexpr bool is_avx512_family = SL2 == SIMDLevel::AVX512 ||
+        SL2 == SIMDLevel::AVX512_VPOPCNT || SL2 == SIMDLevel::AVX512_SPR;
+
 // Returns true if dimension d is compatible with the given SIMD level
 template <SIMDLevel SL2>
 constexpr bool is_dimension_compatible(size_t d) {
-    if constexpr (SL2 == SIMDLevel::AVX512 || SL2 == SIMDLevel::AVX512_SPR) {
+    if constexpr (is_avx512_family<SL2>) {
         return d % 16 == 0;
     } else if constexpr (SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
         return d % 8 == 0;
@@ -416,6 +435,34 @@ ScalarQuantizer::SQuantizer* sq_select_quantizer<THE_LEVEL_TO_DISPATCH>(
             return new QuantizerTurboQuantMSE<4, SL>(d, trained);
         case ScalarQuantizer::QT_8bit_tqmse:
             return new QuantizerTurboQuantMSE<8, SL>(d, trained);
+        case ScalarQuantizer::QT_1bit_eden:
+            return new QuantizerLloydMax<1, SL>(d, trained);
+        case ScalarQuantizer::QT_2bit_eden:
+            return new QuantizerLloydMax<2, SL>(d, trained);
+        case ScalarQuantizer::QT_3bit_eden:
+            return new QuantizerLloydMax<3, SL>(d, trained);
+        case ScalarQuantizer::QT_4bit_eden:
+            return new QuantizerLloydMax<4, SL>(d, trained);
+        case ScalarQuantizer::QT_5bit_eden:
+            if constexpr (SL != SIMDLevel::NONE) {
+                return nullptr;
+            } else {
+                return new QuantizerLloydMax<5, SL>(d, trained);
+            }
+        case ScalarQuantizer::QT_6bit_eden:
+            if constexpr (SL != SIMDLevel::NONE) {
+                return nullptr;
+            } else {
+                return new QuantizerLloydMax<6, SL>(d, trained);
+            }
+        case ScalarQuantizer::QT_7bit_eden:
+            if constexpr (SL != SIMDLevel::NONE) {
+                return nullptr;
+            } else {
+                return new QuantizerLloydMax<7, SL>(d, trained);
+            }
+        case ScalarQuantizer::QT_8bit_eden:
+            return new QuantizerLloydMax<8, SL>(d, trained);
         case ScalarQuantizer::QT_2bit_tq:
             return new QuantizerTurboQuantFull<2, SL>(d, trained);
         case ScalarQuantizer::QT_3bit_tq:
@@ -497,13 +544,13 @@ SQDistanceComputer* select_distance_computer_body(
             return new DCTemplate<QuantizerBF16<SL2>, Sim, SL2>(d, trained);
 
         case ScalarQuantizer::QT_8bit_direct:
-            if constexpr (
-                    SL2 == SIMDLevel::AVX512 || SL2 == SIMDLevel::AVX512_SPR) {
+            if constexpr (is_avx512_family<SL2>) {
                 if (d % 32 == 0) {
                     return new DistanceComputerByte<Sim, SL2>(
                             static_cast<int>(d), trained);
                 }
-            } else if constexpr (SL2 == SIMDLevel::AVX2) {
+            } else if constexpr (
+                    SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
                 if (d % 16 == 0) {
                     return new DistanceComputerByte<Sim, SL2>(
                             static_cast<int>(d), trained);
@@ -518,6 +565,17 @@ SQDistanceComputer* select_distance_computer_body(
                     return new DistanceComputerByteSigned<Sim, SL2>(
                             static_cast<int>(d), trained);
                 }
+            } else if constexpr (is_avx512_family<SL2>) {
+                if (d % 32 == 0) {
+                    return new DistanceComputerByteSigned<Sim, SL2>(
+                            static_cast<int>(d), trained);
+                }
+            } else if constexpr (
+                    SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
+                if (d % 16 == 0) {
+                    return new DistanceComputerByteSigned<Sim, SL2>(
+                            static_cast<int>(d), trained);
+                }
             }
             return new DCTemplate<Quantizer8bitDirectSigned<SL2>, Sim, SL2>(
                     d, trained);
@@ -539,6 +597,42 @@ SQDistanceComputer* select_distance_computer_body(
         case ScalarQuantizer::QT_8bit_tqmse:
             return new DCTemplate<QuantizerTurboQuantMSE<8, SL2>, Sim, SL2>(
                     d, trained);
+        case ScalarQuantizer::QT_1bit_eden:
+            return new DCTemplate<QuantizerLloydMax<1, SL2>, Sim, SL2>(
+                    d, trained);
+        case ScalarQuantizer::QT_2bit_eden:
+            return new DCTemplate<QuantizerLloydMax<2, SL2>, Sim, SL2>(
+                    d, trained);
+        case ScalarQuantizer::QT_3bit_eden:
+            return new DCTemplate<QuantizerLloydMax<3, SL2>, Sim, SL2>(
+                    d, trained);
+        case ScalarQuantizer::QT_4bit_eden:
+            return new DCTemplate<QuantizerLloydMax<4, SL2>, Sim, SL2>(
+                    d, trained);
+        case ScalarQuantizer::QT_5bit_eden:
+            if constexpr (SL2 != SIMDLevel::NONE) {
+                return nullptr;
+            } else {
+                return new DCTemplate<QuantizerLloydMax<5, SL2>, Sim, SL2>(
+                        d, trained);
+            }
+        case ScalarQuantizer::QT_6bit_eden:
+            if constexpr (SL2 != SIMDLevel::NONE) {
+                return nullptr;
+            } else {
+                return new DCTemplate<QuantizerLloydMax<6, SL2>, Sim, SL2>(
+                        d, trained);
+            }
+        case ScalarQuantizer::QT_7bit_eden:
+            if constexpr (SL2 != SIMDLevel::NONE) {
+                return nullptr;
+            } else {
+                return new DCTemplate<QuantizerLloydMax<7, SL2>, Sim, SL2>(
+                        d, trained);
+            }
+        case ScalarQuantizer::QT_8bit_eden:
+            return new DCTemplate<QuantizerLloydMax<8, SL2>, Sim, SL2>(
+                    d, trained);
         case ScalarQuantizer::QT_2bit_tq:
             // FRICTION: bypasses DCTemplate entirely — custom DC
             // that doesn't fit the Quantizer+Similarity decomposition
@@ -663,14 +757,13 @@ InvertedListScanner* sq_select_InvertedListScanner<THE_LEVEL_TO_DISPATCH>(
                 return scan.template
                 operator()<DCTemplate<QuantizerBF16<SL2>, Similarity, SL2>>();
             case ScalarQuantizer::QT_8bit_direct:
-                if constexpr (
-                        SL2 == SIMDLevel::AVX512 ||
-                        SL2 == SIMDLevel::AVX512_SPR) {
+                if constexpr (is_avx512_family<SL2>) {
                     if (d % 32 == 0) {
                         return scan.template
                         operator()<DistanceComputerByte<Similarity, SL2>>();
                     }
-                } else if constexpr (SL2 == SIMDLevel::AVX2) {
+                } else if constexpr (
+                        SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
                     if (d % 16 == 0) {
                         return scan.template
                         operator()<DistanceComputerByte<Similarity, SL2>>();
@@ -686,6 +779,17 @@ InvertedListScanner* sq_select_InvertedListScanner<THE_LEVEL_TO_DISPATCH>(
                         return scan.template operator()<
                                 DistanceComputerByteSigned<Similarity, SL2>>();
                     }
+                } else if constexpr (is_avx512_family<SL2>) {
+                    if (d % 32 == 0) {
+                        return scan.template operator()<
+                                DistanceComputerByteSigned<Similarity, SL2>>();
+                    }
+                } else if constexpr (
+                        SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
+                    if (d % 16 == 0) {
+                        return scan.template operator()<
+                                DistanceComputerByteSigned<Similarity, SL2>>();
+                    }
                 }
                 return scan.template operator()<DCTemplate<
                         Quantizer8bitDirectSigned<SL2>,
@@ -719,6 +823,58 @@ InvertedListScanner* sq_select_InvertedListScanner<THE_LEVEL_TO_DISPATCH>(
                         QuantizerTurboQuantMSE<8, SL2>,
                         Similarity,
                         SL2>>();
+            case ScalarQuantizer::QT_1bit_eden:
+                return scan.template operator()<DCTemplate<
+                        QuantizerLloydMax<1, SL2>,
+                        Similarity,
+                        SL2>>();
+            case ScalarQuantizer::QT_2bit_eden:
+                return scan.template operator()<DCTemplate<
+                        QuantizerLloydMax<2, SL2>,
+                        Similarity,
+                        SL2>>();
+            case ScalarQuantizer::QT_3bit_eden:
+                return scan.template operator()<DCTemplate<
+                        QuantizerLloydMax<3, SL2>,
+                        Similarity,
+                        SL2>>();
+            case ScalarQuantizer::QT_4bit_eden:
+                return scan.template operator()<DCTemplate<
+                        QuantizerLloydMax<4, SL2>,
+                        Similarity,
+                        SL2>>();
+            case ScalarQuantizer::QT_5bit_eden:
+                if constexpr (SL2 != SIMDLevel::NONE) {
+                    return nullptr;
+                } else {
+                    return scan.template operator()<DCTemplate<
+                            QuantizerLloydMax<5, SL2>,
+                            Similarity,
+                            SL2>>();
+                }
+            case ScalarQuantizer::QT_6bit_eden:
+                if constexpr (SL2 != SIMDLevel::NONE) {
+                    return nullptr;
+                } else {
+                    return scan.template operator()<DCTemplate<
+                            QuantizerLloydMax<6, SL2>,
+                            Similarity,
+                            SL2>>();
+                }
+            case ScalarQuantizer::QT_7bit_eden:
+                if constexpr (SL2 != SIMDLevel::NONE) {
+                    return nullptr;
+                } else {
+                    return scan.template operator()<DCTemplate<
+                            QuantizerLloydMax<7, SL2>,
+                            Similarity,
+                            SL2>>();
+                }
+            case ScalarQuantizer::QT_8bit_eden:
+                return scan.template operator()<DCTemplate<
+                        QuantizerLloydMax<8, SL2>,
+                        Similarity,
+                        SL2>>();
             case ScalarQuantizer::QT_2bit_tq:
                 return scan.template
                 operator()<DCTurboQuantFull<2, Similarity, SL2>>();

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp

@@ -223,18 +223,18 @@ struct QuantizerTemplate<
 };
 
 /**********************************************************
- * TurboQuant MSE quantizer
+ * Lloyd-Max scalar quantizer
  **********************************************************/
 
-// NEON TurboQuantMSE: decode via gather, encode stays scalar.
+// NEON Lloyd-Max: decode via gather, encode stays scalar.
 // NEON doesn't have movemask so 1-bit encode is also scalar.
-#define DEFINE_TQMSE_NEON_SPECIALIZATION(NBITS, UNPACK_FN)                   \
+#define DEFINE_LLOYD_MAX_NEON_SPECIALIZATION(NBITS, UNPACK_FN)               \
     template <>                                                              \
-    struct QuantizerTurboQuantMSE<NBITS, SIMDLevel::ARM_NEON>                \
-            : QuantizerTurboQuantMSE<NBITS, SIMDLevel::NONE> {               \
-        using Base = QuantizerTurboQuantMSE<NBITS, SIMDLevel::NONE>;         \
+    struct QuantizerLloydMax<NBITS, SIMDLevel::ARM_NEON>                     \
+            : QuantizerLloydMax<NBITS, SIMDLevel::NONE> {                    \
+        using Base = QuantizerLloydMax<NBITS, SIMDLevel::NONE>;              \
                                                                              \
-        QuantizerTurboQuantMSE(size_t d, const std::vector<float>& trained)  \
+        QuantizerLloydMax(size_t d, const std::vector<float>& trained)       \
                 : Base(d, trained) {                                         \
             assert(d % 8 == 0);                                              \
         }                                                                    \
@@ -256,19 +256,19 @@ struct QuantizerTemplate<
         }                                                                    \
     }
 
-DEFINE_TQMSE_NEON_SPECIALIZATION(1, unpack_8x1bit_to_u8);
-DEFINE_TQMSE_NEON_SPECIALIZATION(2, unpack_8x2bit_to_u8);
-DEFINE_TQMSE_NEON_SPECIALIZATION(3, unpack_8x3bit_to_u8);
-DEFINE_TQMSE_NEON_SPECIALIZATION(4, unpack_8x4bit_to_u8);
+DEFINE_LLOYD_MAX_NEON_SPECIALIZATION(1, unpack_8x1bit_to_u8);
+DEFINE_LLOYD_MAX_NEON_SPECIALIZATION(2, unpack_8x2bit_to_u8);
+DEFINE_LLOYD_MAX_NEON_SPECIALIZATION(3, unpack_8x3bit_to_u8);
+DEFINE_LLOYD_MAX_NEON_SPECIALIZATION(4, unpack_8x4bit_to_u8);
 
-#undef DEFINE_TQMSE_NEON_SPECIALIZATION
+#undef DEFINE_LLOYD_MAX_NEON_SPECIALIZATION
 
 template <>
-struct QuantizerTurboQuantMSE<8, SIMDLevel::ARM_NEON>
-        : QuantizerTurboQuantMSE<8, SIMDLevel::NONE> {
-    using Base = QuantizerTurboQuantMSE<8, SIMDLevel::NONE>;
+struct QuantizerLloydMax<8, SIMDLevel::ARM_NEON>
+        : QuantizerLloydMax<8, SIMDLevel::NONE> {
+    using Base = QuantizerLloydMax<8, SIMDLevel::NONE>;
 
-    QuantizerTurboQuantMSE(size_t d, const std::vector<float>& trained)
+    QuantizerLloydMax(size_t d, const std::vector<float>& trained)
             : Base(d, trained) {
         assert(d % 8 == 0);
     }
@@ -618,6 +618,64 @@ struct DCTemplate<Quantizer, Similarity, SIMDLevel::ARM_NEON>
     }
 };
 
+// Byte-domain kernels for QT_8bit_direct{,_signed}. The dispatch only
+// selects them when d % 16 == 0, so no loop needs a tail.
+
+namespace {
+
+// The accumulator stays unsigned: a vmull_u8 square reaches 255*255 = 65025,
+// which an int16 lane would read as negative.
+FAISS_ALWAYS_INLINE int neon_byte_l2sqr(
+        const uint8_t* code1,
+        const uint8_t* code2,
+        int d) {
+    uint32x4_t accu = vdupq_n_u32(0);
+    for (int i = 0; i < d; i += 16) {
+        const uint8x16_t diff =
+                vabdq_u8(vld1q_u8(code1 + i), vld1q_u8(code2 + i));
+        accu = vpadalq_u16(
+                accu, vmull_u8(vget_low_u8(diff), vget_low_u8(diff)));
+        accu = vpadalq_u16(
+                accu, vmull_u8(vget_high_u8(diff), vget_high_u8(diff)));
+    }
+    return static_cast<int>(vaddvq_u32(accu));
+}
+
+FAISS_ALWAYS_INLINE int neon_byte_ip(
+        const uint8_t* code1,
+        const uint8_t* code2,
+        int d) {
+    uint32x4_t accu = vdupq_n_u32(0);
+    for (int i = 0; i < d; i += 16) {
+        const uint8x16_t c1 = vld1q_u8(code1 + i);
+        const uint8x16_t c2 = vld1q_u8(code2 + i);
+        accu = vpadalq_u16(accu, vmull_u8(vget_low_u8(c1), vget_low_u8(c2)));
+        accu = vpadalq_u16(accu, vmull_u8(vget_high_u8(c1), vget_high_u8(c2)));
+    }
+    return static_cast<int>(vaddvq_u32(accu));
+}
+
+// The codes store value + 128. For x in 0 to 255, x ^ 0x80 read as int8 is
+// exactly x - 128, which is how the bias comes off before vmull_s8.
+FAISS_ALWAYS_INLINE int neon_byte_ip_unbias(
+        const uint8_t* code1,
+        const uint8_t* code2,
+        int d) {
+    const uint8x16_t bias = vdupq_n_u8(0x80);
+    int32x4_t accu = vdupq_n_s32(0);
+    for (int i = 0; i < d; i += 16) {
+        const int8x16_t c1 =
+                vreinterpretq_s8_u8(veorq_u8(vld1q_u8(code1 + i), bias));
+        const int8x16_t c2 =
+                vreinterpretq_s8_u8(veorq_u8(vld1q_u8(code2 + i), bias));
+        accu = vpadalq_s16(accu, vmull_s8(vget_low_s8(c1), vget_low_s8(c2)));
+        accu = vpadalq_s16(accu, vmull_s8(vget_high_s8(c1), vget_high_s8(c2)));
+    }
+    return static_cast<int>(vaddvq_s32(accu));
+}
+
+} // namespace
+
 template <class Similarity>
 struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
         : SQDistanceComputer {
@@ -626,20 +684,17 @@ struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
     int d;
     std::vector<uint8_t> tmp;
 
-    DistanceComputerByte(int d, const std::vector<float>&) : d(d), tmp(d) {}
+    DistanceComputerByte(int d, const std::vector<float>&) : d(d), tmp(d) {
+        FAISS_THROW_IF_NOT(d % 16 == 0);
+    }
 
     int compute_code_distance(const uint8_t* code1, const uint8_t* code2)
             const {
-        int accu = 0;
-        for (int i = 0; i < d; i++) {
-            if (Sim::metric_type == METRIC_INNER_PRODUCT) {
-                accu += int(code1[i]) * code2[i];
-            } else {
-                int diff = int(code1[i]) - code2[i];
-                accu += diff * diff;
-            }
+        if constexpr (Sim::metric_type == METRIC_INNER_PRODUCT) {
+            return neon_byte_ip(code1, code2, d);
+        } else {
+            return neon_byte_l2sqr(code1, code2, d);
         }
-        return accu;
     }
 
     void set_query(const float* x) final {
@@ -663,6 +718,50 @@ struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
     }
 };
 
+template <class Similarity>
+struct DistanceComputerByteSigned<Similarity, SIMDLevel::ARM_NEON>
+        : SQDistanceComputer {
+    using Sim = Similarity;
+
+    int d;
+    std::vector<uint8_t> tmp;
+
+    DistanceComputerByteSigned(int d, const std::vector<float>&)
+            : d(d), tmp(d) {
+        FAISS_THROW_IF_NOT(d % 16 == 0);
+    }
+
+    int compute_code_distance(const uint8_t* code1, const uint8_t* code2)
+            const {
+        if constexpr (Sim::metric_type == METRIC_INNER_PRODUCT) {
+            return neon_byte_ip_unbias(code1, code2, d);
+        } else {
+            // The bias cancels in the difference.
+            return neon_byte_l2sqr(code1, code2, d);
+        }
+    }
+
+    void set_query(const float* x) final {
+        for (int i = 0; i < d; i++) {
+            tmp[i] = uint8_t(int(x[i]) + 128);
+        }
+    }
+
+    int compute_distance(const float* x, const uint8_t* code) {
+        set_query(x);
+        return compute_code_distance(tmp.data(), code);
+    }
+
+    float symmetric_dis(idx_t i, idx_t j) override {
+        return compute_code_distance(
+                codes + i * code_size, codes + j * code_size);
+    }
+
+    float query_to_code(const uint8_t* code) const final {
+        return compute_code_distance(tmp.data(), code);
+    }
+};
+
 /**********************************************************
  * TurboQuant masked_sum NEON specialization (scalar fallback)
  **********************************************************/

data/vendor/faiss/faiss/impl/simd_dispatch.h

@@ -32,30 +32,32 @@ constexpr int AVAILABLE_SIMD_LEVELS_NONE = (1 << int(SIMDLevel::NONE));
 constexpr int AVAILABLE_SIMD_LEVELS_AVX2_NEON = AVAILABLE_SIMD_LEVELS_NONE |
         (1 << int(SIMDLevel::AVX2)) | (1 << int(SIMDLevel::ARM_NEON));
 
-// A0: same + AVX512 + RISCV_RVV
-constexpr int AVAILABLE_SIMD_LEVELS_A0 = AVAILABLE_SIMD_LEVELS_AVX2_NEON |
+// BASE: AVX2_NEON + AVX512 + RISCV_RVV
+constexpr int AVAILABLE_SIMD_LEVELS_BASE = AVAILABLE_SIMD_LEVELS_AVX2_NEON |
         (1 << int(SIMDLevel::AVX512)) | (1 << int(SIMDLevel::RISCV_RVV));
 
-// A0_SPR: same as A0 + AVX512_SPR (for functions with a dedicated SPR
-// specialization on top of an AVX512 fallback). Currently used by the
-// RaBitQ popcount kernels, which use VPOPCNTDQ on SPR+.
-constexpr int AVAILABLE_SIMD_LEVELS_A0_SPR =
-        AVAILABLE_SIMD_LEVELS_A0 | (1 << int(SIMDLevel::AVX512_SPR));
+// BASE_WITH_VPOPCNT: BASE + AVX512_VPOPCNT, for kernels that need only
+// VPOPCNTDQ on top of baseline AVX-512 (Ice Lake, Zen 4, Zen 5).
+constexpr int AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT =
+        AVAILABLE_SIMD_LEVELS_BASE | (1 << int(SIMDLevel::AVX512_VPOPCNT));
 
-// A1: same + ARM_SVE (for functions with dedicated SVE implementations)
-constexpr int AVAILABLE_SIMD_LEVELS_A1 =
-        AVAILABLE_SIMD_LEVELS_A0 | (1 << int(SIMDLevel::ARM_SVE));
+// BASE_WITH_SPR: BASE + AVX512_SPR, for kernels that need the whole SPR
+// feature set rather than VPOPCNTDQ alone.
+constexpr int AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR =
+        AVAILABLE_SIMD_LEVELS_BASE | (1 << int(SIMDLevel::AVX512_SPR));
 
-// A2: NONE + AVX2 + ARM_SVE only (for functions with only these
-// implementations)
-constexpr int AVAILABLE_SIMD_LEVELS_A2 = AVAILABLE_SIMD_LEVELS_NONE |
-        (1 << int(SIMDLevel::AVX2)) | (1 << int(SIMDLevel::ARM_SVE));
+// BASE_WITH_SVE: BASE + ARM_SVE, for functions with dedicated SVE
+// implementations.
+constexpr int AVAILABLE_SIMD_LEVELS_BASE_WITH_SVE =
+        AVAILABLE_SIMD_LEVELS_BASE | (1 << int(SIMDLevel::ARM_SVE));
 
 constexpr int AVAILABLE_SIMD_LEVELS_ALL = -1;
 
 constexpr SIMDLevel get_simd_fallback(SIMDLevel level) {
     switch (level) {
         case SIMDLevel::AVX512_SPR:
+            return SIMDLevel::AVX512_VPOPCNT;
+        case SIMDLevel::AVX512_VPOPCNT:
             return SIMDLevel::AVX512;
         case SIMDLevel::AVX512:
             return SIMDLevel::AVX2;
@@ -84,6 +86,24 @@ inline auto dispatch_with_fallback(LambdaType&& action) {
     }
 }
 
+/** Run action at current_level; on a null result, retry the next-lower level,
+ * down to NONE (terminal). action is called per level tried, so never moved. */
+template <int available_levels, SIMDLevel current_level, typename LambdaType>
+inline auto dispatch_simd_level_or_lower(LambdaType& action) {
+    if constexpr (current_level == SIMDLevel::NONE) {
+        return action.template operator()<SIMDLevel::NONE>();
+    } else {
+        if constexpr (available_levels & (1 << int(current_level))) {
+            if (auto result = action.template operator()<current_level>()) {
+                return result;
+            }
+        }
+        return dispatch_simd_level_or_lower<
+                available_levels,
+                get_simd_fallback(current_level)>(action);
+    }
+}
+
 /** The complete dispatching function. It takes into account:
  * - the currently selected SIMD level
  * - the compiled in SIMD levels (given by COMPILE_SIMD_XXX)
@@ -106,6 +126,15 @@ inline auto with_selected_simd_levels(LambdaType&& action) {
             [[fallthrough]];
 #endif
 
+#ifdef COMPILE_SIMD_AVX512_VPOPCNT
+        case SIMDLevel::AVX512_VPOPCNT:
+            if constexpr (
+                    available_levels & (1 << int(SIMDLevel::AVX512_VPOPCNT))) {
+                return action.template operator()<SIMDLevel::AVX512_VPOPCNT>();
+            }
+            [[fallthrough]];
+#endif
+
 #ifdef COMPILE_SIMD_AVX512
         case SIMDLevel::AVX512:
             if constexpr (available_levels & (1 << int(SIMDLevel::AVX512))) {
@@ -153,7 +182,7 @@ inline auto with_selected_simd_levels(LambdaType&& action) {
     // In static mode, SINGLE_SIMD_LEVEL is a constexpr resolved at compile
     // time. We mirror the DD fallthrough behavior at compile time via
     // dispatch_with_fallback, which recursively walks get_simd_fallback:
-    //   x86:   AVX512_SPR -> AVX512 -> AVX2 -> NONE
+    //   x86:   AVX512_SPR -> AVX512_VPOPCNT -> AVX512 -> AVX2 -> NONE
     //   ARM:   ARM_SVE -> ARM_NEON -> NONE
     //   RISCV: RISCV_RVV -> NONE
     // The first level in the chain that appears in available_levels is
@@ -163,6 +192,18 @@ inline auto with_selected_simd_levels(LambdaType&& action) {
 #endif
 }
 
+/** Like with_selected_simd_levels, but for factory actions that return null to
+ * decline a level (e.g. AVX-512 needing d % 16 == 0). Falls back to the next
+ * lower level, down to NONE. */
+template <int available_levels, typename LambdaType>
+inline auto with_simd_level_fallback(const LambdaType& action) {
+    return with_selected_simd_levels<available_levels>(
+            [&action]<SIMDLevel SL>() {
+                return dispatch_simd_level_or_lower<available_levels, SL>(
+                        action);
+            });
+}
+
 /**
  * Dispatch to a lambda with SIMDLevel as a compile-time constant.
  *
@@ -194,16 +235,7 @@ inline auto with_selected_simd_levels(LambdaType&& action) {
  */
 template <typename LambdaType>
 inline auto with_simd_level(LambdaType&& action) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0>(
-            std::forward<LambdaType>(action));
-}
-
-/**
- * Use for functions with AVX512_SPR-specific implementations.
- */
-template <typename LambdaType>
-inline auto with_simd_level_spr(LambdaType&& action) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE>(
             std::forward<LambdaType>(action));
 }
 
@@ -217,13 +249,20 @@ inline auto with_simd_level_256bit(LambdaType&& action) {
             std::forward<LambdaType>(action));
 }
 
-/**
- * Use for functions that have A0-level implementations plus an AVX512_SPR
- * specialization (e.g. using VPOPCNTDQ).
- */
+// Plain with_simd_level() uses BASE, which omits the optional levels below.
+// A call site must opt in, or its specialization is never instantiated.
+
+/// BASE + ARM_SVE.
+template <typename LambdaType>
+inline auto with_simd_level_with_sve(LambdaType&& action) {
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_SVE>(
+            std::forward<LambdaType>(action));
+}
+
+/// BASE + AVX512_VPOPCNT.
 template <typename LambdaType>
-inline auto with_simd_level_a0_spr(LambdaType&& action) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+inline auto with_simd_level_with_vpopcnt(LambdaType&& action) {
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT>(
             std::forward<LambdaType>(action));
 }