faiss 0.6.3 → 0.6.4

data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h

@@ -29,19 +29,31 @@ struct WrappedSearchResult {
     ResultHandler& res;
     size_t nup = 0;
     idx_t list_no;
-
+    size_t list_size;
     const idx_t* ids;
     const IDSelector* sel;
+    IDSelectorContextDispatch dispatch;
 
     WrappedSearchResult(
             idx_t list_no_in,
+            size_t list_size_in,
             const idx_t* ids_in,
             const IDSelector* sel_in,
             ResultHandler& res_in)
-            : res(res_in), list_no(list_no_in), ids(ids_in), sel(sel_in) {}
+            : res(res_in),
+              list_no(list_no_in),
+              list_size(list_size_in),
+              ids(ids_in),
+              sel(sel_in),
+              // A selector implies real ids, so ids==nullptr iff store_pairs;
+              // that disables the context path exactly when ids[] is synthetic.
+              dispatch(sel_in, /*store_pairs=*/ids_in == nullptr) {}
 
     inline bool skip_entry(idx_t j) {
-        return use_sel && !sel->is_member(ids[j]);
+        return use_sel &&
+                !dispatch.is_member(
+                        ids[j],
+                        IDScanContext{ids, list_size, static_cast<size_t>(j)});
     }
 
     inline void add(idx_t j, float dis) {
@@ -204,41 +216,41 @@ struct IVFPQScannerT : QueryTables {
         }
     }
 
+    float on_the_fly_dis0 = 0;
+
+    void prepare_on_the_fly_distance() {
+        on_the_fly_dis0 = 0;
+        if (!by_residual) {
+            return;
+        }
+        if (METRIC_TYPE == METRIC_INNER_PRODUCT) {
+            ivfpq.quantizer->reconstruct(key, residual_vec);
+            on_the_fly_dis0 = fvec_inner_product_dispatch(residual_vec, qi, d);
+        } else {
+            ivfpq.quantizer->compute_residual(qi, residual_vec, key);
+        }
+    }
+
+    float distance_to_code_on_the_fly(const uint8_t* code) const {
+        pq.decode(code, decoded_vec);
+        if (METRIC_TYPE == METRIC_INNER_PRODUCT) {
+            return on_the_fly_dis0 +
+                    fvec_inner_product_dispatch(decoded_vec, qi, d);
+        }
+        const float* dvec = by_residual ? residual_vec : qi;
+        return fvec_L2sqr_dispatch(decoded_vec, dvec, d);
+    }
+
     /// nothing is precomputed: access residuals on-the-fly
     template <class SearchResultType>
     void scan_on_the_fly_dist(
             size_t ncode,
             const uint8_t* codes,
             SearchResultType& res) const {
-        const float* dvec;
-        float local_dis0 = 0;
-        if (by_residual) {
-            if (METRIC_TYPE == METRIC_INNER_PRODUCT) {
-                ivfpq.quantizer->reconstruct(key, residual_vec);
-                local_dis0 = fvec_inner_product_dispatch(residual_vec, qi, d);
-            } else {
-                ivfpq.quantizer->compute_residual(qi, residual_vec, key);
-            }
-            dvec = residual_vec;
-        } else {
-            dvec = qi;
-            local_dis0 = 0;
-        }
-
         for (size_t j = 0; j < ncode; j++, codes += pq.code_size) {
-            if (res.skip_entry(j)) {
-                continue;
-            }
-            pq.decode(codes, decoded_vec);
-
-            float dis;
-            if (METRIC_TYPE == METRIC_INNER_PRODUCT) {
-                dis = local_dis0 +
-                        fvec_inner_product_dispatch(decoded_vec, qi, d);
-            } else {
-                dis = fvec_L2sqr_dispatch(decoded_vec, dvec, d);
+            if (!res.skip_entry(j)) {
+                res.add(j, distance_to_code_on_the_fly(codes));
             }
-            res.add(j, dis);
         }
     }
 
@@ -426,7 +438,6 @@ struct IVFPQScannerT : QueryTables {
  *
  * precompute_mode is how much we precompute (2 = precompute distance tables,
  * 1 = precompute pointers to distances, 0 = compute distances one by one).
- * Currently only 2 is supported
  *
  * use_sel: store or ignore the IDSelector
  */
@@ -451,20 +462,29 @@ struct IVFPQScanner : IVFPQScannerT<idx_t, METRIC_TYPE, PQCodeDist>,
     }
 
     void set_query(const float* query) override {
-        this->init_query(query);
+        if (precompute_mode == 0) {
+            this->qi = query;
+        } else {
+            this->init_query(query);
+        }
     }
 
     void set_list(idx_t list_no_in, float coarse_dis_in) override {
         this->list_no = list_no_in;
         this->init_list(list_no_in, coarse_dis_in, precompute_mode);
+        if (precompute_mode == 0) {
+            this->prepare_on_the_fly_distance();
+        }
     }
 
     float distance_to_code(const uint8_t* code) const override {
+        if (precompute_mode == 0) {
+            return this->distance_to_code_on_the_fly(code);
+        }
         FAISS_THROW_IF_NOT(precompute_mode == 2);
-        float dis = this->dis0 +
+        return this->dis0 +
                 PQCodeDist::distance_single_code(
-                            this->pq.M, this->pq.nbits, this->sim_table, code);
-        return dis;
+                        this->pq.M, this->pq.nbits, this->sim_table, code);
     }
 
     size_t scan_codes(
@@ -474,6 +494,7 @@ struct IVFPQScanner : IVFPQScannerT<idx_t, METRIC_TYPE, PQCodeDist>,
             ResultHandler& handler) const override {
         WrappedSearchResult<C, use_sel> res(
                 this->key,
+                ncode,
                 this->store_pairs ? nullptr : ids,
                 this->sel,
                 handler);
@@ -498,27 +519,40 @@ template <SIMDLevel SL>
 InvertedListScanner* make_IVFPQInvertedListScanner(
         const IndexIVFPQ& ivfpq,
         bool store_pairs,
-        const IDSelector* sel);
+        const IDSelector* sel,
+        IndexIVFPQ::ScannerMode mode);
+
+template <SIMDLevel SL>
+inline InvertedListScanner* make_IVFPQInvertedListScanner(
+        const IndexIVFPQ& ivfpq,
+        bool store_pairs,
+        const IDSelector* sel) {
+    return make_IVFPQInvertedListScanner<SL>(
+            ivfpq, store_pairs, sel, IndexIVFPQ::ScannerMode::Precomputed);
+}
 
 // NOLINTNEXTLINE(facebook-hte-MisplacedTemplateSpecialization)
 template <>
 InvertedListScanner* make_IVFPQInvertedListScanner<THE_SIMD_LEVEL>(
         const IndexIVFPQ& ivfpq,
         bool store_pairs,
-        const IDSelector* sel) {
+        const IDSelector* sel,
+        IndexIVFPQ::ScannerMode mode) {
+    const int precompute_mode =
+            mode == IndexIVFPQ::ScannerMode::Precomputed ? 2 : 0;
     auto make = [&]<class PQCodeDist, bool use_sel>() -> InvertedListScanner* {
         if (ivfpq.metric_type == METRIC_INNER_PRODUCT) {
             return new IVFPQScanner<
                     METRIC_INNER_PRODUCT,
                     CMin<float, idx_t>,
                     PQCodeDist,
-                    use_sel>(ivfpq, store_pairs, 2, sel);
+                    use_sel>(ivfpq, store_pairs, precompute_mode, sel);
         } else if (ivfpq.metric_type == METRIC_L2) {
             return new IVFPQScanner<
                     METRIC_L2,
                     CMax<float, idx_t>,
                     PQCodeDist,
-                    use_sel>(ivfpq, store_pairs, 2, sel);
+                    use_sel>(ivfpq, store_pairs, precompute_mode, sel);
         } else {
             FAISS_THROW_MSG("unsupported metric type");
         }

data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp

@@ -7,7 +7,7 @@
 
 // This TU provides non-templated PQ code distance dispatch wrappers
 // (pq_code_distance_8bit_single, pq_code_distance_8bit_four) declared
-// in pq_code_distance-inl.h. These use with_simd_level to route to the
+// in pq_code_distance-inl.h. These use with_simd_level_with_sve to route to the
 // best available SIMD implementation via pq_code_distance_8bit_*_impl
 // function template specializations.
 //
@@ -34,7 +34,7 @@ void pq_scan_8bit(
         float* heap_dis,
         int64_t* heap_ids,
         bool max_heap) {
-    with_simd_level([&]<SIMDLevel SL>() {
+    with_simd_level_with_sve([&]<SIMDLevel SL>() {
         pq_scan_8bit_impl<SL>(
                 M, dis_table, codes, ncodes, k, heap_dis, heap_ids, max_heap);
     });
@@ -44,7 +44,7 @@ float pq_code_distance_8bit_single(
         size_t M,
         const float* sim_table,
         const uint8_t* code) {
-    return with_simd_level([&]<SIMDLevel SL>() {
+    return with_simd_level_with_sve([&]<SIMDLevel SL>() {
         return pq_code_distance_8bit_single_impl<SL>(M, sim_table, code);
     });
 }
@@ -60,7 +60,7 @@ void pq_code_distance_8bit_four(
         float& result1,
         float& result2,
         float& result3) {
-    with_simd_level([&]<SIMDLevel SL>() {
+    with_simd_level_with_sve([&]<SIMDLevel SL>() {
         pq_code_distance_8bit_four_impl<SL>(
                 M,
                 sim_table,

data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp

@@ -319,8 +319,13 @@ struct EDENDistanceComputerBase : EDENFlatCodesDistanceComputer {
             const EDENCodeFactors* factors,
             float code_dot_query) const {
         if (metric_type == MetricType::METRIC_L2) {
-            return query_base + factors->l2_norm_term -
+            // The unbiased scale reports an unbiased estimator of the squared
+            // L2 distance rather than a true squared distance, so the raw
+            // value can dip below zero for strongly-aligned residuals. Clamp
+            // to keep the METRIC_L2 contract (distances are never negative).
+            const float dist = query_base + factors->l2_norm_term -
                     2.0f * factors->scale * code_dot_query;
+            return dist < 0.0f ? 0.0f : dist;
         }
         return query_base + factors->scale * code_dot_query;
     }

data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h

@@ -78,8 +78,8 @@ template <class Similarity, SIMDLevel SL>
 struct DistanceComputerByte : SQDistanceComputer {};
 
 // Byte-domain distance computer for QT_8bit_direct_signed (storage is
-// value+128). Only specialized for AVX512_SPR; other levels fall back to
-// the float-domain DCTemplate path via the dispatch logic.
+// value+128). A level added to the sq-dispatch.h chain without a
+// specialization here instantiates this empty template.
 template <class Similarity, SIMDLevel SL>
 struct DistanceComputerByteSigned : SQDistanceComputer {};
 

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp

@@ -733,7 +733,9 @@ struct DistanceComputerByte<Similarity, SIMDLevel::AVX2> : SQDistanceComputer {
     int d;
     std::vector<uint8_t> tmp;
 
-    DistanceComputerByte(int d, const std::vector<float>&) : d(d), tmp(d) {}
+    DistanceComputerByte(int d, const std::vector<float>&) : d(d), tmp(d) {
+        FAISS_THROW_IF_NOT(d % 16 == 0);
+    }
 
     int compute_code_distance(const uint8_t* code1, const uint8_t* code2)
             const {
@@ -796,7 +798,9 @@ struct DistanceComputerByteSigned<Similarity, SIMDLevel::AVX2>
     std::vector<uint8_t> tmp;
 
     DistanceComputerByteSigned(int d, const std::vector<float>&)
-            : d(d), tmp(d) {}
+            : d(d), tmp(d) {
+        FAISS_THROW_IF_NOT(d % 16 == 0);
+    }
 
     int compute_code_distance(const uint8_t* code1, const uint8_t* code2)
             const {

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h

@@ -28,6 +28,14 @@ namespace scalar_quantizer {
 // Define SL as alias for THE_LEVEL_TO_DISPATCH for use in this file
 constexpr SIMDLevel SL = THE_LEVEL_TO_DISPATCH;
 
+// The SPR scalar-quantizer implementation uses VPOPCNTDQ for its RaBitQ
+// popcount helpers, but those helpers belong to the narrower VPOPCNT
+// capability. Keep the SQ implementation at AVX512_SPR while explicitly
+// reusing the independently dispatched VPOPCNT kernels.
+template <SIMDLevel SL0>
+inline constexpr SIMDLevel rabitq_popcount_level =
+        SL0 == SIMDLevel::AVX512_SPR ? SIMDLevel::AVX512_VPOPCNT : SL0;
+
 /*******************************************************************
  * TurboQuant SIMD kernel: masked_sum
  * Compute sum of arr[j] where bit j of the bitmask is set.
@@ -259,9 +267,11 @@ struct DCTurboQuantFull : ScalarQuantizer::TurboQuantRefine::DistanceComputer {
             if (qb > 0) {
                 // Integer popcount path for 1-bit MSE
                 size_t byte_size = (d + 7) / 8;
-                uint64_t and_result = rabitq::bitwise_and_dot_product<SL2>(
+                uint64_t and_result = rabitq::bitwise_and_dot_product<
+                        rabitq_popcount_level<SL2>>(
                         rearranged_q.data(), code, byte_size, qb);
-                uint64_t pop = rabitq::popcount<SL2>(code, byte_size);
+                uint64_t pop = rabitq::popcount<rabitq_popcount_level<SL2>>(
+                        code, byte_size);
                 mse_dot = mse_base +
                         mse_int_scale * static_cast<float>(and_result) +
                         mse_popcnt_scale * static_cast<float>(pop);
@@ -316,9 +326,11 @@ struct DCTurboQuantFull : ScalarQuantizer::TurboQuantRefine::DistanceComputer {
         float qjl_dot;
         if (qb > 0 && int_qjl) {
             size_t byte_size = (d + 7) / 8;
-            uint64_t and_result = rabitq::bitwise_and_dot_product<SL2>(
-                    rearranged_qproj.data(), qjl_code, byte_size, qb);
-            uint64_t pop = rabitq::popcount<SL2>(qjl_code, byte_size);
+            uint64_t and_result =
+                    rabitq::bitwise_and_dot_product<rabitq_popcount_level<SL2>>(
+                            rearranged_qproj.data(), qjl_code, byte_size, qb);
+            uint64_t pop = rabitq::popcount<rabitq_popcount_level<SL2>>(
+                    qjl_code, byte_size);
             float pos_sum = qjl_popcnt_scale * static_cast<float>(pop) +
                     qjl_int_scale * static_cast<float>(and_result);
             qjl_dot = qjl_coeff * gamma * (2.0f * pos_sum - total_qproj_sum);
@@ -342,10 +354,17 @@ struct DCTurboQuantFull : ScalarQuantizer::TurboQuantRefine::DistanceComputer {
     }
 };
 
+// True for every level that runs the 512-bit kernels. AVX512_VPOPCNT is
+// unreachable here until an SQ translation unit is compiled at that level;
+// naming it keeps the 512-bit alignment rule correct when one is.
+template <SIMDLevel SL2>
+constexpr bool is_avx512_family = SL2 == SIMDLevel::AVX512 ||
+        SL2 == SIMDLevel::AVX512_VPOPCNT || SL2 == SIMDLevel::AVX512_SPR;
+
 // Returns true if dimension d is compatible with the given SIMD level
 template <SIMDLevel SL2>
 constexpr bool is_dimension_compatible(size_t d) {
-    if constexpr (SL2 == SIMDLevel::AVX512 || SL2 == SIMDLevel::AVX512_SPR) {
+    if constexpr (is_avx512_family<SL2>) {
         return d % 16 == 0;
     } else if constexpr (SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
         return d % 8 == 0;
@@ -525,13 +544,13 @@ SQDistanceComputer* select_distance_computer_body(
             return new DCTemplate<QuantizerBF16<SL2>, Sim, SL2>(d, trained);
 
         case ScalarQuantizer::QT_8bit_direct:
-            if constexpr (
-                    SL2 == SIMDLevel::AVX512 || SL2 == SIMDLevel::AVX512_SPR) {
+            if constexpr (is_avx512_family<SL2>) {
                 if (d % 32 == 0) {
                     return new DistanceComputerByte<Sim, SL2>(
                             static_cast<int>(d), trained);
                 }
-            } else if constexpr (SL2 == SIMDLevel::AVX2) {
+            } else if constexpr (
+                    SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
                 if (d % 16 == 0) {
                     return new DistanceComputerByte<Sim, SL2>(
                             static_cast<int>(d), trained);
@@ -546,12 +565,13 @@ SQDistanceComputer* select_distance_computer_body(
                     return new DistanceComputerByteSigned<Sim, SL2>(
                             static_cast<int>(d), trained);
                 }
-            } else if constexpr (SL2 == SIMDLevel::AVX512) {
+            } else if constexpr (is_avx512_family<SL2>) {
                 if (d % 32 == 0) {
                     return new DistanceComputerByteSigned<Sim, SL2>(
                             static_cast<int>(d), trained);
                 }
-            } else if constexpr (SL2 == SIMDLevel::AVX2) {
+            } else if constexpr (
+                    SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
                 if (d % 16 == 0) {
                     return new DistanceComputerByteSigned<Sim, SL2>(
                             static_cast<int>(d), trained);
@@ -737,14 +757,13 @@ InvertedListScanner* sq_select_InvertedListScanner<THE_LEVEL_TO_DISPATCH>(
                 return scan.template
                 operator()<DCTemplate<QuantizerBF16<SL2>, Similarity, SL2>>();
             case ScalarQuantizer::QT_8bit_direct:
-                if constexpr (
-                        SL2 == SIMDLevel::AVX512 ||
-                        SL2 == SIMDLevel::AVX512_SPR) {
+                if constexpr (is_avx512_family<SL2>) {
                     if (d % 32 == 0) {
                         return scan.template
                         operator()<DistanceComputerByte<Similarity, SL2>>();
                     }
-                } else if constexpr (SL2 == SIMDLevel::AVX2) {
+                } else if constexpr (
+                        SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
                     if (d % 16 == 0) {
                         return scan.template
                         operator()<DistanceComputerByte<Similarity, SL2>>();
@@ -760,12 +779,13 @@ InvertedListScanner* sq_select_InvertedListScanner<THE_LEVEL_TO_DISPATCH>(
                         return scan.template operator()<
                                 DistanceComputerByteSigned<Similarity, SL2>>();
                     }
-                } else if constexpr (SL2 == SIMDLevel::AVX512) {
+                } else if constexpr (is_avx512_family<SL2>) {
                     if (d % 32 == 0) {
                         return scan.template operator()<
                                 DistanceComputerByteSigned<Similarity, SL2>>();
                     }
-                } else if constexpr (SL2 == SIMDLevel::AVX2) {
+                } else if constexpr (
+                        SL2 == SIMDLevel::AVX2 || SL2 == SIMDLevel::ARM_NEON) {
                     if (d % 16 == 0) {
                         return scan.template operator()<
                                 DistanceComputerByteSigned<Similarity, SL2>>();

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp

@@ -618,6 +618,64 @@ struct DCTemplate<Quantizer, Similarity, SIMDLevel::ARM_NEON>
     }
 };
 
+// Byte-domain kernels for QT_8bit_direct{,_signed}. The dispatch only
+// selects them when d % 16 == 0, so no loop needs a tail.
+
+namespace {
+
+// The accumulator stays unsigned: a vmull_u8 square reaches 255*255 = 65025,
+// which an int16 lane would read as negative.
+FAISS_ALWAYS_INLINE int neon_byte_l2sqr(
+        const uint8_t* code1,
+        const uint8_t* code2,
+        int d) {
+    uint32x4_t accu = vdupq_n_u32(0);
+    for (int i = 0; i < d; i += 16) {
+        const uint8x16_t diff =
+                vabdq_u8(vld1q_u8(code1 + i), vld1q_u8(code2 + i));
+        accu = vpadalq_u16(
+                accu, vmull_u8(vget_low_u8(diff), vget_low_u8(diff)));
+        accu = vpadalq_u16(
+                accu, vmull_u8(vget_high_u8(diff), vget_high_u8(diff)));
+    }
+    return static_cast<int>(vaddvq_u32(accu));
+}
+
+FAISS_ALWAYS_INLINE int neon_byte_ip(
+        const uint8_t* code1,
+        const uint8_t* code2,
+        int d) {
+    uint32x4_t accu = vdupq_n_u32(0);
+    for (int i = 0; i < d; i += 16) {
+        const uint8x16_t c1 = vld1q_u8(code1 + i);
+        const uint8x16_t c2 = vld1q_u8(code2 + i);
+        accu = vpadalq_u16(accu, vmull_u8(vget_low_u8(c1), vget_low_u8(c2)));
+        accu = vpadalq_u16(accu, vmull_u8(vget_high_u8(c1), vget_high_u8(c2)));
+    }
+    return static_cast<int>(vaddvq_u32(accu));
+}
+
+// The codes store value + 128. For x in 0 to 255, x ^ 0x80 read as int8 is
+// exactly x - 128, which is how the bias comes off before vmull_s8.
+FAISS_ALWAYS_INLINE int neon_byte_ip_unbias(
+        const uint8_t* code1,
+        const uint8_t* code2,
+        int d) {
+    const uint8x16_t bias = vdupq_n_u8(0x80);
+    int32x4_t accu = vdupq_n_s32(0);
+    for (int i = 0; i < d; i += 16) {
+        const int8x16_t c1 =
+                vreinterpretq_s8_u8(veorq_u8(vld1q_u8(code1 + i), bias));
+        const int8x16_t c2 =
+                vreinterpretq_s8_u8(veorq_u8(vld1q_u8(code2 + i), bias));
+        accu = vpadalq_s16(accu, vmull_s8(vget_low_s8(c1), vget_low_s8(c2)));
+        accu = vpadalq_s16(accu, vmull_s8(vget_high_s8(c1), vget_high_s8(c2)));
+    }
+    return static_cast<int>(vaddvq_s32(accu));
+}
+
+} // namespace
+
 template <class Similarity>
 struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
         : SQDistanceComputer {
@@ -626,20 +684,17 @@ struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
     int d;
     std::vector<uint8_t> tmp;
 
-    DistanceComputerByte(int d, const std::vector<float>&) : d(d), tmp(d) {}
+    DistanceComputerByte(int d, const std::vector<float>&) : d(d), tmp(d) {
+        FAISS_THROW_IF_NOT(d % 16 == 0);
+    }
 
     int compute_code_distance(const uint8_t* code1, const uint8_t* code2)
             const {
-        int accu = 0;
-        for (int i = 0; i < d; i++) {
-            if (Sim::metric_type == METRIC_INNER_PRODUCT) {
-                accu += int(code1[i]) * code2[i];
-            } else {
-                int diff = int(code1[i]) - code2[i];
-                accu += diff * diff;
-            }
+        if constexpr (Sim::metric_type == METRIC_INNER_PRODUCT) {
+            return neon_byte_ip(code1, code2, d);
+        } else {
+            return neon_byte_l2sqr(code1, code2, d);
         }
-        return accu;
     }
 
     void set_query(const float* x) final {
@@ -663,6 +718,50 @@ struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
     }
 };
 
+template <class Similarity>
+struct DistanceComputerByteSigned<Similarity, SIMDLevel::ARM_NEON>
+        : SQDistanceComputer {
+    using Sim = Similarity;
+
+    int d;
+    std::vector<uint8_t> tmp;
+
+    DistanceComputerByteSigned(int d, const std::vector<float>&)
+            : d(d), tmp(d) {
+        FAISS_THROW_IF_NOT(d % 16 == 0);
+    }
+
+    int compute_code_distance(const uint8_t* code1, const uint8_t* code2)
+            const {
+        if constexpr (Sim::metric_type == METRIC_INNER_PRODUCT) {
+            return neon_byte_ip_unbias(code1, code2, d);
+        } else {
+            // The bias cancels in the difference.
+            return neon_byte_l2sqr(code1, code2, d);
+        }
+    }
+
+    void set_query(const float* x) final {
+        for (int i = 0; i < d; i++) {
+            tmp[i] = uint8_t(int(x[i]) + 128);
+        }
+    }
+
+    int compute_distance(const float* x, const uint8_t* code) {
+        set_query(x);
+        return compute_code_distance(tmp.data(), code);
+    }
+
+    float symmetric_dis(idx_t i, idx_t j) override {
+        return compute_code_distance(
+                codes + i * code_size, codes + j * code_size);
+    }
+
+    float query_to_code(const uint8_t* code) const final {
+        return compute_code_distance(tmp.data(), code);
+    }
+};
+
 /**********************************************************
  * TurboQuant masked_sum NEON specialization (scalar fallback)
  **********************************************************/

data/vendor/faiss/faiss/impl/simd_dispatch.h

@@ -32,30 +32,32 @@ constexpr int AVAILABLE_SIMD_LEVELS_NONE = (1 << int(SIMDLevel::NONE));
 constexpr int AVAILABLE_SIMD_LEVELS_AVX2_NEON = AVAILABLE_SIMD_LEVELS_NONE |
         (1 << int(SIMDLevel::AVX2)) | (1 << int(SIMDLevel::ARM_NEON));
 
-// A0: same + AVX512 + RISCV_RVV
-constexpr int AVAILABLE_SIMD_LEVELS_A0 = AVAILABLE_SIMD_LEVELS_AVX2_NEON |
+// BASE: AVX2_NEON + AVX512 + RISCV_RVV
+constexpr int AVAILABLE_SIMD_LEVELS_BASE = AVAILABLE_SIMD_LEVELS_AVX2_NEON |
         (1 << int(SIMDLevel::AVX512)) | (1 << int(SIMDLevel::RISCV_RVV));
 
-// A0_SPR: same as A0 + AVX512_SPR (for functions with a dedicated SPR
-// specialization on top of an AVX512 fallback). Currently used by the
-// RaBitQ popcount kernels, which use VPOPCNTDQ on SPR+.
-constexpr int AVAILABLE_SIMD_LEVELS_A0_SPR =
-        AVAILABLE_SIMD_LEVELS_A0 | (1 << int(SIMDLevel::AVX512_SPR));
+// BASE_WITH_VPOPCNT: BASE + AVX512_VPOPCNT, for kernels that need only
+// VPOPCNTDQ on top of baseline AVX-512 (Ice Lake, Zen 4, Zen 5).
+constexpr int AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT =
+        AVAILABLE_SIMD_LEVELS_BASE | (1 << int(SIMDLevel::AVX512_VPOPCNT));
 
-// A1: same + ARM_SVE (for functions with dedicated SVE implementations)
-constexpr int AVAILABLE_SIMD_LEVELS_A1 =
-        AVAILABLE_SIMD_LEVELS_A0 | (1 << int(SIMDLevel::ARM_SVE));
+// BASE_WITH_SPR: BASE + AVX512_SPR, for kernels that need the whole SPR
+// feature set rather than VPOPCNTDQ alone.
+constexpr int AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR =
+        AVAILABLE_SIMD_LEVELS_BASE | (1 << int(SIMDLevel::AVX512_SPR));
 
-// A2: NONE + AVX2 + ARM_SVE only (for functions with only these
-// implementations)
-constexpr int AVAILABLE_SIMD_LEVELS_A2 = AVAILABLE_SIMD_LEVELS_NONE |
-        (1 << int(SIMDLevel::AVX2)) | (1 << int(SIMDLevel::ARM_SVE));
+// BASE_WITH_SVE: BASE + ARM_SVE, for functions with dedicated SVE
+// implementations.
+constexpr int AVAILABLE_SIMD_LEVELS_BASE_WITH_SVE =
+        AVAILABLE_SIMD_LEVELS_BASE | (1 << int(SIMDLevel::ARM_SVE));
 
 constexpr int AVAILABLE_SIMD_LEVELS_ALL = -1;
 
 constexpr SIMDLevel get_simd_fallback(SIMDLevel level) {
     switch (level) {
         case SIMDLevel::AVX512_SPR:
+            return SIMDLevel::AVX512_VPOPCNT;
+        case SIMDLevel::AVX512_VPOPCNT:
             return SIMDLevel::AVX512;
         case SIMDLevel::AVX512:
             return SIMDLevel::AVX2;
@@ -124,6 +126,15 @@ inline auto with_selected_simd_levels(LambdaType&& action) {
             [[fallthrough]];
 #endif
 
+#ifdef COMPILE_SIMD_AVX512_VPOPCNT
+        case SIMDLevel::AVX512_VPOPCNT:
+            if constexpr (
+                    available_levels & (1 << int(SIMDLevel::AVX512_VPOPCNT))) {
+                return action.template operator()<SIMDLevel::AVX512_VPOPCNT>();
+            }
+            [[fallthrough]];
+#endif
+
 #ifdef COMPILE_SIMD_AVX512
         case SIMDLevel::AVX512:
             if constexpr (available_levels & (1 << int(SIMDLevel::AVX512))) {
@@ -171,7 +182,7 @@ inline auto with_selected_simd_levels(LambdaType&& action) {
     // In static mode, SINGLE_SIMD_LEVEL is a constexpr resolved at compile
     // time. We mirror the DD fallthrough behavior at compile time via
     // dispatch_with_fallback, which recursively walks get_simd_fallback:
-    //   x86:   AVX512_SPR -> AVX512 -> AVX2 -> NONE
+    //   x86:   AVX512_SPR -> AVX512_VPOPCNT -> AVX512 -> AVX2 -> NONE
     //   ARM:   ARM_SVE -> ARM_NEON -> NONE
     //   RISCV: RISCV_RVV -> NONE
     // The first level in the chain that appears in available_levels is
@@ -224,7 +235,7 @@ inline auto with_simd_level_fallback(const LambdaType& action) {
  */
 template <typename LambdaType>
 inline auto with_simd_level(LambdaType&& action) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0>(
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE>(
             std::forward<LambdaType>(action));
 }
 
@@ -238,13 +249,20 @@ inline auto with_simd_level_256bit(LambdaType&& action) {
             std::forward<LambdaType>(action));
 }
 
-/**
- * Use for functions that have A0-level implementations plus an AVX512_SPR
- * specialization (e.g. using VPOPCNTDQ).
- */
+// Plain with_simd_level() uses BASE, which omits the optional levels below.
+// A call site must opt in, or its specialization is never instantiated.
+
+/// BASE + ARM_SVE.
+template <typename LambdaType>
+inline auto with_simd_level_with_sve(LambdaType&& action) {
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_SVE>(
+            std::forward<LambdaType>(action));
+}
+
+/// BASE + AVX512_VPOPCNT.
 template <typename LambdaType>
-inline auto with_simd_level_a0_spr(LambdaType&& action) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+inline auto with_simd_level_with_vpopcnt(LambdaType&& action) {
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT>(
             std::forward<LambdaType>(action));
 }