faiss 0.6.0 → 0.6.2

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp

@@ -7,6 +7,11 @@
 
 #ifdef COMPILE_SIMD_ARM_NEON
 
+#include <faiss/impl/simdlib/simdlib_neon.h>
+
+#include <algorithm>
+#include <cstring>
+
 #include <faiss/impl/scalar_quantizer/codecs.h>
 #include <faiss/impl/scalar_quantizer/distance_computers.h>
 #include <faiss/impl/scalar_quantizer/quantizers.h>
@@ -17,6 +22,81 @@ namespace faiss {
 
 namespace scalar_quantizer {
 
+using simd8float32 = faiss::simd8float32_tpl<SIMDLevel::ARM_NEON>;
+
+namespace {
+
+FAISS_ALWAYS_INLINE uint16_t load_u16(const uint8_t* ptr) {
+    uint16_t value;
+    std::memcpy(&value, ptr, sizeof(value));
+    return value;
+}
+
+FAISS_ALWAYS_INLINE uint32_t load_u32(const uint8_t* ptr) {
+    uint32_t value;
+    std::memcpy(&value, ptr, sizeof(value));
+    return value;
+}
+
+FAISS_ALWAYS_INLINE uint32_t load_u24(const uint8_t* ptr) {
+    return static_cast<uint32_t>(ptr[0]) |
+            (static_cast<uint32_t>(ptr[1]) << 8) |
+            (static_cast<uint32_t>(ptr[2]) << 16);
+}
+
+FAISS_ALWAYS_INLINE void unpack_8x1bit_to_u8(
+        const uint8_t* code,
+        int i,
+        uint8_t out[8]) {
+    const uint8_t packed = code[static_cast<size_t>(i) >> 3];
+    for (size_t j = 0; j < 8; ++j) {
+        out[j] = (packed >> j) & 0x1;
+    }
+}
+
+FAISS_ALWAYS_INLINE void unpack_8x2bit_to_u8(
+        const uint8_t* code,
+        int i,
+        uint8_t out[8]) {
+    const uint16_t packed = load_u16(code + (static_cast<size_t>(i) >> 2));
+    for (size_t j = 0; j < 8; ++j) {
+        out[j] = (packed >> (2 * j)) & 0x3;
+    }
+}
+
+FAISS_ALWAYS_INLINE void unpack_8x3bit_to_u8(
+        const uint8_t* code,
+        int i,
+        uint8_t out[8]) {
+    const uint32_t packed =
+            load_u24(code + ((static_cast<size_t>(i) >> 3) * 3));
+    for (size_t j = 0; j < 8; ++j) {
+        out[j] = (packed >> (3 * j)) & 0x7;
+    }
+}
+
+FAISS_ALWAYS_INLINE void unpack_8x4bit_to_u8(
+        const uint8_t* code,
+        int i,
+        uint8_t out[8]) {
+    const uint32_t packed = load_u32(code + (static_cast<size_t>(i) >> 1));
+    for (size_t j = 0; j < 8; ++j) {
+        out[j] = (packed >> (4 * j)) & 0xf;
+    }
+}
+
+FAISS_ALWAYS_INLINE simd8float32
+gather_8_components(const float* codebook, const uint8_t indices[8]) {
+    float result[8];
+    for (size_t j = 0; j < 8; ++j) {
+        result[j] = codebook[indices[j]];
+    }
+    return simd8float32(
+            float32x4x2_t{vld1q_f32(result), vld1q_f32(result + 4)});
+}
+
+} // namespace
+
 /**********************************************************
  * Codecs
  **********************************************************/
@@ -101,6 +181,12 @@ struct QuantizerTemplate<
                                 xi.data.val[1],
                                 this->vdiff)});
     }
+
+    /// Raw codec decode without denormalization (for pre-decode opt)
+    FAISS_ALWAYS_INLINE simd8float32
+    decode_8_raw(const uint8_t* code, int i) const {
+        return Codec::decode_8_components(code, i);
+    }
 };
 
 template <class Codec>
@@ -136,6 +222,74 @@ struct QuantizerTemplate<
     }
 };
 
+/**********************************************************
+ * TurboQuant MSE quantizer
+ **********************************************************/
+
+// NEON TurboQuantMSE: decode via gather, encode stays scalar.
+// NEON doesn't have movemask so 1-bit encode is also scalar.
+#define DEFINE_TQMSE_NEON_SPECIALIZATION(NBITS, UNPACK_FN)                   \
+    template <>                                                              \
+    struct QuantizerTurboQuantMSE<NBITS, SIMDLevel::ARM_NEON>                \
+            : QuantizerTurboQuantMSE<NBITS, SIMDLevel::NONE> {               \
+        using Base = QuantizerTurboQuantMSE<NBITS, SIMDLevel::NONE>;         \
+                                                                             \
+        QuantizerTurboQuantMSE(size_t d, const std::vector<float>& trained)  \
+                : Base(d, trained) {                                         \
+            assert(d % 8 == 0);                                              \
+        }                                                                    \
+                                                                             \
+        FAISS_ALWAYS_INLINE simd8float32                                     \
+        reconstruct_8_components(const uint8_t* code, int i) const {         \
+            uint8_t indices[8];                                              \
+            UNPACK_FN(code, i, indices);                                     \
+            return gather_8_components(this->centroids, indices);            \
+        }                                                                    \
+                                                                             \
+        void decode_vector(const uint8_t* code, float* x) const final {      \
+            for (size_t i = 0; i < this->d; i += 8) {                        \
+                simd8float32 xi =                                            \
+                        reconstruct_8_components(code, static_cast<int>(i)); \
+                vst1q_f32(x + i, xi.data.val[0]);                            \
+                vst1q_f32(x + i + 4, xi.data.val[1]);                        \
+            }                                                                \
+        }                                                                    \
+    }
+
+DEFINE_TQMSE_NEON_SPECIALIZATION(1, unpack_8x1bit_to_u8);
+DEFINE_TQMSE_NEON_SPECIALIZATION(2, unpack_8x2bit_to_u8);
+DEFINE_TQMSE_NEON_SPECIALIZATION(3, unpack_8x3bit_to_u8);
+DEFINE_TQMSE_NEON_SPECIALIZATION(4, unpack_8x4bit_to_u8);
+
+#undef DEFINE_TQMSE_NEON_SPECIALIZATION
+
+template <>
+struct QuantizerTurboQuantMSE<8, SIMDLevel::ARM_NEON>
+        : QuantizerTurboQuantMSE<8, SIMDLevel::NONE> {
+    using Base = QuantizerTurboQuantMSE<8, SIMDLevel::NONE>;
+
+    QuantizerTurboQuantMSE(size_t d, const std::vector<float>& trained)
+            : Base(d, trained) {
+        assert(d % 8 == 0);
+    }
+
+    FAISS_ALWAYS_INLINE simd8float32
+    reconstruct_8_components(const uint8_t* code, int i) const {
+        uint8_t indices[8];
+        std::memcpy(indices, code + static_cast<size_t>(i), sizeof(indices));
+        return gather_8_components(this->centroids, indices);
+    }
+
+    void decode_vector(const uint8_t* code, float* x) const final {
+        for (size_t i = 0; i < this->d; i += 8) {
+            simd8float32 xi =
+                    reconstruct_8_components(code, static_cast<int>(i));
+            vst1q_f32(x + i, xi.data.val[0]);
+            vst1q_f32(x + i + 4, xi.data.val[1]);
+        }
+    }
+};
+
 /**********************************************************
  * FP16 Quantizer
  **********************************************************/
@@ -270,6 +424,22 @@ struct SimilarityL2<SIMDLevel::ARM_NEON> {
     FAISS_ALWAYS_INLINE float result_8() {
         return horizontal_add(accu8);
     }
+
+    static void adjust_query_for_raw_decode(
+            const float* x,
+            float* q_adj,
+            size_t d,
+            float vmin,
+            float vdiff,
+            float& scale_factor,
+            float& bias) {
+        float inv_vdiff = (vdiff != 0) ? 1.0f / vdiff : 0.0f;
+        for (size_t i = 0; i < d; i++) {
+            q_adj[i] = (x[i] - vmin) * inv_vdiff;
+        }
+        scale_factor = vdiff * vdiff;
+        bias = 0;
+    }
 };
 
 template <>
@@ -304,6 +474,23 @@ struct SimilarityIP<SIMDLevel::ARM_NEON> {
     FAISS_ALWAYS_INLINE float result_8() {
         return horizontal_add(accu8);
     }
+
+    static void adjust_query_for_raw_decode(
+            const float* x,
+            float* q_adj,
+            size_t d,
+            float vmin,
+            float vdiff,
+            float& scale_factor,
+            float& bias) {
+        float sum_q = 0;
+        for (size_t i = 0; i < d; i++) {
+            q_adj[i] = x[i];
+            sum_q += x[i];
+        }
+        scale_factor = vdiff;
+        bias = vmin * sum_q;
+    }
 };
 
 /**********************************************************
@@ -317,8 +504,23 @@ struct DCTemplate<Quantizer, Similarity, SIMDLevel::ARM_NEON>
 
     Quantizer quant;
 
+    // Pre-adjusted query buffer for uniform quantizers
+    std::vector<float> q_adj;
+    float scale_factor = 0;
+    float bias = 0;
+
+    static constexpr bool has_decode_raw() {
+        return requires(const Quantizer& q, const uint8_t* c, int i) {
+            { q.decode_8_raw(c, i) };
+        };
+    }
+
     DCTemplate(size_t d, const std::vector<float>& trained)
-            : quant(d, trained) {}
+            : quant(d, trained) {
+        if constexpr (has_decode_raw()) {
+            q_adj.resize(d);
+        }
+    }
 
     float compute_distance(const float* x, const uint8_t* code) const {
         Similarity sim(x);
@@ -344,6 +546,26 @@ struct DCTemplate<Quantizer, Similarity, SIMDLevel::ARM_NEON>
 
     void set_query(const float* x) final {
         q = x;
+        if constexpr (has_decode_raw()) {
+            Sim::adjust_query_for_raw_decode(
+                    x,
+                    q_adj.data(),
+                    quant.d,
+                    quant.vmin,
+                    quant.vdiff,
+                    scale_factor,
+                    bias);
+        }
+    }
+
+    float query_to_code_predecoded(const uint8_t* code) const {
+        Similarity sim(q_adj.data());
+        sim.begin_8();
+        for (size_t i = 0; i < quant.d; i += 8) {
+            simd8float32 xi = quant.decode_8_raw(code, i);
+            sim.add_8_components(xi);
+        }
+        return bias + scale_factor * sim.result_8();
     }
 
     float symmetric_dis(idx_t i, idx_t j) override {
@@ -352,7 +574,47 @@ struct DCTemplate<Quantizer, Similarity, SIMDLevel::ARM_NEON>
     }
 
     float query_to_code(const uint8_t* code) const final {
-        return compute_distance(q, code);
+        if constexpr (has_decode_raw()) {
+            return query_to_code_predecoded(code);
+        } else {
+            return compute_distance(q, code);
+        }
+    }
+
+    void query_to_codes_batch_4(
+            const uint8_t* code_0,
+            const uint8_t* code_1,
+            const uint8_t* code_2,
+            const uint8_t* code_3,
+            float& dis0,
+            float& dis1,
+            float& dis2,
+            float& dis3) const final {
+        Similarity sim0(q);
+        Similarity sim1(q);
+        Similarity sim2(q);
+        Similarity sim3(q);
+
+        sim0.begin_8();
+        sim1.begin_8();
+        sim2.begin_8();
+        sim3.begin_8();
+
+        for (size_t i = 0; i < quant.d; i += 8) {
+            simd8float32 xi0 = quant.reconstruct_8_components(code_0, i);
+            simd8float32 xi1 = quant.reconstruct_8_components(code_1, i);
+            simd8float32 xi2 = quant.reconstruct_8_components(code_2, i);
+            simd8float32 xi3 = quant.reconstruct_8_components(code_3, i);
+            sim0.add_8_components(xi0);
+            sim1.add_8_components(xi1);
+            sim2.add_8_components(xi2);
+            sim3.add_8_components(xi3);
+        }
+
+        dis0 = sim0.result_8();
+        dis1 = sim1.result_8();
+        dis2 = sim2.result_8();
+        dis3 = sim3.result_8();
     }
 };
 
@@ -401,6 +663,32 @@ struct DistanceComputerByte<Similarity, SIMDLevel::ARM_NEON>
     }
 };
 
+/**********************************************************
+ * TurboQuant masked_sum NEON specialization (scalar fallback)
+ **********************************************************/
+
+template <SIMDLevel SL0>
+float turboq_masked_sum(const float* arr, const uint8_t* bits, size_t d);
+
+template <>
+float turboq_masked_sum<SIMDLevel::ARM_NEON>(
+        const float* arr,
+        const uint8_t* bits,
+        size_t d) {
+    float result = 0;
+    for (size_t byte_idx = 0; byte_idx < (d + 7) / 8; byte_idx++) {
+        uint8_t b = bits[byte_idx];
+        size_t base = byte_idx * 8;
+        size_t end = std::min(base + 8, d);
+        for (size_t j = base; j < end; j++) {
+            if (b & (1 << (j - base))) {
+                result += arr[j];
+            }
+        }
+    }
+    return result;
+}
+
 } // namespace scalar_quantizer
 } // namespace faiss
 

data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp

@@ -0,0 +1,337 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifdef COMPILE_SIMD_RISCV_RVV
+
+#include <faiss/impl/scalar_quantizer/codecs.h>
+#include <faiss/impl/scalar_quantizer/distance_computers.h>
+#include <faiss/impl/scalar_quantizer/quantizers.h>
+#include <faiss/impl/scalar_quantizer/scanners.h>
+#include <faiss/impl/scalar_quantizer/similarities.h>
+
+#include <riscv_vector.h>
+#include <cmath>
+
+namespace faiss {
+
+namespace scalar_quantizer {
+
+/*************************************************************************
+ * Marker specializations.
+ *
+ * Unlike x86/NEON sq-*.cpp files that expose a fixed 8-wide / 16-wide codec
+ * interface (reconstruct_8_components / reconstruct_16_components), RVV is
+ * variable-width: the native vector length is implementation-defined and
+ * queried at runtime via __riscv_vsetvl. Forcing RVV into a fixed-width
+ * codec would leave performance on the table on wider hardware.
+ *
+ * So the strategy here is: Codec / Quantizer / Similarity classes for
+ * RISCV_RVV act as opaque TAG TYPES — they only need to be complete types
+ * so that baseline's sq-dispatch.h can form template arguments like
+ * `DCTemplate<QuantizerTemplate<Codec4bit<RISCV_RVV>, UNIFORM, RISCV_RVV>,
+ *             SimilarityL2<RISCV_RVV>, RISCV_RVV>`.
+ *
+ * The real SIMD work lives in full DCTemplate specializations below.
+ * Unspecialized combinations fall through to scalar via the fallback
+ * `DCTemplate<Q, Sim, RISCV_RVV> : DCTemplate<Q, Sim, NONE>`.
+ ************************************************************************/
+
+template <>
+struct Codec8bit<SIMDLevel::RISCV_RVV> : Codec8bit<SIMDLevel::NONE> {};
+
+template <>
+struct Codec4bit<SIMDLevel::RISCV_RVV> : Codec4bit<SIMDLevel::NONE> {};
+
+template <>
+struct Codec6bit<SIMDLevel::RISCV_RVV> : Codec6bit<SIMDLevel::NONE> {};
+
+template <class Codec>
+struct QuantizerTemplate<
+        Codec,
+        QuantizerTemplateScaling::UNIFORM,
+        SIMDLevel::RISCV_RVV>
+        : QuantizerTemplate<
+                  Codec,
+                  QuantizerTemplateScaling::UNIFORM,
+                  SIMDLevel::NONE> {
+    QuantizerTemplate(size_t d, const std::vector<float>& trained)
+            : QuantizerTemplate<
+                      Codec,
+                      QuantizerTemplateScaling::UNIFORM,
+                      SIMDLevel::NONE>(d, trained) {}
+};
+
+template <class Codec>
+struct QuantizerTemplate<
+        Codec,
+        QuantizerTemplateScaling::NON_UNIFORM,
+        SIMDLevel::RISCV_RVV>
+        : QuantizerTemplate<
+                  Codec,
+                  QuantizerTemplateScaling::NON_UNIFORM,
+                  SIMDLevel::NONE> {
+    QuantizerTemplate(size_t d, const std::vector<float>& trained)
+            : QuantizerTemplate<
+                      Codec,
+                      QuantizerTemplateScaling::NON_UNIFORM,
+                      SIMDLevel::NONE>(d, trained) {}
+};
+
+template <>
+struct QuantizerFP16<SIMDLevel::RISCV_RVV> : QuantizerFP16<SIMDLevel::NONE> {
+    QuantizerFP16(size_t d, const std::vector<float>& trained)
+            : QuantizerFP16<SIMDLevel::NONE>(d, trained) {}
+};
+
+template <>
+struct QuantizerBF16<SIMDLevel::RISCV_RVV> : QuantizerBF16<SIMDLevel::NONE> {
+    QuantizerBF16(size_t d, const std::vector<float>& trained)
+            : QuantizerBF16<SIMDLevel::NONE>(d, trained) {}
+};
+
+template <>
+struct Quantizer8bitDirect<SIMDLevel::RISCV_RVV>
+        : Quantizer8bitDirect<SIMDLevel::NONE> {
+    Quantizer8bitDirect(size_t d, const std::vector<float>& trained)
+            : Quantizer8bitDirect<SIMDLevel::NONE>(d, trained) {}
+};
+
+template <>
+struct Quantizer8bitDirectSigned<SIMDLevel::RISCV_RVV>
+        : Quantizer8bitDirectSigned<SIMDLevel::NONE> {
+    Quantizer8bitDirectSigned(size_t d, const std::vector<float>& trained)
+            : Quantizer8bitDirectSigned<SIMDLevel::NONE>(d, trained) {}
+};
+
+template <>
+struct SimilarityL2<SIMDLevel::RISCV_RVV> : SimilarityL2<SIMDLevel::NONE> {
+    using SimilarityL2<SIMDLevel::NONE>::SimilarityL2;
+};
+
+template <>
+struct SimilarityIP<SIMDLevel::RISCV_RVV> : SimilarityIP<SIMDLevel::NONE> {
+    using SimilarityIP<SIMDLevel::NONE>::SimilarityIP;
+};
+
+/*************************************************************************
+ * Fallback DCTemplate / DistanceComputerByte for RISCV_RVV.
+ *
+ * Inheriting from the NONE specialization means every (Quantizer, Similarity)
+ * combination that does NOT have a hand-tuned RVV full specialization below
+ * falls through to scalar code. Callers and the dispatcher don't know or care.
+ ************************************************************************/
+
+template <class Quantizer, class Similarity>
+struct DCTemplate<Quantizer, Similarity, SIMDLevel::RISCV_RVV>
+        : DCTemplate<Quantizer, Similarity, SIMDLevel::NONE> {
+    using Base = DCTemplate<Quantizer, Similarity, SIMDLevel::NONE>;
+    using Base::Base;
+};
+
+template <class Similarity>
+struct DistanceComputerByte<Similarity, SIMDLevel::RISCV_RVV>
+        : DistanceComputerByte<Similarity, SIMDLevel::NONE> {
+    using Base = DistanceComputerByte<Similarity, SIMDLevel::NONE>;
+    using Base::Base;
+};
+
+/*************************************************************************
+ * Fast path — QT_4bit_uniform + L2
+ *
+ * 4-bit UNIFORM scaling: every component reconstructs as an affine function
+ * of the 4-bit code,
+ *     recon(c) = vmin + vdiff * (c + 0.5) / 15 = final_scale * c + bias
+ * where final_scale = vdiff / 15. L2 distance between two reconstructions
+ * therefore reduces to final_scale^2 * (q_c - c_c)^2 over integer codes,
+ * so we can stay in the int domain and pay one float multiply at the end.
+ *
+ * The RVV path pre-nibbles the query into q_lo / q_hi (even / odd lanes)
+ * once at set_query time and then processes native-VL-sized chunks of code
+ * without ever decoding to float.
+ ************************************************************************/
+
+template <>
+struct DCTemplate<
+        QuantizerTemplate<
+                Codec4bit<SIMDLevel::RISCV_RVV>,
+                QuantizerTemplateScaling::UNIFORM,
+                SIMDLevel::RISCV_RVV>,
+        SimilarityL2<SIMDLevel::RISCV_RVV>,
+        SIMDLevel::RISCV_RVV> : SQDistanceComputer {
+    using Sim = SimilarityL2<SIMDLevel::RISCV_RVV>;
+
+    size_t d;
+    float vmin;
+    float vdiff;
+    float final_scale_sq;
+    std::vector<uint8_t> q_lo;
+    std::vector<uint8_t> q_hi;
+
+    DCTemplate(size_t d_in, const std::vector<float>& trained)
+            : d(d_in),
+              vmin(trained[0]),
+              vdiff(trained[1]),
+              q_lo((d_in + 1) / 2, 0),
+              q_hi((d_in + 1) / 2, 0) {
+        const float final_scale = vdiff / 15.0f;
+        final_scale_sq = final_scale * final_scale;
+    }
+
+    void set_query(const float* x) final {
+        this->q = x;
+        const float inv_scale = (vdiff == 0.0f) ? 0.0f : 15.0f / vdiff;
+        for (size_t i = 0; i < d; i++) {
+            float val = (x[i] - vmin) * inv_scale;
+            int code = static_cast<int>(val);
+            if (code < 0) {
+                code = 0;
+            }
+            if (code > 15) {
+                code = 15;
+            }
+            if (i % 2 == 0) {
+                q_lo[i / 2] = static_cast<uint8_t>(code);
+            } else {
+                q_hi[i / 2] = static_cast<uint8_t>(code);
+            }
+        }
+    }
+
+    /// Squared integer-domain L2 between pre-nibbled q and packed 4-bit code.
+    /// Uses RVV's native VL; no fixed width assumptions. Returns the raw
+    /// integer sum — caller multiplies by final_scale_sq.
+    int64_t accumulate_int_l2(const uint8_t* code) const {
+        int64_t acc = 0;
+        size_t i = 0;
+        while (i < d) {
+            // Process up to vl codes per iteration. Each code byte packs two
+            // 4-bit codes, so we load (vl + 1) / 2 bytes; keep vl even to
+            // keep the nibble split aligned with the i % 2 split we used at
+            // set_query time.
+            size_t remaining = d - i;
+            size_t vl = __riscv_vsetvl_e8m1(remaining);
+            if (vl & 1) {
+                vl -= 1; // keep even; tail handled on next iter or scalar
+            }
+            if (vl == 0) {
+                break;
+            }
+            const size_t byte_vl = vl / 2;
+
+            vuint8m1_t packed = __riscv_vle8_v_u8m1(code + i / 2, byte_vl);
+            vuint8m1_t ql = __riscv_vle8_v_u8m1(q_lo.data() + i / 2, byte_vl);
+            vuint8m1_t qh = __riscv_vle8_v_u8m1(q_hi.data() + i / 2, byte_vl);
+
+            vuint8m1_t lo_nib = __riscv_vand_vx_u8m1(packed, 0x0F, byte_vl);
+            vuint8m1_t hi_nib = __riscv_vsrl_vx_u8m1(packed, 4, byte_vl);
+
+            // |ql - lo| and |qh - hi| fit in u8 (values are in [0, 15]).
+            vuint8m1_t d_lo = __riscv_vsub_vv_u8m1(
+                    __riscv_vmaxu_vv_u8m1(ql, lo_nib, byte_vl),
+                    __riscv_vminu_vv_u8m1(ql, lo_nib, byte_vl),
+                    byte_vl);
+            vuint8m1_t d_hi = __riscv_vsub_vv_u8m1(
+                    __riscv_vmaxu_vv_u8m1(qh, hi_nib, byte_vl),
+                    __riscv_vminu_vv_u8m1(qh, hi_nib, byte_vl),
+                    byte_vl);
+
+            // Square via widening multiply (each byte squared fits in u16,
+            // since max byte value is 15 -> 225).
+            vuint16m2_t sq_lo = __riscv_vwmulu_vv_u16m2(d_lo, d_lo, byte_vl);
+            vuint16m2_t sq_hi = __riscv_vwmulu_vv_u16m2(d_hi, d_hi, byte_vl);
+            vuint16m2_t sq_sum = __riscv_vadd_vv_u16m2(sq_lo, sq_hi, byte_vl);
+
+            // Reduce to a scalar u32 (safe: byte_vl * 450 fits in u32 for
+            // any realistic d).
+            vuint32m1_t zero = __riscv_vmv_v_x_u32m1(0, 1);
+            vuint32m1_t red =
+                    __riscv_vwredsumu_vs_u16m2_u32m1(sq_sum, zero, byte_vl);
+            acc += __riscv_vmv_x_s_u32m1_u32(red);
+
+            i += vl;
+        }
+        // Scalar tail: cover any leftover odd lane (at most one).
+        for (; i < d; i++) {
+            uint8_t c_code =
+                    (i % 2 == 0) ? (code[i / 2] & 0x0F) : (code[i / 2] >> 4);
+            uint8_t q_code = (i % 2 == 0) ? q_lo[i / 2] : q_hi[i / 2];
+            int diff = int(q_code) - int(c_code);
+            acc += diff * diff;
+        }
+        return acc;
+    }
+
+    float query_to_code(const uint8_t* code) const final {
+        return static_cast<float>(accumulate_int_l2(code)) * final_scale_sq;
+    }
+
+    float symmetric_dis(idx_t i, idx_t j) override {
+        // Not on the critical path for most workloads; reconstruct both
+        // codes into nibbles scalar-style and compute squared distance.
+        const uint8_t* c1 = codes + i * code_size;
+        const uint8_t* c2 = codes + j * code_size;
+        int64_t acc = 0;
+        for (size_t k = 0; k < d; k++) {
+            uint8_t a = (k % 2 == 0) ? (c1[k / 2] & 0x0F) : (c1[k / 2] >> 4);
+            uint8_t b = (k % 2 == 0) ? (c2[k / 2] & 0x0F) : (c2[k / 2] >> 4);
+            int diff = int(a) - int(b);
+            acc += diff * diff;
+        }
+        return static_cast<float>(acc) * final_scale_sq;
+    }
+
+    void query_to_codes_batch_4(
+            const uint8_t* code_0,
+            const uint8_t* code_1,
+            const uint8_t* code_2,
+            const uint8_t* code_3,
+            float& dis0,
+            float& dis1,
+            float& dis2,
+            float& dis3) const final {
+        // Simple 4x unroll of the single-code path; good enough as a first
+        // cut — gives ILP across the four independent accumulate loops.
+        dis0 = static_cast<float>(accumulate_int_l2(code_0)) * final_scale_sq;
+        dis1 = static_cast<float>(accumulate_int_l2(code_1)) * final_scale_sq;
+        dis2 = static_cast<float>(accumulate_int_l2(code_2)) * final_scale_sq;
+        dis3 = static_cast<float>(accumulate_int_l2(code_3)) * final_scale_sq;
+    }
+};
+
+/**********************************************************
+ * TurboQuant masked_sum RVV specialization (scalar fallback)
+ **********************************************************/
+
+template <SIMDLevel SL0>
+float turboq_masked_sum(const float* arr, const uint8_t* bits, size_t d);
+
+template <>
+float turboq_masked_sum<SIMDLevel::RISCV_RVV>(
+        const float* arr,
+        const uint8_t* bits,
+        size_t d) {
+    float result = 0;
+    for (size_t byte_idx = 0; byte_idx < (d + 7) / 8; byte_idx++) {
+        uint8_t b = bits[byte_idx];
+        size_t base = byte_idx * 8;
+        size_t end = std::min(base + 8, d);
+        for (size_t j = base; j < end; j++) {
+            if (b & (1 << (j - base))) {
+                result += arr[j];
+            }
+        }
+    }
+    return result;
+}
+
+} // namespace scalar_quantizer
+} // namespace faiss
+
+#define THE_LEVEL_TO_DISPATCH SIMDLevel::RISCV_RVV
+#include <faiss/impl/scalar_quantizer/sq-dispatch.h>
+
+#endif // COMPILE_SIMD_RISCV_RVV