faiss 0.6.1 → 0.6.3

data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp

@@ -0,0 +1,435 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @file rabitq_avx512_spr.cpp
+ *
+ * RaBitQ SIMD kernels specialized for SIMDLevel::AVX512_SPR.
+ *
+ * Sapphire Rapids (SPR) and later Intel microarchitectures expose
+ * AVX-512 VPOPCNTDQ (vpopcntq), which performs a per-lane 64-bit
+ * popcount in a single instruction. This is used here to replace the
+ * multi-step shuffle/pshufb-based popcount used by the generic AVX-512
+ * specialization in rabitq_avx512.cpp. The popcount-heavy kernels
+ * (bitwise_and_dot_product, bitwise_xor_dot_product, popcount) become
+ * substantially shorter and faster on SPR+ as a result.
+ *
+ * Build / dispatch behavior:
+ *   - faiss_avx512 (AVX-512 only, no SPR features): NOT compiled.
+ *     The existing AVX512 specialization in rabitq_avx512.cpp is used.
+ *   - faiss_avx512_spr (statically built for SPR+): compiled. The
+ *     SINGLE_SIMD_LEVEL is AVX512_SPR, so this specialization is
+ *     selected by static dispatch.
+ *   - faiss with FAISS_OPT_LEVEL=dd (dynamic dispatch): compiled with
+ *     -mavx512vpopcntdq as a per-file flag. Selected at runtime when
+ *     SIMDConfig::level == SIMDLevel::AVX512_SPR.
+ *
+ * The floating-point multi-bit inner-product kernel does not benefit
+ * from VPOPCNTDQ, so this TU forwards compute_inner_product<SPR> to
+ * the AVX512 implementation to avoid duplicating that code path.
+ */
+
+#ifdef COMPILE_SIMD_AVX512_SPR
+
+#include <faiss/utils/popcount.h>
+#include <faiss/utils/rabitq_simd.h>
+#include <immintrin.h>
+#include <cstdint>
+
+#if defined(_MSC_VER)
+#include <intrin.h>
+#endif
+
+namespace faiss::rabitq {
+
+// Forward declarations for the AVX512 specializations defined in
+// rabitq_avx512.cpp. They live in the same TU group on SPR builds, so
+// we can reuse them as a tail handler / fallback. Declaring rather
+// than redefining avoids ODR risk and keeps a single source of truth
+// for the floating-point kernel.
+template <>
+uint64_t bitwise_and_dot_product<SIMDLevel::AVX512>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb);
+template <>
+uint64_t bitwise_xor_dot_product<SIMDLevel::AVX512>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb);
+template <>
+uint64_t popcount<SIMDLevel::AVX512>(const uint8_t* data, size_t size);
+
+namespace {
+
+// 512-bit popcount using AVX-512 VPOPCNTDQ (vpopcntq).
+// Single-instruction per-lane popcount on 8x uint64 lanes.
+inline __m512i popcount_512_vpopcntdq(__m512i v) {
+    return _mm512_popcnt_epi64(v);
+}
+
+// 256-bit popcount using AVX-512VL VPOPCNTDQ.
+// AVX512VL is part of the SPR feature set, so vpopcntq is available
+// on 256-bit registers via _mm256_popcnt_epi64.
+inline __m256i popcount_256_vpopcntdq(__m256i v) {
+    return _mm256_popcnt_epi64(v);
+}
+
+// 128-bit popcount using AVX-512VL VPOPCNTDQ.
+inline __m128i popcount_128_vpopcntdq(__m128i v) {
+    return _mm_popcnt_epi64(v);
+}
+
+inline uint64_t reduce_add_256(__m256i v) {
+    alignas(32) uint64_t lanes[4];
+    _mm256_store_si256(reinterpret_cast<__m256i*>(lanes), v);
+    return lanes[0] + lanes[1] + lanes[2] + lanes[3];
+}
+
+inline uint64_t reduce_add_128(__m128i v) {
+    alignas(16) uint64_t lanes[2];
+    _mm_store_si128(reinterpret_cast<__m128i*>(lanes), v);
+    return lanes[0] + lanes[1];
+}
+
+} // namespace
+
+template <>
+uint64_t bitwise_and_dot_product<SIMDLevel::AVX512_SPR>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    uint64_t sum = 0;
+    size_t offset = 0;
+
+    // 512-bit main loop: vpopcntq replaces the shuffle-based popcount,
+    // halving the instruction count per iteration relative to AVX512.
+    if (size_t step = 512 / 8; offset + step <= size) {
+        __m512i sum_512 = _mm512_setzero_si512();
+        for (; offset + step <= size; offset += step) {
+            __m512i v_x = _mm512_loadu_si512(
+                    reinterpret_cast<const __m512i*>(data + offset));
+            for (size_t j = 0; j < qb; j++) {
+                __m512i v_q = _mm512_loadu_si512(
+                        reinterpret_cast<const __m512i*>(
+                                query + j * size + offset));
+                __m512i v_and = _mm512_and_si512(v_q, v_x);
+                __m512i v_popcnt = popcount_512_vpopcntdq(v_and);
+                __m512i v_shifted = _mm512_slli_epi64(v_popcnt, j);
+                sum_512 = _mm512_add_epi64(sum_512, v_shifted);
+            }
+        }
+        sum += _mm512_reduce_add_epi64(sum_512);
+    }
+
+    // 256-bit tail.
+    if (size_t step = 256 / 8; offset + step <= size) {
+        __m256i sum_256 = _mm256_setzero_si256();
+        for (; offset + step <= size; offset += step) {
+            __m256i v_x = _mm256_loadu_si256(
+                    reinterpret_cast<const __m256i*>(data + offset));
+            for (size_t j = 0; j < qb; j++) {
+                __m256i v_q = _mm256_loadu_si256(
+                        reinterpret_cast<const __m256i*>(
+                                query + j * size + offset));
+                __m256i v_and = _mm256_and_si256(v_q, v_x);
+                __m256i v_popcnt = popcount_256_vpopcntdq(v_and);
+                __m256i v_shifted = _mm256_slli_epi64(v_popcnt, j);
+                sum_256 = _mm256_add_epi64(sum_256, v_shifted);
+            }
+        }
+        sum += reduce_add_256(sum_256);
+    }
+
+    // 128-bit tail.
+    __m128i sum_128 = _mm_setzero_si128();
+    for (size_t step = 128 / 8; offset + step <= size; offset += step) {
+        __m128i v_x = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(data + offset));
+        for (size_t j = 0; j < qb; j++) {
+            __m128i v_q = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(
+                            query + j * size + offset));
+            __m128i v_and = _mm_and_si128(v_q, v_x);
+            __m128i v_popcnt = popcount_128_vpopcntdq(v_and);
+            __m128i v_shifted = _mm_slli_epi64(v_popcnt, j);
+            sum_128 = _mm_add_epi64(sum_128, v_shifted);
+        }
+    }
+    sum += reduce_add_128(sum_128);
+
+    // 64-bit scalar tail.
+    for (size_t step = 64 / 8; offset + step <= size; offset += step) {
+        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        for (size_t j = 0; j < qb; j++) {
+            const auto qv = *reinterpret_cast<const uint64_t*>(
+                    query + j * size + offset);
+            sum += static_cast<uint64_t>(popcount64(qv & yv)) << j;
+        }
+    }
+    // Byte tail.
+    for (; offset < size; ++offset) {
+        const auto yv = *(data + offset);
+        for (size_t j = 0; j < qb; j++) {
+            const auto qv = *(query + j * size + offset);
+            sum += static_cast<uint64_t>(popcount32(qv & yv)) << j;
+        }
+    }
+    return sum;
+}
+
+template <>
+BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
+        SIMDLevel::AVX512_SPR>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    uint64_t dot_product = 0;
+    uint64_t popcount_sum = 0;
+    size_t offset = 0;
+
+    if (size_t step = 512 / 8; offset + step <= size) {
+        __m512i dot_512 = _mm512_setzero_si512();
+        __m512i pop_512 = _mm512_setzero_si512();
+        for (; offset + step <= size; offset += step) {
+            __m512i v_x = _mm512_loadu_si512(
+                    reinterpret_cast<const __m512i*>(data + offset));
+            pop_512 = _mm512_add_epi64(pop_512, popcount_512_vpopcntdq(v_x));
+            for (size_t j = 0; j < qb; j++) {
+                __m512i v_q = _mm512_loadu_si512(
+                        reinterpret_cast<const __m512i*>(
+                                query + j * size + offset));
+                __m512i v_and = _mm512_and_si512(v_q, v_x);
+                __m512i v_popcnt = popcount_512_vpopcntdq(v_and);
+                __m512i v_shifted = _mm512_slli_epi64(v_popcnt, j);
+                dot_512 = _mm512_add_epi64(dot_512, v_shifted);
+            }
+        }
+        dot_product += _mm512_reduce_add_epi64(dot_512);
+        popcount_sum += _mm512_reduce_add_epi64(pop_512);
+    }
+
+    if (size_t step = 256 / 8; offset + step <= size) {
+        __m256i dot_256 = _mm256_setzero_si256();
+        __m256i pop_256 = _mm256_setzero_si256();
+        for (; offset + step <= size; offset += step) {
+            __m256i v_x = _mm256_loadu_si256(
+                    reinterpret_cast<const __m256i*>(data + offset));
+            pop_256 = _mm256_add_epi64(pop_256, popcount_256_vpopcntdq(v_x));
+            for (size_t j = 0; j < qb; j++) {
+                __m256i v_q = _mm256_loadu_si256(
+                        reinterpret_cast<const __m256i*>(
+                                query + j * size + offset));
+                __m256i v_and = _mm256_and_si256(v_q, v_x);
+                __m256i v_popcnt = popcount_256_vpopcntdq(v_and);
+                __m256i v_shifted = _mm256_slli_epi64(v_popcnt, j);
+                dot_256 = _mm256_add_epi64(dot_256, v_shifted);
+            }
+        }
+        dot_product += reduce_add_256(dot_256);
+        popcount_sum += reduce_add_256(pop_256);
+    }
+
+    __m128i dot_128 = _mm_setzero_si128();
+    __m128i pop_128 = _mm_setzero_si128();
+    for (size_t step = 128 / 8; offset + step <= size; offset += step) {
+        __m128i v_x = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(data + offset));
+        pop_128 = _mm_add_epi64(pop_128, popcount_128_vpopcntdq(v_x));
+        for (size_t j = 0; j < qb; j++) {
+            __m128i v_q = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(
+                            query + j * size + offset));
+            __m128i v_and = _mm_and_si128(v_q, v_x);
+            __m128i v_popcnt = popcount_128_vpopcntdq(v_and);
+            __m128i v_shifted = _mm_slli_epi64(v_popcnt, j);
+            dot_128 = _mm_add_epi64(dot_128, v_shifted);
+        }
+    }
+    dot_product += reduce_add_128(dot_128);
+    popcount_sum += reduce_add_128(pop_128);
+
+    for (size_t step = 64 / 8; offset + step <= size; offset += step) {
+        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        popcount_sum += popcount64(yv);
+        for (size_t j = 0; j < qb; j++) {
+            const auto qv = *reinterpret_cast<const uint64_t*>(
+                    query + j * size + offset);
+            dot_product += static_cast<uint64_t>(popcount64(qv & yv)) << j;
+        }
+    }
+    for (; offset < size; ++offset) {
+        const auto yv = *(data + offset);
+        popcount_sum += popcount32(yv);
+        for (size_t j = 0; j < qb; j++) {
+            const auto qv = *(query + j * size + offset);
+            dot_product += static_cast<uint64_t>(popcount32(qv & yv)) << j;
+        }
+    }
+    return {dot_product, popcount_sum};
+}
+
+template <>
+uint64_t bitwise_xor_dot_product<SIMDLevel::AVX512_SPR>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    uint64_t sum = 0;
+    size_t offset = 0;
+
+    if (size_t step = 512 / 8; offset + step <= size) {
+        __m512i sum_512 = _mm512_setzero_si512();
+        for (; offset + step <= size; offset += step) {
+            __m512i v_x = _mm512_loadu_si512(
+                    reinterpret_cast<const __m512i*>(data + offset));
+            for (size_t j = 0; j < qb; j++) {
+                __m512i v_q = _mm512_loadu_si512(
+                        reinterpret_cast<const __m512i*>(
+                                query + j * size + offset));
+                __m512i v_xor = _mm512_xor_si512(v_q, v_x);
+                __m512i v_popcnt = popcount_512_vpopcntdq(v_xor);
+                __m512i v_shifted = _mm512_slli_epi64(v_popcnt, j);
+                sum_512 = _mm512_add_epi64(sum_512, v_shifted);
+            }
+        }
+        sum += _mm512_reduce_add_epi64(sum_512);
+    }
+
+    if (size_t step = 256 / 8; offset + step <= size) {
+        __m256i sum_256 = _mm256_setzero_si256();
+        for (; offset + step <= size; offset += step) {
+            __m256i v_x = _mm256_loadu_si256(
+                    reinterpret_cast<const __m256i*>(data + offset));
+            for (size_t j = 0; j < qb; j++) {
+                __m256i v_q = _mm256_loadu_si256(
+                        reinterpret_cast<const __m256i*>(
+                                query + j * size + offset));
+                __m256i v_xor = _mm256_xor_si256(v_q, v_x);
+                __m256i v_popcnt = popcount_256_vpopcntdq(v_xor);
+                __m256i v_shifted = _mm256_slli_epi64(v_popcnt, j);
+                sum_256 = _mm256_add_epi64(sum_256, v_shifted);
+            }
+        }
+        sum += reduce_add_256(sum_256);
+    }
+
+    __m128i sum_128 = _mm_setzero_si128();
+    for (size_t step = 128 / 8; offset + step <= size; offset += step) {
+        __m128i v_x = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(data + offset));
+        for (size_t j = 0; j < qb; j++) {
+            __m128i v_q = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(
+                            query + j * size + offset));
+            __m128i v_xor = _mm_xor_si128(v_q, v_x);
+            __m128i v_popcnt = popcount_128_vpopcntdq(v_xor);
+            __m128i v_shifted = _mm_slli_epi64(v_popcnt, j);
+            sum_128 = _mm_add_epi64(sum_128, v_shifted);
+        }
+    }
+    sum += reduce_add_128(sum_128);
+
+    for (size_t step = 64 / 8; offset + step <= size; offset += step) {
+        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        for (size_t j = 0; j < qb; j++) {
+            const auto qv = *reinterpret_cast<const uint64_t*>(
+                    query + j * size + offset);
+            sum += static_cast<uint64_t>(popcount64(qv ^ yv)) << j;
+        }
+    }
+    for (; offset < size; ++offset) {
+        const auto yv = *(data + offset);
+        for (size_t j = 0; j < qb; j++) {
+            const auto qv = *(query + j * size + offset);
+            sum += static_cast<uint64_t>(popcount32(qv ^ yv)) << j;
+        }
+    }
+    return sum;
+}
+
+template <>
+uint64_t popcount<SIMDLevel::AVX512_SPR>(const uint8_t* data, size_t size) {
+    uint64_t sum = 0;
+    size_t offset = 0;
+
+    if (offset + 512 / 8 <= size) {
+        __m512i sum_512 = _mm512_setzero_si512();
+        for (size_t end; (end = offset + 512 / 8) <= size; offset = end) {
+            __m512i v_x = _mm512_loadu_si512(
+                    reinterpret_cast<const __m512i*>(data + offset));
+            __m512i v_popcnt = popcount_512_vpopcntdq(v_x);
+            sum_512 = _mm512_add_epi64(sum_512, v_popcnt);
+        }
+        sum += _mm512_reduce_add_epi64(sum_512);
+    }
+
+    if (offset + 256 / 8 <= size) {
+        __m256i sum_256 = _mm256_setzero_si256();
+        for (size_t end; (end = offset + 256 / 8) <= size; offset = end) {
+            __m256i v_x = _mm256_loadu_si256(
+                    reinterpret_cast<const __m256i*>(data + offset));
+            __m256i v_popcnt = popcount_256_vpopcntdq(v_x);
+            sum_256 = _mm256_add_epi64(sum_256, v_popcnt);
+        }
+        sum += reduce_add_256(sum_256);
+    }
+
+    __m128i sum_128 = _mm_setzero_si128();
+    for (size_t step = 128 / 8; offset + step <= size; offset += step) {
+        __m128i v_x = _mm_loadu_si128(
+                reinterpret_cast<const __m128i*>(data + offset));
+        sum_128 = _mm_add_epi64(sum_128, popcount_128_vpopcntdq(v_x));
+    }
+    sum += reduce_add_128(sum_128);
+
+    for (size_t step = 64 / 8; offset + step <= size; offset += step) {
+        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        sum += popcount64(yv);
+    }
+    for (; offset < size; ++offset) {
+        const auto yv = *(data + offset);
+        sum += popcount32(yv);
+    }
+    return sum;
+}
+
+} // namespace faiss::rabitq
+
+namespace faiss::rabitq::multibit {
+
+// Forward-declare the AVX512 floating-point inner-product kernel.
+// VPOPCNTDQ does not help this kernel (it operates on FP32), so we
+// reuse the AVX512 implementation rather than duplicate it.
+template <>
+float compute_inner_product<SIMDLevel::AVX512>(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        size_t ex_bits,
+        float cb);
+
+template <>
+float compute_inner_product<SIMDLevel::AVX512_SPR>(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        size_t ex_bits,
+        float cb) {
+    return compute_inner_product<SIMDLevel::AVX512>(
+            sign_bits, ex_code, rotated_q, d, ex_bits, cb);
+}
+
+} // namespace faiss::rabitq::multibit
+
+#endif // COMPILE_SIMD_AVX512_SPR

data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp

@@ -20,6 +20,17 @@ uint64_t bitwise_and_dot_product<SIMDLevel::ARM_NEON>(
     return bitwise_and_dot_product<SIMDLevel::NONE>(query, data, size, qb);
 }
 
+template <>
+BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
+        SIMDLevel::ARM_NEON>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    return bitwise_and_dot_product_with_popcount<SIMDLevel::NONE>(
+            query, data, size, qb);
+}
+
 template <>
 uint64_t bitwise_xor_dot_product<SIMDLevel::ARM_NEON>(
         const uint8_t* query,

data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp

@@ -5,19 +5,107 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+// -*- c++ -*-
+
 #include <faiss/utils/rabitq_simd.h>
 
 #ifdef COMPILE_SIMD_RISCV_RVV
 
+#include <riscv_vector.h>
+
 namespace faiss::rabitq {
 
+// SWAR per-byte popcount over a u8m4 group: each output byte holds the
+// population count (0..8) of the corresponding input byte.
+static inline vuint8m4_t popcount_u8m4(vuint8m4_t v, size_t vl) {
+    vuint8m4_t t = __riscv_vsrl_vx_u8m4(v, 1, vl);
+    t = __riscv_vand_vx_u8m4(t, 0x55, vl);
+    v = __riscv_vsub_vv_u8m4(v, t, vl);
+    t = __riscv_vsrl_vx_u8m4(v, 2, vl);
+    t = __riscv_vand_vx_u8m4(t, 0x33, vl);
+    v = __riscv_vand_vx_u8m4(v, 0x33, vl);
+    v = __riscv_vadd_vv_u8m4(v, t, vl);
+    t = __riscv_vsrl_vx_u8m4(v, 4, vl);
+    v = __riscv_vadd_vv_u8m4(v, t, vl);
+    return __riscv_vand_vx_u8m4(v, 0x0F, vl);
+}
+
+// Shared body for bitwise_{and,xor}_dot_product. @p combine applies the
+// per-element bit op (AND or XOR) between the data and query bit-planes; the
+// popcount of the result for query bit-plane j is weighted by 2^j.
+template <typename Op>
+static inline uint64_t bitwise_dot_product_rvv(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb,
+        Op combine) {
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t vx = __riscv_vle8_v_u8m4(data + i, vl);
+        for (size_t j = 0; j < qb; j++) {
+            vuint8m4_t vq = __riscv_vle8_v_u8m4(query + j * size + i, vl);
+            vuint8m4_t vp = popcount_u8m4(combine(vx, vq, vl), vl);
+            vuint16m8_t vw = __riscv_vzext_vf2_u16m8(vp, vl);
+            vw = __riscv_vsll_vx_u16m8(vw, j, vl);
+            acc = __riscv_vadd_vv_u16m8_tu(acc, acc, vw, vl);
+        }
+        i += vl;
+    }
+    vuint32m1_t red = __riscv_vmv_v_x_u32m1(0, 1);
+    red = __riscv_vwredsumu_vs_u16m8_u32m1(acc, red, vlmax);
+    return __riscv_vmv_x_s_u32m1_u32(red);
+}
+
 template <>
 uint64_t bitwise_and_dot_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* query,
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_and_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    return bitwise_dot_product_rvv(
+            query, data, size, qb, [](vuint8m4_t a, vuint8m4_t b, size_t vl) {
+                return __riscv_vand_vv_u8m4(a, b, vl);
+            });
+}
+
+template <>
+BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
+        SIMDLevel::RISCV_RVV>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    // Fused single pass over @p data: reuse each loaded data group for both the
+    // doc-side popcount and the qb AND-dot bit-planes, mirroring the separate
+    // popcount<RVV> and bitwise_and_dot_product<RVV> kernels above.
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t dot_acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    vuint16m8_t pop_acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t vx = __riscv_vle8_v_u8m4(data + i, vl);
+        vuint16m8_t vxp = __riscv_vzext_vf2_u16m8(popcount_u8m4(vx, vl), vl);
+        pop_acc = __riscv_vadd_vv_u16m8_tu(pop_acc, pop_acc, vxp, vl);
+        for (size_t j = 0; j < qb; j++) {
+            vuint8m4_t vq = __riscv_vle8_v_u8m4(query + j * size + i, vl);
+            vuint8m4_t vp = popcount_u8m4(__riscv_vand_vv_u8m4(vx, vq, vl), vl);
+            vuint16m8_t vw = __riscv_vzext_vf2_u16m8(vp, vl);
+            vw = __riscv_vsll_vx_u16m8(vw, j, vl);
+            dot_acc = __riscv_vadd_vv_u16m8_tu(dot_acc, dot_acc, vw, vl);
+        }
+        i += vl;
+    }
+    vuint32m1_t dot_red = __riscv_vmv_v_x_u32m1(0, 1);
+    dot_red = __riscv_vwredsumu_vs_u16m8_u32m1(dot_acc, dot_red, vlmax);
+    vuint32m1_t pop_red = __riscv_vmv_v_x_u32m1(0, 1);
+    pop_red = __riscv_vwredsumu_vs_u16m8_u32m1(pop_acc, pop_red, vlmax);
+    return {__riscv_vmv_x_s_u32m1_u32(dot_red),
+            __riscv_vmv_x_s_u32m1_u32(pop_red)};
 }
 
 template <>
@@ -26,28 +114,77 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_xor_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    return bitwise_dot_product_rvv(
+            query, data, size, qb, [](vuint8m4_t a, vuint8m4_t b, size_t vl) {
+                return __riscv_vxor_vv_u8m4(a, b, vl);
+            });
 }
 
 template <>
 uint64_t popcount<SIMDLevel::RISCV_RVV>(const uint8_t* data, size_t size) {
-    return popcount<SIMDLevel::NONE>(data, size);
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t v = popcount_u8m4(__riscv_vle8_v_u8m4(data + i, vl), vl);
+        vuint16m8_t vw = __riscv_vzext_vf2_u16m8(v, vl);
+        acc = __riscv_vadd_vv_u16m8_tu(acc, acc, vw, vl);
+        i += vl;
+    }
+    vuint32m1_t red = __riscv_vmv_v_x_u32m1(0, 1);
+    red = __riscv_vwredsumu_vs_u16m8_u32m1(acc, red, vlmax);
+    return __riscv_vmv_x_s_u32m1_u32(red);
 }
 
 } // namespace faiss::rabitq
 
 namespace faiss::rabitq::multibit {
 
+static float ip_1exbit_rvv(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        float cb) {
+    size_t vlmax = __riscv_vsetvlmax_e32m8();
+    vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0.0f, vlmax);
+    size_t i = 0;
+    while (i < d) {
+        size_t vl = __riscv_vsetvl_e32m8(d - i);
+        vbool4_t sb = __riscv_vlm_v_b4(sign_bits + i / 8, vl);
+        vbool4_t eb = __riscv_vlm_v_b4(ex_code + i / 8, vl);
+        vfloat32m8_t recon = __riscv_vfmv_v_f_f32m8(cb, vl);
+        recon = __riscv_vfadd_vf_f32m8_mu(sb, recon, recon, 2.0f, vl);
+        recon = __riscv_vfadd_vf_f32m8_mu(eb, recon, recon, 1.0f, vl);
+        vfloat32m8_t rq = __riscv_vle32_v_f32m8(rotated_q + i, vl);
+        acc = __riscv_vfmacc_vv_f32m8_tu(acc, rq, recon, vl);
+        i += vl;
+    }
+    vfloat32m1_t sum = __riscv_vfmv_s_f_f32m1(0.0f, 1);
+    sum = __riscv_vfredusum_vs_f32m8_f32m1(acc, sum, vlmax);
+    return __riscv_vfmv_f_s_f32m1_f32(sum);
+}
+
+// no-tree-vectorize keeps the ex_bits >= 2 tail on the existing scalar
+// ip_scalar without GCC turning its 64-bit memcpy window into a vluxei64 gather
+// at unaligned addresses, which SIGBUSes on strict-align RVV cores (SpacemiT
+// X60).
 template <>
-float compute_inner_product<SIMDLevel::RISCV_RVV>(
+__attribute__((optimize("no-tree-vectorize", "no-tree-slp-vectorize"))) float
+compute_inner_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* __restrict sign_bits,
         const uint8_t* __restrict ex_code,
         const float* __restrict rotated_q,
         size_t d,
         size_t ex_bits,
         float cb) {
-    return compute_inner_product<SIMDLevel::NONE>(
-            sign_bits, ex_code, rotated_q, d, ex_bits, cb);
+    if (ex_bits == 1) {
+        return ip_1exbit_rvv(sign_bits, ex_code, rotated_q, d, cb);
+    }
+    // ex_bits >= 2 needs strided bit-plane extraction (PEXT on x86); RVV has no
+    // cheap equivalent without Zvbb, so reuse the scalar kernel.
+    return ip_scalar(sign_bits, ex_code, rotated_q, 0, d, ex_bits, cb);
 }
 
 } // namespace faiss::rabitq::multibit