faiss 0.6.4 → 0.6.5

data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#ifdef COMPILE_SIMD_AVX512
-
 #include <faiss/utils/rabitq_simd.h>
 #include <immintrin.h>
 #include <limits>
@@ -378,9 +376,11 @@ uint64_t bitwise_and_dot_product<SIMDLevel::AVX512>(
     }
     sum += reduce_add_128(sum_128);
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *(const uint64_t*)(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         for (int j = 0; j < qb; j++) {
-            const auto qv = *(const uint64_t*)(query + j * size + offset);
+            uint64_t qv;
+            std::memcpy(&qv, query + j * size + offset, sizeof(qv));
             sum += popcount64(qv & yv) << j;
         }
     }
@@ -457,10 +457,12 @@ BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
     dot_product += reduce_add_128(dot_128);
     popcount_sum += reduce_add_128(pop_128);
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *(const uint64_t*)(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         popcount_sum += popcount64(yv);
         for (int j = 0; j < qb; j++) {
-            const auto qv = *(const uint64_t*)(query + j * size + offset);
+            uint64_t qv;
+            std::memcpy(&qv, query + j * size + offset, sizeof(qv));
             dot_product += popcount64(qv & yv) << j;
         }
     }
@@ -527,9 +529,11 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::AVX512>(
     }
     sum += reduce_add_128(sum_128);
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *(const uint64_t*)(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         for (int j = 0; j < qb; j++) {
-            const auto qv = *(const uint64_t*)(query + j * size + offset);
+            uint64_t qv;
+            std::memcpy(&qv, query + j * size + offset, sizeof(qv));
             sum += popcount64(qv ^ yv) << j;
         }
     }
@@ -572,7 +576,8 @@ uint64_t popcount<SIMDLevel::AVX512>(const uint8_t* data, size_t size) {
     }
     sum += reduce_add_128(sum_128);
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *(const uint64_t*)(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         sum += popcount64(yv);
     }
     for (; offset < size; ++offset) {
@@ -654,13 +659,9 @@ inline float ip_1exbit_avx512(
     return result;
 }
 
-// Needs BMI2 for _pext_u64. Some AVX2 CPUs lack it, and FAISS_BMI2_FLAGS can
-// be empty, so the dispatcher falls back to the scalar path without it.
-#ifdef __BMI2__
-// Bitplane kernel for ex_bits >= 2, 16 dims per iteration. A bitplane is
-// already a bitmask, so it goes into a mask register and one masked add
-// applies its weight. Reads of ex_code run a few bytes past the ex-code
-// section into the record's own trailing factors, so they stay in bounds.
+// Bit-plane kernel for ex_bits >= 2. It scores 16 dims per pass.
+// Each plane acts as a mask for one weighted add. Reads can run
+// past the codes into the row tail, which stays in bounds.
 inline float ip_bitplane_avx512(
         const uint8_t* __restrict sign_bits,
         const uint8_t* __restrict ex_code,
@@ -737,7 +738,6 @@ inline float ip_bitplane_avx512(
     result += ip_scalar(sign_bits, ex_code, rotated_q, i, d, ex_bits, cb);
     return result;
 }
-#endif // __BMI2__
 
 } // namespace
 
@@ -749,19 +749,36 @@ float compute_inner_product<SIMDLevel::AVX512>(
         size_t d,
         size_t ex_bits,
         float cb) {
+    if (ex_bits == 8) {
+        // RBQ9 has one byte per extra code: no bit-plane extraction or BMI2.
+        __m512 acc = _mm512_setzero_ps();
+        const __m512 weight = _mm512_set1_ps(256.f);
+        const __m512 offset = _mm512_set1_ps(cb);
+        size_t i = 0;
+        for (; i + 16 <= d; i += 16) {
+            uint16_t signs;
+            memcpy(&signs, sign_bits + i / 8, sizeof(signs));
+            const __m128i bytes = _mm_loadu_si128(
+                    reinterpret_cast<const __m128i*>(ex_code + i));
+            __m512 recon = _mm512_cvtepi32_ps(_mm512_cvtepu8_epi32(bytes));
+            recon = _mm512_mask_add_ps(recon, signs, recon, weight);
+            acc = _mm512_fmadd_ps(
+                    _mm512_loadu_ps(rotated_q + i),
+                    _mm512_add_ps(recon, offset),
+                    acc);
+        }
+        return _mm512_reduce_add_ps(acc) +
+                ip_scalar(sign_bits, ex_code, rotated_q, i, d, ex_bits, cb);
+    }
     if (ex_bits == 1) {
         return ip_1exbit_avx512(sign_bits, ex_code, rotated_q, d, cb);
     }
 
-#ifdef __BMI2__
     if (ex_bits <= 7) {
         return ip_bitplane_avx512(
                 sign_bits, ex_code, rotated_q, d, ex_bits, cb);
     }
-#endif
     return ip_scalar(sign_bits, ex_code, rotated_q, 0, d, ex_bits, cb);
 }
 
 } // namespace faiss::rabitq::multibit
-
-#endif // COMPILE_SIMD_AVX512

data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_vpopcnt.cpp

@@ -30,8 +30,6 @@
  * the AVX512 implementation to avoid duplicating that code path.
  */
 
-#ifdef COMPILE_SIMD_AVX512_VPOPCNT
-
 #include <faiss/utils/popcount.h>
 #include <faiss/utils/rabitq_simd.h>
 #include <immintrin.h>
@@ -63,6 +61,43 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::AVX512>(
 template <>
 uint64_t popcount<SIMDLevel::AVX512>(const uint8_t* data, size_t size);
 
+template <>
+void bitwise_q4_batch_4<SIMDLevel::AVX512_VPOPCNT>(
+        const uint8_t* query,
+        const uint8_t* const* data,
+        size_t size,
+        BitwiseAndDotProductResult* results) {
+    __m512i dots[4], pops[4];
+    for (int i = 0; i < 4; ++i) {
+        dots[i] = pops[i] = _mm512_setzero_si512();
+    }
+    for (size_t off = 0; off < size; off += 64) {
+        const size_t count = std::min(size - off, size_t(64));
+        const __mmask64 mask =
+                count == 64 ? ~__mmask64(0) : (__mmask64(1) << count) - 1;
+        __m512i x[4];
+        for (int i = 0; i < 4; ++i) {
+            x[i] = _mm512_maskz_loadu_epi8(mask, data[i] + off);
+            pops[i] = _mm512_add_epi64(pops[i], _mm512_popcnt_epi64(x[i]));
+        }
+        for (int bit = 0; bit < 4; ++bit) {
+            // Load each query plane once for four independent database codes.
+            const __m512i q =
+                    _mm512_maskz_loadu_epi8(mask, query + bit * size + off);
+            for (int i = 0; i < 4; ++i) {
+                const __m512i p =
+                        _mm512_popcnt_epi64(_mm512_and_si512(q, x[i]));
+                dots[i] = _mm512_add_epi64(dots[i], _mm512_slli_epi64(p, bit));
+            }
+        }
+    }
+    for (int i = 0; i < 4; ++i) {
+        results[i] = {
+                static_cast<uint64_t>(_mm512_reduce_add_epi64(dots[i])),
+                static_cast<uint64_t>(_mm512_reduce_add_epi64(pops[i]))};
+    }
+}
+
 namespace {
 
 // 512-bit popcount using AVX-512 VPOPCNTDQ (vpopcntq).
@@ -164,10 +199,11 @@ uint64_t bitwise_and_dot_product<SIMDLevel::AVX512_VPOPCNT>(
 
     // 64-bit scalar tail.
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         for (size_t j = 0; j < qb; j++) {
-            const auto qv = *reinterpret_cast<const uint64_t*>(
-                    query + j * size + offset);
+            uint64_t qv;
+            std::memcpy(&qv, query + j * size + offset, sizeof(qv));
             sum += static_cast<uint64_t>(popcount64(qv & yv)) << j;
         }
     }
@@ -255,11 +291,12 @@ BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
     popcount_sum += reduce_add_128(pop_128);
 
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         popcount_sum += popcount64(yv);
         for (size_t j = 0; j < qb; j++) {
-            const auto qv = *reinterpret_cast<const uint64_t*>(
-                    query + j * size + offset);
+            uint64_t qv;
+            std::memcpy(&qv, query + j * size + offset, sizeof(qv));
             dot_product += static_cast<uint64_t>(popcount64(qv & yv)) << j;
         }
     }
@@ -336,10 +373,11 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::AVX512_VPOPCNT>(
     sum += reduce_add_128(sum_128);
 
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         for (size_t j = 0; j < qb; j++) {
-            const auto qv = *reinterpret_cast<const uint64_t*>(
-                    query + j * size + offset);
+            uint64_t qv;
+            std::memcpy(&qv, query + j * size + offset, sizeof(qv));
             sum += static_cast<uint64_t>(popcount64(qv ^ yv)) << j;
         }
     }
@@ -389,7 +427,8 @@ uint64_t popcount<SIMDLevel::AVX512_VPOPCNT>(const uint8_t* data, size_t size) {
     sum += reduce_add_128(sum_128);
 
     for (size_t step = 64 / 8; offset + step <= size; offset += step) {
-        const auto yv = *reinterpret_cast<const uint64_t*>(data + offset);
+        uint64_t yv;
+        std::memcpy(&yv, data + offset, sizeof(yv));
         sum += popcount64(yv);
     }
     for (; offset < size; ++offset) {
@@ -428,5 +467,3 @@ float compute_inner_product<SIMDLevel::AVX512_VPOPCNT>(
 }
 
 } // namespace faiss::rabitq::multibit
-
-#endif // COMPILE_SIMD_AVX512_VPOPCNT

data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp

@@ -7,17 +7,78 @@
 
 #include <faiss/utils/rabitq_simd.h>
 
-#ifdef COMPILE_SIMD_ARM_NEON
+#include <arm_neon.h>
+
+#include <array>
+
+#include <faiss/impl/FaissAssert.h>
 
 namespace faiss::rabitq {
 
+namespace {
+
+// One accumulator per query bit plane. RaBitQuantizer caps qb at 8, so the
+// accumulator arrays below are fixed size rather than heap allocated.
+constexpr size_t kMaxQueryBits = 8;
+
+inline uint32x4_t accumulate_popcount(uint32x4_t acc, uint8x16_t value) {
+    const uint16x8_t pair_counts = vpaddlq_u8(vcntq_u8(value));
+    return vpadalq_u16(acc, pair_counts);
+}
+
+inline uint64_t reduce_popcount(uint32x4_t acc) {
+    return vaddlvq_u32(acc);
+}
+
+} // namespace
+
 template <>
 uint64_t bitwise_and_dot_product<SIMDLevel::ARM_NEON>(
         const uint8_t* query,
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_and_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    FAISS_THROW_IF_NOT(qb <= kMaxQueryBits);
+    std::array<uint32x4_t, kMaxQueryBits> accumulators;
+    for (size_t j = 0; j < qb; j++) {
+        accumulators[j] = vdupq_n_u32(0);
+    }
+
+    size_t offset = 0;
+    for (; offset + 16 <= size; offset += 16) {
+        const uint8x16_t value = vld1q_u8(data + offset);
+        for (size_t j = 0; j < qb; j++) {
+            const uint8x16_t query_value = vld1q_u8(query + j * size + offset);
+            accumulators[j] = accumulate_popcount(
+                    accumulators[j], vandq_u8(query_value, value));
+        }
+    }
+
+    uint64_t result = 0;
+    for (size_t j = 0; j < qb; j++) {
+        result += reduce_popcount(accumulators[j]) << j;
+    }
+    for (; offset + 8 <= size; offset += 8) {
+        uint64_t value;
+        memcpy(&value, data + offset, sizeof(value));
+        for (size_t j = 0; j < qb; j++) {
+            uint64_t query_value;
+            memcpy(&query_value,
+                   query + j * size + offset,
+                   sizeof(query_value));
+            result += static_cast<uint64_t>(popcount64(query_value & value))
+                    << j;
+        }
+    }
+    for (; offset < size; offset++) {
+        const uint8_t value = data[offset];
+        for (size_t j = 0; j < qb; j++) {
+            result += static_cast<uint64_t>(
+                              popcount32(query[j * size + offset] & value))
+                    << j;
+        }
+    }
+    return result;
 }
 
 template <>
@@ -27,8 +88,52 @@ BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_and_dot_product_with_popcount<SIMDLevel::NONE>(
-            query, data, size, qb);
+    FAISS_THROW_IF_NOT(qb <= kMaxQueryBits);
+    std::array<uint32x4_t, kMaxQueryBits> dot_accumulators;
+    for (size_t j = 0; j < qb; j++) {
+        dot_accumulators[j] = vdupq_n_u32(0);
+    }
+    uint32x4_t popcount_accumulator = vdupq_n_u32(0);
+
+    size_t offset = 0;
+    for (; offset + 16 <= size; offset += 16) {
+        const uint8x16_t value = vld1q_u8(data + offset);
+        popcount_accumulator = accumulate_popcount(popcount_accumulator, value);
+        for (size_t j = 0; j < qb; j++) {
+            const uint8x16_t query_value = vld1q_u8(query + j * size + offset);
+            dot_accumulators[j] = accumulate_popcount(
+                    dot_accumulators[j], vandq_u8(query_value, value));
+        }
+    }
+
+    uint64_t dot_product = 0;
+    for (size_t j = 0; j < qb; j++) {
+        dot_product += reduce_popcount(dot_accumulators[j]) << j;
+    }
+    uint64_t popcount_sum = reduce_popcount(popcount_accumulator);
+    for (; offset + 8 <= size; offset += 8) {
+        uint64_t value;
+        memcpy(&value, data + offset, sizeof(value));
+        popcount_sum += popcount64(value);
+        for (size_t j = 0; j < qb; j++) {
+            uint64_t query_value;
+            memcpy(&query_value,
+                   query + j * size + offset,
+                   sizeof(query_value));
+            dot_product +=
+                    static_cast<uint64_t>(popcount64(query_value & value)) << j;
+        }
+    }
+    for (; offset < size; offset++) {
+        const uint8_t value = data[offset];
+        popcount_sum += popcount32(value);
+        for (size_t j = 0; j < qb; j++) {
+            dot_product += static_cast<uint64_t>(
+                                   popcount32(query[j * size + offset] & value))
+                    << j;
+        }
+    }
+    return {dot_product, popcount_sum};
 }
 
 template <>
@@ -37,18 +142,185 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::ARM_NEON>(
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_xor_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    FAISS_THROW_IF_NOT(qb <= kMaxQueryBits);
+    std::array<uint32x4_t, kMaxQueryBits> accumulators;
+    for (size_t j = 0; j < qb; j++) {
+        accumulators[j] = vdupq_n_u32(0);
+    }
+
+    size_t offset = 0;
+    for (; offset + 16 <= size; offset += 16) {
+        const uint8x16_t value = vld1q_u8(data + offset);
+        for (size_t j = 0; j < qb; j++) {
+            const uint8x16_t query_value = vld1q_u8(query + j * size + offset);
+            accumulators[j] = accumulate_popcount(
+                    accumulators[j], veorq_u8(query_value, value));
+        }
+    }
+
+    uint64_t result = 0;
+    for (size_t j = 0; j < qb; j++) {
+        result += reduce_popcount(accumulators[j]) << j;
+    }
+    for (; offset + 8 <= size; offset += 8) {
+        uint64_t value;
+        memcpy(&value, data + offset, sizeof(value));
+        for (size_t j = 0; j < qb; j++) {
+            uint64_t query_value;
+            memcpy(&query_value,
+                   query + j * size + offset,
+                   sizeof(query_value));
+            result += static_cast<uint64_t>(popcount64(query_value ^ value))
+                    << j;
+        }
+    }
+    for (; offset < size; offset++) {
+        const uint8_t value = data[offset];
+        for (size_t j = 0; j < qb; j++) {
+            result += static_cast<uint64_t>(
+                              popcount32(query[j * size + offset] ^ value))
+                    << j;
+        }
+    }
+    return result;
 }
 
 template <>
 uint64_t popcount<SIMDLevel::ARM_NEON>(const uint8_t* data, size_t size) {
-    return popcount<SIMDLevel::NONE>(data, size);
+    uint32x4_t accumulator = vdupq_n_u32(0);
+    size_t offset = 0;
+    for (; offset + 16 <= size; offset += 16) {
+        accumulator = accumulate_popcount(accumulator, vld1q_u8(data + offset));
+    }
+
+    uint64_t result = reduce_popcount(accumulator);
+    for (; offset + 8 <= size; offset += 8) {
+        uint64_t value;
+        memcpy(&value, data + offset, sizeof(value));
+        result += popcount64(value);
+    }
+    for (; offset < size; offset++) {
+        result += popcount32(data[offset]);
+    }
+    return result;
 }
 
 } // namespace faiss::rabitq
 
 namespace faiss::rabitq::multibit {
 
+namespace {
+
+inline float horizontal_sum(float32x4_t low, float32x4_t high) {
+    return vaddvq_f32(vaddq_f32(low, high));
+}
+
+inline float ip_1exbit_neon(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        float cb) {
+    const uint32x4_t low_bit_positions = {1, 2, 4, 8};
+    const uint32x4_t high_bit_positions = {16, 32, 64, 128};
+    const uint32x4_t sign_weight = vdupq_n_u32(2);
+    const uint32x4_t ex_weight = vdupq_n_u32(1);
+    const float32x4_t bias = vdupq_n_f32(cb);
+    float32x4_t low_accumulator = vdupq_n_f32(0.0f);
+    float32x4_t high_accumulator = vdupq_n_f32(0.0f);
+
+    size_t i = 0;
+    for (; i + 8 <= d; i += 8) {
+        const uint32x4_t sign = vdupq_n_u32(sign_bits[i / 8]);
+        const uint32x4_t extra = vdupq_n_u32(ex_code[i / 8]);
+
+        const uint32x4_t low_values = vaddq_u32(
+                vandq_u32(vtstq_u32(sign, low_bit_positions), sign_weight),
+                vandq_u32(vtstq_u32(extra, low_bit_positions), ex_weight));
+        const uint32x4_t high_values = vaddq_u32(
+                vandq_u32(vtstq_u32(sign, high_bit_positions), sign_weight),
+                vandq_u32(vtstq_u32(extra, high_bit_positions), ex_weight));
+
+        const float32x4_t low_reconstruction =
+                vaddq_f32(vcvtq_f32_u32(low_values), bias);
+        const float32x4_t high_reconstruction =
+                vaddq_f32(vcvtq_f32_u32(high_values), bias);
+        low_accumulator = vfmaq_f32(
+                low_accumulator, vld1q_f32(rotated_q + i), low_reconstruction);
+        high_accumulator = vfmaq_f32(
+                high_accumulator,
+                vld1q_f32(rotated_q + i + 4),
+                high_reconstruction);
+    }
+
+    float result = horizontal_sum(low_accumulator, high_accumulator);
+    result += ip_scalar(sign_bits, ex_code, rotated_q, i, d, 1, cb);
+    return result;
+}
+
+template <size_t ExBits>
+inline float ip_multibit_neon(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        float cb) {
+    static_assert(ExBits >= 2 && ExBits <= 7);
+    const uint32x4_t low_bit_positions = {1, 2, 4, 8};
+    const uint32x4_t high_bit_positions = {16, 32, 64, 128};
+    const uint32x4_t sign_weight =
+            vdupq_n_u32(static_cast<uint32_t>(1u << ExBits));
+    const uint64x2_t code_mask =
+            vdupq_n_u64(static_cast<uint64_t>((1u << ExBits) - 1));
+    constexpr int64_t bit_width = static_cast<int64_t>(ExBits);
+    const int64x2_t shifts_01 = {0, -bit_width};
+    const int64x2_t shifts_23 = {-2 * bit_width, -3 * bit_width};
+    const int64x2_t shifts_45 = {-4 * bit_width, -5 * bit_width};
+    const int64x2_t shifts_67 = {-6 * bit_width, -7 * bit_width};
+    const float32x4_t bias = vdupq_n_f32(cb);
+    float32x4_t low_accumulator = vdupq_n_f32(0.0f);
+    float32x4_t high_accumulator = vdupq_n_f32(0.0f);
+
+    size_t i = 0;
+    for (; i + 8 <= d; i += 8) {
+        uint64_t packed_codes = 0;
+        memcpy(&packed_codes, ex_code + (i / 8) * ExBits, ExBits);
+        const uint64x2_t packed = vdupq_n_u64(packed_codes);
+
+        const uint32x4_t low_codes = vcombine_u32(
+                vmovn_u64(vandq_u64(vshlq_u64(packed, shifts_01), code_mask)),
+                vmovn_u64(vandq_u64(vshlq_u64(packed, shifts_23), code_mask)));
+        const uint32x4_t high_codes = vcombine_u32(
+                vmovn_u64(vandq_u64(vshlq_u64(packed, shifts_45), code_mask)),
+                vmovn_u64(vandq_u64(vshlq_u64(packed, shifts_67), code_mask)));
+
+        const uint32x4_t sign = vdupq_n_u32(sign_bits[i / 8]);
+        const uint32x4_t low_values = vaddq_u32(
+                low_codes,
+                vandq_u32(vtstq_u32(sign, low_bit_positions), sign_weight));
+        const uint32x4_t high_values = vaddq_u32(
+                high_codes,
+                vandq_u32(vtstq_u32(sign, high_bit_positions), sign_weight));
+
+        const float32x4_t low_reconstruction =
+                vaddq_f32(vcvtq_f32_u32(low_values), bias);
+        const float32x4_t high_reconstruction =
+                vaddq_f32(vcvtq_f32_u32(high_values), bias);
+        low_accumulator = vfmaq_f32(
+                low_accumulator, vld1q_f32(rotated_q + i), low_reconstruction);
+        high_accumulator = vfmaq_f32(
+                high_accumulator,
+                vld1q_f32(rotated_q + i + 4),
+                high_reconstruction);
+    }
+
+    float result = horizontal_sum(low_accumulator, high_accumulator);
+    result += ip_scalar(sign_bits, ex_code, rotated_q, i, d, ExBits, cb);
+    return result;
+}
+
+} // namespace
+
 template <>
 float compute_inner_product<SIMDLevel::ARM_NEON>(
         const uint8_t* __restrict sign_bits,
@@ -57,10 +329,25 @@ float compute_inner_product<SIMDLevel::ARM_NEON>(
         size_t d,
         size_t ex_bits,
         float cb) {
-    return compute_inner_product<SIMDLevel::NONE>(
-            sign_bits, ex_code, rotated_q, d, ex_bits, cb);
+    if (ex_bits == 1) {
+        return ip_1exbit_neon(sign_bits, ex_code, rotated_q, d, cb);
+    }
+    switch (ex_bits) {
+        case 2:
+            return ip_multibit_neon<2>(sign_bits, ex_code, rotated_q, d, cb);
+        case 3:
+            return ip_multibit_neon<3>(sign_bits, ex_code, rotated_q, d, cb);
+        case 4:
+            return ip_multibit_neon<4>(sign_bits, ex_code, rotated_q, d, cb);
+        case 5:
+            return ip_multibit_neon<5>(sign_bits, ex_code, rotated_q, d, cb);
+        case 6:
+            return ip_multibit_neon<6>(sign_bits, ex_code, rotated_q, d, cb);
+        case 7:
+            return ip_multibit_neon<7>(sign_bits, ex_code, rotated_q, d, cb);
+        default:
+            return ip_scalar(sign_bits, ex_code, rotated_q, 0, d, ex_bits, cb);
+    }
 }
 
 } // namespace faiss::rabitq::multibit
-
-#endif // COMPILE_SIMD_ARM_NEON

data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp

@@ -9,8 +9,6 @@
 
 #include <faiss/utils/rabitq_simd.h>
 
-#ifdef COMPILE_SIMD_RISCV_RVV
-
 #include <riscv_vector.h>
 
 namespace faiss::rabitq {
@@ -188,5 +186,3 @@ compute_inner_product<SIMDLevel::RISCV_RVV>(
 }
 
 } // namespace faiss::rabitq::multibit
-
-#endif // COMPILE_SIMD_RISCV_RVV

data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#ifdef COMPILE_SIMD_AVX2
-
 #include <faiss/utils/simd_impl/super_kmeans_kernels.h>
 
 #include <immintrin.h>
@@ -53,5 +51,3 @@ float block_l2<SIMDLevel::AVX2>(const float* x, const float* y, int n) {
 
 } // namespace detail
 } // namespace faiss
-
-#endif // COMPILE_SIMD_AVX2

data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#ifdef COMPILE_SIMD_AVX512
-
 #include <faiss/utils/simd_impl/super_kmeans_kernels.h>
 
 #include <immintrin.h>
@@ -41,5 +39,3 @@ float block_l2<SIMDLevel::AVX512>(const float* x, const float* y, int n) {
 
 } // namespace detail
 } // namespace faiss
-
-#endif // COMPILE_SIMD_AVX512

data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#ifdef COMPILE_SIMD_ARM_SVE
-
 #include <faiss/utils/simd_impl/super_kmeans_kernels.h>
 
 #include <arm_sve.h>
@@ -30,5 +28,3 @@ float block_l2<SIMDLevel::ARM_SVE>(const float* x, const float* y, int n) {
 
 } // namespace detail
 } // namespace faiss
-
-#endif // COMPILE_SIMD_ARM_SVE