faiss 0.6.2 → 0.6.3

data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp

@@ -5,19 +5,107 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+// -*- c++ -*-
+
 #include <faiss/utils/rabitq_simd.h>
 
 #ifdef COMPILE_SIMD_RISCV_RVV
 
+#include <riscv_vector.h>
+
 namespace faiss::rabitq {
 
+// SWAR per-byte popcount over a u8m4 group: each output byte holds the
+// population count (0..8) of the corresponding input byte.
+static inline vuint8m4_t popcount_u8m4(vuint8m4_t v, size_t vl) {
+    vuint8m4_t t = __riscv_vsrl_vx_u8m4(v, 1, vl);
+    t = __riscv_vand_vx_u8m4(t, 0x55, vl);
+    v = __riscv_vsub_vv_u8m4(v, t, vl);
+    t = __riscv_vsrl_vx_u8m4(v, 2, vl);
+    t = __riscv_vand_vx_u8m4(t, 0x33, vl);
+    v = __riscv_vand_vx_u8m4(v, 0x33, vl);
+    v = __riscv_vadd_vv_u8m4(v, t, vl);
+    t = __riscv_vsrl_vx_u8m4(v, 4, vl);
+    v = __riscv_vadd_vv_u8m4(v, t, vl);
+    return __riscv_vand_vx_u8m4(v, 0x0F, vl);
+}
+
+// Shared body for bitwise_{and,xor}_dot_product. @p combine applies the
+// per-element bit op (AND or XOR) between the data and query bit-planes; the
+// popcount of the result for query bit-plane j is weighted by 2^j.
+template <typename Op>
+static inline uint64_t bitwise_dot_product_rvv(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb,
+        Op combine) {
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t vx = __riscv_vle8_v_u8m4(data + i, vl);
+        for (size_t j = 0; j < qb; j++) {
+            vuint8m4_t vq = __riscv_vle8_v_u8m4(query + j * size + i, vl);
+            vuint8m4_t vp = popcount_u8m4(combine(vx, vq, vl), vl);
+            vuint16m8_t vw = __riscv_vzext_vf2_u16m8(vp, vl);
+            vw = __riscv_vsll_vx_u16m8(vw, j, vl);
+            acc = __riscv_vadd_vv_u16m8_tu(acc, acc, vw, vl);
+        }
+        i += vl;
+    }
+    vuint32m1_t red = __riscv_vmv_v_x_u32m1(0, 1);
+    red = __riscv_vwredsumu_vs_u16m8_u32m1(acc, red, vlmax);
+    return __riscv_vmv_x_s_u32m1_u32(red);
+}
+
 template <>
 uint64_t bitwise_and_dot_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* query,
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_and_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    return bitwise_dot_product_rvv(
+            query, data, size, qb, [](vuint8m4_t a, vuint8m4_t b, size_t vl) {
+                return __riscv_vand_vv_u8m4(a, b, vl);
+            });
+}
+
+template <>
+BitwiseAndDotProductResult bitwise_and_dot_product_with_popcount<
+        SIMDLevel::RISCV_RVV>(
+        const uint8_t* query,
+        const uint8_t* data,
+        size_t size,
+        size_t qb) {
+    // Fused single pass over @p data: reuse each loaded data group for both the
+    // doc-side popcount and the qb AND-dot bit-planes, mirroring the separate
+    // popcount<RVV> and bitwise_and_dot_product<RVV> kernels above.
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t dot_acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    vuint16m8_t pop_acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t vx = __riscv_vle8_v_u8m4(data + i, vl);
+        vuint16m8_t vxp = __riscv_vzext_vf2_u16m8(popcount_u8m4(vx, vl), vl);
+        pop_acc = __riscv_vadd_vv_u16m8_tu(pop_acc, pop_acc, vxp, vl);
+        for (size_t j = 0; j < qb; j++) {
+            vuint8m4_t vq = __riscv_vle8_v_u8m4(query + j * size + i, vl);
+            vuint8m4_t vp = popcount_u8m4(__riscv_vand_vv_u8m4(vx, vq, vl), vl);
+            vuint16m8_t vw = __riscv_vzext_vf2_u16m8(vp, vl);
+            vw = __riscv_vsll_vx_u16m8(vw, j, vl);
+            dot_acc = __riscv_vadd_vv_u16m8_tu(dot_acc, dot_acc, vw, vl);
+        }
+        i += vl;
+    }
+    vuint32m1_t dot_red = __riscv_vmv_v_x_u32m1(0, 1);
+    dot_red = __riscv_vwredsumu_vs_u16m8_u32m1(dot_acc, dot_red, vlmax);
+    vuint32m1_t pop_red = __riscv_vmv_v_x_u32m1(0, 1);
+    pop_red = __riscv_vwredsumu_vs_u16m8_u32m1(pop_acc, pop_red, vlmax);
+    return {__riscv_vmv_x_s_u32m1_u32(dot_red),
+            __riscv_vmv_x_s_u32m1_u32(pop_red)};
 }
 
 template <>
@@ -26,28 +114,77 @@ uint64_t bitwise_xor_dot_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* data,
         size_t size,
         size_t qb) {
-    return bitwise_xor_dot_product<SIMDLevel::NONE>(query, data, size, qb);
+    return bitwise_dot_product_rvv(
+            query, data, size, qb, [](vuint8m4_t a, vuint8m4_t b, size_t vl) {
+                return __riscv_vxor_vv_u8m4(a, b, vl);
+            });
 }
 
 template <>
 uint64_t popcount<SIMDLevel::RISCV_RVV>(const uint8_t* data, size_t size) {
-    return popcount<SIMDLevel::NONE>(data, size);
+    size_t vlmax = __riscv_vsetvlmax_e16m8();
+    vuint16m8_t acc = __riscv_vmv_v_x_u16m8(0, vlmax);
+    size_t i = 0;
+    while (i < size) {
+        size_t vl = __riscv_vsetvl_e8m4(size - i);
+        vuint8m4_t v = popcount_u8m4(__riscv_vle8_v_u8m4(data + i, vl), vl);
+        vuint16m8_t vw = __riscv_vzext_vf2_u16m8(v, vl);
+        acc = __riscv_vadd_vv_u16m8_tu(acc, acc, vw, vl);
+        i += vl;
+    }
+    vuint32m1_t red = __riscv_vmv_v_x_u32m1(0, 1);
+    red = __riscv_vwredsumu_vs_u16m8_u32m1(acc, red, vlmax);
+    return __riscv_vmv_x_s_u32m1_u32(red);
 }
 
 } // namespace faiss::rabitq
 
 namespace faiss::rabitq::multibit {
 
+static float ip_1exbit_rvv(
+        const uint8_t* __restrict sign_bits,
+        const uint8_t* __restrict ex_code,
+        const float* __restrict rotated_q,
+        size_t d,
+        float cb) {
+    size_t vlmax = __riscv_vsetvlmax_e32m8();
+    vfloat32m8_t acc = __riscv_vfmv_v_f_f32m8(0.0f, vlmax);
+    size_t i = 0;
+    while (i < d) {
+        size_t vl = __riscv_vsetvl_e32m8(d - i);
+        vbool4_t sb = __riscv_vlm_v_b4(sign_bits + i / 8, vl);
+        vbool4_t eb = __riscv_vlm_v_b4(ex_code + i / 8, vl);
+        vfloat32m8_t recon = __riscv_vfmv_v_f_f32m8(cb, vl);
+        recon = __riscv_vfadd_vf_f32m8_mu(sb, recon, recon, 2.0f, vl);
+        recon = __riscv_vfadd_vf_f32m8_mu(eb, recon, recon, 1.0f, vl);
+        vfloat32m8_t rq = __riscv_vle32_v_f32m8(rotated_q + i, vl);
+        acc = __riscv_vfmacc_vv_f32m8_tu(acc, rq, recon, vl);
+        i += vl;
+    }
+    vfloat32m1_t sum = __riscv_vfmv_s_f_f32m1(0.0f, 1);
+    sum = __riscv_vfredusum_vs_f32m8_f32m1(acc, sum, vlmax);
+    return __riscv_vfmv_f_s_f32m1_f32(sum);
+}
+
+// no-tree-vectorize keeps the ex_bits >= 2 tail on the existing scalar
+// ip_scalar without GCC turning its 64-bit memcpy window into a vluxei64 gather
+// at unaligned addresses, which SIGBUSes on strict-align RVV cores (SpacemiT
+// X60).
 template <>
-float compute_inner_product<SIMDLevel::RISCV_RVV>(
+__attribute__((optimize("no-tree-vectorize", "no-tree-slp-vectorize"))) float
+compute_inner_product<SIMDLevel::RISCV_RVV>(
         const uint8_t* __restrict sign_bits,
         const uint8_t* __restrict ex_code,
         const float* __restrict rotated_q,
         size_t d,
         size_t ex_bits,
         float cb) {
-    return compute_inner_product<SIMDLevel::NONE>(
-            sign_bits, ex_code, rotated_q, d, ex_bits, cb);
+    if (ex_bits == 1) {
+        return ip_1exbit_rvv(sign_bits, ex_code, rotated_q, d, cb);
+    }
+    // ex_bits >= 2 needs strided bit-plane extraction (PEXT on x86); RVV has no
+    // cheap equivalent without Zvbb, so reuse the scalar kernel.
+    return ip_scalar(sign_bits, ex_code, rotated_q, 0, d, ex_bits, cb);
 }
 
 } // namespace faiss::rabitq::multibit

data/vendor/faiss/faiss/utils/simd_levels.cpp

@@ -20,6 +20,10 @@ SIMDLevel SIMDConfig::level = SIMDLevel::NONE;
 // Bitmask of supported SIMD levels (1 << SIMDLevel)
 uint64_t SIMDConfig::supported_simd_levels = 0;
 
+// Microarchitecture flags (x86). Default false; set by
+// detect_x86_uarch_flags() at load time.
+bool SIMDConfig::avx512_split = false;
+
 // ARM SVE runtime detection
 #if defined(__aarch64__) || defined(_M_ARM64)
 
@@ -53,6 +57,43 @@ static bool has_sve() {
 }
 #endif
 
+// Detect x86 microarchitecture flags used for kernel routing. Uses raw
+// cpuid so it is safe to run on any CPU regardless of compiled SIMD level.
+#if defined(__x86_64__)
+namespace {
+void detect_x86_uarch_flags() {
+    unsigned int eax, ebx, ecx, edx;
+
+    // Vendor string (CPUID.0): "AuthenticAMD" is EBX="Auth", EDX="enti",
+    // ECX="cAMD".
+    eax = 0;
+    ecx = 0;
+    asm volatile("cpuid"
+                 : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
+                 : "a"(eax), "c"(ecx));
+    const bool is_amd =
+            ebx == 0x68747541u && edx == 0x69746e65u && ecx == 0x444d4163u;
+
+    // Family/model (CPUID.1 EAX).
+    eax = 1;
+    ecx = 0;
+    asm volatile("cpuid"
+                 : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
+                 : "a"(eax), "c"(ecx));
+    const unsigned int base_family = (eax >> 8) & 0xfu;
+    const unsigned int display_family =
+            base_family + (base_family == 0xfu ? ((eax >> 20) & 0xffu) : 0u);
+    // AMD Zen 4 / Zen 4c (Bergamo) is family 0x19 and splits AVX-512.
+    // (Zen 5, family 0x1A, has a native 512-bit datapath and is excluded.)
+    SIMDConfig::avx512_split = is_amd && display_family == 0x19u;
+}
+} // namespace
+#else
+namespace {
+void detect_x86_uarch_flags() {}
+} // namespace
+#endif
+
 #ifdef FAISS_ENABLE_DD
 
 // =============================================================================
@@ -101,6 +142,8 @@ bool SIMDConfig::is_simd_level_available(SIMDLevel l) {
 SIMDLevel SIMDConfig::auto_detect_simd_level() {
     SIMDLevel detected_level = SIMDLevel::NONE;
 
+    detect_x86_uarch_flags();
+
 #if defined(__x86_64__) && \
         (defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512))
     unsigned int eax, ebx, ecx, edx;
@@ -264,6 +307,7 @@ bool SIMDConfig::is_simd_level_available(SIMDLevel l) {
 }
 
 SIMDLevel SIMDConfig::auto_detect_simd_level() {
+    detect_x86_uarch_flags();
     // In static mode, return the compiled-in level
 #if defined(COMPILE_SIMD_AVX512_SPR)
     return SIMDLevel::AVX512_SPR;

data/vendor/faiss/faiss/utils/simd_levels.h

@@ -161,8 +161,22 @@ struct FAISS_API SIMDConfig {
     /// Returns bitmask of supported SIMD levels (1 << SIMDLevel).
     static uint64_t supported_simd_levels;
 
+    /// CPU implements AVX-512 by splitting over a 256-bit datapath
+    /// (AMD Zen 4 / Zen 4c "Bergamo", family 0x19). On such CPUs 512-bit
+    /// ops give no throughput gain, so the fast-scan QBS path prefers the
+    /// 256-bit kernel.
+    static bool avx512_split;
+
     static SIMDLevel auto_detect_simd_level();
 
+    static constexpr bool has_dynamic_dispatch() {
+#ifdef FAISS_ENABLE_DD
+        return true;
+#else
+        return false;
+#endif
+    }
+
     SIMDConfig(const char** faiss_simd_level_env = nullptr);
 
     /// Set the SIMD level. Throws FaissException if level is not supported.

data/vendor/faiss/faiss/utils/utils.cpp

@@ -30,6 +30,7 @@
 #include <algorithm>
 #include <set>
 #include <type_traits>
+#include <unordered_set>
 #include <vector>
 
 #include <faiss/impl/AuxIndexStructures.h>
@@ -362,39 +363,20 @@ size_t ranklist_intersection_size(
         const int64_t* v2_in) {
     if (k2 > k1)
         return ranklist_intersection_size(k2, v2_in, k1, v1);
-    int64_t* v2 = new int64_t[k2];
-    memcpy(v2, v2_in, sizeof(int64_t) * k2);
-    std::sort(v2, v2 + k2);
-    { // de-dup v2
-        int64_t prev = -1;
-        size_t wp = 0;
-        for (size_t i = 0; i < k2; i++) {
-            if (v2[i] != prev) {
-                v2[wp++] = prev = v2[i];
-            }
+    // erase-on-hit avoids double-counting; negatives are padding, not IDs
+    std::unordered_set<int64_t> remaining;
+    remaining.reserve(k2);
+    for (size_t i = 0; i < k2; i++) {
+        if (v2_in[i] >= 0) {
+            remaining.insert(v2_in[i]);
         }
-        k2 = wp;
     }
-    const int64_t seen_flag = int64_t{1} << 60;
     size_t count = 0;
     for (size_t i = 0; i < k1; i++) {
-        int64_t q = v1[i];
-        size_t i0 = 0, i1 = k2;
-        while (i0 + 1 < i1) {
-            size_t imed = (i1 + i0) / 2;
-            int64_t piv = v2[imed] & ~seen_flag;
-            if (piv <= q)
-                i0 = imed;
-            else
-                i1 = imed;
-        }
-        if (v2[i0] == q) {
+        if (remaining.erase(v1[i]) != 0) {
             count++;
-            v2[i0] |= seen_flag;
         }
     }
-    delete[] v2;
-
     return count;
 }
 
@@ -462,7 +444,7 @@ uint64_t ivec_checksum(size_t n, const int32_t* assigned) {
 uint64_t bvec_checksum(size_t n, const uint8_t* a) {
     uint64_t cs = ivec_checksum(n / 4, (const int32_t*)a);
     for (size_t i = n / 4 * 4; i < n; i++) {
-        cs = cs * 65713 + a[n] * 1686049;
+        cs = cs * 65713 + a[i] * 1686049;
     }
     return cs;
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Andrew Kane
@@ -85,6 +85,8 @@ files:
 - vendor/faiss/faiss/IndexBinaryHash.h
 - vendor/faiss/faiss/IndexBinaryIVF.cpp
 - vendor/faiss/faiss/IndexBinaryIVF.h
+- vendor/faiss/faiss/IndexEDEN.cpp
+- vendor/faiss/faiss/IndexEDEN.h
 - vendor/faiss/faiss/IndexFastScan.cpp
 - vendor/faiss/faiss/IndexFastScan.h
 - vendor/faiss/faiss/IndexFlat.cpp
@@ -101,6 +103,8 @@ files:
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizer.h
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h
+- vendor/faiss/faiss/IndexIVFEDEN.cpp
+- vendor/faiss/faiss/IndexIVFEDEN.h
 - vendor/faiss/faiss/IndexIVFFastScan.cpp
 - vendor/faiss/faiss/IndexIVFFastScan.h
 - vendor/faiss/faiss/IndexIVFFlat.cpp
@@ -238,11 +242,13 @@ files:
 - vendor/faiss/faiss/gpu_metal/MetalIndex.h
 - vendor/faiss/faiss/gpu_metal/MetalIndexFlat.h
 - vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h
+- vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h
 - vendor/faiss/faiss/gpu_metal/MetalKernels.h
 - vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h
 - vendor/faiss/faiss/gpu_metal/MetalResources.h
 - vendor/faiss/faiss/gpu_metal/StandardMetalResources.h
 - vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h
+- vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h
 - vendor/faiss/faiss/impl/AdSampling.cpp
 - vendor/faiss/faiss/impl/AdSampling.h
 - vendor/faiss/faiss/impl/AdditiveQuantizer.cpp
@@ -258,6 +264,7 @@ files:
 - vendor/faiss/faiss/impl/CodePackerRaBitQ.cpp
 - vendor/faiss/faiss/impl/CodePackerRaBitQ.h
 - vendor/faiss/faiss/impl/DistanceComputer.h
+- vendor/faiss/faiss/impl/EDENQuantizer.h
 - vendor/faiss/faiss/impl/FaissAssert.h
 - vendor/faiss/faiss/impl/FaissException.cpp
 - vendor/faiss/faiss/impl/FaissException.h
@@ -356,6 +363,8 @@ files:
 - vendor/faiss/faiss/impl/mapped_io.h
 - vendor/faiss/faiss/impl/maybe_owned_vector.h
 - vendor/faiss/faiss/impl/platform_macros.h
+- vendor/faiss/faiss/impl/polysemous_training/avx512.cpp
+- vendor/faiss/faiss/impl/polysemous_training/dispatch.h
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.h
@@ -373,6 +382,12 @@ files:
 - vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h
+- vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp
+- vendor/faiss/faiss/impl/result_handler/avx2.cpp
+- vendor/faiss/faiss/impl/result_handler/avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp
 - vendor/faiss/faiss/impl/scalar_quantizer/codecs.h
 - vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h
 - vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h