faiss 0.6.1 → 0.6.3

data/vendor/faiss/faiss/utils/simd_levels.cpp

@@ -20,6 +20,10 @@ SIMDLevel SIMDConfig::level = SIMDLevel::NONE;
 // Bitmask of supported SIMD levels (1 << SIMDLevel)
 uint64_t SIMDConfig::supported_simd_levels = 0;
 
+// Microarchitecture flags (x86). Default false; set by
+// detect_x86_uarch_flags() at load time.
+bool SIMDConfig::avx512_split = false;
+
 // ARM SVE runtime detection
 #if defined(__aarch64__) || defined(_M_ARM64)
 
@@ -53,6 +57,43 @@ static bool has_sve() {
 }
 #endif
 
+// Detect x86 microarchitecture flags used for kernel routing. Uses raw
+// cpuid so it is safe to run on any CPU regardless of compiled SIMD level.
+#if defined(__x86_64__)
+namespace {
+void detect_x86_uarch_flags() {
+    unsigned int eax, ebx, ecx, edx;
+
+    // Vendor string (CPUID.0): "AuthenticAMD" is EBX="Auth", EDX="enti",
+    // ECX="cAMD".
+    eax = 0;
+    ecx = 0;
+    asm volatile("cpuid"
+                 : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
+                 : "a"(eax), "c"(ecx));
+    const bool is_amd =
+            ebx == 0x68747541u && edx == 0x69746e65u && ecx == 0x444d4163u;
+
+    // Family/model (CPUID.1 EAX).
+    eax = 1;
+    ecx = 0;
+    asm volatile("cpuid"
+                 : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
+                 : "a"(eax), "c"(ecx));
+    const unsigned int base_family = (eax >> 8) & 0xfu;
+    const unsigned int display_family =
+            base_family + (base_family == 0xfu ? ((eax >> 20) & 0xffu) : 0u);
+    // AMD Zen 4 / Zen 4c (Bergamo) is family 0x19 and splits AVX-512.
+    // (Zen 5, family 0x1A, has a native 512-bit datapath and is excluded.)
+    SIMDConfig::avx512_split = is_amd && display_family == 0x19u;
+}
+} // namespace
+#else
+namespace {
+void detect_x86_uarch_flags() {}
+} // namespace
+#endif
+
 #ifdef FAISS_ENABLE_DD
 
 // =============================================================================
@@ -101,6 +142,8 @@ bool SIMDConfig::is_simd_level_available(SIMDLevel l) {
 SIMDLevel SIMDConfig::auto_detect_simd_level() {
     SIMDLevel detected_level = SIMDLevel::NONE;
 
+    detect_x86_uarch_flags();
+
 #if defined(__x86_64__) && \
         (defined(COMPILE_SIMD_AVX2) || defined(COMPILE_SIMD_AVX512))
     unsigned int eax, ebx, ecx, edx;
@@ -129,6 +172,9 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
         asm volatile("cpuid"
                      : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
                      : "a"(eax), "c"(ecx));
+        // Save EDX before xgetbv clobbers it — needed for
+        // AVX512_FP16 check (bit 23) in the SPR detection below.
+        unsigned int cpuid7_edx = edx;
 
         unsigned int xcr0;
         asm volatile("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0));
@@ -155,8 +201,15 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
                         (1 << static_cast<int>(SIMDLevel::AVX512));
 
 #if defined(COMPILE_SIMD_AVX512_SPR)
-                // Check for Sapphire Rapids features (AVX512_BF16)
+                // Check for Sapphire Rapids features.
+                // The SPR code path is compiled with -mavx512fp16, so we
+                // must verify both AVX512_BF16 and AVX512_FP16 before
+                // dispatching to it. AMD Zen 4 (bergamo) has BF16 but
+                // not FP16 — using SPR code there causes SIGILL.
                 // CPUID EAX=7, ECX=1: EAX bit 5 = AVX512_BF16
+                // CPUID EAX=7, ECX=0: EDX bit 23 = AVX512_FP16
+                // (Linux: X86_FEATURE_AVX512_FP16 = 18*32+23)
+                bool has_avx512_fp16 = (cpuid7_edx & (1 << 23)) != 0;
                 unsigned int eax1, ebx1, ecx1, edx1;
                 eax1 = 7;
                 ecx1 = 1;
@@ -164,7 +217,7 @@ SIMDLevel SIMDConfig::auto_detect_simd_level() {
                              : "=a"(eax1), "=b"(ebx1), "=c"(ecx1), "=d"(edx1)
                              : "a"(eax1), "c"(ecx1));
                 bool has_avx512_bf16 = (eax1 & (1 << 5)) != 0;
-                if (has_avx512_bf16) {
+                if (has_avx512_bf16 && has_avx512_fp16) {
                     detected_level = SIMDLevel::AVX512_SPR;
                     supported_simd_levels |=
                             (1 << static_cast<int>(SIMDLevel::AVX512_SPR));
@@ -254,6 +307,7 @@ bool SIMDConfig::is_simd_level_available(SIMDLevel l) {
 }
 
 SIMDLevel SIMDConfig::auto_detect_simd_level() {
+    detect_x86_uarch_flags();
     // In static mode, return the compiled-in level
 #if defined(COMPILE_SIMD_AVX512_SPR)
     return SIMDLevel::AVX512_SPR;

data/vendor/faiss/faiss/utils/simd_levels.h

@@ -161,8 +161,22 @@ struct FAISS_API SIMDConfig {
     /// Returns bitmask of supported SIMD levels (1 << SIMDLevel).
     static uint64_t supported_simd_levels;
 
+    /// CPU implements AVX-512 by splitting over a 256-bit datapath
+    /// (AMD Zen 4 / Zen 4c "Bergamo", family 0x19). On such CPUs 512-bit
+    /// ops give no throughput gain, so the fast-scan QBS path prefers the
+    /// 256-bit kernel.
+    static bool avx512_split;
+
     static SIMDLevel auto_detect_simd_level();
 
+    static constexpr bool has_dynamic_dispatch() {
+#ifdef FAISS_ENABLE_DD
+        return true;
+#else
+        return false;
+#endif
+    }
+
     SIMDConfig(const char** faiss_simd_level_env = nullptr);
 
     /// Set the SIMD level. Throws FaissException if level is not supported.

data/vendor/faiss/faiss/utils/utils.cpp

@@ -30,6 +30,7 @@
 #include <algorithm>
 #include <set>
 #include <type_traits>
+#include <unordered_set>
 #include <vector>
 
 #include <faiss/impl/AuxIndexStructures.h>
@@ -362,39 +363,20 @@ size_t ranklist_intersection_size(
         const int64_t* v2_in) {
     if (k2 > k1)
         return ranklist_intersection_size(k2, v2_in, k1, v1);
-    int64_t* v2 = new int64_t[k2];
-    memcpy(v2, v2_in, sizeof(int64_t) * k2);
-    std::sort(v2, v2 + k2);
-    { // de-dup v2
-        int64_t prev = -1;
-        size_t wp = 0;
-        for (size_t i = 0; i < k2; i++) {
-            if (v2[i] != prev) {
-                v2[wp++] = prev = v2[i];
-            }
+    // erase-on-hit avoids double-counting; negatives are padding, not IDs
+    std::unordered_set<int64_t> remaining;
+    remaining.reserve(k2);
+    for (size_t i = 0; i < k2; i++) {
+        if (v2_in[i] >= 0) {
+            remaining.insert(v2_in[i]);
         }
-        k2 = wp;
     }
-    const int64_t seen_flag = int64_t{1} << 60;
     size_t count = 0;
     for (size_t i = 0; i < k1; i++) {
-        int64_t q = v1[i];
-        size_t i0 = 0, i1 = k2;
-        while (i0 + 1 < i1) {
-            size_t imed = (i1 + i0) / 2;
-            int64_t piv = v2[imed] & ~seen_flag;
-            if (piv <= q)
-                i0 = imed;
-            else
-                i1 = imed;
-        }
-        if (v2[i0] == q) {
+        if (remaining.erase(v1[i]) != 0) {
             count++;
-            v2[i0] |= seen_flag;
         }
     }
-    delete[] v2;
-
     return count;
 }
 
@@ -462,7 +444,7 @@ uint64_t ivec_checksum(size_t n, const int32_t* assigned) {
 uint64_t bvec_checksum(size_t n, const uint8_t* a) {
     uint64_t cs = ivec_checksum(n / 4, (const int32_t*)a);
     for (size_t i = n / 4 * 4; i < n; i++) {
-        cs = cs * 65713 + a[n] * 1686049;
+        cs = cs * 65713 + a[i] * 1686049;
     }
     return cs;
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.3
 platform: ruby
 authors:
 - Andrew Kane
@@ -85,6 +85,8 @@ files:
 - vendor/faiss/faiss/IndexBinaryHash.h
 - vendor/faiss/faiss/IndexBinaryIVF.cpp
 - vendor/faiss/faiss/IndexBinaryIVF.h
+- vendor/faiss/faiss/IndexEDEN.cpp
+- vendor/faiss/faiss/IndexEDEN.h
 - vendor/faiss/faiss/IndexFastScan.cpp
 - vendor/faiss/faiss/IndexFastScan.h
 - vendor/faiss/faiss/IndexFlat.cpp
@@ -101,6 +103,8 @@ files:
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizer.h
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h
+- vendor/faiss/faiss/IndexIVFEDEN.cpp
+- vendor/faiss/faiss/IndexIVFEDEN.h
 - vendor/faiss/faiss/IndexIVFFastScan.cpp
 - vendor/faiss/faiss/IndexIVFFastScan.h
 - vendor/faiss/faiss/IndexIVFFlat.cpp
@@ -233,12 +237,18 @@ files:
 - vendor/faiss/faiss/gpu/utils/Timer.h
 - vendor/faiss/faiss/gpu_metal/GpuIndexFlat.h
 - vendor/faiss/faiss/gpu_metal/MetalCloner.h
+- vendor/faiss/faiss/gpu_metal/MetalDistance.h
 - vendor/faiss/faiss/gpu_metal/MetalFlatKernels.h
 - vendor/faiss/faiss/gpu_metal/MetalIndex.h
 - vendor/faiss/faiss/gpu_metal/MetalIndexFlat.h
+- vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h
+- vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h
 - vendor/faiss/faiss/gpu_metal/MetalKernels.h
+- vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h
 - vendor/faiss/faiss/gpu_metal/MetalResources.h
 - vendor/faiss/faiss/gpu_metal/StandardMetalResources.h
+- vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h
+- vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h
 - vendor/faiss/faiss/impl/AdSampling.cpp
 - vendor/faiss/faiss/impl/AdSampling.h
 - vendor/faiss/faiss/impl/AdditiveQuantizer.cpp
@@ -254,6 +264,7 @@ files:
 - vendor/faiss/faiss/impl/CodePackerRaBitQ.cpp
 - vendor/faiss/faiss/impl/CodePackerRaBitQ.h
 - vendor/faiss/faiss/impl/DistanceComputer.h
+- vendor/faiss/faiss/impl/EDENQuantizer.h
 - vendor/faiss/faiss/impl/FaissAssert.h
 - vendor/faiss/faiss/impl/FaissException.cpp
 - vendor/faiss/faiss/impl/FaissException.h
@@ -352,6 +363,8 @@ files:
 - vendor/faiss/faiss/impl/mapped_io.h
 - vendor/faiss/faiss/impl/maybe_owned_vector.h
 - vendor/faiss/faiss/impl/platform_macros.h
+- vendor/faiss/faiss/impl/polysemous_training/avx512.cpp
+- vendor/faiss/faiss/impl/polysemous_training/dispatch.h
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.h
@@ -365,15 +378,24 @@ files:
 - vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.h
 - vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h
 - vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp
+- vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h
 - vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h
+- vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp
+- vendor/faiss/faiss/impl/result_handler/avx2.cpp
+- vendor/faiss/faiss/impl/result_handler/avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp
 - vendor/faiss/faiss/impl/scalar_quantizer/codecs.h
 - vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h
 - vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h
 - vendor/faiss/faiss/impl/scalar_quantizer/scanners.h
 - vendor/faiss/faiss/impl/scalar_quantizer/similarities.h
 - vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h
+- vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp
 - vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp
 - vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h
 - vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp
@@ -453,8 +475,10 @@ files:
 - vendor/faiss/faiss/utils/hamming_distance/common.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp
 - vendor/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp
+- vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx2.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h
+- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-generic.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-neon.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-rvv.h
@@ -489,6 +513,7 @@ files:
 - vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h
 - vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp
 - vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp
+- vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp
 - vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp
 - vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp
 - vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h
@@ -521,7 +546,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.14
 specification_version: 4
 summary: Efficient similarity search and clustering for Ruby
 test_files: []