faiss 0.6.2 → 0.6.4

data/vendor/faiss/faiss/utils/simd_levels.h

@@ -21,13 +21,18 @@ enum class SIMDLevel {
     // x86
     AVX2,
     AVX512,
-    AVX512_SPR, // Sapphire Rapids: AVX512 + BF16 + FP16 + VNNI
+    AVX512_SPR, // Sapphire Rapids: AVX512_VPOPCNT + BF16 + FP16 + VNNI
     // arm & aarch64
     ARM_NEON,
     ARM_SVE, // Scalable Vector Extension (ARMv8.2+)
     // riscv
     RISCV_RVV, // RISC-V Vector Extension (rv64gcv)
 
+    // Appended to preserve the numeric values of the existing public enum.
+    // AVX-512 core features plus AVX512_VPOPCNTDQ and AVX512_BITALG
+    // (Ice Lake, Zen 4, etc.).
+    AVX512_VPOPCNT,
+
     COUNT
 };
 
@@ -52,6 +57,8 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::NONE;
 #else
 #if defined(COMPILE_SIMD_AVX512_SPR)
 inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::AVX512_SPR;
+#elif defined(COMPILE_SIMD_AVX512_VPOPCNT)
+inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::AVX512_VPOPCNT;
 #elif defined(COMPILE_SIMD_AVX512)
 inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::AVX512;
 #elif defined(COMPILE_SIMD_AVX2)
@@ -71,7 +78,7 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::NONE;
  * Helper to select the appropriate 256-bit SIMD level.
  *
  * For 256-bit SIMD types (simd16uint16, simd32uint8, etc.), maps:
- *   AVX512/AVX512_SPR → AVX2 (256-bit ops use AVX2 instructions)
+ *   AVX512/AVX512_VPOPCNT/AVX512_SPR → AVX2
  *   AVX2 → AVX2
  *   ARM_NEON/ARM_SVE → ARM_NEON
  *   NONE → NONE
@@ -79,7 +86,8 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL = SIMDLevel::NONE;
 template <SIMDLevel SL>
 struct simd256_level_selector {
     static constexpr SIMDLevel value =
-            (SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_SPR)
+            (SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_VPOPCNT ||
+             SL == SIMDLevel::AVX512_SPR)
             ? SIMDLevel::AVX2
             : (SL == SIMDLevel::ARM_SVE             ? SIMDLevel::ARM_NEON
                        : SL == SIMDLevel::RISCV_RVV ? SIMDLevel::NONE
@@ -96,13 +104,14 @@ inline constexpr SIMDLevel SINGLE_SIMD_LEVEL_256 =
  * Helper to select the appropriate 512-bit SIMD level.
  *
  * For 512-bit SIMD types (simd32uint16, simd64uint8, etc.), maps:
- *   AVX512_SPR → AVX512 (512-bit ops share the same instructions)
+ *   AVX512_VPOPCNT/AVX512_SPR → AVX512 for generic 512-bit operations
  *   AVX512 → AVX512
  *   NONE → NONE
  ***************************************************************/
 template <SIMDLevel SL>
 struct simd512_level_selector {
-    static constexpr SIMDLevel value = (SL == SIMDLevel::AVX512_SPR)
+    static constexpr SIMDLevel value =
+            (SL == SIMDLevel::AVX512_SPR || SL == SIMDLevel::AVX512_VPOPCNT)
             ? SIMDLevel::AVX512
             : (SL == SIMDLevel::RISCV_RVV) ? SIMDLevel::NONE
                                            : SL;
@@ -124,14 +133,21 @@ constexpr int simd_width() {
     static_assert(
             SL != SIMDLevel::RISCV_RVV,
             "simd_width<RISCV_RVV> is not supported: RVV is variable-width");
-    if constexpr (SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_SPR)
+    if constexpr (
+            SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_VPOPCNT ||
+            SL == SIMDLevel::AVX512_SPR) {
         return 16;
-    else if constexpr (SL == SIMDLevel::AVX2 || SL == SIMDLevel::ARM_NEON)
+    } else if constexpr (SL == SIMDLevel::AVX2 || SL == SIMDLevel::ARM_NEON) {
         return 8;
-    else
+    } else {
         return 1;
+    }
 }
 
+/// Bitmask (1 << SIMDLevel) of the levels this binary holds code for,
+/// whether or not the CPU supports them.
+uint64_t compiled_simd_levels();
+
 /// Convert SIMDLevel to string. Throws FaissException for invalid level.
 std::string to_string(SIMDLevel level);
 
@@ -161,8 +177,17 @@ struct FAISS_API SIMDConfig {
     /// Returns bitmask of supported SIMD levels (1 << SIMDLevel).
     static uint64_t supported_simd_levels;
 
+    /// CPU implements AVX-512 by splitting over a 256-bit datapath
+    /// (AMD Zen 4 / Zen 4c "Bergamo", family 0x19). On such CPUs 512-bit
+    /// ops give no throughput gain, so the fast-scan QBS path prefers the
+    /// 256-bit kernel.
+    static bool avx512_split;
+
     static SIMDLevel auto_detect_simd_level();
 
+    /// Whether this faiss build dispatches SIMD at runtime.
+    static bool has_dynamic_dispatch();
+
     SIMDConfig(const char** faiss_simd_level_env = nullptr);
 
     /// Set the SIMD level. Throws FaissException if level is not supported.

data/vendor/faiss/faiss/utils/utils.cpp

@@ -30,6 +30,7 @@
 #include <algorithm>
 #include <set>
 #include <type_traits>
+#include <unordered_set>
 #include <vector>
 
 #include <faiss/impl/AuxIndexStructures.h>
@@ -362,39 +363,20 @@ size_t ranklist_intersection_size(
         const int64_t* v2_in) {
     if (k2 > k1)
         return ranklist_intersection_size(k2, v2_in, k1, v1);
-    int64_t* v2 = new int64_t[k2];
-    memcpy(v2, v2_in, sizeof(int64_t) * k2);
-    std::sort(v2, v2 + k2);
-    { // de-dup v2
-        int64_t prev = -1;
-        size_t wp = 0;
-        for (size_t i = 0; i < k2; i++) {
-            if (v2[i] != prev) {
-                v2[wp++] = prev = v2[i];
-            }
+    // erase-on-hit avoids double-counting; negatives are padding, not IDs
+    std::unordered_set<int64_t> remaining;
+    remaining.reserve(k2);
+    for (size_t i = 0; i < k2; i++) {
+        if (v2_in[i] >= 0) {
+            remaining.insert(v2_in[i]);
         }
-        k2 = wp;
     }
-    const int64_t seen_flag = int64_t{1} << 60;
     size_t count = 0;
     for (size_t i = 0; i < k1; i++) {
-        int64_t q = v1[i];
-        size_t i0 = 0, i1 = k2;
-        while (i0 + 1 < i1) {
-            size_t imed = (i1 + i0) / 2;
-            int64_t piv = v2[imed] & ~seen_flag;
-            if (piv <= q)
-                i0 = imed;
-            else
-                i1 = imed;
-        }
-        if (v2[i0] == q) {
+        if (remaining.erase(v1[i]) != 0) {
             count++;
-            v2[i0] |= seen_flag;
         }
     }
-    delete[] v2;
-
     return count;
 }
 
@@ -462,7 +444,7 @@ uint64_t ivec_checksum(size_t n, const int32_t* assigned) {
 uint64_t bvec_checksum(size_t n, const uint8_t* a) {
     uint64_t cs = ivec_checksum(n / 4, (const int32_t*)a);
     for (size_t i = n / 4 * 4; i < n; i++) {
-        cs = cs * 65713 + a[n] * 1686049;
+        cs = cs * 65713 + a[i] * 1686049;
     }
     return cs;
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.4
 platform: ruby
 authors:
 - Andrew Kane
@@ -85,6 +85,8 @@ files:
 - vendor/faiss/faiss/IndexBinaryHash.h
 - vendor/faiss/faiss/IndexBinaryIVF.cpp
 - vendor/faiss/faiss/IndexBinaryIVF.h
+- vendor/faiss/faiss/IndexEDEN.cpp
+- vendor/faiss/faiss/IndexEDEN.h
 - vendor/faiss/faiss/IndexFastScan.cpp
 - vendor/faiss/faiss/IndexFastScan.h
 - vendor/faiss/faiss/IndexFlat.cpp
@@ -101,6 +103,8 @@ files:
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizer.h
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp
 - vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h
+- vendor/faiss/faiss/IndexIVFEDEN.cpp
+- vendor/faiss/faiss/IndexIVFEDEN.h
 - vendor/faiss/faiss/IndexIVFFastScan.cpp
 - vendor/faiss/faiss/IndexIVFFastScan.h
 - vendor/faiss/faiss/IndexIVFFlat.cpp
@@ -238,11 +242,13 @@ files:
 - vendor/faiss/faiss/gpu_metal/MetalIndex.h
 - vendor/faiss/faiss/gpu_metal/MetalIndexFlat.h
 - vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h
+- vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h
 - vendor/faiss/faiss/gpu_metal/MetalKernels.h
 - vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h
 - vendor/faiss/faiss/gpu_metal/MetalResources.h
 - vendor/faiss/faiss/gpu_metal/StandardMetalResources.h
 - vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h
+- vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h
 - vendor/faiss/faiss/impl/AdSampling.cpp
 - vendor/faiss/faiss/impl/AdSampling.h
 - vendor/faiss/faiss/impl/AdditiveQuantizer.cpp
@@ -258,6 +264,7 @@ files:
 - vendor/faiss/faiss/impl/CodePackerRaBitQ.cpp
 - vendor/faiss/faiss/impl/CodePackerRaBitQ.h
 - vendor/faiss/faiss/impl/DistanceComputer.h
+- vendor/faiss/faiss/impl/EDENQuantizer.h
 - vendor/faiss/faiss/impl/FaissAssert.h
 - vendor/faiss/faiss/impl/FaissException.cpp
 - vendor/faiss/faiss/impl/FaissException.h
@@ -356,6 +363,8 @@ files:
 - vendor/faiss/faiss/impl/mapped_io.h
 - vendor/faiss/faiss/impl/maybe_owned_vector.h
 - vendor/faiss/faiss/impl/platform_macros.h
+- vendor/faiss/faiss/impl/polysemous_training/avx512.cpp
+- vendor/faiss/faiss/impl/polysemous_training/dispatch.h
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp
 - vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.h
@@ -373,6 +382,12 @@ files:
 - vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h
+- vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp
+- vendor/faiss/faiss/impl/result_handler/avx2.cpp
+- vendor/faiss/faiss/impl/result_handler/avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp
 - vendor/faiss/faiss/impl/scalar_quantizer/codecs.h
 - vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h
 - vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h
@@ -460,10 +475,10 @@ files:
 - vendor/faiss/faiss/utils/hamming_distance/common.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp
 - vendor/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp
-- vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp
+- vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_vpopcnt.cpp
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx2.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h
-- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h
+- vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_vpopcnt.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-generic.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-neon.h
 - vendor/faiss/faiss/utils/hamming_distance/hamming_computer-rvv.h
@@ -498,13 +513,14 @@ files:
 - vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h
 - vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp
 - vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp
-- vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp
+- vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_vpopcnt.cpp
 - vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp
 - vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp
 - vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h
 - vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h
 - vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp
 - vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp
+- vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp
 - vendor/faiss/faiss/utils/simd_levels.cpp
 - vendor/faiss/faiss/utils/simd_levels.h
 - vendor/faiss/faiss/utils/sorting.cpp
@@ -531,7 +547,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.14
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Efficient similarity search and clustering for Ruby
 test_files: []

data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp

@@ -1,15 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#ifdef COMPILE_SIMD_AVX512_SPR
-
-#define THE_SIMD_LEVEL SIMDLevel::AVX512_SPR
-// NOLINTNEXTLINE(facebook-hte-InlineHeader)
-#include <faiss/utils/hamming_distance/hamming_computer-avx512_spr.h>
-#include <faiss/utils/hamming_distance/hamming_impl.h>
-
-#endif // COMPILE_SIMD_AVX512_SPR