faiss 0.5.2 → 0.6.0

data/vendor/faiss/faiss/utils/simd_levels.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+#include <faiss/impl/platform_macros.h>
+
+namespace faiss {
+
+#define COMPILE_SIMD_NONE
+
+enum class SIMDLevel {
+    NONE,
+    // x86
+    AVX2,
+    AVX512,
+    AVX512_SPR, // Sapphire Rapids: AVX512 + BF16 + FP16 + VNNI
+    // arm & aarch64
+    ARM_NEON,
+    ARM_SVE, // Scalable Vector Extension (ARMv8.2+)
+
+    COUNT
+};
+
+/// Number of float32 lanes for a given SIMD level.
+template <SIMDLevel SL>
+constexpr int simd_width() {
+    if constexpr (SL == SIMDLevel::AVX512 || SL == SIMDLevel::AVX512_SPR)
+        return 16;
+    else if constexpr (SL == SIMDLevel::AVX2 || SL == SIMDLevel::ARM_NEON)
+        return 8;
+    else
+        return 1;
+}
+
+/// Convert SIMDLevel to string. Throws FaissException for invalid level.
+std::string to_string(SIMDLevel level);
+
+/// Parse string to SIMDLevel. Throws FaissException for invalid strings.
+SIMDLevel to_simd_level(const std::string& level_str);
+
+/**
+ * Current SIMD configuration.
+ *
+ * This class provides a uniform API for querying and setting the SIMD level,
+ * regardless of whether faiss was built with Dynamic Dispatch (DD) or static
+ * SIMD selection.
+ *
+ * In DD mode:
+ *   - get_level() returns the runtime-detected or user-set level
+ *   - set_level() changes the runtime level (if level is supported)
+ *   - supported_simd_levels() returns bitmask of all compiled-in levels
+ *
+ * In static mode:
+ *   - get_level() returns the compiled-in level
+ *   - set_level() succeeds only if level matches compiled-in level
+ *   - supported_simd_levels() returns bitmask with single level
+ */
+struct FAISS_API SIMDConfig {
+    static SIMDLevel level;
+
+    /// Returns bitmask of supported SIMD levels (1 << SIMDLevel).
+    static uint64_t supported_simd_levels;
+
+    static SIMDLevel auto_detect_simd_level();
+
+    SIMDConfig(const char** faiss_simd_level_env = nullptr);
+
+    /// Set the SIMD level. Throws FaissException if level is not supported.
+    static void set_level(SIMDLevel level);
+    static SIMDLevel get_level();
+    static std::string get_level_name();
+
+    /// Check if a SIMD level is available (compiled in).
+    static bool is_simd_level_available(SIMDLevel level);
+
+    /// Returns the SIMD level via the dispatch mechanism.
+    /// In DD mode, uses DISPATCH_SIMDLevel internally.
+    /// In static mode, returns the compiled-in level.
+    /// Useful for verification: get_level() == get_dispatched_level()
+    static SIMDLevel get_dispatched_level();
+};
+
+} // namespace faiss

data/vendor/faiss/faiss/utils/simdlib_avx2.h

@@ -794,10 +794,21 @@ inline simd8float32 getlow128(simd8float32 a, simd8float32 b) {
     return simd8float32(_mm256_permute2f128_ps(a.f, b.f, 0 | 2 << 4));
 }
 
-inline simd8float32 gethigh128(simd8float32 a, simd8float32 b) {
+inline simd8float32 gethigh128(const simd8float32& a, const simd8float32& b) {
     return simd8float32(_mm256_permute2f128_ps(a.f, b.f, 1 | 3 << 4));
 }
 
+// horizontal add: sum all 8 floats in the register
+inline float horizontal_add(const simd8float32& a) {
+    __m128 sum = _mm_add_ps(
+            _mm256_castps256_ps128(a.f), _mm256_extractf128_ps(a.f, 1));
+    __m128 v0 = _mm_shuffle_ps(sum, sum, _MM_SHUFFLE(0, 0, 3, 2));
+    __m128 v1 = _mm_add_ps(sum, v0);
+    __m128 v2 = _mm_shuffle_ps(v1, v1, _MM_SHUFFLE(0, 0, 0, 1));
+    __m128 v3 = _mm_add_ps(v1, v2);
+    return _mm_cvtss_f32(v3);
+}
+
 } // namespace
 
 } // namespace faiss

data/vendor/faiss/faiss/utils/simdlib_avx512.h

@@ -293,4 +293,73 @@ struct simd64uint8 : simd512bit {
     }
 };
 
+/// vector of 16 32-bit floats
+struct simd16float32 : simd512bit {
+    simd16float32() {}
+
+    explicit simd16float32(__m512 f) : simd512bit(f) {}
+
+    explicit simd16float32(float x) : simd512bit(_mm512_set1_ps(x)) {}
+
+    explicit simd16float32(const float* x) : simd512bit(_mm512_loadu_ps(x)) {}
+
+    void clear() {
+        f = _mm512_setzero_ps();
+    }
+
+    void storeu(float* ptr) const {
+        _mm512_storeu_ps(ptr, f);
+    }
+
+    void loadu(const float* ptr) {
+        f = _mm512_loadu_ps(ptr);
+    }
+
+    void store(float* ptr) const {
+        _mm512_storeu_ps(ptr, f);
+    }
+
+    simd16float32 operator*(const simd16float32& other) const {
+        return simd16float32(_mm512_mul_ps(f, other.f));
+    }
+
+    simd16float32 operator+(const simd16float32& other) const {
+        return simd16float32(_mm512_add_ps(f, other.f));
+    }
+
+    simd16float32 operator-(const simd16float32& other) const {
+        return simd16float32(_mm512_sub_ps(f, other.f));
+    }
+
+    simd16float32& operator+=(const simd16float32& other) {
+        f = _mm512_add_ps(f, other.f);
+        return *this;
+    }
+
+    std::string tostring() const {
+        float tab[16];
+        storeu(tab);
+        char res[1000];
+        char* ptr = res;
+        for (int i = 0; i < 16; i++) {
+            ptr += sprintf(ptr, "%g,", tab[i]);
+        }
+        ptr[-1] = 0;
+        return std::string(res);
+    }
+};
+
+// compute a * b + c
+inline simd16float32 fmadd(
+        const simd16float32& a,
+        const simd16float32& b,
+        const simd16float32& c) {
+    return simd16float32(_mm512_fmadd_ps(a.f, b.f, c.f));
+}
+
+// horizontal add: sum all 16 floats in the register
+inline float horizontal_add(const simd16float32& a) {
+    return _mm512_reduce_add_ps(a.f);
+}
+
 } // namespace faiss

data/vendor/faiss/faiss/utils/simdlib_neon.h

@@ -1355,6 +1355,12 @@ simd8float32 gethigh128(const simd8float32& a, const simd8float32& b) {
     return simd8float32{float32x4x2_t{a.data.val[1], b.data.val[1]}};
 }
 
+// horizontal add: sum all 8 floats in the register
+inline float horizontal_add(const simd8float32& a) {
+    float32x4_t sum = vaddq_f32(a.data.val[0], a.data.val[1]);
+    return vaddvq_f32(sum);
+}
+
 } // namespace
 
 } // namespace faiss

data/vendor/faiss/faiss/utils/sorting.cpp

@@ -795,15 +795,15 @@ void hashtable_int64_to_int64_lookup(
 #pragma omp parallel for
     for (int64_t i = 0; i < n; i++) {
         int64_t k = keys[i];
-        int64_t hk = hash_function(k) & mask;
-        size_t slot = hk;
+        int64_t hashValue = hash_function(k) & mask;
+        size_t slot = hashValue;
 
         if (tab[2 * slot] == -1) { // not in table
             vals[i] = -1;
         } else if (tab[2 * slot] == k) { // found!
             vals[i] = tab[2 * slot + 1];
         } else { // need to search in [k0, k1)
-            size_t bucket = hk >> (log2_capacity - log2_nbucket);
+            size_t bucket = hashValue >> (log2_capacity - log2_nbucket);
             size_t k0 = bucket << (log2_capacity - log2_nbucket);
             size_t k1 = (bucket + 1) << (log2_capacity - log2_nbucket);
             for (;;) {
@@ -815,7 +815,7 @@ void hashtable_int64_to_int64_lookup(
                 if (slot == k1) {
                     slot = k0;
                 }
-                if (slot == hk) { // bucket is full and not found
+                if (slot == hashValue) { // bucket is full and not found
                     vals[i] = -1;
                     break;
                 }

data/vendor/faiss/faiss/utils/utils.cpp

@@ -8,6 +8,7 @@
 // -*- c++ -*-
 
 #include <faiss/Index.h>
+#include <faiss/utils/simd_levels.h>
 #include <faiss/utils/utils.h>
 
 #include <cassert>
@@ -115,16 +116,22 @@ std::string get_compile_options() {
     options += "OPTIMIZE ";
 #endif
 
-#ifdef __AVX512F__
-    options += "AVX512 ";
-#elif defined(__AVX2__)
-    options += "AVX2 ";
-#elif defined(__ARM_FEATURE_SVE)
-    options += "SVE NEON ";
-#elif defined(__aarch64__)
-    options += "NEON ";
+#ifdef FAISS_ENABLE_DD
+    // Dynamic Dispatch mode: report DD and all available SIMD levels
+    options += "DD ";
+    int supported = SIMDConfig::supported_simd_levels;
+    for (int i = 0; i < static_cast<int>(SIMDLevel::COUNT); ++i) {
+        auto level = static_cast<SIMDLevel>(i);
+        if ((supported & (1 << i)) && level != SIMDLevel::NONE) {
+            options += to_string(level) + " ";
+        }
+    }
 #else
-    options += "GENERIC ";
+    // Static mode: report the compiled-in SIMD level
+    SIMDLevel level = SIMDConfig::get_level();
+    if (level != SIMDLevel::NONE) {
+        options += to_string(level) + " ";
+    }
 #endif
 
 #ifdef FAISS_ENABLE_SVS

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -15,28 +15,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.8'
+        version: 4.11.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.8'
+        version: 4.11.5
 - !ruby/object:Gem::Dependency
-  name: numo-narray
+  name: numo-narray-alt
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
 email: andrew@ankane.org
 executables: []
 extensions:
@@ -48,14 +48,14 @@ files:
 - README.md
 - ext/faiss/ext.cpp
 - ext/faiss/extconf.rb
-- ext/faiss/index.cpp
 - ext/faiss/index_binary.cpp
+- ext/faiss/index_rb.cpp
 - ext/faiss/kmeans.cpp
 - ext/faiss/numo.hpp
 - ext/faiss/pca_matrix.cpp
 - ext/faiss/product_quantizer.cpp
-- ext/faiss/utils.cpp
-- ext/faiss/utils.h
+- ext/faiss/utils_rb.cpp
+- ext/faiss/utils_rb.h
 - lib/faiss.rb
 - lib/faiss/version.rb
 - vendor/faiss/LICENSE
@@ -165,8 +165,6 @@ files:
 - vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h
 - vendor/faiss/faiss/cppcontrib/detail/CoarseBitType.h
 - vendor/faiss/faiss/cppcontrib/detail/UintReader.h
-- vendor/faiss/faiss/cppcontrib/factory_tools.cpp
-- vendor/faiss/faiss/cppcontrib/factory_tools.h
 - vendor/faiss/faiss/cppcontrib/sa_decode/Level2-avx2-inl.h
 - vendor/faiss/faiss/cppcontrib/sa_decode/Level2-inl.h
 - vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h
@@ -175,6 +173,8 @@ files:
 - vendor/faiss/faiss/cppcontrib/sa_decode/PQ-avx2-inl.h
 - vendor/faiss/faiss/cppcontrib/sa_decode/PQ-inl.h
 - vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h
+- vendor/faiss/faiss/factory_tools.cpp
+- vendor/faiss/faiss/factory_tools.h
 - vendor/faiss/faiss/gpu/GpuAutoTune.cpp
 - vendor/faiss/faiss/gpu/GpuAutoTune.h
 - vendor/faiss/faiss/gpu/GpuCloner.cpp
@@ -231,8 +231,12 @@ files:
 - vendor/faiss/faiss/impl/AdditiveQuantizer.h
 - vendor/faiss/faiss/impl/AuxIndexStructures.cpp
 - vendor/faiss/faiss/impl/AuxIndexStructures.h
+- vendor/faiss/faiss/impl/ClusteringInitialization.cpp
+- vendor/faiss/faiss/impl/ClusteringInitialization.h
 - vendor/faiss/faiss/impl/CodePacker.cpp
 - vendor/faiss/faiss/impl/CodePacker.h
+- vendor/faiss/faiss/impl/CodePackerRaBitQ.cpp
+- vendor/faiss/faiss/impl/CodePackerRaBitQ.h
 - vendor/faiss/faiss/impl/DistanceComputer.h
 - vendor/faiss/faiss/impl/FaissAssert.h
 - vendor/faiss/faiss/impl/FaissException.cpp
@@ -276,11 +280,9 @@ files:
 - vendor/faiss/faiss/impl/ScalarQuantizer.h
 - vendor/faiss/faiss/impl/ThreadedIndex-inl.h
 - vendor/faiss/faiss/impl/ThreadedIndex.h
-- vendor/faiss/faiss/impl/code_distance/code_distance-avx2.h
-- vendor/faiss/faiss/impl/code_distance/code_distance-avx512.h
-- vendor/faiss/faiss/impl/code_distance/code_distance-generic.h
-- vendor/faiss/faiss/impl/code_distance/code_distance-sve.h
-- vendor/faiss/faiss/impl/code_distance/code_distance.h
+- vendor/faiss/faiss/impl/VisitedTable.cpp
+- vendor/faiss/faiss/impl/VisitedTable.h
+- vendor/faiss/faiss/impl/expanded_scanners.h
 - vendor/faiss/faiss/impl/index_read.cpp
 - vendor/faiss/faiss/impl/index_read_utils.h
 - vendor/faiss/faiss/impl/index_write.cpp
@@ -299,8 +301,25 @@ files:
 - vendor/faiss/faiss/impl/pq4_fast_scan.h
 - vendor/faiss/faiss/impl/pq4_fast_scan_search_1.cpp
 - vendor/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp
+- vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx2.cpp
+- vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx512.cpp
+- vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp
+- vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h
+- vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp
 - vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h
+- vendor/faiss/faiss/impl/scalar_quantizer/codecs.h
+- vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h
+- vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h
+- vendor/faiss/faiss/impl/scalar_quantizer/scanners.h
+- vendor/faiss/faiss/impl/scalar_quantizer/similarities.h
+- vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h
+- vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/training.cpp
+- vendor/faiss/faiss/impl/scalar_quantizer/training.h
+- vendor/faiss/faiss/impl/simd_dispatch.h
 - vendor/faiss/faiss/impl/simd_result_handlers.h
 - vendor/faiss/faiss/impl/svs_io.cpp
 - vendor/faiss/faiss/impl/svs_io.h
@@ -345,6 +364,7 @@ files:
 - vendor/faiss/faiss/utils/bf16.h
 - vendor/faiss/faiss/utils/distances.cpp
 - vendor/faiss/faiss/utils/distances.h
+- vendor/faiss/faiss/utils/distances_dispatch.h
 - vendor/faiss/faiss/utils/distances_fused/avx512.cpp
 - vendor/faiss/faiss/utils/distances_fused/avx512.h
 - vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp
@@ -371,12 +391,21 @@ files:
 - vendor/faiss/faiss/utils/ordered_key_value.h
 - vendor/faiss/faiss/utils/partitioning.cpp
 - vendor/faiss/faiss/utils/partitioning.h
+- vendor/faiss/faiss/utils/pq_code_distance.h
 - vendor/faiss/faiss/utils/prefetch.h
 - vendor/faiss/faiss/utils/quantize_lut.cpp
 - vendor/faiss/faiss/utils/quantize_lut.h
 - vendor/faiss/faiss/utils/rabitq_simd.h
 - vendor/faiss/faiss/utils/random.cpp
 - vendor/faiss/faiss/utils/random.h
+- vendor/faiss/faiss/utils/simd_impl/distances_aarch64.cpp
+- vendor/faiss/faiss/utils/simd_impl/distances_arm_sve.cpp
+- vendor/faiss/faiss/utils/simd_impl/distances_autovec-inl.h
+- vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp
+- vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp
+- vendor/faiss/faiss/utils/simd_impl/distances_sse-inl.h
+- vendor/faiss/faiss/utils/simd_levels.cpp
+- vendor/faiss/faiss/utils/simd_levels.h
 - vendor/faiss/faiss/utils/simdlib.h
 - vendor/faiss/faiss/utils/simdlib_avx2.h
 - vendor/faiss/faiss/utils/simdlib_avx512.h
@@ -400,14 +429,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Efficient similarity search and clustering for Ruby
 test_files: []

data/vendor/faiss/faiss/impl/code_distance/code_distance-generic.h

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-
-namespace faiss {
-
-/// Returns the distance to a single code.
-template <typename PQDecoderT>
-inline float distance_single_code_generic(
-        // number of subquantizers
-        const size_t M,
-        // number of bits per quantization index
-        const size_t nbits,
-        // precomputed distances, layout (M, ksub)
-        const float* sim_table,
-        // the code
-        const uint8_t* code) {
-    PQDecoderT decoder(code, nbits);
-    const size_t ksub = 1 << nbits;
-
-    const float* tab = sim_table;
-    float result = 0;
-
-    for (size_t m = 0; m < M; m++) {
-        result += tab[decoder.decode()];
-        tab += ksub;
-    }
-
-    return result;
-}
-
-/// Combines 4 operations of distance_single_code()
-/// General-purpose version.
-template <typename PQDecoderT>
-inline void distance_four_codes_generic(
-        // number of subquantizers
-        const size_t M,
-        // number of bits per quantization index
-        const size_t nbits,
-        // precomputed distances, layout (M, ksub)
-        const float* sim_table,
-        // codes
-        const uint8_t* __restrict code0,
-        const uint8_t* __restrict code1,
-        const uint8_t* __restrict code2,
-        const uint8_t* __restrict code3,
-        // computed distances
-        float& result0,
-        float& result1,
-        float& result2,
-        float& result3) {
-    PQDecoderT decoder0(code0, nbits);
-    PQDecoderT decoder1(code1, nbits);
-    PQDecoderT decoder2(code2, nbits);
-    PQDecoderT decoder3(code3, nbits);
-    const size_t ksub = 1 << nbits;
-
-    const float* tab = sim_table;
-    result0 = 0;
-    result1 = 0;
-    result2 = 0;
-    result3 = 0;
-
-    for (size_t m = 0; m < M; m++) {
-        result0 += tab[decoder0.decode()];
-        result1 += tab[decoder1.decode()];
-        result2 += tab[decoder2.decode()];
-        result3 += tab[decoder3.decode()];
-        tab += ksub;
-    }
-}
-
-} // namespace faiss