faiss 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e9bd037bbb04bb31eb1856073a7d01d9b1871f1005c2f81b2b3d48c72f737a2
-  data.tar.gz: bfde828c3c7780e2cba3eac0db39ab805c1892e3fb07968870097051ebc7b713
+  metadata.gz: e41b15bbcda6c4d2a250df5b98d86e9baf51b34b90fc2fccb6f0a37f486ef417
+  data.tar.gz: 768074275062ed45f1752e3a5c9d55a9695a6aa453e925aa0a6e607ce3215bab
 SHA512:
-  metadata.gz: c7dd39002fa4f463c552b75fcfeed505816784ca986e65a46826d7982f9e8f9761750931b7e472ef3406d851c63e2038ef65370cd1c0d54113eb556190fb8c65
-  data.tar.gz: 2f1fc38577e089b9a817feabe7a794354a385ca4cf99887253db12a88cfeb39c0ba456a44191b16680158e2c30d1356efa909df57a2823376036f47e25aed0b1
+  metadata.gz: cecc466dd24e03206219b63e750e48b554355c1c5dfc8e911879988a6f31eb628617133f5b584b3de29efcbe65d087cf5b4e219371cee959e8248c989a4dbffc
+  data.tar.gz: 3e0c6be53825949f9c51a0195d85cbed87bc198dd06852c88c537b13e6bcc8e7fa65a3e3c88667eefef44e95278fe2c73ece89d5f92bd24f8c0d27b543488b56

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.3.1 (2024-03-13)
+
+- Updated Faiss to 1.8.0
+- Fixed memory leak with `load` and `index_binary_factory` methods
+
 ## 0.3.0 (2023-05-11)
 
 - Fixed error on Fedora

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) Facebook, Inc. and its affiliates.
-Copyright (c) 2020-2023 Andrew Kane
+Copyright (c) 2020-2024 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -4,7 +4,7 @@
 
 Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
 
-[![Build Status](https://github.com/ankane/faiss-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss-ruby/actions)
+[![Build Status](https://github.com/ankane/faiss-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/faiss-ruby/actions)
 
 ## Installation
 

data/ext/faiss/extconf.rb

@@ -19,9 +19,16 @@ abort "Numo not found" unless find_header("numo/narray.h", numo)
 # for https://bugs.ruby-lang.org/issues/19005
 $LDFLAGS += " -Wl,-undefined,dynamic_lookup" if RbConfig::CONFIG["host_os"] =~ /darwin/i
 
+$CXXFLAGS += " -std=c++17 $(optflags) -DFINTEGER=int"
+$CXXFLAGS += " -Wall -Wno-unused-parameter -Wno-unused-function -Wno-unused-variable -Wno-deprecated-declarations -Wno-sign-compare"
+
 # -march=native not supported with ARM Mac
-default_optflags = RbConfig::CONFIG["host_os"] =~ /darwin/i && RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i ? "" : "-march=native"
-$CXXFLAGS << " -std=c++17 $(optflags) -DFINTEGER=int " << with_config("optflags", default_optflags)
+default_optflags = RbConfig::CONFIG["host_os"] =~ /darwin/i && RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i ? "" : " -march=native"
+$CXXFLAGS += with_config("optflags", default_optflags)
+
+apple_clang = RbConfig::CONFIG["CC_VERSION_MESSAGE"] =~ /apple clang/i
+$CXXFLAGS += " -Xclang" if apple_clang
+$CXXFLAGS += " -fopenmp"
 
 ext = File.expand_path(".", __dir__)
 vendor = File.expand_path("../../vendor/faiss", __dir__)

data/ext/faiss/index.cpp

@@ -157,7 +157,7 @@ void init_index(Rice::Module& m) {
       "load",
       [](Rice::String fname) {
         return faiss::read_index(fname.c_str());
-      });
+      }, Rice::Return().takeOwnership());
 
   Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(m, "IndexFlatL2")
     .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());

data/ext/faiss/index_binary.cpp

@@ -59,7 +59,7 @@ void init_index_binary(Rice::Module& m) {
       "load",
       [](Rice::String fname) {
         return faiss::read_index_binary(fname.c_str());
-      });
+      }, Rice::Return().takeOwnership());
 
   Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(m, "IndexBinaryFlat")
     .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
@@ -71,5 +71,5 @@ void init_index_binary(Rice::Module& m) {
     "index_binary_factory",
     [](int d, Rice::String description) {
       return faiss::index_binary_factory(d, description.c_str());
-    });
+    }, Rice::Return().takeOwnership());
 }

data/ext/faiss/product_quantizer.cpp

@@ -49,5 +49,5 @@ void init_product_quantizer(Rice::Module& m) {
       "load",
       [](Rice::String fname) {
         return faiss::read_ProductQuantizer(fname.c_str());
-      });
+      }, Rice::Return().takeOwnership());
 }

data/lib/faiss/version.rb

@@ -1,3 +1,3 @@
 module Faiss
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/vendor/faiss/faiss/AutoTune.cpp

@@ -152,12 +152,10 @@ bool OperatingPoints::add(
             return false;
         }
     }
-    { // remove non-optimal points from array
-        int i = a.size() - 1;
-        while (i > 0) {
-            if (a[i].t < a[i - 1].t)
-                a.erase(a.begin() + (i - 1));
-            i--;
+    // remove non-optimal points from array
+    for (int i = a.size() - 1; i > 0; --i) {
+        if (a[i].t < a[i - 1].t) {
+            a.erase(a.begin() + (i - 1));
         }
     }
     return true;
@@ -286,6 +284,8 @@ std::string ParameterSpace::combination_name(size_t cno) const {
     char buf[1000], *wp = buf;
     *wp = 0;
     for (int i = 0; i < parameter_ranges.size(); i++) {
+        FAISS_THROW_IF_NOT_MSG(
+                buf + 1000 - wp >= 0, "Overflow detected in snprintf");
         const ParameterRange& pr = parameter_ranges[i];
         size_t j = cno % pr.values.size();
         cno /= pr.values.size();
@@ -334,7 +334,7 @@ ParameterRange& ParameterSpace::add_range(const std::string& name) {
             return pr;
         }
     }
-    parameter_ranges.push_back(ParameterRange());
+    parameter_ranges.emplace_back();
     parameter_ranges.back().name = name;
     return parameter_ranges.back();
 }

data/vendor/faiss/faiss/AutoTune.h

@@ -11,7 +11,6 @@
 #define FAISS_AUTO_TUNE_H
 
 #include <stdint.h>
-#include <unordered_map>
 #include <vector>
 
 #include <faiss/Index.h>

data/vendor/faiss/faiss/Clustering.cpp

@@ -27,20 +27,6 @@
 
 namespace faiss {
 
-ClusteringParameters::ClusteringParameters()
-        : niter(25),
-          nredo(1),
-          verbose(false),
-          spherical(false),
-          int_centroids(false),
-          update_index(false),
-          frozen_centroids(false),
-          min_points_per_centroid(39),
-          max_points_per_centroid(256),
-          seed(1234),
-          decode_block_size(32768) {}
-// 39 corresponds to 10000 / 256 -> to avoid warnings on PQ tests with randu10k
-
 Clustering::Clustering(int d, int k) : d(d), k(k) {}
 
 Clustering::Clustering(int d, int k, const ClusteringParameters& cp)
@@ -231,7 +217,7 @@ int split_clusters(
     for (size_t ci = 0; ci < k; ci++) {
         if (hassign[ci] == 0) { /* need to redefine a centroid */
             size_t cj;
-            for (cj = 0; 1; cj = (cj + 1) % k) {
+            for (cj = 0; true; cj = (cj + 1) % k) {
                 /* probability to pick this cluster for split */
                 float p = (hassign[cj] - 1.0) / (float)(n - k);
                 float r = rng.rand_float();
@@ -264,7 +250,7 @@ int split_clusters(
     return nsplit;
 }
 
-}; // namespace
+} // namespace
 
 void Clustering::train_encoded(
         idx_t nx,
@@ -590,7 +576,7 @@ float kmeans_clustering(
         const float* x,
         float* centroids) {
     Clustering clus(d, k);
-    clus.verbose = d * n * k > (1L << 30);
+    clus.verbose = d * n * k > (size_t(1) << 30);
     // display logs if > 1Gflop per iteration
     IndexFlatL2 index(d);
     clus.train(n, x, index);
@@ -631,7 +617,7 @@ void copy_columns(idx_t n, idx_t d1, const float* src, idx_t d2, float* dest) {
     }
 }
 
-}; // namespace
+} // namespace
 
 void ProgressiveDimClustering::train(
         idx_t n,

data/vendor/faiss/faiss/Clustering.h

@@ -5,7 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
+/** Implementation of k-means clustering with many variants. */
 
 #ifndef FAISS_CLUSTERING_H
 #define FAISS_CLUSTERING_H
@@ -19,25 +19,35 @@ namespace faiss {
  * constructor of the Clustering object.
  */
 struct ClusteringParameters {
-    int niter; ///< clustering iterations
-    int nredo; ///< redo clustering this many times and keep best
-
-    bool verbose;
-    bool spherical;        ///< do we want normalized centroids?
-    bool int_centroids;    ///< round centroids coordinates to integer
-    bool update_index;     ///< re-train index after each iteration?
-    bool frozen_centroids; ///< use the centroids provided as input and do not
-                           ///< change them during iterations
-
-    int min_points_per_centroid; ///< otherwise you get a warning
-    int max_points_per_centroid; ///< to limit size of dataset
-
-    int seed; ///< seed for the random number generator
-
-    size_t decode_block_size; ///< how many vectors at a time to decode
-
-    /// sets reasonable defaults
-    ClusteringParameters();
+    /// number of clustering iterations
+    int niter = 25;
+    /// redo clustering this many times and keep the clusters with the best
+    /// objective
+    int nredo = 1;
+
+    bool verbose = false;
+    /// whether to normalize centroids after each iteration (useful for inner
+    /// product clustering)
+    bool spherical = false;
+    /// round centroids coordinates to integer after each iteration?
+    bool int_centroids = false;
+    /// re-train index after each iteration?
+    bool update_index = false;
+
+    /// Use the subset of centroids provided as input and do not change them
+    /// during iterations
+    bool frozen_centroids = false;
+    /// If fewer than this number of training vectors per centroid are provided,
+    /// writes a warning. Note that fewer than 1 point per centroid raises an
+    /// exception.
+    int min_points_per_centroid = 39;
+    /// to limit size of dataset, otherwise the training set is subsampled
+    int max_points_per_centroid = 256;
+    /// seed for the random number generator
+    int seed = 1234;
+
+    /// when the training set is encoded, batch size of the codec decoder
+    size_t decode_block_size = 32768;
 };
 
 struct ClusteringIterationStats {
@@ -94,7 +104,7 @@ struct Clustering : ClusteringParameters {
      * to decode the input vectors.
      *
      * @param codec      codec used to decode the vectors (nullptr =
-     *                   vectors are in fact floats)     *
+     *                   vectors are in fact floats)
      */
     void train_encoded(
             idx_t nx,

data/vendor/faiss/faiss/IVFlib.cpp

@@ -12,7 +12,9 @@
 
 #include <faiss/IndexAdditiveQuantizer.h>
 #include <faiss/IndexIVFAdditiveQuantizer.h>
+#include <faiss/IndexIVFIndependentQuantizer.h>
 #include <faiss/IndexPreTransform.h>
+#include <faiss/IndexRefine.h>
 #include <faiss/MetaIndexes.h>
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/utils/distances.h>
@@ -57,20 +59,29 @@ void check_compatible_for_merge(const Index* index0, const Index* index1) {
 }
 
 const IndexIVF* try_extract_index_ivf(const Index* index) {
-    if (auto* pt = dynamic_cast<const IndexPreTransform*>(index)) {
-        index = pt->index;
+    auto* ivf = dynamic_cast<const IndexIVF*>(index);
+    if (ivf != nullptr) {
+        return ivf;
     }
 
+    if (auto* pt = dynamic_cast<const IndexPreTransform*>(index)) {
+        return try_extract_index_ivf(pt->index);
+    }
     if (auto* idmap = dynamic_cast<const IndexIDMap*>(index)) {
-        index = idmap->index;
+        return try_extract_index_ivf(idmap->index);
     }
     if (auto* idmap = dynamic_cast<const IndexIDMap2*>(index)) {
-        index = idmap->index;
+        return try_extract_index_ivf(idmap->index);
+    }
+    if (auto* indep =
+                dynamic_cast<const IndexIVFIndependentQuantizer*>(index)) {
+        return try_extract_index_ivf(indep->index_ivf);
+    }
+    if (auto* refine = dynamic_cast<const IndexRefine*>(index)) {
+        return try_extract_index_ivf(refine->base_index);
     }
 
-    auto* ivf = dynamic_cast<const IndexIVF*>(index);
-
-    return ivf;
+    return nullptr;
 }
 
 IndexIVF* try_extract_index_ivf(Index* index) {
@@ -321,14 +332,14 @@ void search_with_parameters(
         double* ms_per_stage) {
     FAISS_THROW_IF_NOT(params);
     const float* prev_x = x;
-    ScopeDeleter<float> del;
+    std::unique_ptr<const float[]> del;
 
     double t0 = getmillisecs();
 
     if (auto ip = dynamic_cast<const IndexPreTransform*>(index)) {
         x = ip->apply_chain(n, x);
         if (x != prev_x) {
-            del.set(x);
+            del.reset(x);
         }
         index = ip->index;
     }
@@ -371,14 +382,14 @@ void range_search_with_parameters(
         double* ms_per_stage) {
     FAISS_THROW_IF_NOT(params);
     const float* prev_x = x;
-    ScopeDeleter<float> del;
+    std::unique_ptr<const float[]> del;
 
     double t0 = getmillisecs();
 
     if (auto ip = dynamic_cast<const IndexPreTransform*>(index)) {
         x = ip->apply_chain(n, x);
         if (x != prev_x) {
-            del.set(x);
+            del.reset(x);
         }
         index = ip->index;
     }

data/vendor/faiss/faiss/Index.cpp

@@ -18,7 +18,7 @@
 
 namespace faiss {
 
-Index::~Index() {}
+Index::~Index() = default;
 
 void Index::train(idx_t /*n*/, const float* /*x*/) {
     // does nothing by default

data/vendor/faiss/faiss/Index.h

@@ -17,8 +17,8 @@
 #include <typeinfo>
 
 #define FAISS_VERSION_MAJOR 1
-#define FAISS_VERSION_MINOR 7
-#define FAISS_VERSION_PATCH 4
+#define FAISS_VERSION_MINOR 8
+#define FAISS_VERSION_PATCH 0
 
 /**
  * @namespace faiss
@@ -99,6 +99,7 @@ struct Index {
      * Vectors are implicitly assigned labels ntotal .. ntotal + n - 1
      * This function slices the input vectors in chunks smaller than
      * blocksize_add and calls add_core.
+     * @param n      number of vectors
      * @param x      input matrix, size n * d
      */
     virtual void add(idx_t n, const float* x) = 0;
@@ -108,7 +109,9 @@ struct Index {
      * The default implementation fails with an assertion, as it is
      * not supported by all indexes.
      *
-     * @param xids if non-null, ids to store for the vectors (size n)
+     * @param n         number of vectors
+     * @param x         input vectors, size n * d
+     * @param xids      if non-null, ids to store for the vectors (size n)
      */
     virtual void add_with_ids(idx_t n, const float* x, const idx_t* xids);
 
@@ -117,9 +120,11 @@ struct Index {
      * return at most k vectors. If there are not enough results for a
      * query, the result array is padded with -1s.
      *
+     * @param n           number of vectors
      * @param x           input vectors to search, size n * d
-     * @param labels      output labels of the NNs, size n*k
+     * @param k           number of extracted vectors
      * @param distances   output pairwise distances, size n*k
+     * @param labels      output labels of the NNs, size n*k
      */
     virtual void search(
             idx_t n,
@@ -135,6 +140,7 @@ struct Index {
      * indexes do not implement the range_search (only the k-NN search
      * is mandatory).
      *
+     * @param n           number of vectors
      * @param x           input vectors to search, size n * d
      * @param radius      search radius
      * @param result      result table
@@ -149,8 +155,10 @@ struct Index {
     /** return the indexes of the k vectors closest to the query x.
      *
      * This function is identical as search but only return labels of neighbors.
+     * @param n           number of vectors
      * @param x           input vectors to search, size n * d
      * @param labels      output labels of the NNs, size n*k
+     * @param k           number of nearest neighbours
      */
     virtual void assign(idx_t n, const float* x, idx_t* labels, idx_t k = 1)
             const;
@@ -174,7 +182,7 @@ struct Index {
     /** Reconstruct several stored vectors (or an approximation if lossy coding)
      *
      * this function may not be defined for some indexes
-     * @param n        number of vectors to reconstruct
+     * @param n           number of vectors to reconstruct
      * @param keys        ids of the vectors to reconstruct (size n)
      * @param recons      reconstucted vector (size n * d)
      */
@@ -184,6 +192,8 @@ struct Index {
     /** Reconstruct vectors i0 to i0 + ni - 1
      *
      * this function may not be defined for some indexes
+     * @param i0          index of the first vector in the sequence
+     * @param ni          number of vectors in the sequence
      * @param recons      reconstucted vector (size ni * d)
      */
     virtual void reconstruct_n(idx_t i0, idx_t ni, float* recons) const;
@@ -194,6 +204,11 @@ struct Index {
      * If there are not enough results for a query, the resulting arrays
      * is padded with -1s.
      *
+     * @param n           number of vectors
+     * @param x           input vectors to search, size n * d
+     * @param k           number of extracted vectors
+     * @param distances   output pairwise distances, size n*k
+     * @param labels      output labels of the NNs, size n*k
      * @param recons      reconstructed vectors size (n, k, d)
      **/
     virtual void search_and_reconstruct(

data/vendor/faiss/faiss/Index2Layer.cpp

@@ -10,10 +10,10 @@
 #include <faiss/Index2Layer.h>
 
 #include <faiss/impl/platform_macros.h>
-#include <stdint.h>
 #include <cassert>
 #include <cinttypes>
 #include <cmath>
+#include <cstdint>
 #include <cstdio>
 
 #ifdef __SSE3__
@@ -47,7 +47,7 @@ Index2Layer::Index2Layer(
           pq(quantizer->d, M, nbit) {
     is_trained = false;
     for (int nbyte = 0; nbyte < 7; nbyte++) {
-        if ((1L << (8 * nbyte)) >= nlist) {
+        if (((size_t)1 << (8 * nbyte)) >= nlist) {
             code_size_1 = nbyte;
             break;
         }
@@ -60,7 +60,7 @@ Index2Layer::Index2Layer() {
     code_size = code_size_1 = code_size_2 = 0;
 }
 
-Index2Layer::~Index2Layer() {}
+Index2Layer::~Index2Layer() = default;
 
 void Index2Layer::train(idx_t n, const float* x) {
     if (verbose) {
@@ -83,7 +83,7 @@ void Index2Layer::train(idx_t n, const float* x) {
             verbose,
             pq.cp.seed);
 
-    ScopeDeleter<float> del_x(x_in == x ? nullptr : x);
+    std::unique_ptr<const float[]> del_x(x_in == x ? nullptr : x);
 
     std::vector<idx_t> assign(n); // assignement to coarse centroids
     q1.quantizer->assign(n, x, assign.data());
@@ -179,7 +179,7 @@ struct DistanceXPQ4 : Distance2Level {
     float operator()(idx_t i) override {
 #ifdef __SSE3__
         const uint8_t* code = storage.codes.data() + i * storage.code_size;
-        long key = 0;
+        idx_t key = 0;
         memcpy(&key, code, storage.code_size_1);
         code += storage.code_size_1;
 
@@ -225,7 +225,7 @@ struct Distance2xXPQ4 : Distance2Level {
 
     float operator()(idx_t i) override {
         const uint8_t* code = storage.codes.data() + i * storage.code_size;
-        long key01 = 0;
+        int64_t key01 = 0;
         memcpy(&key01, code, storage.code_size_1);
         code += storage.code_size_1;
 #ifdef __SSE3__
@@ -237,7 +237,7 @@ struct Distance2xXPQ4 : Distance2Level {
         __m128 accu = _mm_setzero_ps();
 
         for (int mi_m = 0; mi_m < 2; mi_m++) {
-            long l1_idx = key01 & ((1L << mi_nbits) - 1);
+            int64_t l1_idx = key01 & (((int64_t)1 << mi_nbits) - 1);
             const __m128* pq_l1 = pq_l1_t + M_2 * l1_idx;
 
             for (int m = 0; m < M_2; m++) {