faiss 0.2.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 106966f6d5e7f6a3f5237a2ebb59912304bbe319ba2891c45166c753fc0e9df1
-  data.tar.gz: 3d72777c777d75beb15c09cb223c4e63f188b7e89fab06d6fa9268bc2be4ff59
+  metadata.gz: bdc2effbe6e2d827ffe473531be0864cf710ae6f3ad34f8324087695c367d140
+  data.tar.gz: 6e5b80b1f4281766b17d208af58b44e768576d259de1c6edb25630c700215c10
 SHA512:
-  metadata.gz: b1d822fd4e0850dd667aba09a02f81c43e2afde90a6d88fff1e631539a8d47f42b4dfe0705980a6ff8be88b021af805998d1e70b09cd1010302702a1c1363cac
-  data.tar.gz: 24e4febd81142150541199523f0b1e19a0be7c658e504472778173e38006234ea47f7e362c1396c3135b4449b7ab5229dc027c4bcea59c880902e1fa23e3c956
+  metadata.gz: 404d064f14734a23d946ad0fc0576673e5e685d1001884981453e3bec23ecf6ba7b75857e578b47ef5303fde27342dfb25e85bbb7a60883d6ba09a964049bbfa
+  data.tar.gz: 536f403109d3773a3cd0ca27b4b46bfa26eaeedd8d5e2f4e6fa116e3b922d21f8a964a2d6aa1cbbff951da9444e9184bf7c60955263ed68c91ba91342f3f9196

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.2.5 (2022-12-27)
+
+- Updated Faiss to 1.7.3
+- Fixed installation errors on Mac
+
+## 0.2.4 (2022-01-10)
+
+- Updated Faiss to 1.7.2
+
 ## 0.2.3 (2021-12-17)
 
 - Fixed installation error with ARM Mac

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) Facebook, Inc. and its affiliates.
-Copyright (c) 2020-2021 Andrew Kane
+Copyright (c) 2020-2022 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -8,23 +8,25 @@ Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-
 
 ## Installation
 
-First, install BLAS, LAPACK, and OpenMP:
+First, ensure BLAS, LAPACK, and OpenMP are installed. For Mac, use:
 
 ```sh
-# Mac
-brew install openblas lapack libomp
+brew install libomp
+```
+
+For Ubuntu, use:
 
-# Ubuntu
-sudo apt install libblas-dev liblapack-dev
+```sh
+sudo apt-get install libblas-dev liblapack-dev
 ```
 
-Add this line to your application’s Gemfile:
+Then add this line to your application’s Gemfile:
 
 ```ruby
-gem 'faiss'
+gem "faiss"
 ```
 
-It can take a few minutes to compile the gem. Faiss is not available for Windows.
+It can take a few minutes to compile the gem. Windows is not currently supported.
 
 ## Getting Started
 
@@ -69,61 +71,61 @@ index = Faiss::Index.load("index.bin")
 
 Exact search for L2
 
-```rb
+```ruby
 Faiss::IndexFlatL2.new(d)
 ```
 
 Exact search for inner product
 
-```rb
+```ruby
 Faiss::IndexFlatIP.new(d)
 ```
 
 Hierarchical navigable small world graph exploration
 
-```rb
+```ruby
 Faiss::IndexHNSWFlat.new(d, m)
 ```
 
 Inverted file with exact post-verification
 
-```rb
+```ruby
 Faiss::IndexIVFFlat.new(quantizer, d, nlists)
 ```
 
 Locality-sensitive hashing
 
-```rb
+```ruby
 Faiss::IndexLSH.new(d, nbits)
 ```
 
 Scalar quantizer (SQ) in flat mode
 
-```rb
+```ruby
 Faiss::IndexScalarQuantizer.new(d, qtype)
 ```
 
 Product quantizer (PQ) in flat mode
 
-```rb
+```ruby
 Faiss::IndexPQ.new(d, m, nbits)
 ```
 
 IVF and scalar quantizer
 
-```rb
+```ruby
 Faiss::IndexIVFScalarQuantizer.new(quantizer, d, nlists, qtype)
 ```
 
 IVFADC (coarse quantizer+PQ on residuals)
 
-```rb
+```ruby
 Faiss::IndexIVFPQ.new(quantizer, d, nlists, m, nbits)
 ```
 
 IVFADC+R (same as IVFADC with re-ranking based on codes)
 
-```rb
+```ruby
 Faiss::IndexIVFPQR.new(quantizer, d, nlists, m, nbits, m_refine, nbits_refine)
 ```
 
@@ -131,13 +133,13 @@ Faiss::IndexIVFPQR.new(quantizer, d, nlists, m, nbits, m_refine, nbits_refine)
 
 Index binary vectors
 
-```rb
+```ruby
 Faiss::IndexBinaryFlat.new(d)
 ```
 
 Speed up search with an inverse vector file
 
-```rb
+```ruby
 Faiss::IndexBinaryIVF.new(quantizer, d, nlists)
 ```
 
@@ -220,7 +222,7 @@ Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
 
 ## History
 
-View the [changelog](https://github.com/ankane/faiss-ruby/blob/master/CHANGELOG.md)
+View the [changelog](CHANGELOG.md)
 
 ## Contributing
 

data/ext/faiss/extconf.rb

@@ -1,6 +1,14 @@
 require "mkmf-rice"
 require "numo/narray"
 
+# libomp changed to keg-only
+# https://github.com/Homebrew/homebrew-core/issues/112107
+if RbConfig::CONFIG["host_os"] =~ /darwin/i
+  brew_prefix = RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i ? "/opt/homebrew" : "/usr/local"
+  find_library("omp", nil, "#{brew_prefix}/opt/libomp/lib")
+  find_header("omp.h", "#{brew_prefix}/opt/libomp/include")
+end
+
 abort "BLAS not found" unless have_library("blas")
 abort "LAPACK not found" unless have_library("lapack")
 abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
@@ -8,6 +16,9 @@ abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
 numo = File.join(Gem.loaded_specs["numo-narray"].require_path, "numo")
 abort "Numo not found" unless find_header("numo/narray.h", numo)
 
+# for https://bugs.ruby-lang.org/issues/19005
+$LDFLAGS += " -Wl,-undefined,dynamic_lookup" if RbConfig::CONFIG["host_os"] =~ /darwin/i
+
 # -march=native not supported with ARM Mac
 default_optflags = RbConfig::CONFIG["host_os"] =~ /darwin/i && RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i ? "" : "-march=native"
 $CXXFLAGS << " -std=c++17 $(optflags) -DFINTEGER=int " << with_config("optflags", default_optflags)

data/ext/faiss/index.cpp

@@ -140,13 +140,13 @@ void init_index(Rice::Module& m) {
       })
     .define_method(
       "save",
-      [](faiss::Index &self, const char *fname) {
-        faiss::write_index(&self, fname);
+      [](faiss::Index &self, Rice::String fname) {
+        faiss::write_index(&self, fname.c_str());
       })
     .define_singleton_function(
       "load",
-      [](const char *fname) {
-        return faiss::read_index(fname);
+      [](Rice::String fname) {
+        return faiss::read_index(fname.c_str());
       });
 
   Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(m, "IndexFlatL2")

data/ext/faiss/index_binary.cpp

@@ -52,13 +52,13 @@ void init_index_binary(Rice::Module& m) {
       })
     .define_method(
       "save",
-      [](faiss::IndexBinary &self, const char *fname) {
-        faiss::write_index_binary(&self, fname);
+      [](faiss::IndexBinary &self, Rice::String fname) {
+        faiss::write_index_binary(&self, fname.c_str());
       })
     .define_singleton_function(
       "load",
-      [](const char *fname) {
-        return faiss::read_index_binary(fname);
+      [](Rice::String fname) {
+        return faiss::read_index_binary(fname.c_str());
       });
 
   Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(m, "IndexBinaryFlat")
@@ -69,7 +69,7 @@ void init_index_binary(Rice::Module& m) {
 
   m.define_singleton_function(
     "index_binary_factory",
-    [](int d, const char *description) {
-      return faiss::index_binary_factory(d, description);
+    [](int d, Rice::String description) {
+      return faiss::index_binary_factory(d, description.c_str());
     });
 }

data/ext/faiss/product_quantizer.cpp

@@ -42,12 +42,12 @@ void init_product_quantizer(Rice::Module& m) {
       })
     .define_method(
       "save",
-      [](faiss::ProductQuantizer &self, const char *fname) {
-        faiss::write_ProductQuantizer(&self, fname);
+      [](faiss::ProductQuantizer &self, Rice::String fname) {
+        faiss::write_ProductQuantizer(&self, fname.c_str());
       })
     .define_singleton_function(
       "load",
-      [](const char *fname) {
-        return faiss::read_ProductQuantizer(fname);
+      [](Rice::String fname) {
+        return faiss::read_ProductQuantizer(fname.c_str());
       });
 }

data/lib/faiss/version.rb

@@ -1,3 +1,3 @@
 module Faiss
-  VERSION = "0.2.3"
+  VERSION = "0.2.5"
 end

data/vendor/faiss/faiss/AutoTune.cpp

@@ -523,6 +523,19 @@ void ParameterSpace::set_index_parameter(
         }
     }
 
+    if (name == "efConstruction") {
+        if (DC(IndexHNSW)) {
+            ix->hnsw.efConstruction = int(val);
+            return;
+        }
+        if (DC(IndexIVF)) {
+            if (IndexHNSW* cq = dynamic_cast<IndexHNSW*>(ix->quantizer)) {
+                cq->hnsw.efConstruction = int(val);
+                return;
+            }
+        }
+    }
+
     if (name == "efSearch") {
         if (DC(IndexHNSW)) {
             ix->hnsw.efSearch = int(val);

data/vendor/faiss/faiss/Clustering.cpp

@@ -20,6 +20,7 @@
 
 #include <faiss/IndexFlat.h>
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/kmeans1d.h>
 #include <faiss/utils/distances.h>
 #include <faiss/utils/random.h>
 #include <faiss/utils/utils.h>
@@ -553,6 +554,37 @@ void Clustering::train_encoded(
     }
 }
 
+Clustering1D::Clustering1D(int k) : Clustering(1, k) {}
+
+Clustering1D::Clustering1D(int k, const ClusteringParameters& cp)
+        : Clustering(1, k, cp) {}
+
+void Clustering1D::train_exact(idx_t n, const float* x) {
+    const float* xt = x;
+
+    std::unique_ptr<uint8_t[]> del;
+    if (n > k * max_points_per_centroid) {
+        uint8_t* x_new;
+        float* weights_new;
+        n = subsample_training_set(
+                *this,
+                n,
+                (uint8_t*)x,
+                sizeof(float) * d,
+                nullptr,
+                &x_new,
+                &weights_new);
+        del.reset(x_new);
+        xt = (float*)x_new;
+    }
+
+    centroids.resize(k);
+    double uf = kmeans1d(xt, n, k, centroids.data());
+
+    ClusteringIterationStats stats = {0.0, 0.0, 0.0, uf, 0};
+    iteration_stats.push_back(stats);
+}
+
 float kmeans_clustering(
         size_t d,
         size_t n,

data/vendor/faiss/faiss/Clustering.h

@@ -111,6 +111,20 @@ struct Clustering : ClusteringParameters {
     virtual ~Clustering() {}
 };
 
+/** Exact 1D clustering algorithm
+ *
+ * Since it does not use an index, it does not overload the train() function
+ */
+struct Clustering1D : Clustering {
+    explicit Clustering1D(int k);
+
+    Clustering1D(int k, const ClusteringParameters& cp);
+
+    void train_exact(idx_t n, const float* x);
+
+    virtual ~Clustering1D() {}
+};
+
 struct ProgressiveDimClusteringParameters : ClusteringParameters {
     int progressive_dim_steps; ///< number of incremental steps
     bool apply_pca;            ///< apply PCA on input

data/vendor/faiss/faiss/IVFlib.cpp

@@ -5,15 +5,18 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
-
 #include <faiss/IVFlib.h>
+#include <omp.h>
 
 #include <memory>
 
+#include <faiss/IndexAdditiveQuantizer.h>
+#include <faiss/IndexIVFAdditiveQuantizer.h>
 #include <faiss/IndexPreTransform.h>
 #include <faiss/MetaIndexes.h>
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/utils/distances.h>
+#include <faiss/utils/hamming.h>
 #include <faiss/utils/utils.h>
 
 namespace faiss {
@@ -349,6 +352,7 @@ void search_with_parameters(
     index_ivf->search_preassigned(
             n, x, k, Iq.data(), Dq.data(), distances, labels, false, params);
     double t3 = getmillisecs();
+
     if (ms_per_stage) {
         ms_per_stage[0] = t1 - t0;
         ms_per_stage[1] = t2 - t1;
@@ -406,5 +410,100 @@ void range_search_with_parameters(
     }
 }
 
+IndexIVFResidualQuantizer* ivf_residual_from_quantizer(
+        const ResidualQuantizer& rq,
+        int nlevel) {
+    FAISS_THROW_IF_NOT(nlevel > 0 && nlevel + 1 < rq.M);
+
+    std::vector<size_t> nbits(nlevel);
+    std::copy(rq.nbits.begin(), rq.nbits.begin() + nlevel, nbits.begin());
+    std::unique_ptr<ResidualCoarseQuantizer> rcq(
+            new ResidualCoarseQuantizer(rq.d, nbits));
+
+    // set the coarse quantizer from the 2 first quantizers
+    rcq->rq.initialize_from(rq);
+    rcq->is_trained = true;
+    rcq->ntotal = (idx_t)1 << rcq->rq.tot_bits;
+
+    // settings for exhaustive search in RCQ
+    rcq->centroid_norms.resize(rcq->ntotal);
+    rcq->aq->compute_centroid_norms(rcq->centroid_norms.data());
+    rcq->beam_factor = -1.0; // use exact search
+    size_t nlist = rcq->ntotal;
+
+    // build a IVFResidualQuantizer from that
+    std::vector<size_t> nbits_refined;
+    for (int i = nlevel; i < rq.M; i++) {
+        nbits_refined.push_back(rq.nbits[i]);
+    }
+    std::unique_ptr<IndexIVFResidualQuantizer> index(
+            new IndexIVFResidualQuantizer(
+                    rcq.get(),
+                    rq.d,
+                    nlist,
+                    nbits_refined,
+                    faiss::METRIC_L2,
+                    rq.search_type));
+    index->own_fields = true;
+    rcq.release();
+    index->by_residual = true;
+    index->rq.initialize_from(rq, nlevel);
+    index->is_trained = true;
+
+    return index.release();
+}
+
+void ivf_residual_add_from_flat_codes(
+        IndexIVFResidualQuantizer* index,
+        size_t nb,
+        const uint8_t* raw_codes,
+        int64_t code_size) {
+    const ResidualCoarseQuantizer* rcq =
+            dynamic_cast<const faiss::ResidualCoarseQuantizer*>(
+                    index->quantizer);
+    FAISS_THROW_IF_NOT_MSG(rcq, "the coarse quantizer must be a RCQ");
+    if (code_size < 0) {
+        code_size = index->code_size;
+    }
+    InvertedLists& invlists = *index->invlists;
+    const ResidualQuantizer& rq = index->rq;
+
+    // populate inverted lists
+#pragma omp parallel if (nb > 10000)
+    {
+        std::vector<uint8_t> tmp_code(index->code_size);
+        std::vector<float> tmp(rq.d);
+        int nt = omp_get_num_threads();
+        int rank = omp_get_thread_num();
+
+#pragma omp for
+        for (idx_t i = 0; i < nb; i++) {
+            const uint8_t* code = &raw_codes[i * code_size];
+            BitstringReader rd(code, code_size);
+            idx_t list_no = rd.read(rcq->rq.tot_bits);
+
+            if (list_no % nt ==
+                rank) { // each thread takes care of 1/nt of the invlists
+                // copy AQ indexes one by one
+                BitstringWriter wr(tmp_code.data(), tmp_code.size());
+                for (int j = 0; j < rq.M; j++) {
+                    int nbit = rq.nbits[j];
+                    wr.write(rd.read(nbit), nbit);
+                }
+                // we need to recompute the norm
+                // decode first, does not use the norm component, so that's
+                // ok
+                index->rq.decode(tmp_code.data(), tmp.data(), 1);
+                float norm = fvec_norm_L2sqr(tmp.data(), rq.d);
+                wr.write(rq.encode_norm(norm), rq.norm_bits);
+
+                // add code to the inverted list
+                invlists.add_entry(list_no, i, tmp_code.data());
+            }
+        }
+    }
+    index->ntotal += nb;
+}
+
 } // namespace ivflib
 } // namespace faiss

data/vendor/faiss/faiss/IVFlib.h

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
-
 #ifndef FAISS_IVFLIB_H
 #define FAISS_IVFLIB_H
 
@@ -20,6 +18,11 @@
 #include <vector>
 
 namespace faiss {
+
+struct IndexIVFResidualQuantizer;
+struct IndexResidualQuantizer;
+struct ResidualQuantizer;
+
 namespace ivflib {
 
 /** check if two indexes have the same parameters and are trained in
@@ -145,6 +148,27 @@ void range_search_with_parameters(
         size_t* nb_dis = nullptr,
         double* ms_per_stage = nullptr);
 
+/** Build an IndexIVFResidualQuantizer from an ResidualQuantizer, using the
+ * nlevel first components as coarse quantizer and the rest as codes in invlists
+ */
+IndexIVFResidualQuantizer* ivf_residual_from_quantizer(
+        const ResidualQuantizer&,
+        int nlevel);
+
+/** add from codes. NB that the norm component is not used, so the code_size can
+ * be provided.
+ *
+ * @param ivfrq      index to populate with the codes
+ * @param codes      codes to add, size (ncode, code_size)
+ * @param code_size  override the ivfrq's code_size, useful if the norm encoding
+ *                   is different
+ */
+void ivf_residual_add_from_flat_codes(
+        IndexIVFResidualQuantizer* ivfrq,
+        size_t ncode,
+        const uint8_t* codes,
+        int64_t code_size = -1);
+
 } // namespace ivflib
 } // namespace faiss
 

data/vendor/faiss/faiss/Index.cpp

@@ -10,6 +10,7 @@
 #include <faiss/Index.h>
 
 #include <faiss/impl/AuxIndexStructures.h>
+#include <faiss/impl/DistanceComputer.h>
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/utils/distances.h>
 
@@ -23,7 +24,12 @@ void Index::train(idx_t /*n*/, const float* /*x*/) {
     // does nothing by default
 }
 
-void Index::range_search(idx_t, const float*, float, RangeSearchResult*) const {
+void Index::range_search(
+        idx_t,
+        const float*,
+        float,
+        RangeSearchResult*,
+        const SearchParameters* params) const {
     FAISS_THROW_MSG("range search not implemented");
 }
 
@@ -48,7 +54,25 @@ void Index::reconstruct(idx_t, float*) const {
     FAISS_THROW_MSG("reconstruct not implemented for this type of index");
 }
 
+void Index::reconstruct_batch(idx_t n, const idx_t* keys, float* recons) const {
+    std::mutex exception_mutex;
+    std::string exception_string;
+#pragma omp parallel for if (n > 1000)
+    for (idx_t i = 0; i < n; i++) {
+        try {
+            reconstruct(keys[i], &recons[i * d]);
+        } catch (const std::exception& e) {
+            std::lock_guard<std::mutex> lock(exception_mutex);
+            exception_string = e.what();
+        }
+    }
+    if (!exception_string.empty()) {
+        FAISS_THROW_MSG(exception_string.c_str());
+    }
+}
+
 void Index::reconstruct_n(idx_t i0, idx_t ni, float* recons) const {
+#pragma omp parallel for if (ni > 1000)
     for (idx_t i = 0; i < ni; i++) {
         reconstruct(i0 + i, recons + i * d);
     }
@@ -60,10 +84,11 @@ void Index::search_and_reconstruct(
         idx_t k,
         float* distances,
         idx_t* labels,
-        float* recons) const {
+        float* recons,
+        const SearchParameters* params) const {
     FAISS_THROW_IF_NOT(k > 0);
 
-    search(n, x, k, distances, labels);
+    search(n, x, k, distances, labels, params);
     for (idx_t i = 0; i < n; ++i) {
         for (idx_t j = 0; j < k; ++j) {
             idx_t ij = i * k + j;
@@ -149,4 +174,12 @@ DistanceComputer* Index::get_distance_computer() const {
     }
 }
 
+void Index::merge_from(Index& /* otherIndex */, idx_t /* add_id */) {
+    FAISS_THROW_MSG("merge_from() not implemented");
+}
+
+void Index::check_compatible_for_merge(const Index& /* otherIndex */) const {
+    FAISS_THROW_MSG("check_compatible_for_merge() not implemented");
+}
+
 } // namespace faiss