faiss 0.2.5 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bdc2effbe6e2d827ffe473531be0864cf710ae6f3ad34f8324087695c367d140
-  data.tar.gz: 6e5b80b1f4281766b17d208af58b44e768576d259de1c6edb25630c700215c10
+  metadata.gz: 8d0e67c5b23ef30f3632f8f878bba1ed8be7cbf8212c21a799daca8a65c23fcb
+  data.tar.gz: d3bcbf4453ece0531cb3cfa017bf60327fffa47401a27451ccb501c11431911e
 SHA512:
-  metadata.gz: 404d064f14734a23d946ad0fc0576673e5e685d1001884981453e3bec23ecf6ba7b75857e578b47ef5303fde27342dfb25e85bbb7a60883d6ba09a964049bbfa
-  data.tar.gz: 536f403109d3773a3cd0ca27b4b46bfa26eaeedd8d5e2f4e6fa116e3b922d21f8a964a2d6aa1cbbff951da9444e9184bf7c60955263ed68c91ba91342f3f9196
+  metadata.gz: 50f4c6ef3b637b708e0860f524eabb64b708868ce8210153f0868ff846a67ec24c4f229cf9146ab1dea0113b033e2287866ced15e750297ae79822cf9f131fe2
+  data.tar.gz: a3b0df6f2377c7381fdaab54f56e97a80ebaf5e752b92ba04b5d7a256c493a2bcbb14d60e4eeeb26a0d044f3159b4e416699ab6ecf774eb92f6e6014958d20b7

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.2.7 (2023-04-20)
+
+- Updated Faiss to 1.7.4
+
+## 0.2.6 (2023-04-11)
+
+- Added `add_with_ids` method
+- Added `IndexIDMap`
+
 ## 0.2.5 (2022-12-27)
 
 - Updated Faiss to 1.7.3

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) Facebook, Inc. and its affiliates.
-Copyright (c) 2020-2022 Andrew Kane
+Copyright (c) 2020-2023 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/ext/faiss/extconf.rb

@@ -26,7 +26,7 @@ $CXXFLAGS << " -std=c++17 $(optflags) -DFINTEGER=int " << with_config("optflags"
 ext = File.expand_path(".", __dir__)
 vendor = File.expand_path("../../vendor/faiss", __dir__)
 
-$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/{impl,invlists,utils}}/*.{cpp}"]
+$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/{impl,invlists,utils}/**}/*.{cpp}"]
 $objs = $srcs.map { |v| v.sub(/cpp\z/, "o") }
 abort "Faiss not found" unless find_header("faiss/Index.h", vendor)
 $VPATH << vendor

data/ext/faiss/index.cpp

@@ -1,6 +1,7 @@
 #include <faiss/Index.h>
 #include <faiss/IndexFlat.h>
 #include <faiss/IndexHNSW.h>
+#include <faiss/IndexIDMap.h>
 #include <faiss/IndexIVFFlat.h>
 #include <faiss/IndexLSH.h>
 #include <faiss/IndexScalarQuantizer.h>
@@ -118,6 +119,15 @@ void init_index(Rice::Module& m) {
         auto n = check_shape(objects, self.d);
         self.add(n, objects.read_ptr());
       })
+    .define_method(
+      "add_with_ids",
+      [](faiss::Index &self, numo::SFloat objects, numo::Int64 ids) {
+        auto n = check_shape(objects, self.d);
+        if (ids.ndim() != 1 || ids.shape()[0] != n) {
+          throw Rice::Exception(rb_eArgError, "expected ids to be 1d array with size %d", n);
+        }
+        self.add_with_ids(n, objects.read_ptr(), ids.read_ptr());
+      })
     .define_method(
       "search",
       [](faiss::Index &self, numo::SFloat objects, size_t k) {
@@ -186,4 +196,7 @@ void init_index(Rice::Module& m) {
       [](faiss::ParameterSpace& self, faiss::Index* index, const std::string& name, double val) {
         self.set_index_parameter(index, name, val);
       });
+
+  Rice::define_class_under<faiss::IndexIDMap, faiss::Index>(m, "IndexIDMap")
+    .define_constructor(Rice::Constructor<faiss::IndexIDMap, faiss::Index*>());
 }

data/lib/faiss/version.rb

@@ -1,3 +1,3 @@
 module Faiss
-  VERSION = "0.2.5"
+  VERSION = "0.2.7"
 end

data/lib/faiss.rb

@@ -2,7 +2,7 @@
 require "numo/narray"
 
 # ext
-require "faiss/ext"
+require_relative "faiss/ext"
 
 # modules
-require "faiss/version"
+require_relative "faiss/version"

data/vendor/faiss/faiss/AutoTune.cpp

@@ -32,6 +32,7 @@
 #include <faiss/IndexPreTransform.h>
 #include <faiss/IndexRefine.h>
 #include <faiss/IndexScalarQuantizer.h>
+#include <faiss/IndexShardsIVF.h>
 #include <faiss/MetaIndexes.h>
 #include <faiss/VectorTransform.h>
 
@@ -354,7 +355,7 @@ void ParameterSpace::initialize(const Index* index) {
         index = ix->index;
     }
 
-    if (DC(IndexIVF)) {
+    if (DC(IndexIVFInterface)) {
         {
             ParameterRange& pr = add_range("nprobe");
             for (int i = 0; i < 13; i++) {
@@ -461,6 +462,16 @@ void ParameterSpace::set_index_parameter(
         set_index_parameter(ix->index, name, val);
         return;
     }
+    if (DC(IndexShardsIVF)) {
+        // special handling because the nprobe is set at the sub-class level
+        // but other params are set on the class itself
+        if (name.find("quantizer_") == 0 && name != "nprobe" &&
+            name != "quantizer_nprobe") {
+            std::string sub_name = name.substr(strlen("quantizer_"));
+            set_index_parameter(ix->quantizer, sub_name, val);
+            return;
+        }
+    }
     if (DC(ThreadedIndex<Index>)) {
         // call on all sub-indexes
         auto fn = [this, name, val](int /* no */, Index* subIndex) {
@@ -608,7 +619,7 @@ void ParameterSpace::explore(
     if (n_experiments == 0) {
         for (size_t cno = 0; cno < n_comb; cno++) {
             set_index_parameters(index, cno);
-            std::vector<Index::idx_t> I(nq * crit.nnn);
+            std::vector<idx_t> I(nq * crit.nnn);
             std::vector<float> D(nq * crit.nnn);
 
             double t0 = getmillisecs();
@@ -677,7 +688,7 @@ void ParameterSpace::explore(
         }
 
         set_index_parameters(index, cno);
-        std::vector<Index::idx_t> I(nq * crit.nnn);
+        std::vector<idx_t> I(nq * crit.nnn);
         std::vector<float> D(nq * crit.nnn);
 
         double t0 = getmillisecs();
@@ -688,7 +699,7 @@ void ParameterSpace::explore(
         do {
             if (thread_over_batches) {
 #pragma omp parallel for
-                for (Index::idx_t q0 = 0; q0 < nq; q0 += batchsize) {
+                for (idx_t q0 = 0; q0 < nq; q0 += batchsize) {
                     size_t q1 = q0 + batchsize;
                     if (q1 > nq)
                         q1 = nq;

data/vendor/faiss/faiss/AutoTune.h

@@ -24,7 +24,6 @@ namespace faiss {
  * higher is better.
  */
 struct AutoTuneCriterion {
-    typedef Index::idx_t idx_t;
     idx_t nq;     ///< nb of queries this criterion is evaluated on
     idx_t nnn;    ///< nb of NNs that the query should request
     idx_t gt_nnn; ///< nb of GT NNs required to evaluate criterion

data/vendor/faiss/faiss/Clustering.cpp

@@ -88,8 +88,6 @@ void Clustering::train(
 
 namespace {
 
-using idx_t = Clustering::idx_t;
-
 idx_t subsample_training_set(
         const Clustering& clus,
         idx_t nx,
@@ -374,7 +372,7 @@ void Clustering::train_encoded(
     std::unique_ptr<float[]> dis(new float[nx]);
 
     // remember best iteration for redo
-    bool lower_is_better = index.metric_type != METRIC_INNER_PRODUCT;
+    bool lower_is_better = !is_similarity_metric(index.metric_type);
     float best_obj = lower_is_better ? HUGE_VALF : -HUGE_VALF;
     std::vector<ClusteringIterationStats> best_iteration_stats;
     std::vector<float> best_centroids;
@@ -624,8 +622,6 @@ ProgressiveDimClustering::ProgressiveDimClustering(
 
 namespace {
 
-using idx_t = Index::idx_t;
-
 void copy_columns(idx_t n, idx_t d1, const float* src, idx_t d2, float* dest) {
     idx_t d = std::min(d1, d2);
     for (idx_t i = 0; i < n; i++) {

data/vendor/faiss/faiss/Clustering.h

@@ -61,7 +61,6 @@ struct ClusteringIterationStats {
  *
  */
 struct Clustering : ClusteringParameters {
-    typedef Index::idx_t idx_t;
     size_t d; ///< dimension of the vectors
     size_t k; ///< nb of centroids
 
@@ -154,7 +153,6 @@ struct ProgressiveDimIndexFactory {
  * https://arxiv.org/abs/1509.05195
  */
 struct ProgressiveDimClustering : ProgressiveDimClusteringParameters {
-    using idx_t = Index::idx_t;
     size_t d; ///< dimension of the vectors
     size_t k; ///< nb of centroids
 

data/vendor/faiss/faiss/IVFlib.h

@@ -48,8 +48,6 @@ IndexIVF* try_extract_index_ivf(Index* index);
  */
 void merge_into(Index* index0, Index* index1, bool shift_ids);
 
-typedef Index::idx_t idx_t;
-
 /* Returns the cluster the embeddings belong to.
  *
  * @param index      Index, which should be an IVF index

data/vendor/faiss/faiss/Index.h

@@ -18,7 +18,7 @@
 
 #define FAISS_VERSION_MAJOR 1
 #define FAISS_VERSION_MINOR 7
-#define FAISS_VERSION_PATCH 3
+#define FAISS_VERSION_PATCH 4
 
 /**
  * @namespace faiss
@@ -62,7 +62,6 @@ struct SearchParameters {
  * although the internal representation may vary.
  */
 struct Index {
-    using idx_t = int64_t; ///< all indices are this type
     using component_t = float;
     using distance_t = float;
 

data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp

@@ -484,7 +484,7 @@ ResidualCoarseQuantizer::ResidualCoarseQuantizer(
         int d,        ///< dimensionality of the input vectors
         const std::vector<size_t>& nbits,
         MetricType metric)
-        : AdditiveCoarseQuantizer(d, &rq, metric), rq(d, nbits), beam_factor(4.0) {
+        : AdditiveCoarseQuantizer(d, &rq, metric), rq(d, nbits) {
     FAISS_THROW_IF_NOT(rq.tot_bits <= 63);
     is_trained = false;
 }
@@ -520,10 +520,15 @@ void ResidualCoarseQuantizer::search(
         idx_t k,
         float* distances,
         idx_t* labels,
-        const SearchParameters * params
+        const SearchParameters * params_in
         ) const {
 
-    FAISS_THROW_IF_NOT_MSG(!params, "search params not supported for this index");
+    float beam_factor = this->beam_factor;
+    if (params_in) {
+        auto params = dynamic_cast<const SearchParametersResidualCoarseQuantizer*>(params_in);
+        FAISS_THROW_IF_NOT_MSG(params, "need SearchParametersResidualCoarseQuantizer parameters");
+        beam_factor = params->beam_factor;
+    }
 
     if (beam_factor < 0) {
         AdditiveCoarseQuantizer::search(n, x, k, distances, labels);
@@ -585,6 +590,15 @@ void ResidualCoarseQuantizer::search(
     }
 }
 
+void ResidualCoarseQuantizer::initialize_from(const ResidualCoarseQuantizer &other) {
+    FAISS_THROW_IF_NOT(rq.M <= other.rq.M);
+    rq.initialize_from(other.rq);
+    set_beam_factor(other.beam_factor);
+    is_trained = other.is_trained;
+    ntotal = (idx_t)1 << aq->tot_bits;
+}
+
+
 /**************************************************************************************
  * LocalSearchCoarseQuantizer
  **************************************************************************************/

data/vendor/faiss/faiss/IndexAdditiveQuantizer.h

@@ -187,6 +187,11 @@ struct AdditiveCoarseQuantizer : Index {
     void reset() override;
 };
 
+struct SearchParametersResidualCoarseQuantizer : SearchParameters {
+    float beam_factor = 4.0f;
+    ~SearchParametersResidualCoarseQuantizer() {}
+};
+
 /** The ResidualCoarseQuantizer is a bit specialized compared to the
  * default AdditiveCoarseQuantizer because it can use a beam search
  * at search time (slow but may be useful for very large vocabularies) */
@@ -196,7 +201,7 @@ struct ResidualCoarseQuantizer : AdditiveCoarseQuantizer {
 
     /// factor between the beam size and the search k
     /// if negative, use exact search-to-centroid
-    float beam_factor;
+    float beam_factor = 4.0f;
 
     /// computes centroid norms if required
     void set_beam_factor(float new_beam_factor);
@@ -226,6 +231,10 @@ struct ResidualCoarseQuantizer : AdditiveCoarseQuantizer {
             idx_t* labels,
             const SearchParameters* params = nullptr) const override;
 
+    /** Copy the M first codebook levels from other. Useful to crop a
+     * ResidualQuantizer to its first M quantizers. */
+    void initialize_from(const ResidualCoarseQuantizer& other);
+
     ResidualCoarseQuantizer();
 };
 

data/vendor/faiss/faiss/IndexBinary.h

@@ -32,7 +32,6 @@ struct RangeSearchResult;
  * vectors.
  */
 struct IndexBinary {
-    using idx_t = Index::idx_t; ///< all indices are this type
     using component_t = uint8_t;
     using distance_t = int32_t;
 

data/vendor/faiss/faiss/IndexBinaryFlat.cpp

@@ -60,7 +60,8 @@ void IndexBinaryFlat::search(
                     xb.data(),
                     ntotal,
                     code_size,
-                    /* ordered = */ true);
+                    /* ordered = */ true,
+                    approx_topk_mode);
         } else {
             hammings_knn_mc(
                     x + s * code_size,

data/vendor/faiss/faiss/IndexBinaryFlat.h

@@ -14,6 +14,8 @@
 
 #include <faiss/IndexBinary.h>
 
+#include <faiss/utils/approx_topk/mode.h>
+
 namespace faiss {
 
 /** Index that stores the full vectors and performs exhaustive search. */
@@ -28,6 +30,8 @@ struct IndexBinaryFlat : IndexBinary {
 
     size_t query_batch_size = 32;
 
+    ApproxTopK_mode_t approx_topk_mode = ApproxTopK_mode_t::EXACT_TOPK;
+
     explicit IndexBinaryFlat(idx_t d);
 
     void add(idx_t n, const uint8_t* x) override;

data/vendor/faiss/faiss/IndexBinaryHash.cpp

@@ -108,8 +108,6 @@ struct FlipEnumerator {
     }
 };
 
-using idx_t = Index::idx_t;
-
 struct RangeSearchResults {
     int radius;
     RangeQueryResult& qres;
@@ -353,7 +351,7 @@ template <class HammingComputer, class SearchResults>
 static void verify_shortlist(
         const IndexBinaryFlat& index,
         const uint8_t* q,
-        const std::unordered_set<Index::idx_t>& shortlist,
+        const std::unordered_set<idx_t>& shortlist,
         SearchResults& res) {
     size_t code_size = index.code_size;
     size_t nlist = 0, ndis = 0, n0 = 0;