faiss 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97fd4d583754d20a5771e941fd5272205977c34b77eb9252e58f6016b90be52e
-  data.tar.gz: 26df73fc7891efd894e9ebf4ed37db2aaa3af8b62221cdd707cb287f920ad0a7
+  metadata.gz: 4541b0c40468723a8bed3db80d5778fb156afa999cbbd14b83653285b3ae6267
+  data.tar.gz: 2beeacbad82a578e2a410938bc4447d73699ef3998c146b4309e4b44816f5e33
 SHA512:
-  metadata.gz: 197143a79061c299b8f5e5ae76466db1a673e46f6d842615bed46170afadb6145e14df936262ec45f7e0d1eea889037cb3123c5a9348e32774eeb97414f180fd
-  data.tar.gz: 5a8e1ecebd43886d974e52fa216d86d9c03e3f8c04629eddc04a8be8df5c7ab4fc4c57398f2893d44718c788cd1853ae1bee06f64f1d49cdc5330cb03c5c0e6c
+  metadata.gz: 92d87492ff627e094ef29a48c8e4579dd976846ec9a95d94c45b816391dcea332d370a5fbde9c3f2bdc11964e061557d0e150b223ab6ac0209c092f1e1cd4a6e
+  data.tar.gz: 86fc2aaf3151545f24128429cf1369074809f8449f277a2676e9dacea76c156191b3d6df6df83f0d7ac0f061bfdf537b81ef57f6cbbb861e4ba4b592d06e8ca2

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.4 (2020-02-04)
+
+- Updated Faiss to 1.7.0
+
 ## 0.1.3 (2020-10-22)
 
 - Updated Faiss to 1.6.4

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) Facebook, Inc. and its affiliates.
-Copyright (c) 2020 Andrew Kane
+Copyright (c) 2020-2021 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -4,7 +4,7 @@
 
 Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
 
-[![Build Status](https://travis-ci.org/ankane/faiss.svg?branch=master)](https://travis-ci.org/ankane/faiss)
+[![Build Status](https://github.com/ankane/faiss/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss/actions)
 
 ## Installation
 

data/ext/faiss/extconf.rb

@@ -9,7 +9,7 @@ $CXXFLAGS << " -std=c++11 -march=native -DFINTEGER=int"
 ext = File.expand_path(".", __dir__)
 vendor = File.expand_path("../../vendor/faiss", __dir__)
 
-$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/impl,#{vendor}/faiss/utils}/*.{cpp}"]
+$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/{impl,invlists,utils}}/*.{cpp}"]
 $objs = $srcs.map { |v| v.sub(/cpp\z/, "o") }
 $INCFLAGS << " -I#{vendor}"
 $VPATH << vendor

data/lib/faiss/version.rb

@@ -1,3 +1,3 @@
 module Faiss
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/vendor/faiss/faiss/AutoTune.cpp

@@ -15,6 +15,7 @@
 
 #include <cinttypes>
 #include <cmath>
+#include <typeinfo>
 
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/utils/utils.h>
@@ -32,6 +33,7 @@
 #include <faiss/MetaIndexes.h>
 #include <faiss/IndexScalarQuantizer.h>
 #include <faiss/IndexHNSW.h>
+#include <faiss/IndexRefine.h>
 
 #include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexBinaryHNSW.h>
@@ -234,7 +236,7 @@ void OperatingPoints::display (bool only_optimal) const
 {
     const std::vector<OperatingPoint> &pts =
         only_optimal ? optimal_pts : all_pts;
-    printf("Tested %zd operating points, %zd ones are optimal:\n",
+    printf("Tested %zd operating points, %zd ones are Pareto-optimal:\n",
            all_pts.size(), optimal_pts.size());
 
     for (int i = 0; i < pts.size(); i++) {
@@ -333,7 +335,7 @@ static void init_pq_ParameterRange (const ProductQuantizer & pq,
     pr.values.push_back (pq.code_size * 8);
 }
 
-ParameterRange &ParameterSpace::add_range(const char * name)
+ParameterRange &ParameterSpace::add_range(const std::string & name)
 {
     for (auto & pr : parameter_ranges) {
         if (pr.name == name) {
@@ -346,13 +348,13 @@ ParameterRange &ParameterSpace::add_range(const char * name)
 }
 
 
-/// initialize with reasonable parameters for the index
+/// initialize with reasonable parameters for this type of index
 void ParameterSpace::initialize (const Index * index)
 {
     if (DC (IndexPreTransform)) {
         index = ix->index;
     }
-    if (DC (IndexRefineFlat)) {
+    if (DC (IndexRefine)) {
         ParameterRange & pr = add_range("k_factor_rf");
         for (int i = 0; i <= 6; i++) {
             pr.values.push_back (1 << i);
@@ -372,12 +374,14 @@ void ParameterSpace::initialize (const Index * index)
                 pr.values.push_back (nprobe);
             }
         }
-        if (dynamic_cast<const IndexHNSW*>(ix->quantizer)) {
-            ParameterRange & pr = add_range("efSearch");
-            for (int i = 2; i <= 9; i++) {
-                pr.values.push_back (1 << i);
-            }
+        ParameterSpace ivf_pspace;
+        ivf_pspace.initialize(ix->quantizer);
+
+        for (const ParameterRange & p: ivf_pspace.parameter_ranges) {
+            ParameterRange & pr = add_range("quantizer_" + p.name);
+            pr.values = p.values;
         }
+
     }
     if (DC (IndexPQ)) {
         ParameterRange & pr = add_range("ht");
@@ -446,7 +450,7 @@ void ParameterSpace::set_index_parameters (
          tok = strtok_r (nullptr, " ,", &ptr)) {
         char name[100];
         double val;
-        int ret = sscanf (tok, "%100[^=]=%lf", name, &val);
+        int ret = sscanf (tok, "%99[^=]=%lf", name, &val);
         FAISS_THROW_IF_NOT_FMT (
            ret == 2, "could not interpret parameters %s", tok);
         set_index_parameter (index, name, val);
@@ -457,44 +461,38 @@ void ParameterSpace::set_index_parameters (
 void ParameterSpace::set_index_parameter (
         Index * index, const std::string & name, double val) const
 {
-    if (verbose > 1)
-        printf("    set %s=%g\n", name.c_str(), val);
+    if (verbose > 1) {
+        printf("    set_index_parameter %s=%g\n", name.c_str(), val);
+    }
 
     if (name == "verbose") {
         index->verbose = int(val);
         // and fall through to also enable it on sub-indexes
     }
-    if (DC (IndexPreTransform)) {
+    if (DC (IndexIDMap)) {
         set_index_parameter (ix->index, name, val);
         return;
     }
-    if (DC (IndexShards)) {
-        // call on all sub-indexes
-        auto fn =
-          [this, name, val](int, Index* subIndex) {
-            set_index_parameter(subIndex, name, val);
-          };
-
-        ix->runOnIndex(fn);
+    if (DC (IndexPreTransform)) {
+        set_index_parameter (ix->index, name, val);
         return;
     }
-    if (DC (IndexReplicas)) {
+    if (DC (ThreadedIndex<Index>)) {
         // call on all sub-indexes
         auto fn =
-          [this, name, val](int, Index* subIndex) {
-            set_index_parameter(subIndex, name, val);
+            [this, name, val](int /* no */, Index* subIndex) {
+              set_index_parameter(subIndex, name, val);
           };
-
         ix->runOnIndex(fn);
         return;
     }
-    if (DC (IndexRefineFlat)) {
+    if (DC (IndexRefine)) {
         if (name == "k_factor_rf") {
             ix->k_factor = int(val);
             return;
         }
         // otherwise it is for the sub-index
-        set_index_parameter (&ix->refine_index, name, val);
+        set_index_parameter (ix->base_index, name, val);
         return;
     }
 
@@ -504,10 +502,7 @@ void ParameterSpace::set_index_parameter (
     }
 
     if (name == "nprobe") {
-        if (DC (IndexIDMap)) {
-            set_index_parameter (ix->index, name, val);
-            return;
-        } else if (DC (IndexIVF)) {
+        if (DC (IndexIVF)) {
             ix->nprobe = int(val);
             return;
         }
@@ -559,6 +554,14 @@ void ParameterSpace::set_index_parameter (
         }
     }
 
+    if (name.find("quantizer_") == 0) {
+        if (DC(IndexIVF)) {
+            std::string sub_name = name.substr(strlen("quantizer_"));
+            set_index_parameter(ix->quantizer, sub_name, val);
+            return;
+        }
+    }
+
     FAISS_THROW_FMT ("ParameterSpace::set_index_parameter:"
                      "could not set parameter %s",
                      name.c_str());
@@ -707,8 +710,8 @@ void ParameterSpace::explore (Index *index,
         bool keep = ops->add (perf, t_search, combination_name (cno), cno);
 
         if (verbose)
-            printf(" perf %.3f t %.3f (%d runs) %s\n",
-                   perf, t_search, nrun,
+            printf(" perf %.3f t %.3f (%d %s) %s\n",
+                   perf, t_search, nrun, nrun >= 2 ? "runs" : "run",
                    keep ? "*" : "");
     }
 }

data/vendor/faiss/faiss/AutoTune.h

@@ -81,7 +81,10 @@ struct IntersectionCriterion: AutoTuneCriterion {
 /**
  * Maintains a list of experimental results. Each operating point is a
  * (perf, t, key) triplet, where higher perf and lower t is
- * better. The key field is an arbitrary identifier for the operating point
+ * better. The key field is an arbitrary identifier for the operating point.
+ *
+ * Includes primitives to extract the Pareto-optimal operating points in the
+ * (perf, t) space.
  */
 
 struct OperatingPoint {
@@ -168,7 +171,7 @@ struct ParameterSpace {
     void display () const;
 
     /// add a new parameter (or return it if it exists)
-    ParameterRange &add_range(const char * name);
+    ParameterRange &add_range(const std::string & name);
 
     /// initialize with reasonable parameters for the index
     virtual void initialize (const Index * index);
@@ -179,7 +182,7 @@ struct ParameterSpace {
     /// set a combination of parameters described by a string
     void set_index_parameters (Index *index, const char *param_string) const;
 
-    /// set one of the parameters
+    /// set one of the parameters, returns whether setting was successful
     virtual void set_index_parameter (
         Index * index, const std::string & name, double val) const;
 

data/vendor/faiss/faiss/Clustering.cpp

@@ -269,6 +269,7 @@ void Clustering::train_encoded (idx_t nx, const uint8_t *x_in,
                                 const Index * codec, Index & index,
                                 const float *weights) {
 
+
     FAISS_THROW_IF_NOT_FMT (nx >= k,
              "Number of training points (%" PRId64 ") should be at least "
              "as large as number of clusters (%zd)", nx, k);
@@ -350,8 +351,9 @@ void Clustering::train_encoded (idx_t nx, const uint8_t *x_in,
     std::unique_ptr<float []> dis(new float[nx]);
 
     // remember best iteration for redo
-    float best_err = HUGE_VALF;
-    std::vector<ClusteringIterationStats> best_obj;
+    bool lower_is_better = index.metric_type != METRIC_INNER_PRODUCT;
+    float best_obj = lower_is_better ? HUGE_VALF : -HUGE_VALF;
+    std::vector<ClusteringIterationStats> best_iteration_stats;
     std::vector<float> best_centroids;
 
     // support input centroids
@@ -417,7 +419,7 @@ void Clustering::train_encoded (idx_t nx, const uint8_t *x_in,
 
         // k-means iterations
 
-        float err = 0;
+        float obj = 0;
         for (int i = 0; i < niter; i++) {
             double t0s = getmillisecs();
 
@@ -440,10 +442,10 @@ void Clustering::train_encoded (idx_t nx, const uint8_t *x_in,
             InterruptCallback::check();
             t_search_tot += getmillisecs() - t0s;
 
-            // accumulate error
-            err = 0;
+            // accumulate objective
+            obj = 0;
             for (int j = 0; j < nx; j++) {
-                err += dis[j];
+                obj += dis[j];
             }
 
             // update the centroids
@@ -463,8 +465,9 @@ void Clustering::train_encoded (idx_t nx, const uint8_t *x_in,
 
             // collect statistics
             ClusteringIterationStats stats =
-                { err, (getmillisecs() - t0) / 1000.0,
-                  t_search_tot / 1000, imbalance_factor (nx, k, assign.get()),
+                { obj, (getmillisecs() - t0) / 1000.0,
+                  t_search_tot / 1000,
+                  imbalance_factor (nx, k, assign.get()),
                   nsplit };
             iteration_stats.push_back(stats);
 
@@ -491,20 +494,21 @@ void Clustering::train_encoded (idx_t nx, const uint8_t *x_in,
 
         if (verbose) printf("\n");
         if (nredo > 1) {
-            if (err < best_err) {
+            if ((lower_is_better && obj < best_obj) ||
+                (!lower_is_better && obj > best_obj)) {
                 if (verbose) {
                     printf ("Objective improved: keep new clusters\n");
                 }
                 best_centroids = centroids;
-                best_obj = iteration_stats;
-                best_err = err;
+                best_iteration_stats = iteration_stats;
+                best_obj = obj;
             }
             index.reset ();
         }
     }
     if (nredo > 1) {
         centroids = best_centroids;
-        iteration_stats = best_obj;
+        iteration_stats = best_iteration_stats;
         index.reset();
         index.add(k, best_centroids.data());
     }

data/vendor/faiss/faiss/Index.cpp

@@ -34,11 +34,10 @@ void Index::range_search (idx_t , const float *, float,
   FAISS_THROW_MSG ("range search not implemented");
 }
 
-void Index::assign (idx_t n, const float * x, idx_t * labels, idx_t k)
+void Index::assign (idx_t n, const float * x, idx_t * labels, idx_t k) const
 {
-  float * distances = new float[n * k];
-  ScopeDeleter<float> del(distances);
-  search (n, x, k, distances, labels);
+  std::vector<float> distances(n * k);
+  search (n, x, k, distances.data(), labels);
 }
 
 void Index::add_with_ids(

data/vendor/faiss/faiss/Index.h

@@ -17,8 +17,8 @@
 #include <sstream>
 
 #define FAISS_VERSION_MAJOR 1
-#define FAISS_VERSION_MINOR 6
-#define FAISS_VERSION_PATCH 4
+#define FAISS_VERSION_MINOR 7
+#define FAISS_VERSION_PATCH 0
 
 /**
  * @namespace faiss
@@ -133,7 +133,7 @@ struct Index {
      * @param x           input vectors to search, size n * d
      * @param labels      output labels of the NNs, size n*k
      */
-    void assign (idx_t n, const float * x, idx_t * labels, idx_t k = 1);
+    virtual void assign (idx_t n, const float * x, idx_t * labels, idx_t k = 1) const;
 
     /// removes all elements from the database.
     virtual void reset() = 0;

data/vendor/faiss/faiss/IndexBinary.cpp

@@ -26,10 +26,9 @@ void IndexBinary::range_search(idx_t, const uint8_t *, int,
   FAISS_THROW_MSG("range search not implemented");
 }
 
-void IndexBinary::assign(idx_t n, const uint8_t *x, idx_t *labels, idx_t k) {
-  int *distances = new int[n * k];
-  ScopeDeleter<int> del(distances);
-  search(n, x, k, distances, labels);
+void IndexBinary::assign(idx_t n, const uint8_t *x, idx_t *labels, idx_t k) const {
+  std::vector<int> distances(n * k);
+  search(n, x, k, distances.data(), labels);
 }
 
 void IndexBinary::add_with_ids(idx_t, const uint8_t *, const idx_t *) {

data/vendor/faiss/faiss/IndexBinary.h

@@ -120,7 +120,7 @@ struct IndexBinary {
    * @param x           input vectors to search, size n * d / 8
    * @param labels      output labels of the NNs, size n*k
    */
-  void assign(idx_t n, const uint8_t *x, idx_t *labels, idx_t k = 1);
+  void assign(idx_t n, const uint8_t *x, idx_t *labels, idx_t k = 1) const;
 
   /// Removes all elements from the database.
   virtual void reset() = 0;

data/vendor/faiss/faiss/IndexBinaryHash.cpp

@@ -18,16 +18,7 @@
 
 #include <faiss/impl/AuxIndexStructures.h>
 #include <faiss/impl/FaissAssert.h>
-
-#ifdef _MSC_VER
-#include <intrin.h>
-
-static inline int __builtin_ctzll(uint64_t x) {
-    unsigned long ret;
-    _BitScanForward64(&ret, x);
-    return (int)ret;
-}
-#endif // _MSC_VER
+#include <faiss/impl/platform_macros.h>
 
 namespace faiss {
 
@@ -145,8 +136,7 @@ struct KnnSearchResults {
 
     inline void add (float dis, idx_t id) {
         if (dis < heap_sim[0]) {
-            heap_pop<C> (k, heap_sim, heap_ids);
-            heap_push<C> (k, heap_sim, heap_ids, dis, id);
+            heap_replace_top<C> (k, heap_sim, heap_ids, dis, id);
         }
     }
 

data/vendor/faiss/faiss/IndexBinaryIVF.cpp

@@ -319,9 +319,8 @@ struct IVFBinaryScannerL2: BinaryInvertedListScanner {
         for (size_t j = 0; j < n; j++) {
             uint32_t dis = hc.hamming (codes);
             if (dis < simi[0]) {
-                heap_pop<C> (k, simi, idxi);
                 idx_t id = store_pairs ? lo_build(list_no, j) : ids[j];
-                heap_push<C> (k, simi, idxi, dis, id);
+                heap_replace_top<C> (k, simi, idxi, dis, id);
                 nup++;
             }
             codes += code_size;

data/vendor/faiss/faiss/IndexFlat.cpp

@@ -226,155 +226,7 @@ void IndexFlat::sa_decode (idx_t n, const uint8_t *bytes, float *x) const
 
 
 
-/***************************************************
- * IndexFlatL2BaseShift
- ***************************************************/
-
-IndexFlatL2BaseShift::IndexFlatL2BaseShift (idx_t d, size_t nshift, const float *shift):
-    IndexFlatL2 (d), shift (nshift)
-{
-    memcpy (this->shift.data(), shift, sizeof(float) * nshift);
-}
 
-void IndexFlatL2BaseShift::search (
-            idx_t n,
-            const float *x,
-            idx_t k,
-            float *distances,
-            idx_t *labels) const
-{
-    FAISS_THROW_IF_NOT (shift.size() == ntotal);
-
-    float_maxheap_array_t res = {
-        size_t(n), size_t(k), labels, distances};
-    knn_L2sqr_base_shift (x, xb.data(), d, n, ntotal, &res, shift.data());
-}
-
-
-
-/***************************************************
- * IndexRefineFlat
- ***************************************************/
-
-IndexRefineFlat::IndexRefineFlat (Index *base_index):
-    Index (base_index->d, base_index->metric_type),
-    refine_index (base_index->d, base_index->metric_type),
-    base_index (base_index), own_fields (false),
-    k_factor (1)
-{
-    is_trained = base_index->is_trained;
-    FAISS_THROW_IF_NOT_MSG (base_index->ntotal == 0,
-                      "base_index should be empty in the beginning");
-}
-
-IndexRefineFlat::IndexRefineFlat () {
-    base_index = nullptr;
-    own_fields = false;
-    k_factor = 1;
-}
-
-
-void IndexRefineFlat::train (idx_t n, const float *x)
-{
-    base_index->train (n, x);
-    is_trained = true;
-}
-
-void IndexRefineFlat::add (idx_t n, const float *x) {
-    FAISS_THROW_IF_NOT (is_trained);
-    base_index->add (n, x);
-    refine_index.add (n, x);
-    ntotal = refine_index.ntotal;
-}
-
-void IndexRefineFlat::reset ()
-{
-    base_index->reset ();
-    refine_index.reset ();
-    ntotal = 0;
-}
-
-namespace {
-typedef faiss::Index::idx_t idx_t;
-
-template<class C>
-static void reorder_2_heaps (
-      idx_t n,
-      idx_t k, idx_t *labels, float *distances,
-      idx_t k_base, const idx_t *base_labels, const float *base_distances)
-{
-#pragma omp parallel for
-    for (idx_t i = 0; i < n; i++) {
-        idx_t *idxo = labels + i * k;
-        float *diso = distances + i * k;
-        const idx_t *idxi = base_labels + i * k_base;
-        const float *disi = base_distances + i * k_base;
-
-        heap_heapify<C> (k, diso, idxo, disi, idxi, k);
-        if (k_base != k) { // add remaining elements
-            heap_addn<C> (k, diso, idxo, disi + k, idxi + k, k_base - k);
-        }
-        heap_reorder<C> (k, diso, idxo);
-    }
-}
-
-
-}
-
-
-void IndexRefineFlat::search (
-              idx_t n, const float *x, idx_t k,
-              float *distances, idx_t *labels) const
-{
-    FAISS_THROW_IF_NOT (is_trained);
-    idx_t k_base = idx_t (k * k_factor);
-    idx_t * base_labels = labels;
-    float * base_distances = distances;
-    ScopeDeleter<idx_t> del1;
-    ScopeDeleter<float> del2;
-
-
-    if (k != k_base) {
-        base_labels = new idx_t [n * k_base];
-        del1.set (base_labels);
-        base_distances = new float [n * k_base];
-        del2.set (base_distances);
-    }
-
-    base_index->search (n, x, k_base, base_distances, base_labels);
-
-    for (int i = 0; i < n * k_base; i++)
-        assert (base_labels[i] >= -1 &&
-                base_labels[i] < ntotal);
-
-    // compute refined distances
-    refine_index.compute_distance_subset (
-        n, x, k_base, base_distances, base_labels);
-
-    // sort and store result
-    if (metric_type == METRIC_L2) {
-        typedef CMax <float, idx_t> C;
-        reorder_2_heaps<C> (
-            n, k, labels, distances,
-            k_base, base_labels, base_distances);
-
-    } else if (metric_type == METRIC_INNER_PRODUCT) {
-        typedef CMin <float, idx_t> C;
-        reorder_2_heaps<C> (
-            n, k, labels, distances,
-            k_base, base_labels, base_distances);
-    } else {
-        FAISS_THROW_MSG("Metric type not supported");
-    }
-
-}
-
-
-
-IndexRefineFlat::~IndexRefineFlat ()
-{
-    if (own_fields) delete base_index;
-}
 
 /***************************************************
  * IndexFlat1D