faiss 0.2.0 → 0.2.1

data/vendor/faiss/faiss/IVFlib.h

@@ -16,35 +16,34 @@
  * IndexIVFs embedded within an IndexPreTransform.
  */
 
-#include <vector>
 #include <faiss/IndexIVF.h>
+#include <vector>
 
-namespace faiss { namespace ivflib {
-
+namespace faiss {
+namespace ivflib {
 
 /** check if two indexes have the same parameters and are trained in
  * the same way, otherwise throw. */
-void check_compatible_for_merge (const Index * index1,
-                                 const Index * index2);
+void check_compatible_for_merge(const Index* index1, const Index* index2);
 
 /** get an IndexIVF from an index. The index may be an IndexIVF or
  * some wrapper class that encloses an IndexIVF
  *
  * throws an exception if this is not the case.
  */
-const IndexIVF * extract_index_ivf (const Index * index);
-IndexIVF * extract_index_ivf (Index * index);
+const IndexIVF* extract_index_ivf(const Index* index);
+IndexIVF* extract_index_ivf(Index* index);
 
 /// same as above but returns nullptr instead of throwing on failure
-const IndexIVF * try_extract_index_ivf (const Index * index);
-IndexIVF * try_extract_index_ivf (Index * index);
+const IndexIVF* try_extract_index_ivf(const Index* index);
+IndexIVF* try_extract_index_ivf(Index* index);
 
 /** Merge index1 into index0. Works on IndexIVF's and IndexIVF's
  *  embedded in a IndexPreTransform. On output, the index1 is empty.
  *
  * @param shift_ids: translate the ids from index1 to index0->prev_ntotal
  */
-void merge_into(Index *index0, Index *index1, bool shift_ids);
+void merge_into(Index* index0, Index* index1, bool shift_ids);
 
 typedef Index::idx_t idx_t;
 
@@ -57,9 +56,7 @@ typedef Index::idx_t idx_t;
  * @param centroid_ids
  *                   cluster id each object belongs to, size num_objects
  */
-void search_centroid(Index *index,
-                     const float* x, int n,
-                     idx_t* centroid_ids);
+void search_centroid(Index* index, const float* x, int n, idx_t* centroid_ids);
 
 /* Returns the cluster the embeddings belong to.
  *
@@ -71,25 +68,25 @@ void search_centroid(Index *index,
  *                   centroid ids corresponding to the results (size n * k)
  * other arguments are the same as the standard search function
  */
-void search_and_return_centroids(Index *index,
-                                 size_t n,
-                                 const float* xin,
-                                 long k,
-                                 float *distances,
-                                 idx_t* labels,
-                                 idx_t* query_centroid_ids,
-                                 idx_t* result_centroid_ids);
-
+void search_and_return_centroids(
+        Index* index,
+        size_t n,
+        const float* xin,
+        long k,
+        float* distances,
+        idx_t* labels,
+        idx_t* query_centroid_ids,
+        idx_t* result_centroid_ids);
 
 /** A set of IndexIVFs concatenated together in a FIFO fashion.
  * at each "step", the oldest index slice is removed and a new index is added.
  */
 struct SlidingIndexWindow {
     /// common index that contains the sliding window
-    Index * index;
+    Index* index;
 
     /// InvertedLists of index
-    ArrayInvertedLists *ils;
+    ArrayInvertedLists* ils;
 
     /// number of slices currently in index
     int n_slice;
@@ -98,27 +95,23 @@ struct SlidingIndexWindow {
     size_t nlist;
 
     /// cumulative list sizes at each slice
-    std::vector<std::vector<size_t> > sizes;
+    std::vector<std::vector<size_t>> sizes;
 
     /// index should be initially empty and trained
-    SlidingIndexWindow (Index *index);
+    SlidingIndexWindow(Index* index);
 
     /** Add one index to the current index and remove the oldest one.
      *
      * @param sub_index        slice to swap in (can be NULL)
      * @param remove_oldest    if true, remove the oldest slices */
-    void step(const Index *sub_index, bool remove_oldest);
-
+    void step(const Index* sub_index, bool remove_oldest);
 };
 
-
 /// Get a subset of inverted lists [i0, i1)
-ArrayInvertedLists * get_invlist_range (const Index *index,
-                                        long i0, long i1);
+ArrayInvertedLists* get_invlist_range(const Index* index, long i0, long i1);
 
 /// Set a subset of inverted lists
-void set_invlist_range (Index *index, long i0, long i1,
-                        ArrayInvertedLists * src);
+void set_invlist_range(Index* index, long i0, long i1, ArrayInvertedLists* src);
 
 /** search an IndexIVF, possibly embedded in an IndexPreTransform with
  * given parameters. This is a way to set the nprobe and get
@@ -130,25 +123,29 @@ void set_invlist_range (Index *index, long i0, long i1,
  *                 [1]: coarse quantization,
  *                 [2]: list scanning
  */
-void search_with_parameters (
-        const Index *index,
-        idx_t n, const float *x, idx_t k,
-        float *distances, idx_t *labels,
-        const IVFSearchParameters *params,
-        size_t *nb_dis = nullptr,
-        double *ms_per_stage = nullptr);
+void search_with_parameters(
+        const Index* index,
+        idx_t n,
+        const float* x,
+        idx_t k,
+        float* distances,
+        idx_t* labels,
+        const IVFSearchParameters* params,
+        size_t* nb_dis = nullptr,
+        double* ms_per_stage = nullptr);
 
 /** same as search_with_parameters but for range search */
-void range_search_with_parameters (
-        const Index *index,
-        idx_t n, const float *x, float radius,
-        RangeSearchResult *result,
-        const IVFSearchParameters *params,
-        size_t *nb_dis = nullptr,
-        double *ms_per_stage = nullptr);
-
-
-
-} } // namespace faiss::ivflib
+void range_search_with_parameters(
+        const Index* index,
+        idx_t n,
+        const float* x,
+        float radius,
+        RangeSearchResult* result,
+        const IVFSearchParameters* params,
+        size_t* nb_dis = nullptr,
+        double* ms_per_stage = nullptr);
+
+} // namespace ivflib
+} // namespace faiss
 
 #endif

data/vendor/faiss/faiss/Index.cpp

@@ -15,156 +15,138 @@
 
 #include <cstring>
 
-
 namespace faiss {
 
-Index::~Index ()
-{
-}
-
+Index::~Index() {}
 
 void Index::train(idx_t /*n*/, const float* /*x*/) {
     // does nothing by default
 }
 
-
-void Index::range_search (idx_t , const float *, float,
-                          RangeSearchResult *) const
-{
-  FAISS_THROW_MSG ("range search not implemented");
+void Index::range_search(idx_t, const float*, float, RangeSearchResult*) const {
+    FAISS_THROW_MSG("range search not implemented");
 }
 
-void Index::assign (idx_t n, const float * x, idx_t * labels, idx_t k) const
-{
-  std::vector<float> distances(n * k);
-  search (n, x, k, distances.data(), labels);
+void Index::assign(idx_t n, const float* x, idx_t* labels, idx_t k) const {
+    std::vector<float> distances(n * k);
+    search(n, x, k, distances.data(), labels);
 }
 
 void Index::add_with_ids(
-    idx_t /*n*/,
-    const float* /*x*/,
-    const idx_t* /*xids*/) {
-  FAISS_THROW_MSG ("add_with_ids not implemented for this type of index");
+        idx_t /*n*/,
+        const float* /*x*/,
+        const idx_t* /*xids*/) {
+    FAISS_THROW_MSG("add_with_ids not implemented for this type of index");
 }
 
 size_t Index::remove_ids(const IDSelector& /*sel*/) {
-  FAISS_THROW_MSG ("remove_ids not implemented for this type of index");
-  return -1;
+    FAISS_THROW_MSG("remove_ids not implemented for this type of index");
+    return -1;
 }
 
-
-void Index::reconstruct (idx_t, float * ) const {
-  FAISS_THROW_MSG ("reconstruct not implemented for this type of index");
+void Index::reconstruct(idx_t, float*) const {
+    FAISS_THROW_MSG("reconstruct not implemented for this type of index");
 }
 
-
-void Index::reconstruct_n (idx_t i0, idx_t ni, float *recons) const {
-  for (idx_t i = 0; i < ni; i++) {
-    reconstruct (i0 + i, recons + i * d);
-  }
+void Index::reconstruct_n(idx_t i0, idx_t ni, float* recons) const {
+    for (idx_t i = 0; i < ni; i++) {
+        reconstruct(i0 + i, recons + i * d);
+    }
 }
 
-
-void Index::search_and_reconstruct (idx_t n, const float *x, idx_t k,
-                                    float *distances, idx_t *labels,
-                                    float *recons) const {
-  search (n, x, k, distances, labels);
-  for (idx_t i = 0; i < n; ++i) {
-    for (idx_t j = 0; j < k; ++j) {
-      idx_t ij = i * k + j;
-      idx_t key = labels[ij];
-      float* reconstructed = recons + ij * d;
-      if (key < 0) {
-        // Fill with NaNs
-        memset(reconstructed, -1, sizeof(*reconstructed) * d);
-      } else {
-        reconstruct (key, reconstructed);
-      }
+void Index::search_and_reconstruct(
+        idx_t n,
+        const float* x,
+        idx_t k,
+        float* distances,
+        idx_t* labels,
+        float* recons) const {
+    FAISS_THROW_IF_NOT(k > 0);
+
+    search(n, x, k, distances, labels);
+    for (idx_t i = 0; i < n; ++i) {
+        for (idx_t j = 0; j < k; ++j) {
+            idx_t ij = i * k + j;
+            idx_t key = labels[ij];
+            float* reconstructed = recons + ij * d;
+            if (key < 0) {
+                // Fill with NaNs
+                memset(reconstructed, -1, sizeof(*reconstructed) * d);
+            } else {
+                reconstruct(key, reconstructed);
+            }
+        }
     }
-  }
 }
 
-void Index::compute_residual (const float * x,
-                              float * residual, idx_t key) const {
-  reconstruct (key, residual);
-  for (size_t i = 0; i < d; i++) {
-    residual[i] = x[i] - residual[i];
-  }
+void Index::compute_residual(const float* x, float* residual, idx_t key) const {
+    reconstruct(key, residual);
+    for (size_t i = 0; i < d; i++) {
+        residual[i] = x[i] - residual[i];
+    }
 }
 
-void Index::compute_residual_n (idx_t n, const float* xs,
-                                float* residuals,
-                                const idx_t* keys) const {
+void Index::compute_residual_n(
+        idx_t n,
+        const float* xs,
+        float* residuals,
+        const idx_t* keys) const {
 #pragma omp parallel for
-  for (idx_t i = 0; i < n; ++i) {
-    compute_residual(&xs[i * d], &residuals[i * d], keys[i]);
-  }
+    for (idx_t i = 0; i < n; ++i) {
+        compute_residual(&xs[i * d], &residuals[i * d], keys[i]);
+    }
 }
 
-
-
-size_t Index::sa_code_size () const
-{
-    FAISS_THROW_MSG ("standalone codec not implemented for this type of index");
+size_t Index::sa_code_size() const {
+    FAISS_THROW_MSG("standalone codec not implemented for this type of index");
 }
 
-void Index::sa_encode (idx_t, const float *,
-                             uint8_t *) const
-{
-    FAISS_THROW_MSG ("standalone codec not implemented for this type of index");
+void Index::sa_encode(idx_t, const float*, uint8_t*) const {
+    FAISS_THROW_MSG("standalone codec not implemented for this type of index");
 }
 
-void Index::sa_decode (idx_t, const uint8_t *,
-                            float *) const
-{
-    FAISS_THROW_MSG ("standalone codec not implemented for this type of index");
+void Index::sa_decode(idx_t, const uint8_t*, float*) const {
+    FAISS_THROW_MSG("standalone codec not implemented for this type of index");
 }
 
-
 namespace {
 
-
 // storage that explicitly reconstructs vectors before computing distances
 struct GenericDistanceComputer : DistanceComputer {
-  size_t d;
-  const Index& storage;
-  std::vector<float> buf;
-  const float *q;
-
-  explicit GenericDistanceComputer(const Index& storage)
-      : storage(storage) {
-    d = storage.d;
-    buf.resize(d * 2);
-  }
-
-  float operator () (idx_t i) override {
-    storage.reconstruct(i, buf.data());
-    return fvec_L2sqr(q, buf.data(), d);
-  }
-
-  float symmetric_dis(idx_t i, idx_t j) override {
-    storage.reconstruct(i, buf.data());
-    storage.reconstruct(j, buf.data() + d);
-    return fvec_L2sqr(buf.data() + d, buf.data(), d);
-  }
-
-  void set_query(const float *x) override {
-    q = x;
-  }
+    size_t d;
+    const Index& storage;
+    std::vector<float> buf;
+    const float* q;
+
+    explicit GenericDistanceComputer(const Index& storage) : storage(storage) {
+        d = storage.d;
+        buf.resize(d * 2);
+    }
 
-};
+    float operator()(idx_t i) override {
+        storage.reconstruct(i, buf.data());
+        return fvec_L2sqr(q, buf.data(), d);
+    }
 
+    float symmetric_dis(idx_t i, idx_t j) override {
+        storage.reconstruct(i, buf.data());
+        storage.reconstruct(j, buf.data() + d);
+        return fvec_L2sqr(buf.data() + d, buf.data(), d);
+    }
 
-}  // namespace
+    void set_query(const float* x) override {
+        q = x;
+    }
+};
 
+} // namespace
 
-DistanceComputer * Index::get_distance_computer() const {
+DistanceComputer* Index::get_distance_computer() const {
     if (metric_type == METRIC_L2) {
         return new GenericDistanceComputer(*this);
     } else {
-        FAISS_THROW_MSG ("get_distance_computer() not implemented");
+        FAISS_THROW_MSG("get_distance_computer() not implemented");
     }
 }
 
-
-}
+} // namespace faiss

data/vendor/faiss/faiss/Index.h

@@ -12,13 +12,13 @@
 
 #include <faiss/MetricType.h>
 #include <cstdio>
-#include <typeinfo>
-#include <string>
 #include <sstream>
+#include <string>
+#include <typeinfo>
 
 #define FAISS_VERSION_MAJOR 1
 #define FAISS_VERSION_MINOR 7
-#define FAISS_VERSION_PATCH 0
+#define FAISS_VERSION_PATCH 1
 
 /**
  * @namespace faiss
@@ -36,7 +36,6 @@
  * an n*d matrix, which implies a row-major storage.
  */
 
-
 namespace faiss {
 
 /// Forward declarations see AuxIndexStructures.h
@@ -50,13 +49,13 @@ struct DistanceComputer;
  * although the internal representation may vary.
  */
 struct Index {
-    using idx_t = int64_t;  ///< all indices are this type
+    using idx_t = int64_t; ///< all indices are this type
     using component_t = float;
     using distance_t = float;
 
-    int d;                 ///< vector dimension
-    idx_t ntotal;          ///< total nb of indexed vectors
-    bool verbose;          ///< verbosity level
+    int d;        ///< vector dimension
+    idx_t ntotal; ///< total nb of indexed vectors
+    bool verbose; ///< verbosity level
 
     /// set if the Index does not require training, or if training is
     /// done already
@@ -64,18 +63,17 @@ struct Index {
 
     /// type of metric this index uses for search
     MetricType metric_type;
-    float metric_arg;     ///< argument of the metric type
+    float metric_arg; ///< argument of the metric type
 
-    explicit Index (idx_t d = 0, MetricType metric = METRIC_L2):
-                    d(d),
-                    ntotal(0),
-                    verbose(false),
-                    is_trained(true),
-                    metric_type (metric),
-                    metric_arg(0) {}
-
-    virtual ~Index ();
+    explicit Index(idx_t d = 0, MetricType metric = METRIC_L2)
+            : d(d),
+              ntotal(0),
+              verbose(false),
+              is_trained(true),
+              metric_type(metric),
+              metric_arg(0) {}
 
+    virtual ~Index();
 
     /** Perform training on a representative set of vectors
      *
@@ -87,11 +85,11 @@ struct Index {
     /** Add n vectors of dimension d to the index.
      *
      * Vectors are implicitly assigned labels ntotal .. ntotal + n - 1
-     * This function slices the input vectors in chuncks smaller than
+     * This function slices the input vectors in chunks smaller than
      * blocksize_add and calls add_core.
      * @param x      input matrix, size n * d
      */
-    virtual void add (idx_t n, const float *x) = 0;
+    virtual void add(idx_t n, const float* x) = 0;
 
     /** Same as add, but stores xids instead of sequential ids.
      *
@@ -100,7 +98,7 @@ struct Index {
      *
      * @param xids if non-null, ids to store for the vectors (size n)
      */
-    virtual void add_with_ids (idx_t n, const float * x, const idx_t *xids);
+    virtual void add_with_ids(idx_t n, const float* x, const idx_t* xids);
 
     /** query n vectors of dimension d to the index.
      *
@@ -111,8 +109,12 @@ struct Index {
      * @param labels      output labels of the NNs, size n*k
      * @param distances   output pairwise distances, size n*k
      */
-    virtual void search (idx_t n, const float *x, idx_t k,
-                         float *distances, idx_t *labels) const = 0;
+    virtual void search(
+            idx_t n,
+            const float* x,
+            idx_t k,
+            float* distances,
+            idx_t* labels) const = 0;
 
     /** query n vectors of dimension d to the index.
      *
@@ -124,8 +126,11 @@ struct Index {
      * @param radius      search radius
      * @param result      result table
      */
-    virtual void range_search (idx_t n, const float *x, float radius,
-                               RangeSearchResult *result) const;
+    virtual void range_search(
+            idx_t n,
+            const float* x,
+            float radius,
+            RangeSearchResult* result) const;
 
     /** return the indexes of the k vectors closest to the query x.
      *
@@ -133,7 +138,8 @@ struct Index {
      * @param x           input vectors to search, size n * d
      * @param labels      output labels of the NNs, size n*k
      */
-    virtual void assign (idx_t n, const float * x, idx_t * labels, idx_t k = 1) const;
+    virtual void assign(idx_t n, const float* x, idx_t* labels, idx_t k = 1)
+            const;
 
     /// removes all elements from the database.
     virtual void reset() = 0;
@@ -141,7 +147,7 @@ struct Index {
     /** removes IDs from the index. Not supported by all
      * indexes. Returns the number of elements removed.
      */
-    virtual size_t remove_ids (const IDSelector & sel);
+    virtual size_t remove_ids(const IDSelector& sel);
 
     /** Reconstruct a stored vector (or an approximation if lossy coding)
      *
@@ -149,14 +155,14 @@ struct Index {
      * @param key         id of the vector to reconstruct
      * @param recons      reconstucted vector (size d)
      */
-    virtual void reconstruct (idx_t key, float * recons) const;
+    virtual void reconstruct(idx_t key, float* recons) const;
 
     /** Reconstruct vectors i0 to i0 + ni - 1
      *
      * this function may not be defined for some indexes
      * @param recons      reconstucted vector (size ni * d)
      */
-    virtual void reconstruct_n (idx_t i0, idx_t ni, float *recons) const;
+    virtual void reconstruct_n(idx_t i0, idx_t ni, float* recons) const;
 
     /** Similar to search, but also reconstructs the stored vectors (or an
      * approximation in the case of lossy coding) for the search results.
@@ -166,9 +172,13 @@ struct Index {
      *
      * @param recons      reconstructed vectors size (n, k, d)
      **/
-    virtual void search_and_reconstruct (idx_t n, const float *x, idx_t k,
-                                         float *distances, idx_t *labels,
-                                         float *recons) const;
+    virtual void search_and_reconstruct(
+            idx_t n,
+            const float* x,
+            idx_t k,
+            float* distances,
+            idx_t* labels,
+            float* recons) const;
 
     /** Computes a residual vector after indexing encoding.
      *
@@ -181,8 +191,8 @@ struct Index {
      * @param residual    output residual vector, size d
      * @param key         encoded index, as returned by search and assign
      */
-    virtual void compute_residual (const float * x,
-                                   float * residual, idx_t key) const;
+    virtual void compute_residual(const float* x, float* residual, idx_t key)
+            const;
 
     /** Computes a residual vector after indexing encoding (batch form).
      * Equivalent to calling compute_residual for each vector.
@@ -197,9 +207,11 @@ struct Index {
      * @param residuals   output residual vectors, size (n x d)
      * @param keys        encoded index, as returned by search and assign
      */
-    virtual void compute_residual_n (idx_t n, const float* xs,
-                                     float* residuals,
-                                     const idx_t* keys) const;
+    virtual void compute_residual_n(
+            idx_t n,
+            const float* xs,
+            float* residuals,
+            const idx_t* keys) const;
 
     /** Get a DistanceComputer (defined in AuxIndexStructures) object
      * for this kind of index.
@@ -207,13 +219,12 @@ struct Index {
      * DistanceComputer is implemented for indexes that support random
      * access of their vectors.
      */
-    virtual DistanceComputer * get_distance_computer() const;
-
+    virtual DistanceComputer* get_distance_computer() const;
 
     /* The standalone codec interface */
 
     /** size of the produced codes in bytes */
-    virtual size_t sa_code_size () const;
+    virtual size_t sa_code_size() const;
 
     /** encode a set of vectors
      *
@@ -221,8 +232,7 @@ struct Index {
      * @param x       input vectors, size n * d
      * @param bytes   output encoded vectors, size n * sa_code_size()
      */
-    virtual void sa_encode (idx_t n, const float *x,
-                                  uint8_t *bytes) const;
+    virtual void sa_encode(idx_t n, const float* x, uint8_t* bytes) const;
 
     /** encode a set of vectors
      *
@@ -230,13 +240,9 @@ struct Index {
      * @param bytes   input encoded vectors, size n * sa_code_size()
      * @param x       output vectors, size n * d
      */
-    virtual void sa_decode (idx_t n, const uint8_t *bytes,
-                                    float *x) const;
-
-
+    virtual void sa_decode(idx_t n, const uint8_t* bytes, float* x) const;
 };
 
-}
-
+} // namespace faiss
 
 #endif