faiss 0.1.7 → 0.2.3

data/vendor/faiss/faiss/IndexShards.h

@@ -18,82 +18,94 @@ namespace faiss {
  */
 template <typename IndexT>
 struct IndexShardsTemplate : public ThreadedIndex<IndexT> {
-  using idx_t = typename IndexT::idx_t;
-  using component_t = typename IndexT::component_t;
-  using distance_t = typename IndexT::distance_t;
-
-  /**
-   * The dimension that all sub-indices must share will be the dimension of the
-   * first sub-index added
-   *
-   * @param threaded     do we use one thread per sub_index or do
-   *                     queries sequentially?
-   * @param successive_ids should we shift the returned ids by
-   *                     the size of each sub-index or return them
-   *                     as they are?
-   */
-  explicit IndexShardsTemplate(bool threaded = false,
-                               bool successive_ids = true);
-
-  /**
-   * @param threaded     do we use one thread per sub_index or do
-   *                     queries sequentially?
-   * @param successive_ids should we shift the returned ids by
-   *                     the size of each sub-index or return them
-   *                     as they are?
-   */
-  explicit IndexShardsTemplate(idx_t d,
-                               bool threaded = false,
-                               bool successive_ids = true);
-
-  /// int version due to the implicit bool conversion ambiguity of int as
-  /// dimension
-  explicit IndexShardsTemplate(int d,
-                               bool threaded = false,
-                               bool successive_ids = true);
-
-  /// Alias for addIndex()
-  void add_shard(IndexT* index) { this->addIndex(index); }
-
-  /// Alias for removeIndex()
-  void remove_shard(IndexT* index) { this->removeIndex(index); }
-
-  /// supported only for sub-indices that implement add_with_ids
-  void add(idx_t n, const component_t* x) override;
-
-  /**
-   * Cases (successive_ids, xids):
-   * - true, non-NULL       ERROR: it makes no sense to pass in ids and
-   *                        request them to be shifted
-   * - true, NULL           OK, but should be called only once (calls add()
-   *                        on sub-indexes).
-   * - false, non-NULL      OK: will call add_with_ids with passed in xids
-   *                        distributed evenly over shards
-   * - false, NULL          OK: will call add_with_ids on each sub-index,
-   *                        starting at ntotal
-   */
-  void add_with_ids(idx_t n, const component_t* x, const idx_t* xids) override;
-
-  void search(idx_t n, const component_t* x, idx_t k,
-              distance_t* distances, idx_t* labels) const override;
-
-  void train(idx_t n, const component_t* x) override;
-
-  bool successive_ids;
-
-  /// Synchronize the top-level index (IndexShards) with data in the sub-indices
-  void syncWithSubIndexes();
-
- protected:
-  /// Called just after an index is added
-  void onAfterAddIndex(IndexT* index) override;
-
-  /// Called just after an index is removed
-  void onAfterRemoveIndex(IndexT* index) override;
+    using idx_t = typename IndexT::idx_t;
+    using component_t = typename IndexT::component_t;
+    using distance_t = typename IndexT::distance_t;
+
+    /**
+     * The dimension that all sub-indices must share will be the dimension of
+     * the first sub-index added
+     *
+     * @param threaded     do we use one thread per sub_index or do
+     *                     queries sequentially?
+     * @param successive_ids should we shift the returned ids by
+     *                     the size of each sub-index or return them
+     *                     as they are?
+     */
+    explicit IndexShardsTemplate(
+            bool threaded = false,
+            bool successive_ids = true);
+
+    /**
+     * @param threaded     do we use one thread per sub_index or do
+     *                     queries sequentially?
+     * @param successive_ids should we shift the returned ids by
+     *                     the size of each sub-index or return them
+     *                     as they are?
+     */
+    explicit IndexShardsTemplate(
+            idx_t d,
+            bool threaded = false,
+            bool successive_ids = true);
+
+    /// int version due to the implicit bool conversion ambiguity of int as
+    /// dimension
+    explicit IndexShardsTemplate(
+            int d,
+            bool threaded = false,
+            bool successive_ids = true);
+
+    /// Alias for addIndex()
+    void add_shard(IndexT* index) {
+        this->addIndex(index);
+    }
+
+    /// Alias for removeIndex()
+    void remove_shard(IndexT* index) {
+        this->removeIndex(index);
+    }
+
+    /// supported only for sub-indices that implement add_with_ids
+    void add(idx_t n, const component_t* x) override;
+
+    /**
+     * Cases (successive_ids, xids):
+     * - true, non-NULL       ERROR: it makes no sense to pass in ids and
+     *                        request them to be shifted
+     * - true, NULL           OK, but should be called only once (calls add()
+     *                        on sub-indexes).
+     * - false, non-NULL      OK: will call add_with_ids with passed in xids
+     *                        distributed evenly over shards
+     * - false, NULL          OK: will call add_with_ids on each sub-index,
+     *                        starting at ntotal
+     */
+    void add_with_ids(idx_t n, const component_t* x, const idx_t* xids)
+            override;
+
+    void search(
+            idx_t n,
+            const component_t* x,
+            idx_t k,
+            distance_t* distances,
+            idx_t* labels) const override;
+
+    void train(idx_t n, const component_t* x) override;
+
+    bool successive_ids;
+
+    /// Synchronize the top-level index (IndexShards) with data in the
+    /// sub-indices
+    void syncWithSubIndexes();
+
+   protected:
+    /// Called just after an index is added
+    void onAfterAddIndex(IndexT* index) override;
+
+    /// Called just after an index is removed
+    void onAfterRemoveIndex(IndexT* index) override;
 };
 
 using IndexShards = IndexShardsTemplate<Index>;
 using IndexBinaryShards = IndexShardsTemplate<IndexBinary>;
 
-
 } // namespace faiss

data/vendor/faiss/faiss/MatrixStats.cpp

@@ -7,15 +7,13 @@
 
 // -*- c++ -*-
 
-
 #include <faiss/MatrixStats.h>
 
+#include <stdarg.h> /* va_list, va_start, va_arg, va_end */
 
-#include <stdarg.h>     /* va_list, va_start, va_arg, va_end */
-
+#include <faiss/utils/utils.h>
 #include <cmath>
 #include <cstdio>
-#include <faiss/utils/utils.h>
 
 namespace faiss {
 
@@ -23,16 +21,19 @@ namespace faiss {
  * MatrixStats
  *********************************************************************/
 
-MatrixStats::PerDimStats::PerDimStats():
-    n(0), n_nan(0), n_inf(0), n0(0),
-    min(HUGE_VALF), max(-HUGE_VALF),
-    sum(0), sum2(0),
-    mean(NAN), stddev(NAN)
-{}
-
-
-void MatrixStats::PerDimStats::add (float x)
-{
+MatrixStats::PerDimStats::PerDimStats()
+        : n(0),
+          n_nan(0),
+          n_inf(0),
+          n0(0),
+          min(HUGE_VALF),
+          max(-HUGE_VALF),
+          sum(0),
+          sum2(0),
+          mean(NAN),
+          stddev(NAN) {}
+
+void MatrixStats::PerDimStats::add(float x) {
     n++;
     if (std::isnan(x)) {
         n_nan++;
@@ -42,25 +43,26 @@ void MatrixStats::PerDimStats::add (float x)
         n_inf++;
         return;
     }
-    if (x == 0) n0++;
-    if (x < min) min = x;
-    if (x > max) max = x;
+    if (x == 0)
+        n0++;
+    if (x < min)
+        min = x;
+    if (x > max)
+        max = x;
     sum += x;
     sum2 += (double)x * (double)x;
 }
 
-void MatrixStats::PerDimStats::compute_mean_std ()
-{
+void MatrixStats::PerDimStats::compute_mean_std() {
     n_valid = n - n_nan - n_inf;
     mean = sum / n_valid;
     double var = sum2 / n_valid - mean * mean;
-    if (var < 0) var = 0;
+    if (var < 0)
+        var = 0;
     stddev = sqrt(var);
 }
 
-
-void MatrixStats::do_comment (const char *fmt, ...)
-{
+void MatrixStats::do_comment(const char* fmt, ...) {
     va_list ap;
 
     /* Determine required size */
@@ -72,57 +74,60 @@ void MatrixStats::do_comment (const char *fmt, ...)
     buf += size;
 }
 
-
-
-MatrixStats::MatrixStats (size_t n, size_t d, const float *x):
-    n(n), d(d),
-    n_collision(0), n_valid(0), n0(0),
-    min_norm2(HUGE_VAL), max_norm2(0)
-{
-    std::vector<char> comment_buf (10000);
-    buf = comment_buf.data ();
+MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
+        : n(n),
+          d(d),
+          n_collision(0),
+          n_valid(0),
+          n0(0),
+          min_norm2(HUGE_VAL),
+          max_norm2(0) {
+    std::vector<char> comment_buf(10000);
+    buf = comment_buf.data();
     nbuf = comment_buf.size();
 
-    do_comment ("analyzing %ld vectors of size %ld\n", n, d);
+    do_comment("analyzing %ld vectors of size %ld\n", n, d);
 
     if (d > 1024) {
-        do_comment (
-           "indexing this many dimensions is hard, "
-           "please consider dimensionality reducution (with PCAMatrix)\n");
+        do_comment(
+                "indexing this many dimensions is hard, "
+                "please consider dimensionality reducution (with PCAMatrix)\n");
     }
 
-    size_t nbytes = sizeof (x[0]) * d;
-    per_dim_stats.resize (d);
+    size_t nbytes = sizeof(x[0]) * d;
+    per_dim_stats.resize(d);
 
     for (size_t i = 0; i < n; i++) {
-        const float *xi = x + d * i;
+        const float* xi = x + d * i;
         double sum2 = 0;
         for (size_t j = 0; j < d; j++) {
-            per_dim_stats[j].add (xi[j]);
+            per_dim_stats[j].add(xi[j]);
             sum2 += xi[j] * (double)xi[j];
         }
 
-        if (std::isfinite (sum2)) {
+        if (std::isfinite(sum2)) {
             n_valid++;
             if (sum2 == 0) {
-                n0 ++;
+                n0++;
             } else {
-                if (sum2 < min_norm2) min_norm2 = sum2;
-                if (sum2 > max_norm2) max_norm2 = sum2;
+                if (sum2 < min_norm2)
+                    min_norm2 = sum2;
+                if (sum2 > max_norm2)
+                    max_norm2 = sum2;
             }
         }
 
         { // check hash
             uint64_t hash = hash_bytes((const uint8_t*)xi, nbytes);
-            auto elt = occurrences.find (hash);
+            auto elt = occurrences.find(hash);
             if (elt == occurrences.end()) {
                 Occurrence occ = {i, 1};
                 occurrences[hash] = occ;
             } else {
-                if (!memcmp (xi, x + elt->second.first * d, nbytes)) {
-                    elt->second.count ++;
+                if (!memcmp(xi, x + elt->second.first * d, nbytes)) {
+                    elt->second.count++;
                 } else {
-                    n_collision ++;
+                    n_collision++;
                     // we should use a list of collisions but overkill
                 }
             }
@@ -131,50 +136,59 @@ MatrixStats::MatrixStats (size_t n, size_t d, const float *x):
 
     // invalid vecor stats
     if (n_valid == n) {
-        do_comment ("no NaN or Infs in data\n");
+        do_comment("no NaN or Infs in data\n");
     } else {
-        do_comment ("%ld vectors contain NaN or Inf "
-                 "(or have too large components), "
-                 "expect bad results with indexing!\n", n - n_valid);
+        do_comment(
+                "%ld vectors contain NaN or Inf "
+                "(or have too large components), "
+                "expect bad results with indexing!\n",
+                n - n_valid);
     }
 
     // copies in dataset
     if (occurrences.size() == n) {
-        do_comment ("all vectors are distinct\n");
+        do_comment("all vectors are distinct\n");
     } else {
-        do_comment ("%ld vectors are distinct (%.2f%%)\n",
-                 occurrences.size(),
-                 occurrences.size() * 100.0 / n);
+        do_comment(
+                "%ld vectors are distinct (%.2f%%)\n",
+                occurrences.size(),
+                occurrences.size() * 100.0 / n);
 
         if (n_collision > 0) {
-            do_comment ("%ld collisions in hash table, "
-                     "counts may be invalid\n", n_collision);
+            do_comment(
+                    "%ld collisions in hash table, "
+                    "counts may be invalid\n",
+                    n_collision);
         }
 
         Occurrence max = {0, 0};
-        for (auto it = occurrences.begin();
-             it != occurrences.end(); ++it) {
+        for (auto it = occurrences.begin(); it != occurrences.end(); ++it) {
             if (it->second.count > max.count) {
                 max = it->second;
             }
         }
-        do_comment ("vector %ld has %ld copies\n", max.first, max.count);
+        do_comment("vector %ld has %ld copies\n", max.first, max.count);
     }
 
     { // norm stats
-        min_norm2 = sqrt (min_norm2);
-        max_norm2 = sqrt (max_norm2);
-        do_comment ("range of L2 norms=[%g, %g] (%ld null vectors)\n",
-                 min_norm2, max_norm2, n0);
+        min_norm2 = sqrt(min_norm2);
+        max_norm2 = sqrt(max_norm2);
+        do_comment(
+                "range of L2 norms=[%g, %g] (%ld null vectors)\n",
+                min_norm2,
+                max_norm2,
+                n0);
 
         if (max_norm2 < min_norm2 * 1.0001) {
-            do_comment ("vectors are normalized, inner product and "
-                     "L2  search are equivalent\n");
+            do_comment(
+                    "vectors are normalized, inner product and "
+                    "L2  search are equivalent\n");
         }
 
         if (max_norm2 > min_norm2 * 100) {
-            do_comment ("vectors have very large differences in norms, "
-                     "is this normal?\n");
+            do_comment(
+                    "vectors have very large differences in norms, "
+                    "is this normal?\n");
         }
     }
 
@@ -185,68 +199,69 @@ MatrixStats::MatrixStats (size_t n, size_t d, const float *x):
         size_t n_dangerous_range = 0, n_0_range = 0, n0 = 0;
 
         for (size_t j = 0; j < d; j++) {
-            PerDimStats &st = per_dim_stats[j];
-            st.compute_mean_std ();
+            PerDimStats& st = per_dim_stats[j];
+            st.compute_mean_std();
             n0 += st.n0;
 
             if (st.max == st.min) {
-                n_0_range ++;
+                n_0_range++;
             } else if (st.max < 1.001 * st.min) {
-                n_dangerous_range ++;
+                n_dangerous_range++;
             }
 
-            if (st.stddev > max_std) max_std = st.stddev;
-            if (st.stddev < min_std) min_std = st.stddev;
+            if (st.stddev > max_std)
+                max_std = st.stddev;
+            if (st.stddev < min_std)
+                min_std = st.stddev;
         }
 
-
-
         if (n0 == 0) {
-            do_comment ("matrix contains no 0s\n");
+            do_comment("matrix contains no 0s\n");
         } else {
-            do_comment ("matrix contains %.2f %% 0 entries\n",
-                     n0 * 100.0 / (n * d));
+            do_comment(
+                    "matrix contains %.2f %% 0 entries\n",
+                    n0 * 100.0 / (n * d));
         }
 
         if (n_0_range == 0) {
-            do_comment ("no constant dimensions\n");
+            do_comment("no constant dimensions\n");
         } else {
-            do_comment ("%ld dimensions are constant: they can be removed\n",
-                     n_0_range);
+            do_comment(
+                    "%ld dimensions are constant: they can be removed\n",
+                    n_0_range);
         }
 
         if (n_dangerous_range == 0) {
-            do_comment ("no dimension has a too large mean\n");
+            do_comment("no dimension has a too large mean\n");
         } else {
-            do_comment ("%ld dimensions are too large "
-                     "wrt. their variance, may loose precision "
-                     "in IndexFlatL2 (use CenteringTransform)\n",
-                     n_dangerous_range);
+            do_comment(
+                    "%ld dimensions are too large "
+                    "wrt. their variance, may loose precision "
+                    "in IndexFlatL2 (use CenteringTransform)\n",
+                    n_dangerous_range);
         }
 
-        do_comment ("stddevs per dimension are in [%g %g]\n", min_std, max_std);
+        do_comment("stddevs per dimension are in [%g %g]\n", min_std, max_std);
 
         size_t n_small_var = 0;
 
         for (size_t j = 0; j < d; j++) {
-            const PerDimStats &st = per_dim_stats[j];
+            const PerDimStats& st = per_dim_stats[j];
             if (st.stddev < max_std * 1e-4) {
                 n_small_var++;
             }
         }
 
         if (n_small_var > 0) {
-            do_comment ("%ld dimensions have negligible stddev wrt. "
-                     "the largest dimension, they could be ignored",
-                     n_small_var);
+            do_comment(
+                    "%ld dimensions have negligible stddev wrt. "
+                    "the largest dimension, they could be ignored",
+                    n_small_var);
         }
-
     }
-    comments = comment_buf.data ();
+    comments = comment_buf.data();
     buf = nullptr;
     nbuf = 0;
 }
 
-
-
 } // namespace faiss

data/vendor/faiss/faiss/MatrixStats.h

@@ -9,22 +9,20 @@
 
 #pragma once
 
-#include <vector>
+#include <stdint.h>
 #include <string>
 #include <unordered_map>
-#include <stdint.h>
-
+#include <vector>
 
 namespace faiss {
 
-
 /** Reports some statistics on a dataset and comments on them.
  *
  * It is a class rather than a function so that all stats can also be
  * accessed from code */
 
 struct MatrixStats {
-    MatrixStats (size_t n, size_t d, const float *x);
+    MatrixStats(size_t n, size_t d, const float* x);
     std::string comments;
 
     // raw statistics
@@ -42,8 +40,8 @@ struct MatrixStats {
         double mean, stddev;
 
         PerDimStats();
-        void add (float x);
-        void compute_mean_std ();
+        void add(float x);
+        void compute_mean_std();
     };
 
     std::vector<PerDimStats> per_dim_stats;
@@ -53,10 +51,9 @@ struct MatrixStats {
     };
     std::unordered_map<uint64_t, Occurrence> occurrences;
 
-    char *buf;
+    char* buf;
     size_t nbuf;
-    void do_comment (const char *fmt, ...);
-
+    void do_comment(const char* fmt, ...);
 };
 
 } // namespace faiss