faiss 0.3.0 → 0.3.2

data/vendor/faiss/faiss/MatrixStats.cpp

@@ -9,9 +9,10 @@
 
 #include <faiss/MatrixStats.h>
 
-#include <stdarg.h> /* va_list, va_start, va_arg, va_end */
+#include <cstdarg> /* va_list, va_start, va_arg, va_end */
 
 #include <faiss/utils/utils.h>
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 
@@ -21,18 +22,6 @@ namespace faiss {
  * MatrixStats
  *********************************************************************/
 
-MatrixStats::PerDimStats::PerDimStats()
-        : n(0),
-          n_nan(0),
-          n_inf(0),
-          n0(0),
-          min(HUGE_VALF),
-          max(-HUGE_VALF),
-          sum(0),
-          sum2(0),
-          mean(NAN),
-          stddev(NAN) {}
-
 void MatrixStats::PerDimStats::add(float x) {
     n++;
     if (std::isnan(x)) {
@@ -74,19 +63,12 @@ void MatrixStats::do_comment(const char* fmt, ...) {
     buf += size;
 }
 
-MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
-        : n(n),
-          d(d),
-          n_collision(0),
-          n_valid(0),
-          n0(0),
-          min_norm2(HUGE_VAL),
-          max_norm2(0) {
+MatrixStats::MatrixStats(size_t n, size_t d, const float* x) : n(n), d(d) {
     std::vector<char> comment_buf(10000);
     buf = comment_buf.data();
     nbuf = comment_buf.size();
 
-    do_comment("analyzing %ld vectors of size %ld\n", n, d);
+    do_comment("analyzing %zd vectors of size %zd\n", n, d);
 
     if (d > 1024) {
         do_comment(
@@ -94,6 +76,9 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
                 "please consider dimensionality reducution (with PCAMatrix)\n");
     }
 
+    hash_value = hash_bytes((const uint8_t*)x, n * d * sizeof(*x));
+    do_comment("hash value 0x%016" PRIx64 "\n", hash_value);
+
     size_t nbytes = sizeof(x[0]) * d;
     per_dim_stats.resize(d);
 
@@ -156,7 +141,7 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
 
         if (n_collision > 0) {
             do_comment(
-                    "%ld collisions in hash table, "
+                    "%zd collisions in hash table, "
                     "counts may be invalid\n",
                     n_collision);
         }
@@ -167,14 +152,14 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
                 max = it->second;
             }
         }
-        do_comment("vector %ld has %ld copies\n", max.first, max.count);
+        do_comment("vector %zd has %zd copies\n", max.first, max.count);
     }
 
     { // norm stats
         min_norm2 = sqrt(min_norm2);
         max_norm2 = sqrt(max_norm2);
         do_comment(
-                "range of L2 norms=[%g, %g] (%ld null vectors)\n",
+                "range of L2 norms=[%g, %g] (%zd null vectors)\n",
                 min_norm2,
                 max_norm2,
                 n0);
@@ -182,7 +167,7 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
         if (max_norm2 < min_norm2 * 1.0001) {
             do_comment(
                     "vectors are normalized, inner product and "
-                    "L2  search are equivalent\n");
+                    "L2 search are equivalent\n");
         }
 
         if (max_norm2 > min_norm2 * 100) {
@@ -196,12 +181,12 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
 
         double max_std = 0, min_std = HUGE_VAL;
 
-        size_t n_dangerous_range = 0, n_0_range = 0, n0 = 0;
+        size_t n_dangerous_range = 0, n_0_range = 0, n0_2 = 0;
 
         for (size_t j = 0; j < d; j++) {
             PerDimStats& st = per_dim_stats[j];
             st.compute_mean_std();
-            n0 += st.n0;
+            n0_2 += st.n0;
 
             if (st.max == st.min) {
                 n_0_range++;
@@ -215,19 +200,19 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
                 min_std = st.stddev;
         }
 
-        if (n0 == 0) {
+        if (n0_2 == 0) {
             do_comment("matrix contains no 0s\n");
         } else {
             do_comment(
                     "matrix contains %.2f %% 0 entries\n",
-                    n0 * 100.0 / (n * d));
+                    n0_2 * 100.0 / (n * d));
         }
 
         if (n_0_range == 0) {
             do_comment("no constant dimensions\n");
         } else {
             do_comment(
-                    "%ld dimensions are constant: they can be removed\n",
+                    "%zd dimensions are constant: they can be removed\n",
                     n_0_range);
         }
 
@@ -235,7 +220,7 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x)
             do_comment("no dimension has a too large mean\n");
         } else {
             do_comment(
-                    "%ld dimensions are too large "
+                    "%zd dimensions are too large "
                     "wrt. their variance, may loose precision "
                     "in IndexFlatL2 (use CenteringTransform)\n",
                     n_dangerous_range);

data/vendor/faiss/faiss/MatrixStats.h

@@ -10,6 +10,7 @@
 #pragma once
 
 #include <stdint.h>
+#include <cmath>
 #include <string>
 #include <unordered_map>
 #include <vector>
@@ -26,20 +27,31 @@ struct MatrixStats {
     std::string comments;
 
     // raw statistics
-    size_t n, d;
-    size_t n_collision, n_valid, n0;
-    double min_norm2, max_norm2;
+    size_t n = 0, d = 0;
+    size_t n_collision = 0;
+    size_t n_valid = 0;
+    size_t n0 = 0;
+    double min_norm2 = HUGE_VALF;
+    double max_norm2 = 0;
+    uint64_t hash_value = 0;
 
     struct PerDimStats {
-        size_t n, n_nan, n_inf, n0;
+        /// counts of various special entries
+        size_t n = 0;
+        size_t n_nan = 0;
+        size_t n_inf = 0;
+        size_t n0 = 0;
 
-        float min, max;
-        double sum, sum2;
+        /// to get min/max and stddev values
+        float min = HUGE_VALF;
+        float max = -HUGE_VALF;
+        double sum = 0;
+        double sum2 = 0;
 
-        size_t n_valid;
-        double mean, stddev;
+        size_t n_valid = 0;
+        double mean = NAN;
+        double stddev = NAN;
 
-        PerDimStats();
         void add(float x);
         void compute_mean_std();
     };

data/vendor/faiss/faiss/MetaIndexes.cpp

@@ -9,8 +9,8 @@
 
 #include <faiss/MetaIndexes.h>
 
-#include <stdint.h>
 #include <cinttypes>
+#include <cstdint>
 #include <cstdio>
 #include <limits>
 
@@ -70,37 +70,37 @@ void IndexSplitVectors::search(
             sum_d == d, "not enough indexes compared to # dimensions");
 
     int64_t nshard = sub_indexes.size();
-    float* all_distances = new float[nshard * k * n];
-    idx_t* all_labels = new idx_t[nshard * k * n];
-    ScopeDeleter<float> del(all_distances);
-    ScopeDeleter<idx_t> del2(all_labels);
-
-    auto query_func = [n,
-                       x,
-                       k,
-                       distances,
-                       labels,
-                       all_distances,
-                       all_labels,
-                       this](int no) {
-        const IndexSplitVectors* index = this;
-        float* distances1 = no == 0 ? distances : all_distances + no * k * n;
-        idx_t* labels1 = no == 0 ? labels : all_labels + no * k * n;
-        if (index->verbose)
-            printf("begin query shard %d on %" PRId64 " points\n", no, n);
-        const Index* sub_index = index->sub_indexes[no];
-        int64_t sub_d = sub_index->d, d = index->d;
-        idx_t ofs = 0;
-        for (int i = 0; i < no; i++)
-            ofs += index->sub_indexes[i]->d;
-        float* sub_x = new float[sub_d * n];
-        ScopeDeleter<float> del1(sub_x);
-        for (idx_t i = 0; i < n; i++)
-            memcpy(sub_x + i * sub_d, x + ofs + i * d, sub_d * sizeof(sub_x));
-        sub_index->search(n, sub_x, k, distances1, labels1);
-        if (index->verbose)
-            printf("end query shard %d\n", no);
-    };
+
+    std::unique_ptr<float[]> all_distances(new float[nshard * k * n]);
+    std::unique_ptr<idx_t[]> all_labels(new idx_t[nshard * k * n]);
+
+    auto query_func =
+            [n, x, k, distances, labels, &all_distances, &all_labels, this](
+                    int no) {
+                const IndexSplitVectors* index = this;
+                float* distances1 =
+                        no == 0 ? distances : all_distances.get() + no * k * n;
+                idx_t* labels1 =
+                        no == 0 ? labels : all_labels.get() + no * k * n;
+                if (index->verbose)
+                    printf("begin query shard %d on %" PRId64 " points\n",
+                           no,
+                           n);
+                const Index* sub_index = index->sub_indexes[no];
+                int64_t sub_d = sub_index->d, d = index->d;
+                idx_t ofs = 0;
+                for (int i = 0; i < no; i++)
+                    ofs += index->sub_indexes[i]->d;
+
+                std::unique_ptr<float[]> sub_x(new float[sub_d * n]);
+                for (idx_t i = 0; i < n; i++)
+                    memcpy(sub_x.get() + i * sub_d,
+                           x + ofs + i * d,
+                           sub_d * sizeof(float));
+                sub_index->search(n, sub_x.get(), k, distances1, labels1);
+                if (index->verbose)
+                    printf("end query shard %d\n", no);
+            };
 
     if (!threaded) {
         for (int i = 0; i < nshard; i++) {
@@ -125,8 +125,8 @@ void IndexSplitVectors::search(
     int64_t factor = 1;
     for (int i = 0; i < nshard; i++) {
         if (i > 0) { // results of 0 are already in the table
-            const float* distances_i = all_distances + i * k * n;
-            const idx_t* labels_i = all_labels + i * k * n;
+            const float* distances_i = all_distances.get() + i * k * n;
+            const idx_t* labels_i = all_labels.get() + i * k * n;
             for (int64_t j = 0; j < n; j++) {
                 if (labels[j] >= 0 && labels_i[j] >= 0) {
                     labels[j] += labels_i[j] * factor;
@@ -238,6 +238,6 @@ void IndexRandom::reset() {
     ntotal = 0;
 }
 
-IndexRandom::~IndexRandom() {}
+IndexRandom::~IndexRandom() = default;
 
 } // namespace faiss

data/vendor/faiss/faiss/MetricType.h

@@ -31,8 +31,13 @@ enum MetricType {
     METRIC_Canberra = 20,
     METRIC_BrayCurtis,
     METRIC_JensenShannon,
-    METRIC_Jaccard, ///< defined as: sum_i(min(a_i, b_i)) / sum_i(max(a_i, b_i))
-                    ///< where a_i, b_i > 0
+
+    /// sum_i(min(a_i, b_i)) / sum_i(max(a_i, b_i)) where a_i, b_i > 0
+    METRIC_Jaccard,
+    /// Squared Eucliden distance, ignoring NaNs
+    METRIC_NaNEuclidean,
+    /// abs(x | y): the distance to a hyperplane
+    METRIC_ABS_INNER_PRODUCT,
 };
 
 /// all vector indices are this type

data/vendor/faiss/faiss/VectorTransform.cpp

@@ -441,13 +441,10 @@ void eig(size_t d_in, double* cov, double* eigenvalues, int verbose) {
 
 } // namespace
 
-void PCAMatrix::train(idx_t n, const float* x) {
-    const float* x_in = x;
-
-    x = fvecs_maybe_subsample(
-            d_in, (size_t*)&n, max_points_per_d * d_in, x, verbose);
-
-    ScopeDeleter<float> del_x(x != x_in ? x : nullptr);
+void PCAMatrix::train(idx_t n, const float* x_in) {
+    const float* x = fvecs_maybe_subsample(
+            d_in, (size_t*)&n, max_points_per_d * d_in, x_in, verbose);
+    TransformedVectors tv(x_in, x);
 
     // compute mean
     mean.clear();
@@ -884,14 +881,13 @@ ITQTransform::ITQTransform(int d_in, int d_out, bool do_pca)
     is_trained = false;
 }
 
-void ITQTransform::train(idx_t n, const float* x) {
+void ITQTransform::train(idx_t n, const float* x_in) {
     FAISS_THROW_IF_NOT(!is_trained);
 
-    const float* x_in = x;
     size_t max_train_points = std::max(d_in * max_train_per_dim, 32768);
-    x = fvecs_maybe_subsample(d_in, (size_t*)&n, max_train_points, x);
-
-    ScopeDeleter<float> del_x(x != x_in ? x : nullptr);
+    const float* x =
+            fvecs_maybe_subsample(d_in, (size_t*)&n, max_train_points, x_in);
+    TransformedVectors tv(x_in, x);
 
     std::unique_ptr<float[]> x_norm(new float[n * d_in]);
     { // normalize
@@ -988,25 +984,16 @@ void ITQTransform::check_identical(const VectorTransform& other_in) const {
  *********************************************/
 
 OPQMatrix::OPQMatrix(int d, int M, int d2)
-        : LinearTransform(d, d2 == -1 ? d : d2, false),
-          M(M),
-          niter(50),
-          niter_pq(4),
-          niter_pq_0(40),
-          verbose(false),
-          pq(nullptr) {
+        : LinearTransform(d, d2 == -1 ? d : d2, false), M(M) {
     is_trained = false;
     // OPQ is quite expensive to train, so set this right.
     max_train_points = 256 * 256;
-    pq = nullptr;
 }
 
-void OPQMatrix::train(idx_t n, const float* x) {
-    const float* x_in = x;
-
-    x = fvecs_maybe_subsample(d_in, (size_t*)&n, max_train_points, x, verbose);
-
-    ScopeDeleter<float> del_x(x != x_in ? x : nullptr);
+void OPQMatrix::train(idx_t n, const float* x_in) {
+    const float* x = fvecs_maybe_subsample(
+            d_in, (size_t*)&n, max_train_points, x_in, verbose);
+    TransformedVectors tv(x_in, x);
 
     // To support d_out > d_in, we pad input vectors with 0s to d_out
     size_t d = d_out <= d_in ? d_in : d_out;

data/vendor/faiss/faiss/VectorTransform.h

@@ -230,18 +230,18 @@ struct ProductQuantizer;
  *
  */
 struct OPQMatrix : LinearTransform {
-    int M;          ///< nb of subquantizers
-    int niter;      ///< Number of outer training iterations
-    int niter_pq;   ///< Number of training iterations for the PQ
-    int niter_pq_0; ///< same, for the first outer iteration
+    int M;               ///< nb of subquantizers
+    int niter = 50;      ///< Number of outer training iterations
+    int niter_pq = 4;    ///< Number of training iterations for the PQ
+    int niter_pq_0 = 40; ///< same, for the first outer iteration
 
     /// if there are too many training points, resample
-    size_t max_train_points;
-    bool verbose;
+    size_t max_train_points = 256 * 256;
+    bool verbose = false;
 
     /// if non-NULL, use this product quantizer for training
     /// should be constructed with (d_out, M, _)
-    ProductQuantizer* pq;
+    ProductQuantizer* pq = nullptr;
 
     /// if d2 != -1, output vectors of this dimension
     explicit OPQMatrix(int d = 0, int M = 1, int d2 = -1);

data/vendor/faiss/faiss/clone_index.cpp

@@ -17,6 +17,8 @@
 #include <faiss/Index2Layer.h>
 #include <faiss/IndexAdditiveQuantizer.h>
 #include <faiss/IndexAdditiveQuantizerFastScan.h>
+#include <faiss/IndexBinary.h>
+#include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexFlat.h>
 #include <faiss/IndexHNSW.h>
 #include <faiss/IndexIVF.h>
@@ -35,6 +37,7 @@
 #include <faiss/IndexRefine.h>
 #include <faiss/IndexRowwiseMinMax.h>
 #include <faiss/IndexScalarQuantizer.h>
+
 #include <faiss/MetaIndexes.h>
 #include <faiss/VectorTransform.h>
 
@@ -60,9 +63,10 @@ Index* clone_index(const Index* index) {
 // assumes there is a copy constructor ready. Always try from most
 // specific to most general. Most indexes don't have complicated
 // structs, the default copy constructor often just works.
-#define TRYCLONE(classname, obj)                                      \
-    if (const classname* clo = dynamic_cast<const classname*>(obj)) { \
-        return new classname(*clo);                                   \
+#define TRYCLONE(classname, obj)                       \
+    if (const classname* clo##classname =              \
+                dynamic_cast<const classname*>(obj)) { \
+        return new classname(*clo##classname);         \
     } else
 
 VectorTransform* Cloner::clone_VectorTransform(const VectorTransform* vt) {
@@ -234,13 +238,6 @@ Index* clone_AdditiveQuantizerIndex(const Index* index) {
 
 namespace {
 
-IndexHNSW* clone_HNSW(const IndexHNSW* ihnsw) {
-    TRYCLONE(IndexHNSWFlat, ihnsw)
-    TRYCLONE(IndexHNSWPQ, ihnsw)
-    TRYCLONE(IndexHNSWSQ, ihnsw)
-    return new IndexHNSW(*ihnsw);
-}
-
 InvertedLists* clone_InvertedLists(const InvertedLists* invlists) {
     if (auto* ails = dynamic_cast<const ArrayInvertedLists*>(invlists)) {
         return new ArrayInvertedLists(*ails);
@@ -385,4 +382,12 @@ Quantizer* clone_Quantizer(const Quantizer* quant) {
     FAISS_THROW_MSG("Did not recognize quantizer to clone");
 }
 
+IndexBinary* clone_binary_index(const IndexBinary* index) {
+    if (auto ii = dynamic_cast<const IndexBinaryFlat*>(index)) {
+        return new IndexBinaryFlat(*ii);
+    } else {
+        FAISS_THROW_MSG("cannot clone this type of index");
+    }
+}
+
 } // namespace faiss

data/vendor/faiss/faiss/clone_index.h

@@ -17,6 +17,7 @@ struct Index;
 struct IndexIVF;
 struct VectorTransform;
 struct Quantizer;
+struct IndexBinary;
 
 /* cloning functions */
 Index* clone_index(const Index*);
@@ -33,4 +34,6 @@ struct Cloner {
 
 Quantizer* clone_Quantizer(const Quantizer* quant);
 
+IndexBinary* clone_binary_index(const IndexBinary* index);
+
 } // namespace faiss