faiss 0.1.2 → 0.1.3

data/vendor/faiss/{IndexIVF.h → faiss/IndexIVF.h}

@@ -19,6 +19,7 @@
 #include <faiss/InvertedLists.h>
 #include <faiss/DirectMap.h>
 #include <faiss/Clustering.h>
+#include <faiss/impl/platform_macros.h>
 #include <faiss/utils/Heap.h>
 
 
@@ -68,6 +69,7 @@ struct Level1Quantizer {
 struct IVFSearchParameters {
     size_t nprobe;            ///< number of probes at query time
     size_t max_codes;         ///< max nb of codes to visit to do a query
+    IVFSearchParameters(): nprobe(1), max_codes(0) {}
     virtual ~IVFSearchParameters () {}
 };
 
@@ -96,7 +98,7 @@ struct InvertedListScanner;
  * the distance estimation from the query to databse vectors.
  */
 struct IndexIVF: Index, Level1Quantizer {
-    /// Acess to the actual data
+    /// Access to the actual data
     InvertedLists *invlists;
     bool own_invlists;
 
@@ -194,7 +196,9 @@ struct IndexIVF: Index, Level1Quantizer {
 
     void range_search_preassigned(idx_t nx, const float *x, float radius,
                                   const idx_t *keys, const float *coarse_dis,
-                                  RangeSearchResult *result) const;
+                                  RangeSearchResult *result,
+                                  bool store_pairs=false,
+                                  const IVFSearchParameters *params=nullptr) const;
 
     /// get a scanner for this index (store_pairs means ignore labels)
     virtual InvertedListScanner *get_InvertedListScanner (
@@ -363,7 +367,7 @@ struct IndexIVFStats {
 };
 
 // global var that collects them all
-extern IndexIVFStats indexIVF_stats;
+FAISS_API extern IndexIVFStats indexIVF_stats;
 
 
 } // namespace faiss

data/vendor/faiss/{IndexIVFFlat.cpp → faiss/IndexIVFFlat.cpp}

@@ -9,6 +9,7 @@
 
 #include <faiss/IndexIVFFlat.h>
 
+#include <cinttypes>
 #include <cstdio>
 
 #include <faiss/IndexFlat.h>
@@ -75,7 +76,7 @@ void IndexIVFFlat::add_core (idx_t n, const float * x, const int64_t *xids,
     }
 
     if (verbose) {
-        printf("IndexIVFFlat::add_core: added %ld / %ld vectors\n",
+        printf("IndexIVFFlat::add_core: added %" PRId64 " / %" PRId64 " vectors\n",
                n_add, n);
     }
     ntotal += n;
@@ -247,8 +248,8 @@ void IndexIVFFlatDedup::train(idx_t n, const float* x)
         }
     }
     if (verbose) {
-        printf ("IndexIVFFlatDedup::train: train on %ld points after dedup "
-                "(was %ld points)\n", n2, n);
+        printf ("IndexIVFFlatDedup::train: train on %" PRId64 " points after dedup "
+                "(was %" PRId64 " points)\n", n2, n);
     }
     IndexIVFFlat::train (n2, x2);
 }
@@ -303,8 +304,8 @@ void IndexIVFFlatDedup::add_with_ids(
         n_add++;
     }
     if (verbose) {
-        printf("IndexIVFFlat::add_with_ids: added %ld / %ld vectors"
-               " (out of which %ld are duplicates)\n",
+        printf("IndexIVFFlat::add_with_ids: added %" PRId64 " / %" PRId64 " vectors"
+               " (out of which %" PRId64 " are duplicates)\n",
                n_add, na, n_dup);
     }
     ntotal += n_add;

data/vendor/faiss/{IndexIVFPQ.cpp → faiss/IndexIVFPQ.cpp}

@@ -9,6 +9,7 @@
 
 #include <faiss/IndexIVFPQ.h>
 
+#include <cinttypes>
 #include <cmath>
 #include <cstdio>
 #include <cassert>
@@ -91,7 +92,7 @@ void IndexIVFPQ::train_residual_o (idx_t n, const float *x, float *residuals_2)
         trainset = x;
     }
     if (verbose)
-        printf ("training %zdx%zd product quantizer on %ld vectors in %dD\n",
+        printf ("training %zdx%zd product quantizer on %" PRId64 " vectors in %dD\n",
                 pq.M, pq.ksub, n, d);
     pq.verbose = verbose;
     pq.train (n, trainset);
@@ -140,9 +141,9 @@ void IndexIVFPQ::train_residual_o (idx_t n, const float *x, float *residuals_2)
 void IndexIVFPQ::encode (idx_t key, const float * x, uint8_t * code) const
 {
     if (by_residual) {
-        float residual_vec[d];
-        quantizer->compute_residual (x, residual_vec, key);
-        pq.compute_code (residual_vec, code);
+        std::vector<float> residual_vec(d);
+        quantizer->compute_residual (x, residual_vec.data(), key);
+        pq.compute_code (residual_vec.data(), code);
     }
     else pq.compute_code (x, code);
 }
@@ -240,7 +241,7 @@ void IndexIVFPQ::sa_decode (idx_t n, const uint8_t *codes,
         std::vector<float> residual (d);
 
 #pragma omp for
-        for (size_t i = 0; i < n; i++) {
+        for (idx_t i = 0; i < n; i++) {
             const uint8_t *code = codes + i * (code_size + coarse_size);
             int64_t list_no = decode_listno (code);
             float *xi = x + i * d;
@@ -265,7 +266,7 @@ void IndexIVFPQ::add_core_o (idx_t n, const float * x, const idx_t *xids,
         for (idx_t i0 = 0; i0 < n; i0 += bs) {
             idx_t i1 = std::min(i0 + bs, n);
             if (verbose) {
-                printf("IndexIVFPQ::add_core_o: adding %ld:%ld / %ld\n",
+                printf("IndexIVFPQ::add_core_o: adding %" PRId64 ":%" PRId64 " / %" PRId64 "\n",
                        i0, i1, n);
             }
             add_core_o (i1 - i0, x + i0 * d,
@@ -341,7 +342,7 @@ void IndexIVFPQ::add_core_o (idx_t n, const float * x, const idx_t *xids,
     if(verbose) {
         char comment[100] = {0};
         if (n_ignore > 0)
-            snprintf (comment, 100, "(%ld vectors ignored)", n_ignore);
+            snprintf (comment, 100, "(%zd vectors ignored)", n_ignore);
         printf(" add_core times: %.3f %.3f %.3f %s\n",
                t1 - t0, t2 - t1, t3 - t2, comment);
     }
@@ -425,7 +426,7 @@ void IndexIVFPQ::precompute_table ()
                 if (verbose) {
                     printf(
                        "IndexIVFPQ::precompute_table: not precomputing table, "
-                       "it would be too big: %ld bytes (max %ld)\n",
+                       "it would be too big: %zd bytes (max %zd)\n",
                        table_size, precomputed_table_max_bytes);
                     use_precomputed_table = 0;
                 }

data/vendor/faiss/{IndexIVFPQ.h → faiss/IndexIVFPQ.h}

@@ -15,6 +15,7 @@
 
 #include <faiss/IndexIVF.h>
 #include <faiss/IndexPQ.h>
+#include <faiss/impl/platform_macros.h>
 
 
 namespace faiss {
@@ -22,6 +23,7 @@ namespace faiss {
 struct IVFPQSearchParameters: IVFSearchParameters {
     size_t scan_table_threshold;   ///< use table computation or on-the-fly?
     int polysemous_ht;             ///< Hamming thresh for polysemous filtering
+    IVFPQSearchParameters (): scan_table_threshold(0), polysemous_ht(0) {}
     ~IVFPQSearchParameters () {}
 };
 
@@ -29,7 +31,7 @@ struct IVFPQSearchParameters: IVFSearchParameters {
 /** Inverted file with Product Quantizer encoding. Each residual
  * vector is encoded as a product quantizer code.
  */
-struct IndexIVFPQ: IndexIVF {
+struct FAISS_API IndexIVFPQ: IndexIVF {
     bool by_residual;              ///< Encode residual or plain vector?
 
     ProductQuantizer pq;           ///< produces the codes
@@ -149,7 +151,7 @@ struct IndexIVFPQStats {
 };
 
 // global var that collects them all
-extern IndexIVFPQStats indexIVFPQ_stats;
+FAISS_API extern IndexIVFPQStats indexIVFPQ_stats;
 
 
 

data/vendor/faiss/{IndexIVFPQR.cpp → faiss/IndexIVFPQR.cpp}

@@ -9,6 +9,8 @@
 
 #include <faiss/IndexIVFPQR.h>
 
+#include <cinttypes>
+
 #include <faiss/utils/Heap.h>
 #include <faiss/utils/utils.h>
 #include <faiss/utils/distances.h>
@@ -59,7 +61,7 @@ void IndexIVFPQR::train_residual (idx_t n, const float *x)
     train_residual_o (n, x, residual_2);
 
     if (verbose)
-        printf ("training %zdx%zd 2nd level PQ quantizer on %ld %dD-vectors\n",
+        printf ("training %zdx%zd 2nd level PQ quantizer on %" PRId64 " %dD-vectors\n",
                 refine_pq.M, refine_pq.ksub, n, d);
 
     refine_pq.cp.max_points_per_centroid = 1000;

data/vendor/faiss/{IndexIVFSpectralHash.cpp → faiss/IndexIVFSpectralHash.cpp}

@@ -178,7 +178,7 @@ void IndexIVFSpectralHash::encode_vectors(idx_t n, const float* x_in,
 
         // each thread takes care of a subset of lists
 #pragma omp for
-        for (size_t i = 0; i < n; i++) {
+        for (idx_t i = 0; i < n; i++) {
             int64_t list_no = list_nos [i];
 
             if (list_no >= 0) {

data/vendor/faiss/{IndexPQ.cpp → faiss/IndexPQ.cpp}

@@ -9,7 +9,7 @@
 
 #include <faiss/IndexPQ.h>
 
-
+#include <cinttypes>
 #include <cstddef>
 #include <cstring>
 #include <cstdio>
@@ -59,7 +59,7 @@ void IndexPQ::train (idx_t n, const float *x)
         if (ntrain_perm > n / 4)
             ntrain_perm = n / 4;
         if (verbose) {
-            printf ("PQ training on %ld points, remains %ld points: "
+            printf ("PQ training on %" PRId64 " points, remains %" PRId64 " points: "
                     "training polysemous on %s\n",
                     n - ntrain_perm, ntrain_perm,
                     ntrain_perm == 0 ? "centroids" : "these");
@@ -522,8 +522,8 @@ void IndexPQ::hamming_distance_histogram (idx_t n, const float *x,
         hamdis_t *distances = new hamdis_t [nb * bs];
         ScopeDeleter<hamdis_t> del (distances);
 #pragma omp for
-        for (size_t q0 = 0; q0 < n; q0 += bs) {
-            // printf ("dis stats: %ld/%ld\n", q0, n);
+        for (idx_t q0 = 0; q0 < n; q0 += bs) {
+            // printf ("dis stats: %zd/%zd\n", q0, n);
             size_t q1 = q0 + bs;
             if (q1 > n) q1 = n;
 
@@ -835,7 +835,7 @@ struct MinSumK {
             x += ldx;
         }
 
-        { // intial result: take min for all elements
+        { // initial result: take min for all elements
             T sum = 0;
             terms[0] = 0;
             mark_seen (0);
@@ -955,7 +955,7 @@ void MultiIndexQuantizer::search (idx_t n, const float *x, idx_t k,
         for (idx_t i0 = 0; i0 < n; i0 += bs) {
             idx_t i1 = std::min(i0 + bs, n);
             if (verbose) {
-                printf("MultiIndexQuantizer::search: %ld:%ld / %ld\n",
+                printf("MultiIndexQuantizer::search: %" PRId64 ":%" PRId64 " / %" PRId64 "\n",
                        i0, i1, n);
             }
             search (i1 - i0, x + i0 * d, k,

data/vendor/faiss/{IndexPQ.h → faiss/IndexPQ.h}

@@ -17,6 +17,8 @@
 #include <faiss/Index.h>
 #include <faiss/impl/ProductQuantizer.h>
 #include <faiss/impl/PolysemousTraining.h>
+#include <faiss/impl/platform_macros.h>
+
 
 namespace faiss {
 
@@ -138,7 +140,7 @@ struct IndexPQStats {
     void reset ();
 };
 
-extern IndexPQStats indexPQ_stats;
+FAISS_API extern IndexPQStats indexPQ_stats;
 
 
 

data/vendor/faiss/{IndexReplicas.cpp → faiss/IndexReplicas.cpp}

@@ -5,6 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#include <cinttypes>
+
 #include <faiss/IndexReplicas.h>
 #include <faiss/impl/FaissAssert.h>
 
@@ -36,33 +38,65 @@ IndexReplicasTemplate<IndexT>::onAfterAddIndex(IndexT* index) {
     FAISS_THROW_IF_NOT_FMT(index->ntotal == existing->ntotal,
                            "IndexReplicas: newly added index does "
                            "not have same number of vectors as prior index; "
-                           "prior index has %ld vectors, new index has %ld",
+                           "prior index has %" PRId64 " vectors, new index has %" PRId64,
                            existing->ntotal, index->ntotal);
 
     FAISS_THROW_IF_NOT_MSG(index->is_trained == existing->is_trained,
                            "IndexReplicas: newly added index does "
                            "not have same train status as prior index");
+
+    FAISS_THROW_IF_NOT_MSG(index->d == existing->d,
+                           "IndexReplicas: newly added index does "
+                           "not have same dimension as prior index");
   } else {
-    // Set our parameters based on the first index we're adding
-    // (dimension is handled in ThreadedIndex)
-    this->ntotal = index->ntotal;
-    this->verbose = index->verbose;
-    this->is_trained = index->is_trained;
-    this->metric_type = index->metric_type;
+    syncWithSubIndexes();
   }
 }
 
+template <typename IndexT>
+void
+IndexReplicasTemplate<IndexT>::onAfterRemoveIndex(IndexT* index) {
+  syncWithSubIndexes();
+}
+
 template <typename IndexT>
 void
 IndexReplicasTemplate<IndexT>::train(idx_t n, const component_t* x) {
-  this->runOnIndex([n, x](int, IndexT* index){ index->train(n, x); });
+  auto fn =
+    [n, x](int i, IndexT* index) {
+      if (index->verbose) {
+        printf("begin train replica %d on %" PRId64 " points\n", i, n);
+      }
+
+      index->train(n, x);
+
+      if (index->verbose) {
+        printf("end train replica %d\n", i);
+      }
+    };
+
+  this->runOnIndex(fn);
+  syncWithSubIndexes();
 }
 
 template <typename IndexT>
 void
 IndexReplicasTemplate<IndexT>::add(idx_t n, const component_t* x) {
-  this->runOnIndex([n, x](int, IndexT* index){ index->add(n, x); });
-  this->ntotal += n;
+  auto fn =
+    [n, x](int i, IndexT* index) {
+      if (index->verbose) {
+        printf("begin add replica %d on %" PRId64 " points\n", i, n);
+      }
+
+      index->add(n, x);
+
+      if (index->verbose) {
+        printf("end add replica %d\n", i);
+      }
+    };
+
+  this->runOnIndex(fn);
+  syncWithSubIndexes();
 }
 
 template <typename IndexT>
@@ -105,17 +139,75 @@ IndexReplicasTemplate<IndexT>::search(idx_t n,
       if (base < n) {
         auto numForIndex = std::min(queriesPerIndex, n - base);
 
+        if (index->verbose) {
+          printf("begin search replica %d on %" PRId64 " points\n",
+                 i, numForIndex);
+        }
+
         index->search(numForIndex,
                       x + base * componentsPerVec,
                       k,
                       distances + base * k,
                       labels + base * k);
+
+        if (index->verbose) {
+          printf("end search replica %d\n", i);
+        }
       }
     };
 
   this->runOnIndex(fn);
 }
 
+// FIXME: assumes that nothing is currently running on the sub-indexes, which is
+// true with the normal API, but should use the runOnIndex API instead
+template <typename IndexT>
+void
+IndexReplicasTemplate<IndexT>::syncWithSubIndexes() {
+  if (!this->count()) {
+    this->is_trained = false;
+    this->ntotal = 0;
+
+    return;
+  }
+
+  auto firstIndex = this->at(0);
+  this->metric_type = firstIndex->metric_type;
+  this->is_trained = firstIndex->is_trained;
+  this->ntotal = firstIndex->ntotal;
+
+  for (int i = 1; i < this->count(); ++i) {
+    auto index = this->at(i);
+    FAISS_THROW_IF_NOT(this->metric_type == index->metric_type);
+    FAISS_THROW_IF_NOT(this->d == index->d);
+    FAISS_THROW_IF_NOT(this->is_trained == index->is_trained);
+    FAISS_THROW_IF_NOT(this->ntotal == index->ntotal);
+  }
+}
+
+// No metric_type for IndexBinary
+template <>
+void
+IndexReplicasTemplate<IndexBinary>::syncWithSubIndexes() {
+  if (!this->count()) {
+    this->is_trained = false;
+    this->ntotal = 0;
+
+    return;
+  }
+
+  auto firstIndex = this->at(0);
+  this->is_trained = firstIndex->is_trained;
+  this->ntotal = firstIndex->ntotal;
+
+  for (int i = 1; i < this->count(); ++i) {
+    auto index = this->at(i);
+    FAISS_THROW_IF_NOT(this->d == index->d);
+    FAISS_THROW_IF_NOT(this->is_trained == index->is_trained);
+    FAISS_THROW_IF_NOT(this->ntotal == index->ntotal);
+  }
+}
+
 // explicit instantiations
 template struct IndexReplicasTemplate<Index>;
 template struct IndexReplicasTemplate<IndexBinary>;