faiss 0.3.0 → 0.3.1

data/vendor/faiss/faiss/impl/NNDescent.cpp

@@ -195,8 +195,9 @@ void NNDescent::update() {
         int l = 0;
 
         while ((l < maxl) && (c < S)) {
-            if (nn.pool[l].flag)
+            if (nn.pool[l].flag) {
                 ++c;
+            }
             ++l;
         }
         nn.M = l;
@@ -305,8 +306,9 @@ void NNDescent::generate_eval_set(
     for (int i = 0; i < c.size(); i++) {
         std::vector<Neighbor> tmp;
         for (int j = 0; j < N; j++) {
-            if (c[i] == j)
+            if (c[i] == j) {
                 continue; // skip itself
+            }
             float dist = qdis.symmetric_dis(c[i], j);
             tmp.push_back(Neighbor(j, dist, true));
         }
@@ -360,8 +362,9 @@ void NNDescent::init_graph(DistanceComputer& qdis) {
 
             for (int j = 0; j < S; j++) {
                 int id = tmp[j];
-                if (id == i)
+                if (id == i) {
                     continue;
+                }
                 float dist = qdis.symmetric_dis(i, id);
 
                 graph[i].pool.push_back(Neighbor(id, dist, true));
@@ -374,6 +377,10 @@ void NNDescent::init_graph(DistanceComputer& qdis) {
 
 void NNDescent::build(DistanceComputer& qdis, const int n, bool verbose) {
     FAISS_THROW_IF_NOT_MSG(L >= K, "L should be >= K in NNDescent.build");
+    FAISS_THROW_IF_NOT_FMT(
+            n > NUM_EVAL_POINTS,
+            "NNDescent.build cannot build a graph smaller than %d",
+            int(NUM_EVAL_POINTS));
 
     if (verbose) {
         printf("Parameters: K=%d, S=%d, R=%d, L=%d, iter=%d\n",
@@ -403,7 +410,7 @@ void NNDescent::build(DistanceComputer& qdis, const int n, bool verbose) {
     has_built = true;
 
     if (verbose) {
-        printf("Addes %d points into the index\n", ntotal);
+        printf("Added %d points into the index\n", ntotal);
     }
 }
 
@@ -414,30 +421,30 @@ void NNDescent::search(
         float* dists,
         VisitedTable& vt) const {
     FAISS_THROW_IF_NOT_MSG(has_built, "The index is not build yet.");
-    int L = std::max(search_L, topk);
+    int L_2 = std::max(search_L, topk);
 
     // candidate pool, the K best items is the result.
-    std::vector<Neighbor> retset(L + 1);
+    std::vector<Neighbor> retset(L_2 + 1);
 
-    // Randomly choose L points to initialize the candidate pool
-    std::vector<int> init_ids(L);
+    // Randomly choose L_2 points to initialize the candidate pool
+    std::vector<int> init_ids(L_2);
     std::mt19937 rng(random_seed);
 
-    gen_random(rng, init_ids.data(), L, ntotal);
-    for (int i = 0; i < L; i++) {
+    gen_random(rng, init_ids.data(), L_2, ntotal);
+    for (int i = 0; i < L_2; i++) {
         int id = init_ids[i];
         float dist = qdis(id);
         retset[i] = Neighbor(id, dist, true);
     }
 
     // Maintain the candidate pool in ascending order
-    std::sort(retset.begin(), retset.begin() + L);
+    std::sort(retset.begin(), retset.begin() + L_2);
 
     int k = 0;
 
-    // Stop until the smallest position updated is >= L
-    while (k < L) {
-        int nk = L;
+    // Stop until the smallest position updated is >= L_2
+    while (k < L_2) {
+        int nk = L_2;
 
         if (retset[k].flag) {
             retset[k].flag = false;
@@ -445,25 +452,28 @@ void NNDescent::search(
 
             for (int m = 0; m < K; ++m) {
                 int id = final_graph[n * K + m];
-                if (vt.get(id))
+                if (vt.get(id)) {
                     continue;
+                }
 
                 vt.set(id);
                 float dist = qdis(id);
-                if (dist >= retset[L - 1].distance)
+                if (dist >= retset[L_2 - 1].distance) {
                     continue;
+                }
 
                 Neighbor nn(id, dist, true);
-                int r = insert_into_pool(retset.data(), L, nn);
+                int r = insert_into_pool(retset.data(), L_2, nn);
 
                 if (r < nk)
                     nk = r;
             }
         }
-        if (nk <= k)
+        if (nk <= k) {
             k = nk;
-        else
+        } else {
             ++k;
+        }
     }
     for (size_t i = 0; i < topk; i++) {
         indices[i] = retset[i].id;

data/vendor/faiss/faiss/impl/NSG.h

@@ -54,7 +54,7 @@ namespace nsg {
 
 template <class node_t>
 struct Graph {
-    node_t* data;    ///< the flattened adjacency matrix
+    node_t* data;    ///< the flattened adjacency matrix, size N-by-K
     int K;           ///< nb of neighbors per node
     int N;           ///< total nb of nodes
     bool own_fields; ///< the underlying data owned by itself or not

data/vendor/faiss/faiss/impl/PolysemousTraining.cpp

@@ -12,11 +12,11 @@
 #include <omp.h>
 #include <stdint.h>
 
+#include <algorithm>
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
-
-#include <algorithm>
+#include <memory>
 
 #include <faiss/utils/distances.h>
 #include <faiss/utils/hamming.h>
@@ -683,18 +683,21 @@ struct RankingScore2 : Score3Computer<float, double> {
     double accum_gt_weight_diff(
             const std::vector<int>& a,
             const std::vector<int>& b) {
-        int nb = b.size(), na = a.size();
+        const auto nb_2 = b.size();
+        const auto na = a.size();
 
         double accu = 0;
-        int j = 0;
-        for (int i = 0; i < na; i++) {
-            int ai = a[i];
-            while (j < nb && ai >= b[j])
+        size_t j = 0;
+        for (size_t i = 0; i < na; i++) {
+            const auto ai = a[i];
+            while (j < nb_2 && ai >= b[j]) {
                 j++;
+            }
 
             double accu_i = 0;
-            for (int k = j; k < b.size(); k++)
+            for (auto k = j; k < b.size(); k++) {
                 accu_i += rank_weight(b[k] - ai);
+            }
 
             accu += rank_weight(ai) * accu_i;
         }
@@ -882,14 +885,13 @@ void PolysemousTraining::optimize_ranking(
 
         double t0 = getmillisecs();
 
-        PermutationObjective* obj = new RankingScore2(
+        std::unique_ptr<PermutationObjective> obj(new RankingScore2(
                 nbits,
                 nq,
                 nb,
                 codes.data(),
                 codes.data() + nq,
-                gt_distances.data());
-        ScopeDeleter1<PermutationObjective> del(obj);
+                gt_distances.data()));
 
         if (verbose > 0) {
             printf("   m=%d, nq=%zd, nb=%zd, initialize RankingScore "
@@ -900,7 +902,7 @@ void PolysemousTraining::optimize_ranking(
                    getmillisecs() - t0);
         }
 
-        SimulatedAnnealingOptimizer optim(obj, *this);
+        SimulatedAnnealingOptimizer optim(obj.get(), *this);
 
         if (log_pattern.size()) {
             char fname[256];

data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.h

@@ -151,4 +151,4 @@ struct ProductResidualQuantizer : ProductAdditiveQuantizer {
     ProductResidualQuantizer();
 };
 
-}; // namespace faiss
+} // namespace faiss

data/vendor/faiss/faiss/impl/ProductQuantizer.cpp

@@ -135,11 +135,10 @@ void ProductQuantizer::train(size_t n, const float* x) {
             }
         }
 
-        float* xslice = new float[n * dsub];
-        ScopeDeleter<float> del(xslice);
+        std::unique_ptr<float[]> xslice(new float[n * dsub]);
         for (int m = 0; m < M; m++) {
             for (int j = 0; j < n; j++)
-                memcpy(xslice + j * dsub,
+                memcpy(xslice.get() + j * dsub,
                        x + j * d + m * dsub,
                        dsub * sizeof(float));
 
@@ -153,11 +152,19 @@ void ProductQuantizer::train(size_t n, const float* x) {
             switch (final_train_type) {
                 case Train_hypercube:
                     init_hypercube(
-                            dsub, nbits, n, xslice, clus.centroids.data());
+                            dsub,
+                            nbits,
+                            n,
+                            xslice.get(),
+                            clus.centroids.data());
                     break;
                 case Train_hypercube_pca:
                     init_hypercube_pca(
-                            dsub, nbits, n, xslice, clus.centroids.data());
+                            dsub,
+                            nbits,
+                            n,
+                            xslice.get(),
+                            clus.centroids.data());
                     break;
                 case Train_hot_start:
                     memcpy(clus.centroids.data(),
@@ -172,7 +179,7 @@ void ProductQuantizer::train(size_t n, const float* x) {
                 printf("Training PQ slice %d/%zd\n", m, M);
             }
             IndexFlatL2 index(dsub);
-            clus.train(n, xslice, assign_index ? *assign_index : index);
+            clus.train(n, xslice.get(), assign_index ? *assign_index : index);
             set_params(clus.centroids.data(), m);
         }
 
@@ -306,7 +313,8 @@ void ProductQuantizer::decode(const uint8_t* code, float* x) const {
 }
 
 void ProductQuantizer::decode(const uint8_t* code, float* x, size_t n) const {
-    for (size_t i = 0; i < n; i++) {
+#pragma omp parallel for if (n > 100)
+    for (int64_t i = 0; i < n; i++) {
         this->decode(code + code_size * i, x + d * i);
     }
 }
@@ -342,21 +350,20 @@ void ProductQuantizer::compute_codes_with_assign_index(
         assign_index->reset();
         assign_index->add(ksub, get_centroids(m, 0));
         size_t bs = 65536;
-        float* xslice = new float[bs * dsub];
-        ScopeDeleter<float> del(xslice);
-        idx_t* assign = new idx_t[bs];
-        ScopeDeleter<idx_t> del2(assign);
+
+        std::unique_ptr<float[]> xslice(new float[bs * dsub]);
+        std::unique_ptr<idx_t[]> assign(new idx_t[bs]);
 
         for (size_t i0 = 0; i0 < n; i0 += bs) {
             size_t i1 = std::min(i0 + bs, n);
 
             for (size_t i = i0; i < i1; i++) {
-                memcpy(xslice + (i - i0) * dsub,
+                memcpy(xslice.get() + (i - i0) * dsub,
                        x + i * d + m * dsub,
                        dsub * sizeof(float));
             }
 
-            assign_index->assign(i1 - i0, xslice, assign);
+            assign_index->assign(i1 - i0, xslice.get(), assign.get());
 
             if (nbits == 8) {
                 uint8_t* c = codes + code_size * i0 + m;
@@ -405,15 +412,14 @@ void ProductQuantizer::compute_codes(const float* x, uint8_t* codes, size_t n)
         for (int64_t i = 0; i < n; i++)
             compute_code(x + i * d, codes + i * code_size);
 
-    } else { // worthwile to use BLAS
-        float* dis_tables = new float[n * ksub * M];
-        ScopeDeleter<float> del(dis_tables);
-        compute_distance_tables(n, x, dis_tables);
+    } else { // worthwhile to use BLAS
+        std::unique_ptr<float[]> dis_tables(new float[n * ksub * M]);
+        compute_distance_tables(n, x, dis_tables.get());
 
 #pragma omp parallel for
         for (int64_t i = 0; i < n; i++) {
             uint8_t* code = codes + i * code_size;
-            const float* tab = dis_tables + i * ksub * M;
+            const float* tab = dis_tables.get() + i * ksub * M;
             compute_code_from_distance_table(tab, code);
         }
     }
@@ -774,10 +780,6 @@ void ProductQuantizer::search_ip(
             init_finalize_heap);
 }
 
-static float sqr(float x) {
-    return x * x;
-}
-
 void ProductQuantizer::compute_sdc_table() {
     sdc_table.resize(M * ksub * ksub);
 

data/vendor/faiss/faiss/impl/ProductQuantizer.h

@@ -35,7 +35,7 @@ struct ProductQuantizer : Quantizer {
     enum train_type_t {
         Train_default,
         Train_hot_start,     ///< the centroids are already initialized
-        Train_shared,        ///< share dictionary accross PQ segments
+        Train_shared,        ///< share dictionary across PQ segments
         Train_hypercube,     ///< initialize centroids with nbits-D hypercube
         Train_hypercube_pca, ///< initialize centroids with nbits-D hypercube
     };

data/vendor/faiss/faiss/impl/Quantizer.h

@@ -11,7 +11,7 @@
 
 namespace faiss {
 
-/** Product Quantizer. Implemented only for METRIC_L2 */
+/** General interface for quantizer objects */
 struct Quantizer {
     size_t d;         ///< size of the input vectors
     size_t code_size; ///< bytes per indexed vector