faiss 0.1.1 → 0.1.2

data/vendor/faiss/IndexIVFPQR.cpp

@@ -144,8 +144,8 @@ void IndexIVFPQR::search_preassigned (idx_t n, const float *x, idx_t k,
 
                 if (sl == -1) continue;
 
-                int list_no = sl >> 32;
-                int ofs = sl & 0xffffffff;
+                int list_no = lo_listno(sl);
+                int ofs = lo_offset(sl);
 
                 assert (list_no >= 0 && list_no < nlist);
                 assert (ofs >= 0 && ofs < invlists->list_size (list_no));

data/vendor/faiss/IndexIVFSpectralHash.cpp

@@ -270,7 +270,7 @@ struct IVFScanner: InvertedListScanner {
 
             if (dis < simi [0]) {
                 maxheap_pop (k, simi, idxi);
-                int64_t id = store_pairs ? (list_no << 32 | j) : ids[j];
+                int64_t id = store_pairs ? lo_build (list_no, j) : ids[j];
                 maxheap_push (k, simi, idxi, dis, id);
                 nup++;
             }
@@ -288,7 +288,7 @@ struct IVFScanner: InvertedListScanner {
         for (size_t j = 0; j < list_size; j++) {
             float dis = hc.hamming (codes);
             if (dis < radius) {
-                int64_t id = store_pairs ? (list_no << 32 | j) : ids[j];
+                int64_t id = store_pairs ? lo_build (list_no, j) : ids[j];
                 res.add (dis, id);
             }
             codes += code_size;

data/vendor/faiss/IndexLSH.h

@@ -69,7 +69,10 @@ struct IndexLSH:Index {
 
     IndexLSH ();
 
-    /* standalone codec interface */
+    /* standalone codec interface.
+     *
+     * The vectors are decoded to +/- 1 (not 0, 1) */
+
     size_t sa_code_size () const override;
 
     void sa_encode (idx_t n, const float *x,

data/vendor/faiss/IndexPreTransform.cpp

@@ -14,7 +14,6 @@
 #include <cstring>
 #include <memory>
 
-#include <faiss/utils/utils.h>
 #include <faiss/impl/FaissAssert.h>
 
 namespace faiss {

data/vendor/faiss/IndexScalarQuantizer.cpp

@@ -253,6 +253,8 @@ void IndexIVFScalarQuantizer::add_with_ids
     size_t nadd = 0;
     std::unique_ptr<ScalarQuantizer::Quantizer> squant(sq.select_quantizer ());
 
+    DirectMapAdd dm_add (direct_map, n, xids);
+
 #pragma omp parallel reduction(+: nadd)
     {
         std::vector<float> residual (d);
@@ -275,13 +277,18 @@ void IndexIVFScalarQuantizer::add_with_ids
                 memset (one_code.data(), 0, code_size);
                 squant->encode_vector (xi, one_code.data());
 
-                invlists->add_entry (list_no, id, one_code.data());
+                size_t ofs = invlists->add_entry (list_no, id, one_code.data());
 
+                dm_add.add (i, list_no, ofs);
                 nadd++;
 
+            } else if (rank == 0 && list_no == -1) {
+                dm_add.add (i, -1, 0);
             }
         }
     }
+
+
     ntotal += n;
 }
 

data/vendor/faiss/InvertedLists.cpp

@@ -16,8 +16,6 @@
 
 namespace faiss {
 
-using ScopedIds = InvertedLists::ScopedIds;
-using ScopedCodes = InvertedLists::ScopedCodes;
 
 
 /*****************************************

data/vendor/faiss/MetaIndexes.cpp

@@ -22,7 +22,6 @@ namespace faiss {
 
 namespace {
 
-typedef Index::idx_t idx_t;
 
 } // namespace
 

data/vendor/faiss/MetricType.h

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// -*- c++ -*-
+
+#ifndef FAISS_METRIC_TYPE_H
+#define FAISS_METRIC_TYPE_H
+
+namespace faiss {
+
+/// The metric space for vector comparison for Faiss indices and algorithms.
+///
+/// Most algorithms support both inner product and L2, with the flat
+/// (brute-force) indices supporting additional metric types for vector
+/// comparison.
+enum MetricType {
+    METRIC_INNER_PRODUCT = 0,  ///< maximum inner product search
+    METRIC_L2 = 1,             ///< squared L2 search
+    METRIC_L1,                 ///< L1 (aka cityblock)
+    METRIC_Linf,               ///< infinity distance
+    METRIC_Lp,                 ///< L_p distance, p is given by a faiss::Index
+                               /// metric_arg
+
+    /// some additional metrics defined in scipy.spatial.distance
+    METRIC_Canberra = 20,
+    METRIC_BrayCurtis,
+    METRIC_JensenShannon,
+};
+
+}
+
+#endif

data/vendor/faiss/c_api/Clustering_c.cpp

@@ -19,6 +19,7 @@ extern "C" {
 using faiss::Clustering;
 using faiss::ClusteringParameters;
 using faiss::Index;
+using faiss::ClusteringIterationStats;
 
 DEFINE_GETTER(Clustering, int, niter)
 DEFINE_GETTER(Clustering, int, nredo)
@@ -38,6 +39,12 @@ DEFINE_GETTER(Clustering, size_t, d)
 /// getter for k
 DEFINE_GETTER(Clustering, size_t, k)
 
+DEFINE_GETTER(ClusteringIterationStats, float, obj)
+DEFINE_GETTER(ClusteringIterationStats, double, time)
+DEFINE_GETTER(ClusteringIterationStats, double, time_search)
+DEFINE_GETTER(ClusteringIterationStats, double, imbalance_factor)
+DEFINE_GETTER(ClusteringIterationStats, int, nsplit)
+
 void faiss_ClusteringParameters_init(FaissClusteringParameters* params) {
     ClusteringParameters d;
     params->frozen_centroids = d.frozen_centroids;
@@ -78,13 +85,12 @@ void faiss_Clustering_centroids(
     }
 }
 
-/// getter for objective values (sum of distances reported by index)
-/// over iterations
-void faiss_Clustering_obj(
-    FaissClustering* clustering, float** obj, size_t* size) {
-    std::vector<float>& v = reinterpret_cast<Clustering*>(clustering)->obj;
-    if (obj) {
-        *obj = v.data();
+/// getter for iteration stats
+void faiss_Clustering_iteration_stats(
+    FaissClustering* clustering, FaissClusteringIterationStats** iteration_stats, size_t* size) {
+    std::vector<ClusteringIterationStats>& v = reinterpret_cast<Clustering*>(clustering)->iteration_stats;
+    if (iteration_stats) {
+        *iteration_stats = reinterpret_cast<FaissClusteringIterationStats*>(v.data());
     }
     if (size) {
         *size = v.size();

data/vendor/faiss/c_api/Clustering_c.h

@@ -47,7 +47,7 @@ void faiss_ClusteringParameters_init(FaissClusteringParameters* params);
  * points to the centroids. Therefore, at each iteration the centroids
  * are added to the index.
  *
- * On output, the centoids table is set to the latest version
+ * On output, the centroids table is set to the latest version
  * of the centroids and they are also added to the index. If the
  * centroids table it is not empty on input, it is also used for
  * initialization.
@@ -75,14 +75,20 @@ FAISS_DECLARE_GETTER(Clustering, size_t, d)
 /// getter for k
 FAISS_DECLARE_GETTER(Clustering, size_t, k)
 
+FAISS_DECLARE_CLASS(ClusteringIterationStats)
+FAISS_DECLARE_GETTER(ClusteringIterationStats, float, obj)
+FAISS_DECLARE_GETTER(ClusteringIterationStats, double, time)
+FAISS_DECLARE_GETTER(ClusteringIterationStats, double, time_search)
+FAISS_DECLARE_GETTER(ClusteringIterationStats, double, imbalance_factor)
+FAISS_DECLARE_GETTER(ClusteringIterationStats, int, nsplit)
+
 /// getter for centroids (size = k * d)
 void faiss_Clustering_centroids(
     FaissClustering* clustering, float** centroids, size_t* size);
 
-/// getter for objective values (sum of distances reported by index)
-/// over iterations
-void faiss_Clustering_obj(
-    FaissClustering* clustering, float** obj, size_t* size);
+/// getter for iteration stats
+void faiss_Clustering_iteration_stats(
+    FaissClustering* clustering, FaissClusteringIterationStats** iteration_stats, size_t* size);
 
 /// the only mandatory parameters are k and d
 int faiss_Clustering_new(FaissClustering** p_clustering, int d, int k);

data/vendor/faiss/c_api/IndexIVF_c.cpp

@@ -87,6 +87,13 @@ void faiss_IndexIVF_print_stats (const FaissIndexIVF* index) {
     reinterpret_cast<const IndexIVF*>(index)->invlists->print_stats();
 }
 
+/// get inverted lists ids
+void faiss_IndexIVF_invlists_get_ids (const FaissIndexIVF* index, size_t list_no, idx_t* invlist) {
+    const idx_t* list = reinterpret_cast<const IndexIVF*>(index)->invlists->get_ids(list_no);
+    size_t list_size = reinterpret_cast<const IndexIVF*>(index)->get_list_size(list_no);
+    memcpy(invlist, list, list_size*sizeof(idx_t));
+}
+
 void faiss_IndexIVFStats_reset(FaissIndexIVFStats* stats) {
     reinterpret_cast<IndexIVFStats*>(stats)->reset();    
 }

data/vendor/faiss/c_api/IndexIVF_c.h

@@ -114,6 +114,13 @@ double faiss_IndexIVF_imbalance_factor (const FaissIndexIVF* index);
 /// display some stats about the inverted lists of the index
 void faiss_IndexIVF_print_stats (const FaissIndexIVF* index);
 
+/// Get the IDs in an inverted list. IDs are written to `invlist`, which must be large enough
+//// to accommodate the full list.
+///
+/// @param list_no the list ID
+/// @param invlist output pointer to a slice of memory, at least as long as the list's size
+/// @see faiss_IndexIVF_get_list_size(size_t) 
+void faiss_IndexIVF_invlists_get_ids (const FaissIndexIVF* index, size_t list_no, idx_t* invlist);
 
 typedef struct FaissIndexIVFStats {
     size_t nq;       // nb of queries run

data/vendor/faiss/c_api/IndexPreTransform_c.cpp

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "IndexPreTransform_c.h"
+#include "IndexPreTransform.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexPreTransform;
+
+DEFINE_DESTRUCTOR(IndexPreTransform)
+DEFINE_INDEX_DOWNCAST(IndexPreTransform)
+
+DEFINE_GETTER_PERMISSIVE(IndexPreTransform, FaissIndex*, index)

data/vendor/faiss/c_api/IndexPreTransform_c.h

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_INDEX_PRETRANSFORM_C_H
+#define FAISS_INDEX_PRETRANSFORM_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+FAISS_DECLARE_CLASS(IndexPreTransform)
+FAISS_DECLARE_DESTRUCTOR(IndexPreTransform)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexPreTransform)
+
+FAISS_DECLARE_GETTER(IndexPreTransform, FaissIndex*, index)
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/vendor/faiss/demos/demo_weighted_kmeans.cpp

@@ -0,0 +1,185 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <faiss/Clustering.h>
+#include <faiss/utils/random.h>
+#include <faiss/utils/distances.h>
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexHNSW.h>
+
+
+namespace {
+
+
+enum WeightedKMeansType {
+    WKMT_FlatL2,
+    WKMT_FlatIP,
+    WKMT_FlatIP_spherical,
+    WKMT_HNSW,
+};
+
+
+float weighted_kmeans_clustering (size_t d, size_t n, size_t k,
+                                  const float *input,
+                                  const float *weights,
+                                  float *centroids,
+                                  WeightedKMeansType index_num)
+{
+    using namespace faiss;
+    Clustering clus (d, k);
+    clus.verbose = true;
+
+    std::unique_ptr<Index> index;
+
+    switch (index_num) {
+    case WKMT_FlatL2:
+        index.reset(new IndexFlatL2 (d));
+        break;
+    case WKMT_FlatIP:
+        index.reset(new IndexFlatIP (d));
+        break;
+    case WKMT_FlatIP_spherical:
+        index.reset(new IndexFlatIP (d));
+        clus.spherical = true;
+        break;
+    case WKMT_HNSW:
+        IndexHNSWFlat *ihnsw = new IndexHNSWFlat (d, 32);
+        ihnsw->hnsw.efSearch = 128;
+        index.reset(ihnsw);
+        break;
+    }
+
+    clus.train(n, input, *index.get(), weights);
+    // on output the index contains the centroids.
+    memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
+    return clus.iteration_stats.back().obj;
+}
+
+
+int d = 32;
+float sigma = 0.1;
+
+#define BIGTEST
+
+#ifdef BIGTEST
+// the production setup = setting of https://fb.quip.com/CWgnAAYbwtgs
+int nc = 200000;
+int n_big = 4;
+int n_small = 2;
+#else
+int nc = 5;
+int n_big = 100;
+int n_small = 10;
+#endif
+
+int n; // number of training points
+
+void generate_trainset (std::vector<float> & ccent,
+                        std::vector<float> & x,
+                        std::vector<float> & weights)
+{
+    // same sampling as test_build_blocks.py test_weighted
+
+    ccent.resize (d * 2 * nc);
+    faiss::float_randn (ccent.data(), d * 2 * nc, 123);
+    faiss::fvec_renorm_L2 (d, 2 * nc, ccent.data());
+    n = nc * n_big + nc * n_small;
+    x.resize(d * n);
+    weights.resize(n);
+    faiss::float_randn (x.data(), x.size(), 1234);
+
+    float *xi = x.data();
+    float *w = weights.data();
+    for (int ci = 0; ci < nc * 2; ci++) { // loop over centroids
+        int np = ci < nc ? n_big : n_small; // nb of points around this centroid
+        for (int i = 0; i < np; i++) {
+            for (int j = 0; j < d; j++) {
+                xi[j] = xi[j] * sigma + ccent[ci * d + j];
+            }
+            *w++ = ci < nc ? 0.1 : 10;
+            xi += d;
+        }
+    }
+}
+
+}
+
+
+int main(int argc, char **argv) {
+    std::vector<float> ccent;
+    std::vector<float> x;
+    std::vector<float> weights;
+
+    printf("generate training set\n");
+    generate_trainset(ccent, x, weights);
+
+    std::vector<float> centroids;
+    centroids.resize(nc * d);
+
+    int the_index_num = -1;
+    int the_with_weights = -1;
+
+    if (argc == 3) {
+        the_index_num = atoi(argv[1]);
+        the_with_weights = atoi(argv[2]);
+    }
+
+
+    for (int index_num = WKMT_FlatL2;
+         index_num <= WKMT_HNSW;
+         index_num++) {
+
+        if (the_index_num >= 0 && index_num != the_index_num) {
+            continue;
+        }
+
+        for (int with_weights = 0; with_weights <= 1; with_weights++) {
+            if (the_with_weights >= 0 && with_weights != the_with_weights) {
+                continue;
+            }
+
+            printf("=================== index_num=%d Run %s weights\n",
+                   index_num, with_weights ? "with" : "without");
+
+            weighted_kmeans_clustering (
+                 d, n, nc, x.data(),
+                 with_weights ? weights.data() : nullptr,
+                 centroids.data(), (WeightedKMeansType)index_num
+            );
+
+            { // compute distance of points to centroids
+                faiss::IndexFlatL2 cent_index(d);
+                cent_index.add(nc, centroids.data());
+                std::vector<float> dis (n);
+                std::vector<faiss::Index::idx_t> idx (n);
+
+                cent_index.search (nc * 2, ccent.data(), 1,
+                                   dis.data(), idx.data());
+
+                float dis1 = 0, dis2 = 0;
+                for (int i = 0; i < nc ; i++) {
+                    dis1 += dis[i];
+                }
+                printf("average distance of points from big clusters: %g\n",
+                       dis1 / nc);
+
+                for (int i = 0; i < nc ; i++) {
+                    dis2 += dis[i + nc];
+                }
+
+                printf("average distance of points from small clusters: %g\n",
+                       dis2 / nc);
+
+            }
+
+        }
+    }
+    return 0;
+}