faiss 0.1.3 → 0.2.0

data/vendor/faiss/demos/demo_ivfpq_indexing.cpp

@@ -1,151 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <sys/time.h>
-
-
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/index_io.h>
-
-double elapsed ()
-{
-    struct timeval tv;
-    gettimeofday (&tv, NULL);
-    return  tv.tv_sec + tv.tv_usec * 1e-6;
-}
-
-
-int main ()
-{
-
-    double t0 = elapsed();
-
-    // dimension of the vectors to index
-    int d = 128;
-
-    // size of the database we plan to index
-    size_t nb = 200 * 1000;
-
-    // make a set of nt training vectors in the unit cube
-    // (could be the database)
-    size_t nt = 100 * 1000;
-
-    // make the index object and train it
-    faiss::IndexFlatL2 coarse_quantizer (d);
-
-    // a reasonable number of centroids to index nb vectors
-    int ncentroids = int (4 * sqrt (nb));
-
-    // the coarse quantizer should not be dealloced before the index
-    // 4 = nb of bytes per code (d must be a multiple of this)
-    // 8 = nb of bits per sub-code (almost always 8)
-    faiss::IndexIVFPQ index (&coarse_quantizer, d,
-                             ncentroids, 4, 8);
-
-
-    std::mt19937 rng;
-
-    { // training
-        printf ("[%.3f s] Generating %ld vectors in %dD for training\n",
-                elapsed() - t0, nt, d);
-
-        std::vector <float> trainvecs (nt * d);
-        std::uniform_real_distribution<> distrib;
-        for (size_t i = 0; i < nt * d; i++) {
-            trainvecs[i] = distrib(rng);
-        }
-
-        printf ("[%.3f s] Training the index\n",
-                elapsed() - t0);
-        index.verbose = true;
-
-        index.train (nt, trainvecs.data());
-    }
-
-    { // I/O demo
-        const char *outfilename = "/tmp/index_trained.faissindex";
-        printf ("[%.3f s] storing the pre-trained index to %s\n",
-                elapsed() - t0, outfilename);
-
-        write_index (&index, outfilename);
-    }
-
-    size_t nq;
-    std::vector<float> queries;
-
-    { // populating the database
-        printf ("[%.3f s] Building a dataset of %ld vectors to index\n",
-                elapsed() - t0, nb);
-
-        std::vector <float> database (nb * d);
-        std::uniform_real_distribution<> distrib;
-        for (size_t i = 0; i < nb * d; i++) {
-            database[i] = distrib(rng);
-        }
-
-        printf ("[%.3f s] Adding the vectors to the index\n",
-                elapsed() - t0);
-
-        index.add (nb, database.data());
-
-        printf ("[%.3f s] imbalance factor: %g\n",
-                elapsed() - t0, index.invlists->imbalance_factor ());
-
-        // remember a few elements from the database as queries
-        int i0 = 1234;
-        int i1 = 1243;
-
-        nq = i1 - i0;
-        queries.resize (nq * d);
-        for (int i = i0; i < i1; i++) {
-            for (int j = 0; j < d; j++) {
-                queries [(i - i0) * d  + j]  = database [i * d + j];
-            }
-        }
-
-    }
-
-    { // searching the database
-        int k = 5;
-        printf ("[%.3f s] Searching the %d nearest neighbors "
-                "of %ld vectors in the index\n",
-                elapsed() - t0, k, nq);
-
-        std::vector<faiss::Index::idx_t> nns (k * nq);
-        std::vector<float>               dis (k * nq);
-
-        index.search (nq, queries.data(), k, dis.data(), nns.data());
-
-        printf ("[%.3f s] Query results (vector ids, then distances):\n",
-                elapsed() - t0);
-
-        for (int i = 0; i < nq; i++) {
-            printf ("query %2d: ", i);
-            for (int j = 0; j < k; j++) {
-                printf ("%7ld ", nns[j + i * k]);
-            }
-            printf ("\n     dis: ");
-            for (int j = 0; j < k; j++) {
-                printf ("%7g ", dis[j + i * k]);
-            }
-            printf ("\n");
-        }
-
-        printf ("note that the nearest neighbor is not at "
-                "distance 0 due to quantization errors\n");
-    }
-
-    return 0;
-}

data/vendor/faiss/demos/demo_sift1M.cpp

@@ -1,252 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <cassert>
-#include <cstring>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include <sys/time.h>
-
-#include <faiss/AutoTune.h>
-#include <faiss/index_factory.h>
-
-/**
- * To run this demo, please download the ANN_SIFT1M dataset from
- *
- *   http://corpus-texmex.irisa.fr/
- *
- * and unzip it to the sudirectory sift1M.
- **/
-
-/*****************************************************
- * I/O functions for fvecs and ivecs
- *****************************************************/
-
-
-float * fvecs_read (const char *fname,
-                    size_t *d_out, size_t *n_out)
-{
-    FILE *f = fopen(fname, "r");
-    if(!f) {
-        fprintf(stderr, "could not open %s\n", fname);
-        perror("");
-        abort();
-    }
-    int d;
-    fread(&d, 1, sizeof(int), f);
-    assert((d > 0 && d < 1000000) || !"unreasonable dimension");
-    fseek(f, 0, SEEK_SET);
-    struct stat st;
-    fstat(fileno(f), &st);
-    size_t sz = st.st_size;
-    assert(sz % ((d + 1) * 4) == 0 || !"weird file size");
-    size_t n = sz / ((d + 1) * 4);
-
-    *d_out = d; *n_out = n;
-    float *x = new float[n * (d + 1)];
-    size_t nr = fread(x, sizeof(float), n * (d + 1), f);
-    assert(nr == n * (d + 1) || !"could not read whole file");
-
-    // shift array to remove row headers
-    for(size_t i = 0; i < n; i++)
-        memmove(x + i * d, x + 1 + i * (d + 1), d * sizeof(*x));
-
-    fclose(f);
-    return x;
-}
-
-// not very clean, but works as long as sizeof(int) == sizeof(float)
-int *ivecs_read(const char *fname, size_t *d_out, size_t *n_out)
-{
-    return (int*)fvecs_read(fname, d_out, n_out);
-}
-
-double elapsed ()
-{
-    struct timeval tv;
-    gettimeofday (&tv, nullptr);
-    return  tv.tv_sec + tv.tv_usec * 1e-6;
-}
-
-
-
-int main()
-{
-    double t0 = elapsed();
-
-    // this is typically the fastest one.
-    const char *index_key = "IVF4096,Flat";
-
-    // these ones have better memory usage
-    // const char *index_key = "Flat";
-    // const char *index_key = "PQ32";
-    // const char *index_key = "PCA80,Flat";
-    // const char *index_key = "IVF4096,PQ8+16";
-    // const char *index_key = "IVF4096,PQ32";
-    // const char *index_key = "IMI2x8,PQ32";
-    // const char *index_key = "IMI2x8,PQ8+16";
-    // const char *index_key = "OPQ16_64,IMI2x8,PQ8+16";
-
-    faiss::Index * index;
-
-    size_t d;
-
-    {
-        printf ("[%.3f s] Loading train set\n", elapsed() - t0);
-
-        size_t nt;
-        float *xt = fvecs_read("sift1M/sift_learn.fvecs", &d, &nt);
-
-        printf ("[%.3f s] Preparing index \"%s\" d=%ld\n",
-                elapsed() - t0, index_key, d);
-        index = faiss::index_factory(d, index_key);
-
-        printf ("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nt);
-
-        index->train(nt, xt);
-        delete [] xt;
-    }
-
-
-    {
-        printf ("[%.3f s] Loading database\n", elapsed() - t0);
-
-        size_t nb, d2;
-        float *xb = fvecs_read("sift1M/sift_base.fvecs", &d2, &nb);
-        assert(d == d2 || !"dataset does not have same dimension as train set");
-
-        printf ("[%.3f s] Indexing database, size %ld*%ld\n",
-                elapsed() - t0, nb, d);
-
-        index->add(nb, xb);
-
-        delete [] xb;
-    }
-
-    size_t nq;
-    float *xq;
-
-    {
-        printf ("[%.3f s] Loading queries\n", elapsed() - t0);
-
-        size_t d2;
-        xq = fvecs_read("sift1M/sift_query.fvecs", &d2, &nq);
-        assert(d == d2 || !"query does not have same dimension as train set");
-
-    }
-
-    size_t k; // nb of results per query in the GT
-    faiss::Index::idx_t *gt;  // nq * k matrix of ground-truth nearest-neighbors
-
-    {
-        printf ("[%.3f s] Loading ground truth for %ld queries\n",
-                elapsed() - t0, nq);
-
-        // load ground-truth and convert int to long
-        size_t nq2;
-        int *gt_int = ivecs_read("sift1M/sift_groundtruth.ivecs", &k, &nq2);
-        assert(nq2 == nq || !"incorrect nb of ground truth entries");
-
-        gt = new faiss::Index::idx_t[k * nq];
-        for(int i = 0; i < k * nq; i++) {
-            gt[i] = gt_int[i];
-        }
-        delete [] gt_int;
-    }
-
-    // Result of the auto-tuning
-    std::string selected_params;
-
-    { // run auto-tuning
-
-        printf ("[%.3f s] Preparing auto-tune criterion 1-recall at 1 "
-                "criterion, with k=%ld nq=%ld\n", elapsed() - t0, k, nq);
-
-        faiss::OneRecallAtRCriterion crit(nq, 1);
-        crit.set_groundtruth (k, nullptr, gt);
-        crit.nnn = k; // by default, the criterion will request only 1 NN
-
-        printf ("[%.3f s] Preparing auto-tune parameters\n", elapsed() - t0);
-
-        faiss::ParameterSpace params;
-        params.initialize(index);
-
-        printf ("[%.3f s] Auto-tuning over %ld parameters (%ld combinations)\n",
-                elapsed() - t0, params.parameter_ranges.size(),
-                params.n_combinations());
-
-        faiss::OperatingPoints ops;
-        params.explore (index, nq, xq, crit, &ops);
-
-        printf ("[%.3f s] Found the following operating points: \n",
-                elapsed() - t0);
-
-        ops.display ();
-
-        // keep the first parameter that obtains > 0.5 1-recall@1
-        for (int i = 0; i < ops.optimal_pts.size(); i++) {
-            if (ops.optimal_pts[i].perf > 0.5) {
-                selected_params = ops.optimal_pts[i].key;
-                break;
-            }
-        }
-        assert (selected_params.size() >= 0 ||
-                !"could not find good enough op point");
-    }
-
-
-    { // Use the found configuration to perform a search
-
-        faiss::ParameterSpace params;
-
-        printf ("[%.3f s] Setting parameter configuration \"%s\" on index\n",
-                elapsed() - t0, selected_params.c_str());
-
-        params.set_index_parameters (index, selected_params.c_str());
-
-        printf ("[%.3f s] Perform a search on %ld queries\n",
-                elapsed() - t0, nq);
-
-        // output buffers
-        faiss::Index::idx_t *I = new  faiss::Index::idx_t[nq * k];
-        float *D = new float[nq * k];
-
-        index->search(nq, xq, k, D, I);
-
-        printf ("[%.3f s] Compute recalls\n", elapsed() - t0);
-
-        // evaluate result by hand.
-        int n_1 = 0, n_10 = 0, n_100 = 0;
-        for(int i = 0; i < nq; i++) {
-            int gt_nn = gt[i * k];
-            for(int j = 0; j < k; j++) {
-                if (I[i * k + j] == gt_nn) {
-                    if(j < 1) n_1++;
-                    if(j < 10) n_10++;
-                    if(j < 100) n_100++;
-                }
-            }
-        }
-        printf("R@1 = %.4f\n", n_1 / float(nq));
-        printf("R@10 = %.4f\n", n_10 / float(nq));
-        printf("R@100 = %.4f\n", n_100 / float(nq));
-
-    }
-
-    delete [] xq;
-    delete [] gt;
-    delete index;
-    return 0;
-}

data/vendor/faiss/demos/demo_weighted_kmeans.cpp

@@ -1,185 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-
-#include <faiss/Clustering.h>
-#include <faiss/utils/random.h>
-#include <faiss/utils/distances.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/IndexHNSW.h>
-
-
-namespace {
-
-
-enum WeightedKMeansType {
-    WKMT_FlatL2,
-    WKMT_FlatIP,
-    WKMT_FlatIP_spherical,
-    WKMT_HNSW,
-};
-
-
-float weighted_kmeans_clustering (size_t d, size_t n, size_t k,
-                                  const float *input,
-                                  const float *weights,
-                                  float *centroids,
-                                  WeightedKMeansType index_num)
-{
-    using namespace faiss;
-    Clustering clus (d, k);
-    clus.verbose = true;
-
-    std::unique_ptr<Index> index;
-
-    switch (index_num) {
-    case WKMT_FlatL2:
-        index.reset(new IndexFlatL2 (d));
-        break;
-    case WKMT_FlatIP:
-        index.reset(new IndexFlatIP (d));
-        break;
-    case WKMT_FlatIP_spherical:
-        index.reset(new IndexFlatIP (d));
-        clus.spherical = true;
-        break;
-    case WKMT_HNSW:
-        IndexHNSWFlat *ihnsw = new IndexHNSWFlat (d, 32);
-        ihnsw->hnsw.efSearch = 128;
-        index.reset(ihnsw);
-        break;
-    }
-
-    clus.train(n, input, *index.get(), weights);
-    // on output the index contains the centroids.
-    memcpy(centroids, clus.centroids.data(), sizeof(*centroids) * d * k);
-    return clus.iteration_stats.back().obj;
-}
-
-
-int d = 32;
-float sigma = 0.1;
-
-#define BIGTEST
-
-#ifdef BIGTEST
-// the production setup = setting of https://fb.quip.com/CWgnAAYbwtgs
-int nc = 200000;
-int n_big = 4;
-int n_small = 2;
-#else
-int nc = 5;
-int n_big = 100;
-int n_small = 10;
-#endif
-
-int n; // number of training points
-
-void generate_trainset (std::vector<float> & ccent,
-                        std::vector<float> & x,
-                        std::vector<float> & weights)
-{
-    // same sampling as test_build_blocks.py test_weighted
-
-    ccent.resize (d * 2 * nc);
-    faiss::float_randn (ccent.data(), d * 2 * nc, 123);
-    faiss::fvec_renorm_L2 (d, 2 * nc, ccent.data());
-    n = nc * n_big + nc * n_small;
-    x.resize(d * n);
-    weights.resize(n);
-    faiss::float_randn (x.data(), x.size(), 1234);
-
-    float *xi = x.data();
-    float *w = weights.data();
-    for (int ci = 0; ci < nc * 2; ci++) { // loop over centroids
-        int np = ci < nc ? n_big : n_small; // nb of points around this centroid
-        for (int i = 0; i < np; i++) {
-            for (int j = 0; j < d; j++) {
-                xi[j] = xi[j] * sigma + ccent[ci * d + j];
-            }
-            *w++ = ci < nc ? 0.1 : 10;
-            xi += d;
-        }
-    }
-}
-
-}
-
-
-int main(int argc, char **argv) {
-    std::vector<float> ccent;
-    std::vector<float> x;
-    std::vector<float> weights;
-
-    printf("generate training set\n");
-    generate_trainset(ccent, x, weights);
-
-    std::vector<float> centroids;
-    centroids.resize(nc * d);
-
-    int the_index_num = -1;
-    int the_with_weights = -1;
-
-    if (argc == 3) {
-        the_index_num = atoi(argv[1]);
-        the_with_weights = atoi(argv[2]);
-    }
-
-
-    for (int index_num = WKMT_FlatL2;
-         index_num <= WKMT_HNSW;
-         index_num++) {
-
-        if (the_index_num >= 0 && index_num != the_index_num) {
-            continue;
-        }
-
-        for (int with_weights = 0; with_weights <= 1; with_weights++) {
-            if (the_with_weights >= 0 && with_weights != the_with_weights) {
-                continue;
-            }
-
-            printf("=================== index_num=%d Run %s weights\n",
-                   index_num, with_weights ? "with" : "without");
-
-            weighted_kmeans_clustering (
-                 d, n, nc, x.data(),
-                 with_weights ? weights.data() : nullptr,
-                 centroids.data(), (WeightedKMeansType)index_num
-            );
-
-            { // compute distance of points to centroids
-                faiss::IndexFlatL2 cent_index(d);
-                cent_index.add(nc, centroids.data());
-                std::vector<float> dis (n);
-                std::vector<faiss::Index::idx_t> idx (n);
-
-                cent_index.search (nc * 2, ccent.data(), 1,
-                                   dis.data(), idx.data());
-
-                float dis1 = 0, dis2 = 0;
-                for (int i = 0; i < nc ; i++) {
-                    dis1 += dis[i];
-                }
-                printf("average distance of points from big clusters: %g\n",
-                       dis1 / nc);
-
-                for (int i = 0; i < nc ; i++) {
-                    dis2 += dis[i + nc];
-                }
-
-                printf("average distance of points from small clusters: %g\n",
-                       dis2 / nc);
-
-            }
-
-        }
-    }
-    return 0;
-}