faiss 0.1.3 → 0.2.0

data/vendor/faiss/tests/test_transfer_invlists.cpp

@@ -1,159 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <memory>
-#include <cstdio>
-#include <cstdlib>
-
-#include <gtest/gtest.h>
-
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/index_io.h>
-#include <faiss/impl/io.h>
-#include <faiss/AutoTune.h>
-#include <faiss/index_factory.h>
-#include <faiss/clone_index.h>
-#include <faiss/VectorTransform.h>
-#include <faiss/utils/random.h>
-#include <faiss/IVFlib.h>
-
-
-namespace {
-
-// parameters to use for the test
-int d = 64;
-size_t nb = 1000;
-size_t nq = 100;
-size_t nt = 500;
-int k = 10;
-int nlist = 40;
-
-using namespace faiss;
-
-typedef faiss::Index::idx_t idx_t;
-
-
-std::vector<float> get_data (size_t nb, int seed) {
-    std::vector<float> x (nb * d);
-    float_randn (x.data(), nb * d, seed);
-    return x;
-}
-
-
-void test_index_type(const char *factory_string) {
-
-    // transfer inverted lists in nslice slices
-    int nslice = 3;
-
-    /****************************************************************
-     * trained reference index
-     ****************************************************************/
-
-    std::unique_ptr<Index> trained (index_factory (d, factory_string));
-
-    {
-        auto xt = get_data (nt, 123);
-        trained->train (nt, xt.data());
-    }
-
-    // sample nq query vectors to check if results are the same
-    auto xq = get_data (nq, 818);
-
-
-    /****************************************************************
-     * source index
-     ***************************************************************/
-    std::unique_ptr<Index> src_index (clone_index (trained.get()));
-
-    { // add some data to source index
-        auto xb = get_data (nb, 245);
-        src_index->add (nb, xb.data());
-    }
-
-    ParameterSpace().set_index_parameter (src_index.get(), "nprobe", 4);
-
-    // remember reference search result on source index
-    std::vector<idx_t> Iref (nq * k);
-    std::vector<float> Dref (nq * k);
-    src_index->search (nq, xq.data(), k, Dref.data(), Iref.data());
-
-    /****************************************************************
-     * destination index -- should be replaced by source index
-     ***************************************************************/
-
-    std::unique_ptr<Index> dst_index (clone_index (trained.get()));
-
-    { // initial state: filled in with some garbage
-        int nb2 = nb + 10;
-        auto xb = get_data (nb2, 366);
-        dst_index->add (nb2, xb.data());
-    }
-
-    std::vector<idx_t> Inew (nq * k);
-    std::vector<float> Dnew (nq * k);
-
-    ParameterSpace().set_index_parameter (dst_index.get(), "nprobe", 4);
-
-    // transfer from source to destination in nslice slices
-    for (int sl = 0; sl < nslice; sl++) {
-
-        // so far, the indexes are different
-        dst_index->search (nq, xq.data(), k, Dnew.data(), Inew.data());
-        EXPECT_TRUE (Iref != Inew);
-        EXPECT_TRUE (Dref != Dnew);
-
-        // range of inverted list indices to transfer
-        long i0 = sl * nlist / nslice;
-        long i1 = (sl + 1) * nlist / nslice;
-
-        std::vector<uint8_t> data_to_transfer;
-        {
-            std::unique_ptr<ArrayInvertedLists> il
-                (ivflib::get_invlist_range (src_index.get(), i0, i1));
-            // serialize inverted lists
-            VectorIOWriter wr;
-            write_InvertedLists (il.get(), &wr);
-            data_to_transfer.swap (wr.data);
-        }
-
-        // transfer data here from source machine to dest machine
-
-        {
-            VectorIOReader reader;
-            reader.data.swap (data_to_transfer);
-
-            // deserialize inverted lists
-            std::unique_ptr<ArrayInvertedLists> il
-                (dynamic_cast<ArrayInvertedLists *>
-                 (read_InvertedLists (&reader)));
-
-            // swap inverted lists. Block searches here!
-            {
-                ivflib::set_invlist_range (dst_index.get(), i0, i1, il.get());
-            }
-        }
-
-    }
-    EXPECT_EQ (dst_index->ntotal, src_index->ntotal);
-
-    // now, the indexes are the same
-    dst_index->search (nq, xq.data(), k, Dnew.data(), Inew.data());
-    EXPECT_TRUE (Iref == Inew);
-    EXPECT_TRUE (Dref == Dnew);
-
-}
-
-}  // namespace
-
-
-TEST(TRANS, IVFFlat) {
-    test_index_type ("IVF40,Flat");
-}
-
-TEST(TRANS, IVFFlatPreproc) {
-    test_index_type ("PCAR32,IVF40,Flat");
-}

data/vendor/faiss/tutorial/cpp/1-Flat.cpp

@@ -1,104 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <faiss/IndexFlat.h>
-
-// 64-bit int
-using idx_t = faiss::Index::idx_t;
-
-int main() {
-    int d = 64;                            // dimension
-    int nb = 100000;                       // database size
-    int nq = 10000;                        // nb of queries
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-
-    float *xb = new float[d * nb];
-    float *xq = new float[d * nq];
-
-    for(int i = 0; i < nb; i++) {
-        for(int j = 0; j < d; j++)
-            xb[d * i + j] = distrib(rng);
-        xb[d * i] += i / 1000.;
-    }
-
-    for(int i = 0; i < nq; i++) {
-        for(int j = 0; j < d; j++)
-            xq[d * i + j] = distrib(rng);
-        xq[d * i] += i / 1000.;
-    }
-
-    faiss::IndexFlatL2 index(d);           // call constructor
-    printf("is_trained = %s\n", index.is_trained ? "true" : "false");
-    index.add(nb, xb);                     // add vectors to the index
-    printf("ntotal = %zd\n", index.ntotal);
-
-    int k = 4;
-
-    {       // sanity check: search 5 first vectors of xb
-        idx_t *I = new idx_t[k * 5];
-        float *D = new float[k * 5];
-
-        index.search(5, xb, k, D, I);
-
-        // print results
-        printf("I=\n");
-        for(int i = 0; i < 5; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        printf("D=\n");
-        for(int i = 0; i < 5; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%7g ", D[i * k + j]);
-            printf("\n");
-        }
-
-        delete [] I;
-        delete [] D;
-    }
-
-
-    {       // search xq
-        idx_t *I = new idx_t[k * nq];
-        float *D = new float[k * nq];
-
-        index.search(nq, xq, k, D, I);
-
-        // print results
-        printf("I (5 first results)=\n");
-        for(int i = 0; i < 5; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        printf("I (5 last results)=\n");
-        for(int i = nq - 5; i < nq; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        delete [] I;
-        delete [] D;
-    }
-
-
-
-    delete [] xb;
-    delete [] xq;
-
-    return 0;
-}

data/vendor/faiss/tutorial/cpp/2-IVFFlat.cpp

@@ -1,85 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <cassert>
-#include <random>
-
-#include <faiss/IndexFlat.h>
-#include <faiss/IndexIVFFlat.h>
-
-using idx_t = faiss::Index::idx_t;
-
-int main() {
-    int d = 64;                            // dimension
-    int nb = 100000;                       // database size
-    int nq = 10000;                        // nb of queries
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-
-    float *xb = new float[d * nb];
-    float *xq = new float[d * nq];
-
-    for(int i = 0; i < nb; i++) {
-        for(int j = 0; j < d; j++)
-            xb[d * i + j] = distrib(rng);
-        xb[d * i] += i / 1000.;
-    }
-
-    for(int i = 0; i < nq; i++) {
-        for(int j = 0; j < d; j++)
-            xq[d * i + j] = distrib(rng);
-        xq[d * i] += i / 1000.;
-    }
-
-
-    int nlist = 100;
-    int k = 4;
-
-    faiss::IndexFlatL2 quantizer(d);       // the other index
-    faiss::IndexIVFFlat index(&quantizer, d, nlist);
-    assert(!index.is_trained);
-    index.train(nb, xb);
-    assert(index.is_trained);
-    index.add(nb, xb);
-
-    {       // search xq
-        idx_t *I = new idx_t[k * nq];
-        float *D = new float[k * nq];
-
-        index.search(nq, xq, k, D, I);
-
-        printf("I=\n");
-        for(int i = nq - 5; i < nq; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        index.nprobe = 10;
-        index.search(nq, xq, k, D, I);
-
-        printf("I=\n");
-        for(int i = nq - 5; i < nq; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        delete [] I;
-        delete [] D;
-    }
-
-
-
-    delete [] xb;
-    delete [] xq;
-
-    return 0;
-}

data/vendor/faiss/tutorial/cpp/3-IVFPQ.cpp

@@ -1,98 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <faiss/IndexFlat.h>
-#include <faiss/IndexIVFPQ.h>
-
-using idx_t = faiss::Index::idx_t;
-
-int main() {
-    int d = 64;                            // dimension
-    int nb = 100000;                       // database size
-    int nq = 10000;                        // nb of queries
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-
-    float *xb = new float[d * nb];
-    float *xq = new float[d * nq];
-
-    for(int i = 0; i < nb; i++) {
-        for(int j = 0; j < d; j++)
-            xb[d * i + j] = distrib(rng);
-        xb[d * i] += i / 1000.;
-    }
-
-    for(int i = 0; i < nq; i++) {
-        for(int j = 0; j < d; j++)
-            xq[d * i + j] = distrib(rng);
-        xq[d * i] += i / 1000.;
-    }
-
-
-    int nlist = 100;
-    int k = 4;
-    int m = 8;                             // bytes per vector
-    faiss::IndexFlatL2 quantizer(d);       // the other index
-    faiss::IndexIVFPQ index(&quantizer, d, nlist, m, 8);
-
-    index.train(nb, xb);
-    index.add(nb, xb);
-
-    {       // sanity check
-        idx_t *I = new idx_t[k * 5];
-        float *D = new float[k * 5];
-
-        index.search(5, xb, k, D, I);
-
-        printf("I=\n");
-        for(int i = 0; i < 5; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        printf("D=\n");
-        for(int i = 0; i < 5; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%7g ", D[i * k + j]);
-            printf("\n");
-        }
-
-        delete [] I;
-        delete [] D;
-    }
-
-    {       // search xq
-        idx_t *I = new idx_t[k * nq];
-        float *D = new float[k * nq];
-
-        index.nprobe = 10;
-        index.search(nq, xq, k, D, I);
-
-        printf("I=\n");
-        for(int i = nq - 5; i < nq; i++) {
-            for(int j = 0; j < k; j++)
-                printf("%5zd ", I[i * k + j]);
-            printf("\n");
-        }
-
-        delete [] I;
-        delete [] D;
-    }
-
-
-
-    delete [] xb;
-    delete [] xq;
-
-    return 0;
-}