faiss 0.1.0 → 0.1.1

data/vendor/faiss/tests/test_transfer_invlists.cpp

@@ -0,0 +1,159 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <memory>
+#include <cstdio>
+#include <cstdlib>
+
+#include <gtest/gtest.h>
+
+#include <faiss/IndexIVFFlat.h>
+#include <faiss/index_io.h>
+#include <faiss/impl/io.h>
+#include <faiss/AutoTune.h>
+#include <faiss/index_factory.h>
+#include <faiss/clone_index.h>
+#include <faiss/VectorTransform.h>
+#include <faiss/utils/random.h>
+#include <faiss/IVFlib.h>
+
+
+namespace {
+
+// parameters to use for the test
+int d = 64;
+size_t nb = 1000;
+size_t nq = 100;
+size_t nt = 500;
+int k = 10;
+int nlist = 40;
+
+using namespace faiss;
+
+typedef faiss::Index::idx_t idx_t;
+
+
+std::vector<float> get_data (size_t nb, int seed) {
+    std::vector<float> x (nb * d);
+    float_randn (x.data(), nb * d, seed);
+    return x;
+}
+
+
+void test_index_type(const char *factory_string) {
+
+    // transfer inverted lists in nslice slices
+    int nslice = 3;
+
+    /****************************************************************
+     * trained reference index
+     ****************************************************************/
+
+    std::unique_ptr<Index> trained (index_factory (d, factory_string));
+
+    {
+        auto xt = get_data (nt, 123);
+        trained->train (nt, xt.data());
+    }
+
+    // sample nq query vectors to check if results are the same
+    auto xq = get_data (nq, 818);
+
+
+    /****************************************************************
+     * source index
+     ***************************************************************/
+    std::unique_ptr<Index> src_index (clone_index (trained.get()));
+
+    { // add some data to source index
+        auto xb = get_data (nb, 245);
+        src_index->add (nb, xb.data());
+    }
+
+    ParameterSpace().set_index_parameter (src_index.get(), "nprobe", 4);
+
+    // remember reference search result on source index
+    std::vector<idx_t> Iref (nq * k);
+    std::vector<float> Dref (nq * k);
+    src_index->search (nq, xq.data(), k, Dref.data(), Iref.data());
+
+    /****************************************************************
+     * destination index -- should be replaced by source index
+     ***************************************************************/
+
+    std::unique_ptr<Index> dst_index (clone_index (trained.get()));
+
+    { // initial state: filled in with some garbage
+        int nb2 = nb + 10;
+        auto xb = get_data (nb2, 366);
+        dst_index->add (nb2, xb.data());
+    }
+
+    std::vector<idx_t> Inew (nq * k);
+    std::vector<float> Dnew (nq * k);
+
+    ParameterSpace().set_index_parameter (dst_index.get(), "nprobe", 4);
+
+    // transfer from source to destination in nslice slices
+    for (int sl = 0; sl < nslice; sl++) {
+
+        // so far, the indexes are different
+        dst_index->search (nq, xq.data(), k, Dnew.data(), Inew.data());
+        EXPECT_TRUE (Iref != Inew);
+        EXPECT_TRUE (Dref != Dnew);
+
+        // range of inverted list indices to transfer
+        long i0 = sl * nlist / nslice;
+        long i1 = (sl + 1) * nlist / nslice;
+
+        std::vector<uint8_t> data_to_transfer;
+        {
+            std::unique_ptr<ArrayInvertedLists> il
+                (ivflib::get_invlist_range (src_index.get(), i0, i1));
+            // serialize inverted lists
+            VectorIOWriter wr;
+            write_InvertedLists (il.get(), &wr);
+            data_to_transfer.swap (wr.data);
+        }
+
+        // transfer data here from source machine to dest machine
+
+        {
+            VectorIOReader reader;
+            reader.data.swap (data_to_transfer);
+
+            // deserialize inverted lists
+            std::unique_ptr<ArrayInvertedLists> il
+                (dynamic_cast<ArrayInvertedLists *>
+                 (read_InvertedLists (&reader)));
+
+            // swap inverted lists. Block searches here!
+            {
+                ivflib::set_invlist_range (dst_index.get(), i0, i1, il.get());
+            }
+        }
+
+    }
+    EXPECT_EQ (dst_index->ntotal, src_index->ntotal);
+
+    // now, the indexes are the same
+    dst_index->search (nq, xq.data(), k, Dnew.data(), Inew.data());
+    EXPECT_TRUE (Iref == Inew);
+    EXPECT_TRUE (Dref == Dnew);
+
+}
+
+}  // namespace
+
+
+TEST(TRANS, IVFFlat) {
+    test_index_type ("IVF40,Flat");
+}
+
+TEST(TRANS, IVFFlatPreproc) {
+    test_index_type ("PCAR32,IVF40,Flat");
+}

data/vendor/faiss/tutorial/cpp/1-Flat.cpp

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <faiss/IndexFlat.h>
+
+
+int main() {
+    int d = 64;                            // dimension
+    int nb = 100000;                       // database size
+    int nq = 10000;                        // nb of queries
+
+    float *xb = new float[d * nb];
+    float *xq = new float[d * nq];
+
+    for(int i = 0; i < nb; i++) {
+        for(int j = 0; j < d; j++)
+            xb[d * i + j] = drand48();
+        xb[d * i] += i / 1000.;
+    }
+
+    for(int i = 0; i < nq; i++) {
+        for(int j = 0; j < d; j++)
+            xq[d * i + j] = drand48();
+        xq[d * i] += i / 1000.;
+    }
+
+    faiss::IndexFlatL2 index(d);           // call constructor
+    printf("is_trained = %s\n", index.is_trained ? "true" : "false");
+    index.add(nb, xb);                     // add vectors to the index
+    printf("ntotal = %ld\n", index.ntotal);
+
+    int k = 4;
+
+    {       // sanity check: search 5 first vectors of xb
+        long *I = new long[k * 5];
+        float *D = new float[k * 5];
+
+        index.search(5, xb, k, D, I);
+
+        // print results
+        printf("I=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        printf("D=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%7g ", D[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+
+    {       // search xq
+        long *I = new long[k * nq];
+        float *D = new float[k * nq];
+
+        index.search(nq, xq, k, D, I);
+
+        // print results
+        printf("I (5 first results)=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        printf("I (5 last results)=\n");
+        for(int i = nq - 5; i < nq; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+
+
+    delete [] xb;
+    delete [] xq;
+
+    return 0;
+}

data/vendor/faiss/tutorial/cpp/2-IVFFlat.cpp

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+#include <cassert>
+
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexIVFFlat.h>
+
+
+int main() {
+    int d = 64;                            // dimension
+    int nb = 100000;                       // database size
+    int nq = 10000;                        // nb of queries
+
+    float *xb = new float[d * nb];
+    float *xq = new float[d * nq];
+
+    for(int i = 0; i < nb; i++) {
+        for(int j = 0; j < d; j++)
+            xb[d * i + j] = drand48();
+        xb[d * i] += i / 1000.;
+    }
+
+    for(int i = 0; i < nq; i++) {
+        for(int j = 0; j < d; j++)
+            xq[d * i + j] = drand48();
+        xq[d * i] += i / 1000.;
+    }
+
+
+    int nlist = 100;
+    int k = 4;
+
+    faiss::IndexFlatL2 quantizer(d);       // the other index
+    faiss::IndexIVFFlat index(&quantizer, d, nlist, faiss::METRIC_L2);
+    // here we specify METRIC_L2, by default it performs inner-product search
+    assert(!index.is_trained);
+    index.train(nb, xb);
+    assert(index.is_trained);
+    index.add(nb, xb);
+
+    {       // search xq
+        long *I = new long[k * nq];
+        float *D = new float[k * nq];
+
+        index.search(nq, xq, k, D, I);
+
+        printf("I=\n");
+        for(int i = nq - 5; i < nq; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        index.nprobe = 10;
+        index.search(nq, xq, k, D, I);
+
+        printf("I=\n");
+        for(int i = nq - 5; i < nq; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+
+
+    delete [] xb;
+    delete [] xq;
+
+    return 0;
+}

data/vendor/faiss/tutorial/cpp/3-IVFPQ.cpp

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexIVFPQ.h>
+
+
+int main() {
+    int d = 64;                            // dimension
+    int nb = 100000;                       // database size
+    int nq = 10000;                        // nb of queries
+
+    float *xb = new float[d * nb];
+    float *xq = new float[d * nq];
+
+    for(int i = 0; i < nb; i++) {
+        for(int j = 0; j < d; j++)
+            xb[d * i + j] = drand48();
+        xb[d * i] += i / 1000.;
+    }
+
+    for(int i = 0; i < nq; i++) {
+        for(int j = 0; j < d; j++)
+            xq[d * i + j] = drand48();
+        xq[d * i] += i / 1000.;
+    }
+
+
+    int nlist = 100;
+    int k = 4;
+    int m = 8;                             // bytes per vector
+    faiss::IndexFlatL2 quantizer(d);       // the other index
+    faiss::IndexIVFPQ index(&quantizer, d, nlist, m, 8);
+    // here we specify METRIC_L2, by default it performs inner-product search
+    index.train(nb, xb);
+    index.add(nb, xb);
+
+    {       // sanity check
+        long *I = new long[k * 5];
+        float *D = new float[k * 5];
+
+        index.search(5, xb, k, D, I);
+
+        printf("I=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        printf("D=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%7g ", D[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+    {       // search xq
+        long *I = new long[k * nq];
+        float *D = new float[k * nq];
+
+        index.nprobe = 10;
+        index.search(nq, xq, k, D, I);
+
+        printf("I=\n");
+        for(int i = nq - 5; i < nq; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+
+
+    delete [] xb;
+    delete [] xq;
+
+    return 0;
+}

data/vendor/faiss/tutorial/cpp/4-GPU.cpp

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+#include <cassert>
+
+#include <faiss/IndexFlat.h>
+#include <faiss/gpu/GpuIndexFlat.h>
+#include <faiss/gpu/GpuIndexIVFFlat.h>
+#include <faiss/gpu/StandardGpuResources.h>
+
+
+int main() {
+    int d = 64;                            // dimension
+    int nb = 100000;                       // database size
+    int nq = 10000;                        // nb of queries
+
+    float *xb = new float[d * nb];
+    float *xq = new float[d * nq];
+
+    for(int i = 0; i < nb; i++) {
+        for(int j = 0; j < d; j++)
+            xb[d * i + j] = drand48();
+        xb[d * i] += i / 1000.;
+    }
+
+    for(int i = 0; i < nq; i++) {
+        for(int j = 0; j < d; j++)
+            xq[d * i + j] = drand48();
+        xq[d * i] += i / 1000.;
+    }
+
+    faiss::gpu::StandardGpuResources res;
+
+    // Using a flat index
+
+    faiss::gpu::GpuIndexFlatL2 index_flat(&res, d);
+
+    printf("is_trained = %s\n", index_flat.is_trained ? "true" : "false");
+    index_flat.add(nb, xb);  // add vectors to the index
+    printf("ntotal = %ld\n", index_flat.ntotal);
+
+    int k = 4;
+
+    {       // search xq
+        long *I = new long[k * nq];
+        float *D = new float[k * nq];
+
+        index_flat.search(nq, xq, k, D, I);
+
+        // print results
+        printf("I (5 first results)=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        printf("I (5 last results)=\n");
+        for(int i = nq - 5; i < nq; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+    // Using an IVF index
+
+    int nlist = 100;
+    faiss::gpu::GpuIndexIVFFlat index_ivf(&res, d, nlist, faiss::METRIC_L2);
+    // here we specify METRIC_L2, by default it performs inner-product search
+
+    assert(!index_ivf.is_trained);
+    index_ivf.train(nb, xb);
+    assert(index_ivf.is_trained);
+    index_ivf.add(nb, xb);  // add vectors to the index
+
+    printf("is_trained = %s\n", index_ivf.is_trained ? "true" : "false");
+    printf("ntotal = %ld\n", index_ivf.ntotal);
+
+    {       // search xq
+        long *I = new long[k * nq];
+        float *D = new float[k * nq];
+
+        index_ivf.search(nq, xq, k, D, I);
+
+        // print results
+        printf("I (5 first results)=\n");
+        for(int i = 0; i < 5; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        printf("I (5 last results)=\n");
+        for(int i = nq - 5; i < nq; i++) {
+            for(int j = 0; j < k; j++)
+                printf("%5ld ", I[i * k + j]);
+            printf("\n");
+        }
+
+        delete [] I;
+        delete [] D;
+    }
+
+
+    delete [] xb;
+    delete [] xq;
+
+    return 0;
+}