faiss 0.1.3 → 0.2.0

data/vendor/faiss/tests/test_merge.cpp

@@ -1,260 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <gtest/gtest.h>
-
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/MetaIndexes.h>
-#include <faiss/IndexPreTransform.h>
-#include <faiss/OnDiskInvertedLists.h>
-#include <faiss/IVFlib.h>
-
-
-namespace {
-
-
-struct Tempfilename {
-
-    static pthread_mutex_t mutex;
-
-    std::string filename;
-
-    Tempfilename (const char *prefix = nullptr) {
-        pthread_mutex_lock (&mutex);
-        char *cfname = tempnam (nullptr, prefix);
-        filename = cfname;
-        free(cfname);
-        pthread_mutex_unlock (&mutex);
-    }
-
-    ~Tempfilename () {
-        if (access (filename.c_str(), F_OK)) {
-            unlink (filename.c_str());
-        }
-    }
-
-    const char *c_str() {
-        return filename.c_str();
-    }
-
-};
-
-pthread_mutex_t Tempfilename::mutex = PTHREAD_MUTEX_INITIALIZER;
-
-
-typedef faiss::Index::idx_t idx_t;
-
-// parameters to use for the test
-int d = 64;
-size_t nb = 1000;
-size_t nq = 100;
-int nindex = 4;
-int k = 10;
-int nlist = 40;
-
-struct CommonData {
-
-    std::vector <float> database;
-    std::vector <float> queries;
-    std::vector<idx_t> ids;
-    faiss::IndexFlatL2 quantizer;
-
-    CommonData(): database (nb * d), queries (nq * d), ids(nb), quantizer (d) {
-
-        std::mt19937 rng;
-        std::uniform_real_distribution<> distrib;
-        for (size_t i = 0; i < nb * d; i++) {
-            database[i] = distrib(rng);
-        }
-        for (size_t i = 0; i < nq * d; i++) {
-            queries[i] = distrib(rng);
-        }
-        for (int i = 0; i < nb; i++) {
-            ids[i] = 123 + 456 * i;
-        }
-        { // just to train the quantizer
-            faiss::IndexIVFFlat iflat (&quantizer, d, nlist);
-            iflat.train(nb, database.data());
-        }
-    }
-};
-
-CommonData cd;
-
-/// perform a search on shards, then merge and search again and
-/// compare results.
-int compare_merged (faiss::IndexShards *index_shards, bool shift_ids,
-                    bool standard_merge = true)
-{
-
-    std::vector<idx_t> refI(k * nq);
-    std::vector<float> refD(k * nq);
-
-    index_shards->search(nq, cd.queries.data(), k, refD.data(), refI.data());
-    Tempfilename filename;
-
-    std::vector<idx_t> newI(k * nq);
-    std::vector<float> newD(k * nq);
-
-    if (standard_merge) {
-
-        for (int i = 1; i < nindex; i++) {
-            faiss::ivflib::merge_into(
-                   index_shards->at(0), index_shards->at(i),
-                   shift_ids);
-        }
-
-        index_shards->syncWithSubIndexes();
-    } else {
-        std::vector<const faiss::InvertedLists *> lists;
-        faiss::IndexIVF *index0 = nullptr;
-        size_t ntotal = 0;
-        for (int i = 0; i < nindex; i++) {
-            auto index_ivf = dynamic_cast<faiss::IndexIVF*>(index_shards->at(i));
-            assert (index_ivf);
-            if (i == 0) {
-                index0 = index_ivf;
-            }
-            lists.push_back (index_ivf->invlists);
-            ntotal += index_ivf->ntotal;
-        }
-
-        auto il = new faiss::OnDiskInvertedLists(
-                        index0->nlist, index0->code_size,
-                        filename.c_str());
-
-        il->merge_from(lists.data(), lists.size());
-
-        index0->replace_invlists(il, true);
-        index0->ntotal = ntotal;
-    }
-    // search only on first index
-    index_shards->at(0)->search(nq, cd.queries.data(),
-                                k, newD.data(), newI.data());
-
-    size_t ndiff = 0;
-    for (size_t i = 0; i < k * nq; i++) {
-        if (refI[i] != newI[i]) {
-            ndiff ++;
-        }
-    }
-    return ndiff;
-}
-
-}  // namespace
-
-
-// test on IVFFlat with implicit numbering
-TEST(MERGE, merge_flat_no_ids) {
-    faiss::IndexShards index_shards(d);
-    index_shards.own_fields = true;
-    for (int i = 0; i < nindex; i++) {
-        index_shards.add_shard (
-            new faiss::IndexIVFFlat (&cd.quantizer, d, nlist));
-    }
-    EXPECT_TRUE(index_shards.is_trained);
-    index_shards.add(nb, cd.database.data());
-    size_t prev_ntotal = index_shards.ntotal;
-    int ndiff = compare_merged(&index_shards, true);
-    EXPECT_EQ (prev_ntotal, index_shards.ntotal);
-    EXPECT_EQ(0, ndiff);
-}
-
-
-// test on IVFFlat, explicit ids
-TEST(MERGE, merge_flat) {
-    faiss::IndexShards index_shards(d, false, false);
-    index_shards.own_fields = true;
-
-    for (int i = 0; i < nindex; i++) {
-        index_shards.add_shard (
-             new faiss::IndexIVFFlat (&cd.quantizer, d, nlist));
-    }
-
-    EXPECT_TRUE(index_shards.is_trained);
-    index_shards.add_with_ids(nb, cd.database.data(), cd.ids.data());
-    int ndiff = compare_merged(&index_shards, false);
-    EXPECT_GE(0, ndiff);
-}
-
-// test on IVFFlat and a VectorTransform
-TEST(MERGE, merge_flat_vt) {
-    faiss::IndexShards index_shards(d, false, false);
-    index_shards.own_fields = true;
-
-    // here we have to retrain because of the vectorTransform
-    faiss::RandomRotationMatrix rot(d, d);
-    rot.init(1234);
-    faiss::IndexFlatL2 quantizer (d);
-
-    { // just to train the quantizer
-        faiss::IndexIVFFlat iflat (&quantizer, d, nlist);
-        faiss::IndexPreTransform ipt (&rot, &iflat);
-        ipt.train(nb, cd.database.data());
-    }
-
-    for (int i = 0; i < nindex; i++) {
-        faiss::IndexPreTransform * ipt = new faiss::IndexPreTransform (
-             new faiss::RandomRotationMatrix (rot),
-             new faiss::IndexIVFFlat (&quantizer, d, nlist)
-        );
-        ipt->own_fields = true;
-        index_shards.add_shard (ipt);
-    }
-    EXPECT_TRUE(index_shards.is_trained);
-    index_shards.add_with_ids(nb, cd.database.data(), cd.ids.data());
-    size_t prev_ntotal = index_shards.ntotal;
-    int ndiff = compare_merged(&index_shards, false);
-    EXPECT_EQ (prev_ntotal, index_shards.ntotal);
-    EXPECT_GE(0, ndiff);
-}
-
-
-// put the merged invfile on disk
-TEST(MERGE, merge_flat_ondisk) {
-    faiss::IndexShards index_shards(d, false, false);
-    index_shards.own_fields = true;
-    Tempfilename filename;
-
-    for (int i = 0; i < nindex; i++) {
-        auto ivf = new faiss::IndexIVFFlat (&cd.quantizer, d, nlist);
-        if (i == 0) {
-            auto il = new faiss::OnDiskInvertedLists (
-                ivf->nlist, ivf->code_size,
-                filename.c_str());
-            ivf->replace_invlists(il, true);
-        }
-        index_shards.add_shard (ivf);
-    }
-
-    EXPECT_TRUE(index_shards.is_trained);
-    index_shards.add_with_ids(nb, cd.database.data(), cd.ids.data());
-    int ndiff = compare_merged(&index_shards, false);
-
-    EXPECT_EQ(ndiff, 0);
-}
-
-// now use ondisk specific merge
-TEST(MERGE, merge_flat_ondisk_2) {
-    faiss::IndexShards index_shards(d, false, false);
-    index_shards.own_fields = true;
-
-    for (int i = 0; i < nindex; i++) {
-        index_shards.add_shard (
-             new faiss::IndexIVFFlat (&cd.quantizer, d, nlist));
-    }
-    EXPECT_TRUE(index_shards.is_trained);
-    index_shards.add_with_ids(nb, cd.database.data(), cd.ids.data());
-    int ndiff = compare_merged(&index_shards, false, false);
-    EXPECT_GE(0, ndiff);
-}

data/vendor/faiss/tests/test_omp_threads.cpp

@@ -1,14 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <gtest/gtest.h>
-
-#include <faiss/utils/utils.h>
-
-TEST(Threading, openmp) {
-  EXPECT_TRUE(faiss::check_openmp());
-}

data/vendor/faiss/tests/test_ondisk_ivf.cpp

@@ -1,225 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <omp.h>
-
-#include <unordered_map>
-#include <pthread.h>
-
-#include <gtest/gtest.h>
-
-#include <faiss/OnDiskInvertedLists.h>
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/utils/random.h>
-#include <faiss/index_io.h>
-
-
-namespace {
-
-struct Tempfilename {
-
-    static pthread_mutex_t mutex;
-
-    std::string filename;
-
-    Tempfilename (const char *prefix = nullptr) {
-        pthread_mutex_lock (&mutex);
-        char *cfname = tempnam (nullptr, prefix);
-        filename = cfname;
-        free(cfname);
-        pthread_mutex_unlock (&mutex);
-    }
-
-    ~Tempfilename () {
-        if (access (filename.c_str(), F_OK)) {
-            unlink (filename.c_str());
-        }
-    }
-
-    const char *c_str() {
-        return filename.c_str();
-    }
-
-};
-
-pthread_mutex_t Tempfilename::mutex = PTHREAD_MUTEX_INITIALIZER;
-
-}  // namespace
-
-
-TEST(ONDISK, make_invlists) {
-    int nlist = 100;
-    int code_size = 32;
-    int nadd = 1000000;
-    std::unordered_map<int, int> listnos;
-
-    Tempfilename filename;
-
-    faiss::OnDiskInvertedLists ivf (
-                nlist, code_size,
-                filename.c_str());
-
-    {
-        std::vector<uint8_t> code(32);
-        std::mt19937 rng;
-        std::uniform_real_distribution<> distrib;
-        for (int i = 0; i < nadd; i++) {
-            double d = distrib(rng);
-            int list_no = int(nlist * d * d); // skewed distribution
-            int * ar = (int*)code.data();
-            ar[0] = i;
-            ar[1] = list_no;
-            ivf.add_entry (list_no, i, code.data());
-            listnos[i] = list_no;
-        }
-    }
-
-    int ntot = 0;
-    for (int i = 0; i < nlist; i++) {
-        int size = ivf.list_size(i);
-        const faiss::Index::idx_t *ids = ivf.get_ids (i);
-        const uint8_t *codes = ivf.get_codes (i);
-        for (int j = 0; j < size; j++) {
-            faiss::Index::idx_t id = ids[j];
-            const int * ar = (const int*)&codes[code_size * j];
-            EXPECT_EQ (ar[0], id);
-            EXPECT_EQ (ar[1], i);
-            EXPECT_EQ (listnos[id], i);
-            ntot ++;
-        }
-    }
-    EXPECT_EQ (ntot, nadd);
-};
-
-
-TEST(ONDISK, test_add) {
-    int d = 8;
-    int nlist = 30, nq = 200, nb = 1500, k = 10;
-    faiss::IndexFlatL2 quantizer(d);
-    {
-        std::vector<float> x(d * nlist);
-        faiss::float_rand(x.data(), d * nlist, 12345);
-        quantizer.add(nlist, x.data());
-    }
-    std::vector<float> xb(d * nb);
-    faiss::float_rand(xb.data(), d * nb, 23456);
-
-    faiss::IndexIVFFlat index(&quantizer, d, nlist);
-    index.add(nb, xb.data());
-
-    std::vector<float> xq(d * nb);
-    faiss::float_rand(xq.data(), d * nq, 34567);
-
-    std::vector<float> ref_D (nq * k);
-    std::vector<faiss::Index::idx_t> ref_I (nq * k);
-
-    index.search (nq, xq.data(), k,
-                  ref_D.data(), ref_I.data());
-
-    Tempfilename filename, filename2;
-
-    // test add + search
-    {
-        faiss::IndexIVFFlat index2(&quantizer, d, nlist);
-
-        faiss::OnDiskInvertedLists ivf (
-                index.nlist, index.code_size,
-                filename.c_str());
-
-        index2.replace_invlists(&ivf);
-
-        index2.add(nb, xb.data());
-
-        std::vector<float> new_D (nq * k);
-        std::vector<faiss::Index::idx_t> new_I (nq * k);
-
-        index2.search (nq, xq.data(), k,
-                       new_D.data(), new_I.data());
-
-        EXPECT_EQ (ref_D, new_D);
-        EXPECT_EQ (ref_I, new_I);
-
-        write_index(&index2, filename2.c_str());
-
-    }
-
-    // test io
-    {
-        faiss::Index *index3 = faiss::read_index(filename2.c_str());
-
-        std::vector<float> new_D (nq * k);
-        std::vector<faiss::Index::idx_t> new_I (nq * k);
-
-        index3->search (nq, xq.data(), k,
-                        new_D.data(), new_I.data());
-
-        EXPECT_EQ (ref_D, new_D);
-        EXPECT_EQ (ref_I, new_I);
-
-        delete index3;
-    }
-
-};
-
-
-
-// WARN this thest will run multithreaded only in opt mode
-TEST(ONDISK, make_invlists_threaded) {
-    int nlist = 100;
-    int code_size = 32;
-    int nadd = 1000000;
-
-    Tempfilename filename;
-
-    faiss::OnDiskInvertedLists ivf (
-                nlist, code_size,
-                filename.c_str());
-
-    std::vector<int> list_nos (nadd);
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-    for (int i = 0; i < nadd; i++) {
-        double d = distrib(rng);
-        list_nos[i] = int(nlist * d * d); // skewed distribution
-    }
-
-#pragma omp parallel
-    {
-        std::vector<uint8_t> code(32);
-#pragma omp for
-        for (int i = 0; i < nadd; i++) {
-            int list_no = list_nos[i];
-            int * ar = (int*)code.data();
-            ar[0] = i;
-            ar[1] = list_no;
-            ivf.add_entry (list_no, i, code.data());
-        }
-    }
-
-    int ntot = 0;
-    for (int i = 0; i < nlist; i++) {
-        int size = ivf.list_size(i);
-        const faiss::Index::idx_t *ids = ivf.get_ids (i);
-        const uint8_t *codes = ivf.get_codes (i);
-        for (int j = 0; j < size; j++) {
-            faiss::Index::idx_t id = ids[j];
-            const int * ar = (const int*)&codes[code_size * j];
-            EXPECT_EQ (ar[0], id);
-            EXPECT_EQ (ar[1], i);
-            EXPECT_EQ (list_nos[id], i);
-            ntot ++;
-        }
-    }
-    EXPECT_EQ (ntot, nadd);
-
-};