faiss 0.1.3 → 0.2.0

data/vendor/faiss/misc/test_blas.cpp

@@ -1,87 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#undef FINTEGER
-#define FINTEGER long
-
-
-extern "C" {
-
-/* declare BLAS functions, see http://www.netlib.org/clapack/cblas/ */
-
-int sgemm_ (const char *transa, const char *transb, FINTEGER *m, FINTEGER *
-            n, FINTEGER *k, const float *alpha, const float *a,
-            FINTEGER *lda, const float *b, FINTEGER *
-            ldb, float *beta, float *c, FINTEGER *ldc);
-
-/* Lapack functions, see http://www.netlib.org/clapack/old/single/sgeqrf.c */
-
-int sgeqrf_ (FINTEGER *m, FINTEGER *n, float *a, FINTEGER *lda,
-                 float *tau, float *work, FINTEGER *lwork, FINTEGER *info);
-
-}
-
-float *new_random_vec(int size)
-{
-    float *x = new float[size];
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-    for (int i = 0; i < size; i++)
-        x[i] = distrib(rng);
-    return x;
-}
-
-
-int main() {
-
-    FINTEGER m = 10, n = 20, k = 30;
-    float *a = new_random_vec(m * k), *b = new_random_vec(n * k), *c = new float[n * m];
-    float one = 1.0, zero = 0.0;
-
-    printf("BLAS test\n");
-
-    sgemm_("Not transposed", "Not transposed",
-           &m, &n, &k, &one, a, &m, b, &k, &zero, c, &m);
-
-    printf("errors=\n");
-
-    for (int i = 0; i < m; i++) {
-        for (int j = 0; j < n; j++) {
-            float accu = 0;
-            for (int l = 0; l < k; l++)
-                accu += a[i + l * m] * b[l + j * k];
-            printf ("%6.3f ", accu - c[i + j * m]);
-        }
-        printf("\n");
-    }
-
-    long info = 0x64bL << 32;
-    long mi = 0x64bL << 32 | m;
-    float *tau = new float[m];
-    FINTEGER lwork = -1;
-
-    float work1;
-
-    printf("Intentional Lapack error (appears only for 64-bit INTEGER):\n");
-    sgeqrf_ (&mi, &n, c, &m, tau, &work1, &lwork, (FINTEGER*)&info);
-
-    // sgeqrf_ (&m, &n, c, &zeroi, tau, &work1, &lwork, (FINTEGER*)&info);
-    printf("info=%016lx\n", info);
-
-    if(info >> 32 == 0x64b) {
-        printf("Lapack uses 32-bit integers\n");
-    } else {
-        printf("Lapack uses 64-bit integers\n");
-    }
-
-
-    return 0;
-}

data/vendor/faiss/tests/test_binary_flat.cpp

@@ -1,62 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-
-#include <gtest/gtest.h>
-
-#include <faiss/IndexBinaryFlat.h>
-#include <faiss/utils/hamming.h>
-
-TEST(BinaryFlat, accuracy) {
-  // dimension of the vectors to index
-  int d = 64;
-
-  // size of the database we plan to index
-  size_t nb = 1000;
-
-  // make the index object and train it
-  faiss::IndexBinaryFlat index(d);
-
-  std::vector<uint8_t> database(nb * (d / 8));
-  for (size_t i = 0; i < nb * (d / 8); i++) {
-    database[i] = rand() % 0x100;
-  }
-
-  { // populating the database
-    index.add(nb, database.data());
-  }
-
-  size_t nq = 200;
-
-  { // searching the database
-
-    std::vector<uint8_t> queries(nq * (d / 8));
-    for (size_t i = 0; i < nq * (d / 8); i++) {
-      queries[i] = rand() % 0x100;
-    }
-
-    int k = 5;
-    std::vector<faiss::IndexBinary::idx_t> nns(k * nq);
-    std::vector<int>                     dis(k * nq);
-
-    index.search(nq, queries.data(), k, dis.data(), nns.data());
-
-    for (size_t i = 0; i < nq; ++i) {
-      faiss::HammingComputer8 hc(queries.data() + i * (d / 8), d / 8);
-      hamdis_t dist_min = hc.hamming(database.data());
-      for (size_t j = 1; j < nb; ++j) {
-        hamdis_t dist = hc.hamming(database.data() + j * (d / 8));
-        if (dist < dist_min) {
-          dist_min = dist;
-        }
-      }
-      EXPECT_EQ(dist_min, dis[k * i]);
-    }
-  }
-}

data/vendor/faiss/tests/test_dealloc_invlists.cpp

@@ -1,188 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-
-#include <memory>
-#include <vector>
-#include <random>
-
-#include <gtest/gtest.h>
-
-#include <faiss/IndexIVF.h>
-#include <faiss/index_factory.h>
-#include <faiss/AutoTune.h>
-#include <faiss/index_io.h>
-#include <faiss/IVFlib.h>
-
-
-using namespace faiss;
-
-namespace {
-
-typedef Index::idx_t idx_t;
-
-
-// dimension of the vectors to index
-int d = 32;
-
-// nb of training vectors
-size_t nt = 5000;
-
-// size of the database points per window step
-size_t nb = 1000;
-
-// nb of queries
-size_t nq = 200;
-
-std::mt19937 rng;
-
-std::vector<float> make_data(size_t n)
-{
-    std::vector <float> database (n * d);
-    std::uniform_real_distribution<> distrib;
-
-    for (size_t i = 0; i < n * d; i++) {
-        database[i] = distrib(rng);
-    }
-    return database;
-}
-
-std::unique_ptr<Index> make_trained_index(const char *index_type)
-{
-    auto index = std::unique_ptr<Index>(index_factory(d, index_type));
-    auto xt = make_data(nt * d);
-    index->train(nt, xt.data());
-    ParameterSpace().set_index_parameter (index.get(), "nprobe", 4);
-    return index;
-}
-
-std::vector<idx_t> search_index(Index *index, const float *xq) {
-    int k = 10;
-    std::vector<idx_t> I(k * nq);
-    std::vector<float> D(k * nq);
-    index->search (nq, xq, k, D.data(), I.data());
-    return I;
-}
-
-
-
-
-
-/*************************************************************
- * Test functions for a given index type
- *************************************************************/
-
-struct EncapsulateInvertedLists: InvertedLists {
-
-    const InvertedLists *il;
-
-    EncapsulateInvertedLists(const InvertedLists *il):
-        InvertedLists(il->nlist, il->code_size),
-        il(il)
-    {}
-
-    static void * memdup (const void *m, size_t size) {
-        if (size == 0) return nullptr;
-        return memcpy (malloc(size), m, size);
-    }
-
-    size_t list_size(size_t list_no) const override {
-        return il->list_size (list_no);
-    }
-
-    const uint8_t * get_codes (size_t list_no) const override {
-        return (uint8_t*)memdup (il->get_codes(list_no),
-                                 list_size(list_no) * code_size);
-    }
-
-    const idx_t * get_ids (size_t list_no) const override {
-        return (idx_t*)memdup (il->get_ids(list_no),
-                               list_size(list_no) * sizeof(idx_t));
-    }
-
-    void release_codes (size_t, const uint8_t *codes) const override {
-        free ((void*)codes);
-    }
-
-    void release_ids (size_t, const idx_t *ids) const override {
-        free ((void*)ids);
-    }
-
-    const uint8_t * get_single_code (size_t list_no, size_t offset)
-        const override {
-        return (uint8_t*)memdup (il->get_single_code(list_no, offset),
-                                 code_size);
-    }
-
-    size_t add_entries(size_t, size_t, const idx_t*, const uint8_t*) override {
-      assert(!"not implemented");
-      return 0;
-    }
-
-    void update_entries(size_t, size_t, size_t, const idx_t*, const uint8_t*)
-        override {
-      assert(!"not implemented");
-    }
-
-    void resize(size_t, size_t) override {
-      assert(!"not implemented");
-    }
-
-    ~EncapsulateInvertedLists() override {}
-};
-
-
-
-int test_dealloc_invlists (const char *index_key) {
-
-    std::unique_ptr<Index> index = make_trained_index(index_key);
-    IndexIVF * index_ivf = ivflib::extract_index_ivf (index.get());
-
-    auto xb = make_data (nb * d);
-    index->add(nb, xb.data());
-
-    auto xq = make_data (nq * d);
-
-    auto ref_res = search_index (index.get(), xq.data());
-
-    EncapsulateInvertedLists eil(index_ivf->invlists);
-
-    index_ivf->own_invlists = false;
-    index_ivf->replace_invlists (&eil, false);
-
-    // TEST: this could crash or leak mem
-    auto new_res = search_index (index.get(), xq.data());
-
-    // delete explicitly
-    delete eil.il;
-
-    // just to make sure
-    EXPECT_EQ (ref_res, new_res);
-    return 0;
-}
-
-} // anonymous namespace
-
-
-
-/*************************************************************
- * Test entry points
- *************************************************************/
-
-TEST(TestIvlistDealloc, IVFFlat) {
-    test_dealloc_invlists ("IVF32,Flat");
-}
-
-TEST(TestIvlistDealloc, IVFSQ) {
-    test_dealloc_invlists ("IVF32,SQ8");
-}
-
-TEST(TestIvlistDealloc, IVFPQ) {
-    test_dealloc_invlists ("IVF32,PQ4np");
-}

data/vendor/faiss/tests/test_ivfpq_codec.cpp

@@ -1,70 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <gtest/gtest.h>
-
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/utils/utils.h>
-#include <faiss/utils/distances.h>
-
-
-namespace {
-
-// dimension of the vectors to index
-int d = 64;
-
-// size of the database we plan to index
-size_t nb = 8000;
-
-
-double eval_codec_error (long ncentroids, long m, const std::vector<float> &v)
-{
-    faiss::IndexFlatL2 coarse_quantizer (d);
-    faiss::IndexIVFPQ index (&coarse_quantizer, d,
-                             ncentroids, m, 8);
-    index.pq.cp.niter = 10; // speed up train
-    index.train (nb, v.data());
-
-    // encode and decode to compute reconstruction error
-
-    std::vector<faiss::Index::idx_t> keys (nb);
-    std::vector<uint8_t> codes (nb * m);
-    index.encode_multiple (nb, keys.data(), v.data(), codes.data(), true);
-
-    std::vector<float> v2 (nb * d);
-    index.decode_multiple (nb, keys.data(), codes.data(), v2.data());
-
-    return faiss::fvec_L2sqr (v.data(), v2.data(), nb * d);
-}
-
-}  // namespace
-
-
-TEST(IVFPQ, codec) {
-
-    std::vector <float> database (nb * d);
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-    for (size_t i = 0; i < nb * d; i++) {
-        database[i] = distrib(rng);
-    }
-
-    double err0 = eval_codec_error(16, 8, database);
-
-    // should be more accurate as there are more coarse centroids
-    double err1 = eval_codec_error(128, 8, database);
-    EXPECT_GT(err0, err1);
-
-    // should be more accurate as there are more PQ codes
-    double err2 = eval_codec_error(16, 16, database);
-    EXPECT_GT(err0, err2);
-}

data/vendor/faiss/tests/test_ivfpq_indexing.cpp

@@ -1,100 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <gtest/gtest.h>
-
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/index_io.h>
-
-TEST(IVFPQ, accuracy) {
-
-    // dimension of the vectors to index
-    int d = 64;
-
-    // size of the database we plan to index
-    size_t nb = 1000;
-
-    // make a set of nt training vectors in the unit cube
-    // (could be the database)
-    size_t nt = 1500;
-
-    // make the index object and train it
-    faiss::IndexFlatL2 coarse_quantizer (d);
-
-    // a reasonable number of cetroids to index nb vectors
-    int ncentroids = 25;
-
-    faiss::IndexIVFPQ index (&coarse_quantizer, d,
-                             ncentroids, 16, 8);
-
-    // index that gives the ground-truth
-    faiss::IndexFlatL2 index_gt (d);
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-
-    { // training
-
-        std::vector <float> trainvecs (nt * d);
-        for (size_t i = 0; i < nt * d; i++) {
-            trainvecs[i] = distrib(rng);
-        }
-        index.verbose = true;
-        index.train (nt, trainvecs.data());
-    }
-
-    { // populating the database
-
-        std::vector <float> database (nb * d);
-        for (size_t i = 0; i < nb * d; i++) {
-            database[i] = distrib(rng);
-        }
-
-        index.add (nb, database.data());
-        index_gt.add (nb, database.data());
-    }
-
-    int nq = 200;
-    int n_ok;
-
-    { // searching the database
-
-        std::vector <float> queries (nq * d);
-        for (size_t i = 0; i < nq * d; i++) {
-            queries[i] = distrib(rng);
-        }
-
-        std::vector<faiss::Index::idx_t> gt_nns (nq);
-        std::vector<float>               gt_dis (nq);
-
-        index_gt.search (nq, queries.data(), 1,
-                         gt_dis.data(), gt_nns.data());
-
-        index.nprobe = 5;
-        int k = 5;
-        std::vector<faiss::Index::idx_t> nns (k * nq);
-        std::vector<float>               dis (k * nq);
-
-        index.search (nq, queries.data(), k, dis.data(), nns.data());
-
-        n_ok = 0;
-        for (int q = 0; q < nq; q++) {
-
-            for (int i = 0; i < k; i++)
-                if (nns[q * k + i] == gt_nns[q])
-                    n_ok++;
-        }
-        EXPECT_GT(n_ok, nq * 0.4);
-    }
-
-}