faiss 0.1.0 → 0.1.1

data/vendor/faiss/index_factory.h

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// -*- c++ -*-
+
+#pragma once
+
+#include <faiss/Index.h>
+#include <faiss/IndexBinary.h>
+
+namespace faiss {
+
+/** Build and index with the sequence of processing steps described in
+ *  the string. */
+Index *index_factory (int d, const char *description,
+                      MetricType metric = METRIC_L2);
+
+IndexBinary *index_binary_factory (int d, const char *description);
+
+
+}

data/vendor/faiss/index_io.h

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// -*- c++ -*-
+
+// I/O code for indexes
+
+#ifndef FAISS_INDEX_IO_H
+#define FAISS_INDEX_IO_H
+
+
+#include <cstdio>
+
+/** I/O functions can read/write to a filename, a file handle or to an
+ * object that abstracts the medium.
+ *
+ * The read functions return objects that should be deallocated with
+ * delete. All references within these objectes are owned by the
+ * object.
+ */
+
+namespace faiss {
+
+struct Index;
+struct IndexBinary;
+struct VectorTransform;
+struct ProductQuantizer;
+struct IOReader;
+struct IOWriter;
+struct InvertedLists;
+
+void write_index (const Index *idx, const char *fname);
+void write_index (const Index *idx, FILE *f);
+void write_index (const Index *idx, IOWriter *writer);
+
+void write_index_binary (const IndexBinary *idx, const char *fname);
+void write_index_binary (const IndexBinary *idx, FILE *f);
+void write_index_binary (const IndexBinary *idx, IOWriter *writer);
+
+// The read_index flags are implemented only for a subset of index types.
+const int IO_FLAG_MMAP = 1; // try to memmap if possible
+const int IO_FLAG_READ_ONLY = 2;
+// strip directory component from ondisk filename, and assume it's in
+// the same directory as the index file
+const int IO_FLAG_ONDISK_SAME_DIR = 4;
+
+Index *read_index (const char *fname, int io_flags = 0);
+Index *read_index (FILE * f, int io_flags = 0);
+Index *read_index (IOReader *reader, int io_flags = 0);
+
+IndexBinary *read_index_binary (const char *fname, int io_flags = 0);
+IndexBinary *read_index_binary (FILE * f, int io_flags = 0);
+IndexBinary *read_index_binary (IOReader *reader, int io_flags = 0);
+
+void write_VectorTransform (const VectorTransform *vt, const char *fname);
+VectorTransform *read_VectorTransform (const char *fname);
+
+ProductQuantizer * read_ProductQuantizer (const char*fname);
+ProductQuantizer * read_ProductQuantizer (IOReader *reader);
+
+void write_ProductQuantizer (const ProductQuantizer*pq, const char *fname);
+void write_ProductQuantizer (const ProductQuantizer*pq, IOWriter *f);
+
+void write_InvertedLists (const InvertedLists *ils, IOWriter *f);
+InvertedLists *read_InvertedLists (IOReader *reader, int io_flags = 0);
+
+
+} // namespace faiss
+
+
+#endif

data/vendor/faiss/misc/test_blas.cpp

@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#undef FINTEGER
+#define FINTEGER long
+
+
+extern "C" {
+
+/* declare BLAS functions, see http://www.netlib.org/clapack/cblas/ */
+
+int sgemm_ (const char *transa, const char *transb, FINTEGER *m, FINTEGER *
+            n, FINTEGER *k, const float *alpha, const float *a,
+            FINTEGER *lda, const float *b, FINTEGER *
+            ldb, float *beta, float *c, FINTEGER *ldc);
+
+/* Lapack functions, see http://www.netlib.org/clapack/old/single/sgeqrf.c */
+
+int sgeqrf_ (FINTEGER *m, FINTEGER *n, float *a, FINTEGER *lda,
+                 float *tau, float *work, FINTEGER *lwork, FINTEGER *info);
+
+}
+
+float *new_random_vec(int size)
+{
+    float *x = new float[size];
+    for (int i = 0; i < size; i++)
+        x[i] = drand48();
+    return x;
+}
+
+
+int main() {
+
+    FINTEGER m = 10, n = 20, k = 30;
+    float *a = new_random_vec(m * k), *b = new_random_vec(n * k), *c = new float[n * m];
+    float one = 1.0, zero = 0.0;
+
+    printf("BLAS test\n");
+
+    sgemm_("Not transposed", "Not transposed",
+           &m, &n, &k, &one, a, &m, b, &k, &zero, c, &m);
+
+    printf("errors=\n");
+
+    for (int i = 0; i < m; i++) {
+        for (int j = 0; j < n; j++) {
+            float accu = 0;
+            for (int l = 0; l < k; l++)
+                accu += a[i + l * m] * b[l + j * k];
+            printf ("%6.3f ", accu - c[i + j * m]);
+        }
+        printf("\n");
+    }
+
+    long info = 0x64bL << 32;
+    long mi = 0x64bL << 32 | m;
+    float *tau = new float[m];
+    FINTEGER lwork = -1;
+
+    float work1;
+
+    printf("Intentional Lapack error (appears only for 64-bit INTEGER):\n");
+    sgeqrf_ (&mi, &n, c, &m, tau, &work1, &lwork, (FINTEGER*)&info);
+
+    // sgeqrf_ (&m, &n, c, &zeroi, tau, &work1, &lwork, (FINTEGER*)&info);
+    printf("info=%016lx\n", info);
+
+    if(info >> 32 == 0x64b) {
+        printf("Lapack uses 32-bit integers\n");
+    } else {
+        printf("Lapack uses 64-bit integers\n");
+    }
+
+
+    return 0;
+}

data/vendor/faiss/tests/test_binary_flat.cpp

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <gtest/gtest.h>
+
+#include <faiss/IndexBinaryFlat.h>
+#include <faiss/utils/hamming.h>
+
+TEST(BinaryFlat, accuracy) {
+  // dimension of the vectors to index
+  int d = 64;
+
+  // size of the database we plan to index
+  size_t nb = 1000;
+
+  // make the index object and train it
+  faiss::IndexBinaryFlat index(d);
+
+  srand(35);
+
+  std::vector<uint8_t> database(nb * (d / 8));
+  for (size_t i = 0; i < nb * (d / 8); i++) {
+    database[i] = rand() % 0x100;
+  }
+
+  { // populating the database
+    index.add(nb, database.data());
+  }
+
+  size_t nq = 200;
+
+  { // searching the database
+
+    std::vector<uint8_t> queries(nq * (d / 8));
+    for (size_t i = 0; i < nq * (d / 8); i++) {
+      queries[i] = rand() % 0x100;
+    }
+
+    int k = 5;
+    std::vector<faiss::IndexBinary::idx_t> nns(k * nq);
+    std::vector<int>                     dis(k * nq);
+
+    index.search(nq, queries.data(), k, dis.data(), nns.data());
+
+    for (size_t i = 0; i < nq; ++i) {
+      faiss::HammingComputer8 hc(queries.data() + i * (d / 8), d / 8);
+      hamdis_t dist_min = hc.hamming(database.data());
+      for (size_t j = 1; j < nb; ++j) {
+        hamdis_t dist = hc.hamming(database.data() + j * (d / 8));
+        if (dist < dist_min) {
+          dist_min = dist;
+        }
+      }
+      EXPECT_EQ(dist_min, dis[k * i]);
+    }
+  }
+}

data/vendor/faiss/tests/test_dealloc_invlists.cpp

@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <memory>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include <faiss/IndexIVF.h>
+#include <faiss/index_factory.h>
+#include <faiss/AutoTune.h>
+#include <faiss/index_io.h>
+#include <faiss/IVFlib.h>
+
+using namespace faiss;
+
+namespace {
+
+typedef Index::idx_t idx_t;
+
+
+// dimension of the vectors to index
+int d = 32;
+
+// nb of training vectors
+size_t nt = 5000;
+
+// size of the database points per window step
+size_t nb = 1000;
+
+// nb of queries
+size_t nq = 200;
+
+
+std::vector<float> make_data(size_t n)
+{
+    std::vector <float> database (n * d);
+    for (size_t i = 0; i < n * d; i++) {
+        database[i] = drand48();
+    }
+    return database;
+}
+
+std::unique_ptr<Index> make_trained_index(const char *index_type)
+{
+    auto index = std::unique_ptr<Index>(index_factory(d, index_type));
+    auto xt = make_data(nt * d);
+    index->train(nt, xt.data());
+    ParameterSpace().set_index_parameter (index.get(), "nprobe", 4);
+    return index;
+}
+
+std::vector<idx_t> search_index(Index *index, const float *xq) {
+    int k = 10;
+    std::vector<idx_t> I(k * nq);
+    std::vector<float> D(k * nq);
+    index->search (nq, xq, k, D.data(), I.data());
+    return I;
+}
+
+
+
+
+
+/*************************************************************
+ * Test functions for a given index type
+ *************************************************************/
+
+struct EncapsulateInvertedLists: InvertedLists {
+
+    const InvertedLists *il;
+
+    EncapsulateInvertedLists(const InvertedLists *il):
+        InvertedLists(il->nlist, il->code_size),
+        il(il)
+    {}
+
+    static void * memdup (const void *m, size_t size) {
+        if (size == 0) return nullptr;
+        return memcpy (malloc(size), m, size);
+    }
+
+    size_t list_size(size_t list_no) const override {
+        return il->list_size (list_no);
+    }
+
+    const uint8_t * get_codes (size_t list_no) const override {
+        return (uint8_t*)memdup (il->get_codes(list_no),
+                                 list_size(list_no) * code_size);
+    }
+
+    const idx_t * get_ids (size_t list_no) const override {
+        return (idx_t*)memdup (il->get_ids(list_no),
+                               list_size(list_no) * sizeof(idx_t));
+    }
+
+    void release_codes (size_t, const uint8_t *codes) const override {
+        free ((void*)codes);
+    }
+
+    void release_ids (size_t, const idx_t *ids) const override {
+        free ((void*)ids);
+    }
+
+    const uint8_t * get_single_code (size_t list_no, size_t offset)
+        const override {
+        return (uint8_t*)memdup (il->get_single_code(list_no, offset),
+                                 code_size);
+    }
+
+    size_t add_entries(size_t, size_t, const idx_t*, const uint8_t*) override {
+      assert(!"not implemented");
+      return 0;
+    }
+
+    void update_entries(size_t, size_t, size_t, const idx_t*, const uint8_t*)
+        override {
+      assert(!"not implemented");
+    }
+
+    void resize(size_t, size_t) override {
+      assert(!"not implemented");
+    }
+
+    ~EncapsulateInvertedLists() override {}
+};
+
+
+
+int test_dealloc_invlists (const char *index_key) {
+
+    std::unique_ptr<Index> index = make_trained_index(index_key);
+    IndexIVF * index_ivf = ivflib::extract_index_ivf (index.get());
+
+    auto xb = make_data (nb * d);
+    index->add(nb, xb.data());
+
+    auto xq = make_data (nq * d);
+
+    auto ref_res = search_index (index.get(), xq.data());
+
+    EncapsulateInvertedLists eil(index_ivf->invlists);
+
+    index_ivf->own_invlists = false;
+    index_ivf->replace_invlists (&eil, false);
+
+    // TEST: this could crash or leak mem
+    auto new_res = search_index (index.get(), xq.data());
+
+    // delete explicitly
+    delete eil.il;
+
+    // just to make sure
+    EXPECT_EQ (ref_res, new_res);
+    return 0;
+}
+
+} // anonymous namespace
+
+
+
+/*************************************************************
+ * Test entry points
+ *************************************************************/
+
+TEST(TestIvlistDealloc, IVFFlat) {
+    test_dealloc_invlists ("IVF32,Flat");
+}
+
+TEST(TestIvlistDealloc, IVFSQ) {
+    test_dealloc_invlists ("IVF32,SQ8");
+}
+
+TEST(TestIvlistDealloc, IVFPQ) {
+    test_dealloc_invlists ("IVF32,PQ4np");
+}

data/vendor/faiss/tests/test_ivfpq_codec.cpp

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <gtest/gtest.h>
+
+#include <faiss/IndexIVFPQ.h>
+#include <faiss/IndexFlat.h>
+#include <faiss/utils/utils.h>
+#include <faiss/utils/distances.h>
+
+
+namespace {
+
+// dimension of the vectors to index
+int d = 64;
+
+// size of the database we plan to index
+size_t nb = 8000;
+
+
+double eval_codec_error (long ncentroids, long m, const std::vector<float> &v)
+{
+    faiss::IndexFlatL2 coarse_quantizer (d);
+    faiss::IndexIVFPQ index (&coarse_quantizer, d,
+                             ncentroids, m, 8);
+    index.pq.cp.niter = 10; // speed up train
+    index.train (nb, v.data());
+
+    // encode and decode to compute reconstruction error
+
+    std::vector<faiss::Index::idx_t> keys (nb);
+    std::vector<uint8_t> codes (nb * m);
+    index.encode_multiple (nb, keys.data(), v.data(), codes.data(), true);
+
+    std::vector<float> v2 (nb * d);
+    index.decode_multiple (nb, keys.data(), codes.data(), v2.data());
+
+    return faiss::fvec_L2sqr (v.data(), v2.data(), nb * d);
+}
+
+}  // namespace
+
+
+TEST(IVFPQ, codec) {
+
+    std::vector <float> database (nb * d);
+    for (size_t i = 0; i < nb * d; i++) {
+        database[i] = drand48();
+    }
+
+    double err0 = eval_codec_error(16, 8, database);
+
+    // should be more accurate as there are more coarse centroids
+    double err1 = eval_codec_error(128, 8, database);
+    EXPECT_GT(err0, err1);
+
+    // should be more accurate as there are more PQ codes
+    double err2 = eval_codec_error(16, 16, database);
+    EXPECT_GT(err0, err2);
+}