faiss 0.1.3 → 0.2.0

data/vendor/faiss/c_api/index_io_c.cpp

@@ -1,42 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-//  Copyright 2004-present Facebook. All Rights Reserved
-// -*- c++ -*-
-// I/O code for indexes
-
-#include "index_io_c.h"
-#include "index_io.h"
-#include "macros_impl.h"
-
-using faiss::Index;
-
-int faiss_write_index(const FaissIndex *idx, FILE *f) {
-    try {
-        faiss::write_index(reinterpret_cast<const Index*>(idx), f);
-    } CATCH_AND_HANDLE
-}
-
-int faiss_write_index_fname(const FaissIndex *idx, const char *fname) {
-    try {
-        faiss::write_index(reinterpret_cast<const Index*>(idx), fname);
-    } CATCH_AND_HANDLE
-}
-
-int faiss_read_index(FILE *f, int io_flags, FaissIndex **p_out) {
-    try {
-        auto out = faiss::read_index(f, io_flags);
-        *p_out = reinterpret_cast<FaissIndex*>(out);
-    } CATCH_AND_HANDLE
-}
-
-int faiss_read_index_fname(const char *fname, int io_flags, FaissIndex **p_out) {
-    try {
-        auto out = faiss::read_index(fname, io_flags);
-        *p_out = reinterpret_cast<FaissIndex*>(out);
-    } CATCH_AND_HANDLE
-}

data/vendor/faiss/c_api/index_io_c.h

@@ -1,50 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-//  Copyright 2004-present Facebook. All Rights Reserved
-// -*- c++ -*-
-// I/O code for indexes
-
-
-#ifndef FAISS_INDEX_IO_C_H
-#define FAISS_INDEX_IO_C_H
-
-#include <stdio.h>
-#include "faiss_c.h"
-#include "Index_c.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/** Write index to a file.
- * This is equivalent to `faiss::write_index` when a file descriptor is provided.
- */
-int faiss_write_index(const FaissIndex *idx, FILE *f);
-
-/** Write index to a file.
- * This is equivalent to `faiss::write_index` when a file path is provided.
- */
-int faiss_write_index_fname(const FaissIndex *idx, const char *fname);
-
-#define FAISS_IO_FLAG_MMAP 1
-#define FAISS_IO_FLAG_READ_ONLY 2
-
-/** Read index from a file.
- * This is equivalent to `faiss:read_index` when a file descriptor is given.
- */
-int faiss_read_index(FILE *f, int io_flags, FaissIndex **p_out);
-
-/** Read index from a file.
- * This is equivalent to `faiss:read_index` when a file path is given.
- */
-int faiss_read_index_fname(const char *fname, int io_flags, FaissIndex **p_out);
-
-#ifdef __cplusplus
-}
-#endif
-#endif

data/vendor/faiss/c_api/macros_impl.h

@@ -1,110 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-// Copyright 2004-present Facebook. All Rights Reserved.
-// -*- c++ -*-
-
-/// Utility macros for the C wrapper implementation.
-
-#ifndef MACROS_IMPL_H
-#define MACROS_IMPL_H
-
-#include "faiss_c.h"
-#include "FaissException.h"
-#include "error_impl.h"
-#include <stdexcept>
-#include <iostream>
-
-#ifdef NDEBUG
-#define CATCH_AND_HANDLE                              \
-    catch (faiss::FaissException& e) {                \
-        faiss_last_exception =                        \
-            std::make_exception_ptr(e);               \
-        return -2;                                    \
-    } catch (std::exception& e) {                     \
-        faiss_last_exception =                        \
-            std::make_exception_ptr(e);               \
-        return -4;                                    \
-    } catch (...) {                                   \
-        faiss_last_exception =                        \
-            std::make_exception_ptr(                  \
-                std::runtime_error("Unknown error")); \
-        return -1;                                    \
-    } return 0;
-#else
-#define CATCH_AND_HANDLE                              \
-    catch (faiss::FaissException& e) {                \
-        std::cerr << e.what() << '\n';                \
-        faiss_last_exception =                        \
-            std::make_exception_ptr(e);               \
-        return -2;                                    \
-    } catch (std::exception& e) {                     \
-        std::cerr << e.what() << '\n';                \
-        faiss_last_exception =                        \
-            std::make_exception_ptr(e);               \
-        return -4;                                    \
-    } catch (...) {                                   \
-        std::cerr << "Unrecognized exception!\n";     \
-        faiss_last_exception =                        \
-            std::make_exception_ptr(                  \
-                std::runtime_error("Unknown error")); \
-        return -1;                                    \
-    } return 0;
-#endif
-
-#define DEFINE_GETTER(clazz, ty, name)                            \
-    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) { \
-        return static_cast< ty >(                                 \
-            reinterpret_cast< const faiss::clazz *>(obj)-> name   \
-        );                                                        \
-    }
-
-#define DEFINE_GETTER_SUBCLASS(clazz, parent, ty, name)                \
-    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) {      \
-        return static_cast< ty >(                                      \
-            reinterpret_cast<const faiss::parent::clazz *>(obj)-> name \
-        );                                                             \
-    }
-
-#define DEFINE_GETTER_PERMISSIVE(clazz, ty, name)                 \
-    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) { \
-        return ( ty ) (                                           \
-            reinterpret_cast<const faiss::clazz *>(obj)-> name    \
-        );                                                        \
-    }
-
-#define DEFINE_GETTER_SUBCLASS_PERMISSIVE(clazz, parent, ty, name)     \
-    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) {      \
-        return ( ty ) (                                                \
-            reinterpret_cast<const faiss::parent::clazz *>(obj)-> name \
-        );                                                             \
-    }
-
-#define DEFINE_SETTER(clazz, ty, name)                                    \
-    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *obj, ty val) { \
-        reinterpret_cast< faiss::clazz *>(obj)-> name = val;              \
-    }
-
-#define DEFINE_SETTER_STATIC(clazz, ty_to, ty_from, name)                      \
-    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *obj, ty_from val) { \
-        reinterpret_cast< faiss::clazz *>(obj)-> name =                        \
-            static_cast< ty_to >(val);                                         \
-    }
-
-#define DEFINE_DESTRUCTOR(clazz)                           \
-    void faiss_ ## clazz ## _free (Faiss ## clazz *obj) {  \
-        delete reinterpret_cast<faiss::clazz *>(obj);      \
-    }
-
-#define DEFINE_INDEX_DOWNCAST(clazz)                                \
-    Faiss ## clazz * faiss_ ## clazz ## _cast (FaissIndex* index) { \
-        return reinterpret_cast<Faiss ## clazz *>(                  \
-            dynamic_cast< faiss::clazz *>(                          \
-                reinterpret_cast<faiss::Index*>(index)));           \
-    }
-
-#endif

data/vendor/faiss/demos/demo_imi_flat.cpp

@@ -1,154 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <sys/time.h>
-
-
-#include <faiss/IndexPQ.h>
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/index_io.h>
-
-double elapsed ()
-{
-    struct timeval tv;
-    gettimeofday (&tv, nullptr);
-    return  tv.tv_sec + tv.tv_usec * 1e-6;
-}
-
-
-int main ()
-{
-    double t0 = elapsed();
-
-    // dimension of the vectors to index
-    int d = 128;
-
-    // size of the database we plan to index
-    size_t nb = 1000 * 1000;
-
-    // make a set of nt training vectors in the unit cube
-    // (could be the database)
-    size_t nt = 100 * 1000;
-
-    //---------------------------------------------------------------
-    // Define the core quantizer
-    // We choose a multiple inverted index for faster training with less data
-    // and because it usually offers best accuracy/speed trade-offs
-    //
-    // We here assume that its lifespan of this coarse quantizer will cover the
-    // lifespan of the inverted-file quantizer IndexIVFFlat below
-    // With dynamic allocation, one may give the responsability to free the
-    // quantizer to the inverted-file index (with attribute do_delete_quantizer)
-    //
-    // Note: a regular clustering algorithm would be defined as:
-    //       faiss::IndexFlatL2 coarse_quantizer (d);
-    //
-    // Use nhash=2 subquantizers used to define the product coarse quantizer
-    // Number of bits: we will have 2^nbits_coarse centroids per subquantizer
-    //                 meaning (2^12)^nhash distinct inverted lists
-    size_t nhash = 2;
-    size_t nbits_subq = int (log2 (nb+1) / 2);        // good choice in general
-    size_t ncentroids = 1 << (nhash * nbits_subq);  // total # of centroids
-
-    faiss::MultiIndexQuantizer coarse_quantizer (d, nhash, nbits_subq);
-
-    printf ("IMI (%ld,%ld): %ld virtual centroids (target: %ld base vectors)",
-            nhash, nbits_subq, ncentroids, nb);
-
-    // the coarse quantizer should not be dealloced before the index
-    // 4 = nb of bytes per code (d must be a multiple of this)
-    // 8 = nb of bits per sub-code (almost always 8)
-    faiss::MetricType metric = faiss::METRIC_L2; // can be METRIC_INNER_PRODUCT
-    faiss::IndexIVFFlat index (&coarse_quantizer, d, ncentroids, metric);
-    index.quantizer_trains_alone = true;
-
-    // define the number of probes. 2048 is for high-dim, overkilled in practice
-    // Use 4-1024 depending on the trade-off speed accuracy that you want
-    index.nprobe = 2048;
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-
-    { // training
-        printf ("[%.3f s] Generating %ld vectors in %dD for training\n",
-                elapsed() - t0, nt, d);
-
-        std::vector <float> trainvecs (nt * d);
-        for (size_t i = 0; i < nt * d; i++) {
-            trainvecs[i] = distrib(rng);
-        }
-
-        printf ("[%.3f s] Training the index\n", elapsed() - t0);
-        index.verbose = true;
-        index.train (nt, trainvecs.data());
-    }
-
-    size_t nq;
-    std::vector<float> queries;
-
-    { // populating the database
-        printf ("[%.3f s] Building a dataset of %ld vectors to index\n",
-                elapsed() - t0, nb);
-
-        std::vector <float> database (nb * d);
-        for (size_t i = 0; i < nb * d; i++) {
-            database[i] = distrib(rng);
-        }
-
-        printf ("[%.3f s] Adding the vectors to the index\n", elapsed() - t0);
-
-        index.add (nb, database.data());
-
-        // remember a few elements from the database as queries
-        int i0 = 1234;
-        int i1 = 1244;
-
-        nq = i1 - i0;
-        queries.resize (nq * d);
-        for (int i = i0; i < i1; i++) {
-            for (int j = 0; j < d; j++) {
-                queries [(i - i0) * d  + j]  = database [i * d + j];
-            }
-        }
-    }
-
-    { // searching the database
-        int k = 5;
-        printf ("[%.3f s] Searching the %d nearest neighbors "
-                "of %ld vectors in the index\n",
-                elapsed() - t0, k, nq);
-
-        std::vector<faiss::Index::idx_t> nns (k * nq);
-        std::vector<float>               dis (k * nq);
-
-        index.search (nq, queries.data(), k, dis.data(), nns.data());
-
-        printf ("[%.3f s] Query results (vector ids, then distances):\n",
-                elapsed() - t0);
-
-        for (int i = 0; i < nq; i++) {
-            printf ("query %2d: ", i);
-            for (int j = 0; j < k; j++) {
-                printf ("%7ld ", nns[j + i * k]);
-            }
-            printf ("\n     dis: ");
-            for (int j = 0; j < k; j++) {
-                printf ("%7g ", dis[j + i * k]);
-            }
-            printf ("\n");
-        }
-    }
-    return 0;
-}

data/vendor/faiss/demos/demo_imi_pq.cpp

@@ -1,203 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <random>
-
-#include <sys/time.h>
-
-
-#include <faiss/IndexPQ.h>
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexFlat.h>
-#include <faiss/index_io.h>
-
-double elapsed ()
-{
-    struct timeval tv;
-    gettimeofday (&tv, nullptr);
-    return  tv.tv_sec + tv.tv_usec * 1e-6;
-}
-
-
-int main ()
-{
-    double t0 = elapsed();
-
-    // dimension of the vectors to index
-    int d = 64;
-
-    // size of the database we plan to index
-    size_t nb = 1000 * 1000;
-    size_t add_bs = 10000; // # size of the blocks to add
-
-    // make a set of nt training vectors in the unit cube
-    // (could be the database)
-    size_t nt = 100 * 1000;
-
-    //---------------------------------------------------------------
-    // Define the core quantizer
-    // We choose a multiple inverted index for faster training with less data
-    // and because it usually offers best accuracy/speed trade-offs
-    //
-    // We here assume that its lifespan of this coarse quantizer will cover the
-    // lifespan of the inverted-file quantizer IndexIVFFlat below
-    // With dynamic allocation, one may give the responsability to free the
-    // quantizer to the inverted-file index (with attribute do_delete_quantizer)
-    //
-    // Note: a regular clustering algorithm would be defined as:
-    //       faiss::IndexFlatL2 coarse_quantizer (d);
-    //
-    // Use nhash=2 subquantizers used to define the product coarse quantizer
-    // Number of bits: we will have 2^nbits_coarse centroids per subquantizer
-    //                 meaning (2^12)^nhash distinct inverted lists
-    //
-    // The parameter bytes_per_code is determined by the memory
-    // constraint, the dataset will use nb * (bytes_per_code + 8)
-    // bytes.
-    //
-    // The parameter nbits_subq is determined by the size of the dataset to index.
-    //
-    size_t nhash = 2;
-    size_t nbits_subq = 9;
-    size_t ncentroids = 1 << (nhash * nbits_subq);  // total # of centroids
-    int bytes_per_code = 16;
-
-    faiss::MultiIndexQuantizer coarse_quantizer (d, nhash, nbits_subq);
-
-    printf ("IMI (%ld,%ld): %ld virtual centroids (target: %ld base vectors)",
-            nhash, nbits_subq, ncentroids, nb);
-
-    // the coarse quantizer should not be dealloced before the index
-    // 4 = nb of bytes per code (d must be a multiple of this)
-    // 8 = nb of bits per sub-code (almost always 8)
-    faiss::MetricType metric = faiss::METRIC_L2; // can be METRIC_INNER_PRODUCT
-    faiss::IndexIVFPQ index (&coarse_quantizer, d, ncentroids, bytes_per_code, 8);
-    index.quantizer_trains_alone = true;
-
-    // define the number of probes. 2048 is for high-dim, overkill in practice
-    // Use 4-1024 depending on the trade-off speed accuracy that you want
-    index.nprobe = 2048;
-
-
-    std::mt19937 rng;
-    std::uniform_real_distribution<> distrib;
-
-    { // training.
-
-        // The distribution of the training vectors should be the same
-        // as the database vectors. It could be a sub-sample of the
-        // database vectors, if sampling is not biased. Here we just
-        // randomly generate the vectors.
-
-        printf ("[%.3f s] Generating %ld vectors in %dD for training\n",
-                elapsed() - t0, nt, d);
-
-        std::vector <float> trainvecs (nt * d);
-        for (size_t i = 0; i < nt; i++) {
-            for (size_t j = 0; j < d; j++) {
-                trainvecs[i * d + j] = distrib(rng);
-            }
-        }
-
-        printf ("[%.3f s] Training the index\n", elapsed() - t0);
-        index.verbose = true;
-        index.train (nt, trainvecs.data());
-    }
-
-    // the index can be re-loaded later with
-    // faiss::Index * idx = faiss::read_index("/tmp/trained_index.faissindex");
-    faiss::write_index(&index, "/tmp/trained_index.faissindex");
-
-    size_t nq;
-    std::vector<float> queries;
-
-    { // populating the database
-        printf ("[%.3f s] Building a dataset of %ld vectors to index\n",
-                elapsed() - t0, nb);
-
-        std::vector <float> database (nb * d);
-        std::vector <long> ids (nb);
-        for (size_t i = 0; i < nb; i++) {
-            for (size_t j = 0; j < d; j++) {
-                database[i * d + j] = distrib(rng);
-            }
-            ids[i] = 8760000000L + i;
-        }
-
-        printf ("[%.3f s] Adding the vectors to the index\n", elapsed() - t0);
-
-        for (size_t begin = 0; begin < nb; begin += add_bs) {
-            size_t end = std::min (begin + add_bs, nb);
-            index.add_with_ids (end - begin,
-                                database.data() + d * begin,
-                                ids.data() + begin);
-        }
-
-        // remember a few elements from the database as queries
-        int i0 = 1234;
-        int i1 = 1244;
-
-        nq = i1 - i0;
-        queries.resize (nq * d);
-        for (int i = i0; i < i1; i++) {
-            for (int j = 0; j < d; j++) {
-                queries [(i - i0) * d  + j] = database [i * d + j];
-            }
-        }
-    }
-
-    // A few notes on the internal format of the index:
-    //
-    // - the positing lists for PQ codes are index.codes, which is a
-    //    std::vector < std::vector<uint8_t> >
-    //   if n is the length of posting list #i, codes[i] has length bytes_per_code * n
-    //
-    // - the corresponding ids are stored in index.ids
-    //
-    // - given a vector float *x, finding which k centroids are
-    //   closest to it (ie to find the nearest neighbors) can be done with
-    //
-    //   long *centroid_ids = new long[k];
-    //   float *distances = new float[k];
-    //   index.quantizer->search (1, x, k, dis, centroids_ids);
-    //
-
-    faiss::write_index(&index, "/tmp/populated_index.faissindex");
-
-    { // searching the database
-        int k = 5;
-        printf ("[%.3f s] Searching the %d nearest neighbors "
-                "of %ld vectors in the index\n",
-                elapsed() - t0, k, nq);
-
-        std::vector<faiss::Index::idx_t> nns (k * nq);
-        std::vector<float>               dis (k * nq);
-
-        index.search (nq, queries.data(), k, dis.data(), nns.data());
-
-        printf ("[%.3f s] Query results (vector ids, then distances):\n",
-                elapsed() - t0);
-
-        for (int i = 0; i < nq; i++) {
-            printf ("query %2d: ", i);
-            for (int j = 0; j < k; j++) {
-                printf ("%7ld ", nns[j + i * k]);
-            }
-            printf ("\n     dis: ");
-            for (int j = 0; j < k; j++) {
-                printf ("%7g ", dis[j + i * k]);
-            }
-            printf ("\n");
-        }
-    }
-    return 0;
-}