faiss 0.1.0 → 0.1.1

data/vendor/faiss/c_api/impl/AuxIndexStructures_c.h

@@ -0,0 +1,149 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_AUX_INDEX_STRUCTURES_C_H
+#define FAISS_AUX_INDEX_STRUCTURES_C_H
+
+#include "../Index_c.h"
+#include "../faiss_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+FAISS_DECLARE_CLASS(RangeSearchResult)
+
+FAISS_DECLARE_GETTER(RangeSearchResult, size_t, nq)
+
+int faiss_RangeSearchResult_new(FaissRangeSearchResult** p_rsr, idx_t nq);
+
+int faiss_RangeSearchResult_new_with(FaissRangeSearchResult** p_rsr, idx_t nq, int alloc_lims);
+
+/// called when lims contains the nb of elements result entries
+/// for each query
+int faiss_RangeSearchResult_do_allocation(FaissRangeSearchResult* rsr);
+
+FAISS_DECLARE_DESTRUCTOR(RangeSearchResult)
+
+/// getter for buffer_size
+FAISS_DECLARE_GETTER(RangeSearchResult, size_t, buffer_size)
+
+/// getter for lims: size (nq + 1)
+void faiss_RangeSearchResult_lims(
+    FaissRangeSearchResult* rsr, size_t** lims);
+
+/// getter for labels and respective distances (not sorted):
+/// result for query i is labels[lims[i]:lims[i+1]]
+void faiss_RangeSearchResult_labels(
+    FaissRangeSearchResult* rsr, idx_t** labels, float** distances);
+
+
+/** Encapsulates a set of ids to remove. */
+FAISS_DECLARE_CLASS(IDSelector)
+FAISS_DECLARE_DESTRUCTOR(IDSelector)
+
+int faiss_IDSelector_is_member(const FaissIDSelector* sel, idx_t id);
+
+/** remove ids between [imni, imax) */
+FAISS_DECLARE_CLASS(IDSelectorRange)
+FAISS_DECLARE_DESTRUCTOR(IDSelectorRange)
+
+FAISS_DECLARE_GETTER(IDSelectorRange, idx_t, imin)
+FAISS_DECLARE_GETTER(IDSelectorRange, idx_t, imax)
+
+int faiss_IDSelectorRange_new(FaissIDSelectorRange** p_sel, idx_t imin, idx_t imax);
+
+/** Remove ids from a set. Repetitions of ids in the indices set
+ * passed to the constructor does not hurt performance. The hash
+ * function used for the bloom filter and GCC's implementation of
+ * unordered_set are just the least significant bits of the id. This
+ * works fine for random ids or ids in sequences but will produce many
+ * hash collisions if lsb's are always the same */
+FAISS_DECLARE_CLASS(IDSelectorBatch)
+
+FAISS_DECLARE_GETTER(IDSelectorBatch, int, nbits)
+FAISS_DECLARE_GETTER(IDSelectorBatch, idx_t, mask)
+
+int faiss_IDSelectorBatch_new(FaissIDSelectorBatch** p_sel, size_t n, const idx_t* indices);
+
+// Below are structures used only by Index implementations
+
+/** List of temporary buffers used to store results before they are
+ *  copied to the RangeSearchResult object. */
+FAISS_DECLARE_CLASS(BufferList)
+FAISS_DECLARE_DESTRUCTOR(BufferList)
+
+FAISS_DECLARE_GETTER(BufferList, size_t, buffer_size)
+FAISS_DECLARE_GETTER(BufferList, size_t, wp)
+
+typedef struct FaissBuffer {
+    idx_t *ids;
+    float *dis;
+} FaissBuffer;
+
+int faiss_BufferList_append_buffer(FaissBufferList* bl);
+
+int faiss_BufferList_new(FaissBufferList** p_bl, size_t buffer_size);
+
+int faiss_BufferList_add(FaissBufferList* bl, idx_t id, float dis);
+
+/// copy elemnts ofs:ofs+n-1 seen as linear data in the buffers to
+/// tables dest_ids, dest_dis
+int faiss_BufferList_copy_range(
+    FaissBufferList* bl, size_t ofs, size_t n, idx_t *dest_ids, float *dest_dis);
+
+/// the entries in the buffers are split per query
+FAISS_DECLARE_CLASS(RangeSearchPartialResult)
+
+/// result structure for a single query
+FAISS_DECLARE_CLASS(RangeQueryResult)
+FAISS_DECLARE_GETTER(RangeQueryResult, idx_t, qno)
+FAISS_DECLARE_GETTER(RangeQueryResult, size_t, nres)
+FAISS_DECLARE_GETTER(RangeQueryResult, FaissRangeSearchPartialResult*, pres)
+
+int faiss_RangeQueryResult_add(FaissRangeQueryResult* qr, float dis, idx_t id);
+
+
+FAISS_DECLARE_GETTER(RangeSearchPartialResult, FaissRangeSearchResult*, res)
+
+int faiss_RangeSearchPartialResult_new(
+    FaissRangeSearchPartialResult** p_res, FaissRangeSearchResult* res_in);
+
+int faiss_RangeSearchPartialResult_finalize(
+    FaissRangeSearchPartialResult* res);
+
+/// called by range_search before do_allocation
+int faiss_RangeSearchPartialResult_set_lims(
+    FaissRangeSearchPartialResult* res);
+
+int faiss_RangeSearchPartialResult_new_result(
+    FaissRangeSearchPartialResult* res, idx_t qno, FaissRangeQueryResult** qr);
+
+
+FAISS_DECLARE_CLASS(DistanceComputer)
+/// called before computing distances
+int faiss_DistanceComputer_set_query(FaissDistanceComputer *dc, const float *x);
+
+/**
+ * Compute distance of vector i to current query.
+ * This function corresponds to the function call operator: DistanceComputer::operator()
+ */
+int faiss_DistanceComputer_vector_to_query_dis( FaissDistanceComputer *dc, idx_t i, float *qd);
+/// compute distance between two stored vectors
+int faiss_DistanceComputer_symmetric_dis(FaissDistanceComputer *dc, idx_t i, idx_t j, float *vd);
+
+FAISS_DECLARE_DESTRUCTOR(DistanceComputer)
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/faiss/c_api/index_factory_c.cpp

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include <cstring>
+#include "index_factory.h"
+#include "index_factory_c.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+
+/** Build and index with the sequence of processing steps described in
+ *  the string.
+ */
+int faiss_index_factory(FaissIndex** p_index, int d, const char* description, FaissMetricType metric) {
+    try {
+        *p_index = reinterpret_cast<FaissIndex*>(faiss::index_factory(
+            d, description, static_cast<faiss::MetricType>(metric)));
+    } CATCH_AND_HANDLE
+}

data/vendor/faiss/c_api/index_factory_c.h

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_INDEX_FACTORY_C_H
+#define FAISS_INDEX_FACTORY_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Build and index with the sequence of processing steps described in
+ *  the string.
+ */
+int faiss_index_factory(FaissIndex** p_index, int d, const char* description, FaissMetricType metric);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/faiss/c_api/index_io_c.cpp

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//  Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
+// I/O code for indexes
+
+#include "index_io_c.h"
+#include "index_io.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+
+int faiss_write_index(const FaissIndex *idx, FILE *f) {
+    try {
+        faiss::write_index(reinterpret_cast<const Index*>(idx), f);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_write_index_fname(const FaissIndex *idx, const char *fname) {
+    try {
+        faiss::write_index(reinterpret_cast<const Index*>(idx), fname);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_read_index(FILE *f, int io_flags, FaissIndex **p_out) {
+    try {
+        auto out = faiss::read_index(f, io_flags);
+        *p_out = reinterpret_cast<FaissIndex*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_read_index_fname(const char *fname, int io_flags, FaissIndex **p_out) {
+    try {
+        auto out = faiss::read_index(fname, io_flags);
+        *p_out = reinterpret_cast<FaissIndex*>(out);
+    } CATCH_AND_HANDLE
+}

data/vendor/faiss/c_api/index_io_c.h

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//  Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
+// I/O code for indexes
+
+
+#ifndef FAISS_INDEX_IO_C_H
+#define FAISS_INDEX_IO_C_H
+
+#include <stdio.h>
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Write index to a file.
+ * This is equivalent to `faiss::write_index` when a file descriptor is provided.
+ */
+int faiss_write_index(const FaissIndex *idx, FILE *f);
+
+/** Write index to a file.
+ * This is equivalent to `faiss::write_index` when a file path is provided.
+ */
+int faiss_write_index_fname(const FaissIndex *idx, const char *fname);
+
+#define FAISS_IO_FLAG_MMAP 1
+#define FAISS_IO_FLAG_READ_ONLY 2
+
+/** Read index from a file.
+ * This is equivalent to `faiss:read_index` when a file descriptor is given.
+ */
+int faiss_read_index(FILE *f, int io_flags, FaissIndex **p_out);
+
+/** Read index from a file.
+ * This is equivalent to `faiss:read_index` when a file path is given.
+ */
+int faiss_read_index_fname(const char *fname, int io_flags, FaissIndex **p_out);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/vendor/faiss/c_api/macros_impl.h

@@ -0,0 +1,110 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+/// Utility macros for the C wrapper implementation.
+
+#ifndef MACROS_IMPL_H
+#define MACROS_IMPL_H
+
+#include "faiss_c.h"
+#include "FaissException.h"
+#include "error_impl.h"
+#include <stdexcept>
+#include <iostream>
+
+#ifdef NDEBUG
+#define CATCH_AND_HANDLE                              \
+    catch (faiss::FaissException& e) {                \
+        faiss_last_exception =                        \
+            std::make_exception_ptr(e);               \
+        return -2;                                    \
+    } catch (std::exception& e) {                     \
+        faiss_last_exception =                        \
+            std::make_exception_ptr(e);               \
+        return -4;                                    \
+    } catch (...) {                                   \
+        faiss_last_exception =                        \
+            std::make_exception_ptr(                  \
+                std::runtime_error("Unknown error")); \
+        return -1;                                    \
+    } return 0;
+#else
+#define CATCH_AND_HANDLE                              \
+    catch (faiss::FaissException& e) {                \
+        std::cerr << e.what() << '\n';                \
+        faiss_last_exception =                        \
+            std::make_exception_ptr(e);               \
+        return -2;                                    \
+    } catch (std::exception& e) {                     \
+        std::cerr << e.what() << '\n';                \
+        faiss_last_exception =                        \
+            std::make_exception_ptr(e);               \
+        return -4;                                    \
+    } catch (...) {                                   \
+        std::cerr << "Unrecognized exception!\n";     \
+        faiss_last_exception =                        \
+            std::make_exception_ptr(                  \
+                std::runtime_error("Unknown error")); \
+        return -1;                                    \
+    } return 0;
+#endif
+
+#define DEFINE_GETTER(clazz, ty, name)                            \
+    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) { \
+        return static_cast< ty >(                                 \
+            reinterpret_cast< const faiss::clazz *>(obj)-> name   \
+        );                                                        \
+    }
+
+#define DEFINE_GETTER_SUBCLASS(clazz, parent, ty, name)                \
+    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) {      \
+        return static_cast< ty >(                                      \
+            reinterpret_cast<const faiss::parent::clazz *>(obj)-> name \
+        );                                                             \
+    }
+
+#define DEFINE_GETTER_PERMISSIVE(clazz, ty, name)                 \
+    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) { \
+        return ( ty ) (                                           \
+            reinterpret_cast<const faiss::clazz *>(obj)-> name    \
+        );                                                        \
+    }
+
+#define DEFINE_GETTER_SUBCLASS_PERMISSIVE(clazz, parent, ty, name)     \
+    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *obj) {      \
+        return ( ty ) (                                                \
+            reinterpret_cast<const faiss::parent::clazz *>(obj)-> name \
+        );                                                             \
+    }
+
+#define DEFINE_SETTER(clazz, ty, name)                                    \
+    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *obj, ty val) { \
+        reinterpret_cast< faiss::clazz *>(obj)-> name = val;              \
+    }
+
+#define DEFINE_SETTER_STATIC(clazz, ty_to, ty_from, name)                      \
+    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *obj, ty_from val) { \
+        reinterpret_cast< faiss::clazz *>(obj)-> name =                        \
+            static_cast< ty_to >(val);                                         \
+    }
+
+#define DEFINE_DESTRUCTOR(clazz)                           \
+    void faiss_ ## clazz ## _free (Faiss ## clazz *obj) {  \
+        delete reinterpret_cast<faiss::clazz *>(obj);      \
+    }
+
+#define DEFINE_INDEX_DOWNCAST(clazz)                                \
+    Faiss ## clazz * faiss_ ## clazz ## _cast (FaissIndex* index) { \
+        return reinterpret_cast<Faiss ## clazz *>(                  \
+            dynamic_cast< faiss::clazz *>(                          \
+                reinterpret_cast<faiss::Index*>(index)));           \
+    }
+
+#endif

data/vendor/faiss/clone_index.cpp

@@ -0,0 +1,147 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// -*- c++ -*-
+
+#include <faiss/clone_index.h>
+
+#include <cstdio>
+#include <cstdlib>
+
+#include <faiss/impl/FaissAssert.h>
+
+#include <faiss/IndexFlat.h>
+#include <faiss/VectorTransform.h>
+#include <faiss/IndexPreTransform.h>
+#include <faiss/IndexLSH.h>
+#include <faiss/IndexPQ.h>
+#include <faiss/IndexIVF.h>
+#include <faiss/IndexIVFPQ.h>
+#include <faiss/IndexIVFPQR.h>
+#include <faiss/Index2Layer.h>
+#include <faiss/IndexIVFFlat.h>
+#include <faiss/IndexIVFSpectralHash.h>
+#include <faiss/MetaIndexes.h>
+#include <faiss/IndexScalarQuantizer.h>
+#include <faiss/IndexHNSW.h>
+#include <faiss/IndexLattice.h>
+#include <faiss/Index2Layer.h>
+
+namespace faiss {
+
+/*************************************************************
+ * cloning functions
+ **************************************************************/
+
+
+
+Index * clone_index (const Index *index)
+{
+    Cloner cl;
+    return cl.clone_Index (index);
+}
+
+// assumes there is a copy constructor ready. Always try from most
+// specific to most general. Most indexes don't have complicated
+// structs, the default copy constructor often just works.
+#define TRYCLONE(classname, obj) \
+    if (const classname *clo = dynamic_cast<const classname *>(obj)) { \
+        return new classname(*clo); \
+    } else
+
+VectorTransform *Cloner::clone_VectorTransform (const VectorTransform *vt)
+{
+    TRYCLONE (RemapDimensionsTransform, vt)
+    TRYCLONE (OPQMatrix, vt)
+    TRYCLONE (PCAMatrix, vt)
+    TRYCLONE (ITQMatrix, vt)
+    TRYCLONE (RandomRotationMatrix, vt)
+    TRYCLONE (LinearTransform, vt)
+    {
+      FAISS_THROW_MSG("clone not supported for this type of VectorTransform");
+    }
+    return nullptr;
+}
+
+IndexIVF * Cloner::clone_IndexIVF (const IndexIVF *ivf)
+{
+    TRYCLONE (IndexIVFPQR, ivf)
+    TRYCLONE (IndexIVFPQ, ivf)
+    TRYCLONE (IndexIVFFlat, ivf)
+    TRYCLONE (IndexIVFScalarQuantizer, ivf)
+    {
+      FAISS_THROW_MSG("clone not supported for this type of IndexIVF");
+    }
+    return nullptr;
+}
+
+Index *Cloner::clone_Index (const Index *index)
+{
+    TRYCLONE (IndexPQ, index)
+    TRYCLONE (IndexLSH, index)
+    TRYCLONE (IndexFlatL2, index)
+    TRYCLONE (IndexFlatIP, index)
+    TRYCLONE (IndexFlat, index)
+    TRYCLONE (IndexLattice, index)
+    TRYCLONE (IndexScalarQuantizer, index)
+    TRYCLONE (MultiIndexQuantizer, index)
+    if (const IndexIVF * ivf = dynamic_cast<const IndexIVF*>(index)) {
+        IndexIVF *res = clone_IndexIVF (ivf);
+        if (ivf->invlists == nullptr) {
+            res->invlists = nullptr;
+        } else if (auto *ails = dynamic_cast<const ArrayInvertedLists*>
+                   (ivf->invlists)) {
+            res->invlists = new ArrayInvertedLists(*ails);
+            res->own_invlists = true;
+        } else {
+            FAISS_THROW_MSG( "clone not supported for this type of inverted lists");
+        }
+        res->own_fields = true;
+        res->quantizer = clone_Index (ivf->quantizer);
+        return res;
+    } else if (const IndexPreTransform * ipt =
+               dynamic_cast<const IndexPreTransform*> (index)) {
+        IndexPreTransform *res = new IndexPreTransform ();
+        res->d = ipt->d;
+        res->ntotal = ipt->ntotal;
+        res->is_trained = ipt->is_trained;
+        res->metric_type = ipt->metric_type;
+        res->metric_arg = ipt->metric_arg;
+
+
+        res->index = clone_Index (ipt->index);
+        for (int i = 0; i < ipt->chain.size(); i++)
+            res->chain.push_back (clone_VectorTransform (ipt->chain[i]));
+        res->own_fields = true;
+        return res;
+    } else if (const IndexIDMap *idmap =
+               dynamic_cast<const IndexIDMap*> (index)) {
+        IndexIDMap *res = new IndexIDMap (*idmap);
+        res->own_fields = true;
+        res->index = clone_Index (idmap->index);
+        return res;
+    } else if (const IndexHNSW *ihnsw =
+               dynamic_cast<const IndexHNSW*> (index)) {
+        IndexHNSW *res = new IndexHNSW (*ihnsw);
+        res->own_fields = true;
+        res->storage = clone_Index (ihnsw->storage);
+        return res;
+    } else if (const Index2Layer *i2l =
+               dynamic_cast<const Index2Layer*> (index)) {
+        Index2Layer *res = new Index2Layer (*i2l);
+        res->q1.own_fields = true;
+        res->q1.quantizer = clone_Index (i2l->q1.quantizer);
+        return res;
+    } else {
+        FAISS_THROW_MSG( "clone not supported for this type of Index");
+    }
+    return nullptr;
+}
+
+
+
+} // namespace faiss