faiss 0.1.0 → 0.1.1

data/vendor/faiss/c_api/IndexShards_c.h

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#ifndef INDEXSHARDS_C_H
+#define INDEXSHARDS_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Index that concatenates the results from several sub-indexes
+ */
+FAISS_DECLARE_CLASS_INHERITED(IndexShards, Index)
+
+FAISS_DECLARE_GETTER_SETTER(IndexShards, int, own_fields)
+FAISS_DECLARE_GETTER_SETTER(IndexShards, int, successive_ids)
+
+int faiss_IndexShards_new(FaissIndexShards** p_index, idx_t d);
+
+int faiss_IndexShards_new_with_options(FaissIndexShards** p_index, idx_t d, int threaded, int successive_ids);
+
+int faiss_IndexShards_add_shard(FaissIndexShards* index, FaissIndex* shard);
+
+/// update metric_type and ntotal
+int faiss_IndexShards_sync_with_shard_indexes(FaissIndexShards* index);
+
+FaissIndex* faiss_IndexShards_at(FaissIndexShards* index, int i);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/vendor/faiss/c_api/Index_c.cpp

@@ -0,0 +1,105 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "Index_c.h"
+#include "Index.h"
+#include "macros_impl.h"
+
+extern "C" {
+
+DEFINE_DESTRUCTOR(Index)
+
+DEFINE_GETTER(Index, int, d)
+
+DEFINE_GETTER(Index, int, is_trained)
+
+DEFINE_GETTER(Index, idx_t, ntotal)
+
+DEFINE_GETTER(Index, FaissMetricType, metric_type)
+
+int faiss_Index_train(FaissIndex* index, idx_t n, const float* x) {
+    try {
+        reinterpret_cast<faiss::Index*>(index)->train(n, x);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_add(FaissIndex* index, idx_t n, const float* x) {
+    try {
+        reinterpret_cast<faiss::Index*>(index)->add(n, x);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_add_with_ids(FaissIndex* index, idx_t n, const float* x, const idx_t* xids) {
+    try {
+        reinterpret_cast<faiss::Index*>(index)->add_with_ids(n, x, xids);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_search(const FaissIndex* index, idx_t n, const float* x, idx_t k,
+                       float* distances, idx_t* labels) {
+    try {
+        reinterpret_cast<const faiss::Index*>(index)->search(n, x, k, distances, labels);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_range_search(const FaissIndex* index, idx_t n, const float* x, float radius,
+                             FaissRangeSearchResult* result) {
+    try {
+        reinterpret_cast<const faiss::Index*>(index)->range_search(
+            n, x, radius, reinterpret_cast<faiss::RangeSearchResult*>(result));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_assign(FaissIndex* index, idx_t n, const float * x, idx_t * labels, idx_t k) {
+    try {
+        reinterpret_cast<faiss::Index*>(index)->assign(n, x, labels, k);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_reset(FaissIndex* index) {
+    try {
+        reinterpret_cast<faiss::Index*>(index)->reset();
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_remove_ids(FaissIndex* index, const FaissIDSelector* sel, size_t* n_removed) {
+    try {
+        size_t n {reinterpret_cast<faiss::Index*>(index)->remove_ids(
+            *reinterpret_cast<const faiss::IDSelector*>(sel))};
+        if (n_removed) {
+            *n_removed = n;
+        }
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_reconstruct(const FaissIndex* index, idx_t key, float* recons) {
+    try {
+        reinterpret_cast<const faiss::Index*>(index)->reconstruct(key, recons);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_reconstruct_n (const FaissIndex* index, idx_t i0, idx_t ni, float* recons) {
+    try {
+        reinterpret_cast<const faiss::Index*>(index)->reconstruct_n(i0, ni, recons);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_compute_residual(const FaissIndex* index, const float* x, float* residual, idx_t key) {
+    try {
+        reinterpret_cast<const faiss::Index*>(index)->compute_residual(x, residual, key);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_Index_compute_residual_n(const FaissIndex* index, idx_t n, const float* x, float* residuals, const idx_t* keys) {
+    try {
+        reinterpret_cast<const faiss::Index *>(index)->compute_residual_n(n, x, residuals, keys);
+    } CATCH_AND_HANDLE
+}
+}

data/vendor/faiss/c_api/Index_c.h

@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved
+// -*- c -*-
+
+#ifndef FAISS_INDEX_C_H
+#define FAISS_INDEX_C_H
+
+#include <stddef.h>
+#include "faiss_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// forward declaration required here
+FAISS_DECLARE_CLASS(RangeSearchResult)
+
+//typedef struct FaissRangeSearchResult_H FaissRangeSearchResult;
+typedef struct FaissIDSelector_H FaissIDSelector;
+
+/// Some algorithms support both an inner product version and a L2 search version.
+typedef enum FaissMetricType {
+    METRIC_INNER_PRODUCT = 0,  ///< maximum inner product search
+    METRIC_L2 = 1,             ///< squared L2 search
+    METRIC_L1,                 ///< L1 (aka cityblock)
+    METRIC_Linf,               ///< infinity distance
+    METRIC_Lp,                 ///< L_p distance, p is given by metric_arg
+
+    /// some additional metrics defined in scipy.spatial.distance
+    METRIC_Canberra = 20,
+    METRIC_BrayCurtis,
+    METRIC_JensenShannon,    
+} FaissMetricType;
+
+/// Opaque type for referencing to an index object
+FAISS_DECLARE_CLASS(Index)
+FAISS_DECLARE_DESTRUCTOR(Index)
+
+/// Getter for d
+FAISS_DECLARE_GETTER(Index, int, d)
+
+/// Getter for is_trained
+FAISS_DECLARE_GETTER(Index, int, is_trained)
+
+/// Getter for ntotal
+FAISS_DECLARE_GETTER(Index, idx_t, ntotal)
+
+/// Getter for metric_type
+FAISS_DECLARE_GETTER(Index, FaissMetricType, metric_type)
+
+/** Perform training on a representative set of vectors
+ *
+ * @param index  opaque pointer to index object
+ * @param n      nb of training vectors
+ * @param x      training vecors, size n * d
+ */
+int faiss_Index_train(FaissIndex* index, idx_t n, const float* x);
+
+/** Add n vectors of dimension d to the index.
+ *
+ * Vectors are implicitly assigned labels ntotal .. ntotal + n - 1
+ * This function slices the input vectors in chuncks smaller than
+ * blocksize_add and calls add_core.
+ * @param index  opaque pointer to index object
+ * @param x      input matrix, size n * d
+ */
+int faiss_Index_add(FaissIndex* index, idx_t n, const float* x);
+
+/** Same as add, but stores xids instead of sequential ids.
+ *
+ * The default implementation fails with an assertion, as it is
+ * not supported by all indexes.
+ *
+ * @param index  opaque pointer to index object
+ * @param xids   if non-null, ids to store for the vectors (size n)
+ */
+int faiss_Index_add_with_ids(FaissIndex* index, idx_t n, const float* x, const idx_t* xids);
+
+/** query n vectors of dimension d to the index.
+ *
+ * return at most k vectors. If there are not enough results for a
+ * query, the result array is padded with -1s.
+ *
+ * @param index       opaque pointer to index object
+ * @param x           input vectors to search, size n * d
+ * @param labels      output labels of the NNs, size n*k
+ * @param distances   output pairwise distances, size n*k
+ */
+int faiss_Index_search(const FaissIndex* index, idx_t n, const float* x, idx_t k,
+                       float* distances, idx_t* labels);
+
+/** query n vectors of dimension d to the index.
+ *
+ * return all vectors with distance < radius. Note that many
+ * indexes do not implement the range_search (only the k-NN search
+ * is mandatory).
+ *
+ * @param index       opaque pointer to index object
+ * @param x           input vectors to search, size n * d
+ * @param radius      search radius
+ * @param result      result table
+ */
+int faiss_Index_range_search(const FaissIndex* index, idx_t n, const float* x,
+                             float radius, FaissRangeSearchResult* result);
+
+/** return the indexes of the k vectors closest to the query x.
+ *
+ * This function is identical as search but only return labels of neighbors.
+ * @param index       opaque pointer to index object
+ * @param x           input vectors to search, size n * d
+ * @param labels      output labels of the NNs, size n*k
+ */
+int faiss_Index_assign(FaissIndex* index, idx_t n, const float * x, idx_t * labels, idx_t k);
+
+/** removes all elements from the database.
+ * @param index       opaque pointer to index object
+ */
+int faiss_Index_reset(FaissIndex* index);
+
+/** removes IDs from the index. Not supported by all indexes
+ * @param index       opaque pointer to index object
+ * @param nremove     output for the number of IDs removed
+ */
+int faiss_Index_remove_ids(FaissIndex* index, const FaissIDSelector* sel, size_t* n_removed);
+
+/** Reconstruct a stored vector (or an approximation if lossy coding)
+ *
+ * this function may not be defined for some indexes
+ * @param index       opaque pointer to index object
+ * @param key         id of the vector to reconstruct
+ * @param recons      reconstucted vector (size d)
+ */
+int faiss_Index_reconstruct(const FaissIndex* index, idx_t key, float* recons);
+
+/** Reconstruct vectors i0 to i0 + ni - 1
+ *
+ * this function may not be defined for some indexes
+ * @param index       opaque pointer to index object
+ * @param recons      reconstucted vector (size ni * d)
+ */
+int faiss_Index_reconstruct_n (const FaissIndex* index, idx_t i0, idx_t ni, float* recons);
+
+/** Computes a residual vector after indexing encoding.
+ *
+ * The residual vector is the difference between a vector and the
+ * reconstruction that can be decoded from its representation in
+ * the index. The residual can be used for multiple-stage indexing
+ * methods, like IndexIVF's methods.
+ *
+ * @param index       opaque pointer to index object
+ * @param x           input vector, size d
+ * @param residual    output residual vector, size d
+ * @param key         encoded index, as returned by search and assign
+ */
+int faiss_Index_compute_residual(const FaissIndex* index, const float* x, float* residual, idx_t key);
+
+/** Computes a residual vector after indexing encoding.
+ *
+ * The residual vector is the difference between a vector and the
+ * reconstruction that can be decoded from its representation in
+ * the index. The residual can be used for multiple-stage indexing
+ * methods, like IndexIVF's methods.
+ *
+ * @param index       opaque pointer to index object
+ * @param n           number of vectors
+ * @param x           input vector, size (n x d)
+ * @param residuals    output residual vectors, size (n x d)
+ * @param keys         encoded index, as returned by search and assign
+ */
+int faiss_Index_compute_residual_n(const FaissIndex* index, idx_t n, const float* x, float* residuals, const idx_t* keys);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/faiss/c_api/MetaIndexes_c.cpp

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "MetaIndexes_c.h"
+#include "MetaIndexes.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexIDMap;
+using faiss::IndexIDMap2;
+
+DEFINE_GETTER(IndexIDMap, int, own_fields)
+DEFINE_SETTER(IndexIDMap, int, own_fields)
+
+int faiss_IndexIDMap_new(FaissIndexIDMap** p_index, FaissIndex* index) {
+    try {
+        auto out = new IndexIDMap(reinterpret_cast<Index*>(index));
+        *p_index = reinterpret_cast<FaissIndexIDMap*>(out);
+    } CATCH_AND_HANDLE
+}
+
+void faiss_IndexIDMap_id_map(FaissIndexIDMap* index, idx_t** p_id_map, size_t* p_size) {
+    auto idx = reinterpret_cast<IndexIDMap*>(index);
+    if (p_id_map)
+        *p_id_map = idx->id_map.data();
+    if (p_size)
+        *p_size = idx->id_map.size();
+}
+
+int faiss_IndexIDMap2_new(FaissIndexIDMap2** p_index, FaissIndex* index) {
+    try {
+        auto out = new IndexIDMap2(reinterpret_cast<Index*>(index));
+        *p_index = reinterpret_cast<FaissIndexIDMap2*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIDMap2_construct_rev_map(FaissIndexIDMap2* index) {
+    try {
+        reinterpret_cast<IndexIDMap2*>(index)->construct_rev_map();
+    } CATCH_AND_HANDLE
+}
+

data/vendor/faiss/c_api/MetaIndexes_c.h

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#ifndef METAINDEXES_C_H
+#define METAINDEXES_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Index that translates search results to ids */
+FAISS_DECLARE_CLASS_INHERITED(IndexIDMap, Index)
+
+FAISS_DECLARE_GETTER_SETTER(IndexIDMap, int, own_fields)
+
+int faiss_IndexIDMap_new(FaissIndexIDMap** p_index, FaissIndex* index);
+
+/** get a pointer to the index map's internal ID vector (the `id_map` field). The
+ * outputs of this function become invalid after any operation that can modify the index.
+ * 
+ * @param index   opaque pointer to index object
+ * @param p_id_map    output, the pointer to the beginning of `id_map`.
+ * @param p_size  output, the current length of `id_map`.
+ */
+void faiss_IndexIDMap_id_map(FaissIndexIDMap* index, idx_t** p_id_map, size_t* p_size);
+
+/** same as IndexIDMap but also provides an efficient reconstruction
+    implementation via a 2-way index */
+FAISS_DECLARE_CLASS_INHERITED(IndexIDMap2, IndexIDMap)
+
+int faiss_IndexIDMap2_new(FaissIndexIDMap2** p_index, FaissIndex* index);
+
+/// make the rev_map from scratch
+int faiss_IndexIDMap2_construct_rev_map(FaissIndexIDMap2* index);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/vendor/faiss/c_api/clone_index_c.cpp

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//  Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
+// I/O code for indexes
+
+#include "clone_index_c.h"
+#include "clone_index.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+
+int faiss_clone_index (const FaissIndex *idx, FaissIndex **p_out) {
+    try {
+        auto out = faiss::clone_index(reinterpret_cast<const Index*>(idx));
+        *p_out = reinterpret_cast<FaissIndex*>(out);
+    } CATCH_AND_HANDLE
+}