faiss 0.1.0 → 0.1.1

data/vendor/faiss/c_api/IndexIVFFlat_c.cpp

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "Index_c.h"
+#include "Clustering_c.h"
+#include "IndexIVFFlat_c.h"
+#include "IndexIVFFlat.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexIVFFlat;
+using faiss::MetricType;
+
+DEFINE_DESTRUCTOR(IndexIVFFlat)
+DEFINE_INDEX_DOWNCAST(IndexIVFFlat)
+
+int faiss_IndexIVFFlat_new(FaissIndexIVFFlat** p_index) {
+    try {
+        *p_index = reinterpret_cast<FaissIndexIVFFlat*>(new IndexIVFFlat());
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIVFFlat_new_with(FaissIndexIVFFlat** p_index,
+    FaissIndex* quantizer, size_t d, size_t nlist)
+{
+    try {
+        auto q = reinterpret_cast<Index*>(quantizer);
+        *p_index = reinterpret_cast<FaissIndexIVFFlat*>(new IndexIVFFlat(q, d, nlist));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIVFFlat_new_with_metric(
+    FaissIndexIVFFlat** p_index, FaissIndex* quantizer, size_t d, size_t nlist,
+    FaissMetricType metric)
+{
+    try {
+        auto q = reinterpret_cast<Index*>(quantizer);
+        auto m = static_cast<MetricType>(metric);
+        *p_index = reinterpret_cast<FaissIndexIVFFlat*>(new IndexIVFFlat(q, d, nlist, m));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIVFFlat_add_core(FaissIndexIVFFlat* index, idx_t n, 
+    const float * x, const idx_t *xids, const int64_t *precomputed_idx)
+{
+    try {
+        reinterpret_cast<IndexIVFFlat*>(index)->add_core(n, x, xids, precomputed_idx);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIVFFlat_update_vectors(FaissIndexIVFFlat* index, int nv,
+    idx_t *idx, const float *v)
+{
+    try {
+        reinterpret_cast<IndexIVFFlat*>(index)->update_vectors(nv, idx, v);
+    } CATCH_AND_HANDLE
+}

data/vendor/faiss/c_api/IndexIVFFlat_c.h

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_INDEX_IVF_FLAT_C_H
+#define FAISS_INDEX_IVF_FLAT_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+#include "Clustering_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Inverted file with stored vectors. Here the inverted file
+ * pre-selects the vectors to be searched, but they are not otherwise
+ * encoded, the code array just contains the raw float entries.
+ */
+FAISS_DECLARE_CLASS(IndexIVFFlat)
+FAISS_DECLARE_DESTRUCTOR(IndexIVFFlat)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexIVFFlat)
+
+int faiss_IndexIVFFlat_new(FaissIndexIVFFlat** p_index);
+
+int faiss_IndexIVFFlat_new_with(FaissIndexIVFFlat** p_index,
+    FaissIndex* quantizer, size_t d, size_t nlist);
+
+int faiss_IndexIVFFlat_new_with_metric(
+    FaissIndexIVFFlat** p_index, FaissIndex* quantizer, size_t d, size_t nlist,
+    FaissMetricType metric);
+
+int faiss_IndexIVFFlat_add_core(FaissIndexIVFFlat* index, idx_t n, 
+    const float * x, const idx_t *xids, const int64_t *precomputed_idx);
+
+/** Update a subset of vectors.
+ *
+ * The index must have a direct_map
+ *
+ * @param nv     nb of vectors to update
+ * @param idx    vector indices to update, size nv
+ * @param v      vectors of new values, size nv*d
+ */
+int faiss_IndexIVFFlat_update_vectors(FaissIndexIVFFlat* index, int nv,
+    idx_t *idx, const float *v);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/vendor/faiss/c_api/IndexIVF_c.cpp

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "Index_c.h"
+#include "Clustering_c.h"
+#include "IndexIVF_c.h"
+#include "IndexIVF.h"
+#include "macros_impl.h"
+
+using faiss::IndexIVF;
+using faiss::IndexIVFStats;
+
+DEFINE_DESTRUCTOR(IndexIVF)
+DEFINE_INDEX_DOWNCAST(IndexIVF)
+
+/// number of possible key values
+DEFINE_GETTER(IndexIVF, size_t, nlist)
+/// number of probes at query time
+DEFINE_GETTER(IndexIVF, size_t, nprobe)
+/// quantizer that maps vectors to inverted lists
+DEFINE_GETTER_PERMISSIVE(IndexIVF, FaissIndex*, quantizer)
+
+/**
+ * = 0: use the quantizer as index in a kmeans training
+ * = 1: just pass on the training set to the train() of the quantizer
+ * = 2: kmeans training on a flat index + add the centroids to the quantizer
+ */
+DEFINE_GETTER(IndexIVF, char, quantizer_trains_alone)
+
+/// whether object owns the quantizer
+DEFINE_GETTER(IndexIVF, int, own_fields)
+
+using faiss::IndexIVF;
+
+int faiss_IndexIVF_merge_from(
+    FaissIndexIVF* index, FaissIndexIVF* other, idx_t add_id) {
+    try {
+        reinterpret_cast<IndexIVF*>(index)->merge_from(
+            *reinterpret_cast<IndexIVF*>(other), add_id);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIVF_copy_subset_to(
+    const FaissIndexIVF* index, FaissIndexIVF* other, int subset_type, idx_t a1,
+    idx_t a2) {
+    try {
+        reinterpret_cast<const IndexIVF*>(index)->copy_subset_to(
+            *reinterpret_cast<IndexIVF*>(other), subset_type, a1, a2);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexIVF_search_preassigned (const FaissIndexIVF* index,
+    idx_t n, const float *x, idx_t k, const idx_t *assign,
+    const float *centroid_dis, float *distances, idx_t *labels,
+    int store_pairs) {
+    try {
+        reinterpret_cast<const IndexIVF*>(index)->search_preassigned(
+            n, x, k, assign, centroid_dis, distances, labels, store_pairs);
+   } CATCH_AND_HANDLE
+}
+
+size_t faiss_IndexIVF_get_list_size(const FaissIndexIVF* index, size_t list_no) {
+    return reinterpret_cast<const IndexIVF*>(index)->get_list_size(list_no);
+}
+
+int faiss_IndexIVF_make_direct_map(FaissIndexIVF* index,
+    int new_maintain_direct_map) {
+    try {
+        reinterpret_cast<IndexIVF*>(index)->make_direct_map(
+            static_cast<bool>(new_maintain_direct_map));
+    } CATCH_AND_HANDLE
+}
+
+double faiss_IndexIVF_imbalance_factor (const FaissIndexIVF* index) {
+    return reinterpret_cast<const IndexIVF*>(index)->invlists->imbalance_factor();
+}
+
+/// display some stats about the inverted lists
+void faiss_IndexIVF_print_stats (const FaissIndexIVF* index) {
+    reinterpret_cast<const IndexIVF*>(index)->invlists->print_stats();
+}
+
+void faiss_IndexIVFStats_reset(FaissIndexIVFStats* stats) {
+    reinterpret_cast<IndexIVFStats*>(stats)->reset();    
+}

data/vendor/faiss/c_api/IndexIVF_c.h

@@ -0,0 +1,135 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_INDEX_IVF_C_H
+#define FAISS_INDEX_IVF_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+#include "Clustering_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Index based on a inverted file (IVF)
+ *
+ * In the inverted file, the quantizer (an Index instance) provides a
+ * quantization index for each vector to be added. The quantization
+ * index maps to a list (aka inverted list or posting list), where the
+ * id of the vector is then stored.
+ *
+ * At search time, the vector to be searched is also quantized, and
+ * only the list corresponding to the quantization index is
+ * searched. This speeds up the search by making it
+ * non-exhaustive. This can be relaxed using multi-probe search: a few
+ * (nprobe) quantization indices are selected and several inverted
+ * lists are visited.
+ *
+ * Sub-classes implement a post-filtering of the index that refines
+ * the distance estimation from the query to databse vectors.
+ */
+FAISS_DECLARE_CLASS_INHERITED(IndexIVF, Index)
+FAISS_DECLARE_DESTRUCTOR(IndexIVF)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexIVF)
+
+/// number of possible key values
+FAISS_DECLARE_GETTER(IndexIVF, size_t, nlist)
+/// number of probes at query time
+FAISS_DECLARE_GETTER(IndexIVF, size_t, nprobe)
+/// quantizer that maps vectors to inverted lists
+FAISS_DECLARE_GETTER(IndexIVF, FaissIndex*, quantizer)
+/**
+ * = 0: use the quantizer as index in a kmeans training
+ * = 1: just pass on the training set to the train() of the quantizer
+ * = 2: kmeans training on a flat index + add the centroids to the quantizer
+ */
+FAISS_DECLARE_GETTER(IndexIVF, char, quantizer_trains_alone)
+
+/// whether object owns the quantizer
+FAISS_DECLARE_GETTER(IndexIVF, int, own_fields)
+
+/** moves the entries from another dataset to self. On output,
+ * other is empty. add_id is added to all moved ids (for
+ * sequential ids, this would be this->ntotal */
+int faiss_IndexIVF_merge_from(
+    FaissIndexIVF* index, FaissIndexIVF* other, idx_t add_id);
+
+/** copy a subset of the entries index to the other index
+ *
+ * if subset_type == 0: copies ids in [a1, a2)
+ * if subset_type == 1: copies ids if id % a1 == a2
+ * if subset_type == 2: copies inverted lists such that a1
+ *                      elements are left before and a2 elements are after
+ */
+int faiss_IndexIVF_copy_subset_to(
+    const FaissIndexIVF* index, FaissIndexIVF* other, int subset_type, idx_t a1,
+    idx_t a2);
+
+/** search a set of vectors, that are pre-quantized by the IVF
+ *  quantizer. Fill in the corresponding heaps with the query
+ *  results. search() calls this.
+ *
+ * @param n      nb of vectors to query
+ * @param x      query vectors, size nx * d
+ * @param assign coarse quantization indices, size nx * nprobe
+ * @param centroid_dis
+ *               distances to coarse centroids, size nx * nprobe
+ * @param distance
+ *               output distances, size n * k
+ * @param labels output labels, size n * k
+ * @param store_pairs store inv list index + inv list offset
+ *                     instead in upper/lower 32 bit of result,
+ *                     instead of ids (used for reranking).
+ */
+int faiss_IndexIVF_search_preassigned (const FaissIndexIVF* index,
+    idx_t n, const float *x, idx_t k, const idx_t *assign,
+    const float *centroid_dis, float *distances, idx_t *labels,
+    int store_pairs);
+
+size_t faiss_IndexIVF_get_list_size(const FaissIndexIVF* index,
+    size_t list_no);
+
+/** intialize a direct map
+ *
+ * @param new_maintain_direct_map    if true, create a direct map,
+ *                                   else clear it
+ */
+int faiss_IndexIVF_make_direct_map(FaissIndexIVF* index,
+    int new_maintain_direct_map);
+
+/** Check the inverted lists' imbalance factor.
+ *
+ * 1= perfectly balanced, >1: imbalanced
+ */
+double faiss_IndexIVF_imbalance_factor (const FaissIndexIVF* index);
+
+/// display some stats about the inverted lists of the index
+void faiss_IndexIVF_print_stats (const FaissIndexIVF* index);
+
+
+typedef struct FaissIndexIVFStats {
+    size_t nq;       // nb of queries run
+    size_t nlist;    // nb of inverted lists scanned
+    size_t ndis;     // nb of distancs computed
+} FaissIndexIVFStats;
+
+void faiss_IndexIVFStats_reset(FaissIndexIVFStats* stats);
+
+inline void faiss_IndexIVFStats_init(FaissIndexIVFStats* stats) {
+    faiss_IndexIVFStats_reset(stats);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/vendor/faiss/c_api/IndexLSH_c.cpp

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "IndexLSH_c.h"
+#include "IndexLSH.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexLSH;
+
+DEFINE_DESTRUCTOR(IndexLSH)
+DEFINE_INDEX_DOWNCAST(IndexLSH)
+
+DEFINE_GETTER(IndexLSH, int, nbits)
+DEFINE_GETTER(IndexLSH, int, bytes_per_vec)
+DEFINE_GETTER_PERMISSIVE(IndexLSH, int, rotate_data)
+DEFINE_GETTER_PERMISSIVE(IndexLSH, int, train_thresholds)
+
+int faiss_IndexLSH_new(FaissIndexLSH** p_index, idx_t d, int nbits) {
+    try {
+        *p_index = reinterpret_cast<FaissIndexLSH*>(new IndexLSH(d, nbits));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexLSH_new_with_options(FaissIndexLSH** p_index, idx_t d, int nbits, int rotate_data, int train_thresholds) {
+    try {
+        *p_index = reinterpret_cast<FaissIndexLSH*>(
+            new IndexLSH(d, nbits, static_cast<bool>(rotate_data), static_cast<bool>(train_thresholds)));
+    } CATCH_AND_HANDLE
+}

data/vendor/faiss/c_api/IndexLSH_c.h

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#ifndef INDEX_LSH_C_H
+#define INDEX_LSH_C_H
+
+#include "faiss_c.h"
+#include "Index_c.h"
+#include "Clustering_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** The sign of each vector component is put in a binary signature */
+FAISS_DECLARE_CLASS_INHERITED(IndexLSH, Index)
+FAISS_DECLARE_DESTRUCTOR(IndexLSH)
+FAISS_DECLARE_INDEX_DOWNCAST(IndexLSH)
+
+FAISS_DECLARE_GETTER(IndexLSH, int, nbits)
+FAISS_DECLARE_GETTER(IndexLSH, int, bytes_per_vec)
+FAISS_DECLARE_GETTER(IndexLSH, int, rotate_data)
+FAISS_DECLARE_GETTER(IndexLSH, int, train_thresholds)
+
+int faiss_IndexLSH_new(FaissIndexLSH** p_index, idx_t d, int nbits);
+
+int faiss_IndexLSH_new_with_options(FaissIndexLSH** p_index, idx_t d, int nbits, int rotate_data, int train_thresholds);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/faiss/c_api/IndexShards_c.cpp

@@ -0,0 +1,44 @@
+#include "IndexShards_c.h"
+#include "IndexShards.h"
+#include "macros_impl.h"
+
+using faiss::Index;
+using faiss::IndexShards;
+
+DEFINE_GETTER(IndexShards, int, own_fields)
+DEFINE_SETTER(IndexShards, int, own_fields)
+
+DEFINE_GETTER(IndexShards, int, successive_ids)
+DEFINE_SETTER(IndexShards, int, successive_ids)
+
+int faiss_IndexShards_new(FaissIndexShards** p_index, idx_t d) {
+    try {
+        auto out = new IndexShards(d);
+        *p_index = reinterpret_cast<FaissIndexShards*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexShards_new_with_options(FaissIndexShards** p_index, idx_t d, int threaded, int successive_ids) {
+    try {
+        auto out = new IndexShards(d, static_cast<bool>(threaded), static_cast<bool>(successive_ids));
+        *p_index = reinterpret_cast<FaissIndexShards*>(out);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexShards_add_shard(FaissIndexShards* index, FaissIndex* shard) {
+    try {
+        reinterpret_cast<IndexShards*>(index)->add_shard(
+            reinterpret_cast<Index*>(shard));
+    } CATCH_AND_HANDLE
+}
+
+int faiss_IndexShards_sync_with_shard_indexes(FaissIndexShards* index) {
+    try {
+        reinterpret_cast<IndexShards*>(index)->sync_with_shard_indexes();
+    } CATCH_AND_HANDLE
+}
+
+FaissIndex* faiss_IndexShards_at(FaissIndexShards* index, int i) {
+    auto shard = reinterpret_cast<IndexShards*>(index)->at(i);
+    return reinterpret_cast<FaissIndex*>(shard);
+}