faiss 0.1.0 → 0.1.1

data/vendor/faiss/c_api/clone_index_c.h

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+//  Copyright 2004-present Facebook. All Rights Reserved
+// -*- c++ -*-
+// I/O code for indexes
+
+
+#ifndef FAISS_CLONE_INDEX_C_H
+#define FAISS_CLONE_INDEX_C_H
+
+#include <stdio.h>
+#include "faiss_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* cloning functions */
+
+/** Clone an index. This is equivalent to `faiss::clone_index` */
+int faiss_clone_index (const FaissIndex *, FaissIndex ** p_out);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/vendor/faiss/c_api/error_c.h

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_ERROR_C_H
+#define FAISS_ERROR_C_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// An error code which depends on the exception thrown from the previous 
+/// operation. See `faiss_get_last_error` to retrieve the error message.
+typedef enum FaissErrorCode {
+    /// No error
+    OK = 0,
+    /// Any exception other than Faiss or standard C++ library exceptions
+    UNKNOWN_EXCEPT = -1,
+    /// Faiss library exception
+    FAISS_EXCEPT = -2,
+    /// Standard C++ library exception
+    STD_EXCEPT = -4
+} FaissErrorCode;
+
+/**
+ * Get the error message of the last failed operation performed by Faiss.
+ * The given pointer is only invalid until another Faiss function is
+ * called.
+ */
+const char* faiss_get_last_error();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/faiss/c_api/error_impl.cpp

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "error_c.h"
+#include "error_impl.h"
+#include "FaissException.h"
+#include <exception>
+
+thread_local std::exception_ptr faiss_last_exception;
+
+const char* faiss_get_last_error() {
+    if (faiss_last_exception) {
+        try {
+            std::rethrow_exception(faiss_last_exception);
+        } catch (std::exception& e) {
+            return e.what();
+        }
+    }
+    return nullptr;
+}

data/vendor/faiss/c_api/error_impl.h

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include <exception>
+
+/** global variable for holding the last exception thrown by
+ * calls to Faiss functions through the C API
+ */
+extern thread_local std::exception_ptr faiss_last_exception;

data/vendor/faiss/c_api/faiss_c.h

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+/// Macros and typedefs for C wrapper API declarations
+
+#ifndef FAISS_C_H
+#define FAISS_C_H
+
+#include <stdint.h>
+
+typedef int64_t faiss_idx_t;    ///< all indices are this type
+typedef faiss_idx_t idx_t;
+typedef float faiss_component_t;    ///< all vector components are this type
+typedef float faiss_distance_t;    ///< all distances between vectors are this type
+
+/// Declare an opaque type for a class type `clazz`.
+#define FAISS_DECLARE_CLASS(clazz) \
+    typedef struct Faiss ## clazz ## _H Faiss ## clazz;
+
+/// Declare an opaque type for a class type `clazz`, while
+/// actually aliasing it to an existing parent class type `parent`.
+#define FAISS_DECLARE_CLASS_INHERITED(clazz, parent) \
+    typedef struct Faiss ## parent ## _H Faiss ## clazz;
+
+/// Declare a dynamic downcast operation from a base `FaissIndex*` pointer
+/// type to a more specific index type. The function returns the same pointer
+/// if the downcast is valid, and `NULL` otherwise.
+#define FAISS_DECLARE_INDEX_DOWNCAST(clazz) \
+    Faiss ## clazz * faiss_ ## clazz ## _cast (FaissIndex*);
+
+/// Declare a getter for the field `name` in class `clazz`,
+/// of return type `ty`
+#define FAISS_DECLARE_GETTER(clazz, ty, name) \
+    ty faiss_ ## clazz ## _ ## name (const Faiss ## clazz *);
+
+/// Declare a setter for the field `name` in class `clazz`,
+/// in which the user provides a value of type `ty`
+#define FAISS_DECLARE_SETTER(clazz, ty, name) \
+    void faiss_ ## clazz ## _set_ ## name (Faiss ## clazz *, ty);
+
+/// Declare a getter and setter for the field `name` in class `clazz`.
+#define FAISS_DECLARE_GETTER_SETTER(clazz, ty, name) \
+    FAISS_DECLARE_GETTER(clazz, ty, name) \
+    FAISS_DECLARE_SETTER(clazz, ty, name)
+    
+/// Declare a destructor function which frees an object of
+/// type `clazz`.
+#define FAISS_DECLARE_DESTRUCTOR(clazz) \
+    void faiss_ ## clazz ## _free (Faiss ## clazz *obj);
+
+#endif

data/vendor/faiss/c_api/gpu/GpuAutoTune_c.cpp

@@ -0,0 +1,96 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "GpuAutoTune_c.h"
+#include "GpuClonerOptions_c.h"
+#include "macros_impl.h"
+#include "Index.h"
+#include "gpu/GpuAutoTune.h"
+#include "gpu/GpuClonerOptions.h"
+#include <vector>
+
+using faiss::Index;
+using faiss::gpu::GpuResources;
+using faiss::gpu::GpuClonerOptions;
+using faiss::gpu::GpuMultipleClonerOptions;
+
+int faiss_index_gpu_to_cpu(const FaissIndex* gpu_index, FaissIndex** p_out) {
+    try {
+        auto cpu_index = faiss::gpu::index_gpu_to_cpu(
+            reinterpret_cast<const Index*>(gpu_index)
+        );
+        *p_out = reinterpret_cast<FaissIndex*>(cpu_index);
+    } CATCH_AND_HANDLE
+}
+
+/// converts any CPU index that can be converted to GPU
+int faiss_index_cpu_to_gpu(FaissGpuResources* resources, int device, const FaissIndex *index, FaissGpuIndex** p_out) {
+    try {
+        auto res = reinterpret_cast<GpuResources*>(resources);
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu(
+            res, device, reinterpret_cast<const Index*>(index)
+        );
+        *p_out = reinterpret_cast<FaissGpuIndex*>(gpu_index);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_index_cpu_to_gpu_with_options(
+    FaissGpuResources* resources, int device,
+    const FaissIndex *index, const FaissGpuClonerOptions* options,
+    FaissGpuIndex** p_out)
+{
+    try {
+        auto res = reinterpret_cast<GpuResources*>(resources);
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu(
+            res, device, reinterpret_cast<const Index*>(index),
+            reinterpret_cast<const GpuClonerOptions*>(options));
+        *p_out = reinterpret_cast<FaissGpuIndex*>(gpu_index);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_index_cpu_to_gpu_multiple(
+    FaissGpuResources* const* resources_vec,
+    const int* devices, size_t devices_size,
+    const FaissIndex* index, FaissGpuIndex** p_out)
+{
+    try {
+        std::vector<GpuResources*> res(devices_size);
+        for (auto i = 0u; i < devices_size; ++i) {
+            res[i] = reinterpret_cast<GpuResources*>(resources_vec[i]);
+        }
+
+        std::vector<int> dev(devices, devices + devices_size);
+
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu_multiple(
+            res, dev, reinterpret_cast<const Index*>(index));
+        *p_out = reinterpret_cast<FaissGpuIndex*>(gpu_index);
+    } CATCH_AND_HANDLE
+}
+
+int faiss_index_cpu_to_gpu_multiple_with_options(
+    FaissGpuResources** resources_vec, size_t resources_vec_size,
+    int* devices, size_t devices_size,
+    const FaissIndex* index, const FaissGpuMultipleClonerOptions* options,
+    FaissGpuIndex** p_out)
+{
+    try {
+        std::vector<GpuResources*> res(resources_vec_size);
+        for (auto i = 0u; i < resources_vec_size; ++i) {
+            res[i] = reinterpret_cast<GpuResources*>(resources_vec[i]);
+        }
+
+        std::vector<int> dev(devices, devices + devices_size);
+
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu_multiple(
+            res, dev, reinterpret_cast<const Index*>(index),
+            reinterpret_cast<const GpuMultipleClonerOptions*>(options));
+        *p_out = reinterpret_cast<FaissGpuIndex*>(gpu_index);
+    } CATCH_AND_HANDLE
+}

data/vendor/faiss/c_api/gpu/GpuAutoTune_c.h

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_GPU_AUTO_TUNE_C_H
+#define FAISS_GPU_AUTO_TUNE_C_H
+
+#include <stddef.h>
+#include "faiss_c.h"
+#include "GpuClonerOptions_c.h"
+#include "GpuResources_c.h"
+#include "GpuIndex_c.h"
+#include "Index_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// converts any GPU index inside gpu_index to a CPU index
+int faiss_index_gpu_to_cpu(const FaissIndex* gpu_index, FaissIndex** p_out);
+
+/// converts any CPU index that can be converted to GPU
+int faiss_index_cpu_to_gpu(
+    FaissGpuResources* resources, int device,
+    const FaissIndex *index, FaissGpuIndex** p_out);
+
+/// converts any CPU index that can be converted to GPU
+int faiss_index_cpu_to_gpu_with_options(
+    FaissGpuResources* resources, int device,
+    const FaissIndex *index, const FaissGpuClonerOptions* options,
+    FaissGpuIndex** p_out);
+
+/// converts any CPU index that can be converted to GPU
+int faiss_index_cpu_to_gpu_multiple(
+    FaissGpuResources* const* resources_vec, const int* devices, size_t devices_size,
+    const FaissIndex* index, FaissGpuIndex** p_out);
+
+/// converts any CPU index that can be converted to GPU
+int faiss_index_cpu_to_gpu_multiple_with_options(
+    FaissGpuResources* const* resources_vec, const int* devices, size_t devices_size,
+    const FaissIndex* index, const FaissGpuMultipleClonerOptions* options,
+    FaissGpuIndex** p_out);
+
+/// parameter space and setters for GPU indexes
+FAISS_DECLARE_CLASS_INHERITED(GpuParameterSpace, ParameterSpace)
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/vendor/faiss/c_api/gpu/GpuClonerOptions_c.cpp

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c++ -*-
+
+#include "GpuClonerOptions_c.h"
+#include "gpu/GpuClonerOptions.h"
+#include "macros_impl.h"
+
+using faiss::gpu::IndicesOptions;
+using faiss::gpu::GpuClonerOptions;
+using faiss::gpu::GpuMultipleClonerOptions;
+
+int faiss_GpuClonerOptions_new(FaissGpuClonerOptions** p) {
+    try {
+        *p = reinterpret_cast<FaissGpuClonerOptions*>(new GpuClonerOptions());
+    } CATCH_AND_HANDLE
+}
+
+int faiss_GpuMultipleClonerOptions_new(FaissGpuMultipleClonerOptions** p) {
+    try {
+        *p = reinterpret_cast<FaissGpuMultipleClonerOptions*>(new GpuMultipleClonerOptions());
+    } CATCH_AND_HANDLE
+}
+
+DEFINE_DESTRUCTOR(GpuClonerOptions)
+DEFINE_DESTRUCTOR(GpuMultipleClonerOptions)
+
+DEFINE_GETTER(GpuClonerOptions, FaissIndicesOptions, indicesOptions)
+DEFINE_GETTER(GpuClonerOptions, int, useFloat16CoarseQuantizer)
+DEFINE_GETTER(GpuClonerOptions, int, useFloat16)
+DEFINE_GETTER(GpuClonerOptions, int, usePrecomputed)
+DEFINE_GETTER(GpuClonerOptions, long, reserveVecs)
+DEFINE_GETTER(GpuClonerOptions, int, storeTransposed)
+DEFINE_GETTER(GpuClonerOptions, int, verbose)
+DEFINE_GETTER(GpuMultipleClonerOptions, int, shard)
+DEFINE_GETTER(GpuMultipleClonerOptions, int, shard_type)
+
+DEFINE_SETTER_STATIC(GpuClonerOptions, IndicesOptions, FaissIndicesOptions, indicesOptions)
+DEFINE_SETTER_STATIC(GpuClonerOptions, bool, int, useFloat16CoarseQuantizer)
+DEFINE_SETTER_STATIC(GpuClonerOptions, bool, int, useFloat16)
+DEFINE_SETTER_STATIC(GpuClonerOptions, bool, int, usePrecomputed)
+DEFINE_SETTER(GpuClonerOptions, long, reserveVecs)
+DEFINE_SETTER_STATIC(GpuClonerOptions, bool, int, storeTransposed)
+DEFINE_SETTER_STATIC(GpuClonerOptions, bool, int, verbose)
+DEFINE_SETTER_STATIC(GpuMultipleClonerOptions, bool, int, shard)
+DEFINE_SETTER(GpuMultipleClonerOptions, int, shard_type)

data/vendor/faiss/c_api/gpu/GpuClonerOptions_c.h

@@ -0,0 +1,68 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Copyright 2004-present Facebook. All Rights Reserved.
+// -*- c -*-
+
+#ifndef FAISS_GPU_CLONER_OPTIONS_C_H
+#define FAISS_GPU_CLONER_OPTIONS_C_H
+
+#include "faiss_c.h"
+#include "GpuIndicesOptions_c.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+FAISS_DECLARE_CLASS(GpuClonerOptions)
+
+FAISS_DECLARE_DESTRUCTOR(GpuClonerOptions)
+
+/// Default constructor for GpuClonerOptions
+int faiss_GpuClonerOptions_new(FaissGpuClonerOptions**);
+
+/// how should indices be stored on index types that support indices
+/// (anything but GpuIndexFlat*)?
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, FaissIndicesOptions, indicesOptions)
+
+/// (boolean) is the coarse quantizer in float16?
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, int, useFloat16CoarseQuantizer)
+
+/// (boolean) for GpuIndexIVFFlat, is storage in float16?
+/// for GpuIndexIVFPQ, are intermediate calculations in float16?
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, int, useFloat16)
+
+/// (boolean) use precomputed tables?
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, int, usePrecomputed)
+
+/// reserve vectors in the invfiles?
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, long, reserveVecs)
+
+/// (boolean) For GpuIndexFlat, store data in transposed layout?
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, int, storeTransposed)
+
+/// (boolean) Set verbose options on the index
+FAISS_DECLARE_GETTER_SETTER(GpuClonerOptions, int, verbose)
+
+FAISS_DECLARE_CLASS_INHERITED(GpuMultipleClonerOptions, GpuClonerOptions)
+
+FAISS_DECLARE_DESTRUCTOR(GpuMultipleClonerOptions)
+
+/// Default constructor for GpuMultipleClonerOptions
+int faiss_GpuMultipleClonerOptions_new(FaissGpuMultipleClonerOptions**);
+
+/// (boolean) Whether to shard the index across GPUs, versus replication
+/// across GPUs
+FAISS_DECLARE_GETTER_SETTER(GpuMultipleClonerOptions, int, shard)
+
+/// IndexIVF::copy_subset_to subset type
+FAISS_DECLARE_GETTER_SETTER(GpuMultipleClonerOptions, int, shard_type)
+
+#ifdef __cplusplus
+}
+#endif
+#endif