faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/impl/AuxIndexStructures.h

@@ -15,15 +15,15 @@
 
 #include <stdint.h>
 
-#include <vector>
-#include <unordered_set>
+#include <cstring>
 #include <memory>
 #include <mutex>
+#include <unordered_set>
+#include <vector>
 
 #include <faiss/Index.h>
 #include <faiss/impl/platform_macros.h>
 
-
 namespace faiss {
 
 /** The objective is to have a simple result structure while
@@ -31,42 +31,39 @@ namespace faiss {
  *  do_allocation can be overloaded to allocate the result tables in
  *  the matrix type of a scripting language like Lua or Python. */
 struct RangeSearchResult {
-    size_t nq;      ///< nb of queries
-    size_t *lims;   ///< size (nq + 1)
+    size_t nq;    ///< nb of queries
+    size_t* lims; ///< size (nq + 1)
 
     typedef Index::idx_t idx_t;
 
-    idx_t *labels;     ///< result for query i is labels[lims[i]:lims[i+1]]
-    float *distances;  ///< corresponding distances (not sorted)
+    idx_t* labels;    ///< result for query i is labels[lims[i]:lims[i+1]]
+    float* distances; ///< corresponding distances (not sorted)
 
     size_t buffer_size; ///< size of the result buffers used
 
     /// lims must be allocated on input to range_search.
-    explicit RangeSearchResult (idx_t nq, bool alloc_lims=true);
+    explicit RangeSearchResult(idx_t nq, bool alloc_lims = true);
 
     /// called when lims contains the nb of elements result entries
     /// for each query
 
-    virtual void do_allocation ();
+    virtual void do_allocation();
 
-    virtual ~RangeSearchResult ();
+    virtual ~RangeSearchResult();
 };
 
-
 /** Encapsulates a set of ids to remove. */
 struct IDSelector {
     typedef Index::idx_t idx_t;
-    virtual bool is_member (idx_t id) const = 0;
+    virtual bool is_member(idx_t id) const = 0;
     virtual ~IDSelector() {}
 };
 
-
-
 /** remove ids between [imni, imax) */
-struct IDSelectorRange: IDSelector {
+struct IDSelectorRange : IDSelector {
     idx_t imin, imax;
 
-    IDSelectorRange (idx_t imin, idx_t imax);
+    IDSelectorRange(idx_t imin, idx_t imax);
     bool is_member(idx_t id) const override;
     ~IDSelectorRange() override {}
 };
@@ -76,11 +73,11 @@ struct IDSelectorRange: IDSelector {
  * this is inefficient in most cases, except for IndexIVF with
  * maintain_direct_map
  */
-struct IDSelectorArray: IDSelector {
+struct IDSelectorArray : IDSelector {
     size_t n;
-    const idx_t *ids;
+    const idx_t* ids;
 
-    IDSelectorArray (size_t n, const idx_t *ids);
+    IDSelectorArray(size_t n, const idx_t* ids);
     bool is_member(idx_t id) const override;
     ~IDSelectorArray() override {}
 };
@@ -91,8 +88,7 @@ struct IDSelectorArray: IDSelector {
  * unordered_set are just the least significant bits of the id. This
  * works fine for random ids or ids in sequences but will produce many
  * hash collisions if lsb's are always the same */
-struct IDSelectorBatch: IDSelector {
-
+struct IDSelectorBatch : IDSelector {
     std::unordered_set<idx_t> set;
 
     typedef unsigned char uint8_t;
@@ -100,7 +96,7 @@ struct IDSelectorBatch: IDSelector {
     int nbits;
     idx_t mask;
 
-    IDSelectorBatch (size_t n, const idx_t *indices);
+    IDSelectorBatch(size_t n, const idx_t* indices);
     bool is_member(idx_t id) const override;
     ~IDSelectorBatch() override {}
 };
@@ -124,28 +120,26 @@ struct BufferList {
     size_t buffer_size;
 
     struct Buffer {
-        idx_t *ids;
-        float *dis;
+        idx_t* ids;
+        float* dis;
     };
 
     std::vector<Buffer> buffers;
     size_t wp; ///< write pointer in the last buffer.
 
-    explicit BufferList (size_t buffer_size);
+    explicit BufferList(size_t buffer_size);
 
-    ~BufferList ();
+    ~BufferList();
 
     /// create a new buffer
-    void append_buffer ();
+    void append_buffer();
 
     /// add one result, possibly appending a new buffer if needed
-    void add (idx_t id, float dis);
+    void add(idx_t id, float dis);
 
     /// copy elemnts ofs:ofs+n-1 seen as linear data in the buffers to
     /// tables dest_ids, dest_dis
-    void copy_range (size_t ofs, size_t n,
-                     idx_t * dest_ids, float *dest_dis);
-
+    void copy_range(size_t ofs, size_t n, idx_t* dest_ids, float* dest_dis);
 };
 
 struct RangeSearchPartialResult;
@@ -153,46 +147,45 @@ struct RangeSearchPartialResult;
 /// result structure for a single query
 struct RangeQueryResult {
     using idx_t = Index::idx_t;
-    idx_t qno;    //< id of the query
-    size_t nres;  //< nb of results for this query
-    RangeSearchPartialResult * pres;
+    idx_t qno;   //< id of the query
+    size_t nres; //< nb of results for this query
+    RangeSearchPartialResult* pres;
 
     /// called by search function to report a new result
-    void add (float dis, idx_t id);
+    void add(float dis, idx_t id);
 };
 
 /// the entries in the buffers are split per query
-struct RangeSearchPartialResult: BufferList {
-    RangeSearchResult * res;
+struct RangeSearchPartialResult : BufferList {
+    RangeSearchResult* res;
 
     /// eventually the result will be stored in res_in
-    explicit RangeSearchPartialResult (RangeSearchResult * res_in);
+    explicit RangeSearchPartialResult(RangeSearchResult* res_in);
 
     /// query ids + nb of results per query.
     std::vector<RangeQueryResult> queries;
 
     /// begin a new result
-    RangeQueryResult & new_result (idx_t qno);
+    RangeQueryResult& new_result(idx_t qno);
 
     /*****************************************
      * functions used at the end of the search to merge the result
      * lists */
-    void finalize ();
+    void finalize();
 
     /// called by range_search before do_allocation
-    void set_lims ();
+    void set_lims();
 
     /// called by range_search after do_allocation
-    void copy_result (bool incremental = false);
+    void copy_result(bool incremental = false);
 
     /// merge a set of PartialResult's into one RangeSearchResult
     /// on ouptut the partialresults are empty!
-    static void merge (std::vector <RangeSearchPartialResult *> &
-                       partial_results, bool do_delete=true);
-
+    static void merge(
+            std::vector<RangeSearchPartialResult*>& partial_results,
+            bool do_delete = true);
 };
 
-
 /***********************************************************
  * The distance computer maintains a current query and computes
  * distances to elements in an index that supports random access.
@@ -202,19 +195,19 @@ struct RangeSearchPartialResult: BufferList {
  * instantiate one from each thread if needed.
  ***********************************************************/
 struct DistanceComputer {
-     using idx_t = Index::idx_t;
+    using idx_t = Index::idx_t;
 
-     /// called before computing distances. Pointer x should remain valid
-     /// while operator () is called
-     virtual void set_query(const float *x) = 0;
+    /// called before computing distances. Pointer x should remain valid
+    /// while operator () is called
+    virtual void set_query(const float* x) = 0;
 
-     /// compute distance of vector i to current query
-     virtual float operator () (idx_t i) = 0;
+    /// compute distance of vector i to current query
+    virtual float operator()(idx_t i) = 0;
 
-     /// compute distance between two stored vectors
-     virtual float symmetric_dis (idx_t i, idx_t j) = 0;
+    /// compute distance between two stored vectors
+    virtual float symmetric_dis(idx_t i, idx_t j) = 0;
 
-     virtual ~DistanceComputer() {}
+    virtual ~DistanceComputer() {}
 };
 
 /***********************************************************
@@ -222,7 +215,7 @@ struct DistanceComputer {
  ***********************************************************/
 
 struct FAISS_API InterruptCallback {
-    virtual bool want_interrupt () = 0;
+    virtual bool want_interrupt() = 0;
     virtual ~InterruptCallback() {}
 
     // lock that protects concurrent calls to is_interrupted
@@ -230,7 +223,7 @@ struct FAISS_API InterruptCallback {
 
     static std::unique_ptr<InterruptCallback> instance;
 
-    static void clear_instance ();
+    static void clear_instance();
 
     /** check if:
      * - an interrupt callback is set
@@ -238,23 +231,46 @@ struct FAISS_API InterruptCallback {
      * if this is the case, then throw an exception. Should not be called
      * from multiple threads.
      */
-    static void check ();
+    static void check();
 
     /// same as check() but return true if is interrupted instead of
     /// throwing. Can be called from multiple threads.
-    static bool is_interrupted ();
+    static bool is_interrupted();
 
     /** assuming each iteration takes a certain number of flops, what
      * is a reasonable interval to check for interrupts?
      */
-    static size_t get_period_hint (size_t flops);
-
+    static size_t get_period_hint(size_t flops);
 };
 
+/// set implementation optimized for fast access.
+struct VisitedTable {
+    std::vector<uint8_t> visited;
+    int visno;
+
+    explicit VisitedTable(int size) : visited(size), visno(1) {}
+
+    /// set flag #no to true
+    void set(int no) {
+        visited[no] = visno;
+    }
+
+    /// get flag #no
+    bool get(int no) const {
+        return visited[no] == visno;
+    }
+
+    /// reset all flags to false
+    void advance() {
+        visno++;
+        if (visno == 250) {
+            // 250 rather than 255 because sometimes we use visno and visno+1
+            memset(visited.data(), 0, sizeof(visited[0]) * visited.size());
+            visno = 1;
+        }
+    }
+};
 
-
-}; // namespace faiss
-
-
+} // namespace faiss
 
 #endif

data/vendor/faiss/faiss/impl/FaissAssert.h

@@ -12,85 +12,100 @@
 
 #include <faiss/impl/FaissException.h>
 #include <faiss/impl/platform_macros.h>
-#include <cstdlib>
 #include <cstdio>
+#include <cstdlib>
 #include <string>
 
 ///
 /// Assertions
 ///
 
-#define FAISS_ASSERT(X)                                                 \
-  do {                                                                  \
-    if (! (X)) {                                                        \
-      fprintf(stderr, "Faiss assertion '%s' failed in %s "              \
-               "at %s:%d\n",                                            \
-               #X, __PRETTY_FUNCTION__, __FILE__, __LINE__);            \
-      abort();                                                          \
-    }                                                                   \
-  } while (false)
+#define FAISS_ASSERT(X)                                  \
+    do {                                                 \
+        if (!(X)) {                                      \
+            fprintf(stderr,                              \
+                    "Faiss assertion '%s' failed in %s " \
+                    "at %s:%d\n",                        \
+                    #X,                                  \
+                    __PRETTY_FUNCTION__,                 \
+                    __FILE__,                            \
+                    __LINE__);                           \
+            abort();                                     \
+        }                                                \
+    } while (false)
 
-#define FAISS_ASSERT_MSG(X, MSG)                                        \
-  do {                                                                  \
-    if (! (X)) {                                                        \
-      fprintf(stderr, "Faiss assertion '%s' failed in %s "              \
-               "at %s:%d; details: " MSG "\n",                          \
-               #X, __PRETTY_FUNCTION__, __FILE__, __LINE__);            \
-      abort();                                                          \
-    }                                                                   \
-  } while (false)
+#define FAISS_ASSERT_MSG(X, MSG)                         \
+    do {                                                 \
+        if (!(X)) {                                      \
+            fprintf(stderr,                              \
+                    "Faiss assertion '%s' failed in %s " \
+                    "at %s:%d; details: " MSG "\n",      \
+                    #X,                                  \
+                    __PRETTY_FUNCTION__,                 \
+                    __FILE__,                            \
+                    __LINE__);                           \
+            abort();                                     \
+        }                                                \
+    } while (false)
 
-#define FAISS_ASSERT_FMT(X, FMT, ...)                                   \
-  do {                                                                  \
-    if (! (X)) {                                                        \
-      fprintf(stderr, "Faiss assertion '%s' failed in %s "              \
-               "at %s:%d; details: " FMT "\n",                          \
-               #X, __PRETTY_FUNCTION__, __FILE__, __LINE__, __VA_ARGS__); \
-      abort();                                                          \
-    }                                                                   \
-  } while (false)
+#define FAISS_ASSERT_FMT(X, FMT, ...)                    \
+    do {                                                 \
+        if (!(X)) {                                      \
+            fprintf(stderr,                              \
+                    "Faiss assertion '%s' failed in %s " \
+                    "at %s:%d; details: " FMT "\n",      \
+                    #X,                                  \
+                    __PRETTY_FUNCTION__,                 \
+                    __FILE__,                            \
+                    __LINE__,                            \
+                    __VA_ARGS__);                        \
+            abort();                                     \
+        }                                                \
+    } while (false)
 
 ///
 /// Exceptions for returning user errors
 ///
 
-#define FAISS_THROW_MSG(MSG)                                            \
-  do {                                                                  \
-    throw faiss::FaissException(MSG, __PRETTY_FUNCTION__, __FILE__, __LINE__); \
-  } while (false)
+#define FAISS_THROW_MSG(MSG)                                   \
+    do {                                                       \
+        throw faiss::FaissException(                           \
+                MSG, __PRETTY_FUNCTION__, __FILE__, __LINE__); \
+    } while (false)
 
-#define FAISS_THROW_FMT(FMT, ...)                                       \
-  do {                                                                  \
-    std::string __s;                                                    \
-    int __size = snprintf(nullptr, 0, FMT, __VA_ARGS__);                \
-    __s.resize(__size + 1);                                             \
-    snprintf(&__s[0], __s.size(), FMT, __VA_ARGS__);                    \
-    throw faiss::FaissException(__s, __PRETTY_FUNCTION__, __FILE__, __LINE__); \
-  } while (false)
+#define FAISS_THROW_FMT(FMT, ...)                              \
+    do {                                                       \
+        std::string __s;                                       \
+        int __size = snprintf(nullptr, 0, FMT, __VA_ARGS__);   \
+        __s.resize(__size + 1);                                \
+        snprintf(&__s[0], __s.size(), FMT, __VA_ARGS__);       \
+        throw faiss::FaissException(                           \
+                __s, __PRETTY_FUNCTION__, __FILE__, __LINE__); \
+    } while (false)
 
 ///
 /// Exceptions thrown upon a conditional failure
 ///
 
-#define FAISS_THROW_IF_NOT(X)                           \
-  do {                                                  \
-    if (!(X)) {                                         \
-      FAISS_THROW_FMT("Error: '%s' failed", #X);        \
-    }                                                   \
-  } while (false)
+#define FAISS_THROW_IF_NOT(X)                          \
+    do {                                               \
+        if (!(X)) {                                    \
+            FAISS_THROW_FMT("Error: '%s' failed", #X); \
+        }                                              \
+    } while (false)
 
-#define FAISS_THROW_IF_NOT_MSG(X, MSG)                  \
-  do {                                                  \
-    if (!(X)) {                                         \
-      FAISS_THROW_FMT("Error: '%s' failed: " MSG, #X);  \
-    }                                                   \
-  } while (false)
+#define FAISS_THROW_IF_NOT_MSG(X, MSG)                       \
+    do {                                                     \
+        if (!(X)) {                                          \
+            FAISS_THROW_FMT("Error: '%s' failed: " MSG, #X); \
+        }                                                    \
+    } while (false)
 
-#define FAISS_THROW_IF_NOT_FMT(X, FMT, ...)                             \
-  do {                                                                  \
-    if (!(X)) {                                                         \
-      FAISS_THROW_FMT("Error: '%s' failed: " FMT, #X, __VA_ARGS__);     \
-    }                                                                   \
-  } while (false)
+#define FAISS_THROW_IF_NOT_FMT(X, FMT, ...)                               \
+    do {                                                                  \
+        if (!(X)) {                                                       \
+            FAISS_THROW_FMT("Error: '%s' failed: " FMT, #X, __VA_ARGS__); \
+        }                                                                 \
+    } while (false)
 
 #endif

data/vendor/faiss/faiss/impl/FaissException.cpp

@@ -10,71 +10,81 @@
 #include <faiss/impl/FaissException.h>
 #include <sstream>
 
-#ifdef  __GNUG__
+#ifdef __GNUG__
 #include <cxxabi.h>
 #endif
 
 namespace faiss {
 
-FaissException::FaissException(const std::string& m)
-    : msg(m) {
-}
-
-FaissException::FaissException(const std::string& m,
-                               const char* funcName,
-                               const char* file,
-                               int line) {
-  int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s",
-                      funcName, file, line, m.c_str());
-  msg.resize(size + 1);
-  snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s",
-           funcName, file, line, m.c_str());
+FaissException::FaissException(const std::string& m) : msg(m) {}
+
+FaissException::FaissException(
+        const std::string& m,
+        const char* funcName,
+        const char* file,
+        int line) {
+    int size = snprintf(
+            nullptr,
+            0,
+            "Error in %s at %s:%d: %s",
+            funcName,
+            file,
+            line,
+            m.c_str());
+    msg.resize(size + 1);
+    snprintf(
+            &msg[0],
+            msg.size(),
+            "Error in %s at %s:%d: %s",
+            funcName,
+            file,
+            line,
+            m.c_str());
 }
 
-const char*
-FaissException::what() const noexcept {
-  return msg.c_str();
+const char* FaissException::what() const noexcept {
+    return msg.c_str();
 }
 
 void handleExceptions(
-  std::vector<std::pair<int, std::exception_ptr>>& exceptions) {
-  if (exceptions.size() == 1) {
-    // throw the single received exception directly
-    std::rethrow_exception(exceptions.front().second);
-
-  } else if (exceptions.size() > 1) {
-    // multiple exceptions; aggregate them and return a single exception
-    std::stringstream ss;
-
-    for (auto& p : exceptions) {
-      try {
-        std::rethrow_exception(p.second);
-      } catch (std::exception& ex) {
-        if (ex.what()) {
-          // exception message available
-          ss << "Exception thrown from index " << p.first << ": "
-             << ex.what() << "\n";
-        } else {
-          // No message available
-          ss << "Unknown exception thrown from index " << p.first << "\n";
+        std::vector<std::pair<int, std::exception_ptr>>& exceptions) {
+    if (exceptions.size() == 1) {
+        // throw the single received exception directly
+        std::rethrow_exception(exceptions.front().second);
+
+    } else if (exceptions.size() > 1) {
+        // multiple exceptions; aggregate them and return a single exception
+        std::stringstream ss;
+
+        for (auto& p : exceptions) {
+            try {
+                std::rethrow_exception(p.second);
+            } catch (std::exception& ex) {
+                if (ex.what()) {
+                    // exception message available
+                    ss << "Exception thrown from index " << p.first << ": "
+                       << ex.what() << "\n";
+                } else {
+                    // No message available
+                    ss << "Unknown exception thrown from index " << p.first
+                       << "\n";
+                }
+            } catch (...) {
+                ss << "Unknown exception thrown from index " << p.first << "\n";
+            }
         }
-      } catch (...) {
-        ss << "Unknown exception thrown from index " << p.first << "\n";
-      }
-    }
 
-    throw FaissException(ss.str());
-  }
+        throw FaissException(ss.str());
+    }
 }
 
-
 // From
 // https://stackoverflow.com/questions/281818/unmangling-the-result-of-stdtype-infoname
 
 std::string demangle_cpp_symbol(const char* name) {
 #ifdef __GNUG__
     int status = -1;
-    const char * res = abi::__cxa_demangle(name, nullptr, nullptr, &status);
+    const char* res = abi::__cxa_demangle(name, nullptr, nullptr, &status);
     std::string sres;
     if (status == 0) {
         sres = res;
@@ -87,6 +97,4 @@ std::string demangle_cpp_symbol(const char* name) {
 #endif
 }
 
-
-
-} // namespace
+} // namespace faiss