faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.h

@@ -10,28 +10,25 @@
 #ifndef FAISS_ON_DISK_INVERTED_LISTS_H
 #define FAISS_ON_DISK_INVERTED_LISTS_H
 
-#include <vector>
 #include <list>
 #include <typeinfo>
+#include <vector>
 
 #include <faiss/IndexIVF.h>
-#include <faiss/invlists/InvertedListsIOHook.h>
 #include <faiss/index_io.h>
+#include <faiss/invlists/InvertedListsIOHook.h>
 
 namespace faiss {
 
-
 struct LockLevels;
 
-
 struct OnDiskOneList {
     size_t size;     // size of inverted list (entries)
     size_t capacity; // allocated size (entries)
     size_t offset;   // offset in buffer (bytes)
-    OnDiskOneList ();
+    OnDiskOneList();
 };
 
-
 /** On-disk storage of inverted lists.
  *
  * The data is stored in a mmapped chunk of memory (base ptointer ptr,
@@ -60,17 +57,17 @@ struct OnDiskOneList {
  * to call prefetch_lists, that launches a set of threads to read the
  * lists in parallel.
  */
-struct OnDiskInvertedLists: InvertedLists {
+struct OnDiskInvertedLists : InvertedLists {
     using List = OnDiskOneList;
 
     // size nlist
     std::vector<List> lists;
 
     struct Slot {
-        size_t offset;    // bytes
-        size_t capacity;  // bytes
-        Slot (size_t offset, size_t capacity);
-        Slot ();
+        size_t offset;   // bytes
+        size_t capacity; // bytes
+        Slot(size_t offset, size_t capacity);
+        Slot();
     };
 
     // size whatever space remains
@@ -78,73 +75,81 @@ struct OnDiskInvertedLists: InvertedLists {
 
     std::string filename;
     size_t totsize;
-    uint8_t *ptr; // mmap base pointer
-    bool read_only;  /// are inverted lists mapped read-only
+    uint8_t* ptr;   // mmap base pointer
+    bool read_only; /// are inverted lists mapped read-only
 
-    OnDiskInvertedLists (size_t nlist, size_t code_size,
-                         const char *filename);
+    OnDiskInvertedLists(size_t nlist, size_t code_size, const char* filename);
 
     size_t list_size(size_t list_no) const override;
-    const uint8_t * get_codes (size_t list_no) const override;
-    const idx_t * get_ids (size_t list_no) const override;
+    const uint8_t* get_codes(size_t list_no) const override;
+    const idx_t* get_ids(size_t list_no) const override;
 
-    size_t add_entries (
-           size_t list_no, size_t n_entry,
-           const idx_t* ids, const uint8_t *code) override;
+    size_t add_entries(
+            size_t list_no,
+            size_t n_entry,
+            const idx_t* ids,
+            const uint8_t* code) override;
 
-    void update_entries (size_t list_no, size_t offset, size_t n_entry,
-                         const idx_t *ids, const uint8_t *code) override;
+    void update_entries(
+            size_t list_no,
+            size_t offset,
+            size_t n_entry,
+            const idx_t* ids,
+            const uint8_t* code) override;
 
-    void resize (size_t list_no, size_t new_size) override;
+    void resize(size_t list_no, size_t new_size) override;
 
     // copy all inverted lists into *this, in compact form (without
     // allocating slots)
-    size_t merge_from (const InvertedLists **ils, int n_il, bool verbose=false);
+    size_t merge_from(
+            const InvertedLists** ils,
+            int n_il,
+            bool verbose = false);
 
     /// same as merge_from for a single invlist
-    size_t merge_from_1 (const InvertedLists *il, bool verbose=false);
+    size_t merge_from_1(const InvertedLists* il, bool verbose = false);
 
     /// restrict the inverted lists to l0:l1 without touching the mmapped region
     void crop_invlists(size_t l0, size_t l1);
 
-    void prefetch_lists (const idx_t *list_nos, int nlist) const override;
+    void prefetch_lists(const idx_t* list_nos, int nlist) const override;
 
-    ~OnDiskInvertedLists () override;
+    ~OnDiskInvertedLists() override;
 
     // private
 
-    LockLevels * locks;
+    LockLevels* locks;
 
     // encapsulates the threads that are busy prefeteching
     struct OngoingPrefetch;
-    OngoingPrefetch *pf;
+    OngoingPrefetch* pf;
     int prefetch_nthread;
 
-    void do_mmap ();
-    void update_totsize (size_t new_totsize);
-    void resize_locked (size_t list_no, size_t new_size);
-    size_t allocate_slot (size_t capacity);
-    void free_slot (size_t offset, size_t capacity);
+    void do_mmap();
+    void update_totsize(size_t new_totsize);
+    void resize_locked(size_t list_no, size_t new_size);
+    size_t allocate_slot(size_t capacity);
+    void free_slot(size_t offset, size_t capacity);
 
     /// override all list sizes and make a packed storage
-    void set_all_lists_sizes(const size_t *sizes);
+    void set_all_lists_sizes(const size_t* sizes);
 
     // empty constructor for the I/O functions
-    OnDiskInvertedLists ();
+    OnDiskInvertedLists();
 };
 
-struct OnDiskInvertedListsIOHook: InvertedListsIOHook {
+struct OnDiskInvertedListsIOHook : InvertedListsIOHook {
     OnDiskInvertedListsIOHook();
-    void write(const InvertedLists *ils, IOWriter *f) const override;
-    InvertedLists * read(IOReader *f, int io_flags) const override;
-    InvertedLists * read_ArrayInvertedLists(
-            IOReader *f, int io_flags,
-            size_t nlist, size_t code_size,
-            const std::vector<size_t> &sizes) const override;
+    void write(const InvertedLists* ils, IOWriter* f) const override;
+    InvertedLists* read(IOReader* f, int io_flags) const override;
+    InvertedLists* read_ArrayInvertedLists(
+            IOReader* f,
+            int io_flags,
+            size_t nlist,
+            size_t code_size,
+            const std::vector<size_t>& sizes) const override;
 };
 
-
-
 } // namespace faiss
 
 #endif

data/vendor/faiss/faiss/python/python_callbacks.cpp

@@ -5,47 +5,46 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#include <faiss/python/python_callbacks.h>
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/python/python_callbacks.h>
 
 namespace {
 
-struct PyThreadLock  {
+struct PyThreadLock {
     PyGILState_STATE gstate;
 
-    PyThreadLock () {
+    PyThreadLock() {
         gstate = PyGILState_Ensure();
     }
 
-    ~PyThreadLock () {
+    ~PyThreadLock() {
         PyGILState_Release(gstate);
     }
 };
 
-
-};
-
+}; // namespace
 
 /***********************************************************
  * Callbacks for IO reader and writer
  ***********************************************************/
 
-PyCallbackIOWriter::PyCallbackIOWriter(PyObject *callback, size_t bs):
-    callback(callback), bs(bs) {
+PyCallbackIOWriter::PyCallbackIOWriter(PyObject* callback, size_t bs)
+        : callback(callback), bs(bs) {
     PyThreadLock gil;
     Py_INCREF(callback);
     name = "PyCallbackIOWriter";
 }
 
-
-
-size_t PyCallbackIOWriter::operator()(const void *ptrv, size_t size, size_t nitems) {
+size_t PyCallbackIOWriter::operator()(
+        const void* ptrv,
+        size_t size,
+        size_t nitems) {
     size_t ws = size * nitems;
-    const char *ptr = (const char*)ptrv;
+    const char* ptr = (const char*)ptrv;
     PyThreadLock gil;
-    while(ws > 0) {
+    while (ws > 0) {
         size_t wi = ws > bs ? bs : ws;
-        PyObject * result = PyObject_CallFunction(
+        PyObject* result = PyObject_CallFunction(
                 callback, "(N)", PyBytes_FromStringAndSize(ptr, wi));
         if (result == NULL) {
             FAISS_THROW_MSG("py err");
@@ -63,27 +62,25 @@ PyCallbackIOWriter::~PyCallbackIOWriter() {
     Py_DECREF(callback);
 }
 
-
-PyCallbackIOReader::PyCallbackIOReader(PyObject *callback, size_t bs):
-    callback(callback), bs(bs) {
+PyCallbackIOReader::PyCallbackIOReader(PyObject* callback, size_t bs)
+        : callback(callback), bs(bs) {
     PyThreadLock gil;
     Py_INCREF(callback);
     name = "PyCallbackIOReader";
 }
 
-size_t PyCallbackIOReader::operator()(void *ptrv, size_t size, size_t nitems)
-{
+size_t PyCallbackIOReader::operator()(void* ptrv, size_t size, size_t nitems) {
     size_t rs = size * nitems;
     size_t nb = 0;
-    char *ptr = (char*)ptrv;
+    char* ptr = (char*)ptrv;
     PyThreadLock gil;
-    while(rs > 0) {
+    while (rs > 0) {
         size_t ri = rs > bs ? bs : rs;
-        PyObject * result = PyObject_CallFunction(callback, "(n)", ri);
+        PyObject* result = PyObject_CallFunction(callback, "(n)", ri);
         if (result == NULL) {
             FAISS_THROW_MSG("propagate py error");
         }
-        if(!PyBytes_Check(result)) {
+        if (!PyBytes_Check(result)) {
             Py_DECREF(result);
             FAISS_THROW_MSG("read callback did not return a bytes object");
         }
@@ -95,8 +92,8 @@ size_t PyCallbackIOReader::operator()(void *ptrv, size_t size, size_t nitems)
         nb += sz;
         if (sz > rs) {
             Py_DECREF(result);
-            FAISS_THROW_FMT("read callback returned %zd bytes (asked %zd)",
-                            sz, rs);
+            FAISS_THROW_FMT(
+                    "read callback returned %zd bytes (asked %zd)", sz, rs);
         }
         memcpy(ptr, PyBytes_AsString(result), sz);
         Py_DECREF(result);

data/vendor/faiss/faiss/python/python_callbacks.h

@@ -7,46 +7,39 @@
 
 #pragma once
 
-#include "Python.h"
 #include <faiss/impl/io.h>
 #include <faiss/invlists/InvertedLists.h>
+#include "Python.h"
 
 //  all callbacks have to acquire the GIL on input
 
-
 /***********************************************************
  * Callbacks for IO reader and writer
  ***********************************************************/
 
-struct PyCallbackIOWriter: faiss::IOWriter {
-
-    PyObject * callback;
+struct PyCallbackIOWriter : faiss::IOWriter {
+    PyObject* callback;
     size_t bs; // maximum write size
 
     /** Callback: Python function that takes a bytes object and
      *  returns the number of bytes successfully written.
      */
-    explicit PyCallbackIOWriter(PyObject *callback,
-                                size_t bs = 1024 * 1024);
+    explicit PyCallbackIOWriter(PyObject* callback, size_t bs = 1024 * 1024);
 
-    size_t operator()(const void *ptrv, size_t size, size_t nitems) override;
+    size_t operator()(const void* ptrv, size_t size, size_t nitems) override;
 
     ~PyCallbackIOWriter() override;
-
 };
 
-
-struct PyCallbackIOReader: faiss::IOReader {
-    PyObject * callback;
+struct PyCallbackIOReader : faiss::IOReader {
+    PyObject* callback;
     size_t bs; // maximum buffer size
 
     /** Callback: Python function that takes a size and returns a
      * bytes object with the resulting read */
-    explicit PyCallbackIOReader(PyObject *callback,
-                                size_t bs = 1024 * 1024);
+    explicit PyCallbackIOReader(PyObject* callback, size_t bs = 1024 * 1024);
 
-    size_t operator()(void *ptrv, size_t size, size_t nitems) override;
+    size_t operator()(void* ptrv, size_t size, size_t nitems) override;
 
     ~PyCallbackIOReader() override;
-
 };

data/vendor/faiss/faiss/utils/AlignedTable.h

@@ -5,12 +5,11 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
+#include <cassert>
 #include <cstdint>
 #include <cstdlib>
-#include <cassert>
 #include <cstring>
 
 #include <algorithm>
@@ -19,33 +18,34 @@
 
 namespace faiss {
 
-template<int A=32>
-inline bool is_aligned_pointer(const void* x)
-{
+template <int A = 32>
+inline bool is_aligned_pointer(const void* x) {
     size_t xi = (size_t)x;
     return xi % A == 0;
 }
 
 // class that manages suitably aligned arrays for SIMD
 // T should be a POV type. The default alignment is 32 for AVX
-template<class T, int A=32>
+template <class T, int A = 32>
 struct AlignedTableTightAlloc {
-    T * ptr;
+    T* ptr;
     size_t numel;
 
-    AlignedTableTightAlloc(): ptr(nullptr), numel(0)
-    { }
+    AlignedTableTightAlloc() : ptr(nullptr), numel(0) {}
 
-    explicit AlignedTableTightAlloc(size_t n): ptr(nullptr), numel(0)
-    { resize(n); }
+    explicit AlignedTableTightAlloc(size_t n) : ptr(nullptr), numel(0) {
+        resize(n);
+    }
 
-    size_t itemsize() const {return sizeof(T); }
+    size_t itemsize() const {
+        return sizeof(T);
+    }
 
     void resize(size_t n) {
         if (numel == n) {
             return;
         }
-        T * new_ptr;
+        T* new_ptr;
         if (n > 0) {
             int ret = posix_memalign((void**)&new_ptr, A, n * sizeof(T));
             if (ret != 0) {
@@ -62,34 +62,54 @@ struct AlignedTableTightAlloc {
         ptr = new_ptr;
     }
 
-    void clear() {memset(ptr, 0, nbytes()); }
-    size_t size() const {return numel; }
-    size_t nbytes() const {return numel * sizeof(T); }
+    void clear() {
+        memset(ptr, 0, nbytes());
+    }
+    size_t size() const {
+        return numel;
+    }
+    size_t nbytes() const {
+        return numel * sizeof(T);
+    }
 
-    T * get() {return ptr; }
-    const T * get() const {return ptr; }
-    T * data() {return ptr; }
-    const T * data() const {return ptr; }
-    T & operator [] (size_t i)  {return ptr[i]; }
-    T operator [] (size_t i) const {return ptr[i]; }
+    T* get() {
+        return ptr;
+    }
+    const T* get() const {
+        return ptr;
+    }
+    T* data() {
+        return ptr;
+    }
+    const T* data() const {
+        return ptr;
+    }
+    T& operator[](size_t i) {
+        return ptr[i];
+    }
+    T operator[](size_t i) const {
+        return ptr[i];
+    }
 
-    ~AlignedTableTightAlloc() {posix_memalign_free(ptr); }
+    ~AlignedTableTightAlloc() {
+        posix_memalign_free(ptr);
+    }
 
-    AlignedTableTightAlloc<T, A> & operator =
-            (const AlignedTableTightAlloc<T, A> & other) {
+    AlignedTableTightAlloc<T, A>& operator=(
+            const AlignedTableTightAlloc<T, A>& other) {
         resize(other.numel);
         memcpy(ptr, other.ptr, sizeof(T) * numel);
         return *this;
     }
 
-    AlignedTableTightAlloc(const AlignedTableTightAlloc<T, A> & other) {
+    AlignedTableTightAlloc(const AlignedTableTightAlloc<T, A>& other)
+            : ptr(nullptr), numel(0) {
         *this = other;
     }
-
 };
 
 // same as AlignedTableTightAlloc, but with geometric re-allocation
-template<class T, int A=32>
+template <class T, int A = 32>
 struct AlignedTable {
     AlignedTableTightAlloc<T, A> tab;
     size_t numel = 0;
@@ -110,32 +130,47 @@ struct AlignedTable {
 
     AlignedTable() {}
 
-    explicit AlignedTable(size_t n):
-        tab(round_capacity(n)),
-        numel(n)
-    { }
+    explicit AlignedTable(size_t n) : tab(round_capacity(n)), numel(n) {}
 
-    size_t itemsize() const {return sizeof(T); }
+    size_t itemsize() const {
+        return sizeof(T);
+    }
 
     void resize(size_t n) {
         tab.resize(round_capacity(n));
         numel = n;
     }
 
-    void clear() { tab.clear(); }
-    size_t size() const {return numel; }
-    size_t nbytes() const {return numel * sizeof(T); }
+    void clear() {
+        tab.clear();
+    }
+    size_t size() const {
+        return numel;
+    }
+    size_t nbytes() const {
+        return numel * sizeof(T);
+    }
 
-    T * get() {return tab.get(); }
-    const T * get() const {return tab.get(); }
-    T * data() {return tab.get(); }
-    const T * data() const {return tab.get(); }
-    T & operator [] (size_t i)  {return tab.ptr[i]; }
-    T operator [] (size_t i) const {return tab.ptr[i]; }
+    T* get() {
+        return tab.get();
+    }
+    const T* get() const {
+        return tab.get();
+    }
+    T* data() {
+        return tab.get();
+    }
+    const T* data() const {
+        return tab.get();
+    }
+    T& operator[](size_t i) {
+        return tab.ptr[i];
+    }
+    T operator[](size_t i) const {
+        return tab.ptr[i];
+    }
 
     // assign and copy constructor should work as expected
-
 };
 
-
 } // namespace faiss

data/vendor/faiss/faiss/utils/Heap.cpp

@@ -11,86 +11,82 @@
 
 #include <faiss/utils/Heap.h>
 
-
 namespace faiss {
 
-
 template <typename C>
-void HeapArray<C>::heapify ()
-{
+void HeapArray<C>::heapify() {
 #pragma omp parallel for
     for (int64_t j = 0; j < nh; j++)
-        heap_heapify<C> (k, val + j * k, ids + j * k);
+        heap_heapify<C>(k, val + j * k, ids + j * k);
 }
 
 template <typename C>
-void HeapArray<C>::reorder ()
-{
+void HeapArray<C>::reorder() {
 #pragma omp parallel for
     for (int64_t j = 0; j < nh; j++)
-        heap_reorder<C> (k, val + j * k, ids + j * k);
+        heap_reorder<C>(k, val + j * k, ids + j * k);
 }
 
 template <typename C>
-void HeapArray<C>::addn (size_t nj, const T *vin, TI j0,
-                         size_t i0, int64_t ni)
-{
-    if (ni == -1) ni = nh;
-    assert (i0 >= 0 && i0 + ni <= nh);
+void HeapArray<C>::addn(size_t nj, const T* vin, TI j0, size_t i0, int64_t ni) {
+    if (ni == -1)
+        ni = nh;
+    assert(i0 >= 0 && i0 + ni <= nh);
 #pragma omp parallel for
     for (int64_t i = i0; i < i0 + ni; i++) {
-        T * __restrict simi = get_val(i);
-        TI * __restrict idxi = get_ids (i);
-        const T *ip_line = vin + (i - i0) * nj;
+        T* __restrict simi = get_val(i);
+        TI* __restrict idxi = get_ids(i);
+        const T* ip_line = vin + (i - i0) * nj;
 
         for (size_t j = 0; j < nj; j++) {
-            T ip = ip_line [j];
+            T ip = ip_line[j];
             if (C::cmp(simi[0], ip)) {
-                heap_replace_top<C> (k, simi, idxi, ip, j + j0);
+                heap_replace_top<C>(k, simi, idxi, ip, j + j0);
             }
         }
     }
 }
 
 template <typename C>
-void HeapArray<C>::addn_with_ids (
-     size_t nj, const T *vin, const TI *id_in,
-     int64_t id_stride, size_t i0, int64_t ni)
-{
+void HeapArray<C>::addn_with_ids(
+        size_t nj,
+        const T* vin,
+        const TI* id_in,
+        int64_t id_stride,
+        size_t i0,
+        int64_t ni) {
     if (id_in == nullptr) {
-        addn (nj, vin, 0, i0, ni);
+        addn(nj, vin, 0, i0, ni);
         return;
     }
-    if (ni == -1) ni = nh;
-    assert (i0 >= 0 && i0 + ni <= nh);
+    if (ni == -1)
+        ni = nh;
+    assert(i0 >= 0 && i0 + ni <= nh);
 #pragma omp parallel for
     for (int64_t i = i0; i < i0 + ni; i++) {
-        T * __restrict simi = get_val(i);
-        TI * __restrict idxi = get_ids (i);
-        const T *ip_line = vin + (i - i0) * nj;
-        const TI *id_line = id_in + (i - i0) * id_stride;
+        T* __restrict simi = get_val(i);
+        TI* __restrict idxi = get_ids(i);
+        const T* ip_line = vin + (i - i0) * nj;
+        const TI* id_line = id_in + (i - i0) * id_stride;
 
         for (size_t j = 0; j < nj; j++) {
-            T ip = ip_line [j];
+            T ip = ip_line[j];
             if (C::cmp(simi[0], ip)) {
-                heap_replace_top<C> (k, simi, idxi, ip, id_line [j]);
+                heap_replace_top<C>(k, simi, idxi, ip, id_line[j]);
             }
         }
     }
 }
 
 template <typename C>
-void HeapArray<C>::per_line_extrema (
-                   T * out_val,
-                   TI * out_ids) const
-{
+void HeapArray<C>::per_line_extrema(T* out_val, TI* out_ids) const {
 #pragma omp parallel for
     for (int64_t j = 0; j < nh; j++) {
         int64_t imin = -1;
-        typename C::T xval = C::Crev::neutral ();
-        const typename C::T * x_ = val + j * k;
+        typename C::T xval = C::Crev::neutral();
+        const typename C::T* x_ = val + j * k;
         for (size_t i = 0; i < k; i++)
-            if (C::cmp (x_[i], xval)) {
+            if (C::cmp(x_[i], xval)) {
                 xval = x_[i];
                 imin = i;
             }
@@ -99,22 +95,18 @@ void HeapArray<C>::per_line_extrema (
 
         if (out_ids) {
             if (ids && imin != -1)
-                out_ids[j] = ids [j * k + imin];
+                out_ids[j] = ids[j * k + imin];
             else
                 out_ids[j] = imin;
         }
     }
 }
 
-
-
-
 // explicit instanciations
 
-template struct HeapArray<CMin <float, int64_t> >;
-template struct HeapArray<CMax <float, int64_t> >;
-template struct HeapArray<CMin <int, int64_t> >;
-template struct HeapArray<CMax <int, int64_t> >;
-
+template struct HeapArray<CMin<float, int64_t>>;
+template struct HeapArray<CMax<float, int64_t>>;
+template struct HeapArray<CMin<int, int64_t>>;
+template struct HeapArray<CMax<int, int64_t>>;
 
-}  // END namespace fasis
+} // namespace faiss