faiss 0.1.5 → 0.2.2

data/vendor/faiss/faiss/impl/lattice_Zn.h

@@ -9,9 +9,9 @@
 #ifndef FAISS_LATTICE_ZN_H
 #define FAISS_LATTICE_ZN_H
 
-#include <vector>
 #include <stddef.h>
 #include <stdint.h>
+#include <vector>
 
 namespace faiss {
 
@@ -32,23 +32,20 @@ struct ZnSphereSearch {
     ZnSphereSearch(int dim, int r2);
 
     /// find nearest centroid. x does not need to be normalized
-    float search(const float *x, float *c) const;
+    float search(const float* x, float* c) const;
 
     /// full call. Requires externally-allocated temp space
-    float search(const float *x, float *c,
-                 float *tmp, // size 2 *dim
-                 int *tmp_int, // size dim
-                 int *ibest_out = nullptr
-                 ) const;
+    float search(
+            const float* x,
+            float* c,
+            float* tmp,   // size 2 *dim
+            int* tmp_int, // size dim
+            int* ibest_out = nullptr) const;
 
     // multi-threaded
-    void search_multi(int n, const float *x,
-                      float *c_out,
-                      float *dp_out);
-
+    void search_multi(int n, const float* x, float* c_out, float* dp_out);
 };
 
-
 /***************************************************************************
  * Support ids as well.
  *
@@ -60,30 +57,31 @@ struct EnumeratedVectors {
     uint64_t nv;
     int dim;
 
-    explicit EnumeratedVectors(int dim): nv(0), dim(dim) {}
+    explicit EnumeratedVectors(int dim) : nv(0), dim(dim) {}
 
     /// encode a vector from a collection
-    virtual uint64_t encode(const float *x) const = 0;
+    virtual uint64_t encode(const float* x) const = 0;
 
     /// decode it
-    virtual void decode(uint64_t code, float *c) const = 0;
+    virtual void decode(uint64_t code, float* c) const = 0;
 
     // call encode on nc vectors
-    void encode_multi (size_t nc, const float *c,
-                       uint64_t * codes) const;
+    void encode_multi(size_t nc, const float* c, uint64_t* codes) const;
 
     // call decode on nc codes
-    void decode_multi (size_t nc, const uint64_t * codes,
-                       float *c) const;
+    void decode_multi(size_t nc, const uint64_t* codes, float* c) const;
 
     // find the nearest neighbor of each xq
     // (decodes and computes distances)
-    void find_nn (size_t n, const uint64_t * codes,
-                  size_t nq, const float *xq,
-                  int64_t *idx, float *dis);
+    void find_nn(
+            size_t n,
+            const uint64_t* codes,
+            size_t nq,
+            const float* xq,
+            int64_t* idx,
+            float* dis);
 
     virtual ~EnumeratedVectors() {}
-
 };
 
 struct Repeat {
@@ -100,26 +98,24 @@ struct Repeats {
     std::vector<Repeat> repeats;
 
     // initialize from a template of the atom.
-    Repeats(int dim = 0, const float *c = nullptr);
+    Repeats(int dim = 0, const float* c = nullptr);
 
     // count number of possible codes for this atom
     uint64_t count() const;
 
-    uint64_t encode(const float *c) const;
+    uint64_t encode(const float* c) const;
 
-    void decode(uint64_t code, float *c) const;
+    void decode(uint64_t code, float* c) const;
 };
 
-
 /** codec that can return ids for the encoded vectors
  *
  * uses the ZnSphereSearch to encode the vector by encoding the
  * permutation and signs. Depends on ZnSphereSearch because it uses
  * the atom numbers */
-struct ZnSphereCodec: ZnSphereSearch, EnumeratedVectors {
-
-    struct CodeSegment:Repeats {
-        explicit CodeSegment(const Repeats & r): Repeats(r) {}
+struct ZnSphereCodec : ZnSphereSearch, EnumeratedVectors {
+    struct CodeSegment : Repeats {
+        explicit CodeSegment(const Repeats& r) : Repeats(r) {}
         uint64_t c0; // first code assigned to segment
         int signbits;
     };
@@ -130,13 +126,12 @@ struct ZnSphereCodec: ZnSphereSearch, EnumeratedVectors {
 
     ZnSphereCodec(int dim, int r2);
 
-    uint64_t search_and_encode(const float *x) const;
+    uint64_t search_and_encode(const float* x) const;
 
-    void decode(uint64_t code, float *c) const override;
+    void decode(uint64_t code, float* c) const override;
 
     /// takes vectors that do not need to be centroids
-    uint64_t encode(const float *x) const override;
-
+    uint64_t encode(const float* x) const override;
 };
 
 /** recursive sphere codec
@@ -145,8 +140,7 @@ struct ZnSphereCodec: ZnSphereSearch, EnumeratedVectors {
  * centroids found by the ZnSphereSearch. The codes are *not*
  * compatible with the ones of ZnSpehreCodec
  */
-struct ZnSphereCodecRec: EnumeratedVectors {
-
+struct ZnSphereCodecRec : EnumeratedVectors {
     int r2;
 
     int log2_dim;
@@ -154,19 +148,19 @@ struct ZnSphereCodecRec: EnumeratedVectors {
 
     ZnSphereCodecRec(int dim, int r2);
 
-    uint64_t encode_centroid(const float *c) const;
+    uint64_t encode_centroid(const float* c) const;
 
-    void decode(uint64_t code, float *c) const override;
+    void decode(uint64_t code, float* c) const override;
 
     /// vectors need to be centroids (does not work on arbitrary
     /// vectors)
-    uint64_t encode(const float *x) const override;
+    uint64_t encode(const float* x) const override;
 
     std::vector<uint64_t> all_nv;
     std::vector<uint64_t> all_nv_cum;
 
     int decode_cache_ld;
-    std::vector<std::vector<float> > decode_cache;
+    std::vector<std::vector<float>> decode_cache;
 
     // nb of vectors in the sphere in dim 2^ld with r2 radius
     uint64_t get_nv(int ld, int r2a) const;
@@ -174,26 +168,21 @@ struct ZnSphereCodecRec: EnumeratedVectors {
     // cumulative version
     uint64_t get_nv_cum(int ld, int r2t, int r2a) const;
     void set_nv_cum(int ld, int r2t, int r2a, uint64_t v);
-
 };
 
-
 /** Codec that uses the recursive codec if dim is a power of 2 and
  * the regular one otherwise */
-struct ZnSphereCodecAlt: ZnSphereCodec {
+struct ZnSphereCodecAlt : ZnSphereCodec {
     bool use_rec;
     ZnSphereCodecRec znc_rec;
 
-    ZnSphereCodecAlt (int dim, int r2);
-
-    uint64_t encode(const float *x) const override;
-
-    void decode(uint64_t code, float *c) const override;
-
-};
+    ZnSphereCodecAlt(int dim, int r2);
 
+    uint64_t encode(const float* x) const override;
 
+    void decode(uint64_t code, float* c) const override;
 };
 
+} // namespace faiss
 
 #endif

data/vendor/faiss/faiss/impl/platform_macros.h

@@ -7,14 +7,12 @@
 
 #pragma once
 
-
 #ifdef _MSC_VER
 
 /*******************************************************
  * Windows specific macros
  *******************************************************/
 
-
 #ifdef FAISS_MAIN_LIB
 #define FAISS_API __declspec(dllexport)
 #else // _FAISS_MAIN_LIB
@@ -23,7 +21,8 @@
 
 #define __PRETTY_FUNCTION__ __FUNCSIG__
 
-#define posix_memalign(p, a, s) (((*(p)) = _aligned_malloc((s), (a))), *(p) ?0 :errno)
+#define posix_memalign(p, a, s) \
+    (((*(p)) = _aligned_malloc((s), (a))), *(p) ? 0 : errno)
 #define posix_memalign_free _aligned_free
 
 // aligned should be in front of the declaration
@@ -39,18 +38,42 @@ inline int __builtin_ctzll(uint64_t x) {
     return (int)ret;
 }
 
+// cudatoolkit provides __builtin_ctz for NVCC >= 11.0
+#if !defined(__CUDACC__) || __CUDACC_VER_MAJOR__ < 11
 inline int __builtin_ctz(unsigned long x) {
     unsigned long ret;
     _BitScanForward(&ret, x);
     return (int)ret;
 }
+#endif
 
 inline int __builtin_clzll(uint64_t x) {
     return (int)__lzcnt64(x);
 }
 
+#define __builtin_popcount __popcnt
 #define __builtin_popcountl __popcnt64
 
+// MSVC does not define __SSEx__, and _M_IX86_FP is only defined on 32-bit
+// processors cf.
+// https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros
+#ifdef __AVX__
+#define __SSE__ 1
+#define __SSE2__ 1
+#define __SSE3__ 1
+#define __SSE4_1__ 1
+#define __SSE4_2__ 1
+#endif
+
+// MSVC sets FMA and F16C automatically when using AVX2
+// Ref. FMA (under /arch:AVX2):
+// https://docs.microsoft.com/en-us/cpp/build/reference/arch-x64 Ref. F16C (2nd
+// paragraph): https://walbourn.github.io/directxmath-avx2/
+#ifdef __AVX2__
+#define __FMA__ 1
+#define __F16C__ 1
+#endif
+
 #else
 /*******************************************************
  * Linux and OSX
@@ -59,10 +82,8 @@ inline int __builtin_clzll(uint64_t x) {
 #define FAISS_API
 #define posix_memalign_free free
 
-// aligned should be *in front* of the declaration, for compatibility with windows
-#define ALIGNED(x) __attribute__ ((aligned(x)))
+// aligned should be *in front* of the declaration, for compatibility with
+// windows
+#define ALIGNED(x) __attribute__((aligned(x)))
 
 #endif // _MSC_VER
-
-
-

data/vendor/faiss/faiss/impl/pq4_fast_scan.cpp

@@ -5,37 +5,33 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#include <faiss/impl/pq4_fast_scan.h>
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/pq4_fast_scan.h>
 #include <faiss/impl/simd_result_handlers.h>
 
 #include <array>
 
-
 namespace faiss {
 
-
 using namespace simd_result_handlers;
 
-
-
 /***************************************************************
  * Packing functions for codes
  ***************************************************************/
 
-
-
 namespace {
 
 /* extract the column starting at (i, j)
  * from packed matrix src of size (m, n)*/
-template<typename T, class TA>
+template <typename T, class TA>
 void get_matrix_column(
-        T * src,
-        size_t m, size_t n,
-        int64_t i, int64_t j,
-        TA & dest) {
-    for(int64_t k = 0; k < dest.size(); k++) {
+        T* src,
+        size_t m,
+        size_t n,
+        int64_t i,
+        int64_t j,
+        TA& dest) {
+    for (int64_t k = 0; k < dest.size(); k++) {
         if (k + i >= 0 && k + i < m) {
             dest[k] = src[(k + i) * n + j];
         } else {
@@ -46,38 +42,34 @@ void get_matrix_column(
 
 } // anonymous namespace
 
-
 void pq4_pack_codes(
-        const uint8_t *codes,
-        size_t ntotal, size_t M,
-        size_t nb, size_t bbs, size_t nsq,
-        uint8_t *blocks
-)
-{
+        const uint8_t* codes,
+        size_t ntotal,
+        size_t M,
+        size_t nb,
+        size_t bbs,
+        size_t nsq,
+        uint8_t* blocks) {
     FAISS_THROW_IF_NOT(bbs % 32 == 0);
     FAISS_THROW_IF_NOT(nb % bbs == 0);
     FAISS_THROW_IF_NOT(nsq % 2 == 0);
 
     memset(blocks, 0, nb * nsq / 2);
-    const uint8_t perm0[16] =
-            {0,  8,  1,  9,  2, 10,  3, 11,
-            4, 12,  5, 13,  6, 14,  7, 15};
+    const uint8_t perm0[16] = {
+            0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15};
 
-    uint8_t *codes2 = blocks;
-    for(size_t i0 = 0; i0 < nb; i0 += bbs) {
-        for(int sq = 0; sq < nsq; sq += 2) {
-            for(size_t i = 0; i < bbs; i += 32) {
+    uint8_t* codes2 = blocks;
+    for (size_t i0 = 0; i0 < nb; i0 += bbs) {
+        for (int sq = 0; sq < nsq; sq += 2) {
+            for (size_t i = 0; i < bbs; i += 32) {
                 std::array<uint8_t, 32> c, c0, c1;
                 get_matrix_column(
-                    codes, ntotal,
-                    (M + 1) / 2,
-                    i0 + i, sq / 2, c
-                );
-                for(int j = 0; j < 32; j++) {
+                        codes, ntotal, (M + 1) / 2, i0 + i, sq / 2, c);
+                for (int j = 0; j < 32; j++) {
                     c0[j] = c[j] & 15;
                     c1[j] = c[j] >> 4;
                 }
-                for(int j = 0; j < 16; j++) {
+                for (int j = 0; j < 16; j++) {
                     uint8_t d0, d1;
                     d0 = c0[perm0[j]] | (c0[perm0[j] + 16] << 4);
                     d1 = c1[perm0[j]] | (c1[perm0[j] + 16] << 4);
@@ -91,36 +83,33 @@ void pq4_pack_codes(
 }
 
 void pq4_pack_codes_range(
-        const uint8_t *codes,
+        const uint8_t* codes,
         size_t M,
-        size_t i0, size_t i1,
-        size_t bbs, size_t M2,
-        uint8_t * blocks
-) {
-    const uint8_t perm0[16] =
-            {0,  8,  1,  9,  2, 10,  3, 11,
-            4, 12,  5, 13,  6, 14,  7, 15};
+        size_t i0,
+        size_t i1,
+        size_t bbs,
+        size_t M2,
+        uint8_t* blocks) {
+    const uint8_t perm0[16] = {
+            0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15};
 
     // range of affected blocks
     size_t block0 = i0 / bbs;
     size_t block1 = ((i1 - 1) / bbs) + 1;
 
     for (size_t b = block0; b < block1; b++) {
-        uint8_t *codes2 = blocks + b * bbs * M2 / 2;
+        uint8_t* codes2 = blocks + b * bbs * M2 / 2;
         int64_t i_base = b * bbs - i0;
-        for(int sq = 0; sq < M2; sq += 2) {
-            for(size_t i = 0; i < bbs; i += 32) {
+        for (int sq = 0; sq < M2; sq += 2) {
+            for (size_t i = 0; i < bbs; i += 32) {
                 std::array<uint8_t, 32> c, c0, c1;
                 get_matrix_column(
-                    codes, i1 - i0,
-                    (M + 1) / 2,
-                    i_base + i, sq / 2, c
-                );
-                for(int j = 0; j < 32; j++) {
+                        codes, i1 - i0, (M + 1) / 2, i_base + i, sq / 2, c);
+                for (int j = 0; j < 32; j++) {
                     c0[j] = c[j] & 15;
                     c1[j] = c[j] >> 4;
                 }
-                for(int j = 0; j < 16; j++) {
+                for (int j = 0; j < 16; j++) {
                     uint8_t d0, d1;
                     d0 = c0[perm0[j]] | (c0[perm0[j] + 16] << 4);
                     d1 = c1[perm0[j]] | (c1[perm0[j] + 16] << 4);
@@ -131,14 +120,14 @@ void pq4_pack_codes_range(
             }
         }
     }
-
 }
 
-
 uint8_t pq4_get_packed_element(
-        const uint8_t *data, size_t bbs, size_t nsq,
-        size_t i, size_t sq
-) {
+        const uint8_t* data,
+        size_t bbs,
+        size_t nsq,
+        size_t i,
+        size_t sq) {
     // move to correct bbs-sized block
     data += (i / bbs * (nsq / 2) + sq / 2) * bbs;
     sq = sq & 1;
@@ -151,122 +140,86 @@ uint8_t pq4_get_packed_element(
     if (sq == 1) {
         data += 16;
     }
-    const uint8_t iperm0[16] =
-        {0,  2,  4,  6,  8, 10, 12, 14,
-         1,  3,  5,  7,  9, 11, 13, 15};
+    const uint8_t iperm0[16] = {
+            0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};
     if (i < 16) {
         return data[iperm0[i]] & 15;
     } else {
         return data[iperm0[i - 16]] >> 4;
     }
-
 }
 
 /***************************************************************
  * Packing functions for Look-Up Tables (LUT)
  ***************************************************************/
 
-
-
-
-void pq4_pack_LUT(
-        int nq, int nsq,
-        const uint8_t *src,
-        uint8_t *dest)
-{
-
-    for(int q = 0; q < nq; q++) {
-        for(int sq = 0; sq < nsq; sq += 2) {
-            memcpy(
-                dest + (sq / 2 * nq + q) * 32,
-                src + (q * nsq + sq) * 16,
-                16
-            );
-            memcpy(
-                dest + (sq / 2 * nq + q) * 32 + 16,
-                src + (q * nsq + sq + 1) * 16,
-                16
-            );
+void pq4_pack_LUT(int nq, int nsq, const uint8_t* src, uint8_t* dest) {
+    for (int q = 0; q < nq; q++) {
+        for (int sq = 0; sq < nsq; sq += 2) {
+            memcpy(dest + (sq / 2 * nq + q) * 32,
+                   src + (q * nsq + sq) * 16,
+                   16);
+            memcpy(dest + (sq / 2 * nq + q) * 32 + 16,
+                   src + (q * nsq + sq + 1) * 16,
+                   16);
         }
     }
 }
 
-
-int pq4_pack_LUT_qbs(
-        int qbs, int nsq,
-        const uint8_t *src,
-        uint8_t *dest)
-{
+int pq4_pack_LUT_qbs(int qbs, int nsq, const uint8_t* src, uint8_t* dest) {
     FAISS_THROW_IF_NOT(nsq % 2 == 0);
     size_t dim12 = 16 * nsq;
     int i0 = 0;
     int qi = qbs;
-    while(qi) {
+    while (qi) {
         int nq = qi & 15;
         qi >>= 4;
-        pq4_pack_LUT(
-                nq, nsq,
-                src + i0 * dim12,
-                dest + i0 * dim12
-        );
+        pq4_pack_LUT(nq, nsq, src + i0 * dim12, dest + i0 * dim12);
         i0 += nq;
     }
     return i0;
 }
 
-
 namespace {
 
 void pack_LUT_1_q_map(
-        int nq, const int *q_map,
+        int nq,
+        const int* q_map,
         int nsq,
-        const uint8_t *src,
-        uint8_t *dest)
-{
-
-    for(int qi = 0; qi < nq; qi++) {
+        const uint8_t* src,
+        uint8_t* dest) {
+    for (int qi = 0; qi < nq; qi++) {
         int q = q_map[qi];
-        for(int sq = 0; sq < nsq; sq += 2) {
-            memcpy(
-                dest + (sq / 2 * nq + qi) * 32,
-                src + (q * nsq + sq) * 16,
-                16
-            );
-            memcpy(
-                dest + (sq / 2 * nq + qi) * 32 + 16,
-                src + (q * nsq + sq + 1) * 16,
-                16
-            );
+        for (int sq = 0; sq < nsq; sq += 2) {
+            memcpy(dest + (sq / 2 * nq + qi) * 32,
+                   src + (q * nsq + sq) * 16,
+                   16);
+            memcpy(dest + (sq / 2 * nq + qi) * 32 + 16,
+                   src + (q * nsq + sq + 1) * 16,
+                   16);
         }
     }
-
 }
 
 } // anonymous namespace
 
 int pq4_pack_LUT_qbs_q_map(
-        int qbs, int nsq,
-        const uint8_t *src,
-        const int * q_map,
-        uint8_t *dest)
-{
+        int qbs,
+        int nsq,
+        const uint8_t* src,
+        const int* q_map,
+        uint8_t* dest) {
     FAISS_THROW_IF_NOT(nsq % 2 == 0);
     size_t dim12 = 16 * nsq;
     int i0 = 0;
     int qi = qbs;
-    while(qi) {
+    while (qi) {
         int nq = qi & 15;
         qi >>= 4;
-        pack_LUT_1_q_map(
-                nq, q_map + i0, nsq,
-                src,
-                dest + i0 * dim12
-        );
+        pack_LUT_1_q_map(nq, q_map + i0, nsq, src, dest + i0 * dim12);
         i0 += nq;
     }
     return i0;
 }
 
-
-
 } // namespace faiss