faiss 0.1.3 → 0.1.4

data/vendor/faiss/faiss/impl/io.cpp

@@ -242,16 +242,47 @@ BufferedIOWriter::~BufferedIOWriter()
 
 
 uint32_t fourcc (const  char sx[4]) {
-    assert(4 == strlen(sx));
+    FAISS_THROW_IF_NOT (4 == strlen(sx));
     const unsigned char *x = (unsigned char*)sx;
     return x[0] | x[1] << 8 | x[2] << 16 | x[3] << 24;
 }
 
 uint32_t fourcc (const std::string & sx) {
-    assert(sx.length() == 4);
+    FAISS_THROW_IF_NOT (sx.length() == 4);
     const unsigned char *x = (unsigned char*)sx.c_str();
     return x[0] | x[1] << 8 | x[2] << 16 | x[3] << 24;
 }
 
+void fourcc_inv(uint32_t x, char str[5]) {
+    *(uint32_t*)str = x;
+    str[5] = 0;
+}
+
+std::string fourcc_inv(uint32_t x) {
+    char str[5];
+    fourcc_inv(x, str);
+    return std::string(str);
+}
+
+
+std::string fourcc_inv_printable(uint32_t x) {
+    char cstr[5];
+    fourcc_inv(x, cstr);
+    std::string str = "";
+    for (int i = 0; i < 4; i++) {
+        uint8_t c = cstr[i];
+        if (32 <= c && c < 127) {
+            str += c;
+        } else {
+            char buf[10];
+            sprintf(buf, "\\x%02x", c);
+            str += buf;
+        }
+    }
+    return str;
+}
+
+
+
 
 } // namespace faiss

data/vendor/faiss/faiss/impl/io.h

@@ -50,7 +50,7 @@ struct IOWriter {
     // return a file number that can be memory-mapped
     virtual int fileno ();
 
-    virtual ~IOWriter() {}
+    virtual ~IOWriter() noexcept(false) {}
 };
 
 
@@ -139,12 +139,17 @@ struct BufferedIOWriter: IOWriter {
     size_t operator()(const void *ptr, size_t size, size_t nitems) override;
 
     // flushes
-    ~BufferedIOWriter();
+    ~BufferedIOWriter() override;
 };
 
 /// cast a 4-character string to a uint32_t that can be written and read easily
 uint32_t fourcc (const char sx[4]);
 uint32_t fourcc (const std::string & sx);
 
+// decoding of fourcc (int32 -> string)
+void fourcc_inv(uint32_t x, char str[5]);
+std::string fourcc_inv(uint32_t x);
+std::string fourcc_inv_printable(uint32_t x);
+
 
 } // namespace faiss

data/vendor/faiss/faiss/impl/lattice_Zn.cpp

@@ -20,22 +20,8 @@
 #include <algorithm>
 
 #include <faiss/utils/distances.h>
+#include <faiss/impl/platform_macros.h>
 
-#ifdef _MSC_VER
-
-#include <intrin.h>
-
-static inline int __builtin_ctzll(uint64_t x) {
-    unsigned long ret;
-    _BitScanForward64(&ret, x);
-    return (int)ret;
-}
-
-static inline int __builtin_clzll(uint64_t x) {
-    return (int)__lzcnt64(x);
-}
-
-#endif // _MSC_VER
 
 namespace faiss {
 

data/vendor/faiss/faiss/impl/platform_macros.h

@@ -7,8 +7,14 @@
 
 #pragma once
 
+
 #ifdef _MSC_VER
 
+/*******************************************************
+ * Windows specific macros
+ *******************************************************/
+
+
 #ifdef FAISS_MAIN_LIB
 #define FAISS_API __declspec(dllexport)
 #else // _FAISS_MAIN_LIB
@@ -17,8 +23,46 @@
 
 #define __PRETTY_FUNCTION__ __FUNCSIG__
 
+#define posix_memalign(p, a, s) (((*(p)) = _aligned_malloc((s), (a))), *(p) ?0 :errno)
+#define posix_memalign_free _aligned_free
+
+// aligned should be in front of the declaration
+#define ALIGNED(x) __declspec(align(x))
+
+// redefine the GCC intrinsics with Windows equivalents
+
+#include <intrin.h>
+
+inline int __builtin_ctzll(uint64_t x) {
+    unsigned long ret;
+    _BitScanForward64(&ret, x);
+    return (int)ret;
+}
+
+inline int __builtin_ctz(unsigned long x) {
+    unsigned long ret;
+    _BitScanForward(&ret, x);
+    return (int)ret;
+}
+
+inline int __builtin_clzll(uint64_t x) {
+    return (int)__lzcnt64(x);
+}
+
+#define __builtin_popcountl __popcnt64
+
 #else
+/*******************************************************
+ * Linux and OSX
+ *******************************************************/
 
 #define FAISS_API
+#define posix_memalign_free free
+
+// aligned should be *in front* of the declaration, for compatibility with windows
+#define ALIGNED(x) __attribute__ ((aligned(x)))
 
 #endif // _MSC_VER
+
+
+

data/vendor/faiss/faiss/impl/pq4_fast_scan.cpp

@@ -0,0 +1,272 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <faiss/impl/pq4_fast_scan.h>
+#include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/simd_result_handlers.h>
+
+#include <array>
+
+
+namespace faiss {
+
+
+using namespace simd_result_handlers;
+
+
+
+/***************************************************************
+ * Packing functions for codes
+ ***************************************************************/
+
+
+
+namespace {
+
+/* extract the column starting at (i, j)
+ * from packed matrix src of size (m, n)*/
+template<typename T, class TA>
+void get_matrix_column(
+        T * src,
+        size_t m, size_t n,
+        int64_t i, int64_t j,
+        TA & dest) {
+    for(int64_t k = 0; k < dest.size(); k++) {
+        if (k + i >= 0 && k + i < m) {
+            dest[k] = src[(k + i) * n + j];
+        } else {
+            dest[k] = 0;
+        }
+    }
+}
+
+} // anonymous namespace
+
+
+void pq4_pack_codes(
+        const uint8_t *codes,
+        size_t ntotal, size_t M,
+        size_t nb, size_t bbs, size_t nsq,
+        uint8_t *blocks
+)
+{
+    FAISS_THROW_IF_NOT(bbs % 32 == 0);
+    FAISS_THROW_IF_NOT(nb % bbs == 0);
+    FAISS_THROW_IF_NOT(nsq % 2 == 0);
+
+    memset(blocks, 0, nb * nsq / 2);
+    const uint8_t perm0[16] =
+            {0,  8,  1,  9,  2, 10,  3, 11,
+            4, 12,  5, 13,  6, 14,  7, 15};
+
+    uint8_t *codes2 = blocks;
+    for(size_t i0 = 0; i0 < nb; i0 += bbs) {
+        for(int sq = 0; sq < nsq; sq += 2) {
+            for(size_t i = 0; i < bbs; i += 32) {
+                std::array<uint8_t, 32> c, c0, c1;
+                get_matrix_column(
+                    codes, ntotal,
+                    (M + 1) / 2,
+                    i0 + i, sq / 2, c
+                );
+                for(int j = 0; j < 32; j++) {
+                    c0[j] = c[j] & 15;
+                    c1[j] = c[j] >> 4;
+                }
+                for(int j = 0; j < 16; j++) {
+                    uint8_t d0, d1;
+                    d0 = c0[perm0[j]] | (c0[perm0[j] + 16] << 4);
+                    d1 = c1[perm0[j]] | (c1[perm0[j] + 16] << 4);
+                    codes2[j] = d0;
+                    codes2[j + 16] = d1;
+                }
+                codes2 += 32;
+            }
+        }
+    }
+}
+
+void pq4_pack_codes_range(
+        const uint8_t *codes,
+        size_t M,
+        size_t i0, size_t i1,
+        size_t bbs, size_t M2,
+        uint8_t * blocks
+) {
+    const uint8_t perm0[16] =
+            {0,  8,  1,  9,  2, 10,  3, 11,
+            4, 12,  5, 13,  6, 14,  7, 15};
+
+    // range of affected blocks
+    size_t block0 = i0 / bbs;
+    size_t block1 = ((i1 - 1) / bbs) + 1;
+
+    for (size_t b = block0; b < block1; b++) {
+        uint8_t *codes2 = blocks + b * bbs * M2 / 2;
+        int64_t i_base = b * bbs - i0;
+        for(int sq = 0; sq < M2; sq += 2) {
+            for(size_t i = 0; i < bbs; i += 32) {
+                std::array<uint8_t, 32> c, c0, c1;
+                get_matrix_column(
+                    codes, i1 - i0,
+                    (M + 1) / 2,
+                    i_base + i, sq / 2, c
+                );
+                for(int j = 0; j < 32; j++) {
+                    c0[j] = c[j] & 15;
+                    c1[j] = c[j] >> 4;
+                }
+                for(int j = 0; j < 16; j++) {
+                    uint8_t d0, d1;
+                    d0 = c0[perm0[j]] | (c0[perm0[j] + 16] << 4);
+                    d1 = c1[perm0[j]] | (c1[perm0[j] + 16] << 4);
+                    codes2[j] |= d0;
+                    codes2[j + 16] |= d1;
+                }
+                codes2 += 32;
+            }
+        }
+    }
+
+}
+
+
+uint8_t pq4_get_packed_element(
+        const uint8_t *data, size_t bbs, size_t nsq,
+        size_t i, size_t sq
+) {
+    // move to correct bbs-sized block
+    data += (i / bbs * (nsq / 2) + sq / 2) * bbs;
+    sq = sq & 1;
+    i = i % bbs;
+
+    // another step
+    data += (i / 32) * 32;
+    i = i % 32;
+
+    if (sq == 1) {
+        data += 16;
+    }
+    const uint8_t iperm0[16] =
+        {0,  2,  4,  6,  8, 10, 12, 14,
+         1,  3,  5,  7,  9, 11, 13, 15};
+    if (i < 16) {
+        return data[iperm0[i]] & 15;
+    } else {
+        return data[iperm0[i - 16]] >> 4;
+    }
+
+}
+
+/***************************************************************
+ * Packing functions for Look-Up Tables (LUT)
+ ***************************************************************/
+
+
+
+
+void pq4_pack_LUT(
+        int nq, int nsq,
+        const uint8_t *src,
+        uint8_t *dest)
+{
+
+    for(int q = 0; q < nq; q++) {
+        for(int sq = 0; sq < nsq; sq += 2) {
+            memcpy(
+                dest + (sq / 2 * nq + q) * 32,
+                src + (q * nsq + sq) * 16,
+                16
+            );
+            memcpy(
+                dest + (sq / 2 * nq + q) * 32 + 16,
+                src + (q * nsq + sq + 1) * 16,
+                16
+            );
+        }
+    }
+}
+
+
+int pq4_pack_LUT_qbs(
+        int qbs, int nsq,
+        const uint8_t *src,
+        uint8_t *dest)
+{
+    FAISS_THROW_IF_NOT(nsq % 2 == 0);
+    size_t dim12 = 16 * nsq;
+    int i0 = 0;
+    int qi = qbs;
+    while(qi) {
+        int nq = qi & 15;
+        qi >>= 4;
+        pq4_pack_LUT(
+                nq, nsq,
+                src + i0 * dim12,
+                dest + i0 * dim12
+        );
+        i0 += nq;
+    }
+    return i0;
+}
+
+
+namespace {
+
+void pack_LUT_1_q_map(
+        int nq, const int *q_map,
+        int nsq,
+        const uint8_t *src,
+        uint8_t *dest)
+{
+
+    for(int qi = 0; qi < nq; qi++) {
+        int q = q_map[qi];
+        for(int sq = 0; sq < nsq; sq += 2) {
+            memcpy(
+                dest + (sq / 2 * nq + qi) * 32,
+                src + (q * nsq + sq) * 16,
+                16
+            );
+            memcpy(
+                dest + (sq / 2 * nq + qi) * 32 + 16,
+                src + (q * nsq + sq + 1) * 16,
+                16
+            );
+        }
+    }
+
+}
+
+} // anonymous namespace
+
+int pq4_pack_LUT_qbs_q_map(
+        int qbs, int nsq,
+        const uint8_t *src,
+        const int * q_map,
+        uint8_t *dest)
+{
+    FAISS_THROW_IF_NOT(nsq % 2 == 0);
+    size_t dim12 = 16 * nsq;
+    int i0 = 0;
+    int qi = qbs;
+    while(qi) {
+        int nq = qi & 15;
+        qi >>= 4;
+        pack_LUT_1_q_map(
+                nq, q_map + i0, nsq,
+                src,
+                dest + i0 * dim12
+        );
+        i0 += nq;
+    }
+    return i0;
+}
+
+
+
+} // namespace faiss

data/vendor/faiss/faiss/impl/pq4_fast_scan.h

@@ -0,0 +1,169 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <cstdlib>
+
+/** PQ4 SIMD packing and accumulation functions
+ *
+ * The basic kernel accumulates nq query vectors with bbs = nb * 2 * 16 vectors
+ * and produces an output matrix for that. It is interesting for nq * nb <= 4,
+ * otherwise register spilling becomes too large.
+ *
+ * The implementation of these functions is spread over 3 cpp files to reduce
+ * parallel compile times. Templates are instanciated explicitly.
+ */
+
+
+namespace faiss {
+
+
+/** Pack codes for consumption by the SIMD kernels.
+ *  The unused bytes are set to 0.
+ *
+ * @param codes   input codes, size (ntotal, ceil(M / 2))
+ * @param nototal number of input codes
+ * @param nb      output number of codes (ntotal rounded up to a multiple of
+ *                bbs)
+ * @param M2      number of sub-quantizers (=M rounded up to a muliple of 2)
+ * @param bbs     size of database blocks (multiple of 32)
+ * @param blocks  output array, size nb * nsq / 2.
+ */
+void pq4_pack_codes(
+        const uint8_t *codes,
+        size_t ntotal, size_t M,
+        size_t nb, size_t bbs, size_t M2,
+        uint8_t * blocks
+);
+
+/** Same as pack_codes but write in a given range of the output,
+ * leaving the rest untouched. Assumes allocated entries are 0 on input.
+ *
+ * @param codes   input codes, size (i1 - i0, ceil(M / 2))
+ * @param i0      first output code to write
+ * @param i1      last output code to write
+ * @param blocks  output array, size at least ceil(i1 / bbs) * bbs * nsq / 2
+ */
+void pq4_pack_codes_range(
+        const uint8_t *codes,
+        size_t M,
+        size_t i0, size_t i1,
+        size_t bbs, size_t M2,
+        uint8_t * blocks
+);
+
+/** get a single element from a packed codes table
+ *
+ * @param i        vector id
+ * @param sq       subquantizer (< nsq)
+ */
+uint8_t pq4_get_packed_element(
+        const uint8_t *data, size_t bbs, size_t nsq,
+        size_t i, size_t sq
+);
+
+/** Pack Look-up table for consumption by the kernel.
+ *
+ * @param nq      number of queries
+ * @param nsq     number of sub-quantizers (muliple of 2)
+ * @param src     input array, size (nq, 16)
+ * @param dest    output array, size (nq, 16)
+ */
+void pq4_pack_LUT(
+        int nq, int nsq,
+        const uint8_t *src,
+        uint8_t *dest
+);
+
+
+
+/** Loop over database elements and accumulate results into result handler
+ *
+ * @param nq      number of queries
+ * @param nb      number of database elements
+ * @param bbs     size of database blocks (multiple of 32)
+ * @param nsq     number of sub-quantizers (muliple of 2)
+ * @param codes   packed codes array
+ * @param LUT     packed look-up table
+ */
+template<class ResultHandler>
+void pq4_accumulate_loop(
+        int nq,
+        size_t nb, int bbs,
+        int nsq,
+        const uint8_t *codes,
+        const uint8_t *LUT,
+        ResultHandler & res);
+
+
+
+/* qbs versions, supported only for bbs=32.
+ *
+ * The kernel function runs the kernel for *several* query blocks
+ * and bbs database vectors. The sizes of the blocks are encoded in qbs as
+ * base-16 digits.
+ *
+ * For example, qbs = 0x1223 means that the kernel will be run 4 times, the
+ * first time with 3 query vectors, second time with 2 query vectors, then 2
+ * vectors again and finally with 1 query vector. The output block will thus be
+ * nq = 3 + 2 + 2 + 1 = 6 queries. For a given total block size, the optimal
+ * decomposition into sub-blocks (measured empirically) is given by
+ * preferred_qbs().
+*/
+
+
+/* compute the number of queries from a base-16 decomposition */
+int pq4_qbs_to_nq(int qbs);
+
+/** return the preferred decomposition in blocks for a nb of queries. */
+int pq4_preferred_qbs(int nq);
+
+/** Pack Look-up table for consumption by the kernel.
+ *
+ * @param qbs     4-bit encoded number of query blocks, the total number of
+ *                queries handled (nq) is deduced from it
+ * @param nsq     number of sub-quantizers (muliple of 2)
+ * @param src     input array, size (nq, 16)
+ * @param dest    output array, size (nq, 16)
+ * @return nq
+ */
+int pq4_pack_LUT_qbs(
+        int fqbs, int nsq,
+        const uint8_t *src,
+        uint8_t *dest
+);
+
+/** Same as pq4_pack_LUT_qbs, except the source vectors are remapped with q_map */
+int pq4_pack_LUT_qbs_q_map(
+        int qbs, int nsq,
+        const uint8_t *src,
+        const int * q_map,
+        uint8_t *dest);
+
+/** Run accumulation loop.
+ *
+ * @param qbs     4-bit encded number of queries
+ * @param nb      number of database codes (mutliple of bbs)
+ * @param nsq     number of sub-quantizers
+ * @param codes   encoded database vectors (packed)
+ * @param LUT     look-up table (packed)
+ * @param res     call-back for the resutls
+ */
+template<class ResultHandler>
+void pq4_accumulate_loop_qbs(
+        int qbs,
+        size_t nb,
+        int nsq,
+        const uint8_t *codes,
+        const uint8_t *LUT,
+        ResultHandler & res);
+
+
+
+} // namespace faiss