faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/utils/partitioning.h

@@ -7,7 +7,6 @@
 
 #pragma once
 
-
 #include <stdint.h>
 #include <stdio.h>
 
@@ -15,23 +14,27 @@
 
 namespace faiss {
 
-
 /** partitions the table into 0:q and q:n where all elements above q are >= all
  * elements below q (for C = CMax, for CMin comparisons are reversed)
  *
  * Returns the partition threshold. The elements q:n are destroyed on output.
  */
-template<class C>
+template <class C>
 typename C::T partition_fuzzy(
-    typename C::T *vals, typename C::TI * ids, size_t n,
-    size_t q_min, size_t q_max, size_t * q_out);
+        typename C::T* vals,
+        typename C::TI* ids,
+        size_t n,
+        size_t q_min,
+        size_t q_max,
+        size_t* q_out);
 
 /** simplified interface for when the parition is not fuzzy */
-template<class C>
+template <class C>
 inline typename C::T partition(
-    typename C::T *vals, typename C::TI * ids, size_t n,
-    size_t q)
-{
+        typename C::T* vals,
+        typename C::TI* ids,
+        size_t n,
+        size_t q) {
     return partition_fuzzy<C>(vals, ids, n, q, q, nullptr);
 }
 
@@ -41,29 +44,31 @@ inline typename C::T partition(
  * values outside the range are ignored.
  * the data table should be aligned on 32 bytes */
 void simd_histogram_8(
-    const uint16_t *data, int n,
-    uint16_t min, int shift,
-    int *hist);
+        const uint16_t* data,
+        int n,
+        uint16_t min,
+        int shift,
+        int* hist);
 
 /** same for 16-bin histogram */
 void simd_histogram_16(
-    const uint16_t *data, int n,
-    uint16_t min, int shift,
-    int *hist);
-
+        const uint16_t* data,
+        int n,
+        uint16_t min,
+        int shift,
+        int* hist);
 
 struct PartitionStats {
     uint64_t bissect_cycles;
     uint64_t compress_cycles;
 
-    PartitionStats () {reset (); }
-    void reset ();
+    PartitionStats() {
+        reset();
+    }
+    void reset();
 };
 
 // global var that collects them all
 FAISS_API extern PartitionStats partition_stats;
 
-
-
 } // namespace faiss
-

data/vendor/faiss/faiss/utils/quantize_lut.cpp

@@ -5,150 +5,157 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #include <faiss/utils/quantize_lut.h>
 
+#include <algorithm>
 #include <cmath>
 #include <cstring>
 #include <vector>
-#include <algorithm>
 
 #include <faiss/impl/FaissAssert.h>
 
-
 namespace faiss {
 
-
 namespace quantize_lut {
 
-
 /******************************************************
  * Quantize look-up tables
  ******************************************************/
 
 namespace {
 
-float round_uint8_and_mul(float *tab, size_t n) {
+float round_uint8_and_mul(float* tab, size_t n) {
     float max = 0;
-    for(int i = 0; i < n; i++) {
-        if(fabs(tab[i]) > max) {
+    for (int i = 0; i < n; i++) {
+        if (fabs(tab[i]) > max) {
             max = fabs(tab[i]);
         }
     }
     float multiplier = 127 / max;
-    for(int i = 0; i < n; i++) {
+    for (int i = 0; i < n; i++) {
         tab[i] = floorf(tab[i] * multiplier + 128);
     }
     return multiplier;
 }
 
 // there can be NaNs in tables, they should be ignored
-float tab_min(const float *tab, size_t n) {
+float tab_min(const float* tab, size_t n) {
     float min = HUGE_VAL;
-    for(int i = 0; i < n; i++) {
-        if (tab[i] < min) min = tab[i];
+    for (int i = 0; i < n; i++) {
+        if (tab[i] < min)
+            min = tab[i];
     }
     return min;
 }
 
-float tab_max(const float *tab, size_t n) {
+float tab_max(const float* tab, size_t n) {
     float max = -HUGE_VAL;
-    for(int i = 0; i < n; i++) {
-        if (tab[i] > max) max = tab[i];
+    for (int i = 0; i < n; i++) {
+        if (tab[i] > max)
+            max = tab[i];
     }
     return max;
 }
 
-void round_tab(float *tab, size_t n, float a, float bi) {
-    for(int i = 0; i < n; i++) {
+void round_tab(float* tab, size_t n, float a, float bi) {
+    for (int i = 0; i < n; i++) {
         tab[i] = floorf((tab[i] - bi) * a + 0.5);
     }
 }
 
-template<typename T>
-void round_tab(const float *tab, size_t n, float a, float bi, T *tab_out) {
-    for(int i = 0; i < n; i++) {
+template <typename T>
+void round_tab(const float* tab, size_t n, float a, float bi, T* tab_out) {
+    for (int i = 0; i < n; i++) {
         tab_out[i] = (T)floorf((tab[i] - bi) * a + 0.5);
     }
 }
 
-
-
 } // anonymous namespace
 
 void round_uint8_per_column(
-        float *tab, size_t n, size_t d,
-        float *a_out, float *b_out)
-{
+        float* tab,
+        size_t n,
+        size_t d,
+        float* a_out,
+        float* b_out) {
     float max_span = 0;
     std::vector<float> mins(n);
-    for(int i = 0; i < n; i++) {
+    for (int i = 0; i < n; i++) {
         mins[i] = tab_min(tab + i * d, d);
         float span = tab_max(tab + i * d, d) - mins[i];
-        if(span > max_span) {
+        if (span > max_span) {
             max_span = span;
         }
     }
     float a = 255 / max_span;
     float b = 0;
-    for(int i = 0; i < n; i++) {
+    for (int i = 0; i < n; i++) {
         b += mins[i];
         round_tab(tab + i * d, d, a, mins[i]);
     }
-    if (a_out) *a_out = a;
-    if (b_out) *b_out = b;
+    if (a_out)
+        *a_out = a;
+    if (b_out)
+        *b_out = b;
 }
 
 void round_uint8_per_column_multi(
-        float *tab, size_t m, size_t n, size_t d,
-        float *a_out, float *b_out)
-{
+        float* tab,
+        size_t m,
+        size_t n,
+        size_t d,
+        float* a_out,
+        float* b_out) {
     float max_span = 0;
     std::vector<float> mins(n);
-    for(int i = 0; i < n; i++) {
+    for (int i = 0; i < n; i++) {
         float min_i = HUGE_VAL;
         float max_i = -HUGE_VAL;
-        for(int j = 0; j < m; j++) {
+        for (int j = 0; j < m; j++) {
             min_i = std::min(min_i, tab_min(tab + (j * n + i) * d, d));
             max_i = std::max(max_i, tab_max(tab + (j * n + i) * d, d));
         }
         mins[i] = min_i;
         float span = max_i - min_i;
-        if(span > max_span) {
+        if (span > max_span) {
             max_span = span;
         }
     }
     float a = 255 / max_span;
     float b = 0;
-    for(int i = 0; i < n; i++) {
+    for (int i = 0; i < n; i++) {
         b += mins[i];
-        for(int j = 0; j < m; j++) {
+        for (int j = 0; j < m; j++) {
             round_tab(tab + (j * n + i) * d, d, a, mins[i]);
         }
     }
-    if (a_out) *a_out = a;
-    if (b_out) *b_out = b;
+    if (a_out)
+        *a_out = a;
+    if (b_out)
+        *b_out = b;
 }
 
-
 // translation of
 // https://github.com/fairinternal/faiss_improvements/blob/7122c3cc6ddb0a371d8aa6f1309cd8bcf2335e61/LUT_quantization.ipynb
 void quantize_LUT_and_bias(
-        size_t nprobe, size_t M, size_t ksub,
+        size_t nprobe,
+        size_t M,
+        size_t ksub,
         bool lut_is_3d,
-        const float *LUT,
-        const float *bias,
-        uint8_t *LUTq, size_t M2,
-        uint16_t *biasq,
-        float *a_out, float *b_out)
-{
+        const float* LUT,
+        const float* bias,
+        uint8_t* LUTq,
+        size_t M2,
+        uint16_t* biasq,
+        float* a_out,
+        float* b_out) {
     float a, b;
     if (!bias) {
         FAISS_THROW_IF_NOT(!lut_is_3d);
         std::vector<float> mins(M);
         float max_span_LUT = -HUGE_VAL, max_span_dis = 0;
         b = 0;
-        for(int i = 0; i < M; i++) {
+        for (int i = 0; i < M; i++) {
             mins[i] = tab_min(LUT + i * ksub, ksub);
             float span = tab_max(LUT + i * ksub, ksub) - mins[i];
             max_span_LUT = std::max(max_span_LUT, span);
@@ -157,7 +164,7 @@ void quantize_LUT_and_bias(
         }
         a = std::min(255 / max_span_LUT, 65535 / max_span_dis);
 
-        for(int i = 0; i < M; i++) {
+        for (int i = 0; i < M; i++) {
             round_tab(LUT + i * ksub, ksub, a, mins[i], LUTq + i * ksub);
         }
         memset(LUTq + M * ksub, 0, ksub * (M2 - M));
@@ -168,7 +175,7 @@ void quantize_LUT_and_bias(
         float bias_max = tab_max(bias, nprobe);
         max_span_dis = bias_max - bias_min;
         b = 0;
-        for(int i = 0; i < M; i++) {
+        for (int i = 0; i < M; i++) {
             mins[i] = tab_min(LUT + i * ksub, ksub);
             float span = tab_max(LUT + i * ksub, ksub) - mins[i];
             max_span_LUT = std::max(max_span_LUT, span);
@@ -178,7 +185,7 @@ void quantize_LUT_and_bias(
         a = std::min(255 / max_span_LUT, 65535 / max_span_dis);
         b += bias_min;
 
-        for(int i = 0; i < M; i++) {
+        for (int i = 0; i < M; i++) {
             round_tab(LUT + i * ksub, ksub, a, mins[i], LUTq + i * ksub);
         }
         memset(LUTq + M * ksub, 0, ksub * (M2 - M));
@@ -196,7 +203,7 @@ void quantize_LUT_and_bias(
         for (int j = 0; j < nprobe; j++) {
             float max_span_dis_j = bias[j] - bias_min;
             float b2j = bias[j];
-            for(int i = 0; i < M; i++) {
+            for (int i = 0; i < M; i++) {
                 mins[ij] = tab_min(LUT + ij * ksub, ksub);
                 float span = tab_max(LUT + ij * ksub, ksub) - mins[ij];
                 max_span_LUT = std::max(max_span_LUT, span);
@@ -214,9 +221,11 @@ void quantize_LUT_and_bias(
         ij = 0;
         size_t ij_2 = 0;
         for (int j = 0; j < nprobe; j++) {
-            for(int i = 0; i < M; i++) {
-                round_tab(LUT + ij * ksub, ksub, a, mins[ij], LUTq + ij_2 * ksub);
-                ij++; ij_2++;
+            for (int i = 0; i < M; i++) {
+                round_tab(
+                        LUT + ij * ksub, ksub, a, mins[ij], LUTq + ij_2 * ksub);
+                ij++;
+                ij_2++;
             }
             memset(LUTq + ij_2 * ksub, 0, ksub * (M2 - M));
             ij_2 += M2 - M;
@@ -227,11 +236,11 @@ void quantize_LUT_and_bias(
     } else { // !biasq
         // then we integrate the bias into the LUTs
         std::vector<float> LUT2_storage(nprobe * M * ksub);
-        float *LUT2 = LUT2_storage.data();
+        float* LUT2 = LUT2_storage.data();
         size_t ijc = 0;
         for (int j = 0; j < nprobe; j++) {
             float bias_j = bias[j] / M;
-            for(int i = 0; i < M; i++) {
+            for (int i = 0; i < M; i++) {
                 for (int c = 0; c < ksub; c++) {
                     LUT2[ijc] = LUT[ijc] + bias_j;
                     ijc++;
@@ -241,7 +250,7 @@ void quantize_LUT_and_bias(
         std::vector<float> mins(M, HUGE_VAL), maxs(M, -HUGE_VAL);
         size_t ij = 0;
         for (int j = 0; j < nprobe; j++) {
-            for(int i = 0; i < M; i++) {
+            for (int i = 0; i < M; i++) {
                 mins[i] = std::min(mins[i], tab_min(LUT2 + ij * ksub, ksub));
                 maxs[i] = std::max(maxs[i], tab_max(LUT2 + ij * ksub, ksub));
                 ij++;
@@ -250,7 +259,7 @@ void quantize_LUT_and_bias(
 
         float max_span = -HUGE_VAL;
         b = 0;
-        for(int i = 0; i < M; i++) {
+        for (int i = 0; i < M; i++) {
             float span = maxs[i] - mins[i];
             max_span = std::max(max_span, span);
             b += mins[i];
@@ -259,19 +268,22 @@ void quantize_LUT_and_bias(
         ij = 0;
         size_t ij_2 = 0;
         for (int j = 0; j < nprobe; j++) {
-            for(int i = 0; i < M; i++) {
-                round_tab(LUT2 + ij * ksub, ksub, a, mins[i], LUTq + ij_2 * ksub);
-                ij++; ij_2++;
+            for (int i = 0; i < M; i++) {
+                round_tab(
+                        LUT2 + ij * ksub, ksub, a, mins[i], LUTq + ij_2 * ksub);
+                ij++;
+                ij_2++;
             }
             memset(LUTq + ij_2 * ksub, 0, ksub * (M2 - M));
             ij_2 += M2 - M;
         }
     }
-    if (a_out) *a_out = a;
-    if (b_out) *b_out = b;
+    if (a_out)
+        *a_out = a;
+    if (b_out)
+        *b_out = b;
 }
 
-
 } // namespace quantize_lut
 
 } // namespace faiss

data/vendor/faiss/faiss/utils/quantize_lut.h

@@ -5,12 +5,10 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
 #pragma once
 
-
-#include <cstdio>
 #include <cstdint>
+#include <cstdio>
 
 namespace faiss {
 
@@ -32,19 +30,23 @@ namespace quantize_lut {
  * @param tab input/output, size (n, d)
  */
 void round_uint8_per_column(
-        float *tab, size_t n, size_t d,
-        float *a_out = nullptr,
-        float *b_out = nullptr
-);
-
+        float* tab,
+        size_t n,
+        size_t d,
+        float* a_out = nullptr,
+        float* b_out = nullptr);
 
 /* affine quantizer, a and b are the affine coefficients
  *
  * @param tab input/output, size (m, n, d)
  */
 void round_uint8_per_column_multi(
-        float *tab, size_t m, size_t n, size_t d,
-        float *a_out = nullptr, float *b_out = nullptr);
+        float* tab,
+        size_t m,
+        size_t n,
+        size_t d,
+        float* a_out = nullptr,
+        float* b_out = nullptr);
 
 /** LUT quantization to uint8 and bias to uint16.
  *
@@ -63,18 +65,18 @@ void round_uint8_per_column_multi(
  */
 
 void quantize_LUT_and_bias(
-        size_t nprobe, size_t M, size_t ksub,
+        size_t nprobe,
+        size_t M,
+        size_t ksub,
         bool lut_is_3d,
-        const float *LUT,
-        const float *bias,
-        uint8_t *LUTq, size_t M2,
-        uint16_t *biasq,
-        float *a_out = nullptr, float *b_out = nullptr
-);
-
+        const float* LUT,
+        const float* bias,
+        uint8_t* LUTq,
+        size_t M2,
+        uint16_t* biasq,
+        float* a_out = nullptr,
+        float* b_out = nullptr);
 
 } // namespace quantize_lut
 
 } // namespace faiss
-
-

data/vendor/faiss/faiss/utils/random.cpp

@@ -15,79 +15,67 @@ namespace faiss {
  * Random data generation functions
  **************************************************/
 
-RandomGenerator::RandomGenerator (int64_t seed)
-    : mt((unsigned int)seed) {}
+RandomGenerator::RandomGenerator(int64_t seed) : mt((unsigned int)seed) {}
 
-int RandomGenerator::rand_int ()
-{
+int RandomGenerator::rand_int() {
     return mt() & 0x7fffffff;
 }
 
-int64_t RandomGenerator::rand_int64 ()
-{
+int64_t RandomGenerator::rand_int64() {
     return int64_t(rand_int()) | int64_t(rand_int()) << 31;
 }
 
-int RandomGenerator::rand_int (int max)
-{
+int RandomGenerator::rand_int(int max) {
     return mt() % max;
 }
 
-float RandomGenerator::rand_float ()
-{
+float RandomGenerator::rand_float() {
     return mt() / float(mt.max());
 }
 
-double RandomGenerator::rand_double ()
-{
+double RandomGenerator::rand_double() {
     return mt() / double(mt.max());
 }
 
-
 /***********************************************************************
  * Random functions in this C file only exist because Torch
  *  counterparts are slow and not multi-threaded.  Typical use is for
  *  more than 1-100 billion values. */
 
-
 /* Generate a set of random floating point values such that x[i] in [0,1]
    multi-threading. For this reason, we rely on re-entreant functions.  */
-void float_rand (float * x, size_t n, int64_t seed)
-{
+void float_rand(float* x, size_t n, int64_t seed) {
     // only try to parallelize on large enough arrays
     const size_t nblock = n < 1024 ? 1 : 1024;
 
-    RandomGenerator rng0 (seed);
-    int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
+    RandomGenerator rng0(seed);
+    int a0 = rng0.rand_int(), b0 = rng0.rand_int();
 
 #pragma omp parallel for
     for (int64_t j = 0; j < nblock; j++) {
-
-        RandomGenerator rng (a0 + j * b0);
+        RandomGenerator rng(a0 + j * b0);
 
         const size_t istart = j * n / nblock;
         const size_t iend = (j + 1) * n / nblock;
 
         for (size_t i = istart; i < iend; i++)
-            x[i] = rng.rand_float ();
+            x[i] = rng.rand_float();
     }
 }
 
-
-void float_randn (float * x, size_t n, int64_t seed)
-{
+void float_randn(float* x, size_t n, int64_t seed) {
     // only try to parallelize on large enough arrays
     const size_t nblock = n < 1024 ? 1 : 1024;
 
-    RandomGenerator rng0 (seed);
-    int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
+    RandomGenerator rng0(seed);
+    int a0 = rng0.rand_int(), b0 = rng0.rand_int();
 
 #pragma omp parallel for
     for (int64_t j = 0; j < nblock; j++) {
-        RandomGenerator rng (a0 + j * b0);
+        RandomGenerator rng(a0 + j * b0);
 
         double a = 0, b = 0, s = 0;
-        int state = 0;  /* generate two number per "do-while" loop */
+        int state = 0; /* generate two number per "do-while" loop */
 
         const size_t istart = j * n / nblock;
         const size_t iend = (j + 1) * n / nblock;
@@ -96,96 +84,84 @@ void float_randn (float * x, size_t n, int64_t seed)
             /* Marsaglia's method (see Knuth) */
             if (state == 0) {
                 do {
-                    a = 2.0 * rng.rand_double () - 1;
-                    b = 2.0 * rng.rand_double () - 1;
+                    a = 2.0 * rng.rand_double() - 1;
+                    b = 2.0 * rng.rand_double() - 1;
                     s = a * a + b * b;
                 } while (s >= 1.0);
                 x[i] = a * sqrt(-2.0 * log(s) / s);
-            }
-            else
+            } else
                 x[i] = b * sqrt(-2.0 * log(s) / s);
             state = 1 - state;
         }
     }
 }
 
-
 /* Integer versions */
-void int64_rand (int64_t * x, size_t n, int64_t seed)
-{
+void int64_rand(int64_t* x, size_t n, int64_t seed) {
     // only try to parallelize on large enough arrays
     const size_t nblock = n < 1024 ? 1 : 1024;
 
-    RandomGenerator rng0 (seed);
-    int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
+    RandomGenerator rng0(seed);
+    int a0 = rng0.rand_int(), b0 = rng0.rand_int();
 
 #pragma omp parallel for
     for (int64_t j = 0; j < nblock; j++) {
-
-        RandomGenerator rng (a0 + j * b0);
+        RandomGenerator rng(a0 + j * b0);
 
         const size_t istart = j * n / nblock;
         const size_t iend = (j + 1) * n / nblock;
         for (size_t i = istart; i < iend; i++)
-            x[i] = rng.rand_int64 ();
+            x[i] = rng.rand_int64();
     }
 }
 
-void int64_rand_max (int64_t * x, size_t n, uint64_t max, int64_t seed)
-{
+void int64_rand_max(int64_t* x, size_t n, uint64_t max, int64_t seed) {
     // only try to parallelize on large enough arrays
     const size_t nblock = n < 1024 ? 1 : 1024;
 
-    RandomGenerator rng0 (seed);
-    int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
+    RandomGenerator rng0(seed);
+    int a0 = rng0.rand_int(), b0 = rng0.rand_int();
 
 #pragma omp parallel for
     for (int64_t j = 0; j < nblock; j++) {
-
-        RandomGenerator rng (a0 + j * b0);
+        RandomGenerator rng(a0 + j * b0);
 
         const size_t istart = j * n / nblock;
         const size_t iend = (j + 1) * n / nblock;
         for (size_t i = istart; i < iend; i++)
-            x[i] = rng.rand_int64 () % max;
+            x[i] = rng.rand_int64() % max;
     }
 }
 
+void rand_perm(int* perm, size_t n, int64_t seed) {
+    for (size_t i = 0; i < n; i++)
+        perm[i] = i;
 
-void rand_perm (int *perm, size_t n, int64_t seed)
-{
-    for (size_t i = 0; i < n; i++) perm[i] = i;
-
-    RandomGenerator rng (seed);
+    RandomGenerator rng(seed);
 
     for (size_t i = 0; i + 1 < n; i++) {
-        int i2 = i + rng.rand_int (n - i);
+        int i2 = i + rng.rand_int(n - i);
         std::swap(perm[i], perm[i2]);
     }
 }
 
-
-
-
-void byte_rand (uint8_t * x, size_t n, int64_t seed)
-{
+void byte_rand(uint8_t* x, size_t n, int64_t seed) {
     // only try to parallelize on large enough arrays
     const size_t nblock = n < 1024 ? 1 : 1024;
 
-    RandomGenerator rng0 (seed);
-    int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
+    RandomGenerator rng0(seed);
+    int a0 = rng0.rand_int(), b0 = rng0.rand_int();
 
 #pragma omp parallel for
     for (int64_t j = 0; j < nblock; j++) {
-
-        RandomGenerator rng (a0 + j * b0);
+        RandomGenerator rng(a0 + j * b0);
 
         const size_t istart = j * n / nblock;
         const size_t iend = (j + 1) * n / nblock;
 
         size_t i;
         for (i = istart; i < iend; i++)
-            x[i] = rng.rand_int64 ();
+            x[i] = rng.rand_int64();
     }
 }