faiss 0.4.1 → 0.4.2

data/vendor/faiss/faiss/utils/distances.cpp

@@ -77,8 +77,9 @@ void fvec_norms_L2sqr(
         size_t d,
         size_t nx) {
 #pragma omp parallel for if (nx > 10000)
-    for (int64_t i = 0; i < nx; i++)
+    for (int64_t i = 0; i < nx; i++) {
         nr[i] = fvec_norm_L2sqr(x + i * d, d);
+    }
 }
 
 // The following is a workaround to a problem
@@ -208,8 +209,9 @@ void exhaustive_inner_product_blas(
         size_t ny,
         BlockResultHandler& res) {
     // BLAS does not like empty matrices
-    if (nx == 0 || ny == 0)
+    if (nx == 0 || ny == 0) {
         return;
+    }
 
     /* block sizes */
     const size_t bs_x = distance_compute_blas_query_bs;
@@ -218,15 +220,17 @@ void exhaustive_inner_product_blas(
 
     for (size_t i0 = 0; i0 < nx; i0 += bs_x) {
         size_t i1 = i0 + bs_x;
-        if (i1 > nx)
+        if (i1 > nx) {
             i1 = nx;
+        }
 
         res.begin_multiple(i0, i1);
 
         for (size_t j0 = 0; j0 < ny; j0 += bs_y) {
             size_t j1 = j0 + bs_y;
-            if (j1 > ny)
+            if (j1 > ny) {
                 j1 = ny;
+            }
             /* compute the actual dot products */
             {
                 float one = 1, zero = 0;
@@ -265,8 +269,9 @@ void exhaustive_L2sqr_blas_default_impl(
         BlockResultHandler& res,
         const float* y_norms = nullptr) {
     // BLAS does not like empty matrices
-    if (nx == 0 || ny == 0)
+    if (nx == 0 || ny == 0) {
         return;
+    }
 
     /* block sizes */
     const size_t bs_x = distance_compute_blas_query_bs;
@@ -287,15 +292,17 @@ void exhaustive_L2sqr_blas_default_impl(
 
     for (size_t i0 = 0; i0 < nx; i0 += bs_x) {
         size_t i1 = i0 + bs_x;
-        if (i1 > nx)
+        if (i1 > nx) {
             i1 = nx;
+        }
 
         res.begin_multiple(i0, i1);
 
         for (size_t j0 = 0; j0 < ny; j0 += bs_y) {
             size_t j1 = j0 + bs_y;
-            if (j1 > ny)
+            if (j1 > ny) {
                 j1 = ny;
+            }
             /* compute the actual dot products */
             {
                 float one = 1, zero = 0;
@@ -327,8 +334,9 @@ void exhaustive_L2sqr_blas_default_impl(
                     }
                     // negative values can occur for identical vectors
                     // due to roundoff errors
-                    if (dis < 0)
+                    if (dis < 0) {
                         dis = 0;
+                    }
 
                     *ip_line = dis;
                     ip_line++;
@@ -363,8 +371,9 @@ void exhaustive_L2sqr_blas_cmax_avx2(
         Top1BlockResultHandler<CMax<float, int64_t>>& res,
         const float* y_norms) {
     // BLAS does not like empty matrices
-    if (nx == 0 || ny == 0)
+    if (nx == 0 || ny == 0) {
         return;
+    }
 
     /* block sizes */
     const size_t bs_x = distance_compute_blas_query_bs;
@@ -385,15 +394,17 @@ void exhaustive_L2sqr_blas_cmax_avx2(
 
     for (size_t i0 = 0; i0 < nx; i0 += bs_x) {
         size_t i1 = i0 + bs_x;
-        if (i1 > nx)
+        if (i1 > nx) {
             i1 = nx;
+        }
 
         res.begin_multiple(i0, i1);
 
         for (size_t j0 = 0; j0 < ny; j0 += bs_y) {
             size_t j1 = j0 + bs_y;
-            if (j1 > ny)
+            if (j1 > ny) {
                 j1 = ny;
+            }
             /* compute the actual dot products */
             {
                 float one = 1, zero = 0;
@@ -519,8 +530,9 @@ void exhaustive_L2sqr_blas_cmax_avx2(
 
                     // negative values can occur for identical vectors
                     //    due to roundoff errors.
-                    if (distance_candidate < 0)
+                    if (distance_candidate < 0) {
                         distance_candidate = 0;
+                    }
 
                     int64_t index_candidate = min_indices_scalar[jv] + j0;
 
@@ -540,8 +552,9 @@ void exhaustive_L2sqr_blas_cmax_avx2(
                     float dis = x_norms[i] + y_norms[idx_j + j0] - 2 * ip;
                     // negative values can occur for identical vectors
                     //    due to roundoff errors.
-                    if (dis < 0)
+                    if (dis < 0) {
                         dis = 0;
+                    }
 
                     if (current_min_distance > dis) {
                         current_min_distance = dis;
@@ -1142,33 +1155,40 @@ void pairwise_L2sqr(
         int64_t ldq,
         int64_t ldb,
         int64_t ldd) {
-    if (nq == 0 || nb == 0)
+    if (nq == 0 || nb == 0) {
         return;
-    if (ldq == -1)
+    }
+    if (ldq == -1) {
         ldq = d;
-    if (ldb == -1)
+    }
+    if (ldb == -1) {
         ldb = d;
-    if (ldd == -1)
+    }
+    if (ldd == -1) {
         ldd = nb;
+    }
 
     // store in beginning of distance matrix to avoid malloc
     float* b_norms = dis;
 
 #pragma omp parallel for if (nb > 1)
-    for (int64_t i = 0; i < nb; i++)
+    for (int64_t i = 0; i < nb; i++) {
         b_norms[i] = fvec_norm_L2sqr(xb + i * ldb, d);
+    }
 
 #pragma omp parallel for
     for (int64_t i = 1; i < nq; i++) {
         float q_norm = fvec_norm_L2sqr(xq + i * ldq, d);
-        for (int64_t j = 0; j < nb; j++)
+        for (int64_t j = 0; j < nb; j++) {
             dis[i * ldd + j] = q_norm + b_norms[j];
+        }
     }
 
     {
         float q_norm = fvec_norm_L2sqr(xq, d);
-        for (int64_t j = 0; j < nb; j++)
+        for (int64_t j = 0; j < nb; j++) {
             dis[j] += q_norm;
+        }
     }
 
     {
@@ -1200,8 +1220,9 @@ void inner_product_to_L2sqr(
 #pragma omp parallel for
     for (int64_t j = 0; j < n1; j++) {
         float* disj = dis + j * n2;
-        for (size_t i = 0; i < n2; i++)
+        for (size_t i = 0; i < n2; i++) {
             disj[i] = nr1[j] + nr2[i] - 2 * disj[i];
+        }
     }
 }
 

data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp

@@ -7,9 +7,9 @@
 
 #include <faiss/utils/distances_fused/distances_fused.h>
 
-#include <faiss/impl/platform_macros.h>
+#include <faiss/impl/platform_macros.h> // NOLINT
 
-#include <faiss/utils/distances_fused/avx512.h>
+#include <faiss/utils/distances_fused/avx512.h> // NOLINT
 #include <faiss/utils/distances_fused/simdlib_based.h>
 
 namespace faiss {

data/vendor/faiss/faiss/utils/distances_fused/simdlib_based.cpp

@@ -260,7 +260,7 @@ void exhaustive_L2sqr_fused_cmax(
     const size_t nx_p = (nx / NX_POINTS_PER_LOOP) * NX_POINTS_PER_LOOP;
     // the main loop.
 #pragma omp parallel for schedule(dynamic)
-    for (size_t i = 0; i < nx_p; i += NX_POINTS_PER_LOOP) {
+    for (int64_t i = 0; i < nx_p; i += NX_POINTS_PER_LOOP) {
         kernel<DIM, NX_POINTS_PER_LOOP, NY_POINTS_PER_LOOP>(
                 x, y, y_transposed.data(), ny, res, y_norms, i);
     }

data/vendor/faiss/faiss/utils/distances_simd.cpp

@@ -3252,8 +3252,9 @@ void fvec_inner_products_ny(
         float bf,
         const float* b,
         float* c) {
-    for (size_t i = 0; i < n; i++)
+    for (size_t i = 0; i < n; i++) {
         c[i] = a[i] + bf * b[i];
+    }
 }
 
 #if defined(__AVX512F__)
@@ -3536,10 +3537,11 @@ int fvec_madd_and_argmin(
         float bf,
         const float* b,
         float* c) {
-    if ((n & 3) == 0 && ((((long)a) | ((long)b) | ((long)c)) & 15) == 0)
+    if ((n & 3) == 0 && ((((long)a) | ((long)b) | ((long)c)) & 15) == 0) {
         return fvec_madd_and_argmin_sse(n, a, bf, b, c);
-    else
+    } else {
         return fvec_madd_and_argmin_ref(n, a, bf, b, c);
+    }
 }
 
 #elif defined(__aarch64__)

data/vendor/faiss/faiss/utils/extra_distances-inl.h

@@ -153,14 +153,37 @@ inline float VectorDistance<METRIC_NaNEuclidean>::operator()(
 }
 
 template <>
-inline float VectorDistance<METRIC_ABS_INNER_PRODUCT>::operator()(
+inline float VectorDistance<METRIC_GOWER>::operator()(
         const float* x,
         const float* y) const {
     float accu = 0;
+    size_t valid_dims = 0;
+
     for (size_t i = 0; i < d; i++) {
-        accu += fabs(x[i] * y[i]);
+        if (std::isnan(x[i]) || std::isnan(y[i])) {
+            continue;
+        }
+
+        if (x[i] >= 0 && y[i] >= 0) {
+            if (x[i] > 1 || y[i] > 1) {
+                return std::numeric_limits<float>::quiet_NaN();
+            }
+            // Numeric dimensions are in [0,1]
+            accu += fabs(x[i] - y[i]);
+        } else if (x[i] < 0 && y[i] < 0) {
+            // Categorical dimensions are negative values
+            accu += float(int(x[i] != y[i]));
+        } else {
+            // Invalid representation
+            return std::numeric_limits<float>::quiet_NaN();
+        }
+        valid_dims++;
     }
-    return accu;
+
+    if (valid_dims == 0) {
+        return std::numeric_limits<float>::quiet_NaN();
+    }
+    return accu / valid_dims;
 }
 
 /***************************************************************************
@@ -193,7 +216,7 @@ typename Consumer::T dispatch_VectorDistance(
         DISPATCH_VD(METRIC_JensenShannon);
         DISPATCH_VD(METRIC_Jaccard);
         DISPATCH_VD(METRIC_NaNEuclidean);
-        DISPATCH_VD(METRIC_ABS_INNER_PRODUCT);
+        DISPATCH_VD(METRIC_GOWER);
         default:
             FAISS_THROW_FMT("Invalid metric %d", metric);
     }

data/vendor/faiss/faiss/utils/extra_distances.cpp

@@ -155,14 +155,18 @@ void pairwise_extra_distances(
         int64_t ldq,
         int64_t ldb,
         int64_t ldd) {
-    if (nq == 0 || nb == 0)
+    if (nq == 0 || nb == 0) {
         return;
-    if (ldq == -1)
+    }
+    if (ldq == -1) {
         ldq = d;
-    if (ldb == -1)
+    }
+    if (ldb == -1) {
         ldb = d;
-    if (ldd == -1)
+    }
+    if (ldd == -1) {
         ldd = nb;
+    }
 
     Run_pairwise_extra_distances run;
     dispatch_VectorDistance(

data/vendor/faiss/faiss/utils/hamming.cpp

@@ -53,8 +53,9 @@ void hammings(
     for (i = 0; i < n1; i++) {
         const uint64_t* __restrict bs1_ = bs1 + i * nwords;
         hamdis_t* __restrict dis_ = dis + i * n2;
-        for (j = 0; j < n2; j++)
+        for (j = 0; j < n2; j++) {
             dis_[j] = hamming<nbits>(bs1_, bs2 + j * nwords);
+        }
     }
 }
 
@@ -70,8 +71,9 @@ void hammings(
     for (i = 0; i < n1; i++) {
         const uint64_t* __restrict bs1_ = bs1 + i * nwords;
         hamdis_t* __restrict dis_ = dis + i * n2;
-        for (j = 0; j < n2; j++)
+        for (j = 0; j < n2; j++) {
             dis_[j] = hamming(bs1_, bs2 + j * nwords, nwords);
+        }
     }
 }
 
@@ -92,8 +94,9 @@ void hamming_count_thres(
         bs2 = bs2_;
         for (j = 0; j < n2; j++) {
             /* collect the match only if this satisfies the threshold */
-            if (hamming<nbits>(bs1, bs2) <= ht)
+            if (hamming<nbits>(bs1, bs2) <= ht) {
                 posm++;
+            }
             bs2 += nwords;
         }
         bs1 += nwords; /* next signature */
@@ -114,8 +117,9 @@ void crosshamming_count_thres(
         const uint64_t* bs2 = bs1 + 2;
         for (j = i + 1; j < n; j++) {
             /* collect the match only if this satisfies the threshold */
-            if (hamming<nbits>(bs1, bs2) <= ht)
+            if (hamming<nbits>(bs1, bs2) <= ht) {
                 posm++;
+            }
             bs2 += nwords;
         }
         bs1 += nwords;
@@ -175,8 +179,9 @@ void hammings_knn_hc(
         ApproxTopK_mode_t approx_topk_mode = ApproxTopK_mode_t::EXACT_TOPK,
         const faiss::IDSelector* sel = nullptr) {
     size_t k = ha->k;
-    if (init_heap)
+    if (init_heap) {
         ha->heapify();
+    }
 
     const size_t block_size = hamming_batch_size;
     for (size_t j0 = 0; j0 < n2; j0 += block_size) {
@@ -229,8 +234,9 @@ void hammings_knn_hc(
             }
         }
     }
-    if (order)
+    if (order) {
         ha->reorder();
+    }
 }
 
 /* Return closest neighbors w.r.t Hamming distance, using max count. */
@@ -364,8 +370,9 @@ void fvec2bitvec(const float* __restrict x, uint8_t* __restrict b, size_t d) {
         uint8_t mask = 1;
         int nj = i + 8 <= d ? 8 : d - i;
         for (int j = 0; j < nj; j++) {
-            if (x[i + j] >= 0)
+            if (x[i + j] >= 0) {
                 w |= mask;
+            }
             mask <<= 1;
         }
         *b = w;
@@ -382,8 +389,9 @@ void fvecs2bitvecs(
         size_t n) {
     const int64_t ncodes = ((d + 7) / 8);
 #pragma omp parallel for if (n > 100000)
-    for (int64_t i = 0; i < n; i++)
+    for (int64_t i = 0; i < n; i++) {
         fvec2bitvec(x + i * d, b + i * ncodes, d);
+    }
 }
 
 void bitvecs2fvecs(
@@ -667,8 +675,9 @@ void generalized_hammings_knn_hc(
     int na = ha->nh;
     int k = ha->k;
 
-    if (ordered)
+    if (ordered) {
         ha->heapify();
+    }
 
 #pragma omp parallel for
     for (int i = 0; i < na; i++) {
@@ -698,8 +707,9 @@ void generalized_hammings_knn_hc(
         }
     }
 
-    if (ordered)
+    if (ordered) {
         ha->reorder();
+    }
 }
 
 void pack_bitstrings(

data/vendor/faiss/faiss/utils/partitioning.cpp

@@ -1193,12 +1193,14 @@ void simd_histogram_8(
 
     // complete with remaining bins
     for (int i = (n & ~15); i < n; i++) {
-        if (data[i] < min)
+        if (data[i] < min) {
             continue;
+        }
         uint16_t v = data[i] - min;
         v >>= shift;
-        if (v < 8)
+        if (v < 8) {
             hist[v]++;
+        }
     }
 }
 
@@ -1247,12 +1249,14 @@ void simd_histogram_16(
     }
 
     for (int i = (n & ~15); i < n; i++) {
-        if (data[i] < min)
+        if (data[i] < min) {
             continue;
+        }
         uint16_t v = data[i] - min;
         v >>= shift;
-        if (v < 16)
+        if (v < 16) {
             hist[v]++;
+        }
     }
 }
 

data/vendor/faiss/faiss/utils/quantize_lut.cpp

@@ -28,8 +28,9 @@ namespace {
 float tab_min(const float* tab, size_t n) {
     float min = HUGE_VAL;
     for (int i = 0; i < n; i++) {
-        if (tab[i] < min)
+        if (tab[i] < min) {
             min = tab[i];
+        }
     }
     return min;
 }
@@ -37,8 +38,9 @@ float tab_min(const float* tab, size_t n) {
 float tab_max(const float* tab, size_t n) {
     float max = -HUGE_VAL;
     for (int i = 0; i < n; i++) {
-        if (tab[i] > max)
+        if (tab[i] > max) {
             max = tab[i];
+        }
     }
     return max;
 }
@@ -79,10 +81,12 @@ void round_uint8_per_column(
         b += mins[i];
         round_tab(tab + i * d, d, a, mins[i]);
     }
-    if (a_out)
+    if (a_out) {
         *a_out = a;
-    if (b_out)
+    }
+    if (b_out) {
         *b_out = b;
+    }
 }
 
 void round_uint8_per_column_multi(
@@ -115,14 +119,16 @@ void round_uint8_per_column_multi(
             round_tab(tab + (j * n + i) * d, d, a, mins[i]);
         }
     }
-    if (a_out)
+    if (a_out) {
         *a_out = a;
-    if (b_out)
+    }
+    if (b_out) {
         *b_out = b;
+    }
 }
 
 // translation of
-// https://github.com/fairinternal/faiss_improvements/blob/7122c3cc6ddb0a371d8aa6f1309cd8bcf2335e61/LUT_quantization.ipynb
+// https://gist.github.com/mdouze/f3a05bff5186c1874a77356452297357#file-lut_quantization-ipynb
 void quantize_LUT_and_bias(
         size_t nprobe,
         size_t M,
@@ -264,10 +270,12 @@ void quantize_LUT_and_bias(
             ij_2 += M2 - M;
         }
     }
-    if (a_out)
+    if (a_out) {
         *a_out = a;
-    if (b_out)
+    }
+    if (b_out) {
         *b_out = b;
+    }
 }
 
 void aq_quantize_LUT_and_bias(