faiss 0.1.2 → 0.1.3

data/vendor/faiss/{utils → faiss/utils}/distances_simd.cpp

@@ -14,7 +14,7 @@
 #include <cstring>
 #include <cmath>
 
-#ifdef __SSE__
+#ifdef __SSE3__
 #include <immintrin.h>
 #endif
 
@@ -133,7 +133,7 @@ void fvec_L2sqr_ny_ref (float * dis,
  * SSE and AVX implementations
  */
 
-#ifdef __SSE__
+#ifdef __SSE3__
 
 // reads 0 <= d < 4 floats as __m128
 static inline __m128 masked_read (int d, const float *x)
@@ -477,7 +477,7 @@ float fvec_Linf (const float * x, const float * y, size_t d)
     return  _mm_cvtss_f32 (msum2);
 }
 
-#elif defined(__SSE__) // But not AVX
+#elif defined(__SSE3__) // But not AVX
 
 float fvec_L1 (const float * x, const float * y, size_t d)
 {
@@ -677,7 +677,7 @@ static inline void fvec_madd_ref (size_t n, const float *a,
         c[i] = a[i] + bf * b[i];
 }
 
-#ifdef __SSE__
+#ifdef __SSE3__
 
 static inline void fvec_madd_sse (size_t n, const float *a,
                                   float bf, const float *b, float *c) {
@@ -730,7 +730,7 @@ static inline int fvec_madd_and_argmin_ref (size_t n, const float *a,
     return imin;
 }
 
-#ifdef __SSE__
+#ifdef __SSE3__
 
 static inline int fvec_madd_and_argmin_sse (
         size_t n, const float *a,

data/vendor/faiss/{utils → faiss/utils}/extra_distances.cpp

@@ -9,6 +9,7 @@
 
 #include <faiss/utils/distances.h>
 
+#include <algorithm>
 #include <cmath>
 #include <omp.h>
 
@@ -163,7 +164,7 @@ void knn_extra_metrics_template (
         size_t i1 = std::min(i0 + check_period, nx);
 
 #pragma omp parallel for
-        for (size_t i = i0; i < i1; i++) {
+        for (int64_t i = i0; i < i1; i++) {
             const float * x_i = x + i * d;
             const float * y_j = y;
             size_t j;
@@ -245,7 +246,7 @@ void pairwise_extra_distances (
     switch(mt) {
 #define HANDLE_VAR(kw)                                          \
      case METRIC_ ## kw: {                                      \
-        VectorDistance ## kw vd({(size_t)d});                   \
+        VectorDistance ## kw vd = {(size_t)d};                  \
         pairwise_extra_distances_template (vd, nq, xq, nb, xb,  \
                                            dis, ldq, ldb, ldd); \
         break;                                                  \
@@ -258,7 +259,7 @@ void pairwise_extra_distances (
         HANDLE_VAR(JensenShannon);
 #undef HANDLE_VAR
     case METRIC_Lp: {
-        VectorDistanceLp vd({(size_t)d, metric_arg});
+        VectorDistanceLp vd = {(size_t)d, metric_arg};
         pairwise_extra_distances_template (vd, nq, xq, nb, xb,
                                            dis, ldq, ldb, ldd);
         break;
@@ -280,7 +281,7 @@ void knn_extra_metrics (
     switch(mt) {
 #define HANDLE_VAR(kw)                                          \
      case METRIC_ ## kw: {                                      \
-        VectorDistance ## kw vd({(size_t)d});                   \
+        VectorDistance ## kw vd = {(size_t)d};                  \
         knn_extra_metrics_template (vd, x, y, nx, ny, res);     \
         break;                                                  \
     }
@@ -292,7 +293,7 @@ void knn_extra_metrics (
         HANDLE_VAR(JensenShannon);
 #undef HANDLE_VAR
     case METRIC_Lp: {
-        VectorDistanceLp vd({(size_t)d, metric_arg});
+        VectorDistanceLp vd = {(size_t)d, metric_arg};
         knn_extra_metrics_template (vd, x, y, nx, ny, res);
         break;
     }
@@ -311,7 +312,7 @@ DistanceComputer *get_extra_distance_computer (
     switch(mt) {
 #define HANDLE_VAR(kw)                                                  \
      case METRIC_ ## kw: {                                              \
-        VectorDistance ## kw vd({(size_t)d});                           \
+        VectorDistance ## kw vd = {(size_t)d};                          \
         return new ExtraDistanceComputer<VectorDistance ## kw>(vd, xb, nb); \
     }
         HANDLE_VAR(L2);
@@ -322,7 +323,7 @@ DistanceComputer *get_extra_distance_computer (
         HANDLE_VAR(JensenShannon);
 #undef HANDLE_VAR
     case METRIC_Lp: {
-        VectorDistanceLp vd({(size_t)d, metric_arg});
+        VectorDistanceLp vd = {(size_t)d, metric_arg};
         return new ExtraDistanceComputer<VectorDistanceLp> (vd, xb, nb);
         break;
     }

data/vendor/faiss/{utils → faiss/utils}/hamming-inl.h

@@ -5,15 +5,13 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-
-
 namespace faiss {
 
 
 inline BitstringWriter::BitstringWriter(uint8_t *code, int code_size):
     code (code), code_size (code_size), i(0)
 {
-    bzero (code, code_size);
+    memset (code, 0, code_size);
 }
 
 inline void BitstringWriter::write(uint64_t x, int nbit) {

data/vendor/faiss/{utils → faiss/utils}/hamming.cpp

@@ -26,6 +26,7 @@
 
 #include <faiss/utils/hamming.h>
 
+#include <algorithm>
 #include <vector>
 #include <memory>
 #include <stdio.h>
@@ -280,7 +281,7 @@ void hammings_knn_hc (
     for (size_t j0 = 0; j0 < n2; j0 += block_size) {
       const size_t j1 = std::min(j0 + block_size, n2);
 #pragma omp parallel for
-      for (size_t i = 0; i < ha->nh; i++) {
+      for (int64_t i = 0; i < ha->nh; i++) {
         HammingComputer hc (bs1 + i * bytes_per_code, bytes_per_code);
 
         const uint8_t * bs2_ = bs2 + j0 * bytes_per_code;
@@ -332,7 +333,7 @@ void hammings_knn_mc (
   for (size_t j0 = 0; j0 < nb; j0 += block_size) {
     const size_t j1 = std::min(j0 + block_size, nb);
 #pragma omp parallel for
-    for (size_t i = 0; i < na; ++i) {
+    for (int64_t i = 0; i < na; ++i) {
       for (size_t j = j0; j < j1; ++j) {
         cs[i].update_counter(b + j * bytes_per_code, j);
       }
@@ -379,7 +380,7 @@ void hammings_knn_hc_1 (
     }
 
 #pragma omp parallel for
-    for (size_t i = 0; i < ha->nh; i++) {
+    for (int64_t i = 0; i < ha->nh; i++) {
         const uint64_t bs1_ = bs1 [i];
         const uint64_t * bs2_ = bs2;
         hamdis_t dis;
@@ -435,7 +436,7 @@ void fvecs2bitvecs (const float * x, uint8_t * b, size_t d, size_t n)
 {
     const int64_t ncodes = ((d + 7) / 8);
 #pragma omp parallel for if(n > 100000)
-    for (size_t i = 0; i < n; i++)
+    for (int64_t i = 0; i < n; i++)
         fvec2bitvec (x + i * d, b + i * ncodes, d);
 }
 
@@ -449,7 +450,7 @@ void bitvecs2fvecs (
 
     const int64_t ncodes = ((d + 7) / 8);
 #pragma omp parallel for if(n > 100000)
-    for (size_t i = 0; i < n; i++) {
+    for (int64_t i = 0; i < n; i++) {
         binary_to_real (d, b + i * ncodes, x + i * d);
     }
 }
@@ -497,7 +498,7 @@ void bitvec_shuffle (size_t n, size_t da, size_t db,
     size_t ldb = (db + 7) / 8;
 
 #pragma omp parallel for if(n > 10000)
-    for (size_t i = 0; i < n; i++) {
+    for (int64_t i = 0; i < n; i++) {
         const uint8_t *ai = a + i * lda;
         uint8_t *bi = b + i * ldb;
         memset (bi, 0, ldb);
@@ -653,7 +654,7 @@ void hamming_range_search_template (
         RangeSearchPartialResult pres (res);
 
 #pragma omp for
-        for (size_t i = 0; i < na; i++) {
+        for (int64_t i = 0; i < na; i++) {
              HammingComputer hc (a + i * code_size, code_size);
             const uint8_t * yi = b;
             RangeQueryResult & qres = pres.new_result (i);

data/vendor/faiss/{utils → faiss/utils}/hamming.h

@@ -27,6 +27,12 @@
 
 #include <stdint.h>
 
+#ifdef _MSC_VER
+#include <intrin.h>
+#define __builtin_popcountl __popcnt64
+#endif // _MSC_VER
+
+#include <faiss/impl/platform_macros.h>
 #include <faiss/utils/Heap.h>
 
 
@@ -109,7 +115,7 @@ struct BitstringReader {
 
 
 
-extern size_t hamming_batch_size;
+FAISS_API extern size_t hamming_batch_size;
 
 inline int popcount64(uint64_t x) {
     return __builtin_popcountl(x);

data/vendor/faiss/{utils → faiss/utils}/random.cpp

@@ -61,7 +61,7 @@ void float_rand (float * x, size_t n, int64_t seed)
     int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
 
 #pragma omp parallel for
-    for (size_t j = 0; j < nblock; j++) {
+    for (int64_t j = 0; j < nblock; j++) {
 
         RandomGenerator rng (a0 + j * b0);
 
@@ -83,7 +83,7 @@ void float_randn (float * x, size_t n, int64_t seed)
     int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
 
 #pragma omp parallel for
-    for (size_t j = 0; j < nblock; j++) {
+    for (int64_t j = 0; j < nblock; j++) {
         RandomGenerator rng (a0 + j * b0);
 
         double a = 0, b = 0, s = 0;
@@ -120,7 +120,7 @@ void int64_rand (int64_t * x, size_t n, int64_t seed)
     int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
 
 #pragma omp parallel for
-    for (size_t j = 0; j < nblock; j++) {
+    for (int64_t j = 0; j < nblock; j++) {
 
         RandomGenerator rng (a0 + j * b0);
 
@@ -140,7 +140,7 @@ void int64_rand_max (int64_t * x, size_t n, uint64_t max, int64_t seed)
     int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
 
 #pragma omp parallel for
-    for (size_t j = 0; j < nblock; j++) {
+    for (int64_t j = 0; j < nblock; j++) {
 
         RandomGenerator rng (a0 + j * b0);
 
@@ -176,7 +176,7 @@ void byte_rand (uint8_t * x, size_t n, int64_t seed)
     int a0 = rng0.rand_int (), b0 = rng0.rand_int ();
 
 #pragma omp parallel for
-    for (size_t j = 0; j < nblock; j++) {
+    for (int64_t j = 0; j < nblock; j++) {
 
         RandomGenerator rng (a0 + j * b0);
 

data/vendor/faiss/{utils → faiss/utils}/utils.cpp

@@ -14,9 +14,16 @@
 #include <cstring>
 #include <cmath>
 
-#include <sys/time.h>
 #include <sys/types.h>
+
+#ifdef _MSC_VER
+#define NOMINMAX
+#include <windows.h>
+#undef NOMINMAX
+#else
+#include <sys/time.h>
 #include <unistd.h>
+#endif // !_MSC_VER
 
 #include <omp.h>
 
@@ -65,11 +72,22 @@ int sgemv_(const char *trans, FINTEGER *m, FINTEGER *n, float *alpha,
 
 namespace faiss {
 
+#ifdef _MSC_VER
+double getmillisecs() {
+    LARGE_INTEGER ts;
+    LARGE_INTEGER freq;
+    QueryPerformanceFrequency(&freq);
+    QueryPerformanceCounter(&ts);
+
+    return (ts.QuadPart * 1e3) / freq.QuadPart;
+}
+#else // _MSC_VER
 double getmillisecs () {
     struct timeval tv;
     gettimeofday (&tv, nullptr);
     return tv.tv_sec * 1e3 + tv.tv_usec * 1e-3;
 }
+#endif // _MSC_VER
 
 uint64_t get_cycles () {
 #ifdef  __x86_64__
@@ -103,11 +121,11 @@ size_t get_mem_usage_kb ()
     return sz;
 }
 
-#elif __APPLE__
+#else
 
 size_t get_mem_usage_kb ()
 {
-    fprintf(stderr, "WARN: get_mem_usage_kb not implemented on the mac\n");
+    fprintf(stderr, "WARN: get_mem_usage_kb not implemented on current architecture\n");
     return 0;
 }
 
@@ -173,25 +191,6 @@ void reflection_ref (const float * u, float * x, size_t n, size_t d, size_t nu)
  * Some matrix manipulation functions
  ***************************************************************************/
 
-
-/* This function exists because the Torch counterpart is extremly slow
-   (not multi-threaded + unexpected overhead even in single thread).
-   It is here to implement the usual property |x-y|^2=|x|^2+|y|^2-2<x|y>  */
-void inner_product_to_L2sqr (float * __restrict dis,
-                             const float * nr1,
-                             const float * nr2,
-                             size_t n1, size_t n2)
-{
-
-#pragma omp parallel for
-    for (size_t j = 0 ; j < n1 ; j++) {
-        float * disj = dis + j * n2;
-        for (size_t i = 0 ; i < n2 ; i++)
-            disj[i] = nr1[j] + nr2[i] - 2 * disj[i];
-    }
-}
-
-
 void matrix_qr (int m, int n, float *a)
 {
     FAISS_THROW_IF_NOT (m >= n);
@@ -251,7 +250,7 @@ size_t merge_result_table_with (size_t n, size_t k,
         std::vector<float> tmpD (k);
 
 #pragma omp for
-        for (size_t i = 0; i < n; i++) {
+        for (int64_t i = 0; i < n; i++) {
             int64_t *lI0 = I0 + i * k;
             float *lD0 = D0 + i * k;
             const int64_t *lI1 = I1 + i * k;
@@ -319,7 +318,7 @@ size_t ranklist_intersection_size (size_t k1, const int64_t *v1,
         }
         k2 = wp;
     }
-    const int64_t seen_flag = 1L << 60;
+    const int64_t seen_flag = int64_t{1} << 60;
     size_t count = 0;
     for (size_t i = 0; i < k1; i++) {
         int64_t q = v1 [i];
@@ -436,7 +435,7 @@ namespace {
         }
 
         // compute sub-ranges for each thread
-        SegmentS s1s[nt], s2s[nt], sws[nt];
+        std::vector<SegmentS> s1s(nt), s2s(nt), sws(nt);
         s2s[0].i0 = s2.i0;
         s2s[nt - 1].i1 = s2.i1;
 
@@ -529,7 +528,7 @@ void fvec_argsort_parallel (size_t n, const float *vals,
 
     ArgsortComparator comp = {vals};
 
-    SegmentS segs[nt];
+    std::vector<SegmentS> segs(nt);
 
     // independent sorts
 #pragma omp parallel for
@@ -598,8 +597,8 @@ const float *fvecs_maybe_subsample (
 
     size_t n2 = nmax;
     if (verbose) {
-        printf ("  Input training set too big (max size is %ld), sampling "
-                "%ld / %ld vectors\n", nmax, n2, *n);
+        printf ("  Input training set too big (max size is %zd), sampling "
+                "%zd / %zd vectors\n", nmax, n2, *n);
     }
     std::vector<int> subset (*n);
     rand_perm (subset.data (), *n, seed);

data/vendor/faiss/{utils → faiss/utils}/utils.h

@@ -18,6 +18,10 @@
 
 #include <stdint.h>
 
+#ifdef _MSC_VER
+#define strtok_r strtok_s
+#endif // _MSC_VER
+
 #include <faiss/utils/Heap.h>
 
 

data/vendor/faiss/misc/test_blas.cpp

@@ -7,6 +7,7 @@
 
 #include <cstdio>
 #include <cstdlib>
+#include <random>
 
 #undef FINTEGER
 #define FINTEGER long
@@ -31,8 +32,10 @@ int sgeqrf_ (FINTEGER *m, FINTEGER *n, float *a, FINTEGER *lda,
 float *new_random_vec(int size)
 {
     float *x = new float[size];
+    std::mt19937 rng;
+    std::uniform_real_distribution<> distrib;
     for (int i = 0; i < size; i++)
-        x[i] = drand48();
+        x[i] = distrib(rng);
     return x;
 }
 

data/vendor/faiss/tests/test_binary_flat.cpp

@@ -23,8 +23,6 @@ TEST(BinaryFlat, accuracy) {
   // make the index object and train it
   faiss::IndexBinaryFlat index(d);
 
-  srand(35);
-
   std::vector<uint8_t> database(nb * (d / 8));
   for (size_t i = 0; i < nb * (d / 8); i++) {
     database[i] = rand() % 0x100;

data/vendor/faiss/tests/test_dealloc_invlists.cpp

@@ -10,6 +10,7 @@
 
 #include <memory>
 #include <vector>
+#include <random>
 
 #include <gtest/gtest.h>
 
@@ -19,6 +20,7 @@
 #include <faiss/index_io.h>
 #include <faiss/IVFlib.h>
 
+
 using namespace faiss;
 
 namespace {
@@ -38,12 +40,15 @@ size_t nb = 1000;
 // nb of queries
 size_t nq = 200;
 
+std::mt19937 rng;
 
 std::vector<float> make_data(size_t n)
 {
     std::vector <float> database (n * d);
+    std::uniform_real_distribution<> distrib;
+
     for (size_t i = 0; i < n * d; i++) {
-        database[i] = drand48();
+        database[i] = distrib(rng);
     }
     return database;
 }