faiss 0.5.3 → 0.6.1

data/vendor/faiss/faiss/utils/sorting.cpp

@@ -134,9 +134,9 @@ void fvec_argsort(size_t n, const float* vals, size_t* perm) {
 }
 
 void fvec_argsort_parallel(size_t n, const float* vals, size_t* perm) {
-    size_t* perm2 = new size_t[n];
+    std::vector<size_t> perm2(n);
     // 2 result tables, during merging, flip between them
-    size_t *permB = perm2, *permA = perm;
+    size_t *permB = perm2.data(), *permA = perm;
 
     int nt = omp_get_max_threads();
     { // prepare correct permutation so that the result ends in perm
@@ -148,8 +148,8 @@ void fvec_argsort_parallel(size_t n, const float* vals, size_t* perm) {
         }
     }
 
-#pragma omp parallel
-    for (size_t i = 0; i < n; i++) {
+#pragma omp parallel for
+    for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         permA[i] = i;
     }
 
@@ -184,7 +184,6 @@ void fvec_argsort_parallel(size_t n, const float* vals, size_t* perm) {
             } else {
                 int t0 = s * sub_nt / sub_nseg1;
                 int t1 = (s + 1) * sub_nt / sub_nseg1;
-                printf("merge %d %d, %d threads\n", s, s + 1, t1 - t0);
                 parallel_merge(
                         permA, permB, segs[s], segs[s + 1], t1 - t0, comp);
             }
@@ -197,7 +196,6 @@ void fvec_argsort_parallel(size_t n, const float* vals, size_t* perm) {
     }
     assert(permA == perm);
     omp_set_nested(prev_nested);
-    delete[] perm2;
 }
 
 /*****************************************************************************
@@ -226,7 +224,7 @@ void bucket_sort_ref(
     for (size_t i = 0; i < vmax; i++) {
         lims[i + 1] += lims[i];
     }
-    FAISS_THROW_IF_NOT(lims[vmax] == nval);
+    FAISS_THROW_IF_NOT(static_cast<size_t>(lims[vmax]) == nval);
     double t2 = getmillisecs();
     // populate buckets
     for (size_t i = 0; i < nval; i++) {
@@ -286,7 +284,7 @@ void bucket_sort_parallel(
             for (size_t i = 0; i < vmax; i++) {
                 lims[i + 1] += lims[i];
             }
-            FAISS_THROW_IF_NOT(lims[vmax] == nval);
+            FAISS_THROW_IF_NOT(static_cast<size_t>(lims[vmax]) == nval);
         }
 #pragma omp barrier
 
@@ -341,7 +339,8 @@ void bucket_sort_inplace_ref(
     double t0 = getmillisecs();
     size_t nval = nrow * ncol;
     FAISS_THROW_IF_NOT(
-            nbucket < nval); // unclear what would happen in this case...
+            static_cast<size_t>(nbucket) <
+            nval); // unclear what would happen in this case...
 
     memset(lims, 0, sizeof(*lims) * (nbucket + 1));
     for (size_t i = 0; i < nval; i++) {
@@ -350,14 +349,14 @@ void bucket_sort_inplace_ref(
     }
     double t1 = getmillisecs();
     // compute cumulative sum
-    for (size_t i = 0; i < nbucket; i++) {
+    for (size_t i = 0; i < static_cast<size_t>(nbucket); i++) {
         lims[i + 1] += lims[i];
     }
-    FAISS_THROW_IF_NOT(lims[nbucket] == nval);
+    FAISS_THROW_IF_NOT(static_cast<size_t>(lims[nbucket]) == nval);
     double t2 = getmillisecs();
 
     std::vector<size_t> ptrs(nbucket);
-    for (size_t i = 0; i < nbucket; i++) {
+    for (size_t i = 0; i < static_cast<size_t>(nbucket); i++) {
         ptrs[i] = lims[i];
     }
 
@@ -378,7 +377,8 @@ void bucket_sort_inplace_ref(
         } else {
             // start new loop
             for (; init_bucket_no < nbucket; init_bucket_no++) {
-                if (ptrs[init_bucket_no] < lims[init_bucket_no + 1]) {
+                if (ptrs[init_bucket_no] <
+                    static_cast<size_t>(lims[init_bucket_no + 1])) {
                     break;
                 }
             }
@@ -390,7 +390,7 @@ void bucket_sort_inplace_ref(
         }
     }
 
-    for (size_t i = 0; i < nbucket; i++) {
+    for (size_t i = 0; i < static_cast<size_t>(nbucket); i++) {
         assert(ptrs[i] == lims[i + 1]);
     }
     double t3 = getmillisecs();
@@ -407,8 +407,8 @@ struct ToWrite {
     std::vector<TI> rows;
     std::vector<size_t> lims;
 
-    explicit ToWrite(TI nbucket) : nbucket(nbucket) {
-        lims.resize(nbucket + 1);
+    explicit ToWrite(TI nbucket_in) : nbucket(nbucket_in) {
+        lims.resize(nbucket_in + 1);
     }
 
     /// add one element (row) to write in bucket b
@@ -428,7 +428,7 @@ struct ToWrite {
             lims[buckets[i] + 1]++;
         }
         // compute cumulative sum
-        for (size_t i = 0; i < nbucket; i++) {
+        for (size_t i = 0; i < static_cast<size_t>(nbucket); i++) {
             lims[i + 1] += lims[i];
         }
         FAISS_THROW_IF_NOT(lims[nbucket] == buckets.size());
@@ -466,7 +466,8 @@ void bucket_sort_inplace_parallel(
     std::vector<ToWrite<TI>> all_to_write;
     size_t nval = nrow * ncol;
     FAISS_THROW_IF_NOT(
-            nbucket < nval); // unclear what would happen in this case...
+            static_cast<size_t>(nbucket) <
+            nval); // unclear what would happen in this case...
 
     // try to keep size of all_to_write < 5GiB
     // but we need at least one element per bucket
@@ -498,7 +499,7 @@ void bucket_sort_inplace_parallel(
         }
 #pragma omp critical
         { // accumulate histograms (not shifted indices to prepare cumsum)
-            for (size_t i = 0; i < nbucket; i++) {
+            for (size_t i = 0; i < static_cast<size_t>(nbucket); i++) {
                 lims[i + 1] += local_lims[i];
             }
             all_to_write.push_back(ToWrite<TI>(nbucket));
@@ -511,10 +512,10 @@ void bucket_sort_inplace_parallel(
 #pragma omp master
         {
             // compute cumulative sum
-            for (size_t i = 0; i < nbucket; i++) {
+            for (size_t i = 0; i < static_cast<size_t>(nbucket); i++) {
                 lims[i + 1] += lims[i];
             }
-            FAISS_THROW_IF_NOT(lims[nbucket] == nval);
+            FAISS_THROW_IF_NOT(static_cast<size_t>(lims[nbucket]) == nval);
             // at this point lims is final (read only!)
 
             memcpy(ptrs.data(), lims, sizeof(lims[0]) * nbucket);
@@ -559,19 +560,22 @@ void bucket_sort_inplace_parallel(
                     printf("ROUND %d n_to_write=%zd\n", round, n_to_write);
                 }
                 if (verbose > 2) {
-                    for (size_t b = 0; b < nbucket; b++) {
+                    for (size_t b = 0; b < static_cast<size_t>(nbucket); b++) {
                         printf("   b=%zd [", b);
-                        for (size_t i = lims[b]; i < lims[b + 1]; i++) {
+                        for (size_t i = static_cast<size_t>(lims[b]);
+                             i < static_cast<size_t>(lims[b + 1]);
+                             i++) {
                             printf(" %s%d",
                                    ptrs[b] == i ? ">" : "",
                                    int(vals[i]));
                         }
                         printf(" %s] %s\n",
-                               ptrs[b] == lims[b + 1] ? ">" : "",
+                               ptrs[b] == static_cast<size_t>(lims[b + 1]) ? ">"
+                                                                           : "",
                                did_wrap[b] ? "w" : "");
                     }
                     printf("To write\n");
-                    for (size_t b = 0; b < nbucket; b++) {
+                    for (size_t b = 0; b < static_cast<size_t>(nbucket); b++) {
                         printf("   b=%zd ", b);
                         const char* sep = "[";
                         for (const ToWrite<TI>& to_write_2 : all_to_write) {
@@ -609,7 +613,7 @@ void bucket_sort_inplace_parallel(
                                    rank,
                                    idx);
                         }
-                        if (idx < lims[b + 1]) {
+                        if (idx < static_cast<size_t>(lims[b + 1])) {
                             ptrs[b]++;
                         } else {
                             // wrapping around
@@ -709,7 +713,7 @@ inline int64_t hash_function(int64_t x) {
 void hashtable_int64_to_int64_init(int log2_capacity, int64_t* tab) {
     size_t capacity = (size_t)1 << log2_capacity;
 #pragma omp parallel for
-    for (int64_t i = 0; i < capacity; i++) {
+    for (int64_t i = 0; i < static_cast<int64_t>(capacity); i++) {
         tab[2 * i] = -1;
         tab[2 * i + 1] = -1;
     }
@@ -729,7 +733,7 @@ void hashtable_int64_to_int64_add(
     size_t nbucket = (size_t)1 << log2_nbucket;
 
 #pragma omp parallel for
-    for (int64_t i = 0; i < n; i++) {
+    for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         hk[i] = hash_function(keys[i]) & mask;
         bucket_no[i] = hk[i] >> (log2_capacity - log2_nbucket);
     }
@@ -746,11 +750,13 @@ void hashtable_int64_to_int64_add(
 
     int num_errors = 0;
 #pragma omp parallel for reduction(+ : num_errors)
-    for (int64_t bucket = 0; bucket < nbucket; bucket++) {
+    for (int64_t bucket = 0; bucket < static_cast<int64_t>(nbucket); bucket++) {
         size_t k0 = bucket << (log2_capacity - log2_nbucket);
         size_t k1 = (bucket + 1) << (log2_capacity - log2_nbucket);
 
-        for (size_t i = lims[bucket]; i < lims[bucket + 1]; i++) {
+        for (size_t i = static_cast<size_t>(lims[bucket]);
+             i < static_cast<size_t>(lims[bucket + 1]);
+             i++) {
             int64_t j = perm[i];
             assert(bucket_no[j] == bucket);
             assert(hk[j] >= k0 && hk[j] < k1);
@@ -768,7 +774,8 @@ void hashtable_int64_to_int64_add(
                 if (slot == k1) {
                     slot = k0;
                 }
-                if (slot == hk[j]) { // no free slot left in bucket
+                if (slot ==
+                    static_cast<size_t>(hk[j])) { // no free slot left in bucket
                     num_errors++;
                     break;
                 }
@@ -793,20 +800,24 @@ void hashtable_int64_to_int64_lookup(
     int log2_nbucket = log2_capacity_to_log2_nbucket(log2_capacity);
 
 #pragma omp parallel for
-    for (int64_t i = 0; i < n; i++) {
+    for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         int64_t k = keys[i];
-        int64_t hk = hash_function(k) & mask;
-        size_t slot = hk;
+        int64_t hk_i = hash_function(k) & mask;
+        size_t slot = hk_i;
 
         if (tab[2 * slot] == -1) { // not in table
             vals[i] = -1;
         } else if (tab[2 * slot] == k) { // found!
             vals[i] = tab[2 * slot + 1];
         } else { // need to search in [k0, k1)
-            size_t bucket = hk >> (log2_capacity - log2_nbucket);
+            size_t bucket = hk_i >> (log2_capacity - log2_nbucket);
             size_t k0 = bucket << (log2_capacity - log2_nbucket);
             size_t k1 = (bucket + 1) << (log2_capacity - log2_nbucket);
             for (;;) {
+                if (tab[slot * 2] == -1) { // empty slot, key not in table
+                    vals[i] = -1;
+                    break;
+                }
                 if (tab[slot * 2] == k) { // found!
                     vals[i] = tab[2 * slot + 1];
                     break;
@@ -815,7 +826,8 @@ void hashtable_int64_to_int64_lookup(
                 if (slot == k1) {
                     slot = k0;
                 }
-                if (slot == hk) { // bucket is full and not found
+                if (slot ==
+                    static_cast<size_t>(hk_i)) { // bucket is full and not found
                     vals[i] = -1;
                     break;
                 }

data/vendor/faiss/faiss/utils/utils.cpp

@@ -8,6 +8,7 @@
 // -*- c++ -*-
 
 #include <faiss/Index.h>
+#include <faiss/utils/simd_levels.h>
 #include <faiss/utils/utils.h>
 
 #include <cassert>
@@ -115,16 +116,22 @@ std::string get_compile_options() {
     options += "OPTIMIZE ";
 #endif
 
-#ifdef __AVX512F__
-    options += "AVX512 ";
-#elif defined(__AVX2__)
-    options += "AVX2 ";
-#elif defined(__ARM_FEATURE_SVE)
-    options += "SVE NEON ";
-#elif defined(__aarch64__)
-    options += "NEON ";
+#ifdef FAISS_ENABLE_DD
+    // Dynamic Dispatch mode: report DD and all available SIMD levels
+    options += "DD ";
+    int supported = SIMDConfig::supported_simd_levels;
+    for (int i = 0; i < static_cast<int>(SIMDLevel::COUNT); ++i) {
+        auto level = static_cast<SIMDLevel>(i);
+        if ((supported & (1 << i)) && level != SIMDLevel::NONE) {
+            options += to_string(level) + " ";
+        }
+    }
 #else
-    options += "GENERIC ";
+    // Static mode: report the compiled-in SIMD level
+    SIMDLevel level = SIMDConfig::get_level();
+    if (level != SIMDLevel::NONE) {
+        options += to_string(level) + " ";
+    }
 #endif
 
 #ifdef FAISS_ENABLE_SVS
@@ -180,7 +187,7 @@ size_t get_mem_usage_kb() {
         char buf[256];
         if (!fgets(buf, 256, f))
             break;
-        if (sscanf(buf, "VmRSS: %ld kB", &sz) == 1)
+        if (sscanf(buf, "VmRSS: %zu kB", &sz) == 1)
             break;
     }
     fclose(f);
@@ -300,7 +307,7 @@ size_t merge_result_table_with(
         std::vector<float> tmpD(k);
 
 #pragma omp for
-        for (int64_t i = 0; i < n; i++) {
+        for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
             int64_t* lI0 = I0 + i * k;
             float* lD0 = D0 + i * k;
             const int64_t* lI1 = I1 + i * k;
@@ -430,10 +437,10 @@ void bincode_hist(size_t n, size_t nbits, const uint8_t* codes, int* hist) {
     std::vector<int> accu(d * 256);
     const uint8_t* c = codes;
     for (size_t i = 0; i < n; i++)
-        for (int j = 0; j < d; j++)
+        for (size_t j = 0; j < d; j++)
             accu[j * 256 + *c++]++;
     memset(hist, 0, sizeof(*hist) * nbits);
-    for (int i = 0; i < d; i++) {
+    for (size_t i = 0; i < d; i++) {
         const int* ai = accu.data() + i * 256;
         int* hi = hist + i * 8;
         for (int j = 0; j < 256; j++)
@@ -493,7 +500,7 @@ const float* fvecs_maybe_subsample(
     std::vector<int> subset(*n);
     rand_perm(subset.data(), *n, seed);
     float* x_subset = new float[n2 * d];
-    for (int64_t i = 0; i < n2; i++)
+    for (int64_t i = 0; i < static_cast<int64_t>(n2); i++)
         memcpy(&x_subset[i * d], &x[subset[i] * size_t(d)], sizeof(x[0]) * d);
     *n = n2;
     return x_subset;

data/vendor/faiss/faiss/utils/utils.h

@@ -172,8 +172,8 @@ struct CombinerRangeKNN {
     T r2;          /// range search radius
     bool keep_max; /// whether to keep max values instead of min.
 
-    CombinerRangeKNN(int64_t nq, size_t k, T r2, bool keep_max)
-            : nq(nq), k(k), r2(r2), keep_max(keep_max) {}
+    CombinerRangeKNN(int64_t nq_in, size_t k_in, T r2_in, bool keep_max_in)
+            : nq(nq_in), k(k_in), r2(r2_in), keep_max(keep_max_in) {}
 
     /// Knn search results
     const int64_t* I = nullptr; /// size nq * k
@@ -200,7 +200,7 @@ struct CodeSet {
     size_t d;
     std::set<std::vector<uint8_t>> s;
 
-    explicit CodeSet(size_t d) : d(d) {}
+    explicit CodeSet(size_t d_in) : d(d_in) {}
     void insert(size_t n, const uint8_t* codes, bool* inserted);
 };