faiss 0.5.3 → 0.6.1

data/vendor/faiss/faiss/impl/PolysemousTraining.cpp

@@ -18,6 +18,7 @@
 #include <cstring>
 #include <memory>
 
+#include <faiss/impl/simd_dispatch.h>
 #include <faiss/utils/distances.h>
 #include <faiss/utils/hamming.h>
 #include <faiss/utils/random.h>
@@ -53,11 +54,11 @@ double PermutationObjective::cost_update(const int* perm, int iw, int jw)
 }
 
 SimulatedAnnealingOptimizer::SimulatedAnnealingOptimizer(
-        PermutationObjective* obj,
+        PermutationObjective* obj_in,
         const SimulatedAnnealingParameters& p)
         : SimulatedAnnealingParameters(p),
-          obj(obj),
-          n(obj->n),
+          obj(obj_in),
+          n(obj_in->n),
           logfile(nullptr) {
     rnd = new RandomGenerator(p.seed);
     FAISS_THROW_IF_NOT(n < 100000 && n >= 0);
@@ -164,7 +165,7 @@ double SimulatedAnnealingOptimizer::optimize(int* perm) {
  ****************************************************/
 
 static inline int hamming_dis(uint64_t a, uint64_t b) {
-    return __builtin_popcountl(a ^ b);
+    return popcount64(a ^ b);
 }
 
 namespace {
@@ -259,10 +260,10 @@ struct ReproduceWithHammingObjective : PermutationObjective {
     }
 
     ReproduceWithHammingObjective(
-            int nbits,
+            int nbits_in,
             const std::vector<double>& dis_table,
-            double dis_weight_factor)
-            : nbits(nbits), dis_weight_factor(dis_weight_factor) {
+            double dis_weight_factor_in)
+            : nbits(nbits_in), dis_weight_factor(dis_weight_factor_in) {
         n = 1 << nbits;
         FAISS_THROW_IF_NOT(dis_table.size() == n * n);
         set_affine_target_dis(dis_table);
@@ -372,12 +373,12 @@ double ReproduceDistancesObjective::cost_update(const int* perm, int iw, int jw)
 }
 
 ReproduceDistancesObjective::ReproduceDistancesObjective(
-        int n,
+        int n_in,
         const double* source_dis_in,
         const double* target_dis_in,
-        double dis_weight_factor)
-        : dis_weight_factor(dis_weight_factor), target_dis(target_dis_in) {
-    this->n = n;
+        double dis_weight_factor_in)
+        : dis_weight_factor(dis_weight_factor_in), target_dis(target_dis_in) {
+    this->n = n_in;
     set_affine_target_dis(source_dis_in);
 }
 
@@ -387,7 +388,7 @@ void ReproduceDistancesObjective::compute_mean_stdev(
         double* mean_out,
         double* stddev_out) {
     double sum = 0, sum2 = 0;
-    for (int i = 0; i < n2; i++) {
+    for (size_t i = 0; i < n2; i++) {
         sum += tab[i];
         sum2 += tab[i] * tab[i];
     }
@@ -431,6 +432,8 @@ void ReproduceDistancesObjective::set_affine_target_dis(
  * Cost functions: RankingScore
  ****************************************************/
 
+namespace {
+
 /// Maintains a 3D table of elementary costs.
 /// Accumulates elements based on Hamming distance comparisons
 template <typename Ttab, typename Taccu>
@@ -665,18 +668,18 @@ struct RankingScore2 : Score3Computer<float, double> {
     const float* gt_distances;
 
     RankingScore2(
-            int nbits,
-            int nq,
-            int nb,
-            const uint32_t* qcodes,
-            const uint32_t* bcodes,
-            const float* gt_distances)
-            : nbits(nbits),
-              nq(nq),
-              nb(nb),
-              qcodes(qcodes),
-              bcodes(bcodes),
-              gt_distances(gt_distances) {
+            int nbits_in,
+            int nq_in,
+            int nb_in,
+            const uint32_t* qcodes_in,
+            const uint32_t* bcodes_in,
+            const float* gt_distances_in)
+            : nbits(nbits_in),
+              nq(nq_in),
+              nb(nb_in),
+              qcodes(qcodes_in),
+              bcodes(bcodes_in),
+              gt_distances(gt_distances_in) {
         n = nc = 1 << nbits;
         n_gt.resize(nc * nc * nc);
         init_n_gt();
@@ -756,6 +759,8 @@ struct RankingScore2 : Score3Computer<float, double> {
     }
 };
 
+} // namespace
+
 /*****************************************
  * PolysemousTraining
  ******************************************/
@@ -791,19 +796,25 @@ void PolysemousTraining::optimize_reproduce_distances(
     }
 
 #pragma omp parallel for num_threads(nt)
-    for (int m = 0; m < pq.M; m++) {
+    for (int m = 0; m < static_cast<int>(pq.M); m++) {
         std::vector<double> dis_table;
 
         // printf ("Optimizing quantizer %d\n", m);
 
         float* centroids = pq.get_centroids(m, 0);
 
-        for (int i = 0; i < n; i++) {
-            for (int j = 0; j < n; j++) {
-                dis_table.push_back(fvec_L2sqr(
-                        centroids + i * dsub, centroids + j * dsub, dsub));
+        auto compute_dis_table = [&]<SIMDLevel SL>() {
+            for (int i = 0; i < n; i++) {
+                for (int j = 0; j < n; j++) {
+                    dis_table.push_back(
+                            fvec_L2sqr<SL>(
+                                    centroids + i * dsub,
+                                    centroids + j * dsub,
+                                    dsub));
+                }
             }
-        }
+        };
+        with_simd_level(compute_dis_table);
 
         std::vector<int> perm(n);
         ReproduceWithHammingObjective obj(nbits, dis_table, dis_weight_factor);
@@ -812,7 +823,14 @@ void PolysemousTraining::optimize_reproduce_distances(
 
         if (log_pattern.size()) {
             char fname[256];
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
+#endif
             snprintf(fname, 256, log_pattern.c_str(), m);
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
             printf("opening log file %s\n", fname);
             optim.logfile = fopen(fname, "w");
             FAISS_THROW_IF_NOT_MSG(optim.logfile, "could not open logfile");
@@ -861,21 +879,21 @@ void PolysemousTraining::optimize_ranking(
     }
 
 #pragma omp parallel for
-    for (int m = 0; m < pq.M; m++) {
+    for (int m = 0; m < static_cast<int>(pq.M); m++) {
         size_t nq, nb;
         std::vector<uint32_t> codes;     // query codes, then db codes
         std::vector<float> gt_distances; // nq * nb matrix of distances
 
         if (n > 0) {
             std::vector<float> xtrain(n * dsub);
-            for (int i = 0; i < n; i++) {
+            for (size_t i = 0; i < n; i++) {
                 memcpy(xtrain.data() + i * dsub,
                        x + i * pq.d + m * dsub,
                        sizeof(float) * dsub);
             }
 
             codes.resize(n);
-            for (int i = 0; i < n; i++) {
+            for (size_t i = 0; i < n; i++) {
                 codes[i] = all_codes[i * pq.code_size + m];
             }
 
@@ -890,8 +908,8 @@ void PolysemousTraining::optimize_ranking(
         } else {
             nq = nb = pq.ksub;
             codes.resize(2 * nq);
-            for (int i = 0; i < nq; i++) {
-                codes[i] = codes[i + nq] = i;
+            for (size_t i = 0; i < nq; i++) {
+                codes[i] = codes[i + nq] = static_cast<uint32_t>(i);
             }
 
             gt_distances.resize(nq * nb);
@@ -924,7 +942,14 @@ void PolysemousTraining::optimize_ranking(
 
         if (log_pattern.size()) {
             char fname[256];
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
+#endif
             snprintf(fname, 256, log_pattern.c_str(), m);
+#if defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
             printf("opening log file %s\n", fname);
             optim.logfile = fopen(fname, "w");
             FAISS_THROW_IF_NOT_FMT(
@@ -946,11 +971,11 @@ void PolysemousTraining::optimize_ranking(
         float* centroids = pq.get_centroids(m, 0);
 
         std::vector<float> centroids_copy;
-        for (int i = 0; i < dsub * pq.ksub; i++) {
+        for (size_t i = 0; i < dsub * pq.ksub; i++) {
             centroids_copy.push_back(centroids[i]);
         }
 
-        for (int i = 0; i < pq.ksub; i++) {
+        for (size_t i = 0; i < pq.ksub; i++) {
             memcpy(centroids + perm[i] * dsub,
                    centroids_copy.data() + i * dsub,
                    dsub * sizeof(centroids[0]));

data/vendor/faiss/faiss/impl/PolysemousTraining.h

@@ -70,10 +70,10 @@ struct ReproduceDistancesObjective : PermutationObjective {
     double cost_update(const int* perm, int iw, int jw) const override;
 
     ReproduceDistancesObjective(
-            int n,
+            int n_in,
             const double* source_dis_in,
             const double* target_dis_in,
-            double dis_weight_factor);
+            double dis_weight_factor_in);
 
     static void compute_mean_stdev(
             const double* tab,
@@ -95,7 +95,7 @@ struct SimulatedAnnealingOptimizer : SimulatedAnnealingParameters {
     FILE* logfile; /// logs values of the cost function
 
     SimulatedAnnealingOptimizer(
-            PermutationObjective* obj,
+            PermutationObjective* obj_in,
             const SimulatedAnnealingParameters& p);
     RandomGenerator* rnd;
 

data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.cpp

@@ -42,22 +42,22 @@ int sgemm_(
 namespace faiss {
 
 ProductAdditiveQuantizer::ProductAdditiveQuantizer(
-        size_t d,
+        size_t d_,
         const std::vector<AdditiveQuantizer*>& aqs,
-        Search_type_t search_type) {
-    init(d, aqs, search_type);
+        Search_type_t search_type_) {
+    init(d_, aqs, search_type_);
 }
 
 ProductAdditiveQuantizer::ProductAdditiveQuantizer()
         : ProductAdditiveQuantizer(0, {}) {}
 
 void ProductAdditiveQuantizer::init(
-        size_t d,
+        size_t d_in,
         const std::vector<AdditiveQuantizer*>& aqs,
-        Search_type_t search_type) {
+        Search_type_t search_type_in) {
     // AdditiveQuantizer constructor
-    this->d = d;
-    this->search_type = search_type;
+    this->d = d_in;
+    this->search_type = search_type_in;
     M = 0;
     for (const auto& q : aqs) {
         M += q->M;
@@ -98,7 +98,7 @@ void ProductAdditiveQuantizer::train(size_t n, const float* x) {
         xt.resize(q->d * n);
 
 #pragma omp parallel for if (n > 1000)
-        for (idx_t i = 0; i < n; i++) {
+        for (idx_t i = 0; i < static_cast<idx_t>(n); i++) {
             memcpy(xt.data() + i * q->d,
                    x + i * d + offset_d,
                    q->d * sizeof(*x));
@@ -153,7 +153,7 @@ void ProductAdditiveQuantizer::compute_unpacked_codes(
         const float* x,
         int32_t* unpacked_codes,
         size_t n,
-        const float* centroids) const {
+        const float* /*centroids*/) const {
     /// TODO: actually we do not need to unpack and pack
     size_t offset_d = 0, offset_m = 0;
     std::vector<float> xsub;
@@ -165,7 +165,7 @@ void ProductAdditiveQuantizer::compute_unpacked_codes(
         codes.resize(n * q->code_size);
 
 #pragma omp parallel for if (n > 1000)
-        for (idx_t i = 0; i < n; i++) {
+        for (idx_t i = 0; i < static_cast<idx_t>(n); i++) {
             memcpy(xsub.data() + i * q->d,
                    x + i * d + offset_d,
                    q->d * sizeof(float));
@@ -175,7 +175,7 @@ void ProductAdditiveQuantizer::compute_unpacked_codes(
 
         // unpack
 #pragma omp parallel for if (n > 1000)
-        for (idx_t i = 0; i < n; i++) {
+        for (idx_t i = 0; i < static_cast<idx_t>(n); i++) {
             uint8_t* code = codes.data() + i * q->code_size;
             BitstringReader bsr(code, q->code_size);
 
@@ -204,7 +204,7 @@ void ProductAdditiveQuantizer::decode_unpacked(
 
     // product additive quantizer decoding
 #pragma omp parallel for if (n > 1000)
-    for (int64_t i = 0; i < n; i++) {
+    for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         const int32_t* codesi = codes + i * ld_codes;
 
         size_t offset_m = 0, offset_d = 0;
@@ -212,7 +212,7 @@ void ProductAdditiveQuantizer::decode_unpacked(
             const auto q = quantizers[s];
             float* xi = x + i * d + offset_d;
 
-            for (int m = 0; m < q->M; m++) {
+            for (size_t m = 0; m < q->M; m++) {
                 int idx = codesi[offset_m + m];
                 const float* c = codebooks.data() +
                         q->d * (codebook_offsets[offset_m + m] + idx);
@@ -235,7 +235,7 @@ void ProductAdditiveQuantizer::decode(const uint8_t* codes, float* x, size_t n)
             is_trained, "The product additive quantizer is not trained yet.");
 
 #pragma omp parallel for if (n > 1000)
-    for (int64_t i = 0; i < n; i++) {
+    for (int64_t i = 0; i < static_cast<int64_t>(n); i++) {
         BitstringReader bsr(codes + i * code_size, code_size);
 
         size_t offset_m = 0, offset_d = 0;
@@ -243,7 +243,7 @@ void ProductAdditiveQuantizer::decode(const uint8_t* codes, float* x, size_t n)
             const auto q = quantizers[s];
             float* xi = x + i * d + offset_d;
 
-            for (int m = 0; m < q->M; m++) {
+            for (size_t m = 0; m < q->M; m++) {
                 int idx = bsr.read(q->nbits[m]);
                 const float* c = codebooks.data() +
                         q->d * (codebook_offsets[offset_m + m] + idx);
@@ -315,24 +315,24 @@ void ProductAdditiveQuantizer::compute_LUT(
  ************************************/
 
 ProductLocalSearchQuantizer::ProductLocalSearchQuantizer(
-        size_t d,
-        size_t nsplits,
+        size_t d_,
+        size_t nsplits_,
         size_t Msub,
-        size_t nbits,
-        Search_type_t search_type) {
+        size_t nbits_,
+        Search_type_t search_type_) {
     std::vector<AdditiveQuantizer*> aqs;
 
-    if (nsplits > 0) {
-        FAISS_THROW_IF_NOT(d % nsplits == 0);
-        size_t dsub = d / nsplits;
+    if (nsplits_ > 0) {
+        FAISS_THROW_IF_NOT(d_ % nsplits_ == 0);
+        size_t dsub = d_ / nsplits_;
 
-        for (size_t i = 0; i < nsplits; i++) {
+        for (size_t i = 0; i < nsplits_; i++) {
             auto lsq =
-                    new LocalSearchQuantizer(dsub, Msub, nbits, ST_decompress);
+                    new LocalSearchQuantizer(dsub, Msub, nbits_, ST_decompress);
             aqs.push_back(lsq);
         }
     }
-    init(d, aqs, search_type);
+    init(d_, aqs, search_type_);
     for (auto& q : aqs) {
         delete q;
     }
@@ -346,23 +346,23 @@ ProductLocalSearchQuantizer::ProductLocalSearchQuantizer()
  ************************************/
 
 ProductResidualQuantizer::ProductResidualQuantizer(
-        size_t d,
-        size_t nsplits,
+        size_t d_,
+        size_t nsplits_,
         size_t Msub,
-        size_t nbits,
-        Search_type_t search_type) {
+        size_t nbits_,
+        Search_type_t search_type_) {
     std::vector<AdditiveQuantizer*> aqs;
 
-    if (nsplits > 0) {
-        FAISS_THROW_IF_NOT(d % nsplits == 0);
-        size_t dsub = d / nsplits;
+    if (nsplits_ > 0) {
+        FAISS_THROW_IF_NOT(d_ % nsplits_ == 0);
+        size_t dsub = d_ / nsplits_;
 
-        for (size_t i = 0; i < nsplits; i++) {
-            auto rq = new ResidualQuantizer(dsub, Msub, nbits, ST_decompress);
+        for (size_t i = 0; i < nsplits_; i++) {
+            auto rq = new ResidualQuantizer(dsub, Msub, nbits_, ST_decompress);
             aqs.push_back(rq);
         }
     }
-    init(d, aqs, search_type);
+    init(d_, aqs, search_type_);
     for (auto& q : aqs) {
         delete q;
     }

data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h

@@ -10,10 +10,10 @@
 namespace faiss {
 
 inline PQEncoderGeneric::PQEncoderGeneric(
-        uint8_t* code,
-        int nbits,
-        uint8_t offset)
-        : code(code), offset(offset), nbits(nbits), reg(0) {
+        uint8_t* code_in,
+        int nbits_in,
+        uint8_t offset_in)
+        : code(code_in), offset(offset_in), nbits(nbits_in), reg(0) {
     assert(nbits <= 64);
     if (offset > 0) {
         reg = (*code & ((1 << offset) - 1));
@@ -45,28 +45,30 @@ inline PQEncoderGeneric::~PQEncoderGeneric() {
     }
 }
 
-inline PQEncoder8::PQEncoder8(uint8_t* code, int nbits) : code(code) {
-    assert(8 == nbits);
+inline PQEncoder8::PQEncoder8(uint8_t* code_in, int nbits_in) : code(code_in) {
+    assert(8 == nbits_in);
+    (void)nbits_in;
 }
 
 inline void PQEncoder8::encode(uint64_t x) {
     *code++ = (uint8_t)x;
 }
 
-inline PQEncoder16::PQEncoder16(uint8_t* code, int nbits)
-        : code((uint16_t*)code) {
-    assert(16 == nbits);
+inline PQEncoder16::PQEncoder16(uint8_t* code_in, int nbits_in)
+        : code((uint16_t*)code_in) {
+    assert(16 == nbits_in);
+    (void)nbits_in;
 }
 
 inline void PQEncoder16::encode(uint64_t x) {
     *code++ = (uint16_t)x;
 }
 
-inline PQDecoderGeneric::PQDecoderGeneric(const uint8_t* code, int nbits)
-        : code(code),
+inline PQDecoderGeneric::PQDecoderGeneric(const uint8_t* code_in, int nbits_in)
+        : code(code_in),
           offset(0),
-          nbits(nbits),
-          mask((1ull << nbits) - 1),
+          nbits(nbits_in),
+          mask((1ull << nbits_in) - 1),
           reg(0) {
     assert(nbits <= 64);
 }
@@ -98,17 +100,20 @@ inline uint64_t PQDecoderGeneric::decode() {
     return c & mask;
 }
 
-inline PQDecoder8::PQDecoder8(const uint8_t* code, int nbits_in) : code(code) {
+inline PQDecoder8::PQDecoder8(const uint8_t* code_in, int nbits_in)
+        : code(code_in) {
     assert(8 == nbits_in);
+    (void)nbits_in;
 }
 
 inline uint64_t PQDecoder8::decode() {
     return (uint64_t)(*code++);
 }
 
-inline PQDecoder16::PQDecoder16(const uint8_t* code, int nbits_in)
-        : code((uint16_t*)code) {
+inline PQDecoder16::PQDecoder16(const uint8_t* code_in, int nbits_in)
+        : code((uint16_t*)code_in) {
     assert(16 == nbits_in);
+    (void)nbits_in;
 }
 
 inline uint64_t PQDecoder16::decode() {