faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/impl/PolysemousTraining.h

@@ -10,62 +10,56 @@
 #ifndef FAISS_POLYSEMOUS_TRAINING_INCLUDED
 #define FAISS_POLYSEMOUS_TRAINING_INCLUDED
 
-
 #include <faiss/impl/ProductQuantizer.h>
 
-
 namespace faiss {
 
-
 /// parameters used for the simulated annealing method
 struct SimulatedAnnealingParameters {
-
     // optimization parameters
-    double init_temperature;   // init probability of accepting a bad swap
-    double temperature_decay;  // at each iteration the temp is multiplied by this
-    int n_iter; // nb of iterations
-    int n_redo; // nb of runs of the simulation
-    int seed;   // random seed
+    double init_temperature;  // init probability of accepting a bad swap
+    double temperature_decay; // at each iteration the temp is multiplied by
+                              // this
+    int n_iter;               // nb of iterations
+    int n_redo;               // nb of runs of the simulation
+    int seed;                 // random seed
     int verbose;
     bool only_bit_flips; // restrict permutation changes to bit flips
-    bool init_random; // initialize with a random permutation (not identity)
+    bool init_random;    // initialize with a random permutation (not identity)
 
     // set reasonable defaults
-    SimulatedAnnealingParameters ();
-
+    SimulatedAnnealingParameters();
 };
 
-
 /// abstract class for the loss function
 struct PermutationObjective {
-
     int n;
 
-    virtual double compute_cost (const int *perm) const = 0;
+    virtual double compute_cost(const int* perm) const = 0;
 
     // what would the cost update be if iw and jw were swapped?
     // default implementation just computes both and computes the difference
-    virtual double cost_update (const int *perm, int iw, int jw) const;
+    virtual double cost_update(const int* perm, int iw, int jw) const;
 
-    virtual ~PermutationObjective () {}
+    virtual ~PermutationObjective() {}
 };
 
-
 struct ReproduceDistancesObjective : PermutationObjective {
-
     double dis_weight_factor;
 
-    static double sqr (double x) { return x * x; }
+    static double sqr(double x) {
+        return x * x;
+    }
 
     // weighting of distances: it is more important to reproduce small
     // distances well
-    double dis_weight (double x) const;
+    double dis_weight(double x) const;
 
     std::vector<double> source_dis; ///< "real" corrected distances (size n^2)
-    const double *      target_dis; ///< wanted distances (size n^2)
+    const double* target_dis;       ///< wanted distances (size n^2)
     std::vector<double> weights;    ///< weights for each distance (size n^2)
 
-    double get_source_dis (int i, int j) const;
+    double get_source_dis(int i, int j) const;
 
     // cost = quadratic difference between actual distance and Hamming distance
     double compute_cost(const int* perm) const override;
@@ -74,16 +68,19 @@ struct ReproduceDistancesObjective : PermutationObjective {
     // computed in O(n) instead of O(n^2) for the full re-computation
     double cost_update(const int* perm, int iw, int jw) const override;
 
-    ReproduceDistancesObjective (
-           int n,
-           const double *source_dis_in,
-           const double *target_dis_in,
-           double dis_weight_factor);
+    ReproduceDistancesObjective(
+            int n,
+            const double* source_dis_in,
+            const double* target_dis_in,
+            double dis_weight_factor);
 
-    static void compute_mean_stdev (const double *tab, size_t n2,
-                                    double *mean_out, double *stddev_out);
+    static void compute_mean_stdev(
+            const double* tab,
+            size_t n2,
+            double* mean_out,
+            double* stddev_out);
 
-    void set_affine_target_dis (const double *source_dis_in);
+    void set_affine_target_dis(const double* source_dis_in);
 
     ~ReproduceDistancesObjective() override {}
 };
@@ -91,39 +88,36 @@ struct ReproduceDistancesObjective : PermutationObjective {
 struct RandomGenerator;
 
 /// Simulated annealing optimization algorithm for permutations.
- struct SimulatedAnnealingOptimizer: SimulatedAnnealingParameters {
-
-    PermutationObjective *obj;
+struct SimulatedAnnealingOptimizer : SimulatedAnnealingParameters {
+    PermutationObjective* obj;
     int n;         ///< size of the permutation
-    FILE *logfile; /// logs values of the cost function
+    FILE* logfile; /// logs values of the cost function
 
-    SimulatedAnnealingOptimizer (PermutationObjective *obj,
-                                 const SimulatedAnnealingParameters &p);
-    RandomGenerator *rnd;
+    SimulatedAnnealingOptimizer(
+            PermutationObjective* obj,
+            const SimulatedAnnealingParameters& p);
+    RandomGenerator* rnd;
 
     /// remember initial cost of optimization
     double init_cost;
 
     // main entry point. Perform the optimization loop, starting from
     // and modifying permutation in-place
-    double optimize (int *perm);
+    double optimize(int* perm);
 
     // run the optimization and return the best result in best_perm
-    double run_optimization (int * best_perm);
+    double run_optimization(int* best_perm);
 
-    virtual ~SimulatedAnnealingOptimizer ();
+    virtual ~SimulatedAnnealingOptimizer();
 };
 
-
-
-
 /// optimizes the order of indices in a ProductQuantizer
-struct PolysemousTraining: SimulatedAnnealingParameters {
-
+struct PolysemousTraining : SimulatedAnnealingParameters {
     enum Optimization_type_t {
         OT_None,
-        OT_ReproduceDistances_affine,  ///< default
-        OT_Ranking_weighted_diff       ///< same as _2, but use rank of y+ - rank of y-
+        OT_ReproduceDistances_affine, ///< default
+        OT_Ranking_weighted_diff ///< same as _2, but use rank of y+ - rank of
+                                 ///< y-
     };
     Optimization_type_t optimization_type;
 
@@ -133,26 +127,29 @@ struct PolysemousTraining: SimulatedAnnealingParameters {
     int ntrain_permutation;
     double dis_weight_factor; ///< decay of exp that weights distance loss
 
+    /// refuse to train if it would require more than that amount of RAM
+    size_t max_memory;
+
     // filename pattern for the logging of iterations
     std::string log_pattern;
 
     // sets default values
-    PolysemousTraining ();
+    PolysemousTraining();
 
     /// reorder the centroids so that the Hamming distance becomes a
     /// good approximation of the SDC distance (called by train)
-    void optimize_pq_for_hamming (ProductQuantizer & pq,
-                                  size_t n, const float *x) const;
+    void optimize_pq_for_hamming(ProductQuantizer& pq, size_t n, const float* x)
+            const;
 
     /// called by optimize_pq_for_hamming
-    void optimize_ranking (ProductQuantizer &pq, size_t n, const float *x) const;
+    void optimize_ranking(ProductQuantizer& pq, size_t n, const float* x) const;
     /// called by optimize_pq_for_hamming
-    void optimize_reproduce_distances (ProductQuantizer &pq) const;
+    void optimize_reproduce_distances(ProductQuantizer& pq) const;
 
+    /// make sure we don't blow up the memory
+    size_t memory_usage_per_thread(const ProductQuantizer& pq) const;
 };
 
-
 } // namespace faiss
 
-
 #endif

data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h

@@ -7,20 +7,18 @@
 
 namespace faiss {
 
-inline
-PQEncoderGeneric::PQEncoderGeneric(uint8_t *code, int nbits,
-                                                     uint8_t offset)
-    : code(code), offset(offset), nbits(nbits), reg(0)
-{
+inline PQEncoderGeneric::PQEncoderGeneric(
+        uint8_t* code,
+        int nbits,
+        uint8_t offset)
+        : code(code), offset(offset), nbits(nbits), reg(0) {
     assert(nbits <= 64);
     if (offset > 0) {
         reg = (*code & ((1 << offset) - 1));
     }
 }
 
-inline
-void PQEncoderGeneric::encode(uint64_t x)
-{
+inline void PQEncoderGeneric::encode(uint64_t x) {
     reg |= (uint8_t)(x << offset);
     x >>= (8 - offset);
     if (offset + nbits >= 8) {
@@ -39,51 +37,39 @@ void PQEncoderGeneric::encode(uint64_t x)
     }
 }
 
-inline
-PQEncoderGeneric::~PQEncoderGeneric()
-{
+inline PQEncoderGeneric::~PQEncoderGeneric() {
     if (offset > 0) {
         *code = reg;
     }
 }
 
-
-inline
-PQEncoder8::PQEncoder8(uint8_t *code, int nbits)
-    : code(code) {
+inline PQEncoder8::PQEncoder8(uint8_t* code, int nbits) : code(code) {
     assert(8 == nbits);
 }
 
-inline
-void PQEncoder8::encode(uint64_t x) {
+inline void PQEncoder8::encode(uint64_t x) {
     *code++ = (uint8_t)x;
 }
 
-inline
-PQEncoder16::PQEncoder16(uint8_t *code, int nbits)
-    : code((uint16_t *)code) {
+inline PQEncoder16::PQEncoder16(uint8_t* code, int nbits)
+        : code((uint16_t*)code) {
     assert(16 == nbits);
 }
 
-inline
-void PQEncoder16::encode(uint64_t x) {
+inline void PQEncoder16::encode(uint64_t x) {
     *code++ = (uint16_t)x;
 }
 
-
-inline
-PQDecoderGeneric::PQDecoderGeneric(const uint8_t *code,
-                                                     int nbits)
-    : code(code),
-      offset(0),
-      nbits(nbits),
-      mask((1ull << nbits) - 1),
-      reg(0) {
+inline PQDecoderGeneric::PQDecoderGeneric(const uint8_t* code, int nbits)
+        : code(code),
+          offset(0),
+          nbits(nbits),
+          mask((1ull << nbits) - 1),
+          reg(0) {
     assert(nbits <= 64);
 }
 
-inline
-uint64_t PQDecoderGeneric::decode() {
+inline uint64_t PQDecoderGeneric::decode() {
     if (offset == 0) {
         reg = *code;
     }
@@ -110,27 +96,20 @@ uint64_t PQDecoderGeneric::decode() {
     return c & mask;
 }
 
-
-inline
-PQDecoder8::PQDecoder8(const uint8_t *code, int nbits)
-    : code(code) {
-    assert(8 == nbits);
+inline PQDecoder8::PQDecoder8(const uint8_t* code, int nbits_in) : code(code) {
+    assert(8 == nbits_in);
 }
 
-inline
-uint64_t PQDecoder8::decode() {
+inline uint64_t PQDecoder8::decode() {
     return (uint64_t)(*code++);
 }
 
-
-inline
-PQDecoder16::PQDecoder16(const uint8_t *code, int nbits)
-    : code((uint16_t *)code) {
-     assert(16 == nbits);
+inline PQDecoder16::PQDecoder16(const uint8_t* code, int nbits_in)
+        : code((uint16_t*)code) {
+    assert(16 == nbits_in);
 }
 
-inline
-uint64_t PQDecoder16::decode() {
+inline uint64_t PQDecoder16::decode() {
     return (uint64_t)(*code++);
 }