faiss 0.1.7 → 0.2.3

data/vendor/faiss/faiss/Clustering.h

@@ -15,41 +15,39 @@
 
 namespace faiss {
 
-
 /** Class for the clustering parameters. Can be passed to the
  * constructor of the Clustering object.
  */
 struct ClusteringParameters {
-    int niter;          ///< clustering iterations
-    int nredo;          ///< redo clustering this many times and keep best
+    int niter; ///< clustering iterations
+    int nredo; ///< redo clustering this many times and keep best
 
     bool verbose;
-    bool spherical;     ///< do we want normalized centroids?
-    bool int_centroids; ///< round centroids coordinates to integer
-    bool update_index;  ///< re-train index after each iteration?
-    bool frozen_centroids;  ///< use the centroids provided as input and do not change them during iterations
+    bool spherical;        ///< do we want normalized centroids?
+    bool int_centroids;    ///< round centroids coordinates to integer
+    bool update_index;     ///< re-train index after each iteration?
+    bool frozen_centroids; ///< use the centroids provided as input and do not
+                           ///< change them during iterations
 
     int min_points_per_centroid; ///< otherwise you get a warning
-    int max_points_per_centroid;  ///< to limit size of dataset
+    int max_points_per_centroid; ///< to limit size of dataset
 
     int seed; ///< seed for the random number generator
 
-    size_t decode_block_size;  ///< how many vectors at a time to decode
+    size_t decode_block_size; ///< how many vectors at a time to decode
 
     /// sets reasonable defaults
-    ClusteringParameters ();
+    ClusteringParameters();
 };
 
-
 struct ClusteringIterationStats {
-    float obj;               ///< objective values (sum of distances reported by index)
-    double time;             ///< seconds for iteration
+    float obj;   ///< objective values (sum of distances reported by index)
+    double time; ///< seconds for iteration
     double time_search;      ///< seconds for just search
     double imbalance_factor; ///< imbalance factor of iteration
     int nsplit;              ///< number of cluster splits
 };
 
-
 /** K-means clustering based on assignment - centroid update iterations
  *
  * The clustering is based on an Index object that assigns training
@@ -62,21 +60,22 @@ struct ClusteringIterationStats {
  * initialization.
  *
  */
-struct Clustering: ClusteringParameters {
+struct Clustering : ClusteringParameters {
     typedef Index::idx_t idx_t;
-    size_t d;              ///< dimension of the vectors
-    size_t k;              ///< nb of centroids
+    size_t d; ///< dimension of the vectors
+    size_t k; ///< nb of centroids
 
     /** centroids (k * d)
-     * if centroids are set on input to train, they will be used as initialization
+     * if centroids are set on input to train, they will be used as
+     * initialization
      */
     std::vector<float> centroids;
 
     /// stats at every iteration of clustering
     std::vector<ClusteringIterationStats> iteration_stats;
 
-    Clustering (int d, int k);
-    Clustering (int d, int k, const ClusteringParameters &cp);
+    Clustering(int d, int k);
+    Clustering(int d, int k, const ClusteringParameters& cp);
 
     /** run k-means training
      *
@@ -84,9 +83,11 @@ struct Clustering: ClusteringParameters {
      * @param index      index used for assignment
      * @param x_weights  weight associated to each vector: NULL or size n
      */
-    virtual void train (idx_t n, const float * x, faiss::Index & index,
-                        const float *x_weights = nullptr);
-
+    virtual void train(
+            idx_t n,
+            const float* x,
+            faiss::Index& index,
+            const float* x_weights = nullptr);
 
     /** run with encoded vectors
      *
@@ -96,17 +97,69 @@ struct Clustering: ClusteringParameters {
      * @param codec      codec used to decode the vectors (nullptr =
      *                   vectors are in fact floats)     *
      */
-    void train_encoded (idx_t nx, const uint8_t *x_in,
-                        const Index * codec, Index & index,
-                        const float *weights = nullptr);
+    void train_encoded(
+            idx_t nx,
+            const uint8_t* x_in,
+            const Index* codec,
+            Index& index,
+            const float* weights = nullptr);
 
     /// Post-process the centroids after each centroid update.
     /// includes optional L2 normalization and nearest integer rounding
-    void post_process_centroids ();
+    void post_process_centroids();
 
     virtual ~Clustering() {}
 };
 
+struct ProgressiveDimClusteringParameters : ClusteringParameters {
+    int progressive_dim_steps; ///< number of incremental steps
+    bool apply_pca;            ///< apply PCA on input
+
+    ProgressiveDimClusteringParameters();
+};
+
+/** generates an index suitable for clustering when called */
+struct ProgressiveDimIndexFactory {
+    /// ownership transferred to caller
+    virtual Index* operator()(int dim);
+
+    virtual ~ProgressiveDimIndexFactory() {}
+};
+
+/** K-means clustering with progressive dimensions used
+ *
+ * The clustering first happens in dim 1, then with exponentially increasing
+ * dimension until d (I steps). This is typically applied after a PCA
+ * transformation (optional). Reference:
+ *
+ * "Improved Residual Vector Quantization for High-dimensional Approximate
+ * Nearest Neighbor Search"
+ *
+ * Shicong Liu, Hongtao Lu, Junru Shao, AAAI'15
+ *
+ * https://arxiv.org/abs/1509.05195
+ */
+struct ProgressiveDimClustering : ProgressiveDimClusteringParameters {
+    using idx_t = Index::idx_t;
+    size_t d; ///< dimension of the vectors
+    size_t k; ///< nb of centroids
+
+    /** centroids (k * d) */
+    std::vector<float> centroids;
+
+    /// stats at every iteration of clustering
+    std::vector<ClusteringIterationStats> iteration_stats;
+
+    ProgressiveDimClustering(int d, int k);
+    ProgressiveDimClustering(
+            int d,
+            int k,
+            const ProgressiveDimClusteringParameters& cp);
+
+    void train(idx_t n, const float* x, ProgressiveDimIndexFactory& factory);
+
+    virtual ~ProgressiveDimClustering() {}
+};
 
 /** simplified interface
  *
@@ -117,13 +170,13 @@ struct Clustering: ClusteringParameters {
  * @param centroids output centroids (size k * d)
  * @return final quantization error
  */
-float kmeans_clustering (size_t d, size_t n, size_t k,
-                         const float *x,
-                         float *centroids);
-
-
-
-}
-
+float kmeans_clustering(
+        size_t d,
+        size_t n,
+        size_t k,
+        const float* x,
+        float* centroids);
+
+} // namespace faiss
 
 #endif