faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/VectorTransform.h

@@ -14,67 +14,55 @@
  * vectors Often these are pre-processing steps.
  */
 
-#include <vector>
 #include <stdint.h>
+#include <vector>
 
 #include <faiss/Index.h>
 
-
 namespace faiss {
 
-
 /** Any transformation applied on a set of vectors */
 struct VectorTransform {
-
     typedef Index::idx_t idx_t;
 
-    int d_in;      ///! input dimension
-    int d_out;     ///! output dimension
-
-    explicit VectorTransform (int d_in = 0, int d_out = 0):
-    d_in(d_in), d_out(d_out), is_trained(true)
-    {}
+    int d_in;  ///! input dimension
+    int d_out; ///! output dimension
 
+    explicit VectorTransform(int d_in = 0, int d_out = 0)
+            : d_in(d_in), d_out(d_out), is_trained(true) {}
 
     /// set if the VectorTransform does not require training, or if
     /// training is done already
     bool is_trained;
 
-
     /** Perform training on a representative set of vectors. Does
      * nothing by default.
      *
      * @param n      nb of training vectors
      * @param x      training vecors, size n * d
      */
-    virtual void train (idx_t n, const float *x);
+    virtual void train(idx_t n, const float* x);
 
-    /** apply the random roation, return new allocated matrix
+    /** apply the random rotation, return new allocated matrix
      * @param     x size n * d_in
      * @return    size n * d_out
      */
-    float *apply (idx_t n, const float * x) const;
+    float* apply(idx_t n, const float* x) const;
 
     /// same as apply, but result is pre-allocated
-    virtual void apply_noalloc (idx_t n, const float * x,
-                                float *xt) const = 0;
+    virtual void apply_noalloc(idx_t n, const float* x, float* xt) const = 0;
 
     /// reverse transformation. May not be implemented or may return
     /// approximate result
-    virtual void reverse_transform (idx_t n, const float * xt,
-                                    float *x) const;
-
-    virtual ~VectorTransform () {}
+    virtual void reverse_transform(idx_t n, const float* xt, float* x) const;
 
+    virtual ~VectorTransform() {}
 };
 
-
-
 /** Generic linear transformation, with bias term applied on output
  * y = A * x + b
  */
-struct LinearTransform: VectorTransform {
-
+struct LinearTransform : VectorTransform {
     bool have_bias; ///! whether to use the bias term
 
     /// check if matrix A is orthonormal (enables reverse_transform)
@@ -83,58 +71,56 @@ struct LinearTransform: VectorTransform {
     /// Transformation matrix, size d_out * d_in
     std::vector<float> A;
 
-     /// bias vector, size d_out
+    /// bias vector, size d_out
     std::vector<float> b;
 
     /// both d_in > d_out and d_out < d_in are supported
-    explicit LinearTransform (int d_in = 0, int d_out = 0,
-                              bool have_bias = false);
+    explicit LinearTransform(
+            int d_in = 0,
+            int d_out = 0,
+            bool have_bias = false);
 
     /// same as apply, but result is pre-allocated
     void apply_noalloc(idx_t n, const float* x, float* xt) const override;
 
     /// compute x = A^T * (x - b)
     /// is reverse transform if A has orthonormal lines
-    void transform_transpose (idx_t n, const float * y,
-                              float *x) const;
+    void transform_transpose(idx_t n, const float* y, float* x) const;
 
     /// works only if is_orthonormal
-    void reverse_transform (idx_t n, const float * xt,
-                            float *x) const override;
+    void reverse_transform(idx_t n, const float* xt, float* x) const override;
 
     /// compute A^T * A to set the is_orthonormal flag
-    void set_is_orthonormal ();
+    void set_is_orthonormal();
 
     bool verbose;
-    void print_if_verbose (const char*name, const std::vector<double> &mat,
-                           int n, int d) const;
+    void print_if_verbose(
+            const char* name,
+            const std::vector<double>& mat,
+            int n,
+            int d) const;
 
     ~LinearTransform() override {}
 };
 
-
-
 /// Randomly rotate a set of vectors
-struct RandomRotationMatrix: LinearTransform {
-
-     /// both d_in > d_out and d_out < d_in are supported
-     RandomRotationMatrix (int d_in, int d_out):
-         LinearTransform(d_in, d_out, false) {}
+struct RandomRotationMatrix : LinearTransform {
+    /// both d_in > d_out and d_out < d_in are supported
+    RandomRotationMatrix(int d_in, int d_out)
+            : LinearTransform(d_in, d_out, false) {}
 
-     /// must be called before the transform is used
-     void init(int seed);
+    /// must be called before the transform is used
+    void init(int seed);
 
-     // intializes with an arbitrary seed
-     void train(idx_t n, const float* x) override;
+    // intializes with an arbitrary seed
+    void train(idx_t n, const float* x) override;
 
-     RandomRotationMatrix () {}
+    RandomRotationMatrix() {}
 };
 
-
 /** Applies a principal component analysis on a set of vectors,
  *  with optionally whitening and random rotation. */
-struct PCAMatrix: LinearTransform {
-
+struct PCAMatrix : LinearTransform {
     /** after transformation the components are multiplied by
      * eigenvalues^eigen_power
      *
@@ -143,6 +129,9 @@ struct PCAMatrix: LinearTransform {
      */
     float eigen_power;
 
+    /// value added to eigenvalues to avoid division by 0 when whitening
+    float epsilon;
+
     /// random rotation after PCA
     bool random_rotation;
 
@@ -162,22 +151,23 @@ struct PCAMatrix: LinearTransform {
     std::vector<float> PCAMat;
 
     // the final matrix is computed after random rotation and/or whitening
-    explicit PCAMatrix (int d_in = 0, int d_out = 0,
-                        float eigen_power = 0, bool random_rotation = false);
+    explicit PCAMatrix(
+            int d_in = 0,
+            int d_out = 0,
+            float eigen_power = 0,
+            bool random_rotation = false);
 
     /// train on n vectors. If n < d_in then the eigenvector matrix
     /// will be completed with 0s
     void train(idx_t n, const float* x) override;
 
     /// copy pre-trained PCA matrix
-    void copy_from (const PCAMatrix & other);
+    void copy_from(const PCAMatrix& other);
 
     /// called after mean, PCAMat and eigenvalues are computed
     void prepare_Ab();
-
 };
 
-
 /** ITQ implementation from
  *
  *     Iterative quantization: A procrustean approach to learning binary codes
@@ -187,25 +177,21 @@ struct PCAMatrix: LinearTransform {
  * PAMI'12.
  */
 
-struct ITQMatrix: LinearTransform {
-
+struct ITQMatrix : LinearTransform {
     int max_iter;
     int seed;
 
     // force initialization of the rotation (for debugging)
     std::vector<double> init_rotation;
 
-    explicit ITQMatrix (int d = 0);
+    explicit ITQMatrix(int d = 0);
 
-    void train (idx_t n, const float* x) override;
+    void train(idx_t n, const float* x) override;
 };
 
-
-
 /** The full ITQ transform, including normalizations and PCA transformation
  */
-struct ITQTransform: VectorTransform {
-
+struct ITQTransform : VectorTransform {
     std::vector<float> mean;
     bool do_pca;
     ITQMatrix itq;
@@ -216,15 +202,13 @@ struct ITQTransform: VectorTransform {
     // concatenation of PCA + ITQ transformation
     LinearTransform pca_then_itq;
 
-    explicit ITQTransform (int d_in = 0, int d_out = 0, bool do_pca = false);
-
-    void train (idx_t n, const float *x) override;
+    explicit ITQTransform(int d_in = 0, int d_out = 0, bool do_pca = false);
 
-    void apply_noalloc (idx_t n, const float* x, float* xt) const override;
+    void train(idx_t n, const float* x) override;
 
+    void apply_noalloc(idx_t n, const float* x, float* xt) const override;
 };
 
-
 struct ProductQuantizer;
 
 /** Applies a rotation to align the dimensions with a PQ to minimize
@@ -235,8 +219,7 @@ struct ProductQuantizer;
  * Tiezheng Ge, Kaiming He, Qifa Ke, Jian Sun, CVPR'13
  *
  */
-struct OPQMatrix: LinearTransform {
-
+struct OPQMatrix : LinearTransform {
     int M;          ///< nb of subquantizers
     int niter;      ///< Number of outer training iterations
     int niter_pq;   ///< Number of training iterations for the PQ
@@ -248,46 +231,43 @@ struct OPQMatrix: LinearTransform {
 
     /// if non-NULL, use this product quantizer for training
     /// should be constructed with (d_out, M, _)
-    ProductQuantizer * pq;
+    ProductQuantizer* pq;
 
     /// if d2 != -1, output vectors of this dimension
-    explicit OPQMatrix (int d = 0, int M = 1, int d2 = -1);
+    explicit OPQMatrix(int d = 0, int M = 1, int d2 = -1);
 
     void train(idx_t n, const float* x) override;
 };
 
-
 /** remap dimensions for intput vectors, possibly inserting 0s
  * strictly speaking this is also a linear transform but we don't want
  * to compute it with matrix multiplies */
-struct RemapDimensionsTransform: VectorTransform {
-
+struct RemapDimensionsTransform : VectorTransform {
     /// map from output dimension to input, size d_out
     /// -1 -> set output to 0
     std::vector<int> map;
 
-    RemapDimensionsTransform (int d_in, int d_out, const int *map);
+    RemapDimensionsTransform(int d_in, int d_out, const int* map);
 
     /// remap input to output, skipping or inserting dimensions as needed
     /// if uniform: distribute dimensions uniformly
     /// otherwise just take the d_out first ones.
-    RemapDimensionsTransform (int d_in, int d_out, bool uniform = true);
+    RemapDimensionsTransform(int d_in, int d_out, bool uniform = true);
 
     void apply_noalloc(idx_t n, const float* x, float* xt) const override;
 
     /// reverse transform correct only when the mapping is a permutation
     void reverse_transform(idx_t n, const float* xt, float* x) const override;
 
-    RemapDimensionsTransform () {}
+    RemapDimensionsTransform() {}
 };
 
-
 /** per-vector normalization */
-struct NormalizationTransform: VectorTransform {
+struct NormalizationTransform : VectorTransform {
     float norm;
 
-    explicit NormalizationTransform (int d, float norm = 2.0);
-    NormalizationTransform ();
+    explicit NormalizationTransform(int d, float norm = 2.0);
+    NormalizationTransform();
 
     void apply_noalloc(idx_t n, const float* x, float* xt) const override;
 
@@ -296,12 +276,11 @@ struct NormalizationTransform: VectorTransform {
 };
 
 /** Subtract the mean of each component from the vectors. */
-struct CenteringTransform: VectorTransform {
-
+struct CenteringTransform : VectorTransform {
     /// Mean, size d_in = d_out
     std::vector<float> mean;
 
-    explicit CenteringTransform (int d = 0);
+    explicit CenteringTransform(int d = 0);
 
     /// train on n vectors.
     void train(idx_t n, const float* x) override;
@@ -310,13 +289,9 @@ struct CenteringTransform: VectorTransform {
     void apply_noalloc(idx_t n, const float* x, float* xt) const override;
 
     /// add the mean
-    void reverse_transform (idx_t n, const float * xt,
-                            float *x) const override;
-
+    void reverse_transform(idx_t n, const float* xt, float* x) const override;
 };
 
-
 } // namespace faiss
 
-
 #endif

data/vendor/faiss/faiss/clone_index.cpp

@@ -14,22 +14,23 @@
 
 #include <faiss/impl/FaissAssert.h>
 
+#include <faiss/Index2Layer.h>
+#include <faiss/IndexAdditiveQuantizer.h>
 #include <faiss/IndexFlat.h>
-#include <faiss/VectorTransform.h>
-#include <faiss/IndexPreTransform.h>
-#include <faiss/IndexLSH.h>
-#include <faiss/IndexPQ.h>
+#include <faiss/IndexHNSW.h>
 #include <faiss/IndexIVF.h>
+#include <faiss/IndexIVFFlat.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/IndexIVFPQR.h>
-#include <faiss/Index2Layer.h>
-#include <faiss/IndexIVFFlat.h>
 #include <faiss/IndexIVFSpectralHash.h>
-#include <faiss/MetaIndexes.h>
-#include <faiss/IndexScalarQuantizer.h>
-#include <faiss/IndexHNSW.h>
+#include <faiss/IndexLSH.h>
 #include <faiss/IndexLattice.h>
-#include <faiss/Index2Layer.h>
+#include <faiss/IndexNSG.h>
+#include <faiss/IndexPQ.h>
+#include <faiss/IndexPreTransform.h>
+#include <faiss/IndexScalarQuantizer.h>
+#include <faiss/MetaIndexes.h>
+#include <faiss/VectorTransform.h>
 
 namespace faiss {
 
@@ -37,111 +38,115 @@ namespace faiss {
  * cloning functions
  **************************************************************/
 
-
-
-Index * clone_index (const Index *index)
-{
+Index* clone_index(const Index* index) {
     Cloner cl;
-    return cl.clone_Index (index);
+    return cl.clone_Index(index);
 }
 
 // assumes there is a copy constructor ready. Always try from most
 // specific to most general. Most indexes don't have complicated
 // structs, the default copy constructor often just works.
-#define TRYCLONE(classname, obj) \
-    if (const classname *clo = dynamic_cast<const classname *>(obj)) { \
-        return new classname(*clo); \
+#define TRYCLONE(classname, obj)                                      \
+    if (const classname* clo = dynamic_cast<const classname*>(obj)) { \
+        return new classname(*clo);                                   \
     } else
 
-VectorTransform *Cloner::clone_VectorTransform (const VectorTransform *vt)
-{
-    TRYCLONE (RemapDimensionsTransform, vt)
-    TRYCLONE (OPQMatrix, vt)
-    TRYCLONE (PCAMatrix, vt)
-    TRYCLONE (ITQMatrix, vt)
-    TRYCLONE (RandomRotationMatrix, vt)
-    TRYCLONE (LinearTransform, vt)
-    {
-      FAISS_THROW_MSG("clone not supported for this type of VectorTransform");
+VectorTransform* Cloner::clone_VectorTransform(const VectorTransform* vt) {
+    TRYCLONE(RemapDimensionsTransform, vt)
+    TRYCLONE(OPQMatrix, vt)
+    TRYCLONE(PCAMatrix, vt)
+    TRYCLONE(ITQMatrix, vt)
+    TRYCLONE(RandomRotationMatrix, vt)
+    TRYCLONE(LinearTransform, vt) {
+        FAISS_THROW_MSG("clone not supported for this type of VectorTransform");
     }
     return nullptr;
 }
 
-IndexIVF * Cloner::clone_IndexIVF (const IndexIVF *ivf)
-{
-    TRYCLONE (IndexIVFPQR, ivf)
-    TRYCLONE (IndexIVFPQ, ivf)
-    TRYCLONE (IndexIVFFlat, ivf)
-    TRYCLONE (IndexIVFScalarQuantizer, ivf)
-    {
-      FAISS_THROW_MSG("clone not supported for this type of IndexIVF");
+IndexIVF* Cloner::clone_IndexIVF(const IndexIVF* ivf) {
+    TRYCLONE(IndexIVFPQR, ivf)
+    TRYCLONE(IndexIVFPQ, ivf)
+    TRYCLONE(IndexIVFFlat, ivf)
+    TRYCLONE(IndexIVFScalarQuantizer, ivf) {
+        FAISS_THROW_MSG("clone not supported for this type of IndexIVF");
     }
     return nullptr;
 }
 
-Index *Cloner::clone_Index (const Index *index)
-{
-    TRYCLONE (IndexPQ, index)
-    TRYCLONE (IndexLSH, index)
-    TRYCLONE (IndexFlatL2, index)
-    TRYCLONE (IndexFlatIP, index)
-    TRYCLONE (IndexFlat, index)
-    TRYCLONE (IndexLattice, index)
-    TRYCLONE (IndexScalarQuantizer, index)
-    TRYCLONE (MultiIndexQuantizer, index)
-    if (const IndexIVF * ivf = dynamic_cast<const IndexIVF*>(index)) {
-        IndexIVF *res = clone_IndexIVF (ivf);
+Index* Cloner::clone_Index(const Index* index) {
+    TRYCLONE(IndexPQ, index)
+    TRYCLONE(IndexLSH, index)
+    TRYCLONE(IndexFlatL2, index)
+    TRYCLONE(IndexFlatIP, index)
+    TRYCLONE(IndexFlat, index)
+    TRYCLONE(IndexLattice, index)
+    TRYCLONE(IndexResidualQuantizer, index)
+    TRYCLONE(IndexScalarQuantizer, index)
+    TRYCLONE(MultiIndexQuantizer, index)
+    TRYCLONE(ResidualCoarseQuantizer, index)
+    if (const IndexIVF* ivf = dynamic_cast<const IndexIVF*>(index)) {
+        IndexIVF* res = clone_IndexIVF(ivf);
         if (ivf->invlists == nullptr) {
             res->invlists = nullptr;
-        } else if (auto *ails = dynamic_cast<const ArrayInvertedLists*>
-                   (ivf->invlists)) {
+        } else if (
+                auto* ails = dynamic_cast<const ArrayInvertedLists*>(
+                        ivf->invlists)) {
             res->invlists = new ArrayInvertedLists(*ails);
             res->own_invlists = true;
         } else {
-            FAISS_THROW_MSG( "clone not supported for this type of inverted lists");
+            FAISS_THROW_MSG(
+                    "clone not supported for this type of inverted lists");
         }
         res->own_fields = true;
-        res->quantizer = clone_Index (ivf->quantizer);
+        res->quantizer = clone_Index(ivf->quantizer);
         return res;
-    } else if (const IndexPreTransform * ipt =
-               dynamic_cast<const IndexPreTransform*> (index)) {
-        IndexPreTransform *res = new IndexPreTransform ();
+    } else if (
+            const IndexPreTransform* ipt =
+                    dynamic_cast<const IndexPreTransform*>(index)) {
+        IndexPreTransform* res = new IndexPreTransform();
         res->d = ipt->d;
         res->ntotal = ipt->ntotal;
         res->is_trained = ipt->is_trained;
         res->metric_type = ipt->metric_type;
         res->metric_arg = ipt->metric_arg;
 
-
-        res->index = clone_Index (ipt->index);
+        res->index = clone_Index(ipt->index);
         for (int i = 0; i < ipt->chain.size(); i++)
-            res->chain.push_back (clone_VectorTransform (ipt->chain[i]));
+            res->chain.push_back(clone_VectorTransform(ipt->chain[i]));
         res->own_fields = true;
         return res;
-    } else if (const IndexIDMap *idmap =
-               dynamic_cast<const IndexIDMap*> (index)) {
-        IndexIDMap *res = new IndexIDMap (*idmap);
+    } else if (
+            const IndexIDMap* idmap = dynamic_cast<const IndexIDMap*>(index)) {
+        IndexIDMap* res = new IndexIDMap(*idmap);
         res->own_fields = true;
-        res->index = clone_Index (idmap->index);
+        res->index = clone_Index(idmap->index);
         return res;
-    } else if (const IndexHNSW *ihnsw =
-               dynamic_cast<const IndexHNSW*> (index)) {
-        IndexHNSW *res = new IndexHNSW (*ihnsw);
+    } else if (const IndexHNSW* ihnsw = dynamic_cast<const IndexHNSW*>(index)) {
+        IndexHNSW* res = new IndexHNSW(*ihnsw);
         res->own_fields = true;
-        res->storage = clone_Index (ihnsw->storage);
+        res->storage = clone_Index(ihnsw->storage);
         return res;
-    } else if (const Index2Layer *i2l =
-               dynamic_cast<const Index2Layer*> (index)) {
-        Index2Layer *res = new Index2Layer (*i2l);
+    } else if (const IndexNSG* insg = dynamic_cast<const IndexNSG*>(index)) {
+        IndexNSG* res = new IndexNSG(*insg);
+
+        // copy the dynamic allocated graph
+        auto& new_graph = res->nsg.final_graph;
+        auto& old_graph = insg->nsg.final_graph;
+        new_graph = std::make_shared<nsg::Graph<int>>(*old_graph);
+
+        res->own_fields = true;
+        res->storage = clone_Index(insg->storage);
+        return res;
+    } else if (
+            const Index2Layer* i2l = dynamic_cast<const Index2Layer*>(index)) {
+        Index2Layer* res = new Index2Layer(*i2l);
         res->q1.own_fields = true;
-        res->q1.quantizer = clone_Index (i2l->q1.quantizer);
+        res->q1.quantizer = clone_Index(i2l->q1.quantizer);
         return res;
     } else {
-        FAISS_THROW_MSG( "clone not supported for this type of Index");
+        FAISS_THROW_MSG("clone not supported for this type of Index");
     }
     return nullptr;
 }
 
-
-
 } // namespace faiss

data/vendor/faiss/faiss/clone_index.h

@@ -11,28 +11,23 @@
 
 #pragma once
 
-
-
 namespace faiss {
 
 struct Index;
 struct IndexIVF;
 struct VectorTransform;
 
-
 /* cloning functions */
-Index *clone_index (const Index *);
+Index* clone_index(const Index*);
 
 /** Cloner class, useful to override classes with other cloning
  * functions. The cloning function above just calls
  * Cloner::clone_Index. */
 struct Cloner {
-    virtual VectorTransform *clone_VectorTransform (const VectorTransform *);
-    virtual Index *clone_Index (const Index *);
-    virtual IndexIVF *clone_IndexIVF (const IndexIVF *);
+    virtual VectorTransform* clone_VectorTransform(const VectorTransform*);
+    virtual Index* clone_Index(const Index*);
+    virtual IndexIVF* clone_IndexIVF(const IndexIVF*);
     virtual ~Cloner() {}
 };
 
-
-
 } // namespace faiss