faiss 0.2.0 → 0.2.4

data/vendor/faiss/faiss/IndexLSH.cpp

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
-
 #include <faiss/IndexLSH.h>
 
 #include <cstdio>
@@ -14,10 +12,9 @@
 
 #include <algorithm>
 
-#include <faiss/utils/utils.h>
-#include <faiss/utils/hamming.h>
 #include <faiss/impl/FaissAssert.h>
-
+#include <faiss/utils/hamming.h>
+#include <faiss/utils/utils.h>
 
 namespace faiss {
 
@@ -25,143 +22,117 @@ namespace faiss {
  * IndexLSH
  ***************************************************************/
 
-
-IndexLSH::IndexLSH (idx_t d, int nbits, bool rotate_data, bool train_thresholds):
-    Index(d), nbits(nbits), rotate_data(rotate_data),
-    train_thresholds (train_thresholds), rrot(d, nbits)
-{
+IndexLSH::IndexLSH(idx_t d, int nbits, bool rotate_data, bool train_thresholds)
+        : IndexFlatCodes((nbits + 7) / 8, d),
+          nbits(nbits),
+          rotate_data(rotate_data),
+          train_thresholds(train_thresholds),
+          rrot(d, nbits) {
     is_trained = !train_thresholds;
 
-    bytes_per_vec = (nbits + 7) / 8;
-
     if (rotate_data) {
         rrot.init(5);
     } else {
-        FAISS_THROW_IF_NOT (d >= nbits);
+        FAISS_THROW_IF_NOT(d >= nbits);
     }
 }
 
-IndexLSH::IndexLSH ():
-    nbits (0), bytes_per_vec(0), rotate_data (false), train_thresholds (false)
-{
-}
-
+IndexLSH::IndexLSH() : nbits(0), rotate_data(false), train_thresholds(false) {}
 
-const float * IndexLSH::apply_preprocess (idx_t n, const float *x) const
-{
-
-    float *xt = nullptr;
+const float* IndexLSH::apply_preprocess(idx_t n, const float* x) const {
+    float* xt = nullptr;
     if (rotate_data) {
         // also applies bias if exists
-        xt = rrot.apply (n, x);
+        xt = rrot.apply(n, x);
     } else if (d != nbits) {
-        assert (nbits < d);
-        xt = new float [nbits * n];
-        float *xp = xt;
+        assert(nbits < d);
+        xt = new float[nbits * n];
+        float* xp = xt;
         for (idx_t i = 0; i < n; i++) {
-            const float *xl = x + i * d;
+            const float* xl = x + i * d;
             for (int j = 0; j < nbits; j++)
-                *xp++ = xl [j];
+                *xp++ = xl[j];
         }
     }
 
     if (train_thresholds) {
-
         if (xt == NULL) {
-            xt = new float [nbits * n];
-            memcpy (xt, x, sizeof(*x) * n * nbits);
+            xt = new float[nbits * n];
+            memcpy(xt, x, sizeof(*x) * n * nbits);
         }
 
-        float *xp = xt;
+        float* xp = xt;
         for (idx_t i = 0; i < n; i++)
             for (int j = 0; j < nbits; j++)
-                *xp++ -= thresholds [j];
+                *xp++ -= thresholds[j];
     }
 
     return xt ? xt : x;
 }
 
-
-
-void IndexLSH::train (idx_t n, const float *x)
-{
+void IndexLSH::train(idx_t n, const float* x) {
     if (train_thresholds) {
-        thresholds.resize (nbits);
+        thresholds.resize(nbits);
         train_thresholds = false;
-        const float *xt = apply_preprocess (n, x);
-        ScopeDeleter<float> del (xt == x ? nullptr : xt);
+        const float* xt = apply_preprocess(n, x);
+        ScopeDeleter<float> del(xt == x ? nullptr : xt);
         train_thresholds = true;
 
-        float * transposed_x = new float [n * nbits];
-        ScopeDeleter<float> del2 (transposed_x);
+        float* transposed_x = new float[n * nbits];
+        ScopeDeleter<float> del2(transposed_x);
 
         for (idx_t i = 0; i < n; i++)
             for (idx_t j = 0; j < nbits; j++)
-                transposed_x [j * n + i] = xt [i * nbits + j];
+                transposed_x[j * n + i] = xt[i * nbits + j];
 
         for (idx_t i = 0; i < nbits; i++) {
-            float *xi = transposed_x + i * n;
+            float* xi = transposed_x + i * n;
             // std::nth_element
-            std::sort (xi, xi + n);
+            std::sort(xi, xi + n);
             if (n % 2 == 1)
-                thresholds [i] = xi [n / 2];
+                thresholds[i] = xi[n / 2];
             else
-                thresholds [i] = (xi [n / 2 - 1] + xi [n / 2]) / 2;
-
+                thresholds[i] = (xi[n / 2 - 1] + xi[n / 2]) / 2;
         }
     }
     is_trained = true;
 }
 
-
-void IndexLSH::add (idx_t n, const float *x)
-{
-    FAISS_THROW_IF_NOT (is_trained);
-    codes.resize ((ntotal + n) * bytes_per_vec);
-
-    sa_encode (n, x, &codes[ntotal * bytes_per_vec]);
-
-    ntotal += n;
-}
-
-
-void IndexLSH::search (
+void IndexLSH::search(
         idx_t n,
-        const float *x,
+        const float* x,
         idx_t k,
-        float *distances,
-        idx_t *labels) const
-{
-    FAISS_THROW_IF_NOT (is_trained);
-    const float *xt = apply_preprocess (n, x);
-    ScopeDeleter<float> del (xt == x ? nullptr : xt);
+        float* distances,
+        idx_t* labels) const {
+    FAISS_THROW_IF_NOT(k > 0);
 
-    uint8_t * qcodes = new uint8_t [n * bytes_per_vec];
-    ScopeDeleter<uint8_t> del2 (qcodes);
+    FAISS_THROW_IF_NOT(is_trained);
+    const float* xt = apply_preprocess(n, x);
+    ScopeDeleter<float> del(xt == x ? nullptr : xt);
 
-    fvecs2bitvecs (xt, qcodes, nbits, n);
+    uint8_t* qcodes = new uint8_t[n * code_size];
+    ScopeDeleter<uint8_t> del2(qcodes);
 
-    int * idistances = new int [n * k];
-    ScopeDeleter<int> del3 (idistances);
+    fvecs2bitvecs(xt, qcodes, nbits, n);
 
-    int_maxheap_array_t res = { size_t(n), size_t(k), labels, idistances};
+    int* idistances = new int[n * k];
+    ScopeDeleter<int> del3(idistances);
 
-    hammings_knn_hc (&res, qcodes, codes.data(),
-                     ntotal, bytes_per_vec, true);
+    int_maxheap_array_t res = {size_t(n), size_t(k), labels, idistances};
 
+    hammings_knn_hc(&res, qcodes, codes.data(), ntotal, code_size, true);
 
     // convert distances to floats
     for (int i = 0; i < k * n; i++)
         distances[i] = idistances[i];
-
 }
 
-
-void IndexLSH::transfer_thresholds (LinearTransform *vt) {
-    if (!train_thresholds) return;
-    FAISS_THROW_IF_NOT (nbits == vt->d_out);
+void IndexLSH::transfer_thresholds(LinearTransform* vt) {
+    if (!train_thresholds)
+        return;
+    FAISS_THROW_IF_NOT(nbits == vt->d_out);
     if (!vt->have_bias) {
-        vt->b.resize (nbits, 0);
+        vt->b.resize(nbits, 0);
         vt->have_bias = true;
     }
     for (int i = 0; i < nbits; i++)
@@ -170,56 +141,38 @@ void IndexLSH::transfer_thresholds (LinearTransform *vt) {
     thresholds.clear();
 }
 
-void IndexLSH::reset() {
-    codes.clear();
-    ntotal = 0;
+void IndexLSH::sa_encode(idx_t n, const float* x, uint8_t* bytes) const {
+    FAISS_THROW_IF_NOT(is_trained);
+    const float* xt = apply_preprocess(n, x);
+    ScopeDeleter<float> del(xt == x ? nullptr : xt);
+    fvecs2bitvecs(xt, bytes, nbits, n);
 }
 
-
-size_t IndexLSH::sa_code_size () const
-{
-    return bytes_per_vec;
-}
-
-void IndexLSH::sa_encode (idx_t n, const float *x,
-                                uint8_t *bytes) const
-{
-    FAISS_THROW_IF_NOT (is_trained);
-    const float *xt = apply_preprocess (n, x);
-    ScopeDeleter<float> del (xt == x ? nullptr : xt);
-    fvecs2bitvecs (xt, bytes, nbits, n);
-}
-
-void IndexLSH::sa_decode (idx_t n, const uint8_t *bytes,
-                                  float *x) const
-{
-    float *xt = x;
+void IndexLSH::sa_decode(idx_t n, const uint8_t* bytes, float* x) const {
+    float* xt = x;
     ScopeDeleter<float> del;
     if (rotate_data || nbits != d) {
-        xt = new float [n * nbits];
+        xt = new float[n * nbits];
         del.set(xt);
     }
-    bitvecs2fvecs (bytes, xt, nbits, n);
+    bitvecs2fvecs(bytes, xt, nbits, n);
 
     if (train_thresholds) {
-        float *xp = xt;
+        float* xp = xt;
         for (idx_t i = 0; i < n; i++) {
             for (int j = 0; j < nbits; j++) {
-                *xp++ += thresholds [j];
+                *xp++ += thresholds[j];
             }
         }
     }
 
     if (rotate_data) {
-        rrot.reverse_transform (n, xt, x);
+        rrot.reverse_transform(n, xt, x);
     } else if (nbits != d) {
         for (idx_t i = 0; i < n; i++) {
-            memcpy (x + i * d, xt + i * nbits,
-                    nbits * sizeof(xt[0]));
+            memcpy(x + i * d, xt + i * nbits, nbits * sizeof(xt[0]));
         }
     }
 }
 
-
-
 } // namespace faiss

data/vendor/faiss/faiss/IndexLSH.h

@@ -12,30 +12,24 @@
 
 #include <vector>
 
-#include <faiss/Index.h>
+#include <faiss/IndexFlatCodes.h>
 #include <faiss/VectorTransform.h>
 
 namespace faiss {
 
-
 /** The sign of each vector component is put in a binary signature */
-struct IndexLSH:Index {
-    typedef unsigned char uint8_t;
-
-    int nbits;              ///< nb of bits per vector
-    int bytes_per_vec;      ///< nb of 8-bits per encoded vector
-    bool rotate_data;       ///< whether to apply a random rotation to input
-    bool train_thresholds;  ///< whether we train thresholds or use 0
+struct IndexLSH : IndexFlatCodes {
+    int nbits;             ///< nb of bits per vector
+    bool rotate_data;      ///< whether to apply a random rotation to input
+    bool train_thresholds; ///< whether we train thresholds or use 0
 
     RandomRotationMatrix rrot; ///< optional random rotation
 
-    std::vector <float> thresholds; ///< thresholds to compare with
-
-    /// encoded dataset
-    std::vector<uint8_t> codes;
+    std::vector<float> thresholds; ///< thresholds to compare with
 
-    IndexLSH (
-            idx_t d, int nbits,
+    IndexLSH(
+            idx_t d,
+            int nbits,
             bool rotate_data = true,
             bool train_thresholds = false);
 
@@ -46,45 +40,33 @@ struct IndexLSH:Index {
      * @return output vectors, size n * bits. May be the same pointer
      *         as x, otherwise it should be deleted by the caller
      */
-    const float *apply_preprocess (idx_t n, const float *x) const;
+    const float* apply_preprocess(idx_t n, const float* x) const;
 
     void train(idx_t n, const float* x) override;
 
-    void add(idx_t n, const float* x) override;
-
     void search(
-        idx_t n,
-        const float* x,
-        idx_t k,
-        float* distances,
-        idx_t* labels) const override;
-
-    void reset() override;
+            idx_t n,
+            const float* x,
+            idx_t k,
+            float* distances,
+            idx_t* labels) const override;
 
     /// transfer the thresholds to a pre-processing stage (and unset
     /// train_thresholds)
-    void transfer_thresholds (LinearTransform * vt);
+    void transfer_thresholds(LinearTransform* vt);
 
     ~IndexLSH() override {}
 
-    IndexLSH ();
+    IndexLSH();
 
     /* standalone codec interface.
      *
      * The vectors are decoded to +/- 1 (not 0, 1) */
+    void sa_encode(idx_t n, const float* x, uint8_t* bytes) const override;
 
-    size_t sa_code_size () const override;
-
-    void sa_encode (idx_t n, const float *x,
-                          uint8_t *bytes) const override;
-
-    void sa_decode (idx_t n, const uint8_t *bytes,
-                            float *x) const override;
-
+    void sa_decode(idx_t n, const uint8_t* bytes, float* x) const override;
 };
 
-
-}
-
+} // namespace faiss
 
 #endif

data/vendor/faiss/faiss/IndexLattice.cpp

@@ -7,26 +7,23 @@
 
 // -*- c++ -*-
 
-
 #include <faiss/IndexLattice.h>
-#include <faiss/utils/hamming.h>    // for the bitstring routines
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/utils/distances.h>
+#include <faiss/utils/hamming.h> // for the bitstring routines
 
 namespace faiss {
 
-
-IndexLattice::IndexLattice (idx_t d, int nsq, int scale_nbit, int r2):
-    Index (d),
-    nsq (nsq),
-    dsq (d / nsq),
-    zn_sphere_codec (dsq, r2),
-    scale_nbit (scale_nbit)
-{
-    FAISS_THROW_IF_NOT (d % nsq == 0);
+IndexLattice::IndexLattice(idx_t d, int nsq, int scale_nbit, int r2)
+        : Index(d),
+          nsq(nsq),
+          dsq(d / nsq),
+          zn_sphere_codec(dsq, r2),
+          scale_nbit(scale_nbit) {
+    FAISS_THROW_IF_NOT(d % nsq == 0);
 
     lattice_nbit = 0;
-    while (!( ((uint64_t)1 << lattice_nbit) >= zn_sphere_codec.nv)) {
+    while (!(((uint64_t)1 << lattice_nbit) >= zn_sphere_codec.nv)) {
         lattice_nbit++;
     }
 
@@ -37,12 +34,11 @@ IndexLattice::IndexLattice (idx_t d, int nsq, int scale_nbit, int r2):
     is_trained = false;
 }
 
-void IndexLattice::train(idx_t n, const float* x)
-{
+void IndexLattice::train(idx_t n, const float* x) {
     // compute ranges per sub-block
-    trained.resize (nsq * 2);
-    float * mins = trained.data();
-    float * maxs = trained.data() + nsq;
+    trained.resize(nsq * 2);
+    float* mins = trained.data();
+    float* maxs = trained.data() + nsq;
     for (int sq = 0; sq < nsq; sq++) {
         mins[sq] = HUGE_VAL;
         maxs[sq] = -1;
@@ -50,45 +46,43 @@ void IndexLattice::train(idx_t n, const float* x)
 
     for (idx_t i = 0; i < n; i++) {
         for (int sq = 0; sq < nsq; sq++) {
-            float norm2 = fvec_norm_L2sqr (x + i * d + sq * dsq, dsq);
-            if (norm2 > maxs[sq]) maxs[sq] = norm2;
-            if (norm2 < mins[sq]) mins[sq] = norm2;
+            float norm2 = fvec_norm_L2sqr(x + i * d + sq * dsq, dsq);
+            if (norm2 > maxs[sq])
+                maxs[sq] = norm2;
+            if (norm2 < mins[sq])
+                mins[sq] = norm2;
         }
     }
 
     for (int sq = 0; sq < nsq; sq++) {
-        mins[sq] = sqrtf (mins[sq]);
-        maxs[sq] = sqrtf (maxs[sq]);
+        mins[sq] = sqrtf(mins[sq]);
+        maxs[sq] = sqrtf(maxs[sq]);
     }
 
     is_trained = true;
 }
 
 /* The standalone codec interface */
-size_t IndexLattice::sa_code_size () const
-{
+size_t IndexLattice::sa_code_size() const {
     return code_size;
 }
 
-
-
-void IndexLattice::sa_encode (idx_t n, const float *x, uint8_t *codes) const
-{
-
-    const float * mins = trained.data();
-    const float * maxs = mins + nsq;
+void IndexLattice::sa_encode(idx_t n, const float* x, uint8_t* codes) const {
+    const float* mins = trained.data();
+    const float* maxs = mins + nsq;
     int64_t sc = int64_t(1) << scale_nbit;
 
 #pragma omp parallel for
     for (idx_t i = 0; i < n; i++) {
         BitstringWriter wr(codes + i * code_size, code_size);
-        const float *xi = x + i * d;
+        const float* xi = x + i * d;
         for (int j = 0; j < nsq; j++) {
-            float nj =
-                (sqrtf(fvec_norm_L2sqr(xi, dsq)) - mins[j])
-                * sc / (maxs[j] - mins[j]);
-            if (nj < 0) nj = 0;
-            if (nj >= sc) nj = sc - 1;
+            float nj = (sqrtf(fvec_norm_L2sqr(xi, dsq)) - mins[j]) * sc /
+                    (maxs[j] - mins[j]);
+            if (nj < 0)
+                nj = 0;
+            if (nj >= sc)
+                nj = sc - 1;
             wr.write((int64_t)nj, scale_nbit);
             wr.write(zn_sphere_codec.encode(xi), lattice_nbit);
             xi += dsq;
@@ -96,23 +90,22 @@ void IndexLattice::sa_encode (idx_t n, const float *x, uint8_t *codes) const
     }
 }
 
-void IndexLattice::sa_decode (idx_t n, const uint8_t *codes, float *x) const
-{
-    const float * mins = trained.data();
-    const float * maxs = mins + nsq;
+void IndexLattice::sa_decode(idx_t n, const uint8_t* codes, float* x) const {
+    const float* mins = trained.data();
+    const float* maxs = mins + nsq;
     float sc = int64_t(1) << scale_nbit;
     float r = sqrtf(zn_sphere_codec.r2);
 
 #pragma omp parallel for
     for (idx_t i = 0; i < n; i++) {
         BitstringReader rd(codes + i * code_size, code_size);
-        float *xi = x + i * d;
+        float* xi = x + i * d;
         for (int j = 0; j < nsq; j++) {
             float norm =
-                (rd.read (scale_nbit) + 0.5) *
-                (maxs[j] - mins[j]) / sc + mins[j];
+                    (rd.read(scale_nbit) + 0.5) * (maxs[j] - mins[j]) / sc +
+                    mins[j];
             norm /= r;
-            zn_sphere_codec.decode (rd.read (lattice_nbit), xi);
+            zn_sphere_codec.decode(rd.read(lattice_nbit), xi);
             for (int l = 0; l < dsq; l++) {
                 xi[l] *= norm;
             }
@@ -121,23 +114,16 @@ void IndexLattice::sa_decode (idx_t n, const uint8_t *codes, float *x) const
     }
 }
 
-void IndexLattice::add(idx_t , const float* )
-{
+void IndexLattice::add(idx_t, const float*) {
     FAISS_THROW_MSG("not implemented");
 }
 
-
-void  IndexLattice::search(idx_t , const float* , idx_t ,
-                           float* , idx_t* ) const
-{
+void IndexLattice::search(idx_t, const float*, idx_t, float*, idx_t*) const {
     FAISS_THROW_MSG("not implemented");
 }
 
-
-void IndexLattice::reset()
-{
+void IndexLattice::reset() {
     FAISS_THROW_MSG("not implemented");
 }
 
-
-}  // namespace faiss
+} // namespace faiss

data/vendor/faiss/faiss/IndexLattice.h

@@ -10,7 +10,6 @@
 #ifndef FAISS_INDEX_LATTICE_H
 #define FAISS_INDEX_LATTICE_H
 
-
 #include <vector>
 
 #include <faiss/IndexIVF.h>
@@ -18,14 +17,9 @@
 
 namespace faiss {
 
-
-
-
-
 /** Index that encodes a vector with a series of Zn lattice quantizers
  */
-struct IndexLattice: Index {
-
+struct IndexLattice : Index {
     /// number of sub-vectors
     int nsq;
     /// dimension of sub-vectors
@@ -42,25 +36,26 @@ struct IndexLattice: Index {
     /// mins and maxes of the vector norms, per subquantizer
     std::vector<float> trained;
 
-    IndexLattice (idx_t d, int nsq, int scale_nbit, int r2);
+    IndexLattice(idx_t d, int nsq, int scale_nbit, int r2);
 
     void train(idx_t n, const float* x) override;
 
     /* The standalone codec interface */
-    size_t sa_code_size () const override;
+    size_t sa_code_size() const override;
 
-    void sa_encode (idx_t n, const float *x,
-                          uint8_t *bytes) const override;
+    void sa_encode(idx_t n, const float* x, uint8_t* bytes) const override;
 
-    void sa_decode (idx_t n, const uint8_t *bytes,
-                            float *x) const override;
+    void sa_decode(idx_t n, const uint8_t* bytes, float* x) const override;
 
     /// not implemented
     void add(idx_t n, const float* x) override;
-    void search(idx_t n, const float* x, idx_t k,
-                float* distances, idx_t* labels) const override;
+    void search(
+            idx_t n,
+            const float* x,
+            idx_t k,
+            float* distances,
+            idx_t* labels) const override;
     void reset() override;
-
 };
 
 } // namespace faiss