faiss 0.1.1 → 0.1.2

data/vendor/faiss/index_factory.cpp

@@ -14,7 +14,6 @@
 #include <faiss/AutoTune.h>
 
 #include <cmath>
-#include <stdarg.h>     /* va_list, va_start, va_arg, va_end */
 
 
 #include <faiss/impl/FaissAssert.h>

data/vendor/faiss/tests/test_merge.cpp

@@ -14,7 +14,6 @@
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/IndexFlat.h>
 #include <faiss/MetaIndexes.h>
-#include <faiss/impl/FaissAssert.h>
 #include <faiss/IndexPreTransform.h>
 #include <faiss/OnDiskInvertedLists.h>
 #include <faiss/IVFlib.h>

data/vendor/faiss/tests/test_pq_encoding.cpp

@@ -45,13 +45,13 @@ TEST(PQEncoderGeneric, encode) {
 
     // NOTE(hoss): Necessary scope to ensure trailing bits are flushed to mem.
     {
-      faiss::ProductQuantizer::PQEncoderGeneric encoder(codes.get(), nbits);
+      faiss::PQEncoderGeneric encoder(codes.get(), nbits);
       for (const auto& v : values) {
         encoder.encode(v & mask);
       }
     }
 
-    faiss::ProductQuantizer::PQDecoderGeneric decoder(codes.get(), nbits);
+    faiss::PQDecoderGeneric decoder(codes.get(), nbits);
     for (int i = 0; i < nsubcodes; ++i) {
       uint64_t v = decoder.decode();
       EXPECT_EQ(values[i] & mask, v);
@@ -66,12 +66,12 @@ TEST(PQEncoder8, encode) {
   const uint64_t mask = 0xFF;
   std::unique_ptr<uint8_t[]> codes(new uint8_t[nsubcodes]);
 
-  faiss::ProductQuantizer::PQEncoder8 encoder(codes.get(), 8);
+  faiss::PQEncoder8 encoder(codes.get(), 8);
   for (const auto& v : values) {
     encoder.encode(v & mask);
   }
 
-  faiss::ProductQuantizer::PQDecoder8 decoder(codes.get(), 8);
+  faiss::PQDecoder8 decoder(codes.get(), 8);
   for (int i = 0; i < nsubcodes; ++i) {
     uint64_t v = decoder.decode();
     EXPECT_EQ(values[i] & mask, v);
@@ -85,12 +85,12 @@ TEST(PQEncoder16, encode) {
   const uint64_t mask = 0xFFFF;
   std::unique_ptr<uint8_t[]> codes(new uint8_t[2 * nsubcodes]);
 
-  faiss::ProductQuantizer::PQEncoder16 encoder(codes.get(), 16);
+  faiss::PQEncoder16 encoder(codes.get(), 16);
   for (const auto& v : values) {
     encoder.encode(v & mask);
   }
 
-  faiss::ProductQuantizer::PQDecoder16 decoder(codes.get(), 16);
+  faiss::PQDecoder16 decoder(codes.get(), 16);
   for (int i = 0; i < nsubcodes; ++i) {
     uint64_t v = decoder.decode();
     EXPECT_EQ(values[i] & mask, v);

data/vendor/faiss/tutorial/cpp/5-Multiple-GPUs.cpp

@@ -10,6 +10,7 @@
 
 #include <faiss/IndexFlat.h>
 #include <faiss/gpu/GpuAutoTune.h>
+#include <faiss/gpu/GpuCloner.h>
 #include <faiss/gpu/GpuIndexFlat.h>
 #include <faiss/gpu/StandardGpuResources.h>
 #include <faiss/gpu/utils/DeviceUtils.h>

data/vendor/faiss/utils/distances.cpp

@@ -359,7 +359,7 @@ void knn_inner_product (const float * x,
         size_t d, size_t nx, size_t ny,
         float_minheap_array_t * res)
 {
-    if (d % 4 == 0 && nx < distance_compute_blas_threshold) {
+    if (nx < distance_compute_blas_threshold) {
         knn_inner_product_sse (x, y, d, nx, ny, res);
     } else {
         knn_inner_product_blas (x, y, d, nx, ny, res);
@@ -379,7 +379,7 @@ void knn_L2sqr (const float * x,
                 size_t d, size_t nx, size_t ny,
                 float_maxheap_array_t * res)
 {
-    if (d % 4 == 0 && nx < distance_compute_blas_threshold) {
+    if (nx < distance_compute_blas_threshold) {
         knn_L2sqr_sse (x, y, d, nx, ny, res);
     } else {
         NopDistanceCorrection nop;
@@ -643,7 +643,6 @@ static void range_search_sse (const float * x,
                 float radius,
                 RangeSearchResult *res)
 {
-    FAISS_THROW_IF_NOT (d % 4 == 0);
 
 #pragma omp parallel
     {
@@ -693,7 +692,7 @@ void range_search_L2sqr (
         RangeSearchResult *res)
 {
 
-    if (d % 4 == 0 && nx < distance_compute_blas_threshold) {
+    if (nx < distance_compute_blas_threshold) {
         range_search_sse<true> (x, y, d, nx, ny, radius, res);
     } else {
         range_search_blas<true> (x, y, d, nx, ny, radius, res);
@@ -708,7 +707,7 @@ void range_search_inner_product (
         RangeSearchResult *res)
 {
 
-    if (d % 4 == 0 && nx < distance_compute_blas_threshold) {
+    if (nx < distance_compute_blas_threshold) {
         range_search_sse<false> (x, y, d, nx, ny, radius, res);
     } else {
         range_search_blas<false> (x, y, d, nx, ny, radius, res);

data/vendor/faiss/utils/distances_simd.cpp

@@ -22,7 +22,6 @@
 #include <arm_neon.h>
 #endif
 
-#include <omp.h>
 
 namespace faiss {
 

data/vendor/faiss/utils/hamming.cpp

@@ -28,15 +28,13 @@
 
 #include <vector>
 #include <memory>
-#include <stdlib.h>
 #include <stdio.h>
 #include <math.h>
-#include <assert.h>
-#include <limits.h>
 
 #include <faiss/utils/Heap.h>
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/utils/utils.h>
+#include <faiss/impl/AuxIndexStructures.h>
 
 static const size_t BLOCKSIZE_QUERY = 8192;
 
@@ -487,6 +485,30 @@ void bitvec_print (const uint8_t * b, size_t d)
 }
 
 
+void bitvec_shuffle (size_t n, size_t da, size_t db,
+                     const int *order,
+                     const uint8_t *a,
+                     uint8_t *b)
+{
+    for(size_t i = 0; i < db; i++) {
+        FAISS_THROW_IF_NOT (order[i] >= 0 && order[i] < da);
+    }
+    size_t lda = (da + 7) / 8;
+    size_t ldb = (db + 7) / 8;
+
+#pragma omp parallel for if(n > 10000)
+    for (size_t i = 0; i < n; i++) {
+        const uint8_t *ai = a + i * lda;
+        uint8_t *bi = b + i * ldb;
+        memset (bi, 0, ldb);
+        for(size_t i = 0; i < db; i++) {
+            int o = order[i];
+            uint8_t the_bit = (ai[o >> 3] >> (o & 7)) & 1;
+            bi[i >> 3] |= the_bit << (i & 7);
+        }
+    }
+
+}
 
 
 
@@ -530,6 +552,7 @@ void hammings_knn(
 {
     hammings_knn_hc(ha, a, b, nb, ncodes, order);
 }
+
 void hammings_knn_hc (
         int_maxheap_array_t * ha,
         const uint8_t * a,
@@ -613,7 +636,66 @@ void hammings_knn_mc(
         }
     }
 }
+template <class HammingComputer>
+static
+void hamming_range_search_template (
+    const uint8_t * a,
+    const uint8_t * b,
+    size_t na,
+    size_t nb,
+    int radius,
+    size_t code_size,
+    RangeSearchResult *res)
+{
 
+#pragma omp parallel
+    {
+        RangeSearchPartialResult pres (res);
+
+#pragma omp for
+        for (size_t i = 0; i < na; i++) {
+             HammingComputer hc (a + i * code_size, code_size);
+            const uint8_t * yi = b;
+            RangeQueryResult & qres = pres.new_result (i);
+
+            for (size_t j = 0; j < nb; j++) {
+                int dis = hc.hamming (yi);
+                if (dis < radius) {
+                    qres.add(dis, j);
+                }
+                yi += code_size;
+            }
+        }
+        pres.finalize ();
+    }
+}
+
+void hamming_range_search (
+    const uint8_t * a,
+    const uint8_t * b,
+    size_t na,
+    size_t nb,
+    int radius,
+    size_t code_size,
+    RangeSearchResult *result)
+{
+
+#define HC(name) hamming_range_search_template<name> (a, b, na, nb, radius, code_size, result)
+
+    switch(code_size) {
+    case 4: HC(HammingComputer4); break;
+    case 8: HC(HammingComputer8); break;
+    case 16: HC(HammingComputer16); break;
+    case 32: HC(HammingComputer32); break;
+    default:
+        if (code_size % 8 == 0) {
+            HC(HammingComputerM8);
+        } else {
+            HC(HammingComputerDefault);
+        }
+    }
+#undef HC
+}
 
 
 

data/vendor/faiss/utils/hamming.h

@@ -39,6 +39,7 @@ namespace faiss {
  * General bit vector functions
  **************************************************/
 
+struct RangeSearchResult;
 
 void bitvec_print (const uint8_t * b, size_t d);
 
@@ -65,6 +66,14 @@ void bitvecs2fvecs (
 
 void fvec2bitvec (const float * x, uint8_t * b, size_t d);
 
+/** Shuffle the bits from b(i, j) := a(i, order[j])
+ */
+void bitvec_shuffle (size_t n, size_t da, size_t db,
+                     const int *order,
+                     const uint8_t *a,
+                     uint8_t *b);
+
+
 /***********************************************
  * Generic reader/writer for bit strings
  ***********************************************/
@@ -171,6 +180,17 @@ void hammings_knn_mc (
   int32_t *distances,
   int64_t *labels);
 
+/** same as hammings_knn except we are doing a range search with radius */
+void hamming_range_search (
+    const uint8_t * a,
+    const uint8_t * b,
+    size_t na,
+    size_t nb,
+    int radius,
+    size_t ncodes,
+    RangeSearchResult *result);
+
+
 /* Counting the number of matches or of cross-matches (without returning them)
    For use with function that assume pre-allocated memory */
 void hamming_count_thres (

data/vendor/faiss/utils/utils.cpp

@@ -214,102 +214,6 @@ void matrix_qr (int m, int n, float *a)
 }
 
 
-/***************************************************************************
- * Kmeans subroutine
- ***************************************************************************/
-
-// a bit above machine epsilon for float16
-
-#define EPS (1 / 1024.)
-
-/* For k-means, compute centroids given assignment of vectors to centroids */
-int km_update_centroids (const float * x,
-                         float * centroids,
-                         int64_t * assign,
-                         size_t d, size_t k, size_t n,
-                         size_t k_frozen)
-{
-    k -= k_frozen;
-    centroids += k_frozen * d;
-
-    std::vector<size_t> hassign(k);
-    memset (centroids, 0, sizeof(*centroids) * d * k);
-
-#pragma omp parallel
-    {
-        int nt = omp_get_num_threads();
-        int rank = omp_get_thread_num();
-        // this thread is taking care of centroids c0:c1
-        size_t c0 = (k * rank) / nt;
-        size_t c1 = (k * (rank + 1)) / nt;
-        const float *xi = x;
-        size_t nacc = 0;
-
-        for (size_t i = 0; i < n; i++) {
-            int64_t ci = assign[i];
-            assert (ci >= 0 && ci < k + k_frozen);
-            ci -= k_frozen;
-            if (ci >= c0 && ci < c1)  {
-                float * c = centroids + ci * d;
-                hassign[ci]++;
-                for (size_t j = 0; j < d; j++)
-                    c[j] += xi[j];
-                nacc++;
-            }
-            xi += d;
-        }
-
-    }
-
-#pragma omp parallel for
-    for (size_t ci = 0; ci < k; ci++) {
-        float * c = centroids + ci * d;
-        float ni = (float) hassign[ci];
-        if (ni != 0) {
-            for (size_t j = 0; j < d; j++)
-                c[j] /= ni;
-        }
-    }
-
-    /* Take care of void clusters */
-    size_t nsplit = 0;
-    RandomGenerator rng (1234);
-    for (size_t ci = 0; ci < k; ci++) {
-        if (hassign[ci] == 0) { /* need to redefine a centroid */
-            size_t cj;
-            for (cj = 0; 1; cj = (cj + 1) % k) {
-                /* probability to pick this cluster for split */
-                float p = (hassign[cj] - 1.0) / (float) (n - k);
-                float r = rng.rand_float ();
-                if (r < p) {
-                    break; /* found our cluster to be split */
-                }
-            }
-            memcpy (centroids+ci*d, centroids+cj*d, sizeof(*centroids) * d);
-
-            /* small symmetric pertubation. Much better than  */
-            for (size_t j = 0; j < d; j++) {
-                if (j % 2 == 0) {
-                    centroids[ci * d + j] *= 1 + EPS;
-                    centroids[cj * d + j] *= 1 - EPS;
-                } else {
-                    centroids[ci * d + j] *= 1 - EPS;
-                    centroids[cj * d + j] *= 1 + EPS;
-                }
-            }
-
-            /* assume even split of the cluster */
-            hassign[ci] = hassign[cj] / 2;
-            hassign[cj] -= hassign[ci];
-            nsplit++;
-        }
-    }
-
-    return nsplit;
-}
-
-#undef EPS
-
 
 
 /***************************************************************************

data/vendor/faiss/utils/utils.h

@@ -65,21 +65,6 @@ int fvec_madd_and_argmin (size_t n, const float *a,
 void reflection (const float * u, float * x, size_t n, size_t d, size_t nu);
 
 
-/** For k-means: update stage.
- *
- * @param x          training vectors, size n * d
- * @param centroids  centroid vectors, size k * d
- * @param assign     nearest centroid for each training vector, size n
- * @param k_frozen   do not update the k_frozen first centroids
- * @return           nb of spliting operations to fight empty clusters
- */
-int km_update_centroids (
-        const float * x,
-        float * centroids,
-        int64_t * assign,
-        size_t d, size_t k, size_t n,
-        size_t k_frozen);
-
 /** compute the Q of the QR decomposition for m > n
  * @param a   size n * m: input matrix and output Q
  */

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2020-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -107,7 +107,6 @@ files:
 - ext/faiss/ext.cpp
 - ext/faiss/extconf.rb
 - lib/faiss.rb
-- lib/faiss/ext.bundle
 - lib/faiss/index.rb
 - lib/faiss/index_binary.rb
 - lib/faiss/kmeans.rb
@@ -118,6 +117,8 @@ files:
 - vendor/faiss/AutoTune.h
 - vendor/faiss/Clustering.cpp
 - vendor/faiss/Clustering.h
+- vendor/faiss/DirectMap.cpp
+- vendor/faiss/DirectMap.h
 - vendor/faiss/IVFlib.cpp
 - vendor/faiss/IVFlib.h
 - vendor/faiss/Index.cpp
@@ -132,6 +133,8 @@ files:
 - vendor/faiss/IndexBinaryFromFloat.h
 - vendor/faiss/IndexBinaryHNSW.cpp
 - vendor/faiss/IndexBinaryHNSW.h
+- vendor/faiss/IndexBinaryHash.cpp
+- vendor/faiss/IndexBinaryHash.h
 - vendor/faiss/IndexBinaryIVF.cpp
 - vendor/faiss/IndexBinaryIVF.h
 - vendor/faiss/IndexFlat.cpp
@@ -169,6 +172,7 @@ files:
 - vendor/faiss/MatrixStats.h
 - vendor/faiss/MetaIndexes.cpp
 - vendor/faiss/MetaIndexes.h
+- vendor/faiss/MetricType.h
 - vendor/faiss/OnDiskInvertedLists.cpp
 - vendor/faiss/OnDiskInvertedLists.h
 - vendor/faiss/VectorTransform.cpp
@@ -185,6 +189,8 @@ files:
 - vendor/faiss/c_api/IndexIVF_c.h
 - vendor/faiss/c_api/IndexLSH_c.cpp
 - vendor/faiss/c_api/IndexLSH_c.h
+- vendor/faiss/c_api/IndexPreTransform_c.cpp
+- vendor/faiss/c_api/IndexPreTransform_c.h
 - vendor/faiss/c_api/IndexShards_c.cpp
 - vendor/faiss/c_api/IndexShards_c.h
 - vendor/faiss/c_api/Index_c.cpp
@@ -222,6 +228,7 @@ files:
 - vendor/faiss/demos/demo_imi_pq.cpp
 - vendor/faiss/demos/demo_ivfpq_indexing.cpp
 - vendor/faiss/demos/demo_sift1M.cpp
+- vendor/faiss/demos/demo_weighted_kmeans.cpp
 - vendor/faiss/gpu/GpuAutoTune.cpp
 - vendor/faiss/gpu/GpuAutoTune.h
 - vendor/faiss/gpu/GpuCloner.cpp
@@ -276,6 +283,7 @@ files:
 - vendor/faiss/impl/HNSW.h
 - vendor/faiss/impl/PolysemousTraining.cpp
 - vendor/faiss/impl/PolysemousTraining.h
+- vendor/faiss/impl/ProductQuantizer-inl.h
 - vendor/faiss/impl/ProductQuantizer.cpp
 - vendor/faiss/impl/ProductQuantizer.h
 - vendor/faiss/impl/ScalarQuantizer.cpp