faiss 0.1.0 → 0.1.1

data/vendor/faiss/utils/utils.h

@@ -0,0 +1,181 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// -*- c++ -*-
+
+/*
+ *  A few utilitary functions for similarity search:
+ * - optimized exhaustive distance and knn search functions
+ * - some functions reimplemented from torch for speed
+ */
+
+#ifndef FAISS_utils_h
+#define FAISS_utils_h
+
+#include <stdint.h>
+
+#include <faiss/utils/Heap.h>
+
+
+namespace faiss {
+
+
+/**************************************************
+ * Get some stats about the system
+**************************************************/
+
+
+/// ms elapsed since some arbitrary epoch
+double getmillisecs ();
+
+/// get current RSS usage in kB
+size_t get_mem_usage_kb ();
+
+
+uint64_t get_cycles ();
+
+/***************************************************************************
+ * Misc  matrix and vector manipulation functions
+ ***************************************************************************/
+
+
+/** compute c := a + bf * b for a, b and c tables
+ *
+ * @param n   size of the tables
+ * @param a   size n
+ * @param b   size n
+ * @param c   restult table, size n
+ */
+void fvec_madd (size_t n, const float *a,
+                float bf, const float *b, float *c);
+
+
+/** same as fvec_madd, also return index of the min of the result table
+ * @return    index of the min of table c
+ */
+int fvec_madd_and_argmin (size_t n, const float *a,
+                           float bf, const float *b, float *c);
+
+
+/* perform a reflection (not an efficient implementation, just for test ) */
+void reflection (const float * u, float * x, size_t n, size_t d, size_t nu);
+
+
+/** For k-means: update stage.
+ *
+ * @param x          training vectors, size n * d
+ * @param centroids  centroid vectors, size k * d
+ * @param assign     nearest centroid for each training vector, size n
+ * @param k_frozen   do not update the k_frozen first centroids
+ * @return           nb of spliting operations to fight empty clusters
+ */
+int km_update_centroids (
+        const float * x,
+        float * centroids,
+        int64_t * assign,
+        size_t d, size_t k, size_t n,
+        size_t k_frozen);
+
+/** compute the Q of the QR decomposition for m > n
+ * @param a   size n * m: input matrix and output Q
+ */
+void matrix_qr (int m, int n, float *a);
+
+/** distances are supposed to be sorted. Sorts indices with same distance*/
+void ranklist_handle_ties (int k, int64_t *idx, const float *dis);
+
+/** count the number of comon elements between v1 and v2
+ * algorithm = sorting + bissection to avoid double-counting duplicates
+ */
+size_t ranklist_intersection_size (size_t k1, const int64_t *v1,
+                                   size_t k2, const int64_t *v2);
+
+/** merge a result table into another one
+ *
+ * @param I0, D0       first result table, size (n, k)
+ * @param I1, D1       second result table, size (n, k)
+ * @param keep_min     if true, keep min values, otherwise keep max
+ * @param translation  add this value to all I1's indexes
+ * @return             nb of values that were taken from the second table
+ */
+size_t merge_result_table_with (size_t n, size_t k,
+                                int64_t *I0, float *D0,
+                                const int64_t *I1, const float *D1,
+                                bool keep_min = true,
+                                int64_t translation = 0);
+
+
+/// a balanced assignment has a IF of 1
+double imbalance_factor (int n, int k, const int64_t *assign);
+
+/// same, takes a histogram as input
+double imbalance_factor (int k, const int *hist);
+
+
+void fvec_argsort (size_t n, const float *vals,
+                    size_t *perm);
+
+void fvec_argsort_parallel (size_t n, const float *vals,
+                    size_t *perm);
+
+
+/// compute histogram on v
+int ivec_hist (size_t n, const int * v, int vmax, int *hist);
+
+/** Compute histogram of bits on a code array
+ *
+ * @param codes   size(n, nbits / 8)
+ * @param hist    size(nbits): nb of 1s in the array of codes
+ */
+void bincode_hist(size_t n, size_t nbits, const uint8_t *codes, int *hist);
+
+
+/// compute a checksum on a table.
+size_t ivec_checksum (size_t n, const int *a);
+
+
+/** random subsamples a set of vectors if there are too many of them
+ *
+ * @param d      dimension of the vectors
+ * @param n      on input: nb of input vectors, output: nb of output vectors
+ * @param nmax   max nb of vectors to keep
+ * @param x      input array, size *n-by-d
+ * @param seed   random seed to use for sampling
+ * @return       x or an array allocated with new [] with *n vectors
+ */
+const float *fvecs_maybe_subsample (
+       size_t d, size_t *n, size_t nmax, const float *x,
+       bool verbose = false, int64_t seed = 1234);
+
+/** Convert binary vector to +1/-1 valued float vector.
+ *
+ * @param d      dimension of the vector (multiple of 8)
+ * @param x_in   input binary vector (uint8_t table of size d / 8)
+ * @param x_out  output float vector (float table of size d)
+ */
+void binary_to_real(size_t d, const uint8_t *x_in, float *x_out);
+
+/** Convert float vector to binary vector. Components > 0 are converted to 1,
+ * others to 0.
+ *
+ * @param d      dimension of the vector (multiple of 8)
+ * @param x_in   input float vector (float table of size d)
+ * @param x_out  output binary vector (uint8_t table of size d / 8)
+ */
+void real_to_binary(size_t d, const float *x_in, uint8_t *x_out);
+
+
+/** A reasonable hashing function */
+uint64_t hash_bytes (const uint8_t *bytes, int64_t n);
+
+/** Whether OpenMP annotations were respected. */
+bool check_openmp();
+
+} // namspace faiss
+
+
+#endif /* FAISS_utils_h */

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: faiss
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-08 00:00:00.000000000 Z
+date: 2020-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -107,12 +107,226 @@ files:
 - ext/faiss/ext.cpp
 - ext/faiss/extconf.rb
 - lib/faiss.rb
+- lib/faiss/ext.bundle
 - lib/faiss/index.rb
 - lib/faiss/index_binary.rb
 - lib/faiss/kmeans.rb
 - lib/faiss/pca_matrix.rb
 - lib/faiss/product_quantizer.rb
 - lib/faiss/version.rb
+- vendor/faiss/AutoTune.cpp
+- vendor/faiss/AutoTune.h
+- vendor/faiss/Clustering.cpp
+- vendor/faiss/Clustering.h
+- vendor/faiss/IVFlib.cpp
+- vendor/faiss/IVFlib.h
+- vendor/faiss/Index.cpp
+- vendor/faiss/Index.h
+- vendor/faiss/Index2Layer.cpp
+- vendor/faiss/Index2Layer.h
+- vendor/faiss/IndexBinary.cpp
+- vendor/faiss/IndexBinary.h
+- vendor/faiss/IndexBinaryFlat.cpp
+- vendor/faiss/IndexBinaryFlat.h
+- vendor/faiss/IndexBinaryFromFloat.cpp
+- vendor/faiss/IndexBinaryFromFloat.h
+- vendor/faiss/IndexBinaryHNSW.cpp
+- vendor/faiss/IndexBinaryHNSW.h
+- vendor/faiss/IndexBinaryIVF.cpp
+- vendor/faiss/IndexBinaryIVF.h
+- vendor/faiss/IndexFlat.cpp
+- vendor/faiss/IndexFlat.h
+- vendor/faiss/IndexHNSW.cpp
+- vendor/faiss/IndexHNSW.h
+- vendor/faiss/IndexIVF.cpp
+- vendor/faiss/IndexIVF.h
+- vendor/faiss/IndexIVFFlat.cpp
+- vendor/faiss/IndexIVFFlat.h
+- vendor/faiss/IndexIVFPQ.cpp
+- vendor/faiss/IndexIVFPQ.h
+- vendor/faiss/IndexIVFPQR.cpp
+- vendor/faiss/IndexIVFPQR.h
+- vendor/faiss/IndexIVFSpectralHash.cpp
+- vendor/faiss/IndexIVFSpectralHash.h
+- vendor/faiss/IndexLSH.cpp
+- vendor/faiss/IndexLSH.h
+- vendor/faiss/IndexLattice.cpp
+- vendor/faiss/IndexLattice.h
+- vendor/faiss/IndexPQ.cpp
+- vendor/faiss/IndexPQ.h
+- vendor/faiss/IndexPreTransform.cpp
+- vendor/faiss/IndexPreTransform.h
+- vendor/faiss/IndexReplicas.cpp
+- vendor/faiss/IndexReplicas.h
+- vendor/faiss/IndexScalarQuantizer.cpp
+- vendor/faiss/IndexScalarQuantizer.h
+- vendor/faiss/IndexShards.cpp
+- vendor/faiss/IndexShards.h
+- vendor/faiss/InvertedLists.cpp
+- vendor/faiss/InvertedLists.h
+- vendor/faiss/LICENSE
+- vendor/faiss/MatrixStats.cpp
+- vendor/faiss/MatrixStats.h
+- vendor/faiss/MetaIndexes.cpp
+- vendor/faiss/MetaIndexes.h
+- vendor/faiss/OnDiskInvertedLists.cpp
+- vendor/faiss/OnDiskInvertedLists.h
+- vendor/faiss/VectorTransform.cpp
+- vendor/faiss/VectorTransform.h
+- vendor/faiss/c_api/AutoTune_c.cpp
+- vendor/faiss/c_api/AutoTune_c.h
+- vendor/faiss/c_api/Clustering_c.cpp
+- vendor/faiss/c_api/Clustering_c.h
+- vendor/faiss/c_api/IndexFlat_c.cpp
+- vendor/faiss/c_api/IndexFlat_c.h
+- vendor/faiss/c_api/IndexIVFFlat_c.cpp
+- vendor/faiss/c_api/IndexIVFFlat_c.h
+- vendor/faiss/c_api/IndexIVF_c.cpp
+- vendor/faiss/c_api/IndexIVF_c.h
+- vendor/faiss/c_api/IndexLSH_c.cpp
+- vendor/faiss/c_api/IndexLSH_c.h
+- vendor/faiss/c_api/IndexShards_c.cpp
+- vendor/faiss/c_api/IndexShards_c.h
+- vendor/faiss/c_api/Index_c.cpp
+- vendor/faiss/c_api/Index_c.h
+- vendor/faiss/c_api/MetaIndexes_c.cpp
+- vendor/faiss/c_api/MetaIndexes_c.h
+- vendor/faiss/c_api/clone_index_c.cpp
+- vendor/faiss/c_api/clone_index_c.h
+- vendor/faiss/c_api/error_c.h
+- vendor/faiss/c_api/error_impl.cpp
+- vendor/faiss/c_api/error_impl.h
+- vendor/faiss/c_api/faiss_c.h
+- vendor/faiss/c_api/gpu/GpuAutoTune_c.cpp
+- vendor/faiss/c_api/gpu/GpuAutoTune_c.h
+- vendor/faiss/c_api/gpu/GpuClonerOptions_c.cpp
+- vendor/faiss/c_api/gpu/GpuClonerOptions_c.h
+- vendor/faiss/c_api/gpu/GpuIndex_c.cpp
+- vendor/faiss/c_api/gpu/GpuIndex_c.h
+- vendor/faiss/c_api/gpu/GpuIndicesOptions_c.h
+- vendor/faiss/c_api/gpu/GpuResources_c.cpp
+- vendor/faiss/c_api/gpu/GpuResources_c.h
+- vendor/faiss/c_api/gpu/StandardGpuResources_c.cpp
+- vendor/faiss/c_api/gpu/StandardGpuResources_c.h
+- vendor/faiss/c_api/gpu/macros_impl.h
+- vendor/faiss/c_api/impl/AuxIndexStructures_c.cpp
+- vendor/faiss/c_api/impl/AuxIndexStructures_c.h
+- vendor/faiss/c_api/index_factory_c.cpp
+- vendor/faiss/c_api/index_factory_c.h
+- vendor/faiss/c_api/index_io_c.cpp
+- vendor/faiss/c_api/index_io_c.h
+- vendor/faiss/c_api/macros_impl.h
+- vendor/faiss/clone_index.cpp
+- vendor/faiss/clone_index.h
+- vendor/faiss/demos/demo_imi_flat.cpp
+- vendor/faiss/demos/demo_imi_pq.cpp
+- vendor/faiss/demos/demo_ivfpq_indexing.cpp
+- vendor/faiss/demos/demo_sift1M.cpp
+- vendor/faiss/gpu/GpuAutoTune.cpp
+- vendor/faiss/gpu/GpuAutoTune.h
+- vendor/faiss/gpu/GpuCloner.cpp
+- vendor/faiss/gpu/GpuCloner.h
+- vendor/faiss/gpu/GpuClonerOptions.cpp
+- vendor/faiss/gpu/GpuClonerOptions.h
+- vendor/faiss/gpu/GpuDistance.h
+- vendor/faiss/gpu/GpuFaissAssert.h
+- vendor/faiss/gpu/GpuIndex.h
+- vendor/faiss/gpu/GpuIndexBinaryFlat.h
+- vendor/faiss/gpu/GpuIndexFlat.h
+- vendor/faiss/gpu/GpuIndexIVF.h
+- vendor/faiss/gpu/GpuIndexIVFFlat.h
+- vendor/faiss/gpu/GpuIndexIVFPQ.h
+- vendor/faiss/gpu/GpuIndexIVFScalarQuantizer.h
+- vendor/faiss/gpu/GpuIndicesOptions.h
+- vendor/faiss/gpu/GpuResources.cpp
+- vendor/faiss/gpu/GpuResources.h
+- vendor/faiss/gpu/StandardGpuResources.cpp
+- vendor/faiss/gpu/StandardGpuResources.h
+- vendor/faiss/gpu/impl/RemapIndices.cpp
+- vendor/faiss/gpu/impl/RemapIndices.h
+- vendor/faiss/gpu/perf/IndexWrapper-inl.h
+- vendor/faiss/gpu/perf/IndexWrapper.h
+- vendor/faiss/gpu/perf/PerfClustering.cpp
+- vendor/faiss/gpu/perf/PerfIVFPQAdd.cpp
+- vendor/faiss/gpu/perf/WriteIndex.cpp
+- vendor/faiss/gpu/test/TestGpuIndexBinaryFlat.cpp
+- vendor/faiss/gpu/test/TestGpuIndexFlat.cpp
+- vendor/faiss/gpu/test/TestGpuIndexIVFFlat.cpp
+- vendor/faiss/gpu/test/TestGpuIndexIVFPQ.cpp
+- vendor/faiss/gpu/test/TestGpuMemoryException.cpp
+- vendor/faiss/gpu/test/TestUtils.cpp
+- vendor/faiss/gpu/test/TestUtils.h
+- vendor/faiss/gpu/test/demo_ivfpq_indexing_gpu.cpp
+- vendor/faiss/gpu/utils/DeviceMemory.cpp
+- vendor/faiss/gpu/utils/DeviceMemory.h
+- vendor/faiss/gpu/utils/DeviceUtils.h
+- vendor/faiss/gpu/utils/MemorySpace.cpp
+- vendor/faiss/gpu/utils/MemorySpace.h
+- vendor/faiss/gpu/utils/StackDeviceMemory.cpp
+- vendor/faiss/gpu/utils/StackDeviceMemory.h
+- vendor/faiss/gpu/utils/StaticUtils.h
+- vendor/faiss/gpu/utils/Timer.cpp
+- vendor/faiss/gpu/utils/Timer.h
+- vendor/faiss/impl/AuxIndexStructures.cpp
+- vendor/faiss/impl/AuxIndexStructures.h
+- vendor/faiss/impl/FaissAssert.h
+- vendor/faiss/impl/FaissException.cpp
+- vendor/faiss/impl/FaissException.h
+- vendor/faiss/impl/HNSW.cpp
+- vendor/faiss/impl/HNSW.h
+- vendor/faiss/impl/PolysemousTraining.cpp
+- vendor/faiss/impl/PolysemousTraining.h
+- vendor/faiss/impl/ProductQuantizer.cpp
+- vendor/faiss/impl/ProductQuantizer.h
+- vendor/faiss/impl/ScalarQuantizer.cpp
+- vendor/faiss/impl/ScalarQuantizer.h
+- vendor/faiss/impl/ThreadedIndex-inl.h
+- vendor/faiss/impl/ThreadedIndex.h
+- vendor/faiss/impl/index_read.cpp
+- vendor/faiss/impl/index_write.cpp
+- vendor/faiss/impl/io.cpp
+- vendor/faiss/impl/io.h
+- vendor/faiss/impl/lattice_Zn.cpp
+- vendor/faiss/impl/lattice_Zn.h
+- vendor/faiss/index_factory.cpp
+- vendor/faiss/index_factory.h
+- vendor/faiss/index_io.h
+- vendor/faiss/misc/test_blas.cpp
+- vendor/faiss/tests/test_binary_flat.cpp
+- vendor/faiss/tests/test_dealloc_invlists.cpp
+- vendor/faiss/tests/test_ivfpq_codec.cpp
+- vendor/faiss/tests/test_ivfpq_indexing.cpp
+- vendor/faiss/tests/test_lowlevel_ivf.cpp
+- vendor/faiss/tests/test_merge.cpp
+- vendor/faiss/tests/test_omp_threads.cpp
+- vendor/faiss/tests/test_ondisk_ivf.cpp
+- vendor/faiss/tests/test_pairs_decoding.cpp
+- vendor/faiss/tests/test_params_override.cpp
+- vendor/faiss/tests/test_pq_encoding.cpp
+- vendor/faiss/tests/test_sliding_ivf.cpp
+- vendor/faiss/tests/test_threaded_index.cpp
+- vendor/faiss/tests/test_transfer_invlists.cpp
+- vendor/faiss/tutorial/cpp/1-Flat.cpp
+- vendor/faiss/tutorial/cpp/2-IVFFlat.cpp
+- vendor/faiss/tutorial/cpp/3-IVFPQ.cpp
+- vendor/faiss/tutorial/cpp/4-GPU.cpp
+- vendor/faiss/tutorial/cpp/5-Multiple-GPUs.cpp
+- vendor/faiss/utils/Heap.cpp
+- vendor/faiss/utils/Heap.h
+- vendor/faiss/utils/WorkerThread.cpp
+- vendor/faiss/utils/WorkerThread.h
+- vendor/faiss/utils/distances.cpp
+- vendor/faiss/utils/distances.h
+- vendor/faiss/utils/distances_simd.cpp
+- vendor/faiss/utils/extra_distances.cpp
+- vendor/faiss/utils/extra_distances.h
+- vendor/faiss/utils/hamming-inl.h
+- vendor/faiss/utils/hamming.cpp
+- vendor/faiss/utils/hamming.h
+- vendor/faiss/utils/random.cpp
+- vendor/faiss/utils/random.h
+- vendor/faiss/utils/utils.cpp
+- vendor/faiss/utils/utils.h
 homepage: https://github.com/ankane/faiss
 licenses:
 - MIT