faiss 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47b973803ea179379ab292d5d7b8350c12a383387e90b96f38eae4e90e20dad3
-  data.tar.gz: 5697b1ec26dbbb0794e6c11563b0cf76d564391882e7c0c0148f3893e1bf6b59
+  metadata.gz: 9d260b6585de456d0df620c322f5366a9ae29a1d275c0200c726233f5c5156aa
+  data.tar.gz: 1f463792926335213ae59a56c7271593fa9eaf93d4705fa936da02d74127a571
 SHA512:
-  metadata.gz: dccf5a8ddfd4030e70308c3e71b16bb6742839b8b1a263d3a010a381729af869c1755164e25a3941a3f475376e783ec42b43409fede77010e97949092a424e18
-  data.tar.gz: 57c933bc9d3ffbec37be0edbf641bd004e8dbfa13c04d52ca970e8ff03f719f65f27016892b06cd66755f36afc8d3ff505e6062656f2c48fa20d58cf64c2b6fd
+  metadata.gz: 70960a3c2244b8d8dceaa7a2cc98a04b8f765c783ff2d1879d61e2303f7a7337490b67a9a6a56d3f0a4e19939812418122394633f2a68c198a6e0776a3a2b29a
+  data.tar.gz: 0bc2edfdc86a7d3894923dfd98fe6db7dfa8a15cf640340198f92f7ef967b9548ed7a18e80d886018d8f621da01f73c1b061a85cf892318327144209d1a4ed87

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.1.5 (2021-03-14)
+
+- Added `--with-optflags` option
+- Added support for inner product to `IndexHNSWFlat`
+
+## 0.1.4 (2021-02-04)
+
+- Updated Faiss to 1.7.0
+
+## 0.1.3 (2020-10-22)
+
+- Updated Faiss to 1.6.4
+
+## 0.1.2 (2020-08-17)
+
+- Updated Faiss to 1.6.3
+
+## 0.1.1 (2020-03-09)
+
+- Vendored library
+- Added `save` and `load` methods
+
 ## 0.1.0 (2020-03-08)
 
 - First release

data/LICENSE.txt

@@ -1,22 +1,22 @@
-Copyright (c) 2020 Andrew Kane
-
 MIT License
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+Copyright (c) Facebook, Inc. and its affiliates.
+Copyright (c) 2020-2021 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -2,12 +2,20 @@
 
 [Faiss](https://github.com/facebookresearch/faiss) - efficient similarity search and clustering - for Ruby
 
+Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
+
+[![Build Status](https://github.com/ankane/faiss/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss/actions)
+
 ## Installation
 
-First, install the [Faiss C++ library](https://github.com/facebookresearch/faiss/blob/master/INSTALL.md). For Homebrew, use:
+First, install BLAS, LAPACK, and OpenMP:
 
 ```sh
-brew install faiss
+# Mac
+brew install openblas lapack libomp
+
+# Ubuntu
+sudo apt install libblas-dev liblapack-dev
 ```
 
 Add this line to your application’s Gemfile:
@@ -16,6 +24,8 @@ Add this line to your application’s Gemfile:
 gem 'faiss'
 ```
 
+It can take a few minutes to compile the gem. Faiss is not available for Windows.
+
 ## Getting Started
 
 Prep your data
@@ -41,6 +51,84 @@ Search
 distances, ids = index.search(objects, 3)
 ```
 
+Save an index
+
+```ruby
+index.save("index.bin")
+```
+
+Load an index
+
+```ruby
+index = Faiss::Index.load("index.bin")
+```
+
+> Use `Faiss::IndexBinary` to load binary indexes
+
+## Basic Indexes
+
+Exact search for L2
+
+```rb
+Faiss::IndexFlatL2.new(d)
+```
+
+Exact search for inner product
+
+```rb
+Faiss::IndexFlatIP.new(d)
+```
+
+Hierarchical navigable small world graph exploration
+
+```rb
+Faiss::IndexHNSWFlat.new(d, m)
+```
+
+Inverted file with exact post-verification
+
+```rb
+Faiss::IndexIVFFlat.new(quantizer, d, nlists)
+```
+
+Locality-sensitive hashing
+
+```rb
+Faiss::IndexLSH.new(d, nbits)
+```
+
+Product quantizer (PQ) in flat mode
+
+```rb
+Faiss::IndexPQ.new(d, m, nbits)
+```
+
+IVFADC (coarse quantizer+PQ on residuals)
+
+```rb
+Faiss::IndexIVFPQ.new(quantizer, d, nlists, m, nbits)
+```
+
+IVFADC+R (same as IVFADC with re-ranking based on codes)
+
+```rb
+Faiss::IndexIVFPQR.new(quantizer, d, nlists, m, nbits, m_refine, nbits_refine)
+```
+
+## Binary Indexes
+
+Index binary vectors
+
+```rb
+Faiss::IndexBinaryFlat.new(d)
+```
+
+Speed up search with an inverse vector file
+
+```rb
+Faiss::IndexBinaryIVF.new(quantizer, d, nlists)
+```
+
 ## K-means Clustering
 
 Train
@@ -92,6 +180,18 @@ Decode
 pq.decode(codes)
 ```
 
+Save a quantizer
+
+```ruby
+pq.save("pq.bin")
+```
+
+Load a quantizer
+
+```ruby
+pq = Faiss::ProductQuantizer.load("pq.bin")
+```
+
 ## Data
 
 Data can be an array of arrays
@@ -100,10 +200,10 @@ Data can be an array of arrays
 [[1, 2, 3], [4, 5, 6]]
 ```
 
-Or a Numo NArray
+Or a Numo array
 
 ```ruby
-Numo::SFloat.new(3, 2).seq
+Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
 ```
 
 ## History
@@ -122,7 +222,7 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 To get started with development:
 
 ```sh
-git clone https://github.com/ankane/faiss.git
+git clone --recursive https://github.com/ankane/faiss.git
 cd faiss
 bundle install
 bundle exec rake compile

data/ext/faiss/ext.cpp

@@ -1,248 +1,19 @@
-#include <faiss/IndexFlat.h>
-#include <faiss/IndexHNSW.h>
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/IndexLSH.h>
-#include <faiss/IndexScalarQuantizer.h>
-#include <faiss/IndexPQ.h>
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexIVFPQR.h>
-
-#include <faiss/IndexBinaryFlat.h>
-#include <faiss/IndexBinaryIVF.h>
-#include <faiss/index_factory.h>
-
-#include <faiss/Clustering.h>
-#include <faiss/VectorTransform.h>
-
-#include <rice/Array.hpp>
-#include <rice/Class.hpp>
-#include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+void init_index(Rice::Module& m);
+void init_index_binary(Rice::Module& m);
+void init_kmeans(Rice::Module& m);
+void init_pca_matrix(Rice::Module& m);
+void init_product_quantizer(Rice::Module& m);
+
 extern "C"
 void Init_ext()
 {
-  Rice::Module rb_mFaiss = Rice::define_module("Faiss")
-    .define_singleton_method(
-      "index_binary_factory",
-      *[](int d, const char *description) {
-        return faiss::index_binary_factory(d, description);
-      });
-
-  Rice::define_class_under<faiss::Index>(rb_mFaiss, "Index")
-    .define_method(
-      "d",
-      *[](faiss::Index &self) {
-        return self.d;
-      })
-    .define_method(
-      "trained?",
-      *[](faiss::Index &self) {
-        return self.is_trained;
-      })
-    .define_method(
-      "ntotal",
-      *[](faiss::Index &self) {
-        return self.ntotal;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::Index &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
-        self.train(n, x);
-      })
-    .define_method(
-      "_add",
-      *[](faiss::Index &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
-        self.add(n, x);
-      })
-    .define_method(
-      "_search",
-      *[](faiss::Index &self, int64_t n, Rice::String s, int64_t k) {
-        const float *x = (float*) s.c_str();
-        float *distances = new float[k * n];
-        int64_t *labels = new int64_t[k * n];
-
-        self.search(n, x, k, distances, labels);
-
-        auto dstr = std::string((char*) distances, k * n * sizeof(float));
-        auto lstr = std::string((char*) labels, k * n * sizeof(int64_t));
-
-        Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
-        return ret;
-      });
-
-  Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
-    .define_method(
-      "d",
-      *[](faiss::Index &self) {
-        return self.d;
-      })
-    .define_method(
-      "trained?",
-      *[](faiss::IndexBinary &self) {
-        return self.is_trained;
-      })
-    .define_method(
-      "ntotal",
-      *[](faiss::IndexBinary &self) {
-        return self.ntotal;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::String s) {
-        const uint8_t *x = (uint8_t*) s.c_str();
-        self.train(n, x);
-      })
-    .define_method(
-      "_add",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::String s) {
-        const uint8_t *x = (uint8_t*) s.c_str();
-        self.add(n, x);
-      })
-    .define_method(
-      "_search",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::String s, int64_t k) {
-        const uint8_t *x = (uint8_t*) s.c_str();
-        int32_t *distances = new int32_t[k * n];
-        int64_t *labels = new int64_t[k * n];
-
-        self.search(n, x, k, distances, labels);
-
-        auto dstr = std::string((char*) distances, k * n * sizeof(int32_t));
-        auto lstr = std::string((char*) labels, k * n * sizeof(int64_t));
-
-        Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
-        return ret;
-      });
-
-  Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
-    .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
-
-  Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(rb_mFaiss, "IndexFlatIP")
-    .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
-
-  Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(rb_mFaiss, "IndexHNSWFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int>());
-
-  Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(rb_mFaiss, "IndexIVFFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexLSH, faiss::Index>(rb_mFaiss, "IndexLSH")
-    .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
-
-  Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexScalarQuantizer")
-    .define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
-
-  Rice::define_class_under<faiss::IndexPQ, faiss::Index>(rb_mFaiss, "IndexPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexIVFScalarQuantizer")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
-
-  Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(rb_mFaiss, "IndexIVFPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(rb_mFaiss, "IndexIVFPQR")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
-
-  Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryIVF")
-    .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
-
-  Rice::define_class_under<faiss::Clustering>(rb_mFaiss, "Kmeans")
-    .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
-    .define_method(
-      "d",
-      *[](faiss::Clustering &self) {
-        return self.d;
-      })
-    .define_method(
-      "k",
-      *[](faiss::Clustering &self) {
-        return self.k;
-      })
-    .define_method(
-      "_centroids",
-      *[](faiss::Clustering &self) {
-        float *centroids = new float[self.k * self.d];
-        for (size_t i = 0; i < self.centroids.size(); i++) {
-          centroids[i] = self.centroids[i];
-        }
-        return std::string((char*) centroids, self.k * self.d * sizeof(float));
-      })
-    .define_method(
-      "_train",
-      *[](faiss::Clustering &self, int64_t n, Rice::String s, faiss::Index & index) {
-        const float *x = (float*) s.c_str();
-        self.train(n, x, index);
-      });
-
-  Rice::define_class_under<faiss::PCAMatrix>(rb_mFaiss, "PCAMatrix")
-    .define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
-    .define_method(
-      "d_in",
-      *[](faiss::PCAMatrix &self) {
-        return self.d_in;
-      })
-    .define_method(
-      "d_out",
-      *[](faiss::PCAMatrix &self) {
-        return self.d_out;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
-        self.train(n, x);
-      })
-    .define_method(
-      "_apply",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
-        float* res = self.apply(n, x);
-        return std::string((char*) res, n * self.d_out * sizeof(float));
-      });
+  auto m = Rice::define_module("Faiss");
 
-  Rice::define_class_under<faiss::ProductQuantizer>(rb_mFaiss, "ProductQuantizer")
-    .define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
-    .define_method(
-      "d",
-      *[](faiss::ProductQuantizer &self) {
-        return self.d;
-      })
-    .define_method(
-      "m",
-      *[](faiss::ProductQuantizer &self) {
-        return self.M;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
-        const float *x = (float*) s.c_str();
-        self.train(n, x);
-      })
-    .define_method(
-      "_compute_codes",
-      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
-        const float *x = (float*) s.c_str();
-        uint8_t *codes = new uint8_t[n * self.M];
-        self.compute_codes(x, codes, n);
-        return std::string((char*) codes, n * self.M * sizeof(uint8_t));
-      })
-    .define_method(
-      "_decode",
-      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
-        const uint8_t *codes = (uint8_t*) s.c_str();
-        float *x = new float[n * self.d];
-        self.decode(codes, x, n);
-        return std::string((char*) x, n * self.d * sizeof(float));
-      });
+  init_index(m);
+  init_index_binary(m);
+  init_kmeans(m);
+  init_pca_matrix(m);
+  init_product_quantizer(m);
 }