faiss 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47b973803ea179379ab292d5d7b8350c12a383387e90b96f38eae4e90e20dad3
-  data.tar.gz: 5697b1ec26dbbb0794e6c11563b0cf76d564391882e7c0c0148f3893e1bf6b59
+  metadata.gz: a0369e5dda330b1490e48a88863baa01df9cadfa570078892cec439f82efaad1
+  data.tar.gz: bb7d89fa17f782e8163b114b520b8c2c082cf37661b4b6fc4593460dc5958484
 SHA512:
-  metadata.gz: dccf5a8ddfd4030e70308c3e71b16bb6742839b8b1a263d3a010a381729af869c1755164e25a3941a3f475376e783ec42b43409fede77010e97949092a424e18
-  data.tar.gz: 57c933bc9d3ffbec37be0edbf641bd004e8dbfa13c04d52ca970e8ff03f719f65f27016892b06cd66755f36afc8d3ff505e6062656f2c48fa20d58cf64c2b6fd
+  metadata.gz: 0a9f1515d142d11c688f1a9cdbcf9af0c36fa3fc98b240f236554b1067cf2daad1cefa377d18d236674b8fc1b94d64a3acc070c2528c47b68f4d231f29b7648d
+  data.tar.gz: ae02808dbda4831c7165c987b77c72f9d436bba94e3e28d372f69ceee18fb4971c6cc99a2bf7ce9bd9a9a6e4befcd372515a728bf379294ecc870f2c58f85eb2

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.1 (2020-03-09)
+
+- Vendored library
+- Added `save` and `load` methods
+
 ## 0.1.0 (2020-03-08)
 
 - First release

data/README.md

@@ -2,12 +2,20 @@
 
 [Faiss](https://github.com/facebookresearch/faiss) - efficient similarity search and clustering - for Ruby
 
+Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
+
+[![Build Status](https://travis-ci.org/ankane/faiss.svg?branch=master)](https://travis-ci.org/ankane/faiss)
+
 ## Installation
 
-First, install the [Faiss C++ library](https://github.com/facebookresearch/faiss/blob/master/INSTALL.md). For Homebrew, use:
+First, install BLAS, LAPACK, and OpenMP:
 
 ```sh
-brew install faiss
+# Mac
+brew install openblas lapack libomp
+
+# Ubuntu
+sudo apt install libblas-dev liblapack-dev
 ```
 
 Add this line to your application’s Gemfile:
@@ -16,6 +24,8 @@ Add this line to your application’s Gemfile:
 gem 'faiss'
 ```
 
+Faiss is not available for Windows yet
+
 ## Getting Started
 
 Prep your data
@@ -41,6 +51,84 @@ Search
 distances, ids = index.search(objects, 3)
 ```
 
+Save an index
+
+```ruby
+index.save("index.bin")
+```
+
+Load an index
+
+```ruby
+index = Faiss::Index.load("index.bin")
+```
+
+> Use `Faiss::IndexBinary` to load binary indexes
+
+## Basic Indexes
+
+Exact search for L2
+
+```rb
+Faiss::IndexFlatL2.new(d)
+```
+
+Exact search for inner product
+
+```rb
+Faiss::IndexFlatIP.new(d)
+```
+
+Hierarchical navigable small world graph exploration
+
+```rb
+Faiss::IndexHNSWFlat.new(d, m)
+```
+
+Inverted file with exact post-verification
+
+```rb
+Faiss::IndexIVFFlat.new(quantizer, d, nlists)
+```
+
+Locality-sensitive hashing
+
+```rb
+Faiss::IndexLSH.new(d, nbits)
+```
+
+Product quantizer (PQ) in flat mode
+
+```rb
+Faiss::IndexPQ.new(d, m, nbits)
+```
+
+IVFADC (coarse quantizer+PQ on residuals)
+
+```rb
+Faiss::IndexIVFPQ.new(quantizer, d, nlists, m, nbits)
+```
+
+IVFADC+R (same as IVFADC with re-ranking based on codes)
+
+```rb
+Faiss::IndexIVFPQR.new(quantizer, d, nlists, m, nbits, m_refine, nbits_refine)
+```
+
+## Binary Indexes
+
+Index binary vectors
+
+```rb
+Faiss::IndexBinaryFlat.new(d)
+```
+
+Speed up search with an inverse vector file
+
+```rb
+Faiss::IndexBinaryIVF.new(quantizer, d, nlists)
+```
+
 ## K-means Clustering
 
 Train
@@ -92,6 +180,18 @@ Decode
 pq.decode(codes)
 ```
 
+Save a quantizer
+
+```ruby
+pq.save("pq.bin")
+```
+
+Load a quantizer
+
+```ruby
+pq = Faiss::ProductQuantizer.load("pq.bin")
+```
+
 ## Data
 
 Data can be an array of arrays
@@ -122,7 +222,7 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 To get started with development:
 
 ```sh
-git clone https://github.com/ankane/faiss.git
+git clone --recursive https://github.com/ankane/faiss.git
 cd faiss
 bundle install
 bundle exec rake compile

data/ext/faiss/ext.cpp

@@ -10,6 +10,7 @@
 #include <faiss/IndexBinaryFlat.h>
 #include <faiss/IndexBinaryIVF.h>
 #include <faiss/index_factory.h>
+#include <faiss/index_io.h>
 
 #include <faiss/Clustering.h>
 #include <faiss/VectorTransform.h>
@@ -19,6 +20,42 @@
 #include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
+float* float_array(Rice::Object o)
+{
+  Rice::String s = o.call("to_binary");
+  return (float*) s.c_str();
+}
+
+uint8_t* uint8_array(Rice::Object o)
+{
+  Rice::String s = o.call("to_binary");
+  return (uint8_t*) s.c_str();
+}
+
+// TODO return Numo::SFloat
+Rice::String result(float* ptr, int64_t length)
+{
+  return Rice::String(std::string((char*) ptr, length * sizeof(float)));
+}
+
+// TODO return Numo::UInt8
+Rice::String result(uint8_t* ptr, int64_t length)
+{
+  return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
+}
+
+// TODO return Numo::Int32
+Rice::String result(int32_t* ptr, int64_t length)
+{
+  return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
+}
+
+// TODO return Numo::Int64
+Rice::String result(int64_t* ptr, int64_t length)
+{
+  return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
+}
+
 extern "C"
 void Init_ext()
 {
@@ -47,32 +84,42 @@ void Init_ext()
       })
     .define_method(
       "_train",
-      *[](faiss::Index &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::Index &self, int64_t n, Rice::Object o) {
+        const float *x = float_array(o);
         self.train(n, x);
       })
     .define_method(
       "_add",
-      *[](faiss::Index &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::Index &self, int64_t n, Rice::Object o) {
+        const float *x = float_array(o);
         self.add(n, x);
       })
     .define_method(
       "_search",
-      *[](faiss::Index &self, int64_t n, Rice::String s, int64_t k) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
+        const float *x = float_array(o);
         float *distances = new float[k * n];
         int64_t *labels = new int64_t[k * n];
 
         self.search(n, x, k, distances, labels);
 
-        auto dstr = std::string((char*) distances, k * n * sizeof(float));
-        auto lstr = std::string((char*) labels, k * n * sizeof(int64_t));
+        auto dstr = result(distances, k * n);
+        auto lstr = result(labels, k * n);
 
         Rice::Array ret;
         ret.push(dstr);
         ret.push(lstr);
         return ret;
+      })
+    .define_method(
+      "save",
+      *[](faiss::Index &self, const char *fname) {
+        faiss::write_index(&self, fname);
+      })
+    .define_singleton_method(
+      "load",
+      *[](const char *fname) {
+        return faiss::read_index(fname);
       });
 
   Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
@@ -93,32 +140,42 @@ void Init_ext()
       })
     .define_method(
       "_train",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::String s) {
-        const uint8_t *x = (uint8_t*) s.c_str();
+      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
+        const uint8_t *x = uint8_array(o);
         self.train(n, x);
       })
     .define_method(
       "_add",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::String s) {
-        const uint8_t *x = (uint8_t*) s.c_str();
+      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
+        const uint8_t *x = uint8_array(o);
         self.add(n, x);
       })
     .define_method(
       "_search",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::String s, int64_t k) {
-        const uint8_t *x = (uint8_t*) s.c_str();
+      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
+        const uint8_t *x = uint8_array(o);
         int32_t *distances = new int32_t[k * n];
         int64_t *labels = new int64_t[k * n];
 
         self.search(n, x, k, distances, labels);
 
-        auto dstr = std::string((char*) distances, k * n * sizeof(int32_t));
-        auto lstr = std::string((char*) labels, k * n * sizeof(int64_t));
+        auto dstr = result(distances, k * n);
+        auto lstr = result(labels, k * n);
 
         Rice::Array ret;
         ret.push(dstr);
         ret.push(lstr);
         return ret;
+      })
+    .define_method(
+      "save",
+      *[](faiss::IndexBinary &self, const char *fname) {
+        faiss::write_index_binary(&self, fname);
+      })
+    .define_singleton_method(
+      "load",
+      *[](const char *fname) {
+        return faiss::read_index_binary(fname);
       });
 
   Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
@@ -176,12 +233,12 @@ void Init_ext()
         for (size_t i = 0; i < self.centroids.size(); i++) {
           centroids[i] = self.centroids[i];
         }
-        return std::string((char*) centroids, self.k * self.d * sizeof(float));
+        return result(centroids, self.k * self.d);
       })
     .define_method(
       "_train",
-      *[](faiss::Clustering &self, int64_t n, Rice::String s, faiss::Index & index) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
+        const float *x = float_array(o);
         self.train(n, x, index);
       });
 
@@ -199,16 +256,16 @@ void Init_ext()
       })
     .define_method(
       "_train",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
+        const float *x = float_array(o);
         self.train(n, x);
       })
     .define_method(
       "_apply",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::String s) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
+        const float *x = float_array(o);
         float* res = self.apply(n, x);
-        return std::string((char*) res, n * self.d_out * sizeof(float));
+        return result(res, n * self.d_out);
       });
 
   Rice::define_class_under<faiss::ProductQuantizer>(rb_mFaiss, "ProductQuantizer")
@@ -225,24 +282,34 @@ void Init_ext()
       })
     .define_method(
       "_train",
-      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
+        const float *x = float_array(o);
         self.train(n, x);
       })
     .define_method(
       "_compute_codes",
-      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
-        const float *x = (float*) s.c_str();
+      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
+        const float *x = float_array(o);
         uint8_t *codes = new uint8_t[n * self.M];
         self.compute_codes(x, codes, n);
-        return std::string((char*) codes, n * self.M * sizeof(uint8_t));
+        return result(codes, n * self.M);
       })
     .define_method(
       "_decode",
-      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
-        const uint8_t *codes = (uint8_t*) s.c_str();
+      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
+        const uint8_t *codes = uint8_array(o);
         float *x = new float[n * self.d];
         self.decode(codes, x, n);
-        return std::string((char*) x, n * self.d * sizeof(float));
+        return result(x, n * self.d);
+      })
+    .define_method(
+      "save",
+      *[](faiss::ProductQuantizer &self, const char *fname) {
+        faiss::write_ProductQuantizer(&self, fname);
+      })
+    .define_singleton_method(
+      "load",
+      *[](const char *fname) {
+        return faiss::read_ProductQuantizer(fname);
       });
 }

data/ext/faiss/extconf.rb

@@ -1,7 +1,17 @@
 require "mkmf-rice"
 
-abort "Missing faiss" unless have_library("faiss")
+abort "BLAS not found" unless have_library("blas")
+abort "LAPACK not found" unless have_library("lapack")
+abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
 
-$CXXFLAGS << " -std=c++11"
+$CXXFLAGS << " -std=c++11 -march=native -DFINTEGER=int"
+
+ext = File.expand_path(".", __dir__)
+vendor = File.expand_path("../../vendor", __dir__)
+
+$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/impl,#{vendor}/faiss/utils}/*.{cpp}"]
+$objs = $srcs.map { |v| v.sub(/cpp\z/, "o") }
+$INCFLAGS << " -I#{vendor}"
+$VPATH << vendor
 
 create_makefile("faiss/ext")

data/lib/faiss/index.rb

@@ -2,18 +2,18 @@ module Faiss
   class Index
     def train(objects)
       objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
-      _train(objects.shape[0], objects.to_binary)
+      _train(objects.shape[0], objects)
     end
 
     def add(objects)
       objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
-      _add(objects.shape[0], objects.to_binary)
+      _add(objects.shape[0], objects)
     end
 
     def search(objects, k)
       objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
       n = objects.shape[0]
-      distances, labels = _search(n, objects.to_binary, k)
+      distances, labels = _search(n, objects, k)
       [Numo::SFloat.from_binary(distances).reshape(n, k), Numo::Int64.from_binary(labels).reshape(n, k)]
     end
   end

data/lib/faiss/index_binary.rb

@@ -2,18 +2,18 @@ module Faiss
   class IndexBinary
     def train(objects)
       objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
-      _train(objects.shape[0], objects.to_binary)
+      _train(objects.shape[0], objects)
     end
 
     def add(objects)
       objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
-      _add(objects.shape[0], objects.to_binary)
+      _add(objects.shape[0], objects)
     end
 
     def search(objects, k)
       objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
       n = objects.shape[0]
-      distances, labels = _search(n, objects.to_binary, k)
+      distances, labels = _search(n, objects, k)
       [Numo::UInt32.from_binary(distances).reshape(n, k), Numo::Int64.from_binary(labels).reshape(n, k)]
     end
   end