faiss 0.1.7 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6fb14299c0b91d6ddaf74ca2fa2d6895e8621249a5351683b109f35866ab541
-  data.tar.gz: 2ec9d11faf8f3f1af7790d1f39e31cb92d8a79d2539fa7c935dbda63f3052d8c
+  metadata.gz: 106966f6d5e7f6a3f5237a2ebb59912304bbe319ba2891c45166c753fc0e9df1
+  data.tar.gz: 3d72777c777d75beb15c09cb223c4e63f188b7e89fab06d6fa9268bc2be4ff59
 SHA512:
-  metadata.gz: 5afa3a7ca6947fad6f5a4f7693b524411875d212079621bc72bda73df36fdf090fcb566b83248302a83ecb38ee04189c57c812eaad4ffc99f94840eaf3abfd28
-  data.tar.gz: c29cf727d683f82e530ba1788630ae546ddb4470f3145f0a4dd1b6afe31555bb9363dfa52cc2a5e005a077f6e2aae24b3dc1aa44d1ff9c50fc9269987df0ec87
+  metadata.gz: b1d822fd4e0850dd667aba09a02f81c43e2afde90a6d88fff1e631539a8d47f42b4dfe0705980a6ff8be88b021af805998d1e70b09cd1010302702a1c1363cac
+  data.tar.gz: 24e4febd81142150541199523f0b1e19a0be7c658e504472778173e38006234ea47f7e362c1396c3135b4449b7ab5229dc027c4bcea59c880902e1fa23e3c956

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## 0.2.3 (2021-12-17)
+
+- Fixed installation error with ARM Mac
+
+## 0.2.2 (2021-06-08)
+
+- Fixed installation error
+
+## 0.2.1 (2021-06-07)
+
+- Updated Faiss to 1.7.1
+
+## 0.2.0 (2021-05-23)
+
+- Improved performance
+- Updated to Rice 4
+- Dropped support for Ruby < 2.6
+
 ## 0.1.7 (2021-03-26)
 
 - Added `IndexScalarQuantizer` and `IndexIVFScalarQuantizer`

data/README.md

@@ -1,10 +1,10 @@
-# Faiss
+# Faiss Ruby
 
 [Faiss](https://github.com/facebookresearch/faiss) - efficient similarity search and clustering - for Ruby
 
 Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
 
-[![Build Status](https://github.com/ankane/faiss/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss/actions)
+[![Build Status](https://github.com/ankane/faiss-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss-ruby/actions)
 
 ## Installation
 
@@ -220,22 +220,22 @@ Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
 
 ## History
 
-View the [changelog](https://github.com/ankane/faiss/blob/master/CHANGELOG.md)
+View the [changelog](https://github.com/ankane/faiss-ruby/blob/master/CHANGELOG.md)
 
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:
 
-- [Report bugs](https://github.com/ankane/faiss/issues)
-- Fix bugs and [submit pull requests](https://github.com/ankane/faiss/pulls)
+- [Report bugs](https://github.com/ankane/faiss-ruby/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/faiss-ruby/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
 To get started with development:
 
 ```sh
-git clone --recursive https://github.com/ankane/faiss.git
-cd faiss
+git clone --recursive https://github.com/ankane/faiss-ruby.git
+cd faiss-ruby
 bundle install
 bundle exec rake compile
 bundle exec rake test

data/ext/faiss/ext.cpp

@@ -1,4 +1,4 @@
-#include <rice/Module.hpp>
+#include "utils.h"
 
 void init_index(Rice::Module& m);
 void init_index_binary(Rice::Module& m);

data/ext/faiss/extconf.rb

@@ -1,17 +1,23 @@
 require "mkmf-rice"
+require "numo/narray"
 
 abort "BLAS not found" unless have_library("blas")
 abort "LAPACK not found" unless have_library("lapack")
 abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
 
-$CXXFLAGS << " -std=c++11 -DFINTEGER=int " << with_config("optflags", "-march=native")
+numo = File.join(Gem.loaded_specs["numo-narray"].require_path, "numo")
+abort "Numo not found" unless find_header("numo/narray.h", numo)
+
+# -march=native not supported with ARM Mac
+default_optflags = RbConfig::CONFIG["host_os"] =~ /darwin/i && RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i ? "" : "-march=native"
+$CXXFLAGS << " -std=c++17 $(optflags) -DFINTEGER=int " << with_config("optflags", default_optflags)
 
 ext = File.expand_path(".", __dir__)
 vendor = File.expand_path("../../vendor/faiss", __dir__)
 
 $srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/{impl,invlists,utils}}/*.{cpp}"]
 $objs = $srcs.map { |v| v.sub(/cpp\z/, "o") }
-$INCFLAGS << " -I#{vendor}"
+abort "Faiss not found" unless find_header("faiss/Index.h", vendor)
 $VPATH << vendor
 
 create_makefile("faiss/ext")

data/ext/faiss/index.cpp

@@ -10,109 +10,142 @@
 #include <faiss/index_io.h>
 #include <faiss/AutoTune.h>
 
-#include <rice/Array.hpp>
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
-
 #include "utils.h"
 
-template<>
-faiss::MetricType from_ruby<faiss::MetricType>(Rice::Object x)
-{
-  auto s = x.to_s().str();
-  if (s == "inner_product") {
-    return faiss::METRIC_INNER_PRODUCT;
-  } else if (s == "l2") {
-    return faiss::METRIC_L2;
-  } else {
-    // TODO throw argument error
-    throw std::runtime_error("Invalid metric: " + s);
-  }
-}
-
-template<>
-faiss::ScalarQuantizer::QuantizerType from_ruby<faiss::ScalarQuantizer::QuantizerType>(Rice::Object x)
-{
-  auto s = x.to_s().str();
-  if (s == "qt_8bit") {
-    return faiss::ScalarQuantizer::QT_8bit;
-  } else if (s == "qt_4bit") {
-    return faiss::ScalarQuantizer::QT_4bit;
-  } else if (s == "qt_8bit_uniform") {
-    return faiss::ScalarQuantizer::QT_8bit_uniform;
-  } else if (s == "qt_4bit_uniform") {
-    return faiss::ScalarQuantizer::QT_4bit_uniform;
-  } else if (s == "qt_fp16") {
-    return faiss::ScalarQuantizer::QT_fp16;
-  } else if (s == "qt_8bit_direct") {
-    return faiss::ScalarQuantizer::QT_8bit_direct;
-  } else if (s == "qt_6bit") {
-    return faiss::ScalarQuantizer::QT_6bit;
-  } else {
-    // TODO throw argument error
-    throw std::runtime_error("Invalid quantizer type: " + s);
-  }
+namespace Rice::detail {
+  template<>
+  struct Type<faiss::MetricType>
+  {
+    static bool verify()
+    {
+      return true;
+    }
+  };
+
+  template<>
+  class From_Ruby<faiss::MetricType>
+  {
+  public:
+    From_Ruby() = default;
+
+    From_Ruby(Arg* arg) : arg_(arg)
+    {
+    }
+
+    faiss::MetricType convert(VALUE x)
+    {
+      if (x == Qnil && this->arg_ && this->arg_->hasDefaultValue()) {
+        return this->arg_->defaultValue<faiss::MetricType>();
+      }
+
+      auto s = Object(x).to_s().str();
+      if (s == "inner_product") {
+        return faiss::METRIC_INNER_PRODUCT;
+      } else if (s == "l2") {
+        return faiss::METRIC_L2;
+      } else {
+        throw Rice::Exception(rb_eArgError, "Invalid metric: %s", s.c_str());
+      }
+    }
+
+  private:
+    Arg* arg_;
+  };
+
+  template<>
+  struct Type<faiss::ScalarQuantizer::QuantizerType>
+  {
+    static bool verify()
+    {
+      return true;
+    }
+  };
+
+  template<>
+  class From_Ruby<faiss::ScalarQuantizer::QuantizerType>
+  {
+  public:
+    faiss::ScalarQuantizer::QuantizerType convert(VALUE x)
+    {
+      auto s = Object(x).to_s().str();
+      if (s == "qt_8bit") {
+        return faiss::ScalarQuantizer::QT_8bit;
+      } else if (s == "qt_4bit") {
+        return faiss::ScalarQuantizer::QT_4bit;
+      } else if (s == "qt_8bit_uniform") {
+        return faiss::ScalarQuantizer::QT_8bit_uniform;
+      } else if (s == "qt_4bit_uniform") {
+        return faiss::ScalarQuantizer::QT_4bit_uniform;
+      } else if (s == "qt_fp16") {
+        return faiss::ScalarQuantizer::QT_fp16;
+      } else if (s == "qt_8bit_direct") {
+        return faiss::ScalarQuantizer::QT_8bit_direct;
+      } else if (s == "qt_6bit") {
+        return faiss::ScalarQuantizer::QT_6bit;
+      } else {
+        throw Rice::Exception(rb_eArgError, "Invalid quantizer type: %s", s.c_str());
+      }
+    }
+  };
 }
 
 void init_index(Rice::Module& m) {
   Rice::define_class_under<faiss::Index>(m, "Index")
     .define_method(
       "d",
-      *[](faiss::Index &self) {
+      [](faiss::Index &self) {
         return self.d;
       })
     .define_method(
       "trained?",
-      *[](faiss::Index &self) {
+      [](faiss::Index &self) {
         return self.is_trained;
       })
     .define_method(
       "ntotal",
-      *[](faiss::Index &self) {
+      [](faiss::Index &self) {
         return self.ntotal;
       })
     .define_method(
-      "_train",
-      *[](faiss::Index &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
+      "train",
+      [](faiss::Index &self, numo::SFloat objects) {
+        auto n = check_shape(objects, self.d);
+        self.train(n, objects.read_ptr());
       })
     .define_method(
-      "_add",
-      *[](faiss::Index &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.add(n, x);
+      "add",
+      [](faiss::Index &self, numo::SFloat objects) {
+        auto n = check_shape(objects, self.d);
+        self.add(n, objects.read_ptr());
       })
     .define_method(
-      "_search",
-      *[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
-        const float *x = float_array(o);
-        float *distances = new float[k * n];
-        int64_t *labels = new int64_t[k * n];
+      "search",
+      [](faiss::Index &self, numo::SFloat objects, size_t k) {
+        auto n = check_shape(objects, self.d);
 
-        self.search(n, x, k, distances, labels);
+        auto distances = numo::SFloat({n, k});
+        auto labels = numo::Int64({n, k});
 
-        auto dstr = result(distances, k * n);
-        auto lstr = result(labels, k * n);
+        self.search(n, objects.read_ptr(), k, distances.write_ptr(), labels.write_ptr());
 
         Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
+        ret.push(distances);
+        ret.push(labels);
         return ret;
       })
     .define_method(
       "nprobe=",
-      *[](faiss::Index &self, double val) {
+      [](faiss::Index &self, double val) {
         faiss::ParameterSpace().set_index_parameter(&self, "nprobe", val);
       })
     .define_method(
       "save",
-      *[](faiss::Index &self, const char *fname) {
+      [](faiss::Index &self, const char *fname) {
         faiss::write_index(&self, fname);
       })
-    .define_singleton_method(
+    .define_singleton_function(
       "load",
-      *[](const char *fname) {
+      [](const char *fname) {
         return faiss::read_index(fname);
       });
 
@@ -123,10 +156,10 @@ void init_index(Rice::Module& m) {
     .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
 
   Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(m, "IndexHNSWFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int, faiss::MetricType>(), (Rice::Arg("d"), Rice::Arg("M"), Rice::Arg("metric") = faiss::METRIC_L2));
+    .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int, faiss::MetricType>(), Rice::Arg("d"), Rice::Arg("M"), Rice::Arg("metric") = faiss::METRIC_L2);
 
   Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(m, "IndexIVFFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t, faiss::MetricType>(), (Rice::Arg("quantizer"), Rice::Arg("d"), Rice::Arg("nlist"), Rice::Arg("metric") = faiss::METRIC_L2));
+    .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t, faiss::MetricType>(), Rice::Arg("quantizer"), Rice::Arg("d"), Rice::Arg("nlist"), Rice::Arg("metric") = faiss::METRIC_L2);
 
   Rice::define_class_under<faiss::IndexLSH, faiss::Index>(m, "IndexLSH")
     .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
@@ -141,7 +174,7 @@ void init_index(Rice::Module& m) {
     .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
 
   Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(m, "IndexIVFPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t, faiss::MetricType>(), (Rice::Arg("quantizer"), Rice::Arg("d"), Rice::Arg("nlist"), Rice::Arg("M"), Rice::Arg("nbits_per_idx"), Rice::Arg("metric") = faiss::METRIC_L2));
+    .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t, faiss::MetricType>(), Rice::Arg("quantizer"), Rice::Arg("d"), Rice::Arg("nlist"), Rice::Arg("M"), Rice::Arg("nbits_per_idx"), Rice::Arg("metric") = faiss::METRIC_L2);
 
   Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(m, "IndexIVFPQR")
     .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
@@ -150,7 +183,7 @@ void init_index(Rice::Module& m) {
     .define_constructor(Rice::Constructor<faiss::ParameterSpace>())
     .define_method(
       "set_index_parameter",
-      *[](faiss::ParameterSpace& self, faiss::Index* index, const std::string& name, double val) {
+      [](faiss::ParameterSpace& self, faiss::Index* index, const std::string& name, double val) {
         self.set_index_parameter(index, name, val);
       });
 }

data/ext/faiss/index_binary.cpp

@@ -4,66 +4,60 @@
 #include <faiss/index_factory.h>
 #include <faiss/index_io.h>
 
-#include <rice/Array.hpp>
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
-
 #include "utils.h"
 
 void init_index_binary(Rice::Module& m) {
   Rice::define_class_under<faiss::IndexBinary>(m, "IndexBinary")
     .define_method(
       "d",
-      *[](faiss::IndexBinary &self) {
+      [](faiss::IndexBinary &self) {
         return self.d;
       })
     .define_method(
       "trained?",
-      *[](faiss::IndexBinary &self) {
+      [](faiss::IndexBinary &self) {
         return self.is_trained;
       })
     .define_method(
       "ntotal",
-      *[](faiss::IndexBinary &self) {
+      [](faiss::IndexBinary &self) {
         return self.ntotal;
       })
     .define_method(
-      "_train",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
-        const uint8_t *x = uint8_array(o);
-        self.train(n, x);
+      "train",
+      [](faiss::IndexBinary &self, numo::UInt8 objects) {
+        auto n = check_shape(objects, self.d / 8);
+        self.train(n, objects.read_ptr());
       })
     .define_method(
-      "_add",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
-        const uint8_t *x = uint8_array(o);
-        self.add(n, x);
+      "add",
+      [](faiss::IndexBinary &self, numo::UInt8 objects) {
+        auto n = check_shape(objects, self.d / 8);
+        self.add(n, objects.read_ptr());
       })
     .define_method(
-      "_search",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
-        const uint8_t *x = uint8_array(o);
-        int32_t *distances = new int32_t[k * n];
-        int64_t *labels = new int64_t[k * n];
+      "search",
+      [](faiss::IndexBinary &self, numo::UInt8 objects, size_t k) {
+        auto n = check_shape(objects, self.d / 8);
 
-        self.search(n, x, k, distances, labels);
+        auto distances = numo::Int32({n, k});
+        auto labels = numo::Int64({n, k});
 
-        auto dstr = result(distances, k * n);
-        auto lstr = result(labels, k * n);
+        self.search(n, objects.read_ptr(), k, distances.write_ptr(), labels.write_ptr());
 
         Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
+        ret.push(distances);
+        ret.push(labels);
         return ret;
       })
     .define_method(
       "save",
-      *[](faiss::IndexBinary &self, const char *fname) {
+      [](faiss::IndexBinary &self, const char *fname) {
         faiss::write_index_binary(&self, fname);
       })
-    .define_singleton_method(
+    .define_singleton_function(
       "load",
-      *[](const char *fname) {
+      [](const char *fname) {
         return faiss::read_index_binary(fname);
       });
 
@@ -73,9 +67,9 @@ void init_index_binary(Rice::Module& m) {
   Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(m, "IndexBinaryIVF")
     .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
 
-  m.define_singleton_method(
+  m.define_singleton_function(
     "index_binary_factory",
-    *[](int d, const char *description) {
+    [](int d, const char *description) {
       return faiss::index_binary_factory(d, description);
     });
 }

data/ext/faiss/kmeans.cpp

@@ -1,36 +1,40 @@
 #include <faiss/Clustering.h>
-
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
+#include <faiss/IndexFlat.h>
 
 #include "utils.h"
 
 void init_kmeans(Rice::Module& m) {
   Rice::define_class_under<faiss::Clustering>(m, "Kmeans")
     .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
+    .define_method(
+      "index",
+      [](Rice::Object self) {
+        return self.attr_get("@index");
+      })
     .define_method(
       "d",
-      *[](faiss::Clustering &self) {
+      [](faiss::Clustering &self) {
         return self.d;
       })
     .define_method(
       "k",
-      *[](faiss::Clustering &self) {
+      [](faiss::Clustering &self) {
         return self.k;
       })
     .define_method(
-      "_centroids",
-      *[](faiss::Clustering &self) {
-        float *centroids = new float[self.k * self.d];
-        for (size_t i = 0; i < self.centroids.size(); i++) {
-          centroids[i] = self.centroids[i];
-        }
-        return result(centroids, self.k * self.d);
+      "centroids",
+      [](faiss::Clustering &self) {
+        auto centroids = numo::SFloat({self.k, self.d});
+        memcpy(centroids.write_ptr(), self.centroids.data(), self.centroids.size() * sizeof(float));
+        return centroids;
       })
     .define_method(
-      "_train",
-      *[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
-        const float *x = float_array(o);
-        self.train(n, x, index);
+      "train",
+      [](Rice::Object self, numo::SFloat objects) {
+        auto self_ptr = Rice::detail::From_Ruby<faiss::Clustering*>().convert(self.value());
+        auto n = check_shape(objects, self_ptr->d);
+        auto index = faiss::IndexFlatL2(self_ptr->d);
+        self.iv_set("@index", Rice::Object(Rice::detail::To_Ruby<faiss::IndexFlatL2>().convert(index)));
+        self_ptr->train(n, objects.read_ptr(), index);
       });
 }