faiss 0.1.1 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0369e5dda330b1490e48a88863baa01df9cadfa570078892cec439f82efaad1
-  data.tar.gz: bb7d89fa17f782e8163b114b520b8c2c082cf37661b4b6fc4593460dc5958484
+  metadata.gz: '02752578b25cfcc61a7bb0e06e1d0dbc2d78175796cf99a44184a4e7298de073'
+  data.tar.gz: ec678cf766efa90be0e013b342be6f95e42c6e67c4aa6de1b9e3b6f4c59d8699
 SHA512:
-  metadata.gz: 0a9f1515d142d11c688f1a9cdbcf9af0c36fa3fc98b240f236554b1067cf2daad1cefa377d18d236674b8fc1b94d64a3acc070c2528c47b68f4d231f29b7648d
-  data.tar.gz: ae02808dbda4831c7165c987b77c72f9d436bba94e3e28d372f69ceee18fb4971c6cc99a2bf7ce9bd9a9a6e4befcd372515a728bf379294ecc870f2c58f85eb2
+  metadata.gz: 9342c736f6ce775b8561f0fc066b565e38f8d38470f98ea51b9635f371bb8efdbb9e049874588847fb11360da006da3ec0ca28d44fc6f844d970d91f648f1b03
+  data.tar.gz: e70f220fdc76a37fcf5a06beba2b07f0315bbbb3dab7b396a8f932c04746e11454bd57ed527d18d55436352bd823d1e33e341046697fd36b124f7e64fbb72f38

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+## 0.1.6 (2021-03-21)
+
+- Added support for inner product to `IndexIVFFlat` and `IndexIVFPQ`
+- Added `ParameterSpace`
+- Added `nprobe=` method
+
+## 0.1.5 (2021-03-14)
+
+- Added `--with-optflags` option
+- Added support for inner product to `IndexHNSWFlat`
+
+## 0.1.4 (2021-02-04)
+
+- Updated Faiss to 1.7.0
+
+## 0.1.3 (2020-10-22)
+
+- Updated Faiss to 1.6.4
+
+## 0.1.2 (2020-08-17)
+
+- Updated Faiss to 1.6.3
+
 ## 0.1.1 (2020-03-09)
 
 - Vendored library

data/LICENSE.txt

@@ -1,22 +1,22 @@
-Copyright (c) 2020 Andrew Kane
-
 MIT License
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+Copyright (c) Facebook, Inc. and its affiliates.
+Copyright (c) 2020-2021 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -4,7 +4,7 @@
 
 Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
 
-[![Build Status](https://travis-ci.org/ankane/faiss.svg?branch=master)](https://travis-ci.org/ankane/faiss)
+[![Build Status](https://github.com/ankane/faiss/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss/actions)
 
 ## Installation
 
@@ -24,7 +24,7 @@ Add this line to your application’s Gemfile:
 gem 'faiss'
 ```
 
-Faiss is not available for Windows yet
+It can take a few minutes to compile the gem. Faiss is not available for Windows.
 
 ## Getting Started
 
@@ -200,10 +200,10 @@ Data can be an array of arrays
 [[1, 2, 3], [4, 5, 6]]
 ```
 
-Or a Numo NArray
+Or a Numo array
 
 ```ruby
-Numo::SFloat.new(3, 2).seq
+Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
 ```
 
 ## History

data/ext/faiss/ext.cpp

@@ -1,315 +1,19 @@
-#include <faiss/IndexFlat.h>
-#include <faiss/IndexHNSW.h>
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/IndexLSH.h>
-#include <faiss/IndexScalarQuantizer.h>
-#include <faiss/IndexPQ.h>
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexIVFPQR.h>
-
-#include <faiss/IndexBinaryFlat.h>
-#include <faiss/IndexBinaryIVF.h>
-#include <faiss/index_factory.h>
-#include <faiss/index_io.h>
-
-#include <faiss/Clustering.h>
-#include <faiss/VectorTransform.h>
-
-#include <rice/Array.hpp>
-#include <rice/Class.hpp>
-#include <rice/Constructor.hpp>
 #include <rice/Module.hpp>
 
-float* float_array(Rice::Object o)
-{
-  Rice::String s = o.call("to_binary");
-  return (float*) s.c_str();
-}
-
-uint8_t* uint8_array(Rice::Object o)
-{
-  Rice::String s = o.call("to_binary");
-  return (uint8_t*) s.c_str();
-}
-
-// TODO return Numo::SFloat
-Rice::String result(float* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(float)));
-}
-
-// TODO return Numo::UInt8
-Rice::String result(uint8_t* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
-}
-
-// TODO return Numo::Int32
-Rice::String result(int32_t* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
-}
-
-// TODO return Numo::Int64
-Rice::String result(int64_t* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
-}
+void init_index(Rice::Module& m);
+void init_index_binary(Rice::Module& m);
+void init_kmeans(Rice::Module& m);
+void init_pca_matrix(Rice::Module& m);
+void init_product_quantizer(Rice::Module& m);
 
 extern "C"
 void Init_ext()
 {
-  Rice::Module rb_mFaiss = Rice::define_module("Faiss")
-    .define_singleton_method(
-      "index_binary_factory",
-      *[](int d, const char *description) {
-        return faiss::index_binary_factory(d, description);
-      });
-
-  Rice::define_class_under<faiss::Index>(rb_mFaiss, "Index")
-    .define_method(
-      "d",
-      *[](faiss::Index &self) {
-        return self.d;
-      })
-    .define_method(
-      "trained?",
-      *[](faiss::Index &self) {
-        return self.is_trained;
-      })
-    .define_method(
-      "ntotal",
-      *[](faiss::Index &self) {
-        return self.ntotal;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::Index &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_add",
-      *[](faiss::Index &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.add(n, x);
-      })
-    .define_method(
-      "_search",
-      *[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
-        const float *x = float_array(o);
-        float *distances = new float[k * n];
-        int64_t *labels = new int64_t[k * n];
-
-        self.search(n, x, k, distances, labels);
-
-        auto dstr = result(distances, k * n);
-        auto lstr = result(labels, k * n);
-
-        Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
-        return ret;
-      })
-    .define_method(
-      "save",
-      *[](faiss::Index &self, const char *fname) {
-        faiss::write_index(&self, fname);
-      })
-    .define_singleton_method(
-      "load",
-      *[](const char *fname) {
-        return faiss::read_index(fname);
-      });
-
-  Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
-    .define_method(
-      "d",
-      *[](faiss::Index &self) {
-        return self.d;
-      })
-    .define_method(
-      "trained?",
-      *[](faiss::IndexBinary &self) {
-        return self.is_trained;
-      })
-    .define_method(
-      "ntotal",
-      *[](faiss::IndexBinary &self) {
-        return self.ntotal;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
-        const uint8_t *x = uint8_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_add",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
-        const uint8_t *x = uint8_array(o);
-        self.add(n, x);
-      })
-    .define_method(
-      "_search",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
-        const uint8_t *x = uint8_array(o);
-        int32_t *distances = new int32_t[k * n];
-        int64_t *labels = new int64_t[k * n];
-
-        self.search(n, x, k, distances, labels);
-
-        auto dstr = result(distances, k * n);
-        auto lstr = result(labels, k * n);
-
-        Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
-        return ret;
-      })
-    .define_method(
-      "save",
-      *[](faiss::IndexBinary &self, const char *fname) {
-        faiss::write_index_binary(&self, fname);
-      })
-    .define_singleton_method(
-      "load",
-      *[](const char *fname) {
-        return faiss::read_index_binary(fname);
-      });
-
-  Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
-    .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
-
-  Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(rb_mFaiss, "IndexFlatIP")
-    .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
-
-  Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(rb_mFaiss, "IndexHNSWFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int>());
-
-  Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(rb_mFaiss, "IndexIVFFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexLSH, faiss::Index>(rb_mFaiss, "IndexLSH")
-    .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
-
-  Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexScalarQuantizer")
-    .define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
-
-  Rice::define_class_under<faiss::IndexPQ, faiss::Index>(rb_mFaiss, "IndexPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexIVFScalarQuantizer")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
-
-  Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(rb_mFaiss, "IndexIVFPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(rb_mFaiss, "IndexIVFPQR")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
-
-  Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryIVF")
-    .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
-
-  Rice::define_class_under<faiss::Clustering>(rb_mFaiss, "Kmeans")
-    .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
-    .define_method(
-      "d",
-      *[](faiss::Clustering &self) {
-        return self.d;
-      })
-    .define_method(
-      "k",
-      *[](faiss::Clustering &self) {
-        return self.k;
-      })
-    .define_method(
-      "_centroids",
-      *[](faiss::Clustering &self) {
-        float *centroids = new float[self.k * self.d];
-        for (size_t i = 0; i < self.centroids.size(); i++) {
-          centroids[i] = self.centroids[i];
-        }
-        return result(centroids, self.k * self.d);
-      })
-    .define_method(
-      "_train",
-      *[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
-        const float *x = float_array(o);
-        self.train(n, x, index);
-      });
-
-  Rice::define_class_under<faiss::PCAMatrix>(rb_mFaiss, "PCAMatrix")
-    .define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
-    .define_method(
-      "d_in",
-      *[](faiss::PCAMatrix &self) {
-        return self.d_in;
-      })
-    .define_method(
-      "d_out",
-      *[](faiss::PCAMatrix &self) {
-        return self.d_out;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_apply",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        float* res = self.apply(n, x);
-        return result(res, n * self.d_out);
-      });
+  auto m = Rice::define_module("Faiss");
 
-  Rice::define_class_under<faiss::ProductQuantizer>(rb_mFaiss, "ProductQuantizer")
-    .define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
-    .define_method(
-      "d",
-      *[](faiss::ProductQuantizer &self) {
-        return self.d;
-      })
-    .define_method(
-      "m",
-      *[](faiss::ProductQuantizer &self) {
-        return self.M;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_compute_codes",
-      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
-        const float *x = float_array(o);
-        uint8_t *codes = new uint8_t[n * self.M];
-        self.compute_codes(x, codes, n);
-        return result(codes, n * self.M);
-      })
-    .define_method(
-      "_decode",
-      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
-        const uint8_t *codes = uint8_array(o);
-        float *x = new float[n * self.d];
-        self.decode(codes, x, n);
-        return result(x, n * self.d);
-      })
-    .define_method(
-      "save",
-      *[](faiss::ProductQuantizer &self, const char *fname) {
-        faiss::write_ProductQuantizer(&self, fname);
-      })
-    .define_singleton_method(
-      "load",
-      *[](const char *fname) {
-        return faiss::read_ProductQuantizer(fname);
-      });
+  init_index(m);
+  init_index_binary(m);
+  init_kmeans(m);
+  init_pca_matrix(m);
+  init_product_quantizer(m);
 }

data/ext/faiss/extconf.rb

@@ -4,12 +4,12 @@ abort "BLAS not found" unless have_library("blas")
 abort "LAPACK not found" unless have_library("lapack")
 abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
 
-$CXXFLAGS << " -std=c++11 -march=native -DFINTEGER=int"
+$CXXFLAGS << " -std=c++11 -DFINTEGER=int " << with_config("optflags", "-march=native")
 
 ext = File.expand_path(".", __dir__)
-vendor = File.expand_path("../../vendor", __dir__)
+vendor = File.expand_path("../../vendor/faiss", __dir__)
 
-$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/impl,#{vendor}/faiss/utils}/*.{cpp}"]
+$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/{impl,invlists,utils}}/*.{cpp}"]
 $objs = $srcs.map { |v| v.sub(/cpp\z/, "o") }
 $INCFLAGS << " -I#{vendor}"
 $VPATH << vendor