faiss 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97fd4d583754d20a5771e941fd5272205977c34b77eb9252e58f6016b90be52e
-  data.tar.gz: 26df73fc7891efd894e9ebf4ed37db2aaa3af8b62221cdd707cb287f920ad0a7
+  metadata.gz: 04246f7b765dc5680223f91ecc5c0951f6328411dcaf443c4ae88819c17b26a7
+  data.tar.gz: 81698c65e8cca4d7794253775ef06f6e84af005b8dac2872d5d3057a868638ac
 SHA512:
-  metadata.gz: 197143a79061c299b8f5e5ae76466db1a673e46f6d842615bed46170afadb6145e14df936262ec45f7e0d1eea889037cb3123c5a9348e32774eeb97414f180fd
-  data.tar.gz: 5a8e1ecebd43886d974e52fa216d86d9c03e3f8c04629eddc04a8be8df5c7ab4fc4c57398f2893d44718c788cd1853ae1bee06f64f1d49cdc5330cb03c5c0e6c
+  metadata.gz: 6ad4dcb525a55f0b6743fddeb856d6132c8a518eb92be3dc2ce57302ff05b921fb3ba517aecc76bde78aa6e8bf11d9199fa97cbfe4f3a00e4d466d47f0cabe14
+  data.tar.gz: b333df087e863c8ee828b66c1611d8576976064419403710b0b238346d46cc58e69f1c2c708e35bdbc995754b02f50a976389bb0eb102273e7b9f92d771ac395

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+## 0.2.0 (2021-05-23)
+
+- Improved performance
+- Updated to Rice 4
+- Dropped support for Ruby < 2.6
+
+## 0.1.7 (2021-03-26)
+
+- Added `IndexScalarQuantizer` and `IndexIVFScalarQuantizer`
+
+## 0.1.6 (2021-03-21)
+
+- Added support for inner product to `IndexIVFFlat` and `IndexIVFPQ`
+- Added `ParameterSpace`
+- Added `nprobe=` method
+
+## 0.1.5 (2021-03-14)
+
+- Added `--with-optflags` option
+- Added support for inner product to `IndexHNSWFlat`
+
+## 0.1.4 (2021-02-04)
+
+- Updated Faiss to 1.7.0
+
 ## 0.1.3 (2020-10-22)
 
 - Updated Faiss to 1.6.4

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) Facebook, Inc. and its affiliates.
-Copyright (c) 2020 Andrew Kane
+Copyright (c) 2020-2021 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -4,7 +4,7 @@
 
 Learn more about [Faiss](https://engineering.fb.com/data-infrastructure/faiss-a-library-for-efficient-similarity-search/)
 
-[![Build Status](https://travis-ci.org/ankane/faiss.svg?branch=master)](https://travis-ci.org/ankane/faiss)
+[![Build Status](https://github.com/ankane/faiss/workflows/build/badge.svg?branch=master)](https://github.com/ankane/faiss/actions)
 
 ## Installation
 
@@ -24,7 +24,7 @@ Add this line to your application’s Gemfile:
 gem 'faiss'
 ```
 
-Faiss is not available for Windows
+It can take a few minutes to compile the gem. Faiss is not available for Windows.
 
 ## Getting Started
 
@@ -97,12 +97,24 @@ Locality-sensitive hashing
 Faiss::IndexLSH.new(d, nbits)
 ```
 
+Scalar quantizer (SQ) in flat mode
+
+```rb
+Faiss::IndexScalarQuantizer.new(d, qtype)
+```
+
 Product quantizer (PQ) in flat mode
 
 ```rb
 Faiss::IndexPQ.new(d, m, nbits)
 ```
 
+IVF and scalar quantizer
+
+```rb
+Faiss::IndexIVFScalarQuantizer.new(quantizer, d, nlists, qtype)
+```
+
 IVFADC (coarse quantizer+PQ on residuals)
 
 ```rb
@@ -200,10 +212,10 @@ Data can be an array of arrays
 [[1, 2, 3], [4, 5, 6]]
 ```
 
-Or a Numo NArray
+Or a Numo array
 
 ```ruby
-Numo::SFloat.new(3, 2).seq
+Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
 ```
 
 ## History

data/ext/faiss/ext.cpp

@@ -1,315 +1,19 @@
-#include <faiss/IndexFlat.h>
-#include <faiss/IndexHNSW.h>
-#include <faiss/IndexIVFFlat.h>
-#include <faiss/IndexLSH.h>
-#include <faiss/IndexScalarQuantizer.h>
-#include <faiss/IndexPQ.h>
-#include <faiss/IndexIVFPQ.h>
-#include <faiss/IndexIVFPQR.h>
+#include "utils.h"
 
-#include <faiss/IndexBinaryFlat.h>
-#include <faiss/IndexBinaryIVF.h>
-#include <faiss/index_factory.h>
-#include <faiss/index_io.h>
-
-#include <faiss/Clustering.h>
-#include <faiss/VectorTransform.h>
-
-#include <rice/Array.hpp>
-#include <rice/Class.hpp>
-#include <rice/Constructor.hpp>
-#include <rice/Module.hpp>
-
-float* float_array(Rice::Object o)
-{
-  Rice::String s = o.call("to_binary");
-  return (float*) s.c_str();
-}
-
-uint8_t* uint8_array(Rice::Object o)
-{
-  Rice::String s = o.call("to_binary");
-  return (uint8_t*) s.c_str();
-}
-
-// TODO return Numo::SFloat
-Rice::String result(float* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(float)));
-}
-
-// TODO return Numo::UInt8
-Rice::String result(uint8_t* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
-}
-
-// TODO return Numo::Int32
-Rice::String result(int32_t* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
-}
-
-// TODO return Numo::Int64
-Rice::String result(int64_t* ptr, int64_t length)
-{
-  return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
-}
+void init_index(Rice::Module& m);
+void init_index_binary(Rice::Module& m);
+void init_kmeans(Rice::Module& m);
+void init_pca_matrix(Rice::Module& m);
+void init_product_quantizer(Rice::Module& m);
 
 extern "C"
 void Init_ext()
 {
-  Rice::Module rb_mFaiss = Rice::define_module("Faiss")
-    .define_singleton_method(
-      "index_binary_factory",
-      *[](int d, const char *description) {
-        return faiss::index_binary_factory(d, description);
-      });
-
-  Rice::define_class_under<faiss::Index>(rb_mFaiss, "Index")
-    .define_method(
-      "d",
-      *[](faiss::Index &self) {
-        return self.d;
-      })
-    .define_method(
-      "trained?",
-      *[](faiss::Index &self) {
-        return self.is_trained;
-      })
-    .define_method(
-      "ntotal",
-      *[](faiss::Index &self) {
-        return self.ntotal;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::Index &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_add",
-      *[](faiss::Index &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.add(n, x);
-      })
-    .define_method(
-      "_search",
-      *[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
-        const float *x = float_array(o);
-        float *distances = new float[k * n];
-        int64_t *labels = new int64_t[k * n];
-
-        self.search(n, x, k, distances, labels);
-
-        auto dstr = result(distances, k * n);
-        auto lstr = result(labels, k * n);
-
-        Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
-        return ret;
-      })
-    .define_method(
-      "save",
-      *[](faiss::Index &self, const char *fname) {
-        faiss::write_index(&self, fname);
-      })
-    .define_singleton_method(
-      "load",
-      *[](const char *fname) {
-        return faiss::read_index(fname);
-      });
-
-  Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
-    .define_method(
-      "d",
-      *[](faiss::Index &self) {
-        return self.d;
-      })
-    .define_method(
-      "trained?",
-      *[](faiss::IndexBinary &self) {
-        return self.is_trained;
-      })
-    .define_method(
-      "ntotal",
-      *[](faiss::IndexBinary &self) {
-        return self.ntotal;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
-        const uint8_t *x = uint8_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_add",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
-        const uint8_t *x = uint8_array(o);
-        self.add(n, x);
-      })
-    .define_method(
-      "_search",
-      *[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
-        const uint8_t *x = uint8_array(o);
-        int32_t *distances = new int32_t[k * n];
-        int64_t *labels = new int64_t[k * n];
-
-        self.search(n, x, k, distances, labels);
-
-        auto dstr = result(distances, k * n);
-        auto lstr = result(labels, k * n);
-
-        Rice::Array ret;
-        ret.push(dstr);
-        ret.push(lstr);
-        return ret;
-      })
-    .define_method(
-      "save",
-      *[](faiss::IndexBinary &self, const char *fname) {
-        faiss::write_index_binary(&self, fname);
-      })
-    .define_singleton_method(
-      "load",
-      *[](const char *fname) {
-        return faiss::read_index_binary(fname);
-      });
-
-  Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
-    .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
-
-  Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(rb_mFaiss, "IndexFlatIP")
-    .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
-
-  Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(rb_mFaiss, "IndexHNSWFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int>());
-
-  Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(rb_mFaiss, "IndexIVFFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexLSH, faiss::Index>(rb_mFaiss, "IndexLSH")
-    .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
-
-  Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexScalarQuantizer")
-    .define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
-
-  Rice::define_class_under<faiss::IndexPQ, faiss::Index>(rb_mFaiss, "IndexPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexIVFScalarQuantizer")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
-
-  Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(rb_mFaiss, "IndexIVFPQ")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(rb_mFaiss, "IndexIVFPQR")
-    .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
-
-  Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryFlat")
-    .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
-
-  Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryIVF")
-    .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
-
-  Rice::define_class_under<faiss::Clustering>(rb_mFaiss, "Kmeans")
-    .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
-    .define_method(
-      "d",
-      *[](faiss::Clustering &self) {
-        return self.d;
-      })
-    .define_method(
-      "k",
-      *[](faiss::Clustering &self) {
-        return self.k;
-      })
-    .define_method(
-      "_centroids",
-      *[](faiss::Clustering &self) {
-        float *centroids = new float[self.k * self.d];
-        for (size_t i = 0; i < self.centroids.size(); i++) {
-          centroids[i] = self.centroids[i];
-        }
-        return result(centroids, self.k * self.d);
-      })
-    .define_method(
-      "_train",
-      *[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
-        const float *x = float_array(o);
-        self.train(n, x, index);
-      });
-
-  Rice::define_class_under<faiss::PCAMatrix>(rb_mFaiss, "PCAMatrix")
-    .define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
-    .define_method(
-      "d_in",
-      *[](faiss::PCAMatrix &self) {
-        return self.d_in;
-      })
-    .define_method(
-      "d_out",
-      *[](faiss::PCAMatrix &self) {
-        return self.d_out;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_apply",
-      *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
-        const float *x = float_array(o);
-        float* res = self.apply(n, x);
-        return result(res, n * self.d_out);
-      });
+  auto m = Rice::define_module("Faiss");
 
-  Rice::define_class_under<faiss::ProductQuantizer>(rb_mFaiss, "ProductQuantizer")
-    .define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
-    .define_method(
-      "d",
-      *[](faiss::ProductQuantizer &self) {
-        return self.d;
-      })
-    .define_method(
-      "m",
-      *[](faiss::ProductQuantizer &self) {
-        return self.M;
-      })
-    .define_method(
-      "_train",
-      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
-        const float *x = float_array(o);
-        self.train(n, x);
-      })
-    .define_method(
-      "_compute_codes",
-      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
-        const float *x = float_array(o);
-        uint8_t *codes = new uint8_t[n * self.M];
-        self.compute_codes(x, codes, n);
-        return result(codes, n * self.M);
-      })
-    .define_method(
-      "_decode",
-      *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
-        const uint8_t *codes = uint8_array(o);
-        float *x = new float[n * self.d];
-        self.decode(codes, x, n);
-        return result(x, n * self.d);
-      })
-    .define_method(
-      "save",
-      *[](faiss::ProductQuantizer &self, const char *fname) {
-        faiss::write_ProductQuantizer(&self, fname);
-      })
-    .define_singleton_method(
-      "load",
-      *[](const char *fname) {
-        return faiss::read_ProductQuantizer(fname);
-      });
+  init_index(m);
+  init_index_binary(m);
+  init_kmeans(m);
+  init_pca_matrix(m);
+  init_product_quantizer(m);
 }

data/ext/faiss/extconf.rb

@@ -4,14 +4,17 @@ abort "BLAS not found" unless have_library("blas")
 abort "LAPACK not found" unless have_library("lapack")
 abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
 
-$CXXFLAGS << " -std=c++11 -march=native -DFINTEGER=int"
+numo = $LOAD_PATH.find { |v| File.exist?("#{v}/numo/numo/narray.h") }
+abort "Numo not found" unless numo && find_header("numo/narray.h", "#{numo}/numo")
+
+$CXXFLAGS << " -std=c++17 $(optflags) -DFINTEGER=int " << with_config("optflags", "-march=native")
 
 ext = File.expand_path(".", __dir__)
 vendor = File.expand_path("../../vendor/faiss", __dir__)
 
-$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/impl,#{vendor}/faiss/utils}/*.{cpp}"]
+$srcs = Dir["{#{ext},#{vendor}/faiss,#{vendor}/faiss/{impl,invlists,utils}}/*.{cpp}"]
 $objs = $srcs.map { |v| v.sub(/cpp\z/, "o") }
-$INCFLAGS << " -I#{vendor}"
+abort "Faiss not found" unless find_header("faiss/Index.h", vendor)
 $VPATH << vendor
 
 create_makefile("faiss/ext")