faiss 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 47b973803ea179379ab292d5d7b8350c12a383387e90b96f38eae4e90e20dad3
+  data.tar.gz: 5697b1ec26dbbb0794e6c11563b0cf76d564391882e7c0c0148f3893e1bf6b59
+SHA512:
+  metadata.gz: dccf5a8ddfd4030e70308c3e71b16bb6742839b8b1a263d3a010a381729af869c1755164e25a3941a3f475376e783ec42b43409fede77010e97949092a424e18
+  data.tar.gz: 57c933bc9d3ffbec37be0edbf641bd004e8dbfa13c04d52ca970e8ff03f719f65f27016892b06cd66755f36afc8d3ff505e6062656f2c48fa20d58cf64c2b6fd

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2020-03-08)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2020 Andrew Kane
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,130 @@
+# Faiss
+
+[Faiss](https://github.com/facebookresearch/faiss) - efficient similarity search and clustering - for Ruby
+
+## Installation
+
+First, install the [Faiss C++ library](https://github.com/facebookresearch/faiss/blob/master/INSTALL.md). For Homebrew, use:
+
+```sh
+brew install faiss
+```
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'faiss'
+```
+
+## Getting Started
+
+Prep your data
+
+```ruby
+objects = [
+  [1, 1, 2, 1],
+  [5, 4, 6, 5],
+  [1, 2, 1, 2]
+]
+```
+
+Build an index
+
+```ruby
+index = Faiss::IndexFlatL2.new(4)
+index.add(objects)
+```
+
+Search
+
+```ruby
+distances, ids = index.search(objects, 3)
+```
+
+## K-means Clustering
+
+Train
+
+```ruby
+kmeans = Faiss::Kmeans.new(4, 2)
+kmeans.train(objects)
+```
+
+Get the centroids
+
+```ruby
+kmeans.centroids
+```
+
+## PCA
+
+Train
+
+```ruby
+mat = Faiss::PCAMatrix.new(40, 10)
+mat.train(objects)
+```
+
+Apply
+
+```ruby
+mat.apply(mt)
+```
+
+## Product Quantizer
+
+Train
+
+```ruby
+pq = Faiss::ProductQuantizer.new(32, 4, 8)
+pq.train(objects)
+```
+
+Encode
+
+```ruby
+pq.compute_codes(objects)
+```
+
+Decode
+
+```ruby
+pq.decode(codes)
+```
+
+## Data
+
+Data can be an array of arrays
+
+```ruby
+[[1, 2, 3], [4, 5, 6]]
+```
+
+Or a Numo NArray
+
+```ruby
+Numo::SFloat.new(3, 2).seq
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/faiss/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/faiss/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/faiss/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/faiss.git
+cd faiss
+bundle install
+bundle exec rake compile
+bundle exec rake test
+```

data/ext/faiss/ext.cpp

@@ -0,0 +1,248 @@
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexHNSW.h>
+#include <faiss/IndexIVFFlat.h>
+#include <faiss/IndexLSH.h>
+#include <faiss/IndexScalarQuantizer.h>
+#include <faiss/IndexPQ.h>
+#include <faiss/IndexIVFPQ.h>
+#include <faiss/IndexIVFPQR.h>
+
+#include <faiss/IndexBinaryFlat.h>
+#include <faiss/IndexBinaryIVF.h>
+#include <faiss/index_factory.h>
+
+#include <faiss/Clustering.h>
+#include <faiss/VectorTransform.h>
+
+#include <rice/Array.hpp>
+#include <rice/Class.hpp>
+#include <rice/Constructor.hpp>
+#include <rice/Module.hpp>
+
+extern "C"
+void Init_ext()
+{
+  Rice::Module rb_mFaiss = Rice::define_module("Faiss")
+    .define_singleton_method(
+      "index_binary_factory",
+      *[](int d, const char *description) {
+        return faiss::index_binary_factory(d, description);
+      });
+
+  Rice::define_class_under<faiss::Index>(rb_mFaiss, "Index")
+    .define_method(
+      "d",
+      *[](faiss::Index &self) {
+        return self.d;
+      })
+    .define_method(
+      "trained?",
+      *[](faiss::Index &self) {
+        return self.is_trained;
+      })
+    .define_method(
+      "ntotal",
+      *[](faiss::Index &self) {
+        return self.ntotal;
+      })
+    .define_method(
+      "_train",
+      *[](faiss::Index &self, int64_t n, Rice::String s) {
+        const float *x = (float*) s.c_str();
+        self.train(n, x);
+      })
+    .define_method(
+      "_add",
+      *[](faiss::Index &self, int64_t n, Rice::String s) {
+        const float *x = (float*) s.c_str();
+        self.add(n, x);
+      })
+    .define_method(
+      "_search",
+      *[](faiss::Index &self, int64_t n, Rice::String s, int64_t k) {
+        const float *x = (float*) s.c_str();
+        float *distances = new float[k * n];
+        int64_t *labels = new int64_t[k * n];
+
+        self.search(n, x, k, distances, labels);
+
+        auto dstr = std::string((char*) distances, k * n * sizeof(float));
+        auto lstr = std::string((char*) labels, k * n * sizeof(int64_t));
+
+        Rice::Array ret;
+        ret.push(dstr);
+        ret.push(lstr);
+        return ret;
+      });
+
+  Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
+    .define_method(
+      "d",
+      *[](faiss::Index &self) {
+        return self.d;
+      })
+    .define_method(
+      "trained?",
+      *[](faiss::IndexBinary &self) {
+        return self.is_trained;
+      })
+    .define_method(
+      "ntotal",
+      *[](faiss::IndexBinary &self) {
+        return self.ntotal;
+      })
+    .define_method(
+      "_train",
+      *[](faiss::IndexBinary &self, int64_t n, Rice::String s) {
+        const uint8_t *x = (uint8_t*) s.c_str();
+        self.train(n, x);
+      })
+    .define_method(
+      "_add",
+      *[](faiss::IndexBinary &self, int64_t n, Rice::String s) {
+        const uint8_t *x = (uint8_t*) s.c_str();
+        self.add(n, x);
+      })
+    .define_method(
+      "_search",
+      *[](faiss::IndexBinary &self, int64_t n, Rice::String s, int64_t k) {
+        const uint8_t *x = (uint8_t*) s.c_str();
+        int32_t *distances = new int32_t[k * n];
+        int64_t *labels = new int64_t[k * n];
+
+        self.search(n, x, k, distances, labels);
+
+        auto dstr = std::string((char*) distances, k * n * sizeof(int32_t));
+        auto lstr = std::string((char*) labels, k * n * sizeof(int64_t));
+
+        Rice::Array ret;
+        ret.push(dstr);
+        ret.push(lstr);
+        return ret;
+      });
+
+  Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
+    .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
+
+  Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(rb_mFaiss, "IndexFlatIP")
+    .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
+
+  Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(rb_mFaiss, "IndexHNSWFlat")
+    .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int>());
+
+  Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(rb_mFaiss, "IndexIVFFlat")
+    .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
+
+  Rice::define_class_under<faiss::IndexLSH, faiss::Index>(rb_mFaiss, "IndexLSH")
+    .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
+
+  Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexScalarQuantizer")
+    .define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
+
+  Rice::define_class_under<faiss::IndexPQ, faiss::Index>(rb_mFaiss, "IndexPQ")
+    .define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
+
+  Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexIVFScalarQuantizer")
+    .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
+
+  Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(rb_mFaiss, "IndexIVFPQ")
+    .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
+
+  Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(rb_mFaiss, "IndexIVFPQR")
+    .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
+
+  Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryFlat")
+    .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
+
+  Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryIVF")
+    .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
+
+  Rice::define_class_under<faiss::Clustering>(rb_mFaiss, "Kmeans")
+    .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
+    .define_method(
+      "d",
+      *[](faiss::Clustering &self) {
+        return self.d;
+      })
+    .define_method(
+      "k",
+      *[](faiss::Clustering &self) {
+        return self.k;
+      })
+    .define_method(
+      "_centroids",
+      *[](faiss::Clustering &self) {
+        float *centroids = new float[self.k * self.d];
+        for (size_t i = 0; i < self.centroids.size(); i++) {
+          centroids[i] = self.centroids[i];
+        }
+        return std::string((char*) centroids, self.k * self.d * sizeof(float));
+      })
+    .define_method(
+      "_train",
+      *[](faiss::Clustering &self, int64_t n, Rice::String s, faiss::Index & index) {
+        const float *x = (float*) s.c_str();
+        self.train(n, x, index);
+      });
+
+  Rice::define_class_under<faiss::PCAMatrix>(rb_mFaiss, "PCAMatrix")
+    .define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
+    .define_method(
+      "d_in",
+      *[](faiss::PCAMatrix &self) {
+        return self.d_in;
+      })
+    .define_method(
+      "d_out",
+      *[](faiss::PCAMatrix &self) {
+        return self.d_out;
+      })
+    .define_method(
+      "_train",
+      *[](faiss::PCAMatrix &self, int64_t n, Rice::String s) {
+        const float *x = (float*) s.c_str();
+        self.train(n, x);
+      })
+    .define_method(
+      "_apply",
+      *[](faiss::PCAMatrix &self, int64_t n, Rice::String s) {
+        const float *x = (float*) s.c_str();
+        float* res = self.apply(n, x);
+        return std::string((char*) res, n * self.d_out * sizeof(float));
+      });
+
+  Rice::define_class_under<faiss::ProductQuantizer>(rb_mFaiss, "ProductQuantizer")
+    .define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
+    .define_method(
+      "d",
+      *[](faiss::ProductQuantizer &self) {
+        return self.d;
+      })
+    .define_method(
+      "m",
+      *[](faiss::ProductQuantizer &self) {
+        return self.M;
+      })
+    .define_method(
+      "_train",
+      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
+        const float *x = (float*) s.c_str();
+        self.train(n, x);
+      })
+    .define_method(
+      "_compute_codes",
+      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
+        const float *x = (float*) s.c_str();
+        uint8_t *codes = new uint8_t[n * self.M];
+        self.compute_codes(x, codes, n);
+        return std::string((char*) codes, n * self.M * sizeof(uint8_t));
+      })
+    .define_method(
+      "_decode",
+      *[](faiss::ProductQuantizer &self, int n, Rice::String s) {
+        const uint8_t *codes = (uint8_t*) s.c_str();
+        float *x = new float[n * self.d];
+        self.decode(codes, x, n);
+        return std::string((char*) x, n * self.d * sizeof(float));
+      });
+}

data/ext/faiss/extconf.rb

@@ -0,0 +1,7 @@
+require "mkmf-rice"
+
+abort "Missing faiss" unless have_library("faiss")
+
+$CXXFLAGS << " -std=c++11"
+
+create_makefile("faiss/ext")

data/lib/faiss/index.rb

@@ -0,0 +1,20 @@
+module Faiss
+  class Index
+    def train(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      _train(objects.shape[0], objects.to_binary)
+    end
+
+    def add(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      _add(objects.shape[0], objects.to_binary)
+    end
+
+    def search(objects, k)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      n = objects.shape[0]
+      distances, labels = _search(n, objects.to_binary, k)
+      [Numo::SFloat.from_binary(distances).reshape(n, k), Numo::Int64.from_binary(labels).reshape(n, k)]
+    end
+  end
+end

data/lib/faiss/index_binary.rb

@@ -0,0 +1,20 @@
+module Faiss
+  class IndexBinary
+    def train(objects)
+      objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
+      _train(objects.shape[0], objects.to_binary)
+    end
+
+    def add(objects)
+      objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
+      _add(objects.shape[0], objects.to_binary)
+    end
+
+    def search(objects, k)
+      objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
+      n = objects.shape[0]
+      distances, labels = _search(n, objects.to_binary, k)
+      [Numo::UInt32.from_binary(distances).reshape(n, k), Numo::Int64.from_binary(labels).reshape(n, k)]
+    end
+  end
+end

data/lib/faiss/kmeans.rb

@@ -0,0 +1,15 @@
+module Faiss
+  class Kmeans
+    attr_reader :index
+
+    def train(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      @index = IndexFlatL2.new(d)
+      _train(objects.shape[0], objects.to_binary, @index)
+    end
+
+    def centroids
+      Numo::SFloat.from_binary(_centroids).reshape(k, d)
+    end
+  end
+end

data/lib/faiss/pca_matrix.rb

@@ -0,0 +1,15 @@
+module Faiss
+  class PCAMatrix
+    def train(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      _train(objects.shape[0], objects.to_binary)
+    end
+
+    def apply(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      n = objects.shape[0]
+      res = _apply(n, objects.to_binary)
+      Numo::SFloat.from_binary(res).reshape(n, d_out)
+    end
+  end
+end

data/lib/faiss/product_quantizer.rb

@@ -0,0 +1,22 @@
+module Faiss
+  class ProductQuantizer
+    def train(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      _train(objects.shape[0], objects.to_binary)
+    end
+
+    def compute_codes(objects)
+      objects = Numo::SFloat.cast(objects) unless objects.is_a?(Numo::SFloat)
+      n = objects.shape[0]
+      res = _compute_codes(n, objects.to_binary)
+      Numo::UInt8.from_binary(res).reshape(n, m)
+    end
+
+    def decode(objects)
+      objects = Numo::UInt8.cast(objects) unless objects.is_a?(Numo::UInt8)
+      n = objects.shape[0]
+      res = _decode(n, objects.to_binary)
+      Numo::SFloat.from_binary(res).reshape(n, d)
+    end
+  end
+end

data/lib/faiss/version.rb

@@ -0,0 +1,3 @@
+module Faiss
+  VERSION = "0.1.0"
+end

data/lib/faiss.rb

@@ -0,0 +1,13 @@
+# dependencies
+require "numo/narray"
+
+# ext
+require "faiss/ext"
+
+# modules
+require "faiss/index"
+require "faiss/index_binary"
+require "faiss/kmeans"
+require "faiss/pca_matrix"
+require "faiss/product_quantizer"
+require "faiss/version"

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: faiss
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rice
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+description: 
+email: andrew@chartkick.com
+executables: []
+extensions:
+- ext/faiss/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- ext/faiss/ext.cpp
+- ext/faiss/extconf.rb
+- lib/faiss.rb
+- lib/faiss/index.rb
+- lib/faiss/index_binary.rb
+- lib/faiss/kmeans.rb
+- lib/faiss/pca_matrix.rb
+- lib/faiss/product_quantizer.rb
+- lib/faiss/version.rb
+homepage: https://github.com/ankane/faiss
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Efficient similarity search and clustering for Ruby
+test_files: []