ngt 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 967a8b449269b1d56539e294015f925791ca960a8bfe622aeff171d0d3170069
+  data.tar.gz: 2bc6884f36fe47ddf06cfcc6850dbd3b31d1b2044a11e477ea0684a72d325145
+SHA512:
+  metadata.gz: 399e1cb7a347da8ef610ff394fbd84f73d29edc9408563327bb1a58877c9cb5e255394417642fe84a08e4ee09b15083dc7a789a30192ae3ce25474a6b4211a23
+  data.tar.gz: 92b97b7a9d32bf85bf016e81b83fea70f9292a5fda829f4e1433ca8402b9e1c0f7f48b91279898a80e4bbe61ca91e913e83e4805740939a179d85790d618daa0

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2019 Andrew Kane
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,137 @@
+# NGT
+
+[NGT](https://github.com/yahoojapan/NGT) - high-speed approximate nearest neighbors - for Ruby
+
+## Installation
+
+First, [install NGT](https://github.com/yahoojapan/NGT/blob/master/README.md#Installation).
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'ngt'
+```
+
+## Getting Started
+
+Prep your data
+
+```ruby
+objects = [
+  [1, 1, 2, 1],
+  [5, 4, 6, 5],
+  [1, 2, 1, 2]
+]
+```
+
+Create an index
+
+```ruby
+index = Ngt::Index.create(path, dimensions)
+```
+
+Insert objects
+
+```ruby
+index.batch_insert(objects)
+```
+
+Search the index
+
+```ruby
+index.search(query, size: 3)
+```
+
+Save the index
+
+```ruby
+index.save
+```
+
+Load an index
+
+```ruby
+Ngt::Index.new(path)
+```
+
+Get an object by id
+
+```ruby
+index.object(id)
+```
+
+Insert a single object
+
+```ruby
+index.insert(object)
+```
+
+Remove an object by id
+
+```ruby
+index.remove(id)
+```
+
+Build the index
+
+```ruby
+index.build_index
+```
+
+## Full Example
+
+```ruby
+dim = 10
+objects = []
+100.times do |i|
+  objects << dim.times.map { rand(100) }
+end
+
+index = Ngt::Index.create("tmp", dim)
+index.batch_insert(objects)
+index.save
+
+query = objects[0]
+result = index.search(query, size: 3)
+
+result.each do |res|
+  puts "#{res[:id]}, #{res[:distance]}"
+  object = index.object(res[:id])
+  p object
+end
+```
+
+## Data
+
+Data can be an array of arrays
+
+```ruby
+[[1, 2, 3], [4, 5, 6]]
+```
+
+Or a Numo NArray
+
+```ruby
+Numo::DFloat.new(3, 2).seq
+```
+
+## Resources
+
+- [ANN Benchmarks](https://github.com/erikbern/ann-benchmarks)
+
+## Credits
+
+This library is modeled after NGT’s [Python API](https://github.com/yahoojapan/NGT/blob/master/python/README-ngtpy.md).
+
+## History
+
+View the [changelog](https://github.com/ankane/ngt/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/ngt/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/ngt/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features

data/lib/ngt.rb

@@ -0,0 +1,18 @@
+# dependencies
+require "ffi"
+
+# modules
+require "ngt/index"
+require "ngt/version"
+
+module Ngt
+  class Error < StandardError; end
+
+  class << self
+    attr_accessor :ffi_lib
+  end
+  self.ffi_lib = ["ngt"]
+
+  # friendlier error message
+  autoload :FFI, "ngt/ffi"
+end

data/lib/ngt/ffi.rb

@@ -0,0 +1,57 @@
+module Ngt
+  module FFI
+    extend ::FFI::Library
+
+    begin
+      ffi_lib Ngt.ffi_lib
+    rescue LoadError => e
+      raise e if ENV["NGT_DEBUG"]
+      raise LoadError, "Could not find NGT"
+    end
+
+    # https://github.com/yahoojapan/NGT/blob/master/lib/NGT/Capi.h
+    # keep same order
+
+    class ObjectDistance < ::FFI::Struct
+      layout :id, :int,
+        :distance, :float
+    end
+
+    attach_function :ngt_open_index, %i[string pointer], :pointer
+    attach_function :ngt_create_graph_and_tree, %i[string pointer pointer], :pointer
+    attach_function :ngt_create_property, %i[pointer], :pointer
+    attach_function :ngt_save_index, %i[pointer string pointer], :bool
+    attach_function :ngt_get_property, %i[pointer pointer pointer], :bool
+    attach_function :ngt_get_property_dimension, %i[pointer pointer], :int32_t
+    attach_function :ngt_set_property_dimension, %i[pointer int32_t pointer], :bool
+    attach_function :ngt_set_property_edge_size_for_creation, %i[pointer int16_t pointer], :bool
+    attach_function :ngt_set_property_edge_size_for_search, %i[pointer int16_t pointer], :bool
+    attach_function :ngt_is_property_object_type_float, %i[int32_t], :bool
+    attach_function :ngt_get_property_object_type, %i[pointer pointer], :int32_t
+    attach_function :ngt_set_property_object_type_float, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_object_type_integer, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_l1, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_l2, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_angle, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_hamming, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_jaccard, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_cosine, %i[pointer pointer], :bool
+    attach_function :ngt_batch_insert_index, %i[pointer pointer uint32_t pointer pointer], :bool
+    attach_function :ngt_create_index, %i[pointer uint32_t pointer], :bool
+    attach_function :ngt_remove_index, %i[pointer int pointer], :bool
+    attach_function :ngt_insert_index, %i[pointer pointer uint32_t pointer], :int
+    attach_function :ngt_insert_index_as_float, %i[pointer pointer uint32_t pointer], :int
+    attach_function :ngt_create_empty_results, %i[pointer], :pointer
+    attach_function :ngt_search_index, %i[pointer pointer int32_t size_t float float pointer pointer], :bool
+    attach_function :ngt_get_result_size, %i[pointer pointer], :uint32_t
+    attach_function :ngt_get_result, %i[pointer uint32_t pointer], ObjectDistance.by_value
+    attach_function :ngt_get_object_space, %i[pointer pointer], :pointer
+    attach_function :ngt_get_object_as_float, %i[pointer int pointer], :pointer
+    attach_function :ngt_get_object_as_integer, %i[pointer int pointer], :pointer
+    attach_function :ngt_destroy_property, %i[pointer], :void
+    attach_function :ngt_close_index, %i[pointer], :void
+    attach_function :ngt_create_error_object, %i[], :pointer
+    attach_function :ngt_get_error_string, %i[pointer], :string
+    attach_function :ngt_destroy_error_object, %i[pointer], :void
+  end
+end

data/lib/ngt/index.rb

@@ -0,0 +1,161 @@
+module Ngt
+  class Index
+    def initialize(path)
+      @path = path
+      @error = FFI.ngt_create_error_object
+      @index = ffi(:ngt_open_index, path)
+
+      property = ffi(:ngt_create_property)
+      ffi(:ngt_get_property, @index, property)
+
+      @dimension = ffi(:ngt_get_property_dimension, property)
+
+      object_type = ffi(:ngt_get_property_object_type, property)
+      @float = FFI.ngt_is_property_object_type_float(object_type)
+
+      @object_space = ffi(:ngt_get_object_space, @index)
+
+      ObjectSpace.define_finalizer(self, self.class.finalize(@error))
+    end
+
+    def insert(object)
+      ffi(:ngt_insert_index, @index, c_object(object.to_a), @dimension)
+    end
+
+    # TODO make more performant for Numo
+    def batch_insert(objects, num_threads: 8)
+      objects = objects.to_a
+      flat_objects = objects.flatten
+      obj = ::FFI::MemoryPointer.new(:float, flat_objects.size)
+      obj.write_array_of_float(flat_objects)
+
+      ids = ::FFI::MemoryPointer.new(:uint32_t, objects.size)
+      ffi(:ngt_batch_insert_index, @index, obj, objects.size, ids)
+
+      build_index(num_threads: num_threads)
+
+      ids.read_array_of_uint32(objects.size)
+    end
+
+    def build_index(num_threads: 8)
+      ffi(:ngt_create_index, @index, num_threads)
+    end
+
+    def object(id)
+      if float?
+        res = ffi(:ngt_get_object_as_float, @object_space, id)
+        res.read_array_of_float(@dimension)
+      else
+        res = ffi(:ngt_get_object_as_integer, @object_space, id)
+        res.read_array_of_uint8(@dimension)
+      end
+    end
+
+    def remove(id)
+      ffi(:ngt_remove_index, @index, id)
+    end
+
+    def search(query, size: 20, epsilon: 0.1, radius: nil)
+      radius ||= -1.0
+      results = ffi(:ngt_create_empty_results)
+      ffi(:ngt_search_index, @index, c_object(query.to_a), @dimension, size, epsilon, radius, results)
+      result_size = ffi(:ngt_get_result_size, results)
+      ret = []
+      result_size.times do |i|
+        res = ffi(:ngt_get_result, results, i)
+        ret << {
+          id: res[:id],
+          distance: res[:distance]
+        }
+      end
+      ret
+    end
+
+    def save(path: nil)
+      path ||= @path
+      ffi(:ngt_save_index, @index, path)
+    end
+
+    def close
+      FFI.ngt_close_index(@index)
+    end
+
+    def self.create(path, dimension, edge_size_for_creation: 10,
+        edge_size_for_search: 40, object_type: "Float", distance_type: "L2")
+
+      error = FFI.ngt_create_error_object
+      property = ffi(:ngt_create_property, error)
+      ffi(:ngt_set_property_dimension, property, dimension, error)
+      ffi(:ngt_set_property_edge_size_for_creation, property, edge_size_for_creation, error)
+      ffi(:ngt_set_property_edge_size_for_search, property, edge_size_for_search, error)
+
+      case object_type.to_s
+      when "Float", "float"
+        ffi(:ngt_set_property_object_type_float, property, error)
+      when "Integer", "integer"
+        ffi(:ngt_set_property_object_type_integer, property, error)
+      else
+        raise ArgumentError, "Unknown object type: #{object_type}"
+      end
+
+      case distance_type.to_s
+      when "L1"
+        ffi(:ngt_set_property_distance_type_l1, property, error)
+      when "L2"
+        ffi(:ngt_set_property_distance_type_l2, property, error)
+      when "Angle"
+        ffi(:ngt_set_property_distance_type_angle, property, error)
+      when "Hamming"
+        ffi(:ngt_set_property_distance_type_hamming, property, error)
+      when "Jaccard"
+        ffi(:ngt_set_property_distance_type_jaccard, property, error)
+      when "Cosine"
+        ffi(:ngt_set_property_distance_type_cosine, property, error)
+      else
+        raise ArgumentError, "Unknown distance type: #{distance_type}"
+      end
+
+      index = ffi(:ngt_create_graph_and_tree, path, property, error)
+      FFI.ngt_close_index(index)
+      index = nil
+
+      Index.new(path)
+    ensure
+      FFI.ngt_destroy_error_object(error) if error
+      FFI.ngt_destroy_property(property) if property
+      FFI.ngt_close_index(index) if index
+    end
+
+    # private
+    def self.ffi(method, *args)
+      res = FFI.send(method, *args)
+      message = FFI.ngt_get_error_string(args.last)
+      raise Error, message unless message.empty?
+      res
+    end
+
+    def self.finalize(error)
+      # must use proc instead of stabby lambda
+      proc do
+        # TODO clean-up more objects
+        FFI.ngt_destroy_error_object(error)
+      end
+    end
+
+    private
+
+    def ffi(*args)
+      self.class.ffi(*args, @error)
+    end
+
+    def float?
+      @float
+    end
+
+    def c_object(object)
+      c_object = ::FFI::MemoryPointer.new(:double, object.size)
+      c_object.write_array_of_double(object)
+      c_object
+    end
+  end
+end

data/lib/ngt/version.rb

@@ -0,0 +1,3 @@
+module Ngt
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: ngt
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: andrew@chartkick.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/ngt.rb
+- lib/ngt/ffi.rb
+- lib/ngt/index.rb
+- lib/ngt/version.rb
+homepage: https://github.com/ankane/ngt
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: NGT - high-speed approximate nearest neighbors - for Ruby
+test_files: []