ngt 0.2.3 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bff3db4fe839a13d6c1d0c5c959c4e42bf738670c69b0fd4792a1209c7e7165f
-  data.tar.gz: ed3c9d563903be2c8d550bccbeb7efd40cb78b1f5d78e8d6846e9dc4063f78f6
+  metadata.gz: 60b5246463e87f7d93207b6fe80de719ca27dc9d6414305de8f5a81fb8b9cac6
+  data.tar.gz: 56a9d9b119e6c24fd4b9a32e309e7c400315cd17520768752a224110ae50eaa0
 SHA512:
-  metadata.gz: 7e85cd285bac398611fcae5d3bae99f9b13422b1dab068387f787d91b9137eb7ffdf6b96e1d64a5719ce6eef8fef8c2a2e7f8553893ee535f8f9f09ca55ca9b3
-  data.tar.gz: 9645e1859e2c65d88688b23006a94bb272b8ad19c6df64a43427c8842d1e49b26fe3431524c02c74eefacfa0c20360760fb6c31691583190d1d323efb688a7ed
+  metadata.gz: 20602673391ee8d7483eb6b86c3d59b5ccecad6bd25d44abdacfae0d5e599918b6b3ca915fbcd5715f449d77ee18c52fe80b3b2797d86f745e72fb4fb2a03605
+  data.tar.gz: 7785fab0add41014f2586766b4a64096cb08b6869bb69042fe82cb0a3dc2731e193e35edc420275736387190e0e690e07f9a81e0c975b2121e910f017704bb25

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+## 0.3.3 (2021-02-15)
+
+- Added ARM shared library for Mac
+
+## 0.3.2 (2020-12-27)
+
+- Updated NGT to 1.12.2
+
+## 0.3.1 (2020-05-17)
+
+- Updated NGT to 1.11.5
+- Improved error message when OpenMP not found on Mac
+
+## 0.3.0 (2020-03-25)
+
+- Updated NGT to 1.10.0
+- Added support for OpenMP on Mac
+- Create index in memory if no path specified
+- Added `normalized_angle` and `normalized_cosine`
+
+## 0.2.4 (2020-03-09)
+
+- Updated NGT to 1.9.1
+- Added support for passing an index to optimizers
+- Added `dimensions`, `distance_type`, `edge_size_for_creation`, `edge_size_for_search`, and `object_type` methods
+
 ## 0.2.3 (2020-03-08)
 
 - Added `load` method

data/NOTICE.txt

@@ -1,4 +1,5 @@
-Copyright 2019-2020 Andrew Kane
+Copyright 2016-2020 Yahoo Japan Corporation
+Copyright 2019-2021 Andrew Kane
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -2,7 +2,7 @@
 
 [NGT](https://github.com/yahoojapan/NGT) - high-speed approximate nearest neighbors - for Ruby
 
-[![Build Status](https://travis-ci.org/ankane/ngt.svg?branch=master)](https://travis-ci.org/ankane/ngt)
+[![Build Status](https://github.com/ankane/ngt/workflows/build/badge.svg?branch=master)](https://github.com/ankane/ngt/actions)
 
 ## Installation
 
@@ -12,7 +12,13 @@ Add this line to your application’s Gemfile:
 gem 'ngt'
 ```
 
-NGT is not available for Windows yet
+On Mac, also install OpenMP:
+
+```sh
+brew install libomp
+```
+
+NGT is not available for Windows
 
 ## Getting Started
 
@@ -84,7 +90,8 @@ Optimize the index
 
 ```ruby
 optimizer = Ngt::Optimizer.new(outgoing: 10, incoming: 120)
-optimizer.execute(path, new_path)
+optimizer.adjust_search_coefficients(index)
+optimizer.execute(index, new_path)
 ```
 
 ## Full Example
@@ -108,6 +115,38 @@ result.each do |res|
 end
 ```
 
+## Index Options
+
+Defaults shown below
+
+```ruby
+Ngt::Index.new(dimensions,
+  edge_size_for_creation: 10,
+  edge_size_for_search: 40,
+  object_type: :float, # :float, :integer
+  distance_type: :l2,  # :l1, :l2, :hamming, :angle, :cosine, :normalized_angle, :normalized_cosine, :jaccard
+  path: nil
+)
+```
+
+## Optimizer Options
+
+Defaults shown below
+
+```ruby
+Ngt::Optimizer.new(
+  outgoing: 10,
+  incoming: 120,
+  queries: 100,
+  low_accuracy_from: 0.3,
+  low_accuracy_to: 0.5,
+  high_accuracy_from: 0.8,
+  high_accuracy_to: 0.9,
+  gt_epsilon: 0.1,
+  merge: 0.2
+)
+```
+
 ## Data
 
 Data can be an array of arrays

data/lib/ngt.rb

@@ -16,7 +16,12 @@ module Ngt
   class << self
     attr_accessor :ffi_lib
   end
-  lib_name = FFI.map_library_name("ngt")
+  lib_name =
+    if RbConfig::CONFIG["host_os"] =~ /darwin/i && RbConfig::CONFIG["host_cpu"] =~ /arm/i
+      FFI.map_library_name("ngt.arm64")
+    else
+      FFI.map_library_name("ngt")
+    end
   vendor_lib = File.expand_path("../vendor/#{lib_name}", __dir__)
   self.ffi_lib = [vendor_lib]
 

data/lib/ngt/ffi.rb

@@ -5,7 +5,11 @@ module Ngt
     begin
       ffi_lib Ngt.ffi_lib
     rescue LoadError => e
-      raise e
+      if e.message.include?("Library not loaded: /usr/local/opt/libomp/lib/libomp.dylib") && e.message.include?("Reason: image not found")
+        raise LoadError, "OpenMP not found. Run `brew install libomp`"
+      else
+        raise e
+      end
     end
 
     # https://github.com/yahoojapan/NGT/blob/master/lib/NGT/Capi.h
@@ -19,8 +23,11 @@ module Ngt
         :distance, :float
     end
 
+    enum :distance_type, [:l1, :l2, :hamming, :angle, :cosine, :normalized_angle, :normalized_cosine, :jaccard]
+
     attach_function :ngt_open_index, %i[string pointer], :pointer
     attach_function :ngt_create_graph_and_tree, %i[string pointer pointer], :pointer
+    attach_function :ngt_create_graph_and_tree_in_memory, %i[pointer pointer], :pointer
     attach_function :ngt_create_property, %i[pointer], :pointer
     attach_function :ngt_save_index, %i[pointer string pointer], :bool
     attach_function :ngt_get_property, %i[pointer pointer pointer], :bool
@@ -38,20 +45,26 @@ module Ngt
     attach_function :ngt_set_property_distance_type_hamming, %i[pointer pointer], :bool
     attach_function :ngt_set_property_distance_type_jaccard, %i[pointer pointer], :bool
     attach_function :ngt_set_property_distance_type_cosine, %i[pointer pointer], :bool
-    attach_function :ngt_batch_insert_index, %i[pointer pointer uint32 pointer pointer], :bool
-    attach_function :ngt_create_index, %i[pointer uint32 pointer], :bool
-    attach_function :ngt_remove_index, %i[pointer int pointer], :bool
+    attach_function :ngt_set_property_distance_type_normalized_angle, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_normalized_cosine, %i[pointer pointer], :bool
     attach_function :ngt_insert_index, %i[pointer pointer uint32 pointer], :int
     attach_function :ngt_insert_index_as_float, %i[pointer pointer uint32 pointer], :int
     attach_function :ngt_create_empty_results, %i[pointer], :pointer
     attach_function :ngt_search_index, %i[pointer pointer int32 size_t float float pointer pointer], :bool
     attach_function :ngt_get_result_size, %i[pointer pointer], :uint32
     attach_function :ngt_get_result, %i[pointer uint32 pointer], ObjectDistance.by_value
+    attach_function :ngt_batch_insert_index, %i[pointer pointer uint32 pointer pointer], :bool
+    attach_function :ngt_create_index, %i[pointer uint32 pointer], :bool
+    attach_function :ngt_remove_index, %i[pointer int pointer], :bool
     attach_function :ngt_get_object_space, %i[pointer pointer], :pointer
     attach_function :ngt_get_object_as_float, %i[pointer int pointer], :pointer
     attach_function :ngt_get_object_as_integer, %i[pointer int pointer], :pointer
+    attach_function :ngt_destroy_results, %i[pointer], :void
     attach_function :ngt_destroy_property, %i[pointer], :void
     attach_function :ngt_close_index, %i[pointer], :void
+    attach_function :ngt_get_property_edge_size_for_creation, %i[pointer pointer], :int16
+    attach_function :ngt_get_property_edge_size_for_search, %i[pointer pointer], :int16
+    attach_function :ngt_get_property_distance_type, %i[pointer pointer], :distance_type
     attach_function :ngt_create_error_object, %i[], :pointer
     attach_function :ngt_get_error_string, %i[pointer], :string
     attach_function :ngt_destroy_error_object, %i[pointer], :void

data/lib/ngt/index.rb

@@ -2,26 +2,44 @@ module Ngt
   class Index
     include Utils
 
-    def initialize(path)
+    attr_reader :path
+
+    def initialize(index, path)
+      @index = index
       @path = path
+
       @error = FFI.ngt_create_error_object
-      @index = ffi(:ngt_open_index, path)
+      @property = ffi(:ngt_create_property)
+      ffi(:ngt_get_property, @index, @property)
 
-      property = ffi(:ngt_create_property)
-      ffi(:ngt_get_property, @index, property)
+      ObjectSpace.define_finalizer(self, self.class.finalize(@error, @index, @property))
+    end
 
-      @dimension = ffi(:ngt_get_property_dimension, property)
+    def dimensions
+      @dimensions ||= ffi(:ngt_get_property_dimension, @property)
+    end
 
-      object_type = ffi(:ngt_get_property_object_type, property)
-      @float = FFI.ngt_is_property_object_type_float(object_type)
+    def distance_type
+      @distance_type ||= ffi(:ngt_get_property_distance_type, @property)
+    end
 
-      @object_space = ffi(:ngt_get_object_space, @index)
+    def edge_size_for_creation
+      @edge_size_for_creation ||= ffi(:ngt_get_property_edge_size_for_creation, @property)
+    end
+
+    def edge_size_for_search
+      @edge_size_for_search ||= ffi(:ngt_get_property_edge_size_for_search, @property)
+    end
 
-      ObjectSpace.define_finalizer(self, self.class.finalize(@error))
+    def object_type
+      @object_type ||= begin
+        object_type = ffi(:ngt_get_property_object_type, @property)
+        FFI.ngt_is_property_object_type_float(object_type) ? :float : :integer
+      end
     end
 
     def insert(object)
-      ffi(:ngt_insert_index, @index, c_object(object.to_a), @dimension)
+      ffi(:ngt_insert_index, @index, c_object(object.to_a), dimensions)
     end
 
     def batch_insert(objects, num_threads: 8)
@@ -51,12 +69,12 @@ module Ngt
     end
 
     def object(id)
-      if float?
+      if object_type == :float
         res = ffi(:ngt_get_object_as_float, @object_space, id)
-        res.read_array_of_float(@dimension)
+        res.read_array_of_float(dimensions)
       else
         res = ffi(:ngt_get_object_as_integer, @object_space, id)
-        res.read_array_of_uint8(@dimension)
+        res.read_array_of_uint8(dimensions)
       end
     end
 
@@ -67,7 +85,7 @@ module Ngt
     def search(query, size: 20, epsilon: 0.1, radius: nil)
       radius ||= -1.0
       results = ffi(:ngt_create_empty_results)
-      ffi(:ngt_search_index, @index, c_object(query.to_a), @dimension, size, epsilon, radius, results)
+      ffi(:ngt_search_index, @index, c_object(query.to_a), dimensions, size, epsilon, radius, results)
       result_size = ffi(:ngt_get_result_size, results)
       ret = []
       result_size.times do |i|
@@ -78,82 +96,91 @@ module Ngt
         }
       end
       ret
+    ensure
+      FFI.ngt_destroy_results(results) if results
     end
 
     def save(path2 = nil, path: nil)
       warn "[ngt] Passing path as an option is deprecated - use an argument instead" if path
-      path ||= path2 || @path
-      ffi(:ngt_save_index, @index, path)
+      @path = path || path2 || @path || Dir.mktmpdir
+      ffi(:ngt_save_index, @index, @path)
     end
 
     def close
       FFI.ngt_close_index(@index)
     end
 
-    def self.new(dimension, path: nil, edge_size_for_creation: 10,
-        edge_size_for_search: 40, object_type: "Float", distance_type: "L2")
+    def self.new(dimensions, path: nil, edge_size_for_creation: 10,
+        edge_size_for_search: 40, object_type: :float, distance_type: :l2)
 
-      # called from load
-      return super(path) if path && dimension.nil?
+      error = FFI.ngt_create_error_object
 
-      # TODO remove in 0.3.0
-      create = dimension.is_a?(Integer) || path
-      unless create
+      # TODO remove in 0.4.0
+      if !dimensions.is_a?(Integer) && !path
         warn "[ngt] Passing a path to new is deprecated - use load instead"
-        return super(dimension)
+        path = dimensions
+        dimensions = nil
       end
 
-      path ||= Dir.mktmpdir
-      error = FFI.ngt_create_error_object
-      property = ffi(:ngt_create_property, error)
-      ffi(:ngt_set_property_dimension, property, dimension, error)
-      ffi(:ngt_set_property_edge_size_for_creation, property, edge_size_for_creation, error)
-      ffi(:ngt_set_property_edge_size_for_search, property, edge_size_for_search, error)
-
-      case object_type.to_s
-      when "Float", "float"
-        ffi(:ngt_set_property_object_type_float, property, error)
-      when "Integer", "integer"
-        ffi(:ngt_set_property_object_type_integer, property, error)
+      if path && dimensions.nil?
+        index = ffi(:ngt_open_index, path, error)
       else
-        raise ArgumentError, "Unknown object type: #{object_type}"
+        property = ffi(:ngt_create_property, error)
+        ffi(:ngt_set_property_dimension, property, dimensions, error)
+        ffi(:ngt_set_property_edge_size_for_creation, property, edge_size_for_creation, error)
+        ffi(:ngt_set_property_edge_size_for_search, property, edge_size_for_search, error)
+
+        case object_type.to_s.downcase
+        when "float"
+          ffi(:ngt_set_property_object_type_float, property, error)
+        when "integer"
+          ffi(:ngt_set_property_object_type_integer, property, error)
+        else
+          raise ArgumentError, "Unknown object type: #{object_type}"
+        end
+
+        case distance_type.to_s.downcase
+        when "l1"
+          ffi(:ngt_set_property_distance_type_l1, property, error)
+        when "l2"
+          ffi(:ngt_set_property_distance_type_l2, property, error)
+        when "angle"
+          ffi(:ngt_set_property_distance_type_angle, property, error)
+        when "hamming"
+          ffi(:ngt_set_property_distance_type_hamming, property, error)
+        when "jaccard"
+          ffi(:ngt_set_property_distance_type_jaccard, property, error)
+        when "cosine"
+          ffi(:ngt_set_property_distance_type_cosine, property, error)
+        when "normalized_angle"
+          ffi(:ngt_set_property_distance_type_normalized_angle, property, error)
+        when "normalized_cosine"
+          ffi(:ngt_set_property_distance_type_normalized_cosine, property, error)
+        else
+          raise ArgumentError, "Unknown distance type: #{distance_type}"
+        end
+
+        index =
+          if path
+            ffi(:ngt_create_graph_and_tree, path, property, error)
+          else
+            ffi(:ngt_create_graph_and_tree_in_memory, property, error)
+          end
       end
 
-      case distance_type.to_s
-      when "L1"
-        ffi(:ngt_set_property_distance_type_l1, property, error)
-      when "L2"
-        ffi(:ngt_set_property_distance_type_l2, property, error)
-      when "Angle"
-        ffi(:ngt_set_property_distance_type_angle, property, error)
-      when "Hamming"
-        ffi(:ngt_set_property_distance_type_hamming, property, error)
-      when "Jaccard"
-        ffi(:ngt_set_property_distance_type_jaccard, property, error)
-      when "Cosine"
-        ffi(:ngt_set_property_distance_type_cosine, property, error)
-      else
-        raise ArgumentError, "Unknown distance type: #{distance_type}"
-      end
-
-      index = ffi(:ngt_create_graph_and_tree, path, property, error)
-      FFI.ngt_close_index(index)
-      index = nil
-
-      super(path)
+      super(index, path)
     ensure
       FFI.ngt_destroy_error_object(error) if error
       FFI.ngt_destroy_property(property) if property
-      FFI.ngt_close_index(index) if index
     end
 
     def self.load(path)
       new(nil, path: path)
     end
 
-    def self.create(path, dimension, **options)
+    def self.create(path, dimensions, **options)
       warn "[ngt] create is deprecated - use new instead"
-      new(dimension, path: path, **options)
+      new(dimensions, path: path, **options)
     end
 
     # private
@@ -161,11 +188,12 @@ module Ngt
       Utils.ffi(*args)
     end
 
-    def self.finalize(error)
+    def self.finalize(error, index, property)
       # must use proc instead of stabby lambda
       proc do
-        # TODO clean-up more objects
         FFI.ngt_destroy_error_object(error)
+        FFI.ngt_close_index(index)
+        FFI.ngt_destroy_property(property)
       end
     end
 
@@ -175,10 +203,6 @@ module Ngt
       defined?(Numo::NArray) && data.is_a?(Numo::NArray)
     end
 
-    def float?
-      @float
-    end
-
     def c_object(object)
       c_object = ::FFI::MemoryPointer.new(:double, object.size)
       c_object.write_array_of_double(object)

data/lib/ngt/optimizer.rb

@@ -15,11 +15,11 @@ module Ngt
     end
 
     def execute(in_index_path, out_index_path)
-      ffi(:ngt_optimizer_execute, @optimizer, in_index_path, out_index_path)
+      ffi(:ngt_optimizer_execute, @optimizer, path(in_index_path), out_index_path)
     end
 
     def adjust_search_coefficients(index_path)
-      ffi(:ngt_optimizer_adjust_search_coefficients, @optimizer, index_path)
+      ffi(:ngt_optimizer_adjust_search_coefficients, @optimizer, path(index_path))
     end
 
     def self.finalize(optimizer, error)
@@ -29,5 +29,16 @@ module Ngt
         FFI.ngt_destroy_error_object(error)
       end
     end
+
+    private
+
+    def path(obj)
+      if obj.is_a?(Ngt::Index)
+        raise ArgumentError, "Index not saved" unless obj.path
+        obj.path
+      else
+        obj
+      end
+    end
   end
 end

data/lib/ngt/version.rb

@@ -1,3 +1,3 @@
 module Ngt
-  VERSION = "0.2.3"
+  VERSION = "0.3.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ngt
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.3
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2021-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -24,64 +24,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-- !ruby/object:Gem::Dependency
-  name: numo-narray
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -97,13 +41,14 @@ files:
 - lib/ngt/utils.rb
 - lib/ngt/version.rb
 - vendor/LICENSE
+- vendor/libngt.arm64.dylib
 - vendor/libngt.dylib
 - vendor/libngt.so
 homepage: https://github.com/ankane/ngt
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -118,8 +63,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: High-speed approximate nearest neighbors for Ruby
 test_files: []