ngt 0.2.2 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 349ccec555d0066d9ccf024a40cf0ecaa478ac628d7cc1240ac2ffa288735a29
-  data.tar.gz: 6242793a35080d8b52c650ebb3b24bee762b36bcf5035bd99263ac2657f3f12c
+  metadata.gz: 850afe334c78ec69bfb7c975629336a615d92bb2d03efade8ffe148ed4d1dfc9
+  data.tar.gz: 3bc13fbbc90c7323dd4322f3492d355363d16f28baf5bd0b7b88fc35e44b807b
 SHA512:
-  metadata.gz: 9fe092180d1a68e7843089a2a6c1fda1475ca2ad05d302209db501704994b694a5e94ac34daa9898d26efdf739c775741b79e24bd5aa3fbcdd58fb28063c1bef
-  data.tar.gz: 866b54645ab021c69b50967ae5952d159d4e45ff850f450500c550c154a5f9b224ca45e36ead4f4d3ed96d43a26d3ad601040ba91a31671d815789161a981a83
+  metadata.gz: '08cb7235b719ef9bffe6b0557538dd9464d6bbd0dea617d260de8cb57756f4388c65d1d83d5265f6c2036291769186cdb845a7b02e5838af0c964ab0b1dd0752'
+  data.tar.gz: ccfd9d4139d1297be48fa06b8da41055dfbd7d4802062bdad3730007d7d86e6a4262a731e0d27a793c79cce2fb01a60807a975b24619ae7b35c8edf8425b1b2c

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+## 0.3.2 (2020-12-27)
+
+- Updated NGT to 1.12.2
+
+## 0.3.1 (2020-05-17)
+
+- Updated NGT to 1.11.5
+- Improved error message when OpenMP not found on Mac
+
+## 0.3.0 (2020-03-25)
+
+- Updated NGT to 1.10.0
+- Added support for OpenMP on Mac
+- Create index in memory if no path specified
+- Added `normalized_angle` and `normalized_cosine`
+
+## 0.2.4 (2020-03-09)
+
+- Updated NGT to 1.9.1
+- Added support for passing an index to optimizers
+- Added `dimensions`, `distance_type`, `edge_size_for_creation`, `edge_size_for_search`, and `object_type` methods
+
+## 0.2.3 (2020-03-08)
+
+- Added `load` method
+- Deprecated `create` and passing path to `new`
+
 ## 0.2.2 (2020-02-11)
 
 - Fixed `Could not find NGT` error on some Linux platforms
@@ -11,6 +38,7 @@
 - Changed to Apache 2.0 license to match NGT
 - Added shared libraries
 - Added optimizer
+- Improved performance of `batch_insert` for Numo
 
 ## 0.1.1 (2019-10-27)
 

data/NOTICE.txt

@@ -1,3 +1,4 @@
+Copyright 2016-2020 Yahoo Japan Corporation
 Copyright 2019-2020 Andrew Kane
 
 Licensed under the Apache License, Version 2.0 (the "License");

data/README.md

@@ -2,7 +2,7 @@
 
 [NGT](https://github.com/yahoojapan/NGT) - high-speed approximate nearest neighbors - for Ruby
 
-[![Build Status](https://travis-ci.org/ankane/ngt.svg?branch=master)](https://travis-ci.org/ankane/ngt)
+[![Build Status](https://github.com/ankane/ngt/workflows/build/badge.svg?branch=master)](https://github.com/ankane/ngt/actions)
 
 ## Installation
 
@@ -12,7 +12,13 @@ Add this line to your application’s Gemfile:
 gem 'ngt'
 ```
 
-NGT is not available for Windows yet
+On Mac, also install OpenMP:
+
+```sh
+brew install libomp
+```
+
+NGT is not available for Windows
 
 ## Getting Started
 
@@ -29,7 +35,7 @@ objects = [
 Create an index
 
 ```ruby
-index = Ngt::Index.create(path, dimensions)
+index = Ngt::Index.new(dimensions)
 ```
 
 Insert objects
@@ -47,13 +53,13 @@ index.search(query, size: 3)
 Save the index
 
 ```ruby
-index.save
+index.save(path)
 ```
 
 Load an index
 
 ```ruby
-index = Ngt::Index.new(path)
+index = Ngt::Index.load(path)
 ```
 
 Get an object by id
@@ -84,7 +90,8 @@ Optimize the index
 
 ```ruby
 optimizer = Ngt::Optimizer.new(outgoing: 10, incoming: 120)
-optimizer.execute(path, new_path)
+optimizer.adjust_search_coefficients(index)
+optimizer.execute(index, new_path)
 ```
 
 ## Full Example
@@ -96,9 +103,8 @@ objects = []
   objects << dim.times.map { rand(100) }
 end
 
-index = Ngt::Index.create("tmp", dim)
+index = Ngt::Index.new(dim)
 index.batch_insert(objects)
-index.save
 
 query = objects[0]
 result = index.search(query, size: 3)
@@ -109,6 +115,38 @@ result.each do |res|
 end
 ```
 
+## Index Options
+
+Defaults shown below
+
+```ruby
+Ngt::Index.new(dimensions,
+  edge_size_for_creation: 10,
+  edge_size_for_search: 40,
+  object_type: :float, # :float, :integer
+  distance_type: :l2,  # :l1, :l2, :hamming, :angle, :cosine, :normalized_angle, :normalized_cosine, :jaccard
+  path: nil
+)
+```
+
+## Optimizer Options
+
+Defaults shown below
+
+```ruby
+Ngt::Optimizer.new(
+  outgoing: 10,
+  incoming: 120,
+  queries: 100,
+  low_accuracy_from: 0.3,
+  low_accuracy_to: 0.5,
+  high_accuracy_from: 0.8,
+  high_accuracy_to: 0.9,
+  gt_epsilon: 0.1,
+  merge: 0.2
+)
+```
+
 ## Data
 
 Data can be an array of arrays

data/lib/ngt.rb

@@ -1,6 +1,9 @@
 # dependencies
 require "ffi"
 
+# stdlib
+require "tmpdir"
+
 # modules
 require "ngt/utils"
 require "ngt/index"

data/lib/ngt/ffi.rb

@@ -5,7 +5,11 @@ module Ngt
     begin
       ffi_lib Ngt.ffi_lib
     rescue LoadError => e
-      raise e
+      if e.message.include?("Library not loaded: /usr/local/opt/libomp/lib/libomp.dylib") && e.message.include?("Reason: image not found")
+        raise LoadError, "OpenMP not found. Run `brew install libomp`"
+      else
+        raise e
+      end
     end
 
     # https://github.com/yahoojapan/NGT/blob/master/lib/NGT/Capi.h
@@ -19,8 +23,11 @@ module Ngt
         :distance, :float
     end
 
+    enum :distance_type, [:l1, :l2, :hamming, :angle, :cosine, :normalized_angle, :normalized_cosine, :jaccard]
+
     attach_function :ngt_open_index, %i[string pointer], :pointer
     attach_function :ngt_create_graph_and_tree, %i[string pointer pointer], :pointer
+    attach_function :ngt_create_graph_and_tree_in_memory, %i[pointer pointer], :pointer
     attach_function :ngt_create_property, %i[pointer], :pointer
     attach_function :ngt_save_index, %i[pointer string pointer], :bool
     attach_function :ngt_get_property, %i[pointer pointer pointer], :bool
@@ -38,20 +45,26 @@ module Ngt
     attach_function :ngt_set_property_distance_type_hamming, %i[pointer pointer], :bool
     attach_function :ngt_set_property_distance_type_jaccard, %i[pointer pointer], :bool
     attach_function :ngt_set_property_distance_type_cosine, %i[pointer pointer], :bool
-    attach_function :ngt_batch_insert_index, %i[pointer pointer uint32 pointer pointer], :bool
-    attach_function :ngt_create_index, %i[pointer uint32 pointer], :bool
-    attach_function :ngt_remove_index, %i[pointer int pointer], :bool
+    attach_function :ngt_set_property_distance_type_normalized_angle, %i[pointer pointer], :bool
+    attach_function :ngt_set_property_distance_type_normalized_cosine, %i[pointer pointer], :bool
     attach_function :ngt_insert_index, %i[pointer pointer uint32 pointer], :int
     attach_function :ngt_insert_index_as_float, %i[pointer pointer uint32 pointer], :int
     attach_function :ngt_create_empty_results, %i[pointer], :pointer
     attach_function :ngt_search_index, %i[pointer pointer int32 size_t float float pointer pointer], :bool
     attach_function :ngt_get_result_size, %i[pointer pointer], :uint32
     attach_function :ngt_get_result, %i[pointer uint32 pointer], ObjectDistance.by_value
+    attach_function :ngt_batch_insert_index, %i[pointer pointer uint32 pointer pointer], :bool
+    attach_function :ngt_create_index, %i[pointer uint32 pointer], :bool
+    attach_function :ngt_remove_index, %i[pointer int pointer], :bool
     attach_function :ngt_get_object_space, %i[pointer pointer], :pointer
     attach_function :ngt_get_object_as_float, %i[pointer int pointer], :pointer
     attach_function :ngt_get_object_as_integer, %i[pointer int pointer], :pointer
+    attach_function :ngt_destroy_results, %i[pointer], :void
     attach_function :ngt_destroy_property, %i[pointer], :void
     attach_function :ngt_close_index, %i[pointer], :void
+    attach_function :ngt_get_property_edge_size_for_creation, %i[pointer pointer], :int16
+    attach_function :ngt_get_property_edge_size_for_search, %i[pointer pointer], :int16
+    attach_function :ngt_get_property_distance_type, %i[pointer pointer], :distance_type
     attach_function :ngt_create_error_object, %i[], :pointer
     attach_function :ngt_get_error_string, %i[pointer], :string
     attach_function :ngt_destroy_error_object, %i[pointer], :void

data/lib/ngt/index.rb

@@ -2,26 +2,44 @@ module Ngt
   class Index
     include Utils
 
-    def initialize(path)
+    attr_reader :path
+
+    def initialize(index, path)
+      @index = index
       @path = path
+
       @error = FFI.ngt_create_error_object
-      @index = ffi(:ngt_open_index, path)
+      @property = ffi(:ngt_create_property)
+      ffi(:ngt_get_property, @index, @property)
 
-      property = ffi(:ngt_create_property)
-      ffi(:ngt_get_property, @index, property)
+      ObjectSpace.define_finalizer(self, self.class.finalize(@error, @index, @property))
+    end
+
+    def dimensions
+      @dimensions ||= ffi(:ngt_get_property_dimension, @property)
+    end
 
-      @dimension = ffi(:ngt_get_property_dimension, property)
+    def distance_type
+      @distance_type ||= ffi(:ngt_get_property_distance_type, @property)
+    end
 
-      object_type = ffi(:ngt_get_property_object_type, property)
-      @float = FFI.ngt_is_property_object_type_float(object_type)
+    def edge_size_for_creation
+      @edge_size_for_creation ||= ffi(:ngt_get_property_edge_size_for_creation, @property)
+    end
 
-      @object_space = ffi(:ngt_get_object_space, @index)
+    def edge_size_for_search
+      @edge_size_for_search ||= ffi(:ngt_get_property_edge_size_for_search, @property)
+    end
 
-      ObjectSpace.define_finalizer(self, self.class.finalize(@error))
+    def object_type
+      @object_type ||= begin
+        object_type = ffi(:ngt_get_property_object_type, @property)
+        FFI.ngt_is_property_object_type_float(object_type) ? :float : :integer
+      end
     end
 
     def insert(object)
-      ffi(:ngt_insert_index, @index, c_object(object.to_a), @dimension)
+      ffi(:ngt_insert_index, @index, c_object(object.to_a), dimensions)
     end
 
     def batch_insert(objects, num_threads: 8)
@@ -51,12 +69,12 @@ module Ngt
     end
 
     def object(id)
-      if float?
+      if object_type == :float
         res = ffi(:ngt_get_object_as_float, @object_space, id)
-        res.read_array_of_float(@dimension)
+        res.read_array_of_float(dimensions)
       else
         res = ffi(:ngt_get_object_as_integer, @object_space, id)
-        res.read_array_of_uint8(@dimension)
+        res.read_array_of_uint8(dimensions)
       end
     end
 
@@ -67,7 +85,7 @@ module Ngt
     def search(query, size: 20, epsilon: 0.1, radius: nil)
       radius ||= -1.0
       results = ffi(:ngt_create_empty_results)
-      ffi(:ngt_search_index, @index, c_object(query.to_a), @dimension, size, epsilon, radius, results)
+      ffi(:ngt_search_index, @index, c_object(query.to_a), dimensions, size, epsilon, radius, results)
       result_size = ffi(:ngt_get_result_size, results)
       ret = []
       result_size.times do |i|
@@ -78,61 +96,91 @@ module Ngt
         }
       end
       ret
+    ensure
+      FFI.ngt_destroy_results(results) if results
     end
 
-    def save(path: nil)
-      path ||= @path
-      ffi(:ngt_save_index, @index, path)
+    def save(path2 = nil, path: nil)
+      warn "[ngt] Passing path as an option is deprecated - use an argument instead" if path
+      @path = path || path2 || @path || Dir.mktmpdir
+      ffi(:ngt_save_index, @index, @path)
     end
 
     def close
       FFI.ngt_close_index(@index)
     end
 
-    def self.create(path, dimension, edge_size_for_creation: 10,
-        edge_size_for_search: 40, object_type: "Float", distance_type: "L2")
+    def self.new(dimensions, path: nil, edge_size_for_creation: 10,
+        edge_size_for_search: 40, object_type: :float, distance_type: :l2)
 
       error = FFI.ngt_create_error_object
-      property = ffi(:ngt_create_property, error)
-      ffi(:ngt_set_property_dimension, property, dimension, error)
-      ffi(:ngt_set_property_edge_size_for_creation, property, edge_size_for_creation, error)
-      ffi(:ngt_set_property_edge_size_for_search, property, edge_size_for_search, error)
-
-      case object_type.to_s
-      when "Float", "float"
-        ffi(:ngt_set_property_object_type_float, property, error)
-      when "Integer", "integer"
-        ffi(:ngt_set_property_object_type_integer, property, error)
-      else
-        raise ArgumentError, "Unknown object type: #{object_type}"
+
+      # TODO remove in 0.4.0
+      if !dimensions.is_a?(Integer) && !path
+        warn "[ngt] Passing a path to new is deprecated - use load instead"
+        path = dimensions
+        dimensions = nil
       end
 
-      case distance_type.to_s
-      when "L1"
-        ffi(:ngt_set_property_distance_type_l1, property, error)
-      when "L2"
-        ffi(:ngt_set_property_distance_type_l2, property, error)
-      when "Angle"
-        ffi(:ngt_set_property_distance_type_angle, property, error)
-      when "Hamming"
-        ffi(:ngt_set_property_distance_type_hamming, property, error)
-      when "Jaccard"
-        ffi(:ngt_set_property_distance_type_jaccard, property, error)
-      when "Cosine"
-        ffi(:ngt_set_property_distance_type_cosine, property, error)
+      if path && dimensions.nil?
+        index = ffi(:ngt_open_index, path, error)
       else
-        raise ArgumentError, "Unknown distance type: #{distance_type}"
+        property = ffi(:ngt_create_property, error)
+        ffi(:ngt_set_property_dimension, property, dimensions, error)
+        ffi(:ngt_set_property_edge_size_for_creation, property, edge_size_for_creation, error)
+        ffi(:ngt_set_property_edge_size_for_search, property, edge_size_for_search, error)
+
+        case object_type.to_s.downcase
+        when "float"
+          ffi(:ngt_set_property_object_type_float, property, error)
+        when "integer"
+          ffi(:ngt_set_property_object_type_integer, property, error)
+        else
+          raise ArgumentError, "Unknown object type: #{object_type}"
+        end
+
+        case distance_type.to_s.downcase
+        when "l1"
+          ffi(:ngt_set_property_distance_type_l1, property, error)
+        when "l2"
+          ffi(:ngt_set_property_distance_type_l2, property, error)
+        when "angle"
+          ffi(:ngt_set_property_distance_type_angle, property, error)
+        when "hamming"
+          ffi(:ngt_set_property_distance_type_hamming, property, error)
+        when "jaccard"
+          ffi(:ngt_set_property_distance_type_jaccard, property, error)
+        when "cosine"
+          ffi(:ngt_set_property_distance_type_cosine, property, error)
+        when "normalized_angle"
+          ffi(:ngt_set_property_distance_type_normalized_angle, property, error)
+        when "normalized_cosine"
+          ffi(:ngt_set_property_distance_type_normalized_cosine, property, error)
+        else
+          raise ArgumentError, "Unknown distance type: #{distance_type}"
+        end
+
+        index =
+          if path
+            ffi(:ngt_create_graph_and_tree, path, property, error)
+          else
+            ffi(:ngt_create_graph_and_tree_in_memory, property, error)
+          end
       end
 
-      index = ffi(:ngt_create_graph_and_tree, path, property, error)
-      FFI.ngt_close_index(index)
-      index = nil
-
-      Index.new(path)
+      super(index, path)
     ensure
       FFI.ngt_destroy_error_object(error) if error
       FFI.ngt_destroy_property(property) if property
-      FFI.ngt_close_index(index) if index
+    end
+
+    def self.load(path)
+      new(nil, path: path)
+    end
+
+    def self.create(path, dimensions, **options)
+      warn "[ngt] create is deprecated - use new instead"
+      new(dimensions, path: path, **options)
     end
 
     # private
@@ -140,11 +188,12 @@ module Ngt
       Utils.ffi(*args)
     end
 
-    def self.finalize(error)
+    def self.finalize(error, index, property)
       # must use proc instead of stabby lambda
       proc do
-        # TODO clean-up more objects
         FFI.ngt_destroy_error_object(error)
+        FFI.ngt_close_index(index)
+        FFI.ngt_destroy_property(property)
       end
     end
 
@@ -154,10 +203,6 @@ module Ngt
       defined?(Numo::NArray) && data.is_a?(Numo::NArray)
     end
 
-    def float?
-      @float
-    end
-
     def c_object(object)
       c_object = ::FFI::MemoryPointer.new(:double, object.size)
       c_object.write_array_of_double(object)

data/lib/ngt/optimizer.rb

@@ -15,11 +15,11 @@ module Ngt
     end
 
     def execute(in_index_path, out_index_path)
-      ffi(:ngt_optimizer_execute, @optimizer, in_index_path, out_index_path)
+      ffi(:ngt_optimizer_execute, @optimizer, path(in_index_path), out_index_path)
     end
 
     def adjust_search_coefficients(index_path)
-      ffi(:ngt_optimizer_adjust_search_coefficients, @optimizer, index_path)
+      ffi(:ngt_optimizer_adjust_search_coefficients, @optimizer, path(index_path))
     end
 
     def self.finalize(optimizer, error)
@@ -29,5 +29,16 @@ module Ngt
         FFI.ngt_destroy_error_object(error)
       end
     end
+
+    private
+
+    def path(obj)
+      if obj.is_a?(Ngt::Index)
+        raise ArgumentError, "Index not saved" unless obj.path
+        obj.path
+      else
+        obj
+      end
+    end
   end
 end

data/lib/ngt/version.rb

@@ -1,3 +1,3 @@
 module Ngt
-  VERSION = "0.2.2"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ngt
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-12 00:00:00.000000000 Z
+date: 2020-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -80,7 +80,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email: andrew@chartkick.com
 executables: []
 extensions: []
@@ -103,7 +103,7 @@ homepage: https://github.com/ankane/ngt
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -118,8 +118,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: High-speed approximate nearest neighbors for Ruby
 test_files: []