mitie 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea3ef115016c59ecb496ffbbe13c4ac3a2ffda6acf9392ac423103b9c3cfe634
-  data.tar.gz: 6eb77dd514ba3c08c30e1216921cd83619f206a658018c1d4522c598e175e8b2
+  metadata.gz: e0ceaa2d4609a2a1b3b4056d67d88a1b0f55616ac8fd1a0509f070c352a96ea1
+  data.tar.gz: 0b90eba1027ca5a46a405a97411d2b5fe193d20666ee8c8bad347bb2c79c225b
 SHA512:
-  metadata.gz: 682fb3ea1c0be1889f2e1e177204309ea7b6d2989834a4d2bae49ddf567e309b9fef9ea872746626590ec06b9aabb455067d88e696ea3a4f59a9b785b43819c9
-  data.tar.gz: 4260a6dff4eb613278468d9fff2ef19f99f49bed81571781b4d2ac886f39bed9356a4a20a9220fa2b31e8c12965bffac5e86b40318b28aa450ebd60a3c53c3cd
+  metadata.gz: 8281e51659e08157d305535f3cd242082d4173368c36fa167716771b008a82233d02e43d1015c39e27c4c704ae0ec53935834eb0fefb71dac3b515962987d7eb
+  data.tar.gz: 96d3564684f8197651f93238876f2bbeb69f860a8a373f36dcafb578db698555153b320f2bd0c799cd8d5d939ad05c3746c59e907d4ad62cfa4917fbb7ec2313

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.1.5 (2021-01-29)
+
+- Fixed issue with multibyte characters
+
+## 0.1.4 (2020-12-28)
+
+- Added ARM shared library for Mac
+
+## 0.1.3 (2020-12-04)
+
+- Added support for custom tokenization
+
+## 0.1.2 (2020-09-14)
+
+- Added binary relation detection
+- Added `Document` class
+
+## 0.1.1 (2020-09-14)
+
+- Added shared libraries
+- Improved error message when model file does not exist
+
 ## 0.1.0 (2020-09-14)
 
 - First release

data/README.md

@@ -1,14 +1,13 @@
 # MITIE
 
-[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition - for Ruby
+[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition and binary relation detection - for Ruby
 
-## Installation
+- Finds people, organizations, and locations in text
+- Detects relationships between entities, like `PERSON` was born in `LOCATION`
 
-First, install MITIE. For Homebrew, use:
+[![Build Status](https://github.com/ankane/mitie/workflows/build/badge.svg?branch=master)](https://github.com/ankane/mitie/actions)
 
-```sh
-brew install mitie
-```
+## Installation
 
 Add this line to your application’s Gemfile:
 
@@ -24,44 +23,44 @@ And download the pre-trained model for your language:
 
 ## Getting Started
 
-Get your text
+Load an NER model
 
 ```ruby
-text = "Nat Friedman is the CEO of GitHub, which is headquartered in San Francisco"
+model = Mitie::NER.new("ner_model.dat")
 ```
 
-Load an NER model
+Create a document
 
 ```ruby
-model = Mitie::NER.new("ner_model.dat")
+doc = model.doc("Nat works at GitHub in San Francisco")
 ```
 
 Get entities
 
 ```ruby
-model.entities(text)
+doc.entities
 ```
 
 This returns
 
 ```ruby
 [
-  {text: "Nat Friedman",  tag: "PERSON",       score: 1.099661347535191, offset: 0},
-  {text: "GitHub",        tag: "ORGANIZATION", score: 0.344641651251650, offset: 27},
-  {text: "San Francisco", tag: "LOCATION",     score: 1.428241888939011, offset: 61}
+  {text: "Nat",           tag: "PERSON",       score: 0.3112371212688382, offset: 0},
+  {text: "GitHub",        tag: "ORGANIZATION", score: 0.5660115198329334, offset: 13},
+  {text: "San Francisco", tag: "LOCATION",     score: 1.3890524313885309, offset: 23}
 ]
 ```
 
 Get tokens
 
 ```ruby
-model.tokens(text)
+doc.tokens
 ```
 
 Get tokens and their offset
 
 ```ruby
-model.tokens_with_offset(text)
+doc.tokens_with_offset
 ```
 
 Get all tags for a model
@@ -70,6 +69,40 @@ Get all tags for a model
 model.tags
 ```
 
+## Binary Relation Detection
+
+Detect relationships betweens two entities, like:
+
+- `PERSON` was born in `LOCATION`
+- `ORGANIZATION` was founded in `LOCATION`
+- `FILM` was directed by `PERSON`
+
+There are 21 detectors for English. You can find them in the `binary_relations` directory in the model download.
+
+Load a detector
+
+```ruby
+detector = Mitie::BinaryRelationDetector.new("rel_classifier_organization.organization.place_founded.svm")
+```
+
+And create a document
+
+```ruby
+doc = model.doc("Shopify was founded in Ottawa")
+```
+
+Get relations
+
+```ruby
+detector.relations(doc)
+```
+
+This returns
+
+```ruby
+[{first: "Shopify", second: "Ottawa", score: 0.17649169745814464}]
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/mitie/blob/master/CHANGELOG.md)
@@ -89,5 +122,8 @@ To get started with development:
 git clone https://github.com/ankane/mitie.git
 cd mitie
 bundle install
-MITIE_NER_PATH=path/to/ner_model.dat bundle exec rake test
+bundle exec rake vendor:all
+
+export MITIE_MODELS_PATH=path/to/MITIE-models/english
+bundle exec rake test
 ```

data/lib/mitie.rb

@@ -2,6 +2,8 @@
 require "fiddle/import"
 
 # modules
+require "mitie/binary_relation_detector"
+require "mitie/document"
 require "mitie/ner"
 require "mitie/version"
 
@@ -11,14 +13,18 @@ module Mitie
   class << self
     attr_accessor :ffi_lib
   end
-  self.ffi_lib =
+  lib_name =
     if Gem.win_platform?
-      ["mitie.dll"]
+      "mitie.dll"
+    elsif RbConfig::CONFIG["arch"] =~ /arm64-darwin/i
+      "libmitie.arm64.dylib"
     elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
-      ["libmitie.dylib"]
+      "libmitie.dylib"
     else
-      ["libmitie.so"]
+      "libmitie.so"
     end
+  vendor_lib = File.expand_path("../vendor/#{lib_name}", __dir__)
+  self.ffi_lib = [vendor_lib]
 
   # friendlier error message
   autoload :FFI, "mitie/ffi"

data/lib/mitie/binary_relation_detector.rb

@@ -0,0 +1,62 @@
+module Mitie
+  class BinaryRelationDetector
+    def initialize(path)
+      # better error message
+      raise ArgumentError, "File does not exist" unless File.exist?(path)
+      @pointer = FFI.mitie_load_binary_relation_detector(path)
+      ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
+    end
+
+    def name
+      FFI.mitie_binary_relation_detector_name_string(pointer).to_s
+    end
+
+    def relations(doc)
+      raise ArgumentError, "Expected Mitie::Document, not #{doc.class.name}" unless doc.is_a?(Document)
+
+      entities = doc.entities
+      combinations = []
+      (entities.size - 1).times do |i|
+        combinations << [entities[i], entities[i + 1]]
+        combinations << [entities[i + 1], entities[i]]
+      end
+
+      relations = []
+      combinations.each do |entity1, entity2|
+        relation =
+          FFI.mitie_extract_binary_relation(
+            doc.model.pointer,
+            doc.send(:tokens_ptr),
+            entity1[:token_index],
+            entity1[:token_length],
+            entity2[:token_index],
+            entity2[:token_length]
+          )
+
+        score_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
+        status = FFI.mitie_classify_binary_relation(pointer, relation, score_ptr)
+        raise "Bad status: #{status}" if status != 0
+        score = score_ptr.to_s(Fiddle::SIZEOF_DOUBLE).unpack1("d")
+        if score > 0
+          relations << {
+            first: entity1[:text],
+            second: entity2[:text],
+            score: score
+          }
+        end
+      end
+      relations
+    end
+
+    private
+
+    def pointer
+      @pointer
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+  end
+end

data/lib/mitie/document.rb

@@ -0,0 +1,115 @@
+module Mitie
+  class Document
+    attr_reader :model, :text
+
+    def initialize(model, text)
+      @model = model
+      @text = text
+    end
+
+    def tokens
+      @tokens ||= tokens_with_offset.map(&:first)
+    end
+
+    def tokens_with_offset
+      @tokens_with_offset ||= begin
+        if text.is_a?(Array)
+          # offsets are unknown when given tokens
+          text.map { |v| [v, nil] }
+        else
+          i = 0
+          tokens = []
+          loop do
+            token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+            break if token.null?
+            offset = (offsets_ptr.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
+            tokens << [token.to_s.force_encoding(text.encoding), offset]
+            i += 1
+          end
+          tokens
+        end
+      end
+    end
+
+    def entities
+      @entities ||= begin
+        begin
+          entities = []
+          tokens = tokens_with_offset
+          detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
+          num_detections = FFI.mitie_ner_get_num_detections(detections)
+          num_detections.times do |i|
+            pos = FFI.mitie_ner_get_detection_position(detections, i)
+            len = FFI.mitie_ner_get_detection_length(detections, i)
+            tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
+            score = FFI.mitie_ner_get_detection_score(detections, i)
+            tok = tokens[pos, len]
+            offset = tok[0][1]
+
+            entity = {}
+            if offset
+              finish = tok[-1][1] + tok[-1][0].bytesize
+              entity[:text] = text.byteslice(offset...finish)
+            else
+              entity[:text] = tok.map(&:first)
+            end
+            entity[:tag] = tag
+            entity[:score] = score
+            entity[:offset] = offset if offset
+            entity[:token_index] = pos
+            entity[:token_length] = len
+            entities << entity
+          end
+          entities
+        ensure
+          FFI.mitie_free(detections) if detections
+        end
+      end
+    end
+
+    private
+
+    def pointer
+      model.pointer
+    end
+
+    def tokens_ptr
+      tokenize[0]
+    end
+
+    def offsets_ptr
+      tokenize[1]
+    end
+
+    def tokenize
+      @tokenize ||= begin
+        if text.is_a?(Array)
+          # malloc uses memset to set all bytes to 0
+          tokens_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP * (text.size + 1))
+          text.size.times do |i|
+            tokens_ptr[i * Fiddle::SIZEOF_VOIDP, Fiddle::SIZEOF_VOIDP] = Fiddle::Pointer.to_ptr(text[i]).ref
+          end
+          [tokens_ptr, nil]
+        else
+          offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
+          tokens_ptr = FFI.mitie_tokenize_with_offsets(text, offsets_ptr)
+
+          ObjectSpace.define_finalizer(tokens_ptr, self.class.finalize(tokens_ptr))
+          ObjectSpace.define_finalizer(offsets_ptr, self.class.finalize_ptr(offsets_ptr))
+
+          [tokens_ptr, offsets_ptr]
+        end
+      end
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+
+    def self.finalize_ptr(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer.ptr) }
+    end
+  end
+end

data/lib/mitie/ffi.rb

@@ -25,5 +25,11 @@ module Mitie
     extern "unsigned long mitie_ner_get_detection_tag(const mitie_named_entity_detections* dets, unsigned long idx)"
     extern "const char* mitie_ner_get_detection_tagstr(const mitie_named_entity_detections* dets, unsigned long idx)"
     extern "double mitie_ner_get_detection_score(const mitie_named_entity_detections* dets, unsigned long idx)"
+
+    extern "mitie_binary_relation_detector* mitie_load_binary_relation_detector(const char* filename)"
+    extern "const char* mitie_binary_relation_detector_name_string(const mitie_binary_relation_detector* detector)"
+    extern "int mitie_entities_overlap(unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "mitie_binary_relation* mitie_extract_binary_relation(const mitie_named_entity_extractor* ner, char** tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "int mitie_classify_binary_relation(const mitie_binary_relation_detector* detector, const mitie_binary_relation* relation, double* score)"
   end
 end

data/lib/mitie/ner.rb

@@ -1,6 +1,10 @@
 module Mitie
   class NER
+    attr_reader :pointer
+
     def initialize(path)
+      # better error message
+      raise ArgumentError, "File does not exist" unless File.exist?(path)
       @pointer = FFI.mitie_load_named_entity_extractor(path)
       ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
     end
@@ -11,76 +15,20 @@ module Mitie
       end
     end
 
-    def tokens(text)
-      tokens = []
-      ptr = FFI.mitie_tokenize(text)
-      i = 0
-      loop do
-        token = (ptr + i * Fiddle::SIZEOF_VOIDP).ptr
-        break if token.null?
-        tokens << token.to_s.force_encoding(text.encoding)
-        i += 1
-      end
-      tokens
-    ensure
-      FFI.mitie_free(ptr) if ptr
-    end
-
-    def tokens_with_offset(text)
-      tokens, ptr = tokens_with_offset_with_ptr(text)
-      tokens
-    ensure
-      FFI.mitie_free(ptr) if ptr
+    def doc(text)
+      Document.new(self, text)
     end
 
     def entities(text)
-      entities = []
-      tokens, tokens_ptr = tokens_with_offset_with_ptr(text)
-      detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
-      num_detections = FFI.mitie_ner_get_num_detections(detections)
-      num_detections.times do |i|
-        pos = FFI.mitie_ner_get_detection_position(detections, i)
-        len = FFI.mitie_ner_get_detection_length(detections, i)
-        tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
-        score = FFI.mitie_ner_get_detection_score(detections, i)
-        tok = tokens[pos, len]
-        offset = tok[0][1]
-        finish = tok[-1][1] + tok[-1][0].size
-        entities << {
-          text: text[offset...finish],
-          tag: tag,
-          score: score,
-          offset: offset
-        }
-      end
-      entities
-    ensure
-      FFI.mitie_free(tokens_ptr) if tokens_ptr
-      FFI.mitie_free(detections) if detections
+      doc(text).entities
     end
 
-    private
-
-    def pointer
-      @pointer
+    def tokens(text)
+      doc(text).tokens
     end
 
-    def tokens_with_offset_with_ptr(text)
-      token_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
-      ptr = FFI.mitie_tokenize_with_offsets(text, token_offsets)
-      i = 0
-      tokens = []
-      loop do
-        token = (ptr + i * Fiddle::SIZEOF_VOIDP).ptr
-        break if token.null?
-        offset = (token_offsets.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
-        tokens << [token.to_s.force_encoding(text.encoding), offset]
-        i += 1
-      end
-      [tokens, ptr]
-    ensure
-      # use ptr, not token_offsets.ptr
-      FFI.mitie_free(token_offsets.ptr) if ptr
+    def tokens_with_offset(text)
+      doc(text).tokens_with_offset
     end
 
     def self.finalize(pointer)

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

data/vendor/LICENSE.txt

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

metadata

@@ -1,58 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-14 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-description: 
+date: 2021-01-30 00:00:00.000000000 Z
+dependencies: []
+description:
 email: andrew@chartkick.com
 executables: []
 extensions: []
@@ -62,14 +20,21 @@ files:
 - LICENSE.txt
 - README.md
 - lib/mitie.rb
+- lib/mitie/binary_relation_detector.rb
+- lib/mitie/document.rb
 - lib/mitie/ffi.rb
 - lib/mitie/ner.rb
 - lib/mitie/version.rb
+- vendor/LICENSE.txt
+- vendor/libmitie.arm64.dylib
+- vendor/libmitie.dylib
+- vendor/libmitie.so
+- vendor/mitie.dll
 homepage: https://github.com/ankane/mitie
 licenses:
 - BSL-1.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -84,8 +49,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Named-entity recognition for Ruby
 test_files: []