mitie 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66bb5dce2b079310bd5da7da44a5836331f05bfbbb746163dc45d49094d24e58
-  data.tar.gz: a1e1ec8d23c639315f36f4e0e5e5ddfc0a08e1922fd0d774e302914f5974ab83
+  metadata.gz: 519a88b20911f72d0a66ceaca52e1af3171bab683e27392763f52ac4c494d185
+  data.tar.gz: 6c955aa66776ef1ec92ccc151ba6fce1eeef8d968fad2ac8fd47cb0a7ea4f3bf
 SHA512:
-  metadata.gz: c1914841e9c1fef8ac20c007595b8ed56def925710a75e4e74959496dfa9095a7e253212b560035df9c1c5138443b835a3fffe2cf7c5cf3a9a81135a62df6926
-  data.tar.gz: 397f5d45ca142b3d86fa44c65382fa57a64d9a6ea66ee51b6735d4ae5645cc5c01a4a99744b5fc565cea16d1616e251c9bfca2662b6aa1492d2dcf093a19f57e
+  metadata.gz: 1d8d373478c4ae69844959a349598c35f4187ff91e52e5a6be457b5e61769b0c109bb143b4604999665d6a1fce532ad027add26fc3ea03360764bd2025357c91
+  data.tar.gz: b4c13d770bfb8b03108d6a93c757f286d7dc3e0f157ea5b2db94b7028994df632af40bdbd19f1fe7a40a96d4e3b5fa32d28f724df407c6a901f1bb79635cd046

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.2 (2020-09-14)
+
+- Added binary relation detection
+- Added `Document` class
+
 ## 0.1.1 (2020-09-14)
 
 - Added shared libraries

data/README.md

@@ -1,6 +1,9 @@
 # MITIE
 
-[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition - for Ruby
+[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition and binary relation detection - for Ruby
+
+- Finds people, organizations, and locations in text
+- Detects relationships between entities, like `PERSON` was born in `LOCATION`
 
 [![Build Status](https://travis-ci.org/ankane/mitie.svg?branch=master)](https://travis-ci.org/ankane/mitie) [![Build status](https://ci.appveyor.com/api/projects/status/stc89tc57xfva451/branch/master?svg=true)](https://ci.appveyor.com/project/ankane/mitie/branch/master)
 
@@ -20,22 +23,22 @@ And download the pre-trained model for your language:
 
 ## Getting Started
 
-Get your text
+Load an NER model
 
 ```ruby
-text = "Nat Friedman is the CEO of GitHub, which is headquartered in San Francisco"
+model = Mitie::NER.new("ner_model.dat")
 ```
 
-Load an NER model
+Create a document
 
 ```ruby
-model = Mitie::NER.new("ner_model.dat")
+doc = model.doc("Nat Friedman is the CEO of GitHub, which is headquartered in San Francisco")
 ```
 
 Get entities
 
 ```ruby
-model.entities(text)
+doc.entities
 ```
 
 This returns
@@ -51,13 +54,13 @@ This returns
 Get tokens
 
 ```ruby
-model.tokens(text)
+doc.tokens
 ```
 
 Get tokens and their offset
 
 ```ruby
-model.tokens_with_offset(text)
+doc.tokens_with_offset
 ```
 
 Get all tags for a model
@@ -66,6 +69,40 @@ Get all tags for a model
 model.tags
 ```
 
+## Binary Relation Detection
+
+Detect relationships betweens two entities, like:
+
+- `PERSON` was born in `LOCATION`
+- `ORGANIZATION` was founded in `LOCATION`
+- `FILM` was directed by `PERSON`
+
+There are 21 detectors for English. You can find them in the `binary_relations` directory in the model download.
+
+Load a detector
+
+```ruby
+detector = Mitie::BinaryRelationDetector.new("rel_classifier_film.film.directed_by.svm")
+```
+
+And create a document
+
+```ruby
+doc = model.doc("The Shawshank Redemption was directed by Frank Darabont")
+```
+
+Get relations
+
+```ruby
+detector.relations(doc)
+```
+
+This returns
+
+```ruby
+[{first: "Shawshank Redemption", second: "Frank Darabont", score: 1.124211742912441}]
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/mitie/blob/master/CHANGELOG.md)
@@ -86,5 +123,7 @@ git clone https://github.com/ankane/mitie.git
 cd mitie
 bundle install
 bundle exec rake vendor:all
-MITIE_NER_PATH=path/to/ner_model.dat bundle exec rake test
+
+export MITIE_MODELS_PATH=path/to/MITIE-models/english
+bundle exec rake test
 ```

data/lib/mitie.rb

@@ -2,6 +2,8 @@
 require "fiddle/import"
 
 # modules
+require "mitie/binary_relation_detector"
+require "mitie/document"
 require "mitie/ner"
 require "mitie/version"
 

data/lib/mitie/binary_relation_detector.rb

@@ -0,0 +1,62 @@
+module Mitie
+  class BinaryRelationDetector
+    def initialize(path)
+      # better error message
+      raise ArgumentError, "File does not exist" unless File.exist?(path)
+      @pointer = FFI.mitie_load_binary_relation_detector(path)
+      ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
+    end
+
+    def name
+      FFI.mitie_binary_relation_detector_name_string(pointer).to_s
+    end
+
+    def relations(doc)
+      raise ArgumentError, "Expected Mitie::Document, not #{doc.class.name}" unless doc.is_a?(Document)
+
+      entities = doc.entities
+      combinations = []
+      (entities.size - 1).times do |i|
+        combinations << [entities[i], entities[i + 1]]
+        combinations << [entities[i + 1], entities[i]]
+      end
+
+      relations = []
+      combinations.each do |entity1, entity2|
+        relation =
+          FFI.mitie_extract_binary_relation(
+            doc.model.pointer,
+            doc.send(:tokens_ptr),
+            entity1[:token_index],
+            entity1[:token_length],
+            entity2[:token_index],
+            entity2[:token_length]
+          )
+
+        score_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
+        status = FFI.mitie_classify_binary_relation(pointer, relation, score_ptr)
+        raise "Bad status: #{status}" if status != 0
+        score = score_ptr.to_s(Fiddle::SIZEOF_DOUBLE).unpack1("d")
+        if score > 0
+          relations << {
+            first: entity1[:text],
+            second: entity2[:text],
+            score: score
+          }
+        end
+      end
+      relations
+    end
+
+    private
+
+    def pointer
+      @pointer
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+  end
+end

data/lib/mitie/document.rb

@@ -0,0 +1,96 @@
+module Mitie
+  class Document
+    attr_reader :model, :text
+
+    def initialize(model, text)
+      @model = model
+      @text = text.to_s
+    end
+
+    def tokens
+      @tokens ||= tokens_with_offset.map(&:first)
+    end
+
+    def tokens_with_offset
+      @tokens_with_offset ||= begin
+        i = 0
+        tokens = []
+        loop do
+          token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+          break if token.null?
+          offset = (offsets_ptr.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
+          tokens << [token.to_s.force_encoding(text.encoding), offset]
+          i += 1
+        end
+        tokens
+      end
+    end
+
+    def entities
+      @entities ||= begin
+        begin
+          entities = []
+          tokens = tokens_with_offset
+          detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
+          num_detections = FFI.mitie_ner_get_num_detections(detections)
+          num_detections.times do |i|
+            pos = FFI.mitie_ner_get_detection_position(detections, i)
+            len = FFI.mitie_ner_get_detection_length(detections, i)
+            tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
+            score = FFI.mitie_ner_get_detection_score(detections, i)
+            tok = tokens[pos, len]
+            offset = tok[0][1]
+            finish = tok[-1][1] + tok[-1][0].size
+            entities << {
+              text: text[offset...finish],
+              tag: tag,
+              score: score,
+              offset: offset,
+              token_index: pos,
+              token_length: len
+            }
+          end
+          entities
+        ensure
+          FFI.mitie_free(detections) if detections
+        end
+      end
+    end
+
+    private
+
+    def pointer
+      model.pointer
+    end
+
+    def tokens_ptr
+      tokenize[0]
+    end
+
+    def offsets_ptr
+      tokenize[1]
+    end
+
+    def tokenize
+      @tokenize ||= begin
+        offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
+        tokens_ptr = FFI.mitie_tokenize_with_offsets(text, offsets_ptr)
+
+        ObjectSpace.define_finalizer(tokens_ptr, self.class.finalize(tokens_ptr))
+        ObjectSpace.define_finalizer(offsets_ptr, self.class.finalize_ptr(offsets_ptr))
+
+        [tokens_ptr, offsets_ptr]
+      end
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+
+    def self.finalize_ptr(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer.ptr) }
+    end
+  end
+end

data/lib/mitie/ffi.rb

@@ -25,5 +25,11 @@ module Mitie
     extern "unsigned long mitie_ner_get_detection_tag(const mitie_named_entity_detections* dets, unsigned long idx)"
     extern "const char* mitie_ner_get_detection_tagstr(const mitie_named_entity_detections* dets, unsigned long idx)"
     extern "double mitie_ner_get_detection_score(const mitie_named_entity_detections* dets, unsigned long idx)"
+
+    extern "mitie_binary_relation_detector* mitie_load_binary_relation_detector(const char* filename)"
+    extern "const char* mitie_binary_relation_detector_name_string(const mitie_binary_relation_detector* detector)"
+    extern "int mitie_entities_overlap(unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "mitie_binary_relation* mitie_extract_binary_relation(const mitie_named_entity_extractor* ner, char** tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "int mitie_classify_binary_relation(const mitie_binary_relation_detector* detector, const mitie_binary_relation* relation, double* score)"
   end
 end

data/lib/mitie/ner.rb

@@ -1,8 +1,10 @@
 module Mitie
   class NER
+    attr_reader :pointer
+
     def initialize(path)
       # better error message
-      raise ArgumentError, "Model file does not exist" unless File.exist?(path)
+      raise ArgumentError, "File does not exist" unless File.exist?(path)
       @pointer = FFI.mitie_load_named_entity_extractor(path)
       ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
     end
@@ -13,76 +15,20 @@ module Mitie
       end
     end
 
-    def tokens(text)
-      tokens = []
-      ptr = FFI.mitie_tokenize(text)
-      i = 0
-      loop do
-        token = (ptr + i * Fiddle::SIZEOF_VOIDP).ptr
-        break if token.null?
-        tokens << token.to_s.force_encoding(text.encoding)
-        i += 1
-      end
-      tokens
-    ensure
-      FFI.mitie_free(ptr) if ptr
-    end
-
-    def tokens_with_offset(text)
-      tokens, ptr = tokens_with_offset_with_ptr(text)
-      tokens
-    ensure
-      FFI.mitie_free(ptr) if ptr
+    def doc(text)
+      Document.new(self, text)
     end
 
     def entities(text)
-      entities = []
-      tokens, tokens_ptr = tokens_with_offset_with_ptr(text)
-      detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
-      num_detections = FFI.mitie_ner_get_num_detections(detections)
-      num_detections.times do |i|
-        pos = FFI.mitie_ner_get_detection_position(detections, i)
-        len = FFI.mitie_ner_get_detection_length(detections, i)
-        tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
-        score = FFI.mitie_ner_get_detection_score(detections, i)
-        tok = tokens[pos, len]
-        offset = tok[0][1]
-        finish = tok[-1][1] + tok[-1][0].size
-        entities << {
-          text: text[offset...finish],
-          tag: tag,
-          score: score,
-          offset: offset
-        }
-      end
-      entities
-    ensure
-      FFI.mitie_free(tokens_ptr) if tokens_ptr
-      FFI.mitie_free(detections) if detections
+      doc(text).entities
     end
 
-    private
-
-    def pointer
-      @pointer
+    def tokens(text)
+      doc(text).tokens
     end
 
-    def tokens_with_offset_with_ptr(text)
-      token_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
-      ptr = FFI.mitie_tokenize_with_offsets(text, token_offsets)
-      i = 0
-      tokens = []
-      loop do
-        token = (ptr + i * Fiddle::SIZEOF_VOIDP).ptr
-        break if token.null?
-        offset = (token_offsets.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
-        tokens << [token.to_s.force_encoding(text.encoding), offset]
-        i += 1
-      end
-      [tokens, ptr]
-    ensure
-      # use ptr, not token_offsets.ptr
-      FFI.mitie_free(token_offsets.ptr) if ptr
+    def tokens_with_offset(text)
+      doc(text).tokens_with_offset
     end
 
     def self.finalize(pointer)

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-14 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -62,6 +62,8 @@ files:
 - LICENSE.txt
 - README.md
 - lib/mitie.rb
+- lib/mitie/binary_relation_detector.rb
+- lib/mitie/document.rb
 - lib/mitie/ffi.rb
 - lib/mitie/ner.rb
 - lib/mitie/version.rb