mitie 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 519a88b20911f72d0a66ceaca52e1af3171bab683e27392763f52ac4c494d185
-  data.tar.gz: 6c955aa66776ef1ec92ccc151ba6fce1eeef8d968fad2ac8fd47cb0a7ea4f3bf
+  metadata.gz: ca692cd09ac5c4541998bf601b13e95a041ac29e5d3fffcdf1e00f4b1a5fee58
+  data.tar.gz: dec98a81c3e45b2e6f191883e4a331bbfc0a98d9da7630c9ee806addb2284704
 SHA512:
-  metadata.gz: 1d8d373478c4ae69844959a349598c35f4187ff91e52e5a6be457b5e61769b0c109bb143b4604999665d6a1fce532ad027add26fc3ea03360764bd2025357c91
-  data.tar.gz: b4c13d770bfb8b03108d6a93c757f286d7dc3e0f157ea5b2db94b7028994df632af40bdbd19f1fe7a40a96d4e3b5fa32d28f724df407c6a901f1bb79635cd046
+  metadata.gz: f802b2a582dc0362351f7c26441e482bc064affa03828e033a948c26a9b6a3bdb42ad216768510b507bd8be6dc0692a34a8b1a0125cfa9533e7710d5ee2c1cba
+  data.tar.gz: 0bf700445e710a16a871fff6d7089aa44a3a8a53420e7f9e5a7a8cf959ccf6066a5e7b59bd23a7631b2c378571e34f5cfa4d5158a6b599d91bc1caa3560b79c9

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.3 (2020-12-04)
+
+- Added support for custom tokenization
+
 ## 0.1.2 (2020-09-14)
 
 - Added binary relation detection

data/README.md

@@ -5,7 +5,7 @@
 - Finds people, organizations, and locations in text
 - Detects relationships between entities, like `PERSON` was born in `LOCATION`
 
-[![Build Status](https://travis-ci.org/ankane/mitie.svg?branch=master)](https://travis-ci.org/ankane/mitie) [![Build status](https://ci.appveyor.com/api/projects/status/stc89tc57xfva451/branch/master?svg=true)](https://ci.appveyor.com/project/ankane/mitie/branch/master)
+[![Build Status](https://github.com/ankane/mitie/workflows/build/badge.svg?branch=master)](https://github.com/ankane/mitie/actions)
 
 ## Installation
 
@@ -32,7 +32,7 @@ model = Mitie::NER.new("ner_model.dat")
 Create a document
 
 ```ruby
-doc = model.doc("Nat Friedman is the CEO of GitHub, which is headquartered in San Francisco")
+doc = model.doc("Nat works at GitHub in San Francisco")
 ```
 
 Get entities
@@ -45,9 +45,9 @@ This returns
 
 ```ruby
 [
-  {text: "Nat Friedman",  tag: "PERSON",       score: 1.099661347535191, offset: 0},
-  {text: "GitHub",        tag: "ORGANIZATION", score: 0.344641651251650, offset: 27},
-  {text: "San Francisco", tag: "LOCATION",     score: 1.428241888939011, offset: 61}
+  {text: "Nat",           tag: "PERSON",       score: 0.3112371212688382, offset: 0},
+  {text: "GitHub",        tag: "ORGANIZATION", score: 0.5660115198329334, offset: 13},
+  {text: "San Francisco", tag: "LOCATION",     score: 1.3890524313885309, offset: 23}
 ]
 ```
 
@@ -82,13 +82,13 @@ There are 21 detectors for English. You can find them in the `binary_relations`
 Load a detector
 
 ```ruby
-detector = Mitie::BinaryRelationDetector.new("rel_classifier_film.film.directed_by.svm")
+detector = Mitie::BinaryRelationDetector.new("rel_classifier_organization.organization.place_founded.svm")
 ```
 
 And create a document
 
 ```ruby
-doc = model.doc("The Shawshank Redemption was directed by Frank Darabont")
+doc = model.doc("Shopify was founded in Ottawa")
 ```
 
 Get relations
@@ -100,7 +100,7 @@ detector.relations(doc)
 This returns
 
 ```ruby
-[{first: "Shawshank Redemption", second: "Frank Darabont", score: 1.124211742912441}]
+[{first: "Shopify", second: "Ottawa", score: 0.17649169745814464}]
 ```
 
 ## History

data/lib/mitie/document.rb

@@ -4,7 +4,7 @@ module Mitie
 
     def initialize(model, text)
       @model = model
-      @text = text.to_s
+      @text = text
     end
 
     def tokens
@@ -13,16 +13,21 @@ module Mitie
 
     def tokens_with_offset
       @tokens_with_offset ||= begin
-        i = 0
-        tokens = []
-        loop do
-          token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
-          break if token.null?
-          offset = (offsets_ptr.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
-          tokens << [token.to_s.force_encoding(text.encoding), offset]
-          i += 1
+        if text.is_a?(Array)
+          # offsets are unknown when given tokens
+          text.map { |v| [v, nil] }
+        else
+          i = 0
+          tokens = []
+          loop do
+            token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+            break if token.null?
+            offset = (offsets_ptr.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
+            tokens << [token.to_s.force_encoding(text.encoding), offset]
+            i += 1
+          end
+          tokens
         end
-        tokens
       end
     end
 
@@ -40,15 +45,20 @@ module Mitie
             score = FFI.mitie_ner_get_detection_score(detections, i)
             tok = tokens[pos, len]
             offset = tok[0][1]
-            finish = tok[-1][1] + tok[-1][0].size
-            entities << {
-              text: text[offset...finish],
-              tag: tag,
-              score: score,
-              offset: offset,
-              token_index: pos,
-              token_length: len
-            }
+
+            entity = {}
+            if offset
+              finish = tok[-1][1] + tok[-1][0].size
+              entity[:text] = text[offset...finish]
+            else
+              entity[:text] = tok.map(&:first)
+            end
+            entity[:tag] = tag
+            entity[:score] = score
+            entity[:offset] = offset if offset
+            entity[:token_index] = pos
+            entity[:token_length] = len
+            entities << entity
           end
           entities
         ensure
@@ -73,13 +83,22 @@ module Mitie
 
     def tokenize
       @tokenize ||= begin
-        offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
-        tokens_ptr = FFI.mitie_tokenize_with_offsets(text, offsets_ptr)
+        if text.is_a?(Array)
+          # malloc uses memset to set all bytes to 0
+          tokens_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP * (text.size + 1))
+          text.size.times do |i|
+            tokens_ptr[i * Fiddle::SIZEOF_VOIDP, Fiddle::SIZEOF_VOIDP] = Fiddle::Pointer.to_ptr(text[i]).ref
+          end
+          [tokens_ptr, nil]
+        else
+          offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
+          tokens_ptr = FFI.mitie_tokenize_with_offsets(text, offsets_ptr)
 
-        ObjectSpace.define_finalizer(tokens_ptr, self.class.finalize(tokens_ptr))
-        ObjectSpace.define_finalizer(offsets_ptr, self.class.finalize_ptr(offsets_ptr))
+          ObjectSpace.define_finalizer(tokens_ptr, self.class.finalize(tokens_ptr))
+          ObjectSpace.define_finalizer(offsets_ptr, self.class.finalize_ptr(offsets_ptr))
 
-        [tokens_ptr, offsets_ptr]
+          [tokens_ptr, offsets_ptr]
+        end
       end
     end
 

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2020-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,7 +52,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
-description: 
+description:
 email: andrew@chartkick.com
 executables: []
 extensions: []
@@ -75,7 +75,7 @@ homepage: https://github.com/ankane/mitie
 licenses:
 - BSL-1.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -90,8 +90,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Named-entity recognition for Ruby
 test_files: []