mitie 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ea3ef115016c59ecb496ffbbe13c4ac3a2ffda6acf9392ac423103b9c3cfe634
+  data.tar.gz: 6eb77dd514ba3c08c30e1216921cd83619f206a658018c1d4522c598e175e8b2
+SHA512:
+  metadata.gz: 682fb3ea1c0be1889f2e1e177204309ea7b6d2989834a4d2bae49ddf567e309b9fef9ea872746626590ec06b9aabb455067d88e696ea3a4f59a9b785b43819c9
+  data.tar.gz: 4260a6dff4eb613278468d9fff2ef19f99f49bed81571781b4d2ac886f39bed9356a4a20a9220fa2b31e8c12965bffac5e86b40318b28aa450ebd60a3c53c3cd

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2020-09-14)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# MITIE
+
+[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition - for Ruby
+
+## Installation
+
+First, install MITIE. For Homebrew, use:
+
+```sh
+brew install mitie
+```
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'mitie'
+```
+
+And download the pre-trained model for your language:
+
+- [English](https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2.tar.bz2)
+- [Spanish](https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2-Spanish.zip)
+- [German](https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2-German.tar.bz2)
+
+## Getting Started
+
+Get your text
+
+```ruby
+text = "Nat Friedman is the CEO of GitHub, which is headquartered in San Francisco"
+```
+
+Load an NER model
+
+```ruby
+model = Mitie::NER.new("ner_model.dat")
+```
+
+Get entities
+
+```ruby
+model.entities(text)
+```
+
+This returns
+
+```ruby
+[
+  {text: "Nat Friedman",  tag: "PERSON",       score: 1.099661347535191, offset: 0},
+  {text: "GitHub",        tag: "ORGANIZATION", score: 0.344641651251650, offset: 27},
+  {text: "San Francisco", tag: "LOCATION",     score: 1.428241888939011, offset: 61}
+]
+```
+
+Get tokens
+
+```ruby
+model.tokens(text)
+```
+
+Get tokens and their offset
+
+```ruby
+model.tokens_with_offset(text)
+```
+
+Get all tags for a model
+
+```ruby
+model.tags
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/mitie/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/mitie/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/mitie/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/mitie.git
+cd mitie
+bundle install
+MITIE_NER_PATH=path/to/ner_model.dat bundle exec rake test
+```

data/lib/mitie.rb

@@ -0,0 +1,25 @@
+# stdlib
+require "fiddle/import"
+
+# modules
+require "mitie/ner"
+require "mitie/version"
+
+module Mitie
+  class Error < StandardError; end
+
+  class << self
+    attr_accessor :ffi_lib
+  end
+  self.ffi_lib =
+    if Gem.win_platform?
+      ["mitie.dll"]
+    elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
+      ["libmitie.dylib"]
+    else
+      ["libmitie.so"]
+    end
+
+  # friendlier error message
+  autoload :FFI, "mitie/ffi"
+end

data/lib/mitie/ffi.rb

@@ -0,0 +1,29 @@
+module Mitie
+  module FFI
+    extend Fiddle::Importer
+
+    libs = Mitie.ffi_lib.dup
+    begin
+      dlload Fiddle.dlopen(libs.shift)
+    rescue Fiddle::DLError => e
+      retry if libs.any?
+      raise e
+    end
+
+    extern "void mitie_free(void* object)"
+    extern "char** mitie_tokenize(const char* text)"
+    extern "char** mitie_tokenize_with_offsets(const char* text, unsigned long** token_offsets)"
+
+    extern "mitie_named_entity_extractor* mitie_load_named_entity_extractor(const char* filename)"
+    extern "unsigned long mitie_get_num_possible_ner_tags(const mitie_named_entity_extractor* ner)"
+    extern "const char* mitie_get_named_entity_tagstr(const mitie_named_entity_extractor* ner, unsigned long idx)"
+
+    extern "mitie_named_entity_detections* mitie_extract_entities(const mitie_named_entity_extractor* ner, char** tokens)"
+    extern "unsigned long mitie_ner_get_num_detections(const mitie_named_entity_detections* dets)"
+    extern "unsigned long mitie_ner_get_detection_position(const mitie_named_entity_detections* dets, unsigned long idx)"
+    extern "unsigned long mitie_ner_get_detection_length(const mitie_named_entity_detections* dets, unsigned long idx)"
+    extern "unsigned long mitie_ner_get_detection_tag(const mitie_named_entity_detections* dets, unsigned long idx)"
+    extern "const char* mitie_ner_get_detection_tagstr(const mitie_named_entity_detections* dets, unsigned long idx)"
+    extern "double mitie_ner_get_detection_score(const mitie_named_entity_detections* dets, unsigned long idx)"
+  end
+end

data/lib/mitie/ner.rb

@@ -0,0 +1,91 @@
+module Mitie
+  class NER
+    def initialize(path)
+      @pointer = FFI.mitie_load_named_entity_extractor(path)
+      ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
+    end
+
+    def tags
+      FFI.mitie_get_num_possible_ner_tags(pointer).times.map do |i|
+        FFI.mitie_get_named_entity_tagstr(pointer, i).to_s
+      end
+    end
+
+    def tokens(text)
+      tokens = []
+      ptr = FFI.mitie_tokenize(text)
+      i = 0
+      loop do
+        token = (ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+        break if token.null?
+        tokens << token.to_s.force_encoding(text.encoding)
+        i += 1
+      end
+      tokens
+    ensure
+      FFI.mitie_free(ptr) if ptr
+    end
+
+    def tokens_with_offset(text)
+      tokens, ptr = tokens_with_offset_with_ptr(text)
+      tokens
+    ensure
+      FFI.mitie_free(ptr) if ptr
+    end
+
+    def entities(text)
+      entities = []
+      tokens, tokens_ptr = tokens_with_offset_with_ptr(text)
+      detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
+      num_detections = FFI.mitie_ner_get_num_detections(detections)
+      num_detections.times do |i|
+        pos = FFI.mitie_ner_get_detection_position(detections, i)
+        len = FFI.mitie_ner_get_detection_length(detections, i)
+        tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
+        score = FFI.mitie_ner_get_detection_score(detections, i)
+        tok = tokens[pos, len]
+        offset = tok[0][1]
+        finish = tok[-1][1] + tok[-1][0].size
+        entities << {
+          text: text[offset...finish],
+          tag: tag,
+          score: score,
+          offset: offset
+        }
+      end
+      entities
+    ensure
+      FFI.mitie_free(tokens_ptr) if tokens_ptr
+      FFI.mitie_free(detections) if detections
+    end
+
+    private
+
+    def pointer
+      @pointer
+    end
+
+    def tokens_with_offset_with_ptr(text)
+      token_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
+      ptr = FFI.mitie_tokenize_with_offsets(text, token_offsets)
+      i = 0
+      tokens = []
+      loop do
+        token = (ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+        break if token.null?
+        offset = (token_offsets.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
+        tokens << [token.to_s.force_encoding(text.encoding), offset]
+        i += 1
+      end
+      [tokens, ptr]
+    ensure
+      # use ptr, not token_offsets.ptr
+      FFI.mitie_free(token_offsets.ptr) if ptr
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+  end
+end

data/lib/mitie/version.rb

@@ -0,0 +1,3 @@
+module Mitie
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: mitie
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-09-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+description: 
+email: andrew@chartkick.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/mitie.rb
+- lib/mitie/ffi.rb
+- lib/mitie/ner.rb
+- lib/mitie/version.rb
+homepage: https://github.com/ankane/mitie
+licenses:
+- BSL-1.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Named-entity recognition for Ruby
+test_files: []