mitie 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e36228752078ce915d9d6cd49f3d8087c62263b0e41bebb5bcfd5a478398fe6
-  data.tar.gz: 54881a7c47cd855b678cf71a5baa7486b22acdbac802e15eda3984fc2215838d
+  metadata.gz: 50fb12adcd0042b3c09968108ec382694f8f0df20750b88566bc64c8d85d9e8d
+  data.tar.gz: 9d3d34f9839f71fc17e6651a17069f1f0fdb4b5956f9e57c4b66c9567d908967
 SHA512:
-  metadata.gz: 573e0e14ecd284cb9754abe95f2c3cd2ebbd9e2dbebaa5a0f33fad847abe6ebc3d0875c916d05f95fc1fdd9208c12d906315f0f5a557dfc6b3db63b620aa3f6f
-  data.tar.gz: 272335c83fc924aaf696382959d6cd93dd667139f282fef71b78b29a6784bda16f221cdd36691e885db6075312a00d463772f4408ee130ea5dde6a7a49b77d55
+  metadata.gz: 189d5a17f94fff9abbc1d8d961c8f8407ae85bd1284f727e363b9aa7384d748eb1d80b6ffa1b1a44c8c6fadbbb6e91e6d064ea58fa88af972e6fd6e90e3ef71d
+  data.tar.gz: 202ba418ee98636736185a8f4783601acabecc59dde7e3a5c31a12b56827497b6b2b7e7e26d264b61848281b94cb635249b17a9e8fa009c496dacbe24e6466ca

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.0 (2022-06-01)
+
+- Added support for text categorization
+- Added support for training binary relation detectors
+- Dropped support for Ruby < 2.7
+
 ## 0.1.6 (2022-03-20)
 
 - Added support for training NER models

data/README.md

@@ -1,6 +1,6 @@
 # MITIE Ruby
 
-[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition and binary relation detection - for Ruby
+[MITIE](https://github.com/mit-nlp/MITIE) - named-entity recognition, binary relation detection, and text categorization - for Ruby
 
 - Finds people, organizations, and locations in text
 - Detects relationships between entities, like `PERSON` was born in `LOCATION`
@@ -15,7 +15,7 @@ Add this line to your application’s Gemfile:
 gem "mitie"
 ```
 
-And download the pre-trained model for your language:
+And download the pre-trained models for your language:
 
 - [English](https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2.tar.bz2)
 - [Spanish](https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2-Spanish.zip)
@@ -23,6 +23,12 @@ And download the pre-trained model for your language:
 
 ## Getting Started
 
+- [Named Entity Recognition](#named-entity-recognition)
+- [Binary Relation Detection](#binary-relation-detection)
+- [Text Categorization](#text-categorization)
+
+## Named Entity Recognition
+
 Load an NER model
 
 ```ruby
@@ -69,7 +75,7 @@ Get all tags for a model
 model.tags
 ```
 
-## Training
+### Training
 
 Load an NER model into a trainer
 
@@ -138,6 +144,80 @@ This returns
 [{first: "Shopify", second: "Ottawa", score: 0.17649169745814464}]
 ```
 
+### Training
+
+Load an NER model into a trainer
+
+```ruby
+trainer = Mitie::BinaryRelationTrainer.new(model)
+```
+
+Add positive and negative examples to the trainer
+
+```ruby
+tokens = ["Shopify", "was", "founded", "in", "Ottawa"]
+trainer.add_positive_binary_relation(tokens, 0..0, 4..4)
+trainer.add_negative_binary_relation(tokens, 4..4, 0..0)
+```
+
+Train the detector
+
+```ruby
+detector = trainer.train
+```
+
+Save the detector
+
+```ruby
+detector.save_to_disk("binary_relation_detector.svm")
+```
+
+## Text Categorization
+
+Load a model into a trainer
+
+```ruby
+trainer = Mitie::TextCategorizerTrainer.new("total_word_feature_extractor.dat")
+```
+
+Add labeled text to the trainer
+
+```ruby
+trainer.add(["This", "is", "super", "cool"], "positive")
+```
+
+Train the model
+
+```ruby
+model = trainer.train
+```
+
+Save the model
+
+```ruby
+model.save_to_disk("text_categorization_model.dat")
+```
+
+Load a saved model
+
+```ruby
+model = Mitie::TextCategorizer.new("text_categorization_model.dat")
+```
+
+Categorize text
+
+```ruby
+model.categorize(["What", "a", "super", "nice", "day"])
+```
+
+## Deployment
+
+Check out [Trove](https://github.com/ankane/trove) for deploying models.
+
+```sh
+trove push ner_model.dat
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/mitie-ruby/blob/master/CHANGELOG.md)

data/lib/mitie/binary_relation_detector.rb

@@ -1,9 +1,16 @@
 module Mitie
   class BinaryRelationDetector
-    def initialize(path)
-      # better error message
-      raise ArgumentError, "File does not exist" unless File.exist?(path)
-      @pointer = FFI.mitie_load_binary_relation_detector(path)
+    def initialize(path = nil, pointer: nil)
+      if path
+        # better error message
+        raise ArgumentError, "File does not exist" unless File.exist?(path)
+        @pointer = FFI.mitie_load_binary_relation_detector(path)
+      elsif pointer
+        @pointer = pointer
+      else
+        raise ArgumentError, "Must pass either a path or a pointer"
+      end
+
       ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
     end
 
@@ -23,38 +30,52 @@ module Mitie
 
       relations = []
       combinations.each do |entity1, entity2|
-        relation =
-          FFI.mitie_extract_binary_relation(
-            doc.model.pointer,
-            doc.send(:tokens_ptr),
-            entity1[:token_index],
-            entity1[:token_length],
-            entity2[:token_index],
-            entity2[:token_length]
-          )
-
-        score_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
-        status = FFI.mitie_classify_binary_relation(pointer, relation, score_ptr)
-        # TODO make Mitie::Error
-        raise "Bad status: #{status}" if status != 0
-        score = score_ptr.to_s(Fiddle::SIZEOF_DOUBLE).unpack1("d")
-        if score > 0
-          relations << {
-            first: entity1[:text],
-            second: entity2[:text],
-            score: score
-          }
-        end
+        relation = extract_relation(doc, entity1, entity2)
+        relations << relation if relation
       end
       relations
     end
 
+    def save_to_disk(filename)
+      if FFI.mitie_save_binary_relation_detector(filename, pointer) != 0
+        raise Error, "Unable to save detector"
+      end
+      nil
+    end
+
     private
 
     def pointer
       @pointer
     end
 
+    def extract_relation(doc, entity1, entity2)
+      relation =
+        FFI.mitie_extract_binary_relation(
+          doc.model.pointer,
+          doc.send(:tokens_ptr),
+          entity1[:token_index],
+          entity1[:token_length],
+          entity2[:token_index],
+          entity2[:token_length]
+        )
+
+      score_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
+      status = FFI.mitie_classify_binary_relation(pointer, relation, score_ptr)
+      raise Error, "Bad status: #{status}" if status != 0
+
+      score = Utils.read_double(score_ptr)
+      if score > 0
+        {
+          first: entity1[:text],
+          second: entity2[:text],
+          score: score
+        }
+      end
+    ensure
+      FFI.mitie_free(relation) if relation
+    end
+
     def self.finalize(pointer)
       # must use proc instead of stabby lambda
       proc { FFI.mitie_free(pointer) }

data/lib/mitie/binary_relation_trainer.rb

@@ -0,0 +1,87 @@
+module Mitie
+  class BinaryRelationTrainer
+    def initialize(ner, name: "")
+      @pointer = FFI.mitie_create_binary_relation_trainer(name, ner.pointer)
+
+      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+    end
+
+    def add_positive_binary_relation(tokens, range1, range2)
+      check_add(tokens, range1, range2)
+
+      tokens_pointer = Utils.array_to_pointer(tokens)
+      status = FFI.mitie_add_positive_binary_relation(@pointer, tokens_pointer, range1.begin, range1.size, range2.begin, range2.size)
+      if status != 0
+        raise Error, "Unable to add binary relation"
+      end
+    end
+
+    def add_negative_binary_relation(tokens, range1, range2)
+      check_add(tokens, range1, range2)
+
+      tokens_pointer = Utils.array_to_pointer(tokens)
+      status = FFI.mitie_add_negative_binary_relation(@pointer, tokens_pointer, range1.begin, range1.size, range2.begin, range2.size)
+      if status != 0
+        raise Error, "Unable to add binary relation"
+      end
+    end
+
+    def beta
+      FFI.mitie_binary_relation_trainer_get_beta(@pointer)
+    end
+
+    def beta=(value)
+      raise ArgumentError, "beta must be greater than or equal to zero" unless value >= 0
+
+      FFI.mitie_binary_relation_trainer_set_beta(@pointer, value)
+    end
+
+    def num_threads
+      FFI.mitie_binary_relation_trainer_get_num_threads(@pointer)
+    end
+
+    def num_threads=(value)
+      FFI.mitie_binary_relation_trainer_set_num_threads(@pointer, value)
+    end
+
+    def num_positive_examples
+      FFI.mitie_binary_relation_trainer_num_positive_examples(@pointer)
+    end
+
+    def num_negative_examples
+      FFI.mitie_binary_relation_trainer_num_negative_examples(@pointer)
+    end
+
+    def train
+      if num_positive_examples + num_negative_examples == 0
+        raise Error, "You can't call train() on an empty trainer"
+      end
+
+      detector = FFI.mitie_train_binary_relation_detector(@pointer)
+
+      raise Error, "Unable to create binary relation detector. Probably ran out of RAM." if detector.null?
+
+      Mitie::BinaryRelationDetector.new(pointer: detector)
+    end
+
+    private
+
+    def check_add(tokens, range1, range2)
+      Utils.check_range(range1, tokens.size)
+      Utils.check_range(range2, tokens.size)
+
+      if entities_overlap?(range1, range2)
+        raise ArgumentError, "Entities overlap"
+      end
+    end
+
+    def entities_overlap?(range1, range2)
+      FFI.mitie_entities_overlap(range1.begin, range1.size, range2.begin, range2.size) == 1
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+  end
+end

data/lib/mitie/document.rb

@@ -33,37 +33,35 @@ module Mitie
 
     def entities
       @entities ||= begin
-        begin
-          entities = []
-          tokens = tokens_with_offset
-          detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
-          num_detections = FFI.mitie_ner_get_num_detections(detections)
-          num_detections.times do |i|
-            pos = FFI.mitie_ner_get_detection_position(detections, i)
-            len = FFI.mitie_ner_get_detection_length(detections, i)
-            tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
-            score = FFI.mitie_ner_get_detection_score(detections, i)
-            tok = tokens[pos, len]
-            offset = tok[0][1]
+        entities = []
+        tokens = tokens_with_offset
+        detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
+        num_detections = FFI.mitie_ner_get_num_detections(detections)
+        num_detections.times do |i|
+          pos = FFI.mitie_ner_get_detection_position(detections, i)
+          len = FFI.mitie_ner_get_detection_length(detections, i)
+          tag = FFI.mitie_ner_get_detection_tagstr(detections, i).to_s
+          score = FFI.mitie_ner_get_detection_score(detections, i)
+          tok = tokens[pos, len]
+          offset = tok[0][1]
 
-            entity = {}
-            if offset
-              finish = tok[-1][1] + tok[-1][0].bytesize
-              entity[:text] = text.byteslice(offset...finish)
-            else
-              entity[:text] = tok.map(&:first)
-            end
-            entity[:tag] = tag
-            entity[:score] = score
-            entity[:offset] = offset if offset
-            entity[:token_index] = pos
-            entity[:token_length] = len
-            entities << entity
+          entity = {}
+          if offset
+            finish = tok[-1][1] + tok[-1][0].bytesize
+            entity[:text] = text.byteslice(offset...finish)
+          else
+            entity[:text] = tok.map(&:first)
           end
-          entities
-        ensure
-          FFI.mitie_free(detections) if detections
+          entity[:tag] = tag
+          entity[:score] = score
+          entity[:offset] = offset if offset
+          entity[:token_index] = pos
+          entity[:token_length] = len
+          entities << entity
         end
+        entities
+      ensure
+        FFI.mitie_free(detections) if detections
       end
     end
 

data/lib/mitie/ffi.rb

@@ -15,12 +15,11 @@ module Mitie
     extern "void mitie_free(void* object)"
     extern "char** mitie_tokenize(const char* text)"
     extern "char** mitie_tokenize_with_offsets(const char* text, unsigned long** token_offsets)"
-    extern "int mitie_save_named_entity_extractor(const char* filename, const mitie_named_entity_extractor* ner)"
 
+    # ner
     extern "mitie_named_entity_extractor* mitie_load_named_entity_extractor(const char* filename)"
     extern "unsigned long mitie_get_num_possible_ner_tags(const mitie_named_entity_extractor* ner)"
     extern "const char* mitie_get_named_entity_tagstr(const mitie_named_entity_extractor* ner, unsigned long idx)"
-
     extern "mitie_named_entity_detections* mitie_extract_entities(const mitie_named_entity_extractor* ner, char** tokens)"
     extern "unsigned long mitie_ner_get_num_detections(const mitie_named_entity_detections* dets)"
     extern "unsigned long mitie_ner_get_detection_position(const mitie_named_entity_detections* dets, unsigned long idx)"
@@ -29,12 +28,28 @@ module Mitie
     extern "const char* mitie_ner_get_detection_tagstr(const mitie_named_entity_detections* dets, unsigned long idx)"
     extern "double mitie_ner_get_detection_score(const mitie_named_entity_detections* dets, unsigned long idx)"
 
+    # binary relation detector
+    extern "mitie_binary_relation_detector* mitie_load_binary_relation_detector(const char* filename)"
+    extern "const char* mitie_binary_relation_detector_name_string(const mitie_binary_relation_detector* detector)"
+    extern "int mitie_entities_overlap(unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "mitie_binary_relation* mitie_extract_binary_relation(const mitie_named_entity_extractor* ner, char** tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "int mitie_classify_binary_relation(const mitie_binary_relation_detector* detector, const mitie_binary_relation* relation, double* score)"
+
+    # text categorizer
+    extern "mitie_text_categorizer* mitie_load_text_categorizer(const char* filename)"
+    extern "int mitie_categorize_text(const mitie_text_categorizer* tcat, const char** tokens, char** text_tag, double* text_score)"
+
+    # save
+    extern "int mitie_save_named_entity_extractor(const char* filename, const mitie_named_entity_extractor* ner)"
+    extern "int mitie_save_binary_relation_detector(const char* filename, const mitie_binary_relation_detector* detector)"
+    extern "int mitie_save_text_categorizer(const char* filename, const mitie_text_categorizer* tcat)"
+
+    # ner trainer
     extern "mitie_ner_training_instance* mitie_create_ner_training_instance(char** tokens)"
     extern "unsigned long mitie_ner_training_instance_num_entities(const mitie_ner_training_instance* instance)"
     extern "unsigned long mitie_ner_training_instance_num_tokens(const mitie_ner_training_instance* instance)"
     extern "int mitie_overlaps_any_entity(mitie_ner_training_instance* instance, unsigned long start, unsigned long length)"
     extern "int mitie_add_ner_training_entity(mitie_ner_training_instance* instance, unsigned long start, unsigned long length, const char* label)"
-
     extern "mitie_ner_trainer* mitie_create_ner_trainer(const char* filename)"
     extern "unsigned long mitie_ner_trainer_size(const mitie_ner_trainer* trainer)"
     extern "int mitie_add_ner_training_instance(mitie_ner_trainer* trainer, const mitie_ner_training_instance* instance)"
@@ -44,10 +59,26 @@ module Mitie
     extern "unsigned long mitie_ner_trainer_get_num_threads(const mitie_ner_trainer* trainer)"
     extern "mitie_named_entity_extractor* mitie_train_named_entity_extractor(const mitie_ner_trainer* trainer)"
 
-    extern "mitie_binary_relation_detector* mitie_load_binary_relation_detector(const char* filename)"
-    extern "const char* mitie_binary_relation_detector_name_string(const mitie_binary_relation_detector* detector)"
-    extern "int mitie_entities_overlap(unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
-    extern "mitie_binary_relation* mitie_extract_binary_relation(const mitie_named_entity_extractor* ner, char** tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
-    extern "int mitie_classify_binary_relation(const mitie_binary_relation_detector* detector, const mitie_binary_relation* relation, double* score)"
+    # binary relation trainer
+    extern "mitie_binary_relation_trainer* mitie_create_binary_relation_trainer(const char* relation_name, const mitie_named_entity_extractor* ner)"
+    extern "unsigned long mitie_binary_relation_trainer_num_positive_examples(const mitie_binary_relation_trainer* trainer)"
+    extern "unsigned long mitie_binary_relation_trainer_num_negative_examples(const mitie_binary_relation_trainer* trainer)"
+    extern "int mitie_add_positive_binary_relation(mitie_binary_relation_trainer* trainer, char** tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "int mitie_add_negative_binary_relation(mitie_binary_relation_trainer* trainer, char** tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length)"
+    extern "void mitie_binary_relation_trainer_set_beta(mitie_binary_relation_trainer* trainer, double beta)"
+    extern "double mitie_binary_relation_trainer_get_beta(const mitie_binary_relation_trainer* trainer)"
+    extern "void mitie_binary_relation_trainer_set_num_threads(mitie_binary_relation_trainer* trainer, unsigned long num_threads)"
+    extern "unsigned long mitie_binary_relation_trainer_get_num_threads(const mitie_binary_relation_trainer* trainer)"
+    extern "mitie_binary_relation_detector* mitie_train_binary_relation_detector(const mitie_binary_relation_trainer* trainer)"
+
+    # text categorizer trainer
+    extern "mitie_text_categorizer_trainer* mitie_create_text_categorizer_trainer(const char* filename)"
+    extern "unsigned long mitie_text_categorizer_trainer_size(const mitie_text_categorizer_trainer* trainer)"
+    extern "void mitie_text_categorizer_trainer_set_beta(mitie_text_categorizer_trainer* trainer, double beta)"
+    extern "double mitie_text_categorizer_trainer_get_beta(const mitie_text_categorizer_trainer* trainer)"
+    extern "void mitie_text_categorizer_trainer_set_num_threads(mitie_text_categorizer_trainer* trainer, unsigned long num_threads)"
+    extern "unsigned long mitie_text_categorizer_trainer_get_num_threads(const mitie_text_categorizer_trainer* trainer)"
+    extern "int mitie_add_text_categorizer_labeled_text(mitie_text_categorizer_trainer* trainer, const char** tokens, const char* label)"
+    extern "mitie_text_categorizer* mitie_train_text_categorizer(const mitie_text_categorizer_trainer* trainer)"
   end
 end

data/lib/mitie/ner_training_instance.rb

@@ -12,9 +12,7 @@ module Mitie
     end
 
     def add_entity(range, label)
-      if range.none? || range.end >= num_tokens || range.begin < 0
-        raise ArgumentError, "Invalid range"
-      end
+      Utils.check_range(range, num_tokens)
 
       raise ArgumentError, "Range overlaps existing entity" if overlaps_any_entity?(range)
 
@@ -34,9 +32,7 @@ module Mitie
     end
 
     def overlaps_any_entity?(range)
-      if range.none? || range.max >= num_tokens
-        raise ArgumentError, "Invalid range"
-      end
+      Utils.check_range(range, num_tokens)
 
       FFI.mitie_overlaps_any_entity(@pointer, range.begin, range.size) == 1
     end

data/lib/mitie/text_categorizer.rb

@@ -0,0 +1,47 @@
+module Mitie
+  class TextCategorizer
+    def initialize(path = nil, pointer: nil)
+      if path
+        # better error message
+        raise ArgumentError, "File does not exist" unless File.exist?(path)
+        @pointer = FFI.mitie_load_text_categorizer(path)
+      elsif pointer
+        @pointer = pointer
+      else
+        raise ArgumentError, "Must pass either a path or a pointer"
+      end
+
+      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+    end
+
+    def categorize(tokens)
+      tokens_pointer = Utils.array_to_pointer(tokens)
+      text_tag = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
+      text_score = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
+
+      if FFI.mitie_categorize_text(@pointer, tokens_pointer, text_tag, text_score) != 0
+        raise Error, "Unable to categorize"
+      end
+
+      {
+        tag: text_tag.ptr.to_s,
+        score: Utils.read_double(text_score)
+      }
+    ensure
+      # text_tag must be freed
+      FFI.mitie_free(text_tag.ptr) if text_tag
+    end
+
+    def save_to_disk(filename)
+      if FFI.mitie_save_text_categorizer(filename, @pointer) != 0
+        raise Error, "Unable to save model"
+      end
+      nil
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+  end
+end

data/lib/mitie/text_categorizer_trainer.rb

@@ -0,0 +1,52 @@
+module Mitie
+  class TextCategorizerTrainer
+    def initialize(filename)
+      raise ArgumentError, "File does not exist" unless File.exist?(filename)
+      @pointer = FFI.mitie_create_text_categorizer_trainer(filename)
+
+      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+    end
+
+    def add(tokens, label)
+      tokens_pointer = Utils.array_to_pointer(tokens)
+      FFI.mitie_add_text_categorizer_labeled_text(@pointer, tokens_pointer, label)
+    end
+
+    def beta
+      FFI.mitie_text_categorizer_trainer_get_beta(@pointer)
+    end
+
+    def beta=(value)
+      raise ArgumentError, "beta must be greater than or equal to zero" unless value >= 0
+
+      FFI.mitie_text_categorizer_trainer_set_beta(@pointer, value)
+    end
+
+    def num_threads
+      FFI.mitie_text_categorizer_trainer_get_num_threads(@pointer)
+    end
+
+    def num_threads=(value)
+      FFI.mitie_text_categorizer_trainer_set_num_threads(@pointer, value)
+    end
+
+    def size
+      FFI.mitie_text_categorizer_trainer_size(@pointer)
+    end
+
+    def train
+      raise Error, "You can't call train() on an empty trainer" if size.zero?
+
+      categorizer = FFI.mitie_train_text_categorizer(@pointer)
+
+      raise Error, "Unable to create text categorizer. Probably ran out of RAM." if categorizer.null?
+
+      Mitie::TextCategorizer.new(pointer: categorizer)
+    end
+
+    def self.finalize(pointer)
+      # must use proc instead of stabby lambda
+      proc { FFI.mitie_free(pointer) }
+    end
+  end
+end

data/lib/mitie/utils.rb

@@ -8,5 +8,15 @@ module Mitie
       end
       tokens_ptr
     end
+
+    def self.check_range(range, num_tokens)
+      if range.none? || !(0..(num_tokens - 1)).cover?(range)
+        raise ArgumentError, "Invalid range"
+      end
+    end
+
+    def self.read_double(ptr)
+      ptr.to_s(Fiddle::SIZEOF_DOUBLE).unpack1("d")
+    end
   end
 end

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.1.6"
+  VERSION = "0.2.0"
 end

data/lib/mitie.rb

@@ -3,10 +3,13 @@ require "fiddle/import"
 
 # modules
 require "mitie/binary_relation_detector"
+require "mitie/binary_relation_trainer"
 require "mitie/document"
 require "mitie/ner"
 require "mitie/ner_training_instance"
 require "mitie/ner_trainer"
+require "mitie/text_categorizer"
+require "mitie/text_categorizer_trainer"
 require "mitie/utils"
 require "mitie/version"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-21 00:00:00.000000000 Z
+date: 2022-06-01 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org
@@ -21,11 +21,14 @@ files:
 - README.md
 - lib/mitie.rb
 - lib/mitie/binary_relation_detector.rb
+- lib/mitie/binary_relation_trainer.rb
 - lib/mitie/document.rb
 - lib/mitie/ffi.rb
 - lib/mitie/ner.rb
 - lib/mitie/ner_trainer.rb
 - lib/mitie/ner_training_instance.rb
+- lib/mitie/text_categorizer.rb
+- lib/mitie/text_categorizer_trainer.rb
 - lib/mitie/utils.rb
 - lib/mitie/version.rb
 - vendor/LICENSE.txt
@@ -45,7 +48,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="