mitie 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50fb12adcd0042b3c09968108ec382694f8f0df20750b88566bc64c8d85d9e8d
-  data.tar.gz: 9d3d34f9839f71fc17e6651a17069f1f0fdb4b5956f9e57c4b66c9567d908967
+  metadata.gz: 4106166da7718e20cce4c31c7f3974b12b06e1d2d5c7f30327981c0d0d31652a
+  data.tar.gz: 9345cf0bedb85634dd43c2aaef3438870649479add03a3ca48bc0f8892822c4f
 SHA512:
-  metadata.gz: 189d5a17f94fff9abbc1d8d961c8f8407ae85bd1284f727e363b9aa7384d748eb1d80b6ffa1b1a44c8c6fadbbb6e91e6d064ea58fa88af972e6fd6e90e3ef71d
-  data.tar.gz: 202ba418ee98636736185a8f4783601acabecc59dde7e3a5c31a12b56827497b6b2b7e7e26d264b61848281b94cb635249b17a9e8fa009c496dacbe24e6466ca
+  metadata.gz: dc6581ed4865bff3a21724a056946d90526be0fc1d32803e625a918247075331adf7a5e721cfddbdc366c4df47d2e91c739953bccbd7e451cdb344c12fdf27de
+  data.tar.gz: d96cab2092c6cca5f25f51e1fae4977874d1d85e881d1686110c99d2b975de288e57b725035a766ea80c1304013d91e13953c5caa41d6572e7ab9bacbdad7ae8

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.2.2 (2023-06-07)
+
+- Fixed error with `dup` and `clone`
+
+## 0.2.1 (2022-06-12)
+
+- Added `tokenize` and `tokenize_file` methods
+- Added support for untokenized text to text categorization
+
 ## 0.2.0 (2022-06-01)
 
 - Added support for text categorization

data/README.md

@@ -183,7 +183,7 @@ trainer = Mitie::TextCategorizerTrainer.new("total_word_feature_extractor.dat")
 Add labeled text to the trainer
 
 ```ruby
-trainer.add(["This", "is", "super", "cool"], "positive")
+trainer.add("This is super cool", "positive")
 ```
 
 Train the model
@@ -207,7 +207,7 @@ model = Mitie::TextCategorizer.new("text_categorization_model.dat")
 Categorize text
 
 ```ruby
-model.categorize(["What", "a", "super", "nice", "day"])
+model.categorize("What a super nice day")
 ```
 
 ## Deployment

data/lib/mitie/binary_relation_detector.rb

@@ -5,13 +5,12 @@ module Mitie
         # better error message
         raise ArgumentError, "File does not exist" unless File.exist?(path)
         @pointer = FFI.mitie_load_binary_relation_detector(path)
+        @pointer.free = FFI["mitie_free"]
       elsif pointer
         @pointer = pointer
       else
         raise ArgumentError, "Must pass either a path or a pointer"
       end
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(pointer))
     end
 
     def name
@@ -75,10 +74,5 @@ module Mitie
     ensure
       FFI.mitie_free(relation) if relation
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/binary_relation_trainer.rb

@@ -2,8 +2,7 @@ module Mitie
   class BinaryRelationTrainer
     def initialize(ner, name: "")
       @pointer = FFI.mitie_create_binary_relation_trainer(name, ner.pointer)
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+      @pointer.free = FFI["mitie_free"]
     end
 
     def add_positive_binary_relation(tokens, range1, range2)
@@ -78,10 +77,5 @@ module Mitie
     def entities_overlap?(range1, range2)
       FFI.mitie_entities_overlap(range1.begin, range1.size, range2.begin, range2.size) == 1
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/document.rb

@@ -87,8 +87,8 @@ module Mitie
         else
           offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
           tokens_ptr = FFI.mitie_tokenize_with_offsets(text, offsets_ptr)
+          tokens_ptr.free = FFI["mitie_free"]
 
-          ObjectSpace.define_finalizer(tokens_ptr, self.class.finalize(tokens_ptr))
           ObjectSpace.define_finalizer(offsets_ptr, self.class.finalize_ptr(offsets_ptr))
 
           [tokens_ptr, offsets_ptr]
@@ -96,11 +96,6 @@ module Mitie
       end
     end
 
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
-
     def self.finalize_ptr(pointer)
       # must use proc instead of stabby lambda
       proc { FFI.mitie_free(pointer.ptr) }

data/lib/mitie/ffi.rb

@@ -14,6 +14,7 @@ module Mitie
 
     extern "void mitie_free(void* object)"
     extern "char** mitie_tokenize(const char* text)"
+    extern "char** mitie_tokenize_file(const char* filename)"
     extern "char** mitie_tokenize_with_offsets(const char* text, unsigned long** token_offsets)"
 
     # ner

data/lib/mitie/ner.rb

@@ -7,13 +7,12 @@ module Mitie
         # better error message
         raise ArgumentError, "File does not exist" unless File.exist?(path)
         @pointer = FFI.mitie_load_named_entity_extractor(path)
+        @pointer.free = FFI["mitie_free"]
       elsif pointer
         @pointer = pointer
       else
         raise ArgumentError, "Must pass either a path or a pointer"
       end
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
     end
 
     def tags
@@ -44,10 +43,5 @@ module Mitie
     def tokens_with_offset(text)
       doc(text).tokens_with_offset
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/ner_trainer.rb

@@ -3,8 +3,7 @@ module Mitie
     def initialize(filename)
       raise ArgumentError, "File does not exist" unless File.exist?(filename)
       @pointer = FFI.mitie_create_ner_trainer(filename)
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+      @pointer.free = FFI["mitie_free"]
     end
 
     def add(instance)
@@ -42,10 +41,5 @@ module Mitie
 
       Mitie::NER.new(pointer: extractor)
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/ner_training_instance.rb

@@ -7,8 +7,7 @@ module Mitie
 
       @pointer = FFI.mitie_create_ner_training_instance(tokens_pointer)
       raise Error, "Unable to create training instance. Probably ran out of RAM." if @pointer.null?
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+      @pointer.free = FFI["mitie_free"]
     end
 
     def add_entity(range, label)
@@ -36,10 +35,5 @@ module Mitie
 
       FFI.mitie_overlaps_any_entity(@pointer, range.begin, range.size) == 1
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/text_categorizer.rb

@@ -5,16 +5,16 @@ module Mitie
         # better error message
         raise ArgumentError, "File does not exist" unless File.exist?(path)
         @pointer = FFI.mitie_load_text_categorizer(path)
+        @pointer.free = FFI["mitie_free"]
       elsif pointer
         @pointer = pointer
       else
         raise ArgumentError, "Must pass either a path or a pointer"
       end
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
     end
 
-    def categorize(tokens)
+    def categorize(text)
+      tokens = text.is_a?(Array) ? text : Mitie.tokenize(text)
       tokens_pointer = Utils.array_to_pointer(tokens)
       text_tag = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
       text_score = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
@@ -38,10 +38,5 @@ module Mitie
       end
       nil
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/text_categorizer_trainer.rb

@@ -3,11 +3,11 @@ module Mitie
     def initialize(filename)
       raise ArgumentError, "File does not exist" unless File.exist?(filename)
       @pointer = FFI.mitie_create_text_categorizer_trainer(filename)
-
-      ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
+      @pointer.free = FFI["mitie_free"]
     end
 
-    def add(tokens, label)
+    def add(text, label)
+      tokens = text.is_a?(Array) ? text : Mitie.tokenize(text)
       tokens_pointer = Utils.array_to_pointer(tokens)
       FFI.mitie_add_text_categorizer_labeled_text(@pointer, tokens_pointer, label)
     end
@@ -43,10 +43,5 @@ module Mitie
 
       Mitie::TextCategorizer.new(pointer: categorizer)
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer) }
-    end
   end
 end

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

data/lib/mitie.rb

@@ -2,16 +2,16 @@
 require "fiddle/import"
 
 # modules
-require "mitie/binary_relation_detector"
-require "mitie/binary_relation_trainer"
-require "mitie/document"
-require "mitie/ner"
-require "mitie/ner_training_instance"
-require "mitie/ner_trainer"
-require "mitie/text_categorizer"
-require "mitie/text_categorizer_trainer"
-require "mitie/utils"
-require "mitie/version"
+require_relative "mitie/binary_relation_detector"
+require_relative "mitie/binary_relation_trainer"
+require_relative "mitie/document"
+require_relative "mitie/ner"
+require_relative "mitie/ner_training_instance"
+require_relative "mitie/ner_trainer"
+require_relative "mitie/text_categorizer"
+require_relative "mitie/text_categorizer_trainer"
+require_relative "mitie/utils"
+require_relative "mitie/version"
 
 module Mitie
   class Error < StandardError; end
@@ -36,4 +36,37 @@ module Mitie
 
   # friendlier error message
   autoload :FFI, "mitie/ffi"
+
+  class << self
+    def tokenize(text)
+      tokens_ptr = FFI.mitie_tokenize(text.to_s)
+      tokens = read_tokens(tokens_ptr)
+      tokens.each { |t| t.force_encoding(text.encoding) }
+      tokens
+    ensure
+      FFI.mitie_free(tokens_ptr) if tokens_ptr
+    end
+
+    def tokenize_file(filename)
+      raise ArgumentError, "File does not exist" unless File.exist?(filename)
+      tokens_ptr = FFI.mitie_tokenize_file(filename)
+      read_tokens(tokens_ptr)
+    ensure
+      FFI.mitie_free(tokens_ptr) if tokens_ptr
+    end
+
+    private
+
+    def read_tokens(tokens_ptr)
+      i = 0
+      tokens = []
+      loop do
+        token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+        break if token.null?
+        tokens << token.to_s
+        i += 1
+      end
+      tokens
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-01 00:00:00.000000000 Z
+date: 2023-06-08 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org
@@ -55,7 +55,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Named-entity recognition for Ruby