mitie 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 888f1cb252088e179987e76156daf745d4ea7735c69863b8cca376b325c09f55
-  data.tar.gz: e282533847927dd119ca935e969745dd77603c5367a32542550b5425f10b98c2
+  metadata.gz: da12926ff7be8d2b00da34c945544b556c88a9ba6eeb8292d22feb946a39c1fa
+  data.tar.gz: 2f32b90e2b81f6838e43a097a804b0442730b345eccf90bcabcdb181c0a83d1a
 SHA512:
-  metadata.gz: a2fc5998dd1811288b9be022f8dd181865c711aa3b46feac8f339affb2e4befaaed78aa06ab49bd791bf99f1dff117e3604ddd593c40d8374819babe23a95ae2
-  data.tar.gz: 167f9eace3b0fe9ef1863598244802a62650c57457922a04793f1ef9a22e1668f3ad12ac9ac7702cd43063d3197b1e55a6e4be96a9bfd3932a7549870d5302a4
+  metadata.gz: fd2506381d76577a692d31f8fd126e2ab77d86b6e6d7cc7d7b8826c79462c642653f195732ee346ec6e00364ce145fc5e36b28ef0f9e7f2d0a564479829f912c
+  data.tar.gz: 17b1fc7618990192f1f0cdd28a619caf245c94ebbb0799fa9e32a2b86562484d1e420321464b57b97ff3cefa016e9abfbee556c3fffb5a9cdfd72db89686a468

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.3.3 (2025-11-10)
+
+- Added ARM shared library for Linux
+
+## 0.3.2 (2025-05-04)
+
+- Fixed memory leaks
+
 ## 0.3.1 (2024-12-29)
 
 - Fixed warning with Ruby 3.4

data/lib/mitie/binary_relation_detector.rb

@@ -4,7 +4,7 @@ module Mitie
       if path
         # better error message
         raise ArgumentError, "File does not exist" unless File.exist?(path)
-        @pointer = FFI.mitie_load_binary_relation_detector(path)
+        @pointer = FFI.mitie_load_binary_relation_detector(+path)
         @pointer.free = FFI["mitie_free"]
       elsif pointer
         @pointer = pointer
@@ -36,7 +36,7 @@ module Mitie
     end
 
     def save_to_disk(filename)
-      if FFI.mitie_save_binary_relation_detector(filename, pointer) != 0
+      if FFI.mitie_save_binary_relation_detector(+filename, pointer) != 0
         raise Error, "Unable to save detector"
       end
       nil
@@ -58,8 +58,9 @@ module Mitie
           entity2[:token_index],
           entity2[:token_length]
         )
+      relation.free = FFI["mitie_free"]
 
-      score_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
+      score_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE, Fiddle::RUBY_FREE)
       status = FFI.mitie_classify_binary_relation(pointer, relation, score_ptr)
       raise Error, "Bad status: #{status}" if status != 0
 
@@ -71,8 +72,6 @@ module Mitie
           score: score
         }
       end
-    ensure
-      FFI.mitie_free(relation) if relation
     end
   end
 end

data/lib/mitie/binary_relation_trainer.rb

@@ -1,7 +1,7 @@
 module Mitie
   class BinaryRelationTrainer
     def initialize(ner, name: "")
-      @pointer = FFI.mitie_create_binary_relation_trainer(name, ner.pointer)
+      @pointer = FFI.mitie_create_binary_relation_trainer(+name, ner.pointer)
       @pointer.free = FFI["mitie_free"]
     end
 

data/lib/mitie/document.rb

@@ -22,7 +22,7 @@ module Mitie
           loop do
             token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
             break if token.null?
-            offset = (offsets_ptr.ptr + i * Fiddle::SIZEOF_LONG).to_s(Fiddle::SIZEOF_LONG).unpack1("L!")
+            offset = (offsets_ptr + i * Fiddle::SIZEOF_LONG).to_str(Fiddle::SIZEOF_LONG).unpack1("L!")
             tokens << [token.to_s.force_encoding(text.encoding), offset]
             i += 1
           end
@@ -36,6 +36,7 @@ module Mitie
         entities = []
         tokens = tokens_with_offset
         detections = FFI.mitie_extract_entities(pointer, tokens_ptr)
+        detections.free = FFI["mitie_free"]
         num_detections = FFI.mitie_ner_get_num_detections(detections)
         num_detections.times do |i|
           pos = FFI.mitie_ner_get_detection_position(detections, i)
@@ -60,8 +61,6 @@ module Mitie
           entities << entity
         end
         entities
-      ensure
-        FFI.mitie_free(detections) if detections
       end
     end
 
@@ -85,20 +84,15 @@ module Mitie
           tokens_ptr = Utils.array_to_pointer(text)
           [tokens_ptr, nil]
         else
-          offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
-          tokens_ptr = FFI.mitie_tokenize_with_offsets(text, offsets_ptr)
+          offsets_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
+          tokens_ptr = FFI.mitie_tokenize_with_offsets(+text, offsets_ptr)
           tokens_ptr.free = FFI["mitie_free"]
-
-          ObjectSpace.define_finalizer(offsets_ptr, self.class.finalize_ptr(offsets_ptr))
+          offsets_ptr = offsets_ptr.ptr
+          offsets_ptr.free = FFI["mitie_free"]
 
           [tokens_ptr, offsets_ptr]
         end
       end
     end
-
-    def self.finalize_ptr(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.mitie_free(pointer.ptr) }
-    end
   end
 end

data/lib/mitie/ner.rb

@@ -6,7 +6,7 @@ module Mitie
       if path
         # better error message
         raise ArgumentError, "File does not exist" unless File.exist?(path)
-        @pointer = FFI.mitie_load_named_entity_extractor(path)
+        @pointer = FFI.mitie_load_named_entity_extractor(+path)
         @pointer.free = FFI["mitie_free"]
       elsif pointer
         @pointer = pointer
@@ -30,7 +30,7 @@ module Mitie
     end
 
     def save_to_disk(filename)
-      if FFI.mitie_save_named_entity_extractor(filename, pointer) != 0
+      if FFI.mitie_save_named_entity_extractor(+filename, pointer) != 0
         raise Error, "Unable to save model"
       end
       nil

data/lib/mitie/ner_trainer.rb

@@ -2,7 +2,7 @@ module Mitie
   class NERTrainer
     def initialize(filename)
       raise ArgumentError, "File does not exist" unless File.exist?(filename)
-      @pointer = FFI.mitie_create_ner_trainer(filename)
+      @pointer = FFI.mitie_create_ner_trainer(+filename)
       @pointer.free = FFI["mitie_free"]
     end
 

data/lib/mitie/ner_training_instance.rb

@@ -15,7 +15,7 @@ module Mitie
 
       raise ArgumentError, "Range overlaps existing entity" if overlaps_any_entity?(range)
 
-      unless FFI.mitie_add_ner_training_entity(@pointer, range.begin, range.size, label).zero?
+      unless FFI.mitie_add_ner_training_entity(@pointer, range.begin, range.size, +label).zero?
         raise Error, "Unable to add entity to training instance. Probably ran out of RAM."
       end
 

data/lib/mitie/text_categorizer.rb

@@ -4,7 +4,7 @@ module Mitie
       if path
         # better error message
         raise ArgumentError, "File does not exist" unless File.exist?(path)
-        @pointer = FFI.mitie_load_text_categorizer(path)
+        @pointer = FFI.mitie_load_text_categorizer(+path)
         @pointer.free = FFI["mitie_free"]
       elsif pointer
         @pointer = pointer
@@ -16,24 +16,23 @@ module Mitie
     def categorize(text)
       tokens = text.is_a?(Array) ? text : Mitie.tokenize(text)
       tokens_pointer = Utils.array_to_pointer(tokens)
-      text_tag = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
-      text_score = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)
+      text_tag = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP, Fiddle::RUBY_FREE)
+      text_score = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE, Fiddle::RUBY_FREE)
 
       if FFI.mitie_categorize_text(@pointer, tokens_pointer, text_tag, text_score) != 0
         raise Error, "Unable to categorize"
       end
+      text_tag = text_tag.ptr
+      text_tag.free = FFI["mitie_free"]
 
       {
-        tag: text_tag.ptr.to_s,
+        tag: text_tag.to_s,
         score: Utils.read_double(text_score)
       }
-    ensure
-      # text_tag must be freed
-      FFI.mitie_free(text_tag.ptr) if text_tag
     end
 
     def save_to_disk(filename)
-      if FFI.mitie_save_text_categorizer(filename, @pointer) != 0
+      if FFI.mitie_save_text_categorizer(+filename, @pointer) != 0
         raise Error, "Unable to save model"
       end
       nil

data/lib/mitie/text_categorizer_trainer.rb

@@ -2,14 +2,14 @@ module Mitie
   class TextCategorizerTrainer
     def initialize(filename)
       raise ArgumentError, "File does not exist" unless File.exist?(filename)
-      @pointer = FFI.mitie_create_text_categorizer_trainer(filename)
+      @pointer = FFI.mitie_create_text_categorizer_trainer(+filename)
       @pointer.free = FFI["mitie_free"]
     end
 
     def add(text, label)
       tokens = text.is_a?(Array) ? text : Mitie.tokenize(text)
       tokens_pointer = Utils.array_to_pointer(tokens)
-      FFI.mitie_add_text_categorizer_labeled_text(@pointer, tokens_pointer, label)
+      FFI.mitie_add_text_categorizer_labeled_text(@pointer, tokens_pointer, +label)
     end
 
     def beta

data/lib/mitie/utils.rb

@@ -2,10 +2,12 @@ module Mitie
   module Utils
     def self.array_to_pointer(text)
       # malloc uses memset to set all bytes to 0
-      tokens_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP * (text.size + 1))
+      tokens_ptr = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP * (text.size + 1), Fiddle::RUBY_FREE)
+      text_ptrs = text.map { |v| Fiddle::Pointer[v] }
       text.size.times do |i|
-        tokens_ptr[i * Fiddle::SIZEOF_VOIDP, Fiddle::SIZEOF_VOIDP] = Fiddle::Pointer.to_ptr(text[i]).ref
+        tokens_ptr[i * Fiddle::SIZEOF_VOIDP, Fiddle::SIZEOF_VOIDP] = text_ptrs[i].ref
       end
+      tokens_ptr.instance_variable_set(:@mitie_refs, text_ptrs)
       tokens_ptr
     end
 
@@ -16,7 +18,7 @@ module Mitie
     end
 
     def self.read_double(ptr)
-      ptr.to_s(Fiddle::SIZEOF_DOUBLE).unpack1("d")
+      ptr.to_str(Fiddle::SIZEOF_DOUBLE).unpack1("d")
     end
   end
 end

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.3.1"
+  VERSION = "0.3.3"
 end

data/lib/mitie.rb

@@ -29,7 +29,11 @@ module Mitie
         "libmitie.dylib"
       end
     else
-      "libmitie.so"
+      if RbConfig::CONFIG["host_cpu"] =~ /arm|aarch64/i
+        "libmitie.arm64.so"
+      else
+        "libmitie.so"
+      end
     end
   vendor_lib = File.expand_path("../vendor/#{lib_name}", __dir__)
   self.ffi_lib = [vendor_lib]
@@ -39,20 +43,18 @@ module Mitie
 
   class << self
     def tokenize(text)
-      tokens_ptr = FFI.mitie_tokenize(text.to_s)
+      tokens_ptr = FFI.mitie_tokenize(+text.to_s)
+      tokens_ptr.free = FFI["mitie_free"]
       tokens = read_tokens(tokens_ptr)
       tokens.each { |t| t.force_encoding(text.encoding) }
       tokens
-    ensure
-      FFI.mitie_free(tokens_ptr) if tokens_ptr
     end
 
     def tokenize_file(filename)
       raise ArgumentError, "File does not exist" unless File.exist?(filename)
-      tokens_ptr = FFI.mitie_tokenize_file(filename)
+      tokens_ptr = FFI.mitie_tokenize_file(+filename)
+      tokens_ptr.free = FFI["mitie_free"]
       read_tokens(tokens_ptr)
-    ensure
-      FFI.mitie_free(tokens_ptr) if tokens_ptr
     end
 
     private

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Andrew Kane
 bindir: bin
 cert_chain: []
-date: 2024-12-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fiddle
@@ -45,6 +45,7 @@ files:
 - lib/mitie/version.rb
 - vendor/LICENSE.txt
 - vendor/libmitie.arm64.dylib
+- vendor/libmitie.arm64.so
 - vendor/libmitie.dylib
 - vendor/libmitie.so
 - vendor/mitie.dll
@@ -66,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Named-entity recognition for Ruby
 test_files: []