mitie 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50fb12adcd0042b3c09968108ec382694f8f0df20750b88566bc64c8d85d9e8d
-  data.tar.gz: 9d3d34f9839f71fc17e6651a17069f1f0fdb4b5956f9e57c4b66c9567d908967
+  metadata.gz: dc2b7bdba2fba6b335ab9750efab8766190e618c4b8e2542ff6409f6727ec8b2
+  data.tar.gz: 03d85b928082a04b46209694c8e1b294e5c4205057a26b5cc65e695bbe3564a9
 SHA512:
-  metadata.gz: 189d5a17f94fff9abbc1d8d961c8f8407ae85bd1284f727e363b9aa7384d748eb1d80b6ffa1b1a44c8c6fadbbb6e91e6d064ea58fa88af972e6fd6e90e3ef71d
-  data.tar.gz: 202ba418ee98636736185a8f4783601acabecc59dde7e3a5c31a12b56827497b6b2b7e7e26d264b61848281b94cb635249b17a9e8fa009c496dacbe24e6466ca
+  metadata.gz: 4092a2dc005bb76527429454c301179c63f0eeee4913a3a8f56190e13d7f7551ee3bbfcd1098a6c335cf3b886ec0dee4e52f5c062975a4a058d117229f45b340
+  data.tar.gz: 066e5800a520b16002388088fc367939a26738bd638631d3b86ff9074d508cf5cd051154fbbf7be129f07f82c97a851caf7a6486e6fff2fb7f6a4b252d426ec0

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.1 (2022-06-12)
+
+- Added `tokenize` and `tokenize_file` methods
+- Added support for untokenized text to text categorization
+
 ## 0.2.0 (2022-06-01)
 
 - Added support for text categorization

data/README.md

@@ -183,7 +183,7 @@ trainer = Mitie::TextCategorizerTrainer.new("total_word_feature_extractor.dat")
 Add labeled text to the trainer
 
 ```ruby
-trainer.add(["This", "is", "super", "cool"], "positive")
+trainer.add("This is super cool", "positive")
 ```
 
 Train the model
@@ -207,7 +207,7 @@ model = Mitie::TextCategorizer.new("text_categorization_model.dat")
 Categorize text
 
 ```ruby
-model.categorize(["What", "a", "super", "nice", "day"])
+model.categorize("What a super nice day")
 ```
 
 ## Deployment

data/lib/mitie/ffi.rb

@@ -14,6 +14,7 @@ module Mitie
 
     extern "void mitie_free(void* object)"
     extern "char** mitie_tokenize(const char* text)"
+    extern "char** mitie_tokenize_file(const char* filename)"
     extern "char** mitie_tokenize_with_offsets(const char* text, unsigned long** token_offsets)"
 
     # ner

data/lib/mitie/text_categorizer.rb

@@ -14,7 +14,8 @@ module Mitie
       ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
     end
 
-    def categorize(tokens)
+    def categorize(text)
+      tokens = text.is_a?(Array) ? text : Mitie.tokenize(text)
       tokens_pointer = Utils.array_to_pointer(tokens)
       text_tag = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
       text_score = Fiddle::Pointer.malloc(Fiddle::SIZEOF_DOUBLE)

data/lib/mitie/text_categorizer_trainer.rb

@@ -7,7 +7,8 @@ module Mitie
       ObjectSpace.define_finalizer(self, self.class.finalize(@pointer))
     end
 
-    def add(tokens, label)
+    def add(text, label)
+      tokens = text.is_a?(Array) ? text : Mitie.tokenize(text)
       tokens_pointer = Utils.array_to_pointer(tokens)
       FFI.mitie_add_text_categorizer_labeled_text(@pointer, tokens_pointer, label)
     end

data/lib/mitie/version.rb

@@ -1,3 +1,3 @@
 module Mitie
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/mitie.rb

@@ -36,4 +36,36 @@ module Mitie
 
   # friendlier error message
   autoload :FFI, "mitie/ffi"
+
+  class << self
+    def tokenize(text)
+      tokens_ptr = FFI.mitie_tokenize(text)
+      tokens = read_tokens(tokens_ptr)
+      tokens.each { |t| t.force_encoding(text.encoding) }
+      tokens
+    ensure
+      FFI.mitie_free(tokens_ptr) if tokens_ptr
+    end
+
+    def tokenize_file(filename)
+      tokens_ptr = FFI.mitie_tokenize_file(filename)
+      read_tokens(tokens_ptr)
+    ensure
+      FFI.mitie_free(tokens_ptr) if tokens_ptr
+    end
+
+    private
+
+    def read_tokens(tokens_ptr)
+      i = 0
+      tokens = []
+      loop do
+        token = (tokens_ptr + i * Fiddle::SIZEOF_VOIDP).ptr
+        break if token.null?
+        tokens << token.to_s
+        i += 1
+      end
+      tokens
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mitie
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-01 00:00:00.000000000 Z
+date: 2022-06-12 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org