blingfire 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a93f90584141a618ff5bb73860068e2ea662c568a07e2f877a5f2be7e265919e
-  data.tar.gz: 61b49a4bd7eb304f44c041730bce733102f1936c9340cd9c0c3f1390bd9e853a
+  metadata.gz: 225c7163bbbd6bc06e0b7d54b2a0a64c3b622c174e5c6cb0b8fa54b6e68740a3
+  data.tar.gz: b9d8ce2d21a450c53b1628bdab97b07aa2c39dd1b758fa763c39f052a944084c
 SHA512:
-  metadata.gz: fd0242fc13696c620ffd3a7383b9088aaddade1f7280c4f35c54d5165f66e26b783e88fa2d65ced002e8f233af0637e43e9adb68ad7556a16cfdcbb6aff9ba7e
-  data.tar.gz: a1206eaa056ed93639c00c03eb16be5fbbec6b077e1677a916b69c6244c26d3c2cce51299dc0394335c54d12de500a9648f30fa6f3785ef749be528904d5ae60
+  metadata.gz: 98eed6c71ee974381772e032b27d022cd9c9485a83e8dae25c21817a19489953c7c5dfd4bb4498d796b65a569bd3468f55df558603d904ed9293a4c21e7041d5
+  data.tar.gz: 15389737eec2ad109f528bf7cd064184205bc17d822d108031f29eb7cf9d421c0a84f40a3d170461e7fa3c853b1fbc912b28f07e403924f9744cb8f6c2294f2d

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.1 (2020-05-01)
+
+- Updated Bling Fire to 0.1.1
+- Improved error message when model not found
+
 ## 0.1.0 (2020-02-24)
 
 - First release

data/README.md

@@ -1,6 +1,8 @@
-# BlingFire
+# Bling Fire
 
-[BlingFire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
+[Bling Fire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
+
+[![Build Status](https://travis-ci.org/ankane/blingfire.svg?branch=master)](https://travis-ci.org/ankane/blingfire) [![Build status](https://ci.appveyor.com/api/projects/status/3gyca4gsjw2w9ns1/branch/master?svg=true)](https://ci.appveyor.com/project/ankane/blingfire/branch/master)
 
 ## Installation
 
@@ -39,6 +41,8 @@ BlingFire comes with a default model that follows the tokenization logic of NLTK
 - [BERT Chinese](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_chinese.bin)
 - [BERT Multilingual Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_multi_cased.bin)
 - [WBD](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/wbd_chuni.bin)
+- [XLNet](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet.bin)
+- [XLNet No Norm](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet_nonorm.bin)
 
 Load a model
 

data/lib/blingfire.rb

@@ -35,42 +35,34 @@ module BlingFire
     end
 
     def text_to_words(text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToWords(text, text.bytesize, out, out.size)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split(" ")
+      text_to(text, " ") do |t, out|
+        FFI.TextToWords(t, t.bytesize, out, out.size)
+      end
     end
 
     def text_to_words_with_model(model, text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToWordsWithModel(text, text.bytesize, out, out.size, model)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split(" ")
+      text_to(text, " ") do |t, out|
+        FFI.TextToWordsWithModel(t, t.bytesize, out, out.size, model)
+      end
     end
 
     def text_to_sentences(text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToSentences(text, text.bytesize, out, out.size)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split("\n")
+      text_to(text, "\n") do |t, out|
+        FFI.TextToSentences(t, t.bytesize, out, out.size)
+      end
     end
 
     def text_to_sentences_with_model(model, text)
-      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc(text.bytesize * 3)
-      out_size = FFI.TextToSentencesWithModel(text, text.bytesize, out, out.size, model)
-      check_status out_size
-      encode_utf8(out[0, out_size - 1]).split("\n")
+      text_to(text, "\n") do |t, out|
+        FFI.TextToSentencesWithModel(t, t.bytesize, out, out.size, model)
+      end
     end
 
     def text_to_ids(model, text, max_len = nil, unk_id = 0)
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
       ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT)
       out_size = FFI.TextToIds(model, text, text.bytesize, ids, ids.size, unk_id)
-      check_status out_size
+      check_status out_size, ids
       ids[0, (max_len || out_size) * Fiddle::SIZEOF_INT].unpack("i!*")
     end
 
@@ -80,8 +72,17 @@ module BlingFire
 
     private
 
-    def check_status(ret)
-      raise Error, "Bad status" if ret == -1
+    def check_status(ret, ptr)
+      raise Error, "Not enough memory allocated" if ret == -1 || ret > ptr.size
+    end
+
+    def text_to(text, sep)
+      text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
+      # TODO allocate less, and try again if needed
+      out = Fiddle::Pointer.malloc([text.bytesize * 1.5, 20].max)
+      out_size = yield(text, out)
+      check_status out_size, out
+      encode_utf8(out.to_str(out_size - 1)).split(sep)
     end
 
     def encode_utf8(text)

data/lib/blingfire/model.rb

@@ -2,6 +2,7 @@ module BlingFire
   class Model
     def initialize(path = nil)
       if path
+        raise Error, "Model not found" unless File.exist?(path)
         @handle = FFI.LoadModel(path)
         ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
       end

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-24 00:00:00.000000000 Z
+date: 2020-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler