blingfire 0.1.3 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af6d8ad14f2c3f56fb148eca079a81bb48b3cec80790bc184fc99afd03572f12
-  data.tar.gz: 92f351ac35b186b54b2d2c261427b3827ccb40613d3dd3fc1f57ce6eec83aa57
+  metadata.gz: 4c6b1fa4c1af8020140b480f8c3579bd62232b25d3cbe470a5149ecac7279a8c
+  data.tar.gz: ea2c66e829368d3858759edb0e0b5644e2f1abc14458d6a8a11e618aacb33951
 SHA512:
-  metadata.gz: 5714bc6333a94669d1b6b69682dd3671933ffc0611107eeb2ce174263c329b42b57c92205afac8519b6626dcae631f724682fccfde57c6514c9cac9233653c62
-  data.tar.gz: 46a6de8f8a081c893db5d446b6e981a170efc4cdcfdaac158f58ba423592f0274990345066a703a687e67d2f2496cb46b2a46cfe8a0f476f19b30afd37fd4697
+  metadata.gz: 714e09d5190152a1009c33000651675405a3f6402f1be056ecce74d73c1aa16c265d3c68245cb0118b5fe6e693d35f21b64b372b8f9d91ae445f063d92c22678
+  data.tar.gz: 3ec81c37b184e4363b4faf16d89a9869362c0655e19be451540dada56f32cba607fc7fe94dbe9210da3a864dc22dd692fc161619d2c80e6cdf0847b8f8029305

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 0.1.7 (2021-09-24)
+
+- Updated Bling Fire to 0.1.8
+- Added `ids_to_text` method
+
+## 0.1.6 (2021-06-07)
+
+- Updated Bling Fire to 0.1.7
+- Added `prefix` option
+
+## 0.1.5 (2021-03-14)
+
+- Updated Bling Fire to 0.1.5
+- Added ARM shared library for Linux
+
+## 0.1.4 (2020-12-28)
+
+- Added ARM shared library for Mac
+
 ## 0.1.3 (2020-10-01)
 
 - Added `text_to_words_with_offsets` method

data/README.md

@@ -2,7 +2,7 @@
 
 [Bling Fire](https://github.com/microsoft/BlingFire) - high speed text tokenization - for Ruby
 
-[![Build Status](https://travis-ci.org/ankane/blingfire.svg?branch=master)](https://travis-ci.org/ankane/blingfire) [![Build status](https://ci.appveyor.com/api/projects/status/3gyca4gsjw2w9ns1/branch/master?svg=true)](https://ci.appveyor.com/project/ankane/blingfire/branch/master)
+[![Build Status](https://github.com/ankane/blingfire/workflows/build/badge.svg?branch=master)](https://github.com/ankane/blingfire/actions)
 
 ## Installation
 
@@ -46,18 +46,16 @@ sentences, start_offsets, end_offsets = model.text_to_sentences_with_offsets(tex
 
 ## Pre-trained Models
 
-BlingFire comes with a default model that follows the tokenization logic of NLTK with a few changes. You can also download other models:
-
-- [BERT Base](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_tok.bin)
-- [BERT Base Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_cased_tok.bin)
-- [BERT Chinese](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_chinese.bin)
-- [BERT Multilingual Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_multi_cased.bin)
-- [Laser 100k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser100k.bin)
-- [Laser 250k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser250k.bin)
-- [Laser 500k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser500k.bin)
-- [XLM Roberta](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlm_roberta_base.bin)
-- [XLNet](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet.bin)
-- [XLNet No Norm](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet_nonorm.bin)
+Bling Fire comes with a default model that follows the tokenization logic of NLTK with a few changes. You can also download other models:
+
+- [BERT Base](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_tok.bin), [BERT Base Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_base_cased_tok.bin), [BERT Chinese](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_chinese.bin), [BERT Multilingual Cased](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/bert_multi_cased.bin)
+- [GPT-2](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/gpt2.bin)
+- [Laser 100k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser100k.bin), [Laser 250k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser250k.bin), [Laser 500k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/laser500k.bin)
+- [RoBERTa](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/roberta.bin)
+- [Syllab](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/syllab.bin)
+- [URI 100k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/uri100k.bin), [URI 250k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/uri250k.bin), [URI 500k](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/uri500k.bin)
+- [XLM-RoBERTa](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlm_roberta_base.bin)
+- [XLNet](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet.bin), [XLNet No Norm](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/xlnet_nonorm.bin)
 - [WBD](https://github.com/microsoft/BlingFire/blob/master/dist-pypi/blingfire/wbd_chuni.bin)
 
 Load a model
@@ -78,6 +76,26 @@ Get offsets for ids
 ids, start_offsets, end_offsets = model.text_to_ids_with_offsets(text)
 ```
 
+Disable prefix space
+
+```ruby
+model = BlingFire.load_model("roberta.bin", prefix: false)
+```
+
+## Ids to Text [experimental]
+
+Load a model
+
+```ruby
+model = BlingFire.load_model("bert_base_tok.i2w")
+```
+
+Convert ids to text
+
+```ruby
+model.ids_to_text(ids)
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/blingfire/blob/master/CHANGELOG.md)

data/lib/blingfire/ffi.rb

@@ -10,6 +10,8 @@ module BlingFire
       raise e
     end
 
+    typealias "bool", "char"
+
     # https://github.com/microsoft/BlingFire/blob/master/blingfiretools/blingfiretokdll/blingfiretokdll.cpp
 
     # version
@@ -40,5 +42,11 @@ module BlingFire
 
     # free model
     extern "int FreeModel(void* ModelPtr)"
+
+    # prefix
+    extern "int SetNoDummyPrefix(void* ModelPtr, bool fNoDummyPrefix)"
+
+    # ids to text
+    extern "int IdsToText(void* ModelPtr, int32_t * pIdsArr, int IdsCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, bool SkipSpecialTokens)"
   end
 end

data/lib/blingfire/model.rb

@@ -1,11 +1,15 @@
 module BlingFire
   class Model
-    def initialize(path = nil)
+    def initialize(path = nil, prefix: nil)
       @handle = nil
       if path
         raise Error, "Model not found" unless File.exist?(path)
         @handle = FFI.LoadModel(path)
         ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
+
+        BlingFire.change_settings_dummy_prefix(@handle, prefix) unless prefix.nil?
+      else
+        raise Error, "prefix option requires path" unless prefix.nil?
       end
     end
 
@@ -57,6 +61,14 @@ module BlingFire
       end
     end
 
+    def ids_to_text(ids, skip_special_tokens: true, output_buffer_size: nil)
+      if @handle
+        BlingFire.ids_to_text(@handle, ids, skip_special_tokens: skip_special_tokens, output_buffer_size: output_buffer_size)
+      else
+        raise "Not implemented"
+      end
+    end
+
     def to_ptr
       @handle
     end

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.3"
+  VERSION = "0.1.7"
 end

data/lib/blingfire.rb

@@ -15,9 +15,17 @@ module BlingFire
     if Gem.win_platform?
       "blingfiretokdll.dll"
     elsif RbConfig::CONFIG["host_os"] =~ /darwin/i
-      "libblingfiretokdll.dylib"
+      if RbConfig::CONFIG["host_cpu"] =~ /arm/i
+        "libblingfiretokdll.arm64.dylib"
+      else
+        "libblingfiretokdll.dylib"
+      end
     else
-      "libblingfiretokdll.so"
+      if RbConfig::CONFIG["host_cpu"] =~ /aarch64/i
+        "libblingfiretokdll.arm64.so"
+      else
+        "libblingfiretokdll.so"
+      end
     end
   vendor_lib = File.expand_path("../vendor/#{lib_name}", __dir__)
   self.ffi_lib = [vendor_lib]
@@ -30,8 +38,8 @@ module BlingFire
       FFI.GetBlingFireTokVersion
     end
 
-    def load_model(path)
-      Model.new(path)
+    def load_model(path, **options)
+      Model.new(path, **options)
     end
 
     def text_to_words(text)
@@ -105,6 +113,15 @@ module BlingFire
       [result].concat(unpack_offsets(start_offsets, end_offsets, result, text))
     end
 
+    def ids_to_text(model, ids, skip_special_tokens: true, output_buffer_size: nil)
+      output_buffer_size ||= ids.size * 32
+      c_ids = Fiddle::Pointer[ids.pack("i*")]
+      out = Fiddle::Pointer.malloc(output_buffer_size)
+      out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
+      check_status out_size, out
+      encode_utf8(out.to_str(out_size - 1))
+    end
+
     def free_model(model)
       FFI.FreeModel(model)
     end
@@ -118,6 +135,12 @@ module BlingFire
       encode_utf8(out.to_str(out_size))
     end
 
+    def change_settings_dummy_prefix(model, value)
+      # use opposite of value
+      ret = FFI.SetNoDummyPrefix(model, value ? 0 : 1)
+      raise Error, "Bad status: #{ret}" if ret != 1
+    end
+
     private
 
     def check_status(ret, ptr)
@@ -163,7 +186,7 @@ module BlingFire
       # TODO see if more efficient to store next_pos in variable
       pos = 0
       text.each_char.with_index do |c, i|
-        while pos == start_bytes[starts.size]
+        while pos == start_bytes[starts.size] || start_bytes[starts.size] == -1
           starts << i
         end
         pos += c.bytesize

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.7
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-02 00:00:00.000000000 Z
+date: 2021-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,7 +52,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
-description: 
+description:
 email: andrew@chartkick.com
 executables: []
 extensions: []
@@ -67,13 +67,15 @@ files:
 - lib/blingfire/version.rb
 - vendor/LICENSE
 - vendor/blingfiretokdll.dll
+- vendor/libblingfiretokdll.arm64.dylib
+- vendor/libblingfiretokdll.arm64.so
 - vendor/libblingfiretokdll.dylib
 - vendor/libblingfiretokdll.so
 homepage: https://github.com/ankane/blingfire
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -88,8 +90,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: High speed text tokenization for Ruby
 test_files: []