blingfire 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa9f7ebf09e5745b6865d4c5645b36f42b7c484ed667ce493febee80887e89ad
-  data.tar.gz: 7fb6e8716138c35396964081b8ad60d7c01db88e71e4fbe219d0ea27a6eff94b
+  metadata.gz: 4c6b1fa4c1af8020140b480f8c3579bd62232b25d3cbe470a5149ecac7279a8c
+  data.tar.gz: ea2c66e829368d3858759edb0e0b5644e2f1abc14458d6a8a11e618aacb33951
 SHA512:
-  metadata.gz: 4f1e20f3520f4be17af2df5db4b3e05b169756f1667dccb08dbafb89ee07f633862d16649fa8e3be505073ca58a11989cc5c9b99d3f571aae2eaff3d4053dd8c
-  data.tar.gz: 8f70ecb06ddcad466508a0debe63fd99e9a5cbab35e3702296c8ff522ffbb5075ad52b2ce68d2f316de6685256b756ff098bc5a2b29e807652b89de84f9b4661
+  metadata.gz: 714e09d5190152a1009c33000651675405a3f6402f1be056ecce74d73c1aa16c265d3c68245cb0118b5fe6e693d35f21b64b372b8f9d91ae445f063d92c22678
+  data.tar.gz: 3ec81c37b184e4363b4faf16d89a9869362c0655e19be451540dada56f32cba607fc7fe94dbe9210da3a864dc22dd692fc161619d2c80e6cdf0847b8f8029305

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.7 (2021-09-24)
+
+- Updated Bling Fire to 0.1.8
+- Added `ids_to_text` method
+
 ## 0.1.6 (2021-06-07)
 
 - Updated Bling Fire to 0.1.7

data/README.md

@@ -82,6 +82,20 @@ Disable prefix space
 model = BlingFire.load_model("roberta.bin", prefix: false)
 ```
 
+## Ids to Text [experimental]
+
+Load a model
+
+```ruby
+model = BlingFire.load_model("bert_base_tok.i2w")
+```
+
+Convert ids to text
+
+```ruby
+model.ids_to_text(ids)
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/blingfire/blob/master/CHANGELOG.md)

data/lib/blingfire/ffi.rb

@@ -45,5 +45,8 @@ module BlingFire
 
     # prefix
     extern "int SetNoDummyPrefix(void* ModelPtr, bool fNoDummyPrefix)"
+
+    # ids to text
+    extern "int IdsToText(void* ModelPtr, int32_t * pIdsArr, int IdsCount, char * pOutUtf8Str, int MaxOutUtf8StrByteCount, bool SkipSpecialTokens)"
   end
 end

data/lib/blingfire/model.rb

@@ -61,6 +61,14 @@ module BlingFire
       end
     end
 
+    def ids_to_text(ids, skip_special_tokens: true, output_buffer_size: nil)
+      if @handle
+        BlingFire.ids_to_text(@handle, ids, skip_special_tokens: skip_special_tokens, output_buffer_size: output_buffer_size)
+      else
+        raise "Not implemented"
+      end
+    end
+
     def to_ptr
       @handle
     end

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/blingfire.rb

@@ -113,6 +113,15 @@ module BlingFire
       [result].concat(unpack_offsets(start_offsets, end_offsets, result, text))
     end
 
+    def ids_to_text(model, ids, skip_special_tokens: true, output_buffer_size: nil)
+      output_buffer_size ||= ids.size * 32
+      c_ids = Fiddle::Pointer[ids.pack("i*")]
+      out = Fiddle::Pointer.malloc(output_buffer_size)
+      out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
+      check_status out_size, out
+      encode_utf8(out.to_str(out_size - 1))
+    end
+
     def free_model(model)
       FFI.FreeModel(model)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-07 00:00:00.000000000 Z
+date: 2021-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: High speed text tokenization for Ruby