blingfire 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2585ca0684a0d6af6beaae99ab7b5250eb928c5200cbfb6274dcd2cbc914dccc
-  data.tar.gz: fe4681abb0c63e7d8fd0e8a777abaed95e578bbb0cfd7238bacbe9beecd3d07c
+  metadata.gz: fa9f7ebf09e5745b6865d4c5645b36f42b7c484ed667ce493febee80887e89ad
+  data.tar.gz: 7fb6e8716138c35396964081b8ad60d7c01db88e71e4fbe219d0ea27a6eff94b
 SHA512:
-  metadata.gz: 9a02c3f87eea7ea989f73032388f4abe69b30055f1ea7a96aa302385a2f6038eb833cffe00b15e5b69124d7a11bf8dd031067a4a16b415999955b8ce393f25d4
-  data.tar.gz: e696df2343cab6ee82af95bedcfc45457b69287113f819514851ef5f943a2f0f8c165b21abff0f944dcd96d4cab45b8faa1190940bf36c294c9c98d55bd9d980
+  metadata.gz: 4f1e20f3520f4be17af2df5db4b3e05b169756f1667dccb08dbafb89ee07f633862d16649fa8e3be505073ca58a11989cc5c9b99d3f571aae2eaff3d4053dd8c
+  data.tar.gz: 8f70ecb06ddcad466508a0debe63fd99e9a5cbab35e3702296c8ff522ffbb5075ad52b2ce68d2f316de6685256b756ff098bc5a2b29e807652b89de84f9b4661

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.6 (2021-06-07)
+
+- Updated Bling Fire to 0.1.7
+- Added `prefix` option
+
 ## 0.1.5 (2021-03-14)
 
 - Updated Bling Fire to 0.1.5

data/README.md

@@ -76,6 +76,12 @@ Get offsets for ids
 ids, start_offsets, end_offsets = model.text_to_ids_with_offsets(text)
 ```
 
+Disable prefix space
+
+```ruby
+model = BlingFire.load_model("roberta.bin", prefix: false)
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/blingfire/blob/master/CHANGELOG.md)

data/lib/blingfire.rb

@@ -38,8 +38,8 @@ module BlingFire
       FFI.GetBlingFireTokVersion
     end
 
-    def load_model(path)
-      Model.new(path)
+    def load_model(path, **options)
+      Model.new(path, **options)
     end
 
     def text_to_words(text)
@@ -126,6 +126,12 @@ module BlingFire
       encode_utf8(out.to_str(out_size))
     end
 
+    def change_settings_dummy_prefix(model, value)
+      # use opposite of value
+      ret = FFI.SetNoDummyPrefix(model, value ? 0 : 1)
+      raise Error, "Bad status: #{ret}" if ret != 1
+    end
+
     private
 
     def check_status(ret, ptr)
@@ -171,7 +177,7 @@ module BlingFire
       # TODO see if more efficient to store next_pos in variable
       pos = 0
       text.each_char.with_index do |c, i|
-        while pos == start_bytes[starts.size]
+        while pos == start_bytes[starts.size] || start_bytes[starts.size] == -1
           starts << i
         end
         pos += c.bytesize

data/lib/blingfire/ffi.rb

@@ -10,6 +10,8 @@ module BlingFire
       raise e
     end
 
+    typealias "bool", "char"
+
     # https://github.com/microsoft/BlingFire/blob/master/blingfiretools/blingfiretokdll/blingfiretokdll.cpp
 
     # version
@@ -40,5 +42,8 @@ module BlingFire
 
     # free model
     extern "int FreeModel(void* ModelPtr)"
+
+    # prefix
+    extern "int SetNoDummyPrefix(void* ModelPtr, bool fNoDummyPrefix)"
   end
 end

data/lib/blingfire/model.rb

@@ -1,11 +1,15 @@
 module BlingFire
   class Model
-    def initialize(path = nil)
+    def initialize(path = nil, prefix: nil)
       @handle = nil
       if path
         raise Error, "Model not found" unless File.exist?(path)
         @handle = FFI.LoadModel(path)
         ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
+
+        BlingFire.change_settings_dummy_prefix(@handle, prefix) unless prefix.nil?
+      else
+        raise Error, "prefix option requires path" unless prefix.nil?
       end
     end
 

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-15 00:00:00.000000000 Z
+date: 2021-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler