blingfire 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1d50b260b52505d373156fad02a822da03c95be9c4934f1c3beb5f7c38e650a
-  data.tar.gz: '067318eeb1f9901d36dbd33131823f70247d38ca49b755b8da407f45c14ee2e6'
+  metadata.gz: 890fb5c4a1c6bf935b733e3c004f7a0128d81112d55cacd43d7d5d9b62cbaf8f
+  data.tar.gz: b986e941724657af926a50db274a038957c1937b68ca6dd5dbfc733a59c160db
 SHA512:
-  metadata.gz: 8d5bd5569ed024dd9ebfe841cb6f136e15625cf00194463f5208e81d3d27630426ae944af43be8628aa28e076c40eeb541f336fa33b81f9da505c3a9109ee8cf
-  data.tar.gz: 8c93fb8cda66ef23d61cd427a5e94d9fccdd89205680ed9d221da91b34941486a02b12b4e83b5e1e1a55bc7080d54d4040d8b61e8e0ecbddc852f8003f99fe7c
+  metadata.gz: af31055e834a91bb51c95e60ba6dc35bbd2ca5c844bc75ae0cf5681f71c7e5acacd3d9f2d5ca4ef9e962d022997f594cf35651a0478d53182f3522a5dfcf9840
+  data.tar.gz: 4aff0c7edc5c09675b438167b2449cfcce8aca98bf0b14b9c17fb89192dd5cd8e7c51894a341fbda30411ad7ae29972a37e3ad0f029a9056825f017c1ae65f8e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1 (2023-12-12)
+
+- Fixed error with `ids_to_text` and all special tokens
+
 ## 0.2.0 (2023-09-20)
 
 - Fixed error with `dup` and `clone`

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/blingfire.rb

@@ -119,7 +119,7 @@ module BlingFire
       out = Fiddle::Pointer.malloc(output_buffer_size)
       out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
       check_status out_size, out
-      encode_utf8(out.to_str(out_size - 1))
+      out_size <= 0 ? "" : encode_utf8(out.to_str(out_size - 1))
     end
 
     def free_model(model)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-21 00:00:00.000000000 Z
+date: 2023-12-12 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org