blingfire 0.1.8 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 466b528fca6404415ad072b597cbb6fc96628a325504aba0e7becc4793b35b3f
-  data.tar.gz: c1e02e1f8c48578c6407ac9f7d7a996ecd78205ffdcaa3158144012f61d8922e
+  metadata.gz: 890fb5c4a1c6bf935b733e3c004f7a0128d81112d55cacd43d7d5d9b62cbaf8f
+  data.tar.gz: b986e941724657af926a50db274a038957c1937b68ca6dd5dbfc733a59c160db
 SHA512:
-  metadata.gz: cbfd60e8e54b42a1e7258afcebb51d27029bf66a2969daf4badc62f3117d7113b8d02f99c3445513dc37f08698205c21ba97787ba761512166257438b73d3956
-  data.tar.gz: 1aa595a55116d5e084d31e087180ed04392d1b7d91932a875d53cab88bc72b3a1ae179c15094b9679aa9fa78dc4419afc70f144ba971343d0c465a7842ca634e
+  metadata.gz: af31055e834a91bb51c95e60ba6dc35bbd2ca5c844bc75ae0cf5681f71c7e5acacd3d9f2d5ca4ef9e962d022997f594cf35651a0478d53182f3522a5dfcf9840
+  data.tar.gz: 4aff0c7edc5c09675b438167b2449cfcce8aca98bf0b14b9c17fb89192dd5cd8e7c51894a341fbda30411ad7ae29972a37e3ad0f029a9056825f017c1ae65f8e

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.2.1 (2023-12-12)
+
+- Fixed error with `ids_to_text` and all special tokens
+
+## 0.2.0 (2023-09-20)
+
+- Fixed error with `dup` and `clone`
+- Dropped support for Ruby < 3
+
 ## 0.1.8 (2023-02-01)
 
 - Improved ARM detection

data/LICENSE.txt

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) Microsoft Corporation. All rights reserved.
-Copyright (c) 2020 Andrew Kane
+Copyright (c) 2020-2023 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -82,7 +82,7 @@ Disable prefix space
 model = BlingFire.load_model("roberta.bin", prefix: false)
 ```
 
-## Ids to Text [experimental]
+## Ids to Text
 
 Load a model
 

data/lib/blingfire/model.rb

@@ -5,7 +5,7 @@ module BlingFire
       if path
         raise Error, "Model not found" unless File.exist?(path)
         @handle = FFI.LoadModel(path)
-        ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
+        @handle.free = FFI["FreeModel"]
 
         BlingFire.change_settings_dummy_prefix(@handle, prefix) unless prefix.nil?
       else
@@ -72,10 +72,5 @@ module BlingFire
     def to_ptr
       @handle
     end
-
-    def self.finalize(pointer)
-      # must use proc instead of stabby lambda
-      proc { FFI.FreeModel(pointer) }
-    end
   end
 end

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.1.8"
+  VERSION = "0.2.1"
 end

data/lib/blingfire.rb

@@ -2,8 +2,8 @@
 require "fiddle/import"
 
 # modules
-require "blingfire/model"
-require "blingfire/version"
+require_relative "blingfire/model"
+require_relative "blingfire/version"
 
 module BlingFire
   class Error < StandardError; end
@@ -119,7 +119,7 @@ module BlingFire
       out = Fiddle::Pointer.malloc(output_buffer_size)
       out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
       check_status out_size, out
-      encode_utf8(out.to_str(out_size - 1))
+      out_size <= 0 ? "" : encode_utf8(out.to_str(out_size - 1))
     end
 
     def free_model(model)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-02 00:00:00.000000000 Z
+date: 2023-12-12 00:00:00.000000000 Z
 dependencies: []
 description:
 email: andrew@ankane.org
@@ -41,14 +41,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: High speed text tokenization for Ruby