blingfire 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a63930943ea4fe6a3dcbf4dd30219c7986b5f9e1cdcff9e6ada001a00399761
-  data.tar.gz: d14acf9c98a6ffc8d1917d3d29fb14be4f68ce9dd7519a7e4d79a8ba4dda3a3d
+  metadata.gz: a8bcd44e1517ca140f88c58672f9ce09644e4498add5763ec41a5551e8c281ab
+  data.tar.gz: 47c29e51a1f442ecc3a033a73d0c4916e8eebd8be194d46389e3cb64a98a1e8a
 SHA512:
-  metadata.gz: 538838c9d5c59fbd2fcdec2742500d76b9d4cf6a115c673fccf87c84f393af463524e6e0c19bf7bbaafa26237e3a61e96e68732fc59c97d97a7a7605c7e0e63c
-  data.tar.gz: dc75f4af27a2d62d3a0edba001dc99c848c534ea54a2bec32f3fb7e1eadb5ff8e7c2864ce6dd771cf3d7676ed355074f278f18b819f8422566038dc985958daf
+  metadata.gz: 43d1be1d30fcadf809aab8728d8a7d6d7b57128dc4f7e8784072e469e43fd01ff3a463ecc915563d3bc3c84c7265bfad7b854c26759bc6eee0b47196c79c1456
+  data.tar.gz: 6fc78f981153589681ba6016a9d0e2aaa264d68a551b7656db03e353aaa2bd9441ba9653bece7d6f886063ebca25d8006804d54328c207a6b01719a63f664e53

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.3.1 (2025-05-04)
+
+- Fixed issue with Fiddle 1.1.7+
+- Fixed memory leaks
+
 ## 0.3.0 (2025-04-03)
 
 - Dropped support for Ruby < 3.2

data/lib/blingfire/ffi.rb

@@ -10,8 +10,6 @@ module BlingFire
       raise e
     end
 
-    typealias "bool", "char"
-
     # https://github.com/microsoft/BlingFire/blob/master/blingfiretools/blingfiretokdll/blingfiretokdll.cpp
 
     # version

data/lib/blingfire/model.rb

@@ -4,7 +4,7 @@ module BlingFire
       @handle = nil
       if path
         raise Error, "Model not found" unless File.exist?(path)
-        @handle = FFI.LoadModel(path)
+        @handle = FFI.LoadModel(+path)
         @handle.free = FFI["FreeModel"]
 
         BlingFire.change_settings_dummy_prefix(@handle, prefix) unless prefix.nil?

data/lib/blingfire/version.rb

@@ -1,3 +1,3 @@
 module BlingFire
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/lib/blingfire.rb

@@ -92,20 +92,20 @@ module BlingFire
 
     def text_to_ids(model, text, max_len = nil, unk_id = 0)
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT)
-      out_size = FFI.TextToIds(model, text, text.bytesize, ids, ids.size, unk_id)
+      ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT, Fiddle::RUBY_FREE)
+      out_size = FFI.TextToIds(model, +text, text.bytesize, ids, ids.size, unk_id)
       check_status out_size, ids
       ids[0, (max_len || out_size) * Fiddle::SIZEOF_INT].unpack("i!*")
     end
 
     def text_to_ids_with_offsets(model, text, max_len = nil, unk_id = 0)
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT)
+      ids = Fiddle::Pointer.malloc((max_len || text.size) * Fiddle::SIZEOF_INT, Fiddle::RUBY_FREE)
 
-      start_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * ids.size)
-      end_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * ids.size)
+      start_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * ids.size, Fiddle::RUBY_FREE)
+      end_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * ids.size, Fiddle::RUBY_FREE)
 
-      out_size = FFI.TextToIdsWithOffsets(model, text, text.bytesize, ids, start_offsets, end_offsets, ids.size, unk_id)
+      out_size = FFI.TextToIdsWithOffsets(model, +text, text.bytesize, ids, start_offsets, end_offsets, ids.size, unk_id)
 
       check_status out_size, ids
 
@@ -116,8 +116,8 @@ module BlingFire
     def ids_to_text(model, ids, skip_special_tokens: true, output_buffer_size: nil)
       output_buffer_size ||= ids.size * 32
       c_ids = Fiddle::Pointer[ids.pack("i*")]
-      out = Fiddle::Pointer.malloc(output_buffer_size)
-      out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
+      out = Fiddle::Pointer.malloc(output_buffer_size, Fiddle::RUBY_FREE)
+      out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, !!skip_special_tokens)
       check_status out_size, out
       out_size <= 0 ? "" : encode_utf8(out.to_str(out_size - 1))
     end
@@ -129,15 +129,15 @@ module BlingFire
     def normalize_spaces(text)
       u_space = 0x20
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
-      out = Fiddle::Pointer.malloc([text.bytesize * 1.5, 20].max)
-      out_size = FFI.NormalizeSpaces(text, text.bytesize, out, out.size, u_space)
+      out = Fiddle::Pointer.malloc([text.bytesize * 1.5, 20].max, Fiddle::RUBY_FREE)
+      out_size = FFI.NormalizeSpaces(+text, text.bytesize, out, out.size, u_space)
       check_status out_size, out
       encode_utf8(out.to_str(out_size))
     end
 
     def change_settings_dummy_prefix(model, value)
       # use opposite of value
-      ret = FFI.SetNoDummyPrefix(model, value ? 0 : 1)
+      ret = FFI.SetNoDummyPrefix(model, !value)
       raise Error, "Bad status: #{ret}" if ret != 1
     end
 
@@ -150,8 +150,8 @@ module BlingFire
     def text_to(text, sep)
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
       # TODO allocate less, and try again if needed
-      out = Fiddle::Pointer.malloc([text.bytesize * 3, 20].max)
-      out_size = yield(text, out)
+      out = Fiddle::Pointer.malloc([text.bytesize * 3, 20].max, Fiddle::RUBY_FREE)
+      out_size = yield(+text, out)
       check_status out_size, out
       encode_utf8(out.to_str(out_size - 1)).split(sep)
     end
@@ -159,12 +159,12 @@ module BlingFire
     def text_to_with_offsets(text, sep)
       text = encode_utf8(text.dup) unless text.encoding == Encoding::UTF_8
       # TODO allocate less, and try again if needed
-      out = Fiddle::Pointer.malloc([text.bytesize * 3, 20].max)
+      out = Fiddle::Pointer.malloc([text.bytesize * 3, 20].max, Fiddle::RUBY_FREE)
 
-      start_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * out.size)
-      end_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * out.size)
+      start_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * out.size, Fiddle::RUBY_FREE)
+      end_offsets = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT * out.size, Fiddle::RUBY_FREE)
 
-      out_size = yield(text, out, start_offsets, end_offsets)
+      out_size = yield(+text, out, start_offsets, end_offsets)
 
       check_status out_size, out
 
@@ -177,8 +177,8 @@ module BlingFire
     end
 
     def unpack_offsets(start_offsets, end_offsets, result, text)
-      start_bytes = start_offsets.to_s(Fiddle::SIZEOF_INT * result.size).unpack("i*")
-      end_bytes = end_offsets.to_s(Fiddle::SIZEOF_INT * result.size).unpack("i*")
+      start_bytes = start_offsets.to_str(Fiddle::SIZEOF_INT * result.size).unpack("i*")
+      end_bytes = end_offsets.to_str(Fiddle::SIZEOF_INT * result.size).unpack("i*")
       starts = []
       ends = []
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: blingfire
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Andrew Kane
 bindir: bin
 cert_chain: []
-date: 2025-04-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fiddle
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.7
 email: andrew@ankane.org
 executables: []
 extensions: []
@@ -59,7 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: High speed text tokenization for Ruby
 test_files: []