blingfire 0.1.8 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -0
  3. data/LICENSE.txt +1 -1
  4. data/README.md +1 -1
  5. data/lib/blingfire/model.rb +1 -6
  6. data/lib/blingfire/version.rb +1 -1
  7. data/lib/blingfire.rb +3 -3
  8. metadata +4 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 466b528fca6404415ad072b597cbb6fc96628a325504aba0e7becc4793b35b3f
4
- data.tar.gz: c1e02e1f8c48578c6407ac9f7d7a996ecd78205ffdcaa3158144012f61d8922e
3
+ metadata.gz: 890fb5c4a1c6bf935b733e3c004f7a0128d81112d55cacd43d7d5d9b62cbaf8f
4
+ data.tar.gz: b986e941724657af926a50db274a038957c1937b68ca6dd5dbfc733a59c160db
5
5
  SHA512:
6
- metadata.gz: cbfd60e8e54b42a1e7258afcebb51d27029bf66a2969daf4badc62f3117d7113b8d02f99c3445513dc37f08698205c21ba97787ba761512166257438b73d3956
7
- data.tar.gz: 1aa595a55116d5e084d31e087180ed04392d1b7d91932a875d53cab88bc72b3a1ae179c15094b9679aa9fa78dc4419afc70f144ba971343d0c465a7842ca634e
6
+ metadata.gz: af31055e834a91bb51c95e60ba6dc35bbd2ca5c844bc75ae0cf5681f71c7e5acacd3d9f2d5ca4ef9e962d022997f594cf35651a0478d53182f3522a5dfcf9840
7
+ data.tar.gz: 4aff0c7edc5c09675b438167b2449cfcce8aca98bf0b14b9c17fb89192dd5cd8e7c51894a341fbda30411ad7ae29972a37e3ad0f029a9056825f017c1ae65f8e
data/CHANGELOG.md CHANGED
@@ -1,3 +1,12 @@
1
+ ## 0.2.1 (2023-12-12)
2
+
3
+ - Fixed error with `ids_to_text` and all special tokens
4
+
5
+ ## 0.2.0 (2023-09-20)
6
+
7
+ - Fixed error with `dup` and `clone`
8
+ - Dropped support for Ruby < 3
9
+
1
10
  ## 0.1.8 (2023-02-01)
2
11
 
3
12
  - Improved ARM detection
data/LICENSE.txt CHANGED
@@ -1,7 +1,7 @@
1
1
  MIT License
2
2
 
3
3
  Copyright (c) Microsoft Corporation. All rights reserved.
4
- Copyright (c) 2020 Andrew Kane
4
+ Copyright (c) 2020-2023 Andrew Kane
5
5
 
6
6
  Permission is hereby granted, free of charge, to any person obtaining a copy
7
7
  of this software and associated documentation files (the "Software"), to deal
data/README.md CHANGED
@@ -82,7 +82,7 @@ Disable prefix space
82
82
  model = BlingFire.load_model("roberta.bin", prefix: false)
83
83
  ```
84
84
 
85
- ## Ids to Text [experimental]
85
+ ## Ids to Text
86
86
 
87
87
  Load a model
88
88
 
data/lib/blingfire/model.rb CHANGED
@@ -5,7 +5,7 @@ module BlingFire
5
5
  if path
6
6
  raise Error, "Model not found" unless File.exist?(path)
7
7
  @handle = FFI.LoadModel(path)
8
- ObjectSpace.define_finalizer(self, self.class.finalize(@handle))
8
+ @handle.free = FFI["FreeModel"]
9
9
 
10
10
  BlingFire.change_settings_dummy_prefix(@handle, prefix) unless prefix.nil?
11
11
  else
@@ -72,10 +72,5 @@ module BlingFire
72
72
  def to_ptr
73
73
  @handle
74
74
  end
75
-
76
- def self.finalize(pointer)
77
- # must use proc instead of stabby lambda
78
- proc { FFI.FreeModel(pointer) }
79
- end
80
75
  end
81
76
  end
data/lib/blingfire/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module BlingFire
2
- VERSION = "0.1.8"
2
+ VERSION = "0.2.1"
3
3
  end
data/lib/blingfire.rb CHANGED
@@ -2,8 +2,8 @@
2
2
  require "fiddle/import"
3
3
 
4
4
  # modules
5
- require "blingfire/model"
6
- require "blingfire/version"
5
+ require_relative "blingfire/model"
6
+ require_relative "blingfire/version"
7
7
 
8
8
  module BlingFire
9
9
  class Error < StandardError; end
@@ -119,7 +119,7 @@ module BlingFire
119
119
  out = Fiddle::Pointer.malloc(output_buffer_size)
120
120
  out_size = FFI.IdsToText(model, c_ids, ids.size, out, output_buffer_size, skip_special_tokens ? 1 : 0)
121
121
  check_status out_size, out
122
- encode_utf8(out.to_str(out_size - 1))
122
+ out_size <= 0 ? "" : encode_utf8(out.to_str(out_size - 1))
123
123
  end
124
124
 
125
125
  def free_model(model)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: blingfire
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.8
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-02 00:00:00.000000000 Z
11
+ date: 2023-12-12 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: andrew@ankane.org
@@ -41,14 +41,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
41
41
  requirements:
42
42
  - - ">="
43
43
  - !ruby/object:Gem::Version
44
- version: '2.4'
44
+ version: '3'
45
45
  required_rubygems_version: !ruby/object:Gem::Requirement
46
46
  requirements:
47
47
  - - ">="
48
48
  - !ruby/object:Gem::Version
49
49
  version: '0'
50
50
  requirements: []
51
- rubygems_version: 3.4.1
51
+ rubygems_version: 3.4.10
52
52
  signing_key:
53
53
  specification_version: 4
54
54
  summary: High speed text tokenization for Ruby