tokenizers 0.6.1-x64-mingw-ucrt → 0.6.2-x64-mingw-ucrt

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +5 -0
  3. data/Cargo.lock +6 -6
  4. data/LICENSE-THIRD-PARTY.txt +2 -2
  5. data/lib/tokenizers/3.2/tokenizers.so +0 -0
  6. data/lib/tokenizers/3.3/tokenizers.so +0 -0
  7. data/lib/tokenizers/3.4/tokenizers.so +0 -0
  8. data/lib/tokenizers/from_pretrained.rb +23 -15
  9. data/lib/tokenizers/version.rb +1 -1
  10. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9498c84d09b186624a1b54dc55f8a83c8d847fe852a433078aa70e42bc88304d
4
- data.tar.gz: 5894b4dc5fdb184afc2a780f7b0883bcf6f1dae4614715cc8b081ad64abcc6c9
3
+ metadata.gz: 8aa07d83801712c72cfc0fac5601f8fd4d59b8672a50785408c14333d28992d5
4
+ data.tar.gz: c6e76f9bc88ed251dbef80680390eb78c252edc3b8be1d72562ca25c9db13623
5
5
  SHA512:
6
- metadata.gz: db650dabb20393eea82fbacb1cceb27dd10bcf2dd0f1064e8ee01c6a038c17d8c186e6c6b3b6891bd7badb480c9f53022f623f8e2eb7bb3b91af8875d5f84198
7
- data.tar.gz: 390d24dfac78c46c9ceaa47f7e57c535abc1189f02b81e2b7d6562210bbbc3242ef592bce31e6f882a607d138c60bf1d70191df39c5af06cd4779cea7d291296
6
+ metadata.gz: eec2eca0c73b18c679e9b5ce1241c8b5b44ebd2d7973d8c1f832b6063e46643178e6c1e288ce0cc5f5f998088651520103c20c817e9cfc876e64ffbeae7b2bd4
7
+ data.tar.gz: a7707d2de91a1e09cd0b0d4113e32430f27a69e82935f74f61993d690768076e6fbabb976669e90d2e0a15de56e2eb85995c8b19c45ede1ce0772b01f6c5b1ee
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 0.6.2 (2025-11-24)
2
+
3
+ - Updated Tokenizers to 0.22.1
4
+ - Fixed caching on Windows
5
+
1
6
  ## 0.6.1 (2025-09-17)
2
7
 
3
8
  - Fixed return type of `encode_batch` method
data/Cargo.lock CHANGED
@@ -385,9 +385,9 @@ checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
385
385
 
386
386
  [[package]]
387
387
  name = "magnus"
388
- version = "0.8.1"
388
+ version = "0.8.2"
389
389
  source = "registry+https://github.com/rust-lang/crates.io-index"
390
- checksum = "bd2ac6e71886be00ac34db92aa732c793c5107c95191805b9a1c7e70e6d342e0"
390
+ checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
391
391
  dependencies = [
392
392
  "magnus-macros",
393
393
  "rb-sys",
@@ -784,20 +784,20 @@ dependencies = [
784
784
 
785
785
  [[package]]
786
786
  name = "tokenizers"
787
- version = "0.6.1"
787
+ version = "0.6.2"
788
788
  dependencies = [
789
789
  "ahash",
790
790
  "magnus",
791
791
  "onig",
792
792
  "serde",
793
- "tokenizers 0.22.0",
793
+ "tokenizers 0.22.1",
794
794
  ]
795
795
 
796
796
  [[package]]
797
797
  name = "tokenizers"
798
- version = "0.22.0"
798
+ version = "0.22.1"
799
799
  source = "registry+https://github.com/rust-lang/crates.io-index"
800
- checksum = "af10f51be57162b69d90a15cb226eef12c9e4faecbd5e3ea98a86bfb920b3d71"
800
+ checksum = "6475a27088c98ea96d00b39a9ddfb63780d1ad4cceb6f48374349a96ab2b7842"
801
801
  dependencies = [
802
802
  "ahash",
803
803
  "aho-corasick",
data/LICENSE-THIRD-PARTY.txt CHANGED
@@ -162,7 +162,7 @@ macro_rules_attribute-proc_macro v0.2.0
162
162
  https://github.com/danielhenrymantilla/macro_rules_attribute-rs
163
163
  MIT
164
164
 
165
- magnus v0.8.1
165
+ magnus v0.8.2
166
166
  https://github.com/matsadler/magnus
167
167
  MIT
168
168
 
@@ -342,7 +342,7 @@ thiserror-impl v2.0.12
342
342
  https://github.com/dtolnay/thiserror
343
343
  MIT OR Apache-2.0
344
344
 
345
- tokenizers v0.22.0
345
+ tokenizers v0.22.1
346
346
  https://github.com/huggingface/tokenizers
347
347
  Apache-2.0
348
348
 
data/lib/tokenizers/3.2/tokenizers.so CHANGED
Binary file
data/lib/tokenizers/3.3/tokenizers.so CHANGED
Binary file
data/lib/tokenizers/3.4/tokenizers.so CHANGED
Binary file
data/lib/tokenizers/from_pretrained.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  module Tokenizers
2
2
  module FromPretrained
3
3
  # for user agent
4
- TOKENIZERS_VERSION = "0.22.0"
4
+ TOKENIZERS_VERSION = "0.22.1"
5
5
 
6
6
  # use Ruby for downloads
7
7
  # this avoids the need to vendor OpenSSL on Linux
@@ -94,21 +94,29 @@ module Tokenizers
94
94
  end
95
95
 
96
96
  def cache_dir
97
- if ENV["TOKENIZERS_CACHE"]
98
- ENV["TOKENIZERS_CACHE"]
99
- else
100
- # use same directory as Rust version
101
- # https://docs.rs/dirs/latest/dirs/fn.cache_dir.html
102
- dir =
103
- if Gem.win_platform?
104
- ENV.fetch("LOCALAPPDATA")
105
- elsif mac?
106
- File.join(ENV.fetch("HOME"), "Library", "Caches")
107
- else
108
- ENV["XDG_CACHE_HOME"] || File.join(ENV.fetch("HOME"), ".cache")
109
- end
97
+ cache_dir =
98
+ if ENV["TOKENIZERS_CACHE"]
99
+ ENV["TOKENIZERS_CACHE"]
100
+ else
101
+ # use same directory as Rust version
102
+ # https://docs.rs/dirs/latest/dirs/fn.cache_dir.html
103
+ dir =
104
+ if Gem.win_platform?
105
+ ENV.fetch("LOCALAPPDATA")
106
+ elsif mac?
107
+ File.join(ENV.fetch("HOME"), "Library", "Caches")
108
+ else
109
+ ENV["XDG_CACHE_HOME"] || File.join(ENV.fetch("HOME"), ".cache")
110
+ end
111
+
112
+ File.join(dir, "huggingface", "tokenizers")
113
+ end
110
114
 
111
- File.join(dir, "huggingface", "tokenizers")
115
+ if Gem.win_platform?
116
+ # cannot use backslash for glob on Windows
117
+ cache_dir.gsub("\\", "/")
118
+ else
119
+ cache_dir
112
120
  end
113
121
  end
114
122
 
data/lib/tokenizers/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Tokenizers
2
- VERSION = "0.6.1"
2
+ VERSION = "0.6.2"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tokenizers
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 0.6.2
5
5
  platform: x64-mingw-ucrt
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2025-09-17 00:00:00.000000000 Z
11
+ date: 2025-11-25 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: andrew@ankane.org