tokenizers 0.6.2-x86_64-darwin → 0.6.3-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a793276da84b2b3c38df06471e9128072cf4c87e61c8f11f9c8885ecc79d5a6
-  data.tar.gz: 9bb03308d9699b8464bdba326e45777d94a0d990ccad1997eef38ecbe113904f
+  metadata.gz: ab7df68f887ed98c7592e0553d82dcb0989033b3983aae43d093ffd8d9883ac0
+  data.tar.gz: e17dcd450c6e7f984cde6ecc9d2e9a5d8a695583c226d3e8ab2524a4a3c78c1d
 SHA512:
-  metadata.gz: c63ff72dda8763dad39ec49ec18f3e7e9a05e0939f630a03d2fe9fdc70e590c1e8cd17108e3f5e9d6e9899241a1913c55d8c932abfc066ca632df19a28597fa4
-  data.tar.gz: e1cf8d65b051f288d14d7e92622fe3ea07843acd1288b3aec39e324427b2fca88a38680a2c6b29e5559923614c6f25aac9c249692323da8534ba11696b7d936a
+  metadata.gz: 7067a8fc83ad516f1ac5f063b1c1b891ddd33490d5379e64b8b750aaf164c1a971a0e8053a91f7ae0b7c6eef4b311610c5cd6f3a30264f237b4f0b107557ff8d
+  data.tar.gz: '0986b6b549512d6308e3a273e66b7b68f2a84c3438fd390a3987b73d85b2449cb6478d35234a258c21c01c7f604bf1c5c61b18b0cb4c0d9969237da1dd12f50a'

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.3 (2026-01-05)
+
+- Updated Tokenizers to 0.22.2
+- Added support for Ruby 4.0
+- Fixed `from_pretrained` method with namespaces
+
 ## 0.6.2 (2025-11-24)
 
 - Updated Tokenizers to 0.22.1

data/Cargo.lock

@@ -124,9 +124,9 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.15.11"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4"
 dependencies = [
  "encode_unicode",
  "libc",
@@ -288,14 +288,14 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "indicatif"
-version = "0.17.11"
+version = "0.18.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88"
 dependencies = [
  "console",
- "number_prefix",
  "portable-atomic",
  "unicode-width",
+ "unit-prefix",
  "web-time",
 ]
 
@@ -449,12 +449,6 @@ dependencies = [
  "minimal-lexical",
 ]
 
-[[package]]
-name = "number_prefix"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
-
 [[package]]
 name = "once_cell"
 version = "1.21.3"
@@ -596,18 +590,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.117"
+version = "0.9.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f900d1ce4629a2ebffaf5de74bd8f9c1188d4c5ed406df02f97e22f77a006f44"
+checksum = "c85c4188462601e2aa1469def389c17228566f82ea72f137ed096f21591bc489"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.117"
+version = "0.9.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef1e9c857028f631056bcd6d88cec390c751e343ce2223ddb26d23eb4a151d59"
+checksum = "568068db4102230882e6d4ae8de6632e224ca75fe5970f6e026a04e91ed635d3"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -784,20 +778,9 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.6.2"
-dependencies = [
- "ahash",
- "magnus",
- "onig",
- "serde",
- "tokenizers 0.22.1",
-]
-
-[[package]]
-name = "tokenizers"
-version = "0.22.1"
+version = "0.22.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6475a27088c98ea96d00b39a9ddfb63780d1ad4cceb6f48374349a96ab2b7842"
+checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -827,6 +810,17 @@ dependencies = [
  "unicode_categories",
 ]
 
+[[package]]
+name = "tokenizers-ruby"
+version = "0.6.3"
+dependencies = [
+ "ahash",
+ "magnus",
+ "onig",
+ "serde",
+ "tokenizers",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.18"
@@ -860,6 +854,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -942,13 +942,19 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
 [[package]]
 name = "windows-sys"
-version = "0.59.0"
+version = "0.61.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
 dependencies = [
- "windows-targets",
+ "windows-link",
 ]
 
 [[package]]

data/LICENSE-THIRD-PARTY.txt

@@ -46,7 +46,7 @@ compact_str v0.9.0
 https://github.com/ParkMyCar/compact_str
 MIT
 
-console v0.15.11
+console v0.16.1
 https://github.com/console-rs/console
 MIT
 
@@ -114,7 +114,7 @@ ident_case v1.0.1
 https://github.com/TedDriggs/ident_case
 MIT/Apache-2.0
 
-indicatif v0.17.11
+indicatif v0.18.3
 https://github.com/console-rs/indicatif
 MIT
 
@@ -186,10 +186,6 @@ nom v7.1.3
 https://github.com/Geal/nom
 MIT
 
-number_prefix v0.4.0
-https://github.com/ogham/rust-number-prefix
-MIT
-
 once_cell v1.21.3
 https://github.com/matklad/once_cell
 MIT OR Apache-2.0
@@ -250,11 +246,11 @@ rayon-core v1.12.1
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.117
+rb-sys v0.9.124
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.117
+rb-sys-build v0.9.124
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -338,7 +334,7 @@ thiserror-impl v2.0.12
 https://github.com/dtolnay/thiserror
 MIT OR Apache-2.0
 
-tokenizers v0.22.1
+tokenizers v0.22.2
 https://github.com/huggingface/tokenizers
 Apache-2.0
 
@@ -362,6 +358,10 @@ unicode_categories v0.1.1
 https://github.com/swgillespie/unicode-categories
 MIT OR Apache-2.0
 
+unit-prefix v0.5.2
+https://codeberg.org/commons-rs/unit-prefix
+MIT
+
 version_check v0.9.5
 https://github.com/SergioBenitez/version_check
 MIT/Apache-2.0
@@ -8031,32 +8031,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-================================================================================
-number_prefix LICENCE
-================================================================================
-
-MIT License
-
-Copyright (c) 2018 Benjamin Sago
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
 ================================================================================
 once_cell LICENSE-APACHE
 ================================================================================
@@ -16984,6 +16958,32 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+================================================================================
+unit-prefix LICENSE
+================================================================================
+
+MIT License
+
+Copyright (c) 2024 Benjamin Sago, Fabio Valentini
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ================================================================================
 version_check LICENSE-APACHE
 ================================================================================

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.22.1"
+    TOKENIZERS_VERSION = "0.22.2"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
@@ -27,7 +27,8 @@ module Tokenizers
         headers["Authorization"] = "Bearer #{auth_token}"
       end
 
-      url = "https://huggingface.co/%s/resolve/%s/tokenizer.json" % [identifier, revision].map { |v| CGI.escape(v) }
+      escaped_identifier = identifier.split("/", 2).map { |v| CGI.escape(v) }.join("/")
+      url = "https://huggingface.co/%s/resolve/%s/tokenizer.json" % [escaped_identifier, CGI.escape(revision)]
 
       path =
         begin
@@ -53,14 +54,7 @@ module Tokenizers
         esum = Digest::SHA256.hexdigest(etag)
         resource_path = File.join(cache_dir, "#{fsum}.#{esum}")
         if File.exist?(resource_path)
-          uri = URI(url)
-          req = Net::HTTP::Head.new(uri)
-          headers.each do |k, v|
-            req[k] = v
-          end
-          res = Net::HTTP.start(uri.hostname, uri.port, options.merge(use_ssl: true)) do |http|
-            http.request(req)
-          end
+          res = head_request(url, headers, options)
           if res["etag"] == etag
             return resource_path
           end
@@ -93,6 +87,25 @@ module Tokenizers
       resource_path
     end
 
+    def head_request(url, headers, options, redirects = 0)
+      uri = URI(url)
+      req = Net::HTTP::Head.new(uri)
+      headers.each do |k, v|
+        req[k] = v
+      end
+      res = Net::HTTP.start(uri.hostname, uri.port, options.merge(use_ssl: true)) do |http|
+        http.request(req)
+      end
+      if res.is_a?(Net::HTTPRedirection) && redirects < 3
+        location = URI.parse(res["location"])
+        # follow relative redirects only
+        if location.relative?
+          return head_request(uri.merge(location), headers, options, redirects + 1)
+        end
+      end
+      res
+    end
+
     def cache_dir
       cache_dir =
         if ENV["TOKENIZERS_CACHE"]

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.6.2"
+  VERSION = "0.6.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: x86_64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-11-25 00:00:00.000000000 Z
+date: 2026-01-06 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.2/tokenizers.bundle
 - lib/tokenizers/3.3/tokenizers.bundle
 - lib/tokenizers/3.4/tokenizers.bundle
+- lib/tokenizers/4.0/tokenizers.bundle
 - lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
@@ -71,7 +72,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.2'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.5.dev
+      version: 4.1.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="