tokenizers 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caf36f6de3318df84c66a3ff99963e1b57aeca78736892ad143f86c82bfde737
-  data.tar.gz: 1e8689c437f736aeb85e2163f5b4eb7a30b1b01ba3ec6c472afde16a7dd25040
+  metadata.gz: b7e76174884c06417a6829e89ef1f5785957d92f10d8a9acd6586e19ec84737c
+  data.tar.gz: 04b8e76f25f59e404978f2af68829142890c7bdd4ffbe7cd86a5af4f72de2d5e
 SHA512:
-  metadata.gz: 88a48300b336c3afaf6ba2119835d548945419133e3f194b3ec8ed78c9a6b477389a4f60325c3233358e5f52b2e7b3ad79550a8e5b2828436757cad6796093ab
-  data.tar.gz: af05786f53957827b7094bbf464dd281c44839de13ecbb3811e2897a95c5fd42fe27573259b8d337b1c975746c20ea9051b128bf7a048784654bba6f5fa003e9
+  metadata.gz: b9b2583c6c2aac22d835c045f6674a95a1f1a9dcdddd7d2406b34c7f64cb04bd8900de05feaa85a8f9d5601392636b41ed300326f362e9e5de29098506801cc6
+  data.tar.gz: '08f6e8e3c4187a5f3bd57e75141add2973857a1fe4211b86acd8ab43a3ec5fc5550309910c606d09267dfe3242c7088b7ffa2d5ccb0f94c5ee922deb0fd0c943'

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.3 (2026-01-05)
+
+- Updated Tokenizers to 0.22.2
+- Added support for Ruby 4.0
+- Fixed `from_pretrained` method with namespaces
+
 ## 0.6.2 (2025-11-24)
 
 - Updated Tokenizers to 0.22.1

data/Cargo.lock

@@ -124,9 +124,9 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.15.11"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4"
 dependencies = [
  "encode_unicode",
  "libc",
@@ -288,14 +288,14 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "indicatif"
-version = "0.17.11"
+version = "0.18.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88"
 dependencies = [
  "console",
- "number_prefix",
  "portable-atomic",
  "unicode-width",
+ "unit-prefix",
  "web-time",
 ]
 
@@ -449,12 +449,6 @@ dependencies = [
  "minimal-lexical",
 ]
 
-[[package]]
-name = "number_prefix"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
-
 [[package]]
 name = "once_cell"
 version = "1.21.3"
@@ -596,18 +590,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.117"
+version = "0.9.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f900d1ce4629a2ebffaf5de74bd8f9c1188d4c5ed406df02f97e22f77a006f44"
+checksum = "c85c4188462601e2aa1469def389c17228566f82ea72f137ed096f21591bc489"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.117"
+version = "0.9.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef1e9c857028f631056bcd6d88cec390c751e343ce2223ddb26d23eb4a151d59"
+checksum = "568068db4102230882e6d4ae8de6632e224ca75fe5970f6e026a04e91ed635d3"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -784,20 +778,9 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.6.2"
-dependencies = [
- "ahash",
- "magnus",
- "onig",
- "serde",
- "tokenizers 0.22.1",
-]
-
-[[package]]
-name = "tokenizers"
-version = "0.22.1"
+version = "0.22.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6475a27088c98ea96d00b39a9ddfb63780d1ad4cceb6f48374349a96ab2b7842"
+checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -827,6 +810,17 @@ dependencies = [
  "unicode_categories",
 ]
 
+[[package]]
+name = "tokenizers-ruby"
+version = "0.6.3"
+dependencies = [
+ "ahash",
+ "magnus",
+ "onig",
+ "serde",
+ "tokenizers",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.18"
@@ -860,6 +854,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
@@ -942,13 +942,19 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
 [[package]]
 name = "windows-sys"
-version = "0.59.0"
+version = "0.61.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
 dependencies = [
- "windows-targets",
+ "windows-link",
 ]
 
 [[package]]

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
-name = "tokenizers"
-version = "0.6.2"
+name = "tokenizers-ruby"
+version = "0.6.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -8,6 +8,7 @@ rust-version = "1.63.0"
 publish = false
 
 [lib]
+name = "tokenizers"
 crate-type = ["cdylib"]
 
 [dependencies]
@@ -17,6 +18,6 @@ onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.22.1" # also update in from_pretrained.rb
+version = "=0.22.2" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/lib.rs

@@ -49,7 +49,7 @@ static PROCESSORS: Lazy<RModule> =
 static TRAINERS: Lazy<RModule> =
     Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Trainers").unwrap());
 
-#[magnus::init]
+#[magnus::init(name = "tokenizers")]
 fn init(ruby: &Ruby) -> RbResult<()> {
     let module = ruby.define_module("Tokenizers")?;
 

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.22.1"
+    TOKENIZERS_VERSION = "0.22.2"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
@@ -27,7 +27,8 @@ module Tokenizers
         headers["Authorization"] = "Bearer #{auth_token}"
       end
 
-      url = "https://huggingface.co/%s/resolve/%s/tokenizer.json" % [identifier, revision].map { |v| CGI.escape(v) }
+      escaped_identifier = identifier.split("/", 2).map { |v| CGI.escape(v) }.join("/")
+      url = "https://huggingface.co/%s/resolve/%s/tokenizer.json" % [escaped_identifier, CGI.escape(revision)]
 
       path =
         begin
@@ -53,14 +54,7 @@ module Tokenizers
         esum = Digest::SHA256.hexdigest(etag)
         resource_path = File.join(cache_dir, "#{fsum}.#{esum}")
         if File.exist?(resource_path)
-          uri = URI(url)
-          req = Net::HTTP::Head.new(uri)
-          headers.each do |k, v|
-            req[k] = v
-          end
-          res = Net::HTTP.start(uri.hostname, uri.port, options.merge(use_ssl: true)) do |http|
-            http.request(req)
-          end
+          res = head_request(url, headers, options)
           if res["etag"] == etag
             return resource_path
           end
@@ -93,6 +87,25 @@ module Tokenizers
       resource_path
     end
 
+    def head_request(url, headers, options, redirects = 0)
+      uri = URI(url)
+      req = Net::HTTP::Head.new(uri)
+      headers.each do |k, v|
+        req[k] = v
+      end
+      res = Net::HTTP.start(uri.hostname, uri.port, options.merge(use_ssl: true)) do |http|
+        http.request(req)
+      end
+      if res.is_a?(Net::HTTPRedirection) && redirects < 3
+        location = URI.parse(res["location"])
+        # follow relative redirects only
+        if location.relative?
+          return head_request(uri.merge(location), headers, options, redirects + 1)
+        end
+      end
+      res
+    end
+
     def cache_dir
       cache_dir =
         if ENV["TOKENIZERS_CACHE"]

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.6.2"
+  VERSION = "0.6.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Andrew Kane
@@ -98,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby
 test_files: []