tokenizers 0.5.3-x64-mingw-ucrt → 0.5.4-x64-mingw-ucrt

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2575d80ae3967271482554c3a2e871dca95ce6545864a0222ac8571cd52aa5c9
-  data.tar.gz: 1970e935847b5ff6c2f4fa87ee0219796e1734eb8e777fca6f2a4c7c46108022
+  metadata.gz: c6bb67a2361bc8e34a9561f9b5d6afa684f9315e2a24a21519ede0e0143d4545
+  data.tar.gz: 3835ac6ca9ac2a2fc50b351c97663a870344a995b1bf68920a2c289f236c5568
 SHA512:
-  metadata.gz: 83c10038a44a00931c296cc6549cf5a1614446a094bfb9a87923fa57361e4c5eadfc52b81e27edfcd32d3b9e3ca8dde9d4a40d84df9c1844b16826a936784373
-  data.tar.gz: 6d7659d3928d03ffeb20632d3d0c44a34cda7882a96963e22d6e009b15c8c2935f3e45e879a515569b71e7dc524c5143b5220d76af46f322213399a94269c59f
+  metadata.gz: de40170d02d6359ba3f53b8e8a53c552af7618654045f4888c6eb44e06c28b6ce7df99c0b31acb3c8da4bea566e4515973d43fb5fd01cde3a3c68c05f675e3b2
+  data.tar.gz: '082ae45456ce3543afbb3f551d21df793b3fdf23f60705c12bf9a0a1be71a8ef28d903cdebb41e1d66e5453c919f41db21cb757a4e8f2940ebff555d8623a1f2'

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.4 (2024-12-28)
+
+- Updated Tokenizers to 0.21.0
+- Added support for Ruby 3.4
+
 ## 0.5.3 (2024-09-17)
 
 - Added `AddedToken` class

data/Cargo.lock

@@ -19,9 +19,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "bindgen"
-version = "0.69.4"
+version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
  "bitflags 2.6.0",
  "cexpr",
@@ -49,6 +49,12 @@ version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
+[[package]]
+name = "bumpalo"
+version = "3.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -57,9 +63,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.1.15"
+version = "1.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+checksum = "8d6dbb628b8f8555f86d0323c2eb39e3ec81901f4b83e091db8a6a76d316a333"
 dependencies = [
  "shlex",
 ]
@@ -92,22 +98,22 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.15.8"
+version = "0.15.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
+checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b"
 dependencies = [
  "encode_unicode",
- "lazy_static",
  "libc",
+ "once_cell",
  "unicode-width",
  "windows-sys",
 ]
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
 dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
@@ -124,9 +130,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.20"
+version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "darling"
@@ -165,18 +171,18 @@ dependencies = [
 
 [[package]]
 name = "derive_builder"
-version = "0.20.0"
+version = "0.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
 dependencies = [
  "derive_builder_macro",
 ]
 
 [[package]]
 name = "derive_builder_core"
-version = "0.20.0"
+version = "0.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -186,9 +192,9 @@ dependencies = [
 
 [[package]]
 name = "derive_builder_macro"
-version = "0.20.0"
+version = "0.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
 dependencies = [
  "derive_builder_core",
  "syn",
@@ -202,9 +208,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "encode_unicode"
-version = "0.3.6"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
 
 [[package]]
 name = "esaxx-rs"
@@ -234,9 +240,9 @@ dependencies = [
 
 [[package]]
 name = "glob"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
 
 [[package]]
 name = "ident_case"
@@ -246,24 +252,15 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "indicatif"
-version = "0.17.8"
+version = "0.17.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3"
+checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281"
 dependencies = [
  "console",
- "instant",
  "number_prefix",
  "portable-atomic",
  "unicode-width",
-]
-
-[[package]]
-name = "instant"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
-dependencies = [
- "cfg-if",
+ "web-time",
 ]
 
 [[package]]
@@ -286,9 +283,19 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.11"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
+
+[[package]]
+name = "js-sys"
+version = "0.3.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7"
+dependencies = [
+ "once_cell",
+ "wasm-bindgen",
+]
 
 [[package]]
 name = "lazy_static"
@@ -304,15 +311,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.158"
+version = "0.2.169"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
 
 [[package]]
 name = "libloading"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
+checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
 dependencies = [
  "cfg-if",
  "windows-targets",
@@ -414,9 +421,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "once_cell"
-version = "1.19.0"
+version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
 [[package]]
 name = "onig"
@@ -448,15 +455,15 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
 [[package]]
 name = "portable-atomic"
-version = "1.7.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
 
 [[package]]
 name = "ppv-lite86"
@@ -469,18 +476,18 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.86"
+version = "1.0.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
 dependencies = [
  "proc-macro2",
 ]
@@ -548,18 +555,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.102"
+version = "0.9.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
+checksum = "4b3a1f3ce8e7c36d777d52fe7a99039fe4fea7c8ec355a4c4f3a17f92a14029f"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.102"
+version = "0.9.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
+checksum = "3e6b246c29c0809e1cbe60a1ba9e093da72a4676d02adc68469297d1e589bbf0"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -578,9 +585,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
 
 [[package]]
 name = "regex"
-version = "1.10.6"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -590,9 +597,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -601,9 +608,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "rustc-hash"
@@ -625,18 +632,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.209"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.209"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -645,9 +652,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.127"
+version = "1.0.134"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
+checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d"
 dependencies = [
  "itoa",
  "memchr",
@@ -693,9 +700,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.76"
+version = "2.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
+checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -704,18 +711,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.63"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.63"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -724,19 +731,19 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.3"
+version = "0.5.4"
 dependencies = [
  "magnus",
  "onig",
  "serde",
- "tokenizers 0.20.0",
+ "tokenizers 0.21.0",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.20.0"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
+checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae"
 dependencies = [
  "aho-corasick",
  "derive_builder",
@@ -766,9 +773,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
 [[package]]
 name = "unicode-normalization-alignments"
@@ -781,15 +788,15 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
 
 [[package]]
 name = "unicode-width"
-version = "0.1.13"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
 
 [[package]]
 name = "unicode_categories"
@@ -803,11 +810,75 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
+dependencies = [
+ "bumpalo",
+ "log",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "windows-sys"
-version = "0.52.0"
+version = "0.59.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
 dependencies = [
  "windows-targets",
 ]

data/LICENSE-THIRD-PARTY.txt

@@ -10,7 +10,7 @@ base64 v0.13.1
 https://github.com/marshallpierce/rust-base64
 MIT/Apache-2.0
 
-bindgen v0.69.4
+bindgen v0.69.5
 https://rust-lang.github.io/rust-bindgen/
 BSD-3-Clause
 
@@ -26,7 +26,7 @@ byteorder v1.5.0
 https://github.com/BurntSushi/byteorder
 Unlicense OR MIT
 
-cc v1.1.15
+cc v1.2.6
 https://github.com/rust-lang/cc-rs
 MIT OR Apache-2.0
 
@@ -42,11 +42,11 @@ clang-sys v1.8.1
 https://github.com/KyleMayes/clang-sys
 Apache-2.0
 
-console v0.15.8
+console v0.15.10
 https://github.com/console-rs/console
 MIT
 
-crossbeam-deque v0.8.5
+crossbeam-deque v0.8.6
 https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-deque
 MIT OR Apache-2.0
 
@@ -54,7 +54,7 @@ crossbeam-epoch v0.9.18
 https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-epoch
 MIT OR Apache-2.0
 
-crossbeam-utils v0.8.20
+crossbeam-utils v0.8.21
 https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils
 MIT OR Apache-2.0
 
@@ -70,15 +70,15 @@ darling_macro v0.20.10
 https://github.com/TedDriggs/darling
 MIT
 
-derive_builder v0.20.0
+derive_builder v0.20.2
 https://github.com/colin-kiegel/rust-derive-builder
 MIT OR Apache-2.0
 
-derive_builder_core v0.20.0
+derive_builder_core v0.20.2
 https://github.com/colin-kiegel/rust-derive-builder
 MIT OR Apache-2.0
 
-derive_builder_macro v0.20.0
+derive_builder_macro v0.20.2
 https://github.com/colin-kiegel/rust-derive-builder
 MIT OR Apache-2.0
 
@@ -86,9 +86,9 @@ either v1.13.0
 https://github.com/rayon-rs/either
 MIT OR Apache-2.0
 
-encode_unicode v0.3.6
+encode_unicode v1.0.0
 https://github.com/tormol/encode_unicode
-MIT/Apache-2.0
+Apache-2.0 OR MIT
 
 esaxx-rs v0.1.10
 https://github.com/Narsil/esaxx-rs
@@ -102,7 +102,7 @@ getrandom v0.2.15
 https://github.com/rust-random/getrandom
 MIT OR Apache-2.0
 
-glob v0.3.1
+glob v0.3.2
 https://github.com/rust-lang/glob
 MIT OR Apache-2.0
 
@@ -110,7 +110,7 @@ ident_case v1.0.1
 https://github.com/TedDriggs/ident_case
 MIT/Apache-2.0
 
-indicatif v0.17.8
+indicatif v0.17.9
 https://github.com/console-rs/indicatif
 MIT
 
@@ -122,7 +122,7 @@ itertools v0.12.1
 https://github.com/rust-itertools/itertools
 MIT OR Apache-2.0
 
-itoa v1.0.11
+itoa v1.0.14
 https://github.com/dtolnay/itoa
 MIT OR Apache-2.0
 
@@ -134,11 +134,11 @@ lazycell v1.3.0
 https://github.com/indiv0/lazycell
 MIT/Apache-2.0
 
-libc v0.2.158
+libc v0.2.169
 https://github.com/rust-lang/libc
 MIT OR Apache-2.0
 
-libloading v0.8.5
+libloading v0.8.6
 https://github.com/nagisa/rust_libloading/
 ISC
 
@@ -186,7 +186,7 @@ number_prefix v0.4.0
 https://github.com/ogham/rust-number-prefix
 MIT
 
-once_cell v1.19.0
+once_cell v1.20.2
 https://github.com/matklad/once_cell
 MIT OR Apache-2.0
 
@@ -202,11 +202,11 @@ paste v1.0.15
 https://github.com/dtolnay/paste
 MIT OR Apache-2.0
 
-pkg-config v0.3.30
+pkg-config v0.3.31
 https://github.com/rust-lang/pkg-config-rs
 MIT OR Apache-2.0
 
-portable-atomic v1.7.0
+portable-atomic v1.10.0
 https://github.com/taiki-e/portable-atomic
 Apache-2.0 OR MIT
 
@@ -214,11 +214,11 @@ ppv-lite86 v0.2.20
 https://github.com/cryptocorrosion/cryptocorrosion
 MIT/Apache-2.0
 
-proc-macro2 v1.0.86
+proc-macro2 v1.0.92
 https://github.com/dtolnay/proc-macro2
 MIT OR Apache-2.0
 
-quote v1.0.37
+quote v1.0.38
 https://github.com/dtolnay/quote
 MIT OR Apache-2.0
 
@@ -246,11 +246,11 @@ rayon-core v1.12.1
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.102
+rb-sys v0.9.105
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.102
+rb-sys-build v0.9.105
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -258,15 +258,15 @@ rb-sys-env v0.1.2
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-regex v1.10.6
+regex v1.11.1
 https://github.com/rust-lang/regex
 MIT OR Apache-2.0
 
-regex-automata v0.4.7
+regex-automata v0.4.9
 https://github.com/rust-lang/regex/tree/master/regex-automata
 MIT OR Apache-2.0
 
-regex-syntax v0.8.4
+regex-syntax v0.8.5
 https://github.com/rust-lang/regex/tree/master/regex-syntax
 MIT OR Apache-2.0
 
@@ -282,15 +282,15 @@ seq-macro v0.3.5
 https://github.com/dtolnay/seq-macro
 MIT OR Apache-2.0
 
-serde v1.0.209
+serde v1.0.217
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_derive v1.0.209
+serde_derive v1.0.217
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_json v1.0.127
+serde_json v1.0.134
 https://github.com/serde-rs/json
 MIT OR Apache-2.0
 
@@ -314,35 +314,35 @@ strsim v0.11.1
 https://github.com/rapidfuzz/strsim-rs
 MIT
 
-syn v2.0.76
+syn v2.0.93
 https://github.com/dtolnay/syn
 MIT OR Apache-2.0
 
-thiserror v1.0.63
+thiserror v1.0.69
 https://github.com/dtolnay/thiserror
 MIT OR Apache-2.0
 
-thiserror-impl v1.0.63
+thiserror-impl v1.0.69
 https://github.com/dtolnay/thiserror
 MIT OR Apache-2.0
 
-tokenizers v0.20.0
+tokenizers v0.21.0
 https://github.com/huggingface/tokenizers
 Apache-2.0
 
-unicode-ident v1.0.12
+unicode-ident v1.0.14
 https://github.com/dtolnay/unicode-ident
-(MIT OR Apache-2.0) AND Unicode-DFS-2016
+(MIT OR Apache-2.0) AND Unicode-3.0
 
 unicode-normalization-alignments v0.1.12
 https://github.com/n1t0/unicode-normalization
 MIT/Apache-2.0
 
-unicode-segmentation v1.11.0
+unicode-segmentation v1.12.0
 https://github.com/unicode-rs/unicode-segmentation
-MIT/Apache-2.0
+MIT OR Apache-2.0
 
-unicode-width v0.1.13
+unicode-width v0.2.0
 https://github.com/unicode-rs/unicode-width
 MIT OR Apache-2.0
 
@@ -350,7 +350,7 @@ unicode_categories v0.1.1
 https://github.com/swgillespie/unicode-categories
 MIT OR Apache-2.0
 
-windows-sys v0.52.0
+windows-sys v0.59.0
 https://github.com/microsoft/windows-rs
 MIT OR Apache-2.0
 
@@ -15619,52 +15619,45 @@ DEALINGS IN THE SOFTWARE.
 unicode-ident LICENSE-UNICODE
 ================================================================================
 
-UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
-
-See Terms of Use <https://www.unicode.org/copyright.html>
-for definitions of Unicode Inc.’s Data Files and Software.
-
-NOTICE TO USER: Carefully read the following legal agreement.
-BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
-DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
-YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
-TERMS AND CONDITIONS OF THIS AGREEMENT.
-IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
-THE DATA FILES OR SOFTWARE.
+UNICODE LICENSE V3
 
 COPYRIGHT AND PERMISSION NOTICE
 
-Copyright © 1991-2022 Unicode, Inc. All rights reserved.
-Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of the Unicode data files and any associated documentation
-(the "Data Files") or Unicode software and any associated documentation
-(the "Software") to deal in the Data Files or Software
-without restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, and/or sell copies of
-the Data Files or Software, and to permit persons to whom the Data Files
-or Software are furnished to do so, provided that either
-(a) this copyright and permission notice appear with all copies
-of the Data Files or Software, or
-(b) this copyright and permission notice appear in associated
-Documentation.
-
-THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT OF THIRD PARTY RIGHTS.
-IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
-NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
-DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
-DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THE DATA FILES OR SOFTWARE.
-
-Except as contained in this notice, the name of a copyright holder
-shall not be used in advertising or otherwise to promote the sale,
-use or other dealings in these Data Files or Software without prior
-written authorization of the copyright holder.
+Copyright © 1991-2023 Unicode, Inc.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY
+DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
+SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
+TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
+DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of data files and any associated documentation (the "Data Files") or
+software and any associated documentation (the "Software") to deal in the
+Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Data Files or Software, and to permit persons to whom the
+Data Files or Software are furnished to do so, provided that either (a)
+this copyright and permission notice appear with all copies of the Data
+Files or Software, or (b) this copyright and permission notice appear in
+associated Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS.
+
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
+BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
+OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
+FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
 
 ================================================================================
 unicode-normalization-alignments LICENSE-APACHE

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.20.0"
+    TOKENIZERS_VERSION = "0.21.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.3"
+  VERSION = "0.5.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: x64-mingw-ucrt
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-09-17 00:00:00.000000000 Z
+date: 2024-12-29 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/3.3/tokenizers.so
+- lib/tokenizers/3.4/tokenizers.so
 - lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
@@ -71,14 +72,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.1'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.4.dev
+      version: 3.5.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.4
+rubygems_version: 3.5.23
 signing_key: 
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby