tokenizers 0.5.5 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60cae3e7048bb169293ce830303dc1859d4a18cd992708f18a159b3c7258313e
-  data.tar.gz: 7cac87b564479b6409b98edfca4a499a641b789c9c155a483cea2e897c4651c8
+  metadata.gz: 0f9e999132cb832793cc26a1cc80462332e71c5c3fc06a90bc652d88f5d850c0
+  data.tar.gz: 0705c9463baed06e0c2fc887514fa983e635f8bb99094bc6b5859ad6adbeaccb
 SHA512:
-  metadata.gz: 4fb4bc02b416c13fd179f9f27a73748dd1ad1683ce8b6a0d1ac2400b3dbb1b42519b04488db19fa85340fbc5ac292fa312c1a05b615391def017806e203ebca7
-  data.tar.gz: d8ddeea648974bd7772f2f0200727db1ad781c66bdd483551023db7e30b4926bec9a7677a23183b2ae80fc775e11e54d8f385f118fed962609f7d70d19094b4c
+  metadata.gz: eb18eb98388d80ad6043d3dac967e40f0c9d7dfaa9edd615dca49f1b42c8ca1acffe5096b3250c5a94f230c17bd660ba1c4c55a16e65294354734e20331017b6
+  data.tar.gz: 8ac036c36f7f43eb050d65bab50b03cb434bd7d02f7abb00b0056b5555b226a9db82bbde56bd3b8d7826491e53343342b273e0ebfa131575fc529d90ed163978

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.0 (2025-08-29)
+
+- Updated Tokenizers to 0.22.0
+- Dropped support for Ruby < 3.2
+
 ## 0.5.5 (2025-05-04)
 
 - Updated Tokenizers to 0.21.1

data/Cargo.lock

@@ -2,6 +2,20 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -23,7 +37,7 @@ version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
- "bitflags 2.9.0",
+ "bitflags",
  "cexpr",
  "clang-sys",
  "itertools 0.12.1",
@@ -37,12 +51,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bitflags"
 version = "2.9.0"
@@ -55,6 +63,15 @@ version = "3.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
 
+[[package]]
+name = "castaway"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "cc"
 version = "1.2.21"
@@ -90,6 +107,21 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "compact_str"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "serde",
+ "static_assertions",
+]
+
 [[package]]
 name = "console"
 version = "0.15.11"
@@ -163,6 +195,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "dary_heap"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "derive_builder"
 version = "0.20.2"
@@ -223,12 +264,13 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "getrandom"
-version = "0.2.16"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
 dependencies = [
  "cfg-if",
  "libc",
+ "r-efi",
  "wasi",
 ]
 
@@ -257,15 +299,6 @@ dependencies = [
  "web-time",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.12.1"
@@ -277,9 +310,9 @@ dependencies = [
 
 [[package]]
 name = "itertools"
-version = "0.13.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
 dependencies = [
  "either",
 ]
@@ -352,9 +385,9 @@ checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
 
 [[package]]
 name = "magnus"
-version = "0.7.1"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
+checksum = "3f14d3cc31b2dc4fce6cd447a83c7a7ca2ab8a9f1e535dcb2f796ff972b0e68b"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -364,9 +397,9 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
+checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -430,11 +463,11 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
 [[package]]
 name = "onig"
-version = "6.4.0"
+version = "6.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
  "libc",
  "once_cell",
  "onig_sys",
@@ -442,9 +475,9 @@ dependencies = [
 
 [[package]]
 name = "onig_sys"
-version = "69.8.1"
+version = "69.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"
 dependencies = [
  "cc",
  "pkg-config",
@@ -495,22 +528,27 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
 [[package]]
 name = "rand"
-version = "0.8.5"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
 dependencies = [
- "libc",
  "rand_chacha",
  "rand_core",
 ]
 
 [[package]]
 name = "rand_chacha"
-version = "0.3.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
 dependencies = [
  "ppv-lite86",
  "rand_core",
@@ -518,9 +556,9 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.6.4"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
 dependencies = [
  "getrandom",
 ]
@@ -537,12 +575,12 @@ dependencies = [
 
 [[package]]
 name = "rayon-cond"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
+checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
 dependencies = [
  "either",
- "itertools 0.11.0",
+ "itertools 0.14.0",
  "rayon",
 ]
 
@@ -558,18 +596,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.111"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "becea799ce051c16fb140be80f5e7cf781070f99ca099332383c2b17861249af"
+checksum = "f900d1ce4629a2ebffaf5de74bd8f9c1188d4c5ed406df02f97e22f77a006f44"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.111"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64691175abc704862f60a9ca8ef06174080cc50615f2bf1d4759f46db18b4d29"
+checksum = "ef1e9c857028f631056bcd6d88cec390c751e343ce2223ddb26d23eb4a151d59"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -582,9 +620,9 @@ dependencies = [
 
 [[package]]
 name = "rb-sys-env"
-version = "0.1.2"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
+checksum = "08f8d2924cf136a1315e2b4c7460a39f62ef11ee5d522df9b2750fab55b868b6"
 
 [[package]]
 name = "regex"
@@ -621,6 +659,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustversion"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
+
 [[package]]
 name = "ryu"
 version = "1.0.20"
@@ -695,6 +739,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -734,27 +784,30 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.5"
+version = "0.6.0"
 dependencies = [
+ "ahash",
  "magnus",
  "onig",
  "serde",
- "tokenizers 0.21.1",
+ "tokenizers 0.22.0",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.21.1"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3169b3195f925496c895caee7978a335d49218488ef22375267fba5a46a40bd7"
+checksum = "af10f51be57162b69d90a15cb226eef12c9e4faecbd5e3ea98a86bfb920b3d71"
 dependencies = [
+ "ahash",
  "aho-corasick",
+ "compact_str",
+ "dary_heap",
  "derive_builder",
  "esaxx-rs",
  "getrandom",
  "indicatif",
- "itertools 0.13.0",
- "lazy_static",
+ "itertools 0.14.0",
  "log",
  "macro_rules_attribute",
  "monostate",
@@ -807,11 +860,20 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
 [[package]]
 name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
+version = "0.14.2+wasi-0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
+dependencies = [
+ "wit-bindgen-rt",
+]
 
 [[package]]
 name = "wasm-bindgen"
@@ -953,6 +1015,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.8.25"

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.5.5"
+version = "0.6.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -11,11 +11,12 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = "0.7"
+ahash = { version = "0.8.11", features = ["serde"] }
+magnus = "0.8"
 onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.21.1" # also update in from_pretrained.rb
+version = "=0.22.0" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/encoding.rs

@@ -1,4 +1,4 @@
-use magnus::RArray;
+use magnus::{RArray, Ruby};
 use tk::{Encoding, Offsets};
 
 #[magnus::wrap(class = "Tokenizers::Encoding")]
@@ -50,13 +50,15 @@ impl RbEncoding {
         self.encoding.get_attention_mask().to_vec()
     }
 
-    pub fn overflowing(&self) -> RArray {
-        self.encoding
-            .get_overflowing()
-            .clone()
-            .into_iter()
-            .map(Into::<RbEncoding>::into)
-            .collect()
+    pub fn overflowing(ruby: &Ruby, rb_self: &Self) -> RArray {
+        ruby.ary_from_iter(
+            rb_self
+                .encoding
+                .get_overflowing()
+                .clone()
+                .into_iter()
+                .map(Into::<RbEncoding>::into),
+        )
     }
 
     pub fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {

data/ext/tokenizers/src/models.rs

@@ -3,14 +3,14 @@ use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 
 use crate::trainers::RbTrainer;
+use ahash::AHashMap;
 use magnus::prelude::*;
 use magnus::{
-    data_type_builder, exception, function, method, value::Lazy, Class, DataType,
-    DataTypeFunctions, Error, Module, Object, RClass, RHash, RModule, Ruby, Symbol, TryConvert,
-    TypedData, Value,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error,
+    Module, Object, RClass, RHash, RModule, Ruby, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
-use tk::models::bpe::{BpeBuilder, Merges, Vocab, BPE};
+use tk::models::bpe::{BpeBuilder, Merges, BPE};
 use tk::models::unigram::Unigram;
 use tk::models::wordlevel::WordLevel;
 use tk::models::wordpiece::{WordPiece, WordPieceBuilder};
@@ -72,52 +72,59 @@ pub struct RbBPE {}
 
 impl RbBPE {
     fn with_builder(mut builder: BpeBuilder, kwargs: RHash) -> RbResult<RbModel> {
-        let value: Value = kwargs.delete(Symbol::new("cache_capacity"))?;
+        let ruby = Ruby::get().unwrap();
+
+        let value: Value = kwargs.delete(ruby.to_symbol("cache_capacity"))?;
         if !value.is_nil() {
             builder = builder.cache_capacity(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("dropout"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("dropout"))?;
         if !value.is_nil() {
             builder = builder.dropout(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("unk_token"))?;
         if !value.is_nil() {
             builder = builder.unk_token(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("continuing_subword_prefix"))?;
         if !value.is_nil() {
             builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("end_of_word_suffix"))?;
         if !value.is_nil() {
             builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("fuse_unk"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("fuse_unk"))?;
         if !value.is_nil() {
             builder = builder.fuse_unk(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("byte_fallback"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("byte_fallback"))?;
         if !value.is_nil() {
             builder = builder.byte_fallback(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         builder.build().map(|v| v.into()).map_err(RbError::from)
     }
 
-    pub fn new(vocab: Option<Vocab>, merges: Option<Merges>, kwargs: RHash) -> RbResult<RbModel> {
+    pub fn new(
+        vocab: Option<HashMap<String, u32>>,
+        merges: Option<Merges>,
+        kwargs: RHash,
+    ) -> RbResult<RbModel> {
         let mut builder = BPE::builder();
         if let (Some(vocab), Some(merges)) = (vocab, merges) {
+            let vocab: AHashMap<_, _> = vocab.into_iter().collect();
             builder = builder.vocab_and_merges(vocab, merges);
         }
         RbBPE::with_builder(builder, kwargs)
@@ -125,7 +132,7 @@ impl RbBPE {
 
     pub fn from_file(vocab: String, merges: String, kwargs: RHash) -> RbResult<RbModel> {
         let (vocab, merges) = BPE::read_file(&vocab, &merges).map_err(RbError::from)?;
-
+        let vocab = vocab.into_iter().collect();
         RbBPE::new(Some(vocab), Some(merges), kwargs)
     }
 }
@@ -251,6 +258,7 @@ pub struct RbUnigram {}
 
 impl RbUnigram {
     fn new(
+        ruby: &Ruby,
         vocab: Option<Vec<(String, f64)>>,
         unk_id: Option<usize>,
         byte_fallback: Option<bool>,
@@ -263,7 +271,7 @@ impl RbUnigram {
             }
             (None, None, _) => Ok(Unigram::default().into()),
             _ => Err(Error::new(
-                exception::arg_error(),
+                ruby.exception_arg_error(),
                 "`vocab` and `unk_id` must be both specified",
             )),
         }
@@ -279,7 +287,7 @@ impl RbWordLevel {
     ) -> RbResult<RbModel> {
         let mut builder = WordLevel::builder();
         if let Some(vocab) = vocab {
-            builder = builder.vocab(vocab);
+            builder = builder.vocab(vocab.into_iter().collect());
         }
         if let Some(unk_token) = unk_token {
             builder = builder.unk_token(unk_token);
@@ -287,13 +295,15 @@ impl RbWordLevel {
         builder.build().map(|v| v.into()).map_err(RbError::from)
     }
 
-    pub fn read_file(vocab: String) -> RbResult<Vocab> {
-        WordLevel::read_file(&vocab).map_err(RbError::from)
+    pub fn read_file(vocab: String) -> RbResult<HashMap<String, u32>> {
+        let vocab = WordLevel::read_file(&vocab).map_err(RbError::from)?;
+        let vocab: HashMap<_, _> = vocab.into_iter().collect();
+        Ok(vocab)
     }
 
     pub fn from_file(vocab: String, unk_token: Option<String>) -> RbResult<RbModel> {
         let vocab = WordLevel::read_file(&vocab).map_err(RbError::from)?;
-
+        let vocab = vocab.into_iter().collect();
         RbWordLevel::new(Some(vocab), unk_token)
     }
 }
@@ -302,24 +312,26 @@ pub struct RbWordPiece {}
 
 impl RbWordPiece {
     fn with_builder(mut builder: WordPieceBuilder, kwargs: RHash) -> RbResult<RbModel> {
-        let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
+        let ruby = Ruby::get().unwrap();
+
+        let value: Value = kwargs.delete(ruby.to_symbol("unk_token"))?;
         if !value.is_nil() {
             builder = builder.unk_token(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("max_input_chars_per_word"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("max_input_chars_per_word"))?;
         if !value.is_nil() {
             builder = builder.max_input_chars_per_word(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("continuing_subword_prefix"))?;
         if !value.is_nil() {
             builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         builder.build().map(|v| v.into()).map_err(RbError::from)
@@ -328,6 +340,7 @@ impl RbWordPiece {
     pub fn new(vocab: Option<HashMap<String, u32>>, kwargs: RHash) -> RbResult<RbModel> {
         let mut builder = WordPiece::builder();
         if let Some(vocab) = vocab {
+            let vocab: AHashMap<_, _> = vocab.into_iter().collect();
             builder = builder.vocab(vocab);
         }
         RbWordPiece::with_builder(builder, kwargs)
@@ -336,7 +349,7 @@ impl RbWordPiece {
     pub fn from_file(vocab: String, kwargs: RHash) -> RbResult<RbModel> {
         let vocab = WordPiece::read_file(&vocab).map_err(RbError::from)?;
 
-        RbWordPiece::new(Some(vocab), kwargs)
+        RbWordPiece::new(Some(vocab.into_iter().collect()), kwargs)
     }
 }
 

data/ext/tokenizers/src/normalizers.rs

@@ -199,8 +199,7 @@ impl RbPrecompiled {
         Precompiled::from(&precompiled_charsmap)
             .map_err(|e| {
                 RbError::new_err(format!(
-                    "Error while attempting to build Precompiled normalizer: {}",
-                    e
+                    "Error while attempting to build Precompiled normalizer: {e}"
                 ))
             })
             .map(|v| v.into())