tokenizers 0.5.4 → 0.6.0

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.5.4"
+version = "0.6.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -11,11 +11,12 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = "0.7"
+ahash = { version = "0.8.11", features = ["serde"] }
+magnus = "0.8"
 onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.21.0" # also update in from_pretrained.rb
+version = "=0.22.0" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/encoding.rs

@@ -1,4 +1,4 @@
-use magnus::RArray;
+use magnus::{RArray, Ruby};
 use tk::{Encoding, Offsets};
 
 #[magnus::wrap(class = "Tokenizers::Encoding")]
@@ -50,13 +50,15 @@ impl RbEncoding {
         self.encoding.get_attention_mask().to_vec()
     }
 
-    pub fn overflowing(&self) -> RArray {
-        self.encoding
-            .get_overflowing()
-            .clone()
-            .into_iter()
-            .map(Into::<RbEncoding>::into)
-            .collect()
+    pub fn overflowing(ruby: &Ruby, rb_self: &Self) -> RArray {
+        ruby.ary_from_iter(
+            rb_self
+                .encoding
+                .get_overflowing()
+                .clone()
+                .into_iter()
+                .map(Into::<RbEncoding>::into),
+        )
     }
 
     pub fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {

data/ext/tokenizers/src/models.rs

@@ -3,14 +3,14 @@ use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 
 use crate::trainers::RbTrainer;
+use ahash::AHashMap;
 use magnus::prelude::*;
 use magnus::{
-    data_type_builder, exception, function, method, value::Lazy, Class, DataType,
-    DataTypeFunctions, Error, Module, Object, RClass, RHash, RModule, Ruby, Symbol, TryConvert,
-    TypedData, Value,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error,
+    Module, Object, RClass, RHash, RModule, Ruby, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
-use tk::models::bpe::{BpeBuilder, Merges, Vocab, BPE};
+use tk::models::bpe::{BpeBuilder, Merges, BPE};
 use tk::models::unigram::Unigram;
 use tk::models::wordlevel::WordLevel;
 use tk::models::wordpiece::{WordPiece, WordPieceBuilder};
@@ -72,52 +72,59 @@ pub struct RbBPE {}
 
 impl RbBPE {
     fn with_builder(mut builder: BpeBuilder, kwargs: RHash) -> RbResult<RbModel> {
-        let value: Value = kwargs.delete(Symbol::new("cache_capacity"))?;
+        let ruby = Ruby::get().unwrap();
+
+        let value: Value = kwargs.delete(ruby.to_symbol("cache_capacity"))?;
         if !value.is_nil() {
             builder = builder.cache_capacity(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("dropout"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("dropout"))?;
         if !value.is_nil() {
             builder = builder.dropout(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("unk_token"))?;
         if !value.is_nil() {
             builder = builder.unk_token(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("continuing_subword_prefix"))?;
         if !value.is_nil() {
             builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("end_of_word_suffix"))?;
         if !value.is_nil() {
             builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("fuse_unk"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("fuse_unk"))?;
         if !value.is_nil() {
             builder = builder.fuse_unk(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("byte_fallback"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("byte_fallback"))?;
         if !value.is_nil() {
             builder = builder.byte_fallback(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         builder.build().map(|v| v.into()).map_err(RbError::from)
     }
 
-    pub fn new(vocab: Option<Vocab>, merges: Option<Merges>, kwargs: RHash) -> RbResult<RbModel> {
+    pub fn new(
+        vocab: Option<HashMap<String, u32>>,
+        merges: Option<Merges>,
+        kwargs: RHash,
+    ) -> RbResult<RbModel> {
         let mut builder = BPE::builder();
         if let (Some(vocab), Some(merges)) = (vocab, merges) {
+            let vocab: AHashMap<_, _> = vocab.into_iter().collect();
             builder = builder.vocab_and_merges(vocab, merges);
         }
         RbBPE::with_builder(builder, kwargs)
@@ -125,7 +132,7 @@ impl RbBPE {
 
     pub fn from_file(vocab: String, merges: String, kwargs: RHash) -> RbResult<RbModel> {
         let (vocab, merges) = BPE::read_file(&vocab, &merges).map_err(RbError::from)?;
-
+        let vocab = vocab.into_iter().collect();
         RbBPE::new(Some(vocab), Some(merges), kwargs)
     }
 }
@@ -251,6 +258,7 @@ pub struct RbUnigram {}
 
 impl RbUnigram {
     fn new(
+        ruby: &Ruby,
         vocab: Option<Vec<(String, f64)>>,
         unk_id: Option<usize>,
         byte_fallback: Option<bool>,
@@ -263,7 +271,7 @@ impl RbUnigram {
             }
             (None, None, _) => Ok(Unigram::default().into()),
             _ => Err(Error::new(
-                exception::arg_error(),
+                ruby.exception_arg_error(),
                 "`vocab` and `unk_id` must be both specified",
             )),
         }
@@ -279,7 +287,7 @@ impl RbWordLevel {
     ) -> RbResult<RbModel> {
         let mut builder = WordLevel::builder();
         if let Some(vocab) = vocab {
-            builder = builder.vocab(vocab);
+            builder = builder.vocab(vocab.into_iter().collect());
         }
         if let Some(unk_token) = unk_token {
             builder = builder.unk_token(unk_token);
@@ -287,13 +295,15 @@ impl RbWordLevel {
         builder.build().map(|v| v.into()).map_err(RbError::from)
     }
 
-    pub fn read_file(vocab: String) -> RbResult<Vocab> {
-        WordLevel::read_file(&vocab).map_err(RbError::from)
+    pub fn read_file(vocab: String) -> RbResult<HashMap<String, u32>> {
+        let vocab = WordLevel::read_file(&vocab).map_err(RbError::from)?;
+        let vocab: HashMap<_, _> = vocab.into_iter().collect();
+        Ok(vocab)
     }
 
     pub fn from_file(vocab: String, unk_token: Option<String>) -> RbResult<RbModel> {
         let vocab = WordLevel::read_file(&vocab).map_err(RbError::from)?;
-
+        let vocab = vocab.into_iter().collect();
         RbWordLevel::new(Some(vocab), unk_token)
     }
 }
@@ -302,24 +312,26 @@ pub struct RbWordPiece {}
 
 impl RbWordPiece {
     fn with_builder(mut builder: WordPieceBuilder, kwargs: RHash) -> RbResult<RbModel> {
-        let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
+        let ruby = Ruby::get().unwrap();
+
+        let value: Value = kwargs.delete(ruby.to_symbol("unk_token"))?;
         if !value.is_nil() {
             builder = builder.unk_token(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("max_input_chars_per_word"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("max_input_chars_per_word"))?;
         if !value.is_nil() {
             builder = builder.max_input_chars_per_word(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("continuing_subword_prefix"))?;
         if !value.is_nil() {
             builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         builder.build().map(|v| v.into()).map_err(RbError::from)
@@ -328,6 +340,7 @@ impl RbWordPiece {
     pub fn new(vocab: Option<HashMap<String, u32>>, kwargs: RHash) -> RbResult<RbModel> {
         let mut builder = WordPiece::builder();
         if let Some(vocab) = vocab {
+            let vocab: AHashMap<_, _> = vocab.into_iter().collect();
             builder = builder.vocab(vocab);
         }
         RbWordPiece::with_builder(builder, kwargs)
@@ -336,7 +349,7 @@ impl RbWordPiece {
     pub fn from_file(vocab: String, kwargs: RHash) -> RbResult<RbModel> {
         let vocab = WordPiece::read_file(&vocab).map_err(RbError::from)?;
 
-        RbWordPiece::new(Some(vocab), kwargs)
+        RbWordPiece::new(Some(vocab.into_iter().collect()), kwargs)
     }
 }
 

data/ext/tokenizers/src/normalizers.rs

@@ -199,8 +199,7 @@ impl RbPrecompiled {
         Precompiled::from(&precompiled_charsmap)
             .map_err(|e| {
                 RbError::new_err(format!(
-                    "Error while attempting to build Precompiled normalizer: {}",
-                    e
+                    "Error while attempting to build Precompiled normalizer: {e}"
                 ))
             })
             .map(|v| v.into())

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -1,8 +1,8 @@
 use std::sync::{Arc, RwLock};
 
 use magnus::{
-    data_type_builder, exception, function, method, value::Lazy, Class, DataType,
-    DataTypeFunctions, Error, Module, Object, RArray, RClass, RModule, Ruby, TryConvert, TypedData,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error,
+    Module, Object, RArray, RClass, RModule, Ruby, TryConvert, TypedData,
 };
 
 use serde::ser::SerializeStruct;
@@ -278,16 +278,16 @@ impl RbSequence {
 }
 
 pub(crate) fn from_string(string: String) -> RbResult<PrependScheme> {
+    let ruby = Ruby::get().unwrap();
     let scheme = match string.as_str() {
         "first" => PrependScheme::First,
         "never" => PrependScheme::Never,
         "always" => PrependScheme::Always,
         _ => {
             return Err(Error::new(
-                exception::arg_error(),
+                ruby.exception_arg_error(),
                 format!(
-                    "{} is an unknown variant, should be one of ['first', 'never', 'always']",
-                    string
+                    "{string} is an unknown variant, should be one of ['first', 'never', 'always']"
                 ),
             ));
         }

data/ext/tokenizers/src/tokenizer.rs

@@ -4,7 +4,7 @@ use std::path::PathBuf;
 use std::str::FromStr;
 
 use magnus::prelude::*;
-use magnus::{exception, Error, RArray, RHash, RString, Symbol, TryConvert, Value};
+use magnus::{Error, RArray, RHash, RString, Ruby, TryConvert, Value};
 use tk::tokenizer::{
     Model, PaddingDirection, PaddingParams, PaddingStrategy, TokenizerImpl, TruncationDirection,
     TruncationParams, TruncationStrategy,
@@ -78,37 +78,37 @@ impl From<tk::AddedToken> for RbAddedToken {
 }
 
 impl RbAddedToken {
-    pub fn new(content: Option<String>, kwargs: RHash) -> RbResult<Self> {
+    pub fn new(ruby: &Ruby, content: Option<String>, kwargs: RHash) -> RbResult<Self> {
         let mut token = RbAddedToken::from(content.unwrap_or("".to_string()), None);
 
-        let value: Value = kwargs.delete(Symbol::new("single_word"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("single_word"))?;
         if !value.is_nil() {
             token.single_word = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("lstrip"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("lstrip"))?;
         if !value.is_nil() {
             token.lstrip = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("rstrip"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("rstrip"))?;
         if !value.is_nil() {
             token.rstrip = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("normalized"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("normalized"))?;
         if !value.is_nil() {
             token.normalized = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("special"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("special"))?;
         if !value.is_nil() {
             token.special = TryConvert::try_convert(value)?;
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         Ok(token)
@@ -141,7 +141,7 @@ impl RbAddedToken {
 
 struct TextInputSequence<'s>(tk::InputSequence<'s>);
 
-impl<'s> TryConvert for TextInputSequence<'s> {
+impl TryConvert for TextInputSequence<'_> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         Ok(Self(String::try_convert(ob)?.into()))
     }
@@ -170,7 +170,7 @@ impl From<RbArrayStr> for tk::InputSequence<'_> {
 
 struct PreTokenizedInputSequence<'s>(tk::InputSequence<'s>);
 
-impl<'s> TryConvert for PreTokenizedInputSequence<'s> {
+impl TryConvert for PreTokenizedInputSequence<'_> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         if let Ok(seq) = RbArrayStr::try_convert(ob) {
             return Ok(Self(seq.into()));
@@ -187,8 +187,10 @@ impl<'s> From<PreTokenizedInputSequence<'s>> for tk::InputSequence<'s> {
 
 struct TextEncodeInput<'s>(tk::EncodeInput<'s>);
 
-impl<'s> TryConvert for TextEncodeInput<'s> {
+impl TryConvert for TextEncodeInput<'_> {
     fn try_convert(ob: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(ob);
+
         if let Ok(i) = TextInputSequence::try_convert(ob) {
             return Ok(Self(i.into()));
         }
@@ -204,7 +206,7 @@ impl<'s> TryConvert for TextEncodeInput<'s> {
             }
         }
         Err(Error::new(
-            exception::type_error(),
+            ruby.exception_type_error(),
             "TextEncodeInput must be a string or pair of strings",
         ))
     }
@@ -218,8 +220,10 @@ impl<'s> From<TextEncodeInput<'s>> for tk::tokenizer::EncodeInput<'s> {
 
 struct PreTokenizedEncodeInput<'s>(tk::EncodeInput<'s>);
 
-impl<'s> TryConvert for PreTokenizedEncodeInput<'s> {
+impl TryConvert for PreTokenizedEncodeInput<'_> {
     fn try_convert(ob: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(ob);
+
         if let Ok(i) = PreTokenizedInputSequence::try_convert(ob) {
             return Ok(Self(i.into()));
         }
@@ -237,7 +241,7 @@ impl<'s> TryConvert for PreTokenizedEncodeInput<'s> {
             }
         }
         Err(Error::new(
-            exception::type_error(),
+            ruby.exception_type_error(),
             "PreTokenizedEncodeInput must be an array of strings or pair of arrays",
         ))
     }
@@ -351,7 +355,8 @@ impl RbTokenizer {
     }
 
     pub fn encode_batch(
-        &self,
+        ruby: &Ruby,
+        rb_self: &Self,
         input: RArray,
         is_pretokenized: bool,
         add_special_tokens: bool,
@@ -367,16 +372,16 @@ impl RbTokenizer {
                 Ok(input)
             })
             .collect::<RbResult<Vec<tk::EncodeInput>>>()?;
-        self.tokenizer
-            .borrow()
-            .encode_batch_char_offsets(input, add_special_tokens)
-            .map(|encodings| {
-                encodings
-                    .into_iter()
-                    .map(Into::<RbEncoding>::into)
-                    .collect()
-            })
-            .map_err(RbError::from)
+        Ok(ruby.ary_from_iter(
+            rb_self
+                .tokenizer
+                .borrow()
+                .encode_batch_char_offsets(input, add_special_tokens)
+                .map(|encodings| {
+                    ruby.ary_from_iter(encodings.into_iter().map(Into::<RbEncoding>::into))
+                })
+                .map_err(RbError::from),
+        ))
     }
 
     pub fn decode(&self, ids: Vec<u32>, skip_special_tokens: bool) -> RbResult<String> {
@@ -453,10 +458,10 @@ impl RbTokenizer {
     }
 
     // TODO support more kwargs
-    pub fn enable_padding(&self, kwargs: RHash) -> RbResult<()> {
+    pub fn enable_padding(ruby: &Ruby, rb_self: &Self, kwargs: RHash) -> RbResult<()> {
         let mut params = PaddingParams::default();
 
-        let value: Value = kwargs.delete(Symbol::new("direction"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("direction"))?;
         if !value.is_nil() {
             let dir_str = String::try_convert(value)?;
             params.direction = match dir_str.as_str() {
@@ -464,34 +469,34 @@ impl RbTokenizer {
                 "right" => PaddingDirection::Right,
                 _ => {
                     return Err(Error::new(
-                        exception::arg_error(),
+                        ruby.exception_arg_error(),
                         "The direction value must be 'left' or 'right'",
                     ))
                 }
             }
         }
 
-        let value: Value = kwargs.delete(Symbol::new("pad_to_multiple_of"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("pad_to_multiple_of"))?;
         if !value.is_nil() {
             params.pad_to_multiple_of = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("pad_id"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("pad_id"))?;
         if !value.is_nil() {
             params.pad_id = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("pad_type_id"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("pad_type_id"))?;
         if !value.is_nil() {
             params.pad_type_id = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("pad_token"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("pad_token"))?;
         if !value.is_nil() {
             params.pad_token = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("length"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("length"))?;
         if value.is_nil() {
             params.strategy = PaddingStrategy::BatchLongest;
         } else {
@@ -500,10 +505,10 @@ impl RbTokenizer {
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
-        self.tokenizer.borrow_mut().with_padding(Some(params));
+        rb_self.tokenizer.borrow_mut().with_padding(Some(params));
 
         Ok(())
     }
@@ -512,12 +517,13 @@ impl RbTokenizer {
         self.tokenizer.borrow_mut().with_padding(None);
     }
 
-    pub fn padding(&self) -> RbResult<Option<RHash>> {
-        self.tokenizer
+    pub fn padding(ruby: &Ruby, rb_self: &Self) -> RbResult<Option<RHash>> {
+        rb_self
+            .tokenizer
             .borrow()
             .get_padding()
             .map_or(Ok(None), |params| {
-                let ret_hash = RHash::new();
+                let ret_hash = ruby.hash_new();
 
                 ret_hash.aset(
                     "length",
@@ -536,18 +542,23 @@ impl RbTokenizer {
             })
     }
 
-    pub fn enable_truncation(&self, max_length: usize, kwargs: RHash) -> RbResult<()> {
+    pub fn enable_truncation(
+        ruby: &Ruby,
+        rb_self: &Self,
+        max_length: usize,
+        kwargs: RHash,
+    ) -> RbResult<()> {
         let mut params = TruncationParams {
             max_length,
             ..Default::default()
         };
 
-        let value: Value = kwargs.delete(Symbol::new("stride"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("stride"))?;
         if !value.is_nil() {
             params.stride = TryConvert::try_convert(value)?;
         }
 
-        let value: Value = kwargs.delete(Symbol::new("strategy"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("strategy"))?;
         if !value.is_nil() {
             let strategy_str = String::try_convert(value)?;
             params.strategy = match strategy_str.as_str() {
@@ -555,13 +566,13 @@ impl RbTokenizer {
                 "only_first" => TruncationStrategy::OnlyFirst,
                 "only_second" => TruncationStrategy::OnlySecond,
                 _ => return Err(Error::new(
-                    exception::arg_error(),
+                    ruby.exception_arg_error(),
                     "The strategy value must be 'longest_first', 'only_first', or 'only_second'",
                 )),
             }
         }
 
-        let value: Value = kwargs.delete(Symbol::new("direction"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("direction"))?;
         if !value.is_nil() {
             let dir_str = String::try_convert(value)?;
             params.direction = match dir_str.as_str() {
@@ -569,7 +580,7 @@ impl RbTokenizer {
                 "right" => TruncationDirection::Right,
                 _ => {
                     return Err(Error::new(
-                        exception::arg_error(),
+                        ruby.exception_arg_error(),
                         "The direction value must be 'left' or 'right'",
                     ))
                 }
@@ -578,12 +589,12 @@ impl RbTokenizer {
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
-        if let Err(error_message) = self.tokenizer.borrow_mut().with_truncation(Some(params)) {
+        if let Err(error_message) = rb_self.tokenizer.borrow_mut().with_truncation(Some(params)) {
             return Err(Error::new(
-                exception::arg_error(),
+                ruby.exception_arg_error(),
                 error_message.to_string(),
             ));
         }
@@ -598,12 +609,13 @@ impl RbTokenizer {
             .expect("Failed to set truncation to `None`! This should never happen");
     }
 
-    pub fn truncation(&self) -> RbResult<Option<RHash>> {
-        self.tokenizer
+    pub fn truncation(ruby: &Ruby, rb_self: &Self) -> RbResult<Option<RHash>> {
+        rb_self
+            .tokenizer
             .borrow()
             .get_truncation()
             .map_or(Ok(None), |params| {
-                let ret_hash = RHash::new();
+                let ret_hash = ruby.hash_new();
 
                 ret_hash.aset("max_length", params.max_length)?;
                 ret_hash.aset("stride", params.stride)?;
@@ -629,10 +641,10 @@ impl RbTokenizer {
         self.tokenizer.borrow().get_vocab_size(with_added_tokens)
     }
 
-    pub fn get_added_tokens_decoder(&self) -> RbResult<RHash> {
-        let sorted_map = RHash::new();
+    pub fn get_added_tokens_decoder(ruby: &Ruby, rb_self: &Self) -> RbResult<RHash> {
+        let sorted_map = ruby.hash_new();
 
-        for (key, value) in self.tokenizer.borrow().get_added_tokens_decoder() {
+        for (key, value) in rb_self.tokenizer.borrow().get_added_tokens_decoder() {
             sorted_map.aset::<u32, RbAddedToken>(key, value.into())?;
         }