tokenizers 0.3.2 → 0.4.0

data/ext/tokenizers/src/tokenizer.rs

@@ -2,6 +2,7 @@ use std::cell::RefCell;
 use std::collections::HashMap;
 use std::path::PathBuf;
 
+use magnus::prelude::*;
 use magnus::{exception, Error, RArray, RHash, Symbol, TryConvert, Value};
 use tk::tokenizer::{
     Model, PaddingDirection, PaddingParams, PaddingStrategy,
@@ -78,7 +79,7 @@ struct TextInputSequence<'s>(tk::InputSequence<'s>);
 
 impl<'s> TryConvert for TextInputSequence<'s> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        Ok(Self(ob.try_convert::<String>()?.into()))
+        Ok(Self(String::try_convert(ob)?.into()))
     }
 }
 
@@ -92,7 +93,7 @@ struct RbArrayStr(Vec<String>);
 
 impl TryConvert for RbArrayStr {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        let seq = ob.try_convert::<Vec<String>>()?;
+        let seq = <Vec<String>>::try_convert(ob)?;
         Ok(Self(seq))
     }
 }
@@ -107,7 +108,7 @@ struct PreTokenizedInputSequence<'s>(tk::InputSequence<'s>);
 
 impl<'s> TryConvert for PreTokenizedInputSequence<'s> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        if let Ok(seq) = ob.try_convert::<RbArrayStr>() {
+        if let Ok(seq) = RbArrayStr::try_convert(ob) {
             return Ok(Self(seq.into()));
         }
         todo!()
@@ -124,14 +125,14 @@ struct TextEncodeInput<'s>(tk::EncodeInput<'s>);
 
 impl<'s> TryConvert for TextEncodeInput<'s> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        if let Ok(i) = ob.try_convert::<TextInputSequence>() {
+        if let Ok(i) = TextInputSequence::try_convert(ob) {
             return Ok(Self(i.into()));
         }
-        if let Ok((i1, i2)) = ob.try_convert::<(TextInputSequence, TextInputSequence)>() {
+        if let Ok((i1, i2)) = <(TextInputSequence, TextInputSequence)>::try_convert(ob) {
             return Ok(Self((i1, i2).into()));
         }
         // TODO check if this branch is needed
-        if let Ok(arr) = ob.try_convert::<RArray>() {
+        if let Ok(arr) = RArray::try_convert(ob) {
             if arr.len() == 2 {
                 let first = arr.entry::<TextInputSequence>(0).unwrap();
                 let second = arr.entry::<TextInputSequence>(1).unwrap();
@@ -155,16 +156,16 @@ struct PreTokenizedEncodeInput<'s>(tk::EncodeInput<'s>);
 
 impl<'s> TryConvert for PreTokenizedEncodeInput<'s> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        if let Ok(i) = ob.try_convert::<PreTokenizedInputSequence>() {
+        if let Ok(i) = PreTokenizedInputSequence::try_convert(ob) {
             return Ok(Self(i.into()));
         }
         if let Ok((i1, i2)) =
-            ob.try_convert::<(PreTokenizedInputSequence, PreTokenizedInputSequence)>()
+            <(PreTokenizedInputSequence, PreTokenizedInputSequence)>::try_convert(ob)
         {
             return Ok(Self((i1, i2).into()));
         }
         // TODO check if this branch is needed
-        if let Ok(arr) = ob.try_convert::<RArray>() {
+        if let Ok(arr) = RArray::try_convert(ob) {
             if arr.len() == 2 {
                 let first = arr.entry::<PreTokenizedInputSequence>(0).unwrap();
                 let second = arr.entry::<PreTokenizedInputSequence>(1).unwrap();
@@ -251,16 +252,16 @@ impl RbTokenizer {
         add_special_tokens: bool,
     ) -> RbResult<RbEncoding> {
         let sequence: tk::InputSequence = if is_pretokenized {
-            sequence.try_convert::<PreTokenizedInputSequence>()?.into()
+            PreTokenizedInputSequence::try_convert(sequence)?.into()
         } else {
-            sequence.try_convert::<TextInputSequence>()?.into()
+            TextInputSequence::try_convert(sequence)?.into()
         };
         let input = match pair {
             Some(pair) => {
                 let pair: tk::InputSequence = if is_pretokenized {
-                    pair.try_convert::<PreTokenizedInputSequence>()?.into()
+                    PreTokenizedInputSequence::try_convert(pair)?.into()
                 } else {
-                    pair.try_convert::<TextInputSequence>()?.into()
+                    TextInputSequence::try_convert(pair)?.into()
                 };
                 tk::EncodeInput::Dual(sequence, pair)
             }
@@ -284,9 +285,9 @@ impl RbTokenizer {
             .each()
             .map(|o| {
                 let input: tk::EncodeInput = if is_pretokenized {
-                    o?.try_convert::<PreTokenizedEncodeInput>()?.into()
+                    PreTokenizedEncodeInput::try_convert(o?)?.into()
                 } else {
-                    o?.try_convert::<TextEncodeInput>()?.into()
+                    TextEncodeInput::try_convert(o?)?.into()
                 };
                 Ok(input)
             })
@@ -306,14 +307,15 @@ impl RbTokenizer {
     pub fn decode(&self, ids: Vec<u32>, skip_special_tokens: bool) -> RbResult<String> {
         self.tokenizer
             .borrow()
-            .decode(ids, skip_special_tokens)
+            .decode(&ids, skip_special_tokens)
             .map_err(RbError::from)
     }
 
     pub fn decode_batch(&self, sequences: Vec<Vec<u32>>, skip_special_tokens: bool) -> RbResult<Vec<String>> {
+        let slices = sequences.iter().map(|v| &v[..]).collect::<Vec<&[u32]>>();
         self.tokenizer
             .borrow()
-            .decode_batch(sequences, skip_special_tokens)
+            .decode_batch(&slices, skip_special_tokens)
             .map_err(RbError::from)
     }
 
@@ -353,7 +355,7 @@ impl RbTokenizer {
 
         let value: Value = kwargs.delete(Symbol::new("direction"))?;
         if !value.is_nil() {
-            let dir_str: String = value.try_convert()?;
+            let dir_str = String::try_convert(value)?;
             params.direction = match dir_str.as_str() {
                 "left" => PaddingDirection::Left,
                 "right" => PaddingDirection::Right,
@@ -363,29 +365,29 @@ impl RbTokenizer {
 
         let value: Value = kwargs.delete(Symbol::new("pad_to_multiple_of"))?;
         if !value.is_nil() {
-            params.pad_to_multiple_of = value.try_convert()?;
+            params.pad_to_multiple_of = TryConvert::try_convert(value)?;
         }
 
         let value: Value = kwargs.delete(Symbol::new("pad_id"))?;
         if !value.is_nil() {
-            params.pad_id = value.try_convert()?;
+            params.pad_id = TryConvert::try_convert(value)?;
         }
 
         let value: Value = kwargs.delete(Symbol::new("pad_type_id"))?;
         if !value.is_nil() {
-            params.pad_type_id = value.try_convert()?;
+            params.pad_type_id = TryConvert::try_convert(value)?;
         }
 
         let value: Value = kwargs.delete(Symbol::new("pad_token"))?;
         if !value.is_nil() {
-            params.pad_token = value.try_convert()?;
+            params.pad_token = TryConvert::try_convert(value)?;
         }
 
         let value: Value = kwargs.delete(Symbol::new("length"))?;
         if value.is_nil() {
             params.strategy = PaddingStrategy::BatchLongest;
         } else {
-            params.strategy = PaddingStrategy::Fixed(value.try_convert()?);
+            params.strategy = PaddingStrategy::Fixed(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -431,12 +433,12 @@ impl RbTokenizer {
 
         let value: Value = kwargs.delete(Symbol::new("stride"))?;
         if !value.is_nil() {
-            params.stride = value.try_convert()?;
+            params.stride = TryConvert::try_convert(value)?;
         }
 
         let value: Value = kwargs.delete(Symbol::new("strategy"))?;
         if !value.is_nil() {
-            let strategy_str: String = value.try_convert()?;
+            let strategy_str = String::try_convert(value)?;
             params.strategy = match strategy_str.as_str() {
                 "longest_first" => TruncationStrategy::LongestFirst,
                 "only_first" => TruncationStrategy::OnlyFirst,
@@ -447,7 +449,7 @@ impl RbTokenizer {
 
         let value: Value = kwargs.delete(Symbol::new("direction"))?;
         if !value.is_nil() {
-            let dir_str: String = value.try_convert()?;
+            let dir_str = String::try_convert(value)?;
             params.direction = match dir_str.as_str() {
                 "left" => TruncationDirection::Left,
                 "right" => TruncationDirection::Right,
@@ -460,13 +462,18 @@ impl RbTokenizer {
             return Err(Error::new(exception::arg_error(), "unknown keyword"));
         }
 
-        self.tokenizer.borrow_mut().with_truncation(Some(params));
+        if let Err(error_message) = self.tokenizer.borrow_mut().with_truncation(Some(params)) {
+            return Err(Error::new(exception::arg_error(), error_message.to_string()));
+        }
 
         Ok(())
     }
 
     pub fn no_truncation(&self) {
-        self.tokenizer.borrow_mut().with_truncation(None);
+        self.tokenizer
+            .borrow_mut()
+            .with_truncation(None)
+            .expect("Failed to set truncation to `None`! This should never happen");
     }
 
     pub fn truncation(&self) -> RbResult<Option<RHash>> {

data/ext/tokenizers/src/trainers.rs

@@ -3,16 +3,16 @@ use std::sync::{Arc, RwLock};
 
 use crate::models::RbModel;
 use crate::tokenizer::RbAddedToken;
-use magnus::typed_data::DataTypeBuilder;
+use magnus::prelude::*;
 use magnus::{
-    exception, function, memoize, method, Class, DataType, DataTypeFunctions, Error, Module, Object,
-    RArray, RClass, RHash, RModule, Symbol, TypedData, Value,
+    data_type_builder, exception, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error, Module, Object,
+    RArray, RClass, RHash, RModule, Ruby, Symbol, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
 use tk::models::TrainerWrapper;
 use tk::Trainer;
 
-use super::RbResult;
+use super::{RbResult, TRAINERS};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
 pub struct RbTrainer {
@@ -112,7 +112,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -144,7 +144,7 @@ impl RbTrainer {
             self,
             BpeTrainer,
             initial_alphabet,
-            alphabet.into_iter().map(|c| c).collect()
+            alphabet.into_iter().collect()
         );
     }
 
@@ -199,7 +199,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -223,7 +223,7 @@ impl RbTrainer {
             self,
             UnigramTrainer,
             initial_alphabet,
-            alphabet.into_iter().map(|c| c).collect()
+            alphabet.into_iter().collect()
         );
     }
 
@@ -270,7 +270,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -324,7 +324,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -356,7 +356,7 @@ impl RbTrainer {
             self,
             WordPieceTrainer,
             @set_initial_alphabet,
-            alphabet.into_iter().map(|c| c).collect()
+            alphabet.into_iter().collect()
         );
     }
 
@@ -397,11 +397,10 @@ impl RbBpeTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder = builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -413,39 +412,39 @@ impl RbBpeTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = value.try_convert::<Vec<char>>()?;
+            let arr = <Vec<char>>::try_convert(value)?;
             let set: HashSet<char> = HashSet::from_iter(arr);
             builder = builder.initial_alphabet(set);
         }
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder = builder.vocab_size(value.try_convert()?);
+            builder = builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
         if !value.is_nil() {
-            builder = builder.min_frequency(value.try_convert()?);
+            builder = builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder = builder.show_progress(value.try_convert()?);
+            builder = builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("limit_alphabet"))?;
         if !value.is_nil() {
-            builder = builder.limit_alphabet(value.try_convert()?);
+            builder = builder.limit_alphabet(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
         if !value.is_nil() {
-            builder = builder.continuing_subword_prefix(value.try_convert()?);
+            builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
         if !value.is_nil() {
-            builder = builder.end_of_word_suffix(value.try_convert()?);
+            builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -466,11 +465,10 @@ impl RbUnigramTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -482,44 +480,44 @@ impl RbUnigramTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = value.try_convert::<Vec<char>>()?;
+            let arr = <Vec<char>>::try_convert(value)?;
             let set: HashSet<char> = HashSet::from_iter(arr);
             builder.initial_alphabet(set);
         }
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder.vocab_size(value.try_convert()?);
+            builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder.show_progress(value.try_convert()?);
+            builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("n_sub_iterations"))?;
         if !value.is_nil() {
-            builder.n_sub_iterations(value.try_convert()?);
+            builder.n_sub_iterations(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
         if !value.is_nil() {
-            builder.unk_token(Some(value.try_convert()?));
+            builder.unk_token(Some(TryConvert::try_convert(value)?));
         }
 
         let value: Value = kwargs.delete(Symbol::new("max_piece_length"))?;
         if !value.is_nil() {
-            builder.max_piece_length(value.try_convert()?);
+            builder.max_piece_length(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("seed_size"))?;
         if !value.is_nil() {
-            builder.seed_size(value.try_convert()?);
+            builder.seed_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("shrinking_factor"))?;
         if !value.is_nil() {
-            builder.shrinking_factor(value.try_convert()?);
+            builder.shrinking_factor(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -541,11 +539,10 @@ impl RbWordLevelTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -557,17 +554,17 @@ impl RbWordLevelTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder.vocab_size(value.try_convert()?);
+            builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
         if !value.is_nil() {
-            builder.min_frequency(value.try_convert()?);
+            builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder.show_progress(value.try_convert()?);
+            builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         Ok(builder.build().expect("WordLevelTrainerBuilder cannot fail").into())
@@ -583,11 +580,10 @@ impl RbWordPieceTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder = builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -599,39 +595,39 @@ impl RbWordPieceTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = value.try_convert::<Vec<char>>()?;
+            let arr = <Vec<char>>::try_convert(value)?;
             let set: HashSet<char> = HashSet::from_iter(arr);
             builder = builder.initial_alphabet(set);
         }
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder = builder.vocab_size(value.try_convert()?);
+            builder = builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
         if !value.is_nil() {
-            builder = builder.min_frequency(value.try_convert()?);
+            builder = builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder = builder.show_progress(value.try_convert()?);
+            builder = builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("limit_alphabet"))?;
         if !value.is_nil() {
-            builder = builder.limit_alphabet(value.try_convert()?);
+            builder = builder.limit_alphabet(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
         if !value.is_nil() {
-            builder = builder.continuing_subword_prefix(value.try_convert()?);
+            builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
         if !value.is_nil() {
-            builder = builder.end_of_word_suffix(value.try_convert()?);
+            builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -644,46 +640,52 @@ impl RbWordPieceTrainer {
 }
 
 unsafe impl TypedData for RbTrainer {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::trainers().const_get("Trainer").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("Trainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbTrainer>::new("Tokenizers::Trainers::Trainer").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbTrainer, "Tokenizers::Trainers::Trainer").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static BPE_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("BpeTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static UNIGRAM_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("UnigramTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_LEVEL_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("WordLevelTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_PIECE_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("WordPieceTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match *value.trainer.read().unwrap() {
-            TrainerWrapper::BpeTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("BpeTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            TrainerWrapper::UnigramTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("UnigramTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            TrainerWrapper::WordLevelTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("WordLevelTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            TrainerWrapper::WordPieceTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("WordPieceTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            TrainerWrapper::BpeTrainer(_) => ruby.get_inner(&BPE_TRAINER),
+            TrainerWrapper::UnigramTrainer(_) => ruby.get_inner(&UNIGRAM_TRAINER),
+            TrainerWrapper::WordLevelTrainer(_) => ruby.get_inner(&WORD_LEVEL_TRAINER),
+            TrainerWrapper::WordPieceTrainer(_) => ruby.get_inner(&WORD_PIECE_TRAINER),
         }
     }
 }
 
-pub fn trainers(module: &RModule) -> RbResult<()> {
-    let trainer = module.define_class("Trainer", Default::default())?;
+pub fn init_trainers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let trainer = module.define_class("Trainer", ruby.class_object())?;
 
     let class = module.define_class("BpeTrainer", trainer)?;
     class.define_singleton_method("_new", function!(RbBpeTrainer::new, 1))?;

data/ext/tokenizers/src/utils/normalization.rs

@@ -1,5 +1,6 @@
 use super::regex::{regex, RbRegex};
 use crate::RbResult;
+use magnus::prelude::*;
 use magnus::{exception, Error, TryConvert, Value};
 use tk::normalizer::SplitDelimiterBehavior;
 use tk::pattern::Pattern;
@@ -13,9 +14,9 @@ pub enum RbPattern<'p> {
 impl TryConvert for RbPattern<'_> {
     fn try_convert(obj: Value) -> RbResult<Self> {
         if obj.is_kind_of(regex()) {
-            Ok(RbPattern::Regex(obj.try_convert()?))
+            Ok(RbPattern::Regex(TryConvert::try_convert(obj)?))
         } else {
-            Ok(RbPattern::Str(obj.try_convert()?))
+            Ok(RbPattern::Str(TryConvert::try_convert(obj)?))
         }
     }
 }
@@ -61,7 +62,7 @@ pub struct RbSplitDelimiterBehavior(pub SplitDelimiterBehavior);
 
 impl TryConvert for RbSplitDelimiterBehavior {
     fn try_convert(obj: Value) -> RbResult<Self> {
-        let s = obj.try_convert::<String>()?;
+        let s = String::try_convert(obj)?;
 
         Ok(Self(match s.as_str() {
             "removed" => Ok(SplitDelimiterBehavior::Removed),

data/ext/tokenizers/src/utils/regex.rs

@@ -1,6 +1,6 @@
 use onig::Regex;
-use magnus::{exception, memoize, Error, Module, RClass};
-use crate::{module, RbResult};
+use magnus::{exception, prelude::*, value::Lazy, Error, RClass, Ruby};
+use crate::{RbResult, TOKENIZERS};
 
 #[magnus::wrap(class = "Tokenizers::Regex")]
 pub struct RbRegex {
@@ -17,6 +17,8 @@ impl RbRegex {
     }
 }
 
+static REGEX: Lazy<RClass> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Regex").unwrap());
+
 pub fn regex() -> RClass {
-    *memoize!(RClass: module().const_get("Regex").unwrap())
+    Ruby::get().unwrap().get_inner(&REGEX)
 }

data/lib/tokenizers/decoders/strip.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class Strip
+      def self.new(content: " ", start: 0, stop: 0)
+        _new(content, start, stop)
+      end
+    end
+  end
+end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.13.2"
+    TOKENIZERS_VERSION = "0.14.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux