tokenizers 0.3.3 → 0.4.0

data/ext/tokenizers/src/models.rs

@@ -3,10 +3,10 @@ use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 
 use crate::trainers::RbTrainer;
-use magnus::typed_data::DataTypeBuilder;
+use magnus::prelude::*;
 use magnus::{
-    exception, function, memoize, method, Class, DataType, DataTypeFunctions, Error, Module, Object,
-    RClass, RHash, RModule, Symbol, TypedData, Value,
+    data_type_builder, exception, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error, Module, Object,
+    RClass, RHash, RModule, Ruby, Symbol, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
 use tk::models::bpe::{BpeBuilder, Merges, Vocab, BPE};
@@ -16,7 +16,7 @@ use tk::models::wordlevel::WordLevel;
 use tk::models::wordpiece::{WordPiece, WordPieceBuilder};
 use tk::{Model, Token};
 
-use super::{RbError, RbResult};
+use super::{MODELS, RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbModel {
@@ -73,37 +73,37 @@ impl RbBPE {
     fn with_builder(mut builder: BpeBuilder, kwargs: RHash) -> RbResult<RbModel> {
         let value: Value = kwargs.delete(Symbol::new("cache_capacity"))?;
         if !value.is_nil() {
-            builder = builder.cache_capacity(value.try_convert()?);
+            builder = builder.cache_capacity(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("dropout"))?;
         if !value.is_nil() {
-            builder = builder.dropout(value.try_convert()?);
+            builder = builder.dropout(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
         if !value.is_nil() {
-            builder = builder.unk_token(value.try_convert()?);
+            builder = builder.unk_token(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
         if !value.is_nil() {
-            builder = builder.continuing_subword_prefix(value.try_convert()?);
+            builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
         if !value.is_nil() {
-            builder = builder.end_of_word_suffix(value.try_convert()?);
+            builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("fuse_unk"))?;
         if !value.is_nil() {
-            builder = builder.fuse_unk(value.try_convert()?);
+            builder = builder.fuse_unk(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("byte_fallback"))?;
         if !value.is_nil() {
-            builder = builder.byte_fallback(value.try_convert()?);
+            builder = builder.byte_fallback(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -234,13 +234,13 @@ impl RbModel {
 pub struct RbUnigram {}
 
 impl RbUnigram {
-    fn new(vocab: Option<Vec<(String, f64)>>, unk_id: Option<usize>) -> RbResult<RbModel> {
-        match (vocab, unk_id) {
-            (Some(vocab), unk_id) => {
-                let model = Unigram::from(vocab, unk_id).map_err(RbError::from)?;
+    fn new(vocab: Option<Vec<(String, f64)>>, unk_id: Option<usize>, byte_fallback: Option<bool>) -> RbResult<RbModel> {
+        match (vocab, unk_id, byte_fallback) {
+            (Some(vocab), unk_id, byte_fallback) => {
+                let model = Unigram::from(vocab, unk_id, byte_fallback.unwrap_or(false)).map_err(RbError::from)?;
                 Ok(model.into())
             }
-            (None, None) => Ok(Unigram::default().into()),
+            (None, None, _) => Ok(Unigram::default().into()),
             _ => Err(Error::new(exception::arg_error(), "`vocab` and `unk_id` must be both specified")),
         }
     }
@@ -277,17 +277,17 @@ impl RbWordPiece {
     fn with_builder(mut builder: WordPieceBuilder, kwargs: RHash) -> RbResult<RbModel> {
         let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
         if !value.is_nil() {
-            builder = builder.unk_token(value.try_convert()?);
+            builder = builder.unk_token(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("max_input_chars_per_word"))?;
         if !value.is_nil() {
-            builder = builder.max_input_chars_per_word(value.try_convert()?);
+            builder = builder.max_input_chars_per_word(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
         if !value.is_nil() {
-            builder = builder.continuing_subword_prefix(value.try_convert()?);
+            builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -314,46 +314,52 @@ impl RbWordPiece {
 }
 
 unsafe impl TypedData for RbModel {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::models().const_get("Model").unwrap();
-          class.undef_alloc_func();
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+          let class: RClass = ruby.get_inner(&MODELS).const_get("Model").unwrap();
+          class.undef_default_alloc_func();
           class
-        })
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbModel>::new("Tokenizers::Models::Model").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbModel, "Tokenizers::Models::Model").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static BPE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&MODELS).const_get("BPE").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static UNIGRAM: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&MODELS).const_get("Unigram").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_LEVEL: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&MODELS).const_get("WordLevel").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_PIECE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&MODELS).const_get("WordPiece").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match *value.model.read().unwrap() {
-            ModelWrapper::BPE(_) => *memoize!(RClass: {
-                let class: RClass = crate::models().const_get("BPE").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            ModelWrapper::Unigram(_) => *memoize!(RClass: {
-                let class: RClass = crate::models().const_get("Unigram").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            ModelWrapper::WordLevel(_) => *memoize!(RClass: {
-                let class: RClass = crate::models().const_get("WordLevel").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            ModelWrapper::WordPiece(_) => *memoize!(RClass: {
-                let class: RClass = crate::models().const_get("WordPiece").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            ModelWrapper::BPE(_) => ruby.get_inner(&BPE),
+            ModelWrapper::Unigram(_) => ruby.get_inner(&UNIGRAM),
+            ModelWrapper::WordLevel(_) => ruby.get_inner(&WORD_LEVEL),
+            ModelWrapper::WordPiece(_) => ruby.get_inner(&WORD_PIECE),
         }
     }
 }
 
-pub fn models(module: &RModule) -> RbResult<()> {
-    let model = module.define_class("Model", Default::default())?;
+pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let model = module.define_class("Model", ruby.class_object())?;
 
     let class = module.define_class("BPE", model)?;
     class.define_singleton_method("_new", function!(RbBPE::new, 3))?;
@@ -372,7 +378,7 @@ pub fn models(module: &RModule) -> RbResult<()> {
     class.define_method("byte_fallback=", method!(RbModel::bpe_set_byte_fallback, 1))?;
 
     let class = module.define_class("Unigram", model)?;
-    class.define_singleton_method("_new", function!(RbUnigram::new, 2))?;
+    class.define_singleton_method("_new", function!(RbUnigram::new, 3))?;
 
     let class = module.define_class("WordLevel", model)?;
     class.define_singleton_method("_new", function!(RbWordLevel::new, 2))?;

data/ext/tokenizers/src/normalizers.rs

@@ -1,9 +1,8 @@
 use std::sync::{Arc, RwLock};
 
-use magnus::typed_data::DataTypeBuilder;
 use magnus::{
-    function, memoize, method, Class, DataType, DataTypeFunctions, Module, Object, RArray, RClass, RModule,
-    TypedData,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object, RArray, RClass, RModule,
+    Ruby, TryConvert, TypedData,
 };
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Serialize, Serializer};
@@ -14,7 +13,7 @@ use tk::normalizers::{
 use tk::{NormalizedString, Normalizer};
 
 use super::utils::*;
-use super::{RbError, RbResult};
+use super::{NORMALIZERS, RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbNormalizer {
@@ -224,7 +223,7 @@ impl RbSequence {
     fn new(normalizers: RArray) -> RbResult<RbNormalizer> {
         let mut sequence = Vec::with_capacity(normalizers.len());
         for n in normalizers.each() {
-            let normalizer: &RbNormalizer = n?.try_convert()?;
+            let normalizer: &RbNormalizer = TryConvert::try_convert(n?)?;
             match &normalizer.normalizer {
                 RbNormalizerTypeWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
                 RbNormalizerTypeWrapper::Single(inner) => sequence.push(inner.clone()),
@@ -327,82 +326,96 @@ impl Normalizer for RbNormalizerWrapper {
 }
 
 unsafe impl TypedData for RbNormalizer {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::normalizers().const_get("Normalizer").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Normalizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbNormalizer>::new("Tokenizers::Normalizers::Normalizer").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbNormalizer, "Tokenizers::Normalizers::Normalizer").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static SEQUENCE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Sequence").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BERT_NORMALIZER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("BertNormalizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static LOWERCASE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Lowercase").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFD: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFD").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFC: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFC").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFKC: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFKC").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFKD: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFKD").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NMT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Nmt").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static REPLACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Replace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static PREPEND: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Prepend").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static STRIP: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Strip").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static STRIP_ACCENTS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("StripAccents").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match &value.normalizer {
-            RbNormalizerTypeWrapper::Sequence(_seq) => *memoize!(RClass: {
-                let class: RClass = crate::normalizers().const_get("Sequence").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            RbNormalizerTypeWrapper::Sequence(_seq) => ruby.get_inner(&SEQUENCE),
             RbNormalizerTypeWrapper::Single(inner) => match &*inner.read().unwrap() {
                 RbNormalizerWrapper::Wrapped(wrapped) => match &wrapped {
-                    NormalizerWrapper::BertNormalizer(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("BertNormalizer").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Lowercase(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Lowercase").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFD(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFD").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFC(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFC").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFKC(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFKC").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFKD(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFKD").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Nmt(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Nmt").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Replace(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Replace").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Prepend(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Prepend").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::StripNormalizer(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Strip").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::StripAccents(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("StripAccents").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
+                    NormalizerWrapper::BertNormalizer(_) => ruby.get_inner(&BERT_NORMALIZER),
+                    NormalizerWrapper::Lowercase(_) => ruby.get_inner(&LOWERCASE),
+                    NormalizerWrapper::NFD(_) => ruby.get_inner(&NFD),
+                    NormalizerWrapper::NFC(_) => ruby.get_inner(&NFC),
+                    NormalizerWrapper::NFKC(_) => ruby.get_inner(&NFKC),
+                    NormalizerWrapper::NFKD(_) => ruby.get_inner(&NFKD),
+                    NormalizerWrapper::Nmt(_) => ruby.get_inner(&NMT),
+                    NormalizerWrapper::Replace(_) => ruby.get_inner(&REPLACE),
+                    NormalizerWrapper::Prepend(_) => ruby.get_inner(&PREPEND),
+                    NormalizerWrapper::StripNormalizer(_) => ruby.get_inner(&STRIP),
+                    NormalizerWrapper::StripAccents(_) => ruby.get_inner(&STRIP_ACCENTS),
                     _ => todo!(),
                 },
             },
@@ -410,8 +423,8 @@ unsafe impl TypedData for RbNormalizer {
     }
 }
 
-pub fn normalizers(module: &RModule) -> RbResult<()> {
-    let normalizer = module.define_class("Normalizer", Default::default())?;
+pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let normalizer = module.define_class("Normalizer", ruby.class_object())?;
     normalizer.define_method("normalize_str", method!(RbNormalizer::normalize_str, 1))?;
 
     let class = module.define_class("Sequence", normalizer)?;

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -1,9 +1,8 @@
 use std::sync::{Arc, RwLock};
 
-use magnus::typed_data::DataTypeBuilder;
 use magnus::{
-    function, memoize, method, Class, DataType, DataTypeFunctions, Module, Object,
-    RArray, RClass, RModule, TypedData,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object,
+    RArray, RClass, RModule, Ruby, TryConvert, TypedData,
 };
 
 use serde::ser::SerializeStruct;
@@ -23,7 +22,7 @@ use tk::tokenizer::Offsets;
 use tk::{PreTokenizedString, PreTokenizer};
 
 use super::utils::*;
-use super::{RbError, RbResult};
+use super::{PRE_TOKENIZERS, RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbPreTokenizer {
@@ -215,7 +214,7 @@ pub struct RbWhitespace {}
 
 impl RbWhitespace {
     pub fn new() -> RbPreTokenizer {
-        Whitespace::default().into()
+        Whitespace.into()
     }
 }
 
@@ -241,7 +240,7 @@ impl RbSequence {
     fn new(pre_tokenizers: RArray) -> RbResult<RbPreTokenizer> {
         let mut sequence = Vec::with_capacity(pre_tokenizers.len());
         for n in pre_tokenizers.each() {
-            let pretokenizer: &RbPreTokenizer = n?.try_convert()?;
+            let pretokenizer: &RbPreTokenizer = TryConvert::try_convert(n?)?;
             match &pretokenizer.pretok {
                 RbPreTokenizerTypeWrapper::Sequence(inner) => {
                     sequence.extend(inner.iter().cloned())
@@ -346,77 +345,90 @@ impl PreTokenizer for RbPreTokenizerWrapper {
 }
 
 unsafe impl TypedData for RbPreTokenizer {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::pre_tokenizers().const_get("PreTokenizer").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("PreTokenizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbPreTokenizer>::new("Tokenizers::PreTokenizers::PreTokenizer").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbPreTokenizer, "Tokenizers::PreTokenizers::PreTokenizer").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static SEQUENCE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Sequence").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BERT_PRE_TOKENIZER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("BertPreTokenizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BYTE_LEVEL: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("ByteLevel").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static CHAR_DELIMITER_SPLIT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("CharDelimiterSplit").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static DIGITS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Digits").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static METASPACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Metaspace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static PUNCTUATION: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Punctuation").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static SPLIT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Split").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static UNICODE_SCRIPTS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("UnicodeScripts").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WHITESPACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Whitespace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WHITESPACE_SPLIT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("WhitespaceSplit").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match &value.pretok {
-            RbPreTokenizerTypeWrapper::Sequence(_seq) => *memoize!(RClass: {
-                let class: RClass = crate::pre_tokenizers().const_get("Sequence").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            RbPreTokenizerTypeWrapper::Sequence(_seq) => ruby.get_inner(&SEQUENCE),
             RbPreTokenizerTypeWrapper::Single(inner) => match &*inner.read().unwrap() {
                 RbPreTokenizerWrapper::Wrapped(wrapped) => match &wrapped {
-                    PreTokenizerWrapper::BertPreTokenizer(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("BertPreTokenizer").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::ByteLevel(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("ByteLevel").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Delimiter(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("CharDelimiterSplit").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Digits(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Digits").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Metaspace(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Metaspace").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Punctuation(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Punctuation").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Split(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Split").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::UnicodeScripts(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("UnicodeScripts").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Whitespace(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Whitespace").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::WhitespaceSplit(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("WhitespaceSplit").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
+                    PreTokenizerWrapper::BertPreTokenizer(_) => ruby.get_inner(&BERT_PRE_TOKENIZER),
+                    PreTokenizerWrapper::ByteLevel(_) => ruby.get_inner(&BYTE_LEVEL),
+                    PreTokenizerWrapper::Delimiter(_) => ruby.get_inner(&CHAR_DELIMITER_SPLIT),
+                    PreTokenizerWrapper::Digits(_) => ruby.get_inner(&DIGITS),
+                    PreTokenizerWrapper::Metaspace(_) => ruby.get_inner(&METASPACE),
+                    PreTokenizerWrapper::Punctuation(_) => ruby.get_inner(&PUNCTUATION),
+                    PreTokenizerWrapper::Split(_) => ruby.get_inner(&SPLIT),
+                    PreTokenizerWrapper::UnicodeScripts(_) => ruby.get_inner(&UNICODE_SCRIPTS),
+                    PreTokenizerWrapper::Whitespace(_) => ruby.get_inner(&WHITESPACE),
+                    PreTokenizerWrapper::WhitespaceSplit(_) => ruby.get_inner(&WHITESPACE_SPLIT),
                     _ => todo!(),
                 },
             },
@@ -424,8 +436,8 @@ unsafe impl TypedData for RbPreTokenizer {
     }
 }
 
-pub fn pre_tokenizers(module: &RModule) -> RbResult<()> {
-    let pre_tokenizer = module.define_class("PreTokenizer", Default::default())?;
+pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let pre_tokenizer = module.define_class("PreTokenizer", ruby.class_object())?;
     pre_tokenizer.define_method("pre_tokenize_str", method!(RbPreTokenizer::pre_tokenize_str, 1))?;
 
     let class = module.define_class("Sequence", pre_tokenizer)?;