tokenizers 0.3.2 → 0.4.0

data/ext/tokenizers/src/normalizers.rs

@@ -1,20 +1,19 @@
 use std::sync::{Arc, RwLock};
 
-use magnus::typed_data::DataTypeBuilder;
 use magnus::{
-    function, memoize, method, Class, DataType, DataTypeFunctions, Module, Object, RArray, RClass, RModule,
-    TypedData,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object, RArray, RClass, RModule,
+    Ruby, TryConvert, TypedData,
 };
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Serialize, Serializer};
 use tk::normalizers::{
-    BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Replace, Strip, StripAccents,
+    BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Replace, Prepend, Strip, StripAccents,
     NFC, NFD, NFKC, NFKD,
 };
 use tk::{NormalizedString, Normalizer};
 
 use super::utils::*;
-use super::{RbError, RbResult};
+use super::{NORMALIZERS, RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbNormalizer {
@@ -44,7 +43,7 @@ macro_rules! getter {
     ($self: ident, $variant: ident, $name: ident) => {{
         if let RbNormalizerTypeWrapper::Single(ref norm) = &$self.normalizer {
             let wrapper = norm.read().unwrap();
-            if let RbNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = *wrapper {
+            if let RbNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (*wrapper).clone() {
                 o.$name
             } else {
                 unreachable!()
@@ -105,6 +104,14 @@ impl RbNormalizer {
         setter!(self, BertNormalizer, lowercase, lowercase)
     }
 
+    fn prepend_prepend(&self) -> String {
+        getter!(self, Prepend, prepend)
+    }
+
+    fn prepend_set_prepend(&self, prepend: String) {
+        setter!(self, Prepend, prepend, prepend)
+    }
+
     fn strip_left(&self) -> bool {
         getter!(self, StripNormalizer, strip_left)
     }
@@ -186,6 +193,14 @@ impl RbReplace {
     }
 }
 
+pub struct RbPrepend {}
+
+impl RbPrepend {
+    pub fn new(prepend: String) -> RbNormalizer {
+        Prepend::new(prepend).into()
+    }
+}
+
 pub struct RbStrip {}
 
 impl RbStrip {
@@ -208,7 +223,7 @@ impl RbSequence {
     fn new(normalizers: RArray) -> RbResult<RbNormalizer> {
         let mut sequence = Vec::with_capacity(normalizers.len());
         for n in normalizers.each() {
-            let normalizer: &RbNormalizer = n?.try_convert()?;
+            let normalizer: &RbNormalizer = TryConvert::try_convert(n?)?;
             match &normalizer.normalizer {
                 RbNormalizerTypeWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
                 RbNormalizerTypeWrapper::Single(inner) => sequence.push(inner.clone()),
@@ -311,77 +326,96 @@ impl Normalizer for RbNormalizerWrapper {
 }
 
 unsafe impl TypedData for RbNormalizer {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::normalizers().const_get("Normalizer").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Normalizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbNormalizer>::new("Tokenizers::Normalizers::Normalizer").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbNormalizer, "Tokenizers::Normalizers::Normalizer").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static SEQUENCE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Sequence").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BERT_NORMALIZER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("BertNormalizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static LOWERCASE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Lowercase").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFD: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFD").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFC: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFC").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFKC: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFKC").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NFKD: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("NFKD").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static NMT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Nmt").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static REPLACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Replace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static PREPEND: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Prepend").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static STRIP: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Strip").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static STRIP_ACCENTS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("StripAccents").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match &value.normalizer {
-            RbNormalizerTypeWrapper::Sequence(_seq) => *memoize!(RClass: {
-                let class: RClass = crate::normalizers().const_get("Sequence").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            RbNormalizerTypeWrapper::Sequence(_seq) => ruby.get_inner(&SEQUENCE),
             RbNormalizerTypeWrapper::Single(inner) => match &*inner.read().unwrap() {
                 RbNormalizerWrapper::Wrapped(wrapped) => match &wrapped {
-                    NormalizerWrapper::BertNormalizer(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("BertNormalizer").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Lowercase(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Lowercase").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFD(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFD").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFC(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFC").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFKC(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFKC").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::NFKD(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("NFKD").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Nmt(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Nmt").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::Replace(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Replace").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::StripNormalizer(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("Strip").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    NormalizerWrapper::StripAccents(_) => *memoize!(RClass: {
-                        let class: RClass = crate::normalizers().const_get("StripAccents").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
+                    NormalizerWrapper::BertNormalizer(_) => ruby.get_inner(&BERT_NORMALIZER),
+                    NormalizerWrapper::Lowercase(_) => ruby.get_inner(&LOWERCASE),
+                    NormalizerWrapper::NFD(_) => ruby.get_inner(&NFD),
+                    NormalizerWrapper::NFC(_) => ruby.get_inner(&NFC),
+                    NormalizerWrapper::NFKC(_) => ruby.get_inner(&NFKC),
+                    NormalizerWrapper::NFKD(_) => ruby.get_inner(&NFKD),
+                    NormalizerWrapper::Nmt(_) => ruby.get_inner(&NMT),
+                    NormalizerWrapper::Replace(_) => ruby.get_inner(&REPLACE),
+                    NormalizerWrapper::Prepend(_) => ruby.get_inner(&PREPEND),
+                    NormalizerWrapper::StripNormalizer(_) => ruby.get_inner(&STRIP),
+                    NormalizerWrapper::StripAccents(_) => ruby.get_inner(&STRIP_ACCENTS),
                     _ => todo!(),
                 },
             },
@@ -389,8 +423,8 @@ unsafe impl TypedData for RbNormalizer {
     }
 }
 
-pub fn normalizers(module: &RModule) -> RbResult<()> {
-    let normalizer = module.define_class("Normalizer", Default::default())?;
+pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let normalizer = module.define_class("Normalizer", ruby.class_object())?;
     normalizer.define_method("normalize_str", method!(RbNormalizer::normalize_str, 1))?;
 
     let class = module.define_class("Sequence", normalizer)?;
@@ -428,6 +462,11 @@ pub fn normalizers(module: &RModule) -> RbResult<()> {
     let class = module.define_class("Replace", normalizer)?;
     class.define_singleton_method("new", function!(RbReplace::new, 2))?;
 
+    let class = module.define_class("Prepend", normalizer)?;
+    class.define_singleton_method("_new", function!(RbPrepend::new, 1))?;
+    class.define_method("prepend", method!(RbNormalizer::prepend_prepend, 0))?;
+    class.define_method("prepend=", method!(RbNormalizer::prepend_set_prepend, 1))?;
+
     let class = module.define_class("Strip", normalizer)?;
     class.define_singleton_method("_new", function!(RbStrip::new, 2))?;
     class.define_method("left", method!(RbNormalizer::strip_left, 0))?;

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -1,9 +1,8 @@
 use std::sync::{Arc, RwLock};
 
-use magnus::typed_data::DataTypeBuilder;
 use magnus::{
-    function, memoize, method, Class, DataType, DataTypeFunctions, Module, Object,
-    RArray, RClass, RModule, TypedData,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object,
+    RArray, RClass, RModule, Ruby, TryConvert, TypedData,
 };
 
 use serde::ser::SerializeStruct;
@@ -23,7 +22,7 @@ use tk::tokenizer::Offsets;
 use tk::{PreTokenizedString, PreTokenizer};
 
 use super::utils::*;
-use super::{RbError, RbResult};
+use super::{PRE_TOKENIZERS, RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbPreTokenizer {
@@ -215,7 +214,7 @@ pub struct RbWhitespace {}
 
 impl RbWhitespace {
     pub fn new() -> RbPreTokenizer {
-        Whitespace::default().into()
+        Whitespace.into()
     }
 }
 
@@ -241,7 +240,7 @@ impl RbSequence {
     fn new(pre_tokenizers: RArray) -> RbResult<RbPreTokenizer> {
         let mut sequence = Vec::with_capacity(pre_tokenizers.len());
         for n in pre_tokenizers.each() {
-            let pretokenizer: &RbPreTokenizer = n?.try_convert()?;
+            let pretokenizer: &RbPreTokenizer = TryConvert::try_convert(n?)?;
             match &pretokenizer.pretok {
                 RbPreTokenizerTypeWrapper::Sequence(inner) => {
                     sequence.extend(inner.iter().cloned())
@@ -346,77 +345,90 @@ impl PreTokenizer for RbPreTokenizerWrapper {
 }
 
 unsafe impl TypedData for RbPreTokenizer {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::pre_tokenizers().const_get("PreTokenizer").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("PreTokenizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbPreTokenizer>::new("Tokenizers::PreTokenizers::PreTokenizer").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbPreTokenizer, "Tokenizers::PreTokenizers::PreTokenizer").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static SEQUENCE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Sequence").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BERT_PRE_TOKENIZER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("BertPreTokenizer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BYTE_LEVEL: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("ByteLevel").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static CHAR_DELIMITER_SPLIT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("CharDelimiterSplit").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static DIGITS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Digits").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static METASPACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Metaspace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static PUNCTUATION: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Punctuation").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static SPLIT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Split").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static UNICODE_SCRIPTS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("UnicodeScripts").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WHITESPACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("Whitespace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WHITESPACE_SPLIT: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PRE_TOKENIZERS).const_get("WhitespaceSplit").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match &value.pretok {
-            RbPreTokenizerTypeWrapper::Sequence(_seq) => *memoize!(RClass: {
-                let class: RClass = crate::pre_tokenizers().const_get("Sequence").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            RbPreTokenizerTypeWrapper::Sequence(_seq) => ruby.get_inner(&SEQUENCE),
             RbPreTokenizerTypeWrapper::Single(inner) => match &*inner.read().unwrap() {
                 RbPreTokenizerWrapper::Wrapped(wrapped) => match &wrapped {
-                    PreTokenizerWrapper::BertPreTokenizer(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("BertPreTokenizer").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::ByteLevel(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("ByteLevel").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Delimiter(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("CharDelimiterSplit").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Digits(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Digits").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Metaspace(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Metaspace").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Punctuation(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Punctuation").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Split(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Split").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::UnicodeScripts(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("UnicodeScripts").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::Whitespace(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("Whitespace").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
-                    PreTokenizerWrapper::WhitespaceSplit(_) => *memoize!(RClass: {
-                        let class: RClass = crate::pre_tokenizers().const_get("WhitespaceSplit").unwrap();
-                        class.undef_alloc_func();
-                        class
-                    }),
+                    PreTokenizerWrapper::BertPreTokenizer(_) => ruby.get_inner(&BERT_PRE_TOKENIZER),
+                    PreTokenizerWrapper::ByteLevel(_) => ruby.get_inner(&BYTE_LEVEL),
+                    PreTokenizerWrapper::Delimiter(_) => ruby.get_inner(&CHAR_DELIMITER_SPLIT),
+                    PreTokenizerWrapper::Digits(_) => ruby.get_inner(&DIGITS),
+                    PreTokenizerWrapper::Metaspace(_) => ruby.get_inner(&METASPACE),
+                    PreTokenizerWrapper::Punctuation(_) => ruby.get_inner(&PUNCTUATION),
+                    PreTokenizerWrapper::Split(_) => ruby.get_inner(&SPLIT),
+                    PreTokenizerWrapper::UnicodeScripts(_) => ruby.get_inner(&UNICODE_SCRIPTS),
+                    PreTokenizerWrapper::Whitespace(_) => ruby.get_inner(&WHITESPACE),
+                    PreTokenizerWrapper::WhitespaceSplit(_) => ruby.get_inner(&WHITESPACE_SPLIT),
                     _ => todo!(),
                 },
             },
@@ -424,8 +436,8 @@ unsafe impl TypedData for RbPreTokenizer {
     }
 }
 
-pub fn pre_tokenizers(module: &RModule) -> RbResult<()> {
-    let pre_tokenizer = module.define_class("PreTokenizer", Default::default())?;
+pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let pre_tokenizer = module.define_class("PreTokenizer", ruby.class_object())?;
     pre_tokenizer.define_method("pre_tokenize_str", method!(RbPreTokenizer::pre_tokenize_str, 1))?;
 
     let class = module.define_class("Sequence", pre_tokenizer)?;

data/ext/tokenizers/src/processors.rs

@@ -1,9 +1,8 @@
 use std::sync::Arc;
 
-use magnus::typed_data::DataTypeBuilder;
 use magnus::{
-    function, memoize, Class, DataType, DataTypeFunctions, Module, Object, RClass, RModule,
-    TryConvert, TypedData, Value,
+    data_type_builder, function, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object, RClass, RModule,
+    Ruby, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
 use tk::processors::bert::BertProcessing;
@@ -13,7 +12,7 @@ use tk::processors::template::{SpecialToken, Template};
 use tk::processors::PostProcessorWrapper;
 use tk::{Encoding, PostProcessor};
 
-use super::RbResult;
+use super::{PROCESSORS, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
 pub struct RbPostProcessor {
@@ -53,9 +52,9 @@ impl From<RbSpecialToken> for SpecialToken {
 
 impl TryConvert for RbSpecialToken {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        if let Ok(v) = ob.try_convert::<(String, u32)>() {
+        if let Ok(v) = <(String, u32)>::try_convert(ob) {
             Ok(Self(v.into()))
-        } else if let Ok(v) = ob.try_convert::<(u32, String)>() {
+        } else if let Ok(v) = <(u32, String)>::try_convert(ob) {
             Ok(Self(v.into()))
         } else {
             todo!()
@@ -74,11 +73,11 @@ impl From<RbTemplate> for Template {
 
 impl TryConvert for RbTemplate {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        if let Ok(s) = ob.try_convert::<String>() {
+        if let Ok(s) = String::try_convert(ob) {
             Ok(Self(
                 s.try_into().unwrap(), //.map_err(RbError::from)?,
             ))
-        } else if let Ok(s) = ob.try_convert::<Vec<String>>() {
+        } else if let Ok(s) = <Vec<String>>::try_convert(ob) {
             Ok(Self(
                 s.try_into().unwrap(), //.map_err(RbError::from)?,
             ))
@@ -152,47 +151,53 @@ impl RbTemplateProcessing {
 }
 
 unsafe impl TypedData for RbPostProcessor {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::processors().const_get("PostProcessor").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PROCESSORS).const_get("PostProcessor").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbPostProcessor>::new("Tokenizers::Processors::PostProcessor").build())
+        static DATA_TYPE: DataType = data_type_builder!(RbPostProcessor, "Tokenizers::Processors::PostProcessor").build();
+        &DATA_TYPE
     }
 
-    fn class_for(value: &Self) -> RClass {
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static BERT_PROCESSING: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PROCESSORS).const_get("BertProcessing").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BYTE_LEVEL: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PROCESSORS).const_get("ByteLevel").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static ROBERTA_PROCESSING: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PROCESSORS).const_get("RobertaProcessing").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static TEMPLATE_PROCESSING: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PROCESSORS).const_get("TemplateProcessing").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match *value.processor {
-            PostProcessorWrapper::Bert(_) => *memoize!(RClass: {
-                let class: RClass = crate::processors().const_get("BertProcessing").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            PostProcessorWrapper::ByteLevel(_) => *memoize!(RClass: {
-                let class: RClass = crate::processors().const_get("ByteLevel").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            PostProcessorWrapper::Roberta(_) => *memoize!(RClass: {
-                let class: RClass = crate::processors().const_get("RobertaProcessing").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            PostProcessorWrapper::Template(_) => *memoize!(RClass: {
-                let class: RClass = crate::processors().const_get("TemplateProcessing").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            PostProcessorWrapper::Bert(_) => ruby.get_inner(&BERT_PROCESSING),
+            PostProcessorWrapper::ByteLevel(_) => ruby.get_inner(&BYTE_LEVEL),
+            PostProcessorWrapper::Roberta(_) => ruby.get_inner(&ROBERTA_PROCESSING),
+            PostProcessorWrapper::Template(_) => ruby.get_inner(&TEMPLATE_PROCESSING),
             _ => todo!(),
         }
     }
 }
 
-pub fn processors(module: &RModule) -> RbResult<()> {
-    let post_processor = module.define_class("PostProcessor", Default::default())?;
+pub fn init_processors(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let post_processor = module.define_class("PostProcessor", ruby.class_object())?;
 
     let class = module.define_class("BertProcessing", post_processor)?;
     class.define_singleton_method("new", function!(RbBertProcessing::new, 2))?;