tokenizers 0.5.2 → 0.5.4

data/ext/tokenizers/src/error.rs

@@ -9,9 +9,14 @@ impl RbError {
     pub fn from(e: Box<dyn std::error::Error + Send + Sync>) -> Error {
         Error::new(error(), e.to_string())
     }
+
+    pub fn new_err(s: String) -> Error {
+        Error::new(error(), s)
+    }
 }
 
-static ERROR: Lazy<ExceptionClass> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Error").unwrap());
+static ERROR: Lazy<ExceptionClass> =
+    Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Error").unwrap());
 
 fn error() -> ExceptionClass {
     Ruby::get().unwrap().get_inner(&ERROR)

data/ext/tokenizers/src/lib.rs

@@ -15,26 +15,39 @@ mod utils;
 
 use encoding::RbEncoding;
 use error::RbError;
-use tokenizer::RbTokenizer;
+use tokenizer::{RbAddedToken, RbTokenizer};
 use utils::RbRegex;
 
 use magnus::{function, method, prelude::*, value::Lazy, Error, RModule, Ruby};
 
 type RbResult<T> = Result<T, Error>;
 
-static TOKENIZERS: Lazy<RModule> = Lazy::new(|ruby| ruby.class_object().const_get("Tokenizers").unwrap());
+static TOKENIZERS: Lazy<RModule> =
+    Lazy::new(|ruby| ruby.class_object().const_get("Tokenizers").unwrap());
 
-static DECODERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Decoders").unwrap());
+static DECODERS: Lazy<RModule> =
+    Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Decoders").unwrap());
 
-static MODELS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Models").unwrap());
+static MODELS: Lazy<RModule> =
+    Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Models").unwrap());
 
-static NORMALIZERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Normalizers").unwrap());
+static NORMALIZERS: Lazy<RModule> = Lazy::new(|ruby| {
+    ruby.get_inner(&TOKENIZERS)
+        .const_get("Normalizers")
+        .unwrap()
+});
 
-static PRE_TOKENIZERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("PreTokenizers").unwrap());
+static PRE_TOKENIZERS: Lazy<RModule> = Lazy::new(|ruby| {
+    ruby.get_inner(&TOKENIZERS)
+        .const_get("PreTokenizers")
+        .unwrap()
+});
 
-static PROCESSORS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Processors").unwrap());
+static PROCESSORS: Lazy<RModule> =
+    Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Processors").unwrap());
 
-static TRAINERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Trainers").unwrap());
+static TRAINERS: Lazy<RModule> =
+    Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Trainers").unwrap());
 
 #[magnus::init]
 fn init(ruby: &Ruby) -> RbResult<()> {
@@ -56,12 +69,15 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("_decode", method!(RbTokenizer::decode, 2))?;
     class.define_method("_decode_batch", method!(RbTokenizer::decode_batch, 2))?;
     class.define_method("model", method!(RbTokenizer::get_model, 0))?;
-    class.define_method("model=", method!(RbTokenizer::set_model,1))?;
+    class.define_method("model=", method!(RbTokenizer::set_model, 1))?;
     class.define_method("decoder", method!(RbTokenizer::get_decoder, 0))?;
     class.define_method("decoder=", method!(RbTokenizer::set_decoder, 1))?;
     class.define_method("pre_tokenizer", method!(RbTokenizer::get_pre_tokenizer, 0))?;
     class.define_method("pre_tokenizer=", method!(RbTokenizer::set_pre_tokenizer, 1))?;
-    class.define_method("post_processor", method!(RbTokenizer::get_post_processor, 0))?;
+    class.define_method(
+        "post_processor",
+        method!(RbTokenizer::get_post_processor, 0),
+    )?;
     class.define_method(
         "post_processor=",
         method!(RbTokenizer::set_post_processor, 1),
@@ -73,12 +89,22 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("_enable_padding", method!(RbTokenizer::enable_padding, 1))?;
     class.define_method("padding", method!(RbTokenizer::padding, 0))?;
     class.define_method("no_padding", method!(RbTokenizer::no_padding, 0))?;
-    class.define_method("_enable_truncation", method!(RbTokenizer::enable_truncation, 2))?;
+    class.define_method(
+        "_enable_truncation",
+        method!(RbTokenizer::enable_truncation, 2),
+    )?;
     class.define_method("truncation", method!(RbTokenizer::truncation, 0))?;
     class.define_method("no_truncation", method!(RbTokenizer::no_truncation, 0))?;
-    class.define_method("num_special_tokens_to_add", method!(RbTokenizer::num_special_tokens_to_add, 1))?;
+    class.define_method(
+        "num_special_tokens_to_add",
+        method!(RbTokenizer::num_special_tokens_to_add, 1),
+    )?;
     class.define_method("_vocab", method!(RbTokenizer::vocab, 1))?;
     class.define_method("_vocab_size", method!(RbTokenizer::vocab_size, 1))?;
+    class.define_method(
+        "added_tokens_decoder",
+        method!(RbTokenizer::get_added_tokens_decoder, 0),
+    )?;
     class.define_method("_to_s", method!(RbTokenizer::to_str, 1))?;
 
     let class = module.define_class("Encoding", ruby.class_object())?;
@@ -109,6 +135,15 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     let class = module.define_class("Regex", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbRegex::new, 1))?;
 
+    let class = module.define_class("AddedToken", ruby.class_object())?;
+    class.define_singleton_method("_new", function!(RbAddedToken::new, 2))?;
+    class.define_method("content", method!(RbAddedToken::get_content, 0))?;
+    class.define_method("rstrip", method!(RbAddedToken::get_rstrip, 0))?;
+    class.define_method("lstrip", method!(RbAddedToken::get_lstrip, 0))?;
+    class.define_method("single_word", method!(RbAddedToken::get_single_word, 0))?;
+    class.define_method("normalized", method!(RbAddedToken::get_normalized, 0))?;
+    class.define_method("special", method!(RbAddedToken::get_special, 0))?;
+
     let models = module.define_module("Models")?;
     let pre_tokenizers = module.define_module("PreTokenizers")?;
     let decoders = module.define_module("Decoders")?;

data/ext/tokenizers/src/models.rs

@@ -5,18 +5,19 @@ use std::sync::{Arc, RwLock};
 use crate::trainers::RbTrainer;
 use magnus::prelude::*;
 use magnus::{
-    data_type_builder, exception, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error, Module, Object,
-    RClass, RHash, RModule, Ruby, Symbol, TryConvert, TypedData, Value,
+    data_type_builder, exception, function, method, value::Lazy, Class, DataType,
+    DataTypeFunctions, Error, Module, Object, RClass, RHash, RModule, Ruby, Symbol, TryConvert,
+    TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
 use tk::models::bpe::{BpeBuilder, Merges, Vocab, BPE};
-use tk::models::ModelWrapper;
 use tk::models::unigram::Unigram;
 use tk::models::wordlevel::WordLevel;
 use tk::models::wordpiece::{WordPiece, WordPieceBuilder};
+use tk::models::ModelWrapper;
 use tk::{Model, Token};
 
-use super::{MODELS, RbError, RbResult};
+use super::{RbError, RbResult, MODELS};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbModel {
@@ -187,7 +188,12 @@ impl RbModel {
     }
 
     pub fn bpe_set_continuing_subword_prefix(&self, continuing_subword_prefix: Option<String>) {
-        setter!(self, BPE, continuing_subword_prefix, continuing_subword_prefix);
+        setter!(
+            self,
+            BPE,
+            continuing_subword_prefix,
+            continuing_subword_prefix
+        );
     }
 
     pub fn bpe_end_of_word_suffix(&self) -> Option<String> {
@@ -219,7 +225,12 @@ impl RbModel {
     }
 
     pub fn word_piece_set_continuing_subword_prefix(&self, continuing_subword_prefix: String) {
-        setter!(self, WordPiece, continuing_subword_prefix, continuing_subword_prefix);
+        setter!(
+            self,
+            WordPiece,
+            continuing_subword_prefix,
+            continuing_subword_prefix
+        );
     }
 
     pub fn word_piece_max_input_chars_per_word(&self) -> usize {
@@ -227,21 +238,34 @@ impl RbModel {
     }
 
     pub fn word_piece_set_max_input_chars_per_word(&self, max_input_chars_per_word: usize) {
-        setter!(self, WordPiece, max_input_chars_per_word, max_input_chars_per_word);
+        setter!(
+            self,
+            WordPiece,
+            max_input_chars_per_word,
+            max_input_chars_per_word
+        );
     }
 }
 
 pub struct RbUnigram {}
 
 impl RbUnigram {
-    fn new(vocab: Option<Vec<(String, f64)>>, unk_id: Option<usize>, byte_fallback: Option<bool>) -> RbResult<RbModel> {
+    fn new(
+        vocab: Option<Vec<(String, f64)>>,
+        unk_id: Option<usize>,
+        byte_fallback: Option<bool>,
+    ) -> RbResult<RbModel> {
         match (vocab, unk_id, byte_fallback) {
             (Some(vocab), unk_id, byte_fallback) => {
-                let model = Unigram::from(vocab, unk_id, byte_fallback.unwrap_or(false)).map_err(RbError::from)?;
+                let model = Unigram::from(vocab, unk_id, byte_fallback.unwrap_or(false))
+                    .map_err(RbError::from)?;
                 Ok(model.into())
             }
             (None, None, _) => Ok(Unigram::default().into()),
-            _ => Err(Error::new(exception::arg_error(), "`vocab` and `unk_id` must be both specified")),
+            _ => Err(Error::new(
+                exception::arg_error(),
+                "`vocab` and `unk_id` must be both specified",
+            )),
         }
     }
 }
@@ -249,7 +273,10 @@ impl RbUnigram {
 pub struct RbWordLevel {}
 
 impl RbWordLevel {
-    pub fn new(vocab: Option<HashMap<String, u32>>, unk_token: Option<String>) -> RbResult<RbModel> {
+    pub fn new(
+        vocab: Option<HashMap<String, u32>>,
+        unk_token: Option<String>,
+    ) -> RbResult<RbModel> {
         let mut builder = WordLevel::builder();
         if let Some(vocab) = vocab {
             builder = builder.vocab(vocab);
@@ -316,15 +343,16 @@ impl RbWordPiece {
 unsafe impl TypedData for RbModel {
     fn class(ruby: &Ruby) -> RClass {
         static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
-          let class: RClass = ruby.get_inner(&MODELS).const_get("Model").unwrap();
-          class.undef_default_alloc_func();
-          class
+            let class: RClass = ruby.get_inner(&MODELS).const_get("Model").unwrap();
+            class.undef_default_alloc_func();
+            class
         });
         ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        static DATA_TYPE: DataType = data_type_builder!(RbModel, "Tokenizers::Models::Model").build();
+        static DATA_TYPE: DataType =
+            data_type_builder!(RbModel, "Tokenizers::Models::Model").build();
         &DATA_TYPE
     }
 
@@ -368,10 +396,22 @@ pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_method("dropout=", method!(RbModel::bpe_set_dropout, 1))?;
     class.define_method("unk_token", method!(RbModel::bpe_unk_token, 0))?;
     class.define_method("unk_token=", method!(RbModel::bpe_set_unk_token, 1))?;
-    class.define_method("continuing_subword_prefix", method!(RbModel::bpe_continuing_subword_prefix, 0))?;
-    class.define_method("continuing_subword_prefix=", method!(RbModel::bpe_set_continuing_subword_prefix, 1))?;
-    class.define_method("end_of_word_suffix", method!(RbModel::bpe_end_of_word_suffix, 0))?;
-    class.define_method("end_of_word_suffix=", method!(RbModel::bpe_set_end_of_word_suffix, 1))?;
+    class.define_method(
+        "continuing_subword_prefix",
+        method!(RbModel::bpe_continuing_subword_prefix, 0),
+    )?;
+    class.define_method(
+        "continuing_subword_prefix=",
+        method!(RbModel::bpe_set_continuing_subword_prefix, 1),
+    )?;
+    class.define_method(
+        "end_of_word_suffix",
+        method!(RbModel::bpe_end_of_word_suffix, 0),
+    )?;
+    class.define_method(
+        "end_of_word_suffix=",
+        method!(RbModel::bpe_set_end_of_word_suffix, 1),
+    )?;
     class.define_method("fuse_unk", method!(RbModel::bpe_fuse_unk, 0))?;
     class.define_method("fuse_unk=", method!(RbModel::bpe_set_fuse_unk, 1))?;
     class.define_method("byte_fallback", method!(RbModel::bpe_byte_fallback, 0))?;
@@ -392,10 +432,22 @@ pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("_from_file", function!(RbWordPiece::from_file, 2))?;
     class.define_method("unk_token", method!(RbModel::word_piece_unk_token, 0))?;
     class.define_method("unk_token=", method!(RbModel::word_piece_set_unk_token, 1))?;
-    class.define_method("continuing_subword_prefix", method!(RbModel::word_piece_continuing_subword_prefix, 0))?;
-    class.define_method("continuing_subword_prefix=", method!(RbModel::word_piece_set_continuing_subword_prefix, 1))?;
-    class.define_method("max_input_chars_per_word", method!(RbModel::word_piece_max_input_chars_per_word, 0))?;
-    class.define_method("max_input_chars_per_word=", method!(RbModel::word_piece_set_max_input_chars_per_word, 1))?;
+    class.define_method(
+        "continuing_subword_prefix",
+        method!(RbModel::word_piece_continuing_subword_prefix, 0),
+    )?;
+    class.define_method(
+        "continuing_subword_prefix=",
+        method!(RbModel::word_piece_set_continuing_subword_prefix, 1),
+    )?;
+    class.define_method(
+        "max_input_chars_per_word",
+        method!(RbModel::word_piece_max_input_chars_per_word, 0),
+    )?;
+    class.define_method(
+        "max_input_chars_per_word=",
+        method!(RbModel::word_piece_set_max_input_chars_per_word, 1),
+    )?;
 
     Ok(())
 }

data/ext/tokenizers/src/normalizers.rs

@@ -1,19 +1,19 @@
 use std::sync::{Arc, RwLock};
 
 use magnus::{
-    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object, RArray, RClass, RModule,
-    Ruby, TryConvert, TypedData,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module,
+    Object, RArray, RClass, RModule, Ruby, TryConvert, TypedData,
 };
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Serialize, Serializer};
 use tk::normalizers::{
-    BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Replace, Prepend, Strip, StripAccents,
-    NFC, NFD, NFKC, NFKD,
+    BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Precompiled, Prepend, Replace, Strip,
+    StripAccents, NFC, NFD, NFKC, NFKD,
 };
 use tk::{NormalizedString, Normalizer};
 
 use super::utils::*;
-use super::{NORMALIZERS, RbError, RbResult};
+use super::{RbError, RbResult, NORMALIZERS};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
 pub struct RbNormalizer {
@@ -28,7 +28,9 @@ impl RbNormalizer {
 
     pub fn normalize_str(&self, sequence: String) -> RbResult<String> {
         let mut normalized = NormalizedString::from(sequence);
-        self.normalizer.normalize(&mut normalized).map_err(RbError::from)?;
+        self.normalizer
+            .normalize(&mut normalized)
+            .map_err(RbError::from)?;
         Ok(normalized.get().to_owned())
     }
 }
@@ -43,7 +45,8 @@ macro_rules! getter {
     ($self: ident, $variant: ident, $name: ident) => {{
         if let RbNormalizerTypeWrapper::Single(ref norm) = &$self.normalizer {
             let wrapper = norm.read().unwrap();
-            if let RbNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (*wrapper).clone() {
+            if let RbNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (*wrapper).clone()
+            {
                 o.$name
             } else {
                 unreachable!()
@@ -66,7 +69,6 @@ macro_rules! setter {
 }
 
 impl RbNormalizer {
-
     fn bert_clean_text(&self) -> bool {
         getter!(self, BertNormalizer, clean_text)
     }
@@ -101,7 +103,7 @@ impl RbNormalizer {
     }
 
     fn bert_set_lowercase(&self, lowercase: bool) {
-        setter!(self, BertNormalizer, lowercase, lowercase)
+        setter!(self, BertNormalizer, lowercase, lowercase);
     }
 
     fn prepend_prepend(&self) -> String {
@@ -109,7 +111,7 @@ impl RbNormalizer {
     }
 
     fn prepend_set_prepend(&self, prepend: String) {
-        setter!(self, Prepend, prepend, prepend)
+        setter!(self, Prepend, prepend, prepend);
     }
 
     fn strip_left(&self) -> bool {
@@ -117,7 +119,7 @@ impl RbNormalizer {
     }
 
     fn strip_set_left(&self, left: bool) {
-        setter!(self, StripNormalizer, strip_left, left)
+        setter!(self, StripNormalizer, strip_left, left);
     }
 
     fn strip_right(&self) -> bool {
@@ -125,14 +127,19 @@ impl RbNormalizer {
     }
 
     fn strip_set_right(&self, right: bool) {
-        setter!(self, StripNormalizer, strip_right, right)
+        setter!(self, StripNormalizer, strip_right, right);
     }
 }
 
 pub struct RbBertNormalizer {}
 
 impl RbBertNormalizer {
-    pub fn new(clean_text: bool, handle_chinese_chars: bool, strip_accents: Option<bool>, lowercase: bool) -> RbNormalizer {
+    pub fn new(
+        clean_text: bool,
+        handle_chinese_chars: bool,
+        strip_accents: Option<bool>,
+        lowercase: bool,
+    ) -> RbNormalizer {
         BertNormalizer::new(clean_text, handle_chinese_chars, strip_accents, lowercase).into()
     }
 }
@@ -185,11 +192,28 @@ impl RbNmt {
     }
 }
 
+pub struct RbPrecompiled {}
+
+impl RbPrecompiled {
+    pub fn new(precompiled_charsmap: Vec<u8>) -> RbResult<RbNormalizer> {
+        Precompiled::from(&precompiled_charsmap)
+            .map_err(|e| {
+                RbError::new_err(format!(
+                    "Error while attempting to build Precompiled normalizer: {}",
+                    e
+                ))
+            })
+            .map(|v| v.into())
+    }
+}
+
 pub struct RbReplace {}
 
 impl RbReplace {
     pub fn new(pattern: RbPattern, content: String) -> RbResult<RbNormalizer> {
-        Replace::new(pattern, content).map(|v| v.into()).map_err(RbError::from)
+        Replace::new(pattern, content)
+            .map(|v| v.into())
+            .map_err(RbError::from)
     }
 }
 
@@ -222,14 +246,16 @@ pub struct RbSequence {}
 impl RbSequence {
     fn new(normalizers: RArray) -> RbResult<RbNormalizer> {
         let mut sequence = Vec::with_capacity(normalizers.len());
-        for n in normalizers.into_iter() {
+        for n in normalizers {
             let normalizer: &RbNormalizer = TryConvert::try_convert(n)?;
             match &normalizer.normalizer {
                 RbNormalizerTypeWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
                 RbNormalizerTypeWrapper::Single(inner) => sequence.push(inner.clone()),
             }
         }
-        Ok(RbNormalizer::new(RbNormalizerTypeWrapper::Sequence(sequence)))
+        Ok(RbNormalizer::new(RbNormalizerTypeWrapper::Sequence(
+            sequence,
+        )))
     }
 }
 
@@ -328,7 +354,10 @@ impl Normalizer for RbNormalizerWrapper {
 unsafe impl TypedData for RbNormalizer {
     fn class(ruby: &Ruby) -> RClass {
         static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
-            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Normalizer").unwrap();
+            let class: RClass = ruby
+                .get_inner(&NORMALIZERS)
+                .const_get("Normalizer")
+                .unwrap();
             class.undef_default_alloc_func();
             class
         });
@@ -336,7 +365,8 @@ unsafe impl TypedData for RbNormalizer {
     }
 
     fn data_type() -> &'static DataType {
-        static DATA_TYPE: DataType = data_type_builder!(RbNormalizer, "Tokenizers::Normalizers::Normalizer").build();
+        static DATA_TYPE: DataType =
+            data_type_builder!(RbNormalizer, "Tokenizers::Normalizers::Normalizer").build();
         &DATA_TYPE
     }
 
@@ -347,7 +377,10 @@ unsafe impl TypedData for RbNormalizer {
             class
         });
         static BERT_NORMALIZER: Lazy<RClass> = Lazy::new(|ruby| {
-            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("BertNormalizer").unwrap();
+            let class: RClass = ruby
+                .get_inner(&NORMALIZERS)
+                .const_get("BertNormalizer")
+                .unwrap();
             class.undef_default_alloc_func();
             class
         });
@@ -381,6 +414,14 @@ unsafe impl TypedData for RbNormalizer {
             class.undef_default_alloc_func();
             class
         });
+        static PRECOMPILED: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby
+                .get_inner(&NORMALIZERS)
+                .const_get("Precompiled")
+                .unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         static REPLACE: Lazy<RClass> = Lazy::new(|ruby| {
             let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("Replace").unwrap();
             class.undef_default_alloc_func();
@@ -397,7 +438,10 @@ unsafe impl TypedData for RbNormalizer {
             class
         });
         static STRIP_ACCENTS: Lazy<RClass> = Lazy::new(|ruby| {
-            let class: RClass = ruby.get_inner(&NORMALIZERS).const_get("StripAccents").unwrap();
+            let class: RClass = ruby
+                .get_inner(&NORMALIZERS)
+                .const_get("StripAccents")
+                .unwrap();
             class.undef_default_alloc_func();
             class
         });
@@ -412,6 +456,7 @@ unsafe impl TypedData for RbNormalizer {
                     NormalizerWrapper::NFKC(_) => ruby.get_inner(&NFKC),
                     NormalizerWrapper::NFKD(_) => ruby.get_inner(&NFKD),
                     NormalizerWrapper::Nmt(_) => ruby.get_inner(&NMT),
+                    NormalizerWrapper::Precompiled(_) => ruby.get_inner(&PRECOMPILED),
                     NormalizerWrapper::Replace(_) => ruby.get_inner(&REPLACE),
                     NormalizerWrapper::Prepend(_) => ruby.get_inner(&PREPEND),
                     NormalizerWrapper::StripNormalizer(_) => ruby.get_inner(&STRIP),
@@ -434,10 +479,22 @@ pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("_new", function!(RbBertNormalizer::new, 4))?;
     class.define_method("clean_text", method!(RbNormalizer::bert_clean_text, 0))?;
     class.define_method("clean_text=", method!(RbNormalizer::bert_set_clean_text, 1))?;
-    class.define_method("handle_chinese_chars", method!(RbNormalizer::bert_handle_chinese_chars, 0))?;
-    class.define_method("handle_chinese_chars=", method!(RbNormalizer::bert_set_handle_chinese_chars, 1))?;
-    class.define_method("strip_accents", method!(RbNormalizer::bert_strip_accents, 0))?;
-    class.define_method("strip_accents=", method!(RbNormalizer::bert_set_strip_accents, 1))?;
+    class.define_method(
+        "handle_chinese_chars",
+        method!(RbNormalizer::bert_handle_chinese_chars, 0),
+    )?;
+    class.define_method(
+        "handle_chinese_chars=",
+        method!(RbNormalizer::bert_set_handle_chinese_chars, 1),
+    )?;
+    class.define_method(
+        "strip_accents",
+        method!(RbNormalizer::bert_strip_accents, 0),
+    )?;
+    class.define_method(
+        "strip_accents=",
+        method!(RbNormalizer::bert_set_strip_accents, 1),
+    )?;
     class.define_method("lowercase", method!(RbNormalizer::bert_lowercase, 0))?;
     class.define_method("lowercase=", method!(RbNormalizer::bert_set_lowercase, 1))?;
 
@@ -459,6 +516,9 @@ pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     let class = module.define_class("Nmt", normalizer)?;
     class.define_singleton_method("new", function!(RbNmt::new, 0))?;
 
+    let class = module.define_class("Precompiled", normalizer)?;
+    class.define_singleton_method("new", function!(RbPrecompiled::new, 1))?;
+
     let class = module.define_class("Replace", normalizer)?;
     class.define_singleton_method("new", function!(RbReplace::new, 2))?;