tokenizers 0.6.3 → 0.7.0

data/ext/tokenizers/src/models.rs

@@ -20,8 +20,8 @@ use tk::{Model, Token};
 use super::{RbError, RbResult, MODELS};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
+#[serde(transparent)]
 pub struct RbModel {
-    #[serde(flatten)]
     pub model: Arc<RwLock<ModelWrapper>>,
 }
 
@@ -158,7 +158,7 @@ macro_rules! setter {
 }
 
 impl RbModel {
-    pub fn bpe_dropout(&self) -> Option<f32> {
+    pub fn bpe_get_dropout(&self) -> Option<f32> {
         getter!(self, BPE, dropout)
     }
 
@@ -166,7 +166,7 @@ impl RbModel {
         setter!(self, BPE, dropout, dropout);
     }
 
-    pub fn bpe_unk_token(&self) -> Option<String> {
+    pub fn bpe_get_unk_token(&self) -> Option<String> {
         getter!(self, BPE, unk_token.clone())
     }
 
@@ -174,7 +174,7 @@ impl RbModel {
         setter!(self, BPE, unk_token, unk_token);
     }
 
-    pub fn bpe_fuse_unk(&self) -> bool {
+    pub fn bpe_get_fuse_unk(&self) -> bool {
         getter!(self, BPE, fuse_unk)
     }
 
@@ -182,7 +182,7 @@ impl RbModel {
         setter!(self, BPE, fuse_unk, fuse_unk);
     }
 
-    pub fn bpe_byte_fallback(&self) -> bool {
+    pub fn bpe_get_byte_fallback(&self) -> bool {
         getter!(self, BPE, byte_fallback)
     }
 
@@ -190,7 +190,7 @@ impl RbModel {
         setter!(self, BPE, byte_fallback, byte_fallback);
     }
 
-    pub fn bpe_continuing_subword_prefix(&self) -> Option<String> {
+    pub fn bpe_get_continuing_subword_prefix(&self) -> Option<String> {
         getter!(self, BPE, continuing_subword_prefix.clone())
     }
 
@@ -203,7 +203,7 @@ impl RbModel {
         );
     }
 
-    pub fn bpe_end_of_word_suffix(&self) -> Option<String> {
+    pub fn bpe_get_end_of_word_suffix(&self) -> Option<String> {
         getter!(self, BPE, end_of_word_suffix.clone())
     }
 
@@ -211,7 +211,7 @@ impl RbModel {
         setter!(self, BPE, end_of_word_suffix, end_of_word_suffix);
     }
 
-    pub fn word_level_unk_token(&self) -> String {
+    pub fn word_level_get_unk_token(&self) -> String {
         getter!(self, WordLevel, unk_token.clone())
     }
 
@@ -219,7 +219,7 @@ impl RbModel {
         setter!(self, WordLevel, unk_token, unk_token);
     }
 
-    pub fn word_piece_unk_token(&self) -> String {
+    pub fn word_piece_get_unk_token(&self) -> String {
         getter!(self, WordPiece, unk_token.clone())
     }
 
@@ -227,7 +227,7 @@ impl RbModel {
         setter!(self, WordPiece, unk_token, unk_token);
     }
 
-    pub fn word_piece_continuing_subword_prefix(&self) -> String {
+    pub fn word_piece_get_continuing_subword_prefix(&self) -> String {
         getter!(self, WordPiece, continuing_subword_prefix.clone())
     }
 
@@ -240,7 +240,7 @@ impl RbModel {
         );
     }
 
-    pub fn word_piece_max_input_chars_per_word(&self) -> usize {
+    pub fn word_piece_get_max_input_chars_per_word(&self) -> usize {
         getter!(self, WordPiece, max_input_chars_per_word.clone())
     }
 
@@ -405,13 +405,13 @@ pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     let class = module.define_class("BPE", model)?;
     class.define_singleton_method("_new", function!(RbBPE::new, 3))?;
     class.define_singleton_method("_from_file", function!(RbBPE::from_file, 3))?;
-    class.define_method("dropout", method!(RbModel::bpe_dropout, 0))?;
+    class.define_method("dropout", method!(RbModel::bpe_get_dropout, 0))?;
     class.define_method("dropout=", method!(RbModel::bpe_set_dropout, 1))?;
-    class.define_method("unk_token", method!(RbModel::bpe_unk_token, 0))?;
+    class.define_method("unk_token", method!(RbModel::bpe_get_unk_token, 0))?;
     class.define_method("unk_token=", method!(RbModel::bpe_set_unk_token, 1))?;
     class.define_method(
         "continuing_subword_prefix",
-        method!(RbModel::bpe_continuing_subword_prefix, 0),
+        method!(RbModel::bpe_get_continuing_subword_prefix, 0),
     )?;
     class.define_method(
         "continuing_subword_prefix=",
@@ -419,15 +419,15 @@ pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     )?;
     class.define_method(
         "end_of_word_suffix",
-        method!(RbModel::bpe_end_of_word_suffix, 0),
+        method!(RbModel::bpe_get_end_of_word_suffix, 0),
     )?;
     class.define_method(
         "end_of_word_suffix=",
         method!(RbModel::bpe_set_end_of_word_suffix, 1),
     )?;
-    class.define_method("fuse_unk", method!(RbModel::bpe_fuse_unk, 0))?;
+    class.define_method("fuse_unk", method!(RbModel::bpe_get_fuse_unk, 0))?;
     class.define_method("fuse_unk=", method!(RbModel::bpe_set_fuse_unk, 1))?;
-    class.define_method("byte_fallback", method!(RbModel::bpe_byte_fallback, 0))?;
+    class.define_method("byte_fallback", method!(RbModel::bpe_get_byte_fallback, 0))?;
     class.define_method("byte_fallback=", method!(RbModel::bpe_set_byte_fallback, 1))?;
 
     let class = module.define_class("Unigram", model)?;
@@ -437,17 +437,17 @@ pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("_new", function!(RbWordLevel::new, 2))?;
     class.define_singleton_method("_from_file", function!(RbWordLevel::from_file, 2))?;
     class.define_singleton_method("read_file", function!(RbWordLevel::read_file, 1))?;
-    class.define_method("unk_token", method!(RbModel::word_level_unk_token, 0))?;
+    class.define_method("unk_token", method!(RbModel::word_level_get_unk_token, 0))?;
     class.define_method("unk_token=", method!(RbModel::word_level_set_unk_token, 1))?;
 
     let class = module.define_class("WordPiece", model)?;
     class.define_singleton_method("_new", function!(RbWordPiece::new, 2))?;
     class.define_singleton_method("_from_file", function!(RbWordPiece::from_file, 2))?;
-    class.define_method("unk_token", method!(RbModel::word_piece_unk_token, 0))?;
+    class.define_method("unk_token", method!(RbModel::word_piece_get_unk_token, 0))?;
     class.define_method("unk_token=", method!(RbModel::word_piece_set_unk_token, 1))?;
     class.define_method(
         "continuing_subword_prefix",
-        method!(RbModel::word_piece_continuing_subword_prefix, 0),
+        method!(RbModel::word_piece_get_continuing_subword_prefix, 0),
     )?;
     class.define_method(
         "continuing_subword_prefix=",
@@ -455,7 +455,7 @@ pub fn init_models(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     )?;
     class.define_method(
         "max_input_chars_per_word",
-        method!(RbModel::word_piece_max_input_chars_per_word, 0),
+        method!(RbModel::word_piece_get_max_input_chars_per_word, 0),
     )?;
     class.define_method(
         "max_input_chars_per_word=",

data/ext/tokenizers/src/normalizers.rs

@@ -16,8 +16,8 @@ use super::utils::*;
 use super::{RbError, RbResult, NORMALIZERS};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
+#[serde(transparent)]
 pub struct RbNormalizer {
-    #[serde(flatten)]
     pub(crate) normalizer: RbNormalizerTypeWrapper,
 }
 
@@ -69,7 +69,7 @@ macro_rules! setter {
 }
 
 impl RbNormalizer {
-    fn bert_clean_text(&self) -> bool {
+    fn bert_get_clean_text(&self) -> bool {
         getter!(self, BertNormalizer, clean_text)
     }
 
@@ -77,7 +77,7 @@ impl RbNormalizer {
         setter!(self, BertNormalizer, clean_text, clean_text);
     }
 
-    fn bert_handle_chinese_chars(&self) -> bool {
+    fn bert_get_handle_chinese_chars(&self) -> bool {
         getter!(self, BertNormalizer, handle_chinese_chars)
     }
 
@@ -90,7 +90,7 @@ impl RbNormalizer {
         );
     }
 
-    fn bert_strip_accents(&self) -> Option<bool> {
+    fn bert_get_strip_accents(&self) -> Option<bool> {
         getter!(self, BertNormalizer, strip_accents)
     }
 
@@ -98,7 +98,7 @@ impl RbNormalizer {
         setter!(self, BertNormalizer, strip_accents, strip_accents);
     }
 
-    fn bert_lowercase(&self) -> bool {
+    fn bert_get_lowercase(&self) -> bool {
         getter!(self, BertNormalizer, lowercase)
     }
 
@@ -106,7 +106,7 @@ impl RbNormalizer {
         setter!(self, BertNormalizer, lowercase, lowercase);
     }
 
-    fn prepend_prepend(&self) -> String {
+    fn prepend_get_prepend(&self) -> String {
         getter!(self, Prepend, prepend)
     }
 
@@ -114,7 +114,7 @@ impl RbNormalizer {
         setter!(self, Prepend, prepend, prepend);
     }
 
-    fn strip_left(&self) -> bool {
+    fn strip_get_left(&self) -> bool {
         getter!(self, StripNormalizer, strip_left)
     }
 
@@ -122,7 +122,7 @@ impl RbNormalizer {
         setter!(self, StripNormalizer, strip_left, left);
     }
 
-    fn strip_right(&self) -> bool {
+    fn strip_get_right(&self) -> bool {
         getter!(self, StripNormalizer, strip_right)
     }
 
@@ -476,11 +476,11 @@ pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
 
     let class = module.define_class("BertNormalizer", normalizer)?;
     class.define_singleton_method("_new", function!(RbBertNormalizer::new, 4))?;
-    class.define_method("clean_text", method!(RbNormalizer::bert_clean_text, 0))?;
+    class.define_method("clean_text", method!(RbNormalizer::bert_get_clean_text, 0))?;
     class.define_method("clean_text=", method!(RbNormalizer::bert_set_clean_text, 1))?;
     class.define_method(
         "handle_chinese_chars",
-        method!(RbNormalizer::bert_handle_chinese_chars, 0),
+        method!(RbNormalizer::bert_get_handle_chinese_chars, 0),
     )?;
     class.define_method(
         "handle_chinese_chars=",
@@ -488,13 +488,13 @@ pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     )?;
     class.define_method(
         "strip_accents",
-        method!(RbNormalizer::bert_strip_accents, 0),
+        method!(RbNormalizer::bert_get_strip_accents, 0),
     )?;
     class.define_method(
         "strip_accents=",
         method!(RbNormalizer::bert_set_strip_accents, 1),
     )?;
-    class.define_method("lowercase", method!(RbNormalizer::bert_lowercase, 0))?;
+    class.define_method("lowercase", method!(RbNormalizer::bert_get_lowercase, 0))?;
     class.define_method("lowercase=", method!(RbNormalizer::bert_set_lowercase, 1))?;
 
     let class = module.define_class("Lowercase", normalizer)?;
@@ -523,14 +523,14 @@ pub fn init_normalizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
 
     let class = module.define_class("Prepend", normalizer)?;
     class.define_singleton_method("_new", function!(RbPrepend::new, 1))?;
-    class.define_method("prepend", method!(RbNormalizer::prepend_prepend, 0))?;
+    class.define_method("prepend", method!(RbNormalizer::prepend_get_prepend, 0))?;
     class.define_method("prepend=", method!(RbNormalizer::prepend_set_prepend, 1))?;
 
     let class = module.define_class("Strip", normalizer)?;
     class.define_singleton_method("_new", function!(RbStrip::new, 2))?;
-    class.define_method("left", method!(RbNormalizer::strip_left, 0))?;
+    class.define_method("left", method!(RbNormalizer::strip_get_left, 0))?;
     class.define_method("left=", method!(RbNormalizer::strip_set_left, 1))?;
-    class.define_method("right", method!(RbNormalizer::strip_right, 0))?;
+    class.define_method("right", method!(RbNormalizer::strip_get_right, 0))?;
     class.define_method("right=", method!(RbNormalizer::strip_set_right, 1))?;
 
     let class = module.define_class("StripAccents", normalizer)?;

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -25,8 +25,8 @@ use super::utils::*;
 use super::{RbError, RbResult, PRE_TOKENIZERS};
 
 #[derive(DataTypeFunctions, Clone, Serialize, Deserialize)]
+#[serde(transparent)]
 pub struct RbPreTokenizer {
-    #[serde(flatten)]
     pub(crate) pretok: RbPreTokenizerTypeWrapper,
 }
 
@@ -88,7 +88,7 @@ impl RbPreTokenizer {
         RbPreTokenizer { pretok }
     }
 
-    fn byte_level_add_prefix_space(&self) -> bool {
+    fn byte_level_get_add_prefix_space(&self) -> bool {
         getter!(self, ByteLevel, add_prefix_space)
     }
 
@@ -96,7 +96,7 @@ impl RbPreTokenizer {
         setter!(self, ByteLevel, add_prefix_space, add_prefix_space);
     }
 
-    fn byte_level_use_regex(&self) -> bool {
+    fn byte_level_get_use_regex(&self) -> bool {
         getter!(self, ByteLevel, use_regex)
     }
 
@@ -104,7 +104,7 @@ impl RbPreTokenizer {
         setter!(self, ByteLevel, use_regex, use_regex);
     }
 
-    fn char_delimiter_split_delimiter(&self) -> String {
+    fn char_delimiter_split_get_delimiter(&self) -> String {
         getter!(self, Delimiter, delimiter.to_string())
     }
 
@@ -112,7 +112,7 @@ impl RbPreTokenizer {
         setter!(self, Delimiter, delimiter, delimiter);
     }
 
-    fn digits_individual_digits(&self) -> bool {
+    fn digits_get_individual_digits(&self) -> bool {
         getter!(self, Digits, individual_digits)
     }
 
@@ -120,7 +120,7 @@ impl RbPreTokenizer {
         setter!(self, Digits, individual_digits, individual_digits);
     }
 
-    fn metaspace_replacement(&self) -> String {
+    fn metaspace_get_replacement(&self) -> String {
         getter!(self, Metaspace, get_replacement().to_string())
     }
 
@@ -128,7 +128,7 @@ impl RbPreTokenizer {
         setter!(self, Metaspace, @set_replacement, replacement);
     }
 
-    fn metaspace_split(&self) -> bool {
+    fn metaspace_get_split(&self) -> bool {
         getter!(self, Metaspace, get_split())
     }
 
@@ -136,7 +136,7 @@ impl RbPreTokenizer {
         setter!(self, Metaspace, @set_split, split);
     }
 
-    fn metaspace_prepend_scheme(&self) -> String {
+    fn metaspace_get_prepend_scheme(&self) -> String {
         // Assuming Metaspace has a method to get the prepend_scheme as a string
         let scheme: PrependScheme = getter!(self, Metaspace, get_prepend_scheme());
         match scheme {
@@ -528,7 +528,7 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("alphabet", function!(RbByteLevel::alphabet, 0))?;
     class.define_method(
         "add_prefix_space",
-        method!(RbPreTokenizer::byte_level_add_prefix_space, 0),
+        method!(RbPreTokenizer::byte_level_get_add_prefix_space, 0),
     )?;
     class.define_method(
         "add_prefix_space=",
@@ -536,7 +536,7 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     )?;
     class.define_method(
         "use_regex",
-        method!(RbPreTokenizer::byte_level_use_regex, 0),
+        method!(RbPreTokenizer::byte_level_get_use_regex, 0),
     )?;
     class.define_method(
         "use_regex=",
@@ -547,7 +547,7 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("new", function!(RbCharDelimiterSplit::new, 1))?;
     class.define_method(
         "delimiter",
-        method!(RbPreTokenizer::char_delimiter_split_delimiter, 0),
+        method!(RbPreTokenizer::char_delimiter_split_get_delimiter, 0),
     )?;
     class.define_method(
         "delimiter=",
@@ -558,7 +558,7 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("_new", function!(RbDigits::new, 1))?;
     class.define_method(
         "individual_digits",
-        method!(RbPreTokenizer::digits_individual_digits, 0),
+        method!(RbPreTokenizer::digits_get_individual_digits, 0),
     )?;
     class.define_method(
         "individual_digits=",
@@ -569,7 +569,7 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("_new", function!(RbMetaspace::new, 3))?;
     class.define_method(
         "prepend_scheme",
-        method!(RbPreTokenizer::metaspace_prepend_scheme, 0),
+        method!(RbPreTokenizer::metaspace_get_prepend_scheme, 0),
     )?;
     class.define_method(
         "prepend_scheme=",
@@ -577,13 +577,13 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     )?;
     class.define_method(
         "replacement",
-        method!(RbPreTokenizer::metaspace_replacement, 0),
+        method!(RbPreTokenizer::metaspace_get_replacement, 0),
     )?;
     class.define_method(
         "replacement=",
         method!(RbPreTokenizer::metaspace_set_replacement, 1),
     )?;
-    class.define_method("split", method!(RbPreTokenizer::metaspace_split, 0))?;
+    class.define_method("split", method!(RbPreTokenizer::metaspace_get_split, 0))?;
     class.define_method("split=", method!(RbPreTokenizer::metaspace_set_split, 1))?;
 
     let class = module.define_class("Punctuation", pre_tokenizer)?;

data/ext/tokenizers/src/processors.rs

@@ -1,10 +1,12 @@
 use std::sync::Arc;
+use std::sync::RwLock;
 
 use magnus::{
     data_type_builder, function, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object,
-    RClass, RModule, Ruby, TryConvert, TypedData, Value,
+    RArray, RClass, RModule, Ruby, TryConvert, TypedData, Value,
 };
-use serde::{Deserialize, Serialize};
+use serde::ser::SerializeStruct;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use tk::processors::bert::BertProcessing;
 use tk::processors::byte_level::ByteLevel;
 use tk::processors::roberta::RobertaProcessing;
@@ -15,17 +17,28 @@ use tk::{Encoding, PostProcessor};
 use super::{RbResult, PROCESSORS};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
+#[serde(transparent)]
 pub struct RbPostProcessor {
-    #[serde(flatten)]
-    pub processor: Arc<PostProcessorWrapper>,
+    pub processor: RbPostProcessorTypeWrapper,
 }
 
 impl RbPostProcessor {
-    pub fn new(processor: Arc<PostProcessorWrapper>) -> Self {
+    pub fn new(processor: RbPostProcessorTypeWrapper) -> Self {
         RbPostProcessor { processor }
     }
 }
 
+impl<I> From<I> for RbPostProcessor
+where
+    I: Into<PostProcessorWrapper>,
+{
+    fn from(processor: I) -> Self {
+        RbPostProcessor {
+            processor: processor.into().into(),
+        }
+    }
+}
+
 impl PostProcessor for RbPostProcessor {
     fn added_tokens(&self, is_pair: bool) -> usize {
         self.processor.added_tokens(is_pair)
@@ -41,6 +54,92 @@ impl PostProcessor for RbPostProcessor {
     }
 }
 
+#[derive(Clone)]
+pub(crate) enum RbPostProcessorTypeWrapper {
+    Sequence(Vec<Arc<RwLock<PostProcessorWrapper>>>),
+    Single(Arc<RwLock<PostProcessorWrapper>>),
+}
+
+impl PostProcessor for RbPostProcessorTypeWrapper {
+    fn added_tokens(&self, is_pair: bool) -> usize {
+        match self {
+            RbPostProcessorTypeWrapper::Single(inner) => inner
+                .read()
+                .expect("RwLock synchronisation primitive is poisoned, cannot get subtype of RbPostProcessor")
+                .added_tokens(is_pair),
+            RbPostProcessorTypeWrapper::Sequence(inner) => inner.iter().map(|p| {
+                p.read()
+                    .expect("RwLock synchronisation primitive is poisoned, cannot get subtype of RbPostProcessor")
+                    .added_tokens(is_pair)
+            }).sum::<usize>(),
+        }
+    }
+
+    fn process_encodings(
+        &self,
+        mut encodings: Vec<Encoding>,
+        add_special_tokens: bool,
+    ) -> tk::Result<Vec<Encoding>> {
+        match self {
+            RbPostProcessorTypeWrapper::Single(inner) => inner
+                .read()
+                .expect("RwLock synchronisation primitive is poisoned, cannot get subtype of RbPreTokenizer")
+                .process_encodings(encodings, add_special_tokens),
+            RbPostProcessorTypeWrapper::Sequence(inner) => {
+                for processor in inner.iter() {
+                    encodings = processor
+                        .read()
+                        .expect("RwLock synchronisation primitive is poisoned, cannot get subtype of RbPreTokenizer")
+                        .process_encodings(encodings, add_special_tokens)?;
+                }
+                Ok(encodings)
+            },
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for RbPostProcessorTypeWrapper {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let wrapper = PostProcessorWrapper::deserialize(deserializer)?;
+        Ok(wrapper.into())
+    }
+}
+
+impl Serialize for RbPostProcessorTypeWrapper {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        match self {
+            RbPostProcessorTypeWrapper::Sequence(seq) => {
+                let mut ser = serializer.serialize_struct("Sequence", 2)?;
+                ser.serialize_field("type", "Sequence")?;
+                ser.serialize_field("processors", seq)?;
+                ser.end()
+            }
+            RbPostProcessorTypeWrapper::Single(inner) => inner.serialize(serializer),
+        }
+    }
+}
+
+impl<I> From<I> for RbPostProcessorTypeWrapper
+where
+    I: Into<PostProcessorWrapper>,
+{
+    fn from(processor: I) -> Self {
+        let processor = processor.into();
+        match processor {
+            PostProcessorWrapper::Sequence(seq) => RbPostProcessorTypeWrapper::Sequence(
+                seq.into_iter().map(|p| Arc::new(RwLock::new(p))).collect(),
+            ),
+            _ => RbPostProcessorTypeWrapper::Single(Arc::new(RwLock::new(processor.clone()))),
+        }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct RbSpecialToken(SpecialToken);
 
@@ -91,7 +190,7 @@ pub struct RbBertProcessing {}
 
 impl RbBertProcessing {
     pub fn new(sep: (String, u32), cls: (String, u32)) -> RbPostProcessor {
-        RbPostProcessor::new(Arc::new(BertProcessing::new(sep, cls).into()))
+        BertProcessing::new(sep, cls).into()
     }
 }
 
@@ -104,7 +203,7 @@ impl RbByteLevel {
         if let Some(to) = trim_offsets {
             byte_level = byte_level.trim_offsets(to);
         }
-        RbPostProcessor::new(Arc::new(byte_level.into()))
+        byte_level.into()
     }
 }
 
@@ -120,7 +219,7 @@ impl RbRobertaProcessing {
         let proc = RobertaProcessing::new(sep, cls)
             .trim_offsets(trim_offsets)
             .add_prefix_space(add_prefix_space);
-        RbPostProcessor::new(Arc::new(proc.into()))
+        proc.into()
     }
 }
 
@@ -145,7 +244,27 @@ impl RbTemplateProcessing {
         }
         let processor = builder.build().unwrap(); //.map_err(RbError::from)?;
 
-        Ok(RbPostProcessor::new(Arc::new(processor.into())))
+        Ok(processor.into())
+    }
+}
+
+pub struct RbSequence {}
+
+impl RbSequence {
+    fn new(processors_rb: RArray) -> RbResult<RbPostProcessor> {
+        let mut processors = Vec::with_capacity(processors_rb.len());
+        for n in processors_rb {
+            let processor = <&RbPostProcessor>::try_convert(n)?;
+            match &processor.processor {
+                RbPostProcessorTypeWrapper::Sequence(inner) => {
+                    processors.extend(inner.iter().cloned())
+                }
+                RbPostProcessorTypeWrapper::Single(inner) => processors.push(inner.clone()),
+            }
+        }
+        Ok(RbPostProcessor::new(RbPostProcessorTypeWrapper::Sequence(
+            processors,
+        )))
     }
 }
 
@@ -198,12 +317,20 @@ unsafe impl TypedData for RbPostProcessor {
             class.undef_default_alloc_func();
             class
         });
-        match *value.processor {
-            PostProcessorWrapper::Bert(_) => ruby.get_inner(&BERT_PROCESSING),
-            PostProcessorWrapper::ByteLevel(_) => ruby.get_inner(&BYTE_LEVEL),
-            PostProcessorWrapper::Roberta(_) => ruby.get_inner(&ROBERTA_PROCESSING),
-            PostProcessorWrapper::Template(_) => ruby.get_inner(&TEMPLATE_PROCESSING),
-            _ => todo!(),
+        static SEQUENCE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&PROCESSORS).const_get("Sequence").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        match &value.processor {
+            RbPostProcessorTypeWrapper::Single(inner) => match &*inner.read().unwrap() {
+                PostProcessorWrapper::Bert(_) => ruby.get_inner(&BERT_PROCESSING),
+                PostProcessorWrapper::ByteLevel(_) => ruby.get_inner(&BYTE_LEVEL),
+                PostProcessorWrapper::Roberta(_) => ruby.get_inner(&ROBERTA_PROCESSING),
+                PostProcessorWrapper::Template(_) => ruby.get_inner(&TEMPLATE_PROCESSING),
+                _ => todo!(),
+            },
+            RbPostProcessorTypeWrapper::Sequence(_) => ruby.get_inner(&SEQUENCE),
         }
     }
 }
@@ -223,5 +350,8 @@ pub fn init_processors(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     let class = module.define_class("TemplateProcessing", post_processor)?;
     class.define_singleton_method("_new", function!(RbTemplateProcessing::new, 3))?;
 
+    let class = module.define_class("Sequence", post_processor)?;
+    class.define_singleton_method("_new", function!(RbSequence::new, 1))?;
+
     Ok(())
 }

data/ext/tokenizers/src/ruby.rs

@@ -0,0 +1,51 @@
+use std::ffi::c_void;
+use std::ptr::null_mut;
+
+use magnus::Ruby;
+use rb_sys::rb_thread_call_without_gvl;
+
+pub trait GvlExt {
+    fn detach<T, F>(&self, func: F) -> T
+    where
+        F: Send + FnOnce() -> T,
+        T: Send;
+}
+
+impl GvlExt for Ruby {
+    fn detach<T, F>(&self, func: F) -> T
+    where
+        F: Send + FnOnce() -> T,
+        T: Send,
+    {
+        let mut data = CallbackData {
+            func: Some(func),
+            result: None,
+        };
+
+        unsafe {
+            rb_thread_call_without_gvl(
+                Some(call_without_gvl::<F, T>),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                null_mut(),
+            );
+        }
+
+        data.result.unwrap()
+    }
+}
+
+struct CallbackData<F, T> {
+    func: Option<F>,
+    result: Option<T>,
+}
+
+extern "C" fn call_without_gvl<F, T>(data: *mut c_void) -> *mut c_void
+where
+    F: FnOnce() -> T,
+{
+    let data = unsafe { &mut *(data as *mut CallbackData<F, T>) };
+    let func = data.func.take().unwrap();
+    data.result = Some(func());
+    null_mut()
+}