tokenizers 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91014464d4ecdb18ad06efe86ddea530b6563b29a06a04524840655fc9a18489
-  data.tar.gz: 8f4fa939f8591779d78f35ec44348b6e6701dbf43cd7eaa34247c09711a72772
+  metadata.gz: e11d6d7b7b6adb870221d30c3f11e3dee86a57e1334cd2a419a38959a6523712
+  data.tar.gz: 04a6be127e354dcb9f8f4f0656c242a2e8df12ce91b5614379404d6e35bb219f
 SHA512:
-  metadata.gz: e1251fbc94e365106d3e07cfbb5c92ab5c769da4d778af6c1fb1723b9deb138525e703e6d018923ae1d62dd6074422f847ced25fa3ff3ddb4f045ae1f75aae05
-  data.tar.gz: 4b433e560e22af6095d7d50ff3026a6d43ec114caaf5a6e6f7ee697c32fab1790c36d82f2d3f8bff69c4326238e706af2d17c4f9be95ddd11478af339e750f58
+  metadata.gz: 364207bd71c3aa9fe2760d4ec7ae58666274e7ba7fe0e753b33316b1b61bb411e2592a8cf2c8dff1ae37a3082607d2d07259375a50ff345769274f8aeedd89c1
+  data.tar.gz: 6a6a572a5925f3d140dcbbd93c23bae774d28898921b743dbe1626b02ebff046f54ba5231886e78d5b2ea51ca5c235582c1a440be2a89f35e13584bdbcf186d0

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.7.0 (2026-04-27)
+
+- Updated Tokenizers to 0.23.1
+- Added support for releasing GVL
+- Added `encode_batch_fast` method to `Tokenizer`
+- Dropped support for Ruby < 3.3
+
 ## 0.6.4 (2026-04-09)
 
 - Fixed caching

data/Cargo.lock

@@ -33,16 +33,14 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "bindgen"
-version = "0.69.5"
+version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
  "bitflags",
  "cexpr",
  "clang-sys",
  "itertools 0.12.1",
- "lazy_static",
- "lazycell",
  "proc-macro2",
  "quote",
  "regex",
@@ -160,6 +158,12 @@ version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
+[[package]]
+name = "daachorse"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2"
+
 [[package]]
 name = "darling"
 version = "0.20.11"
@@ -339,12 +343,6 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.172"
@@ -530,9 +528,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
 [[package]]
 name = "rand"
-version = "0.9.1"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
 dependencies = [
  "rand_chacha",
  "rand_core",
@@ -590,18 +588,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.124"
+version = "0.9.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85c4188462601e2aa1469def389c17228566f82ea72f137ed096f21591bc489"
+checksum = "d7d7c9560fe42dcffa576941394075f18a17dce89fcf718a2fa90b7dc2134d12"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.124"
+version = "0.9.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "568068db4102230882e6d4ae8de6632e224ca75fe5970f6e026a04e91ed635d3"
+checksum = "f1688e8f32967ba48c89e4dfa283b57f901075f542fc7ee9c3d7c5f9091ca1d9"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -649,9 +647,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "rustc-hash"
-version = "1.1.0"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
 
 [[package]]
 name = "rustversion"
@@ -778,13 +776,13 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.22.2"
+version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
+checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112"
 dependencies = [
  "ahash",
- "aho-corasick",
  "compact_str",
+ "daachorse",
  "dary_heap",
  "derive_builder",
  "esaxx-rs",
@@ -812,11 +810,12 @@ dependencies = [
 
 [[package]]
 name = "tokenizers-ruby"
-version = "0.6.4"
+version = "0.7.0"
 dependencies = [
  "ahash",
  "magnus",
  "onig",
+ "rb-sys",
  "serde",
  "tokenizers",
 ]

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers-ruby"
-version = "0.6.4"
+version = "0.7.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -15,9 +15,10 @@ crate-type = ["cdylib"]
 ahash = { version = "0.8.11", features = ["serde"] }
 magnus = "0.8"
 onig = { version = "6", default-features = false }
+rb-sys = "0.9"
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.22.2" # also update in from_pretrained.rb
+version = "=0.23.1" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/decoders.rs

@@ -23,8 +23,8 @@ use super::utils::*;
 use super::{RbError, RbResult, DECODERS};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
+#[serde(transparent)]
 pub struct RbDecoder {
-    #[serde(flatten)]
     pub(crate) decoder: RbDecoderWrapper,
 }
 
@@ -69,7 +69,7 @@ macro_rules! setter {
     }};
 }
 impl RbDecoder {
-    pub fn bpe_suffix(&self) -> String {
+    pub fn bpe_get_suffix(&self) -> String {
         getter!(self, BPE, suffix.clone())
     }
 
@@ -77,7 +77,7 @@ impl RbDecoder {
         setter!(self, BPE, suffix, suffix);
     }
 
-    pub fn ctc_cleanup(&self) -> bool {
+    pub fn ctc_get_cleanup(&self) -> bool {
         getter!(self, CTC, cleanup)
     }
 
@@ -85,7 +85,7 @@ impl RbDecoder {
         setter!(self, CTC, cleanup, cleanup);
     }
 
-    pub fn ctc_pad_token(&self) -> String {
+    pub fn ctc_get_pad_token(&self) -> String {
         getter!(self, CTC, pad_token.clone())
     }
 
@@ -93,7 +93,7 @@ impl RbDecoder {
         setter!(self, CTC, pad_token, pad_token);
     }
 
-    pub fn ctc_word_delimiter_token(&self) -> String {
+    pub fn ctc_get_word_delimiter_token(&self) -> String {
         getter!(self, CTC, word_delimiter_token.clone())
     }
 
@@ -101,31 +101,31 @@ impl RbDecoder {
         setter!(self, CTC, word_delimiter_token, word_delimiter_token);
     }
 
-    fn strip_content(&self) -> char {
+    pub fn strip_get_content(&self) -> char {
         getter!(self, Strip, content)
     }
 
-    fn strip_set_content(&self, content: char) {
+    pub fn strip_set_content(&self, content: char) {
         setter!(self, Strip, content, content);
     }
 
-    fn strip_start(&self) -> usize {
+    pub fn strip_get_start(&self) -> usize {
         getter!(self, Strip, start)
     }
 
-    fn strip_set_start(&self, start: usize) {
+    pub fn strip_set_start(&self, start: usize) {
         setter!(self, Strip, start, start);
     }
 
-    fn strip_stop(&self) -> usize {
+    pub fn strip_get_stop(&self) -> usize {
         getter!(self, Strip, stop)
     }
 
-    fn strip_set_stop(&self, stop: usize) {
+    pub fn strip_set_stop(&self, stop: usize) {
         setter!(self, Strip, stop, stop);
     }
 
-    pub fn metaspace_replacement(&self) -> char {
+    pub fn metaspace_get_replacement(&self) -> char {
         getter!(self, Metaspace, get_replacement().clone())
     }
 
@@ -133,7 +133,7 @@ impl RbDecoder {
         setter!(self, Metaspace, @set_replacement, replacement);
     }
 
-    pub fn metaspace_split(&self) -> bool {
+    pub fn metaspace_get_split(&self) -> bool {
         getter!(self, Metaspace, get_split())
     }
 
@@ -141,7 +141,7 @@ impl RbDecoder {
         setter!(self, Metaspace, @set_split, split);
     }
 
-    pub fn metaspace_prepend_scheme(&self) -> String {
+    pub fn metaspace_get_prepend_scheme(&self) -> String {
         // Assuming Metaspace has a method to get the prepend_scheme as a string
         let scheme: PrependScheme = getter!(self, Metaspace, get_prepend_scheme());
         match scheme {
@@ -158,7 +158,7 @@ impl RbDecoder {
         Ok(())
     }
 
-    pub fn word_piece_cleanup(&self) -> bool {
+    pub fn word_piece_get_cleanup(&self) -> bool {
         getter!(self, WordPiece, cleanup)
     }
 
@@ -166,7 +166,7 @@ impl RbDecoder {
         setter!(self, WordPiece, cleanup, cleanup);
     }
 
-    pub fn word_piece_prefix(&self) -> String {
+    pub fn word_piece_get_prefix(&self) -> String {
         getter!(self, WordPiece, prefix.clone())
     }
 
@@ -371,7 +371,7 @@ pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
 
     let class = module.define_class("BPEDecoder", decoder)?;
     class.define_singleton_method("_new", function!(RbBPEDecoder::new, 1))?;
-    class.define_method("suffix", method!(RbDecoder::bpe_suffix, 0))?;
+    class.define_method("suffix", method!(RbDecoder::bpe_get_suffix, 0))?;
     class.define_method("suffix=", method!(RbDecoder::bpe_set_suffix, 1))?;
 
     let class = module.define_class("ByteFallback", decoder)?;
@@ -382,13 +382,13 @@ pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
 
     let class = module.define_class("CTC", decoder)?;
     class.define_singleton_method("_new", function!(RbCTC::new, 3))?;
-    class.define_method("cleanup", method!(RbDecoder::ctc_cleanup, 0))?;
+    class.define_method("cleanup", method!(RbDecoder::ctc_get_cleanup, 0))?;
     class.define_method("cleanup=", method!(RbDecoder::ctc_set_cleanup, 1))?;
-    class.define_method("pad_token", method!(RbDecoder::ctc_pad_token, 0))?;
+    class.define_method("pad_token", method!(RbDecoder::ctc_get_pad_token, 0))?;
     class.define_method("pad_token=", method!(RbDecoder::ctc_set_pad_token, 1))?;
     class.define_method(
         "word_delimiter_token",
-        method!(RbDecoder::ctc_word_delimiter_token, 0),
+        method!(RbDecoder::ctc_get_word_delimiter_token, 0),
     )?;
     class.define_method(
         "word_delimiter_token=",
@@ -402,18 +402,21 @@ pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("_new", function!(RbMetaspaceDecoder::new, 3))?;
     class.define_method(
         "prepend_scheme",
-        method!(RbDecoder::metaspace_prepend_scheme, 0),
+        method!(RbDecoder::metaspace_get_prepend_scheme, 0),
     )?;
     class.define_method(
         "prepend_scheme=",
         method!(RbDecoder::metaspace_set_prepend_scheme, 1),
     )?;
-    class.define_method("replacement", method!(RbDecoder::metaspace_replacement, 0))?;
+    class.define_method(
+        "replacement",
+        method!(RbDecoder::metaspace_get_replacement, 0),
+    )?;
     class.define_method(
         "replacement=",
         method!(RbDecoder::metaspace_set_replacement, 1),
     )?;
-    class.define_method("split", method!(RbDecoder::metaspace_split, 0))?;
+    class.define_method("split", method!(RbDecoder::metaspace_get_split, 0))?;
     class.define_method("split=", method!(RbDecoder::metaspace_set_split, 1))?;
 
     let class = module.define_class("Replace", decoder)?;
@@ -421,18 +424,18 @@ pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
 
     let class = module.define_class("Strip", decoder)?;
     class.define_singleton_method("_new", function!(RbStripDecoder::new, 3))?;
-    class.define_method("content", method!(RbDecoder::strip_content, 0))?;
+    class.define_method("content", method!(RbDecoder::strip_get_content, 0))?;
     class.define_method("content=", method!(RbDecoder::strip_set_content, 1))?;
-    class.define_method("start", method!(RbDecoder::strip_start, 0))?;
+    class.define_method("start", method!(RbDecoder::strip_get_start, 0))?;
     class.define_method("start=", method!(RbDecoder::strip_set_start, 1))?;
-    class.define_method("stop", method!(RbDecoder::strip_stop, 0))?;
+    class.define_method("stop", method!(RbDecoder::strip_get_stop, 0))?;
     class.define_method("stop=", method!(RbDecoder::strip_set_stop, 1))?;
 
     let class = module.define_class("WordPiece", decoder)?;
     class.define_singleton_method("_new", function!(RbWordPieceDecoder::new, 2))?;
-    class.define_method("cleanup", method!(RbDecoder::word_piece_cleanup, 0))?;
+    class.define_method("cleanup", method!(RbDecoder::word_piece_get_cleanup, 0))?;
     class.define_method("cleanup=", method!(RbDecoder::word_piece_set_cleanup, 1))?;
-    class.define_method("prefix", method!(RbDecoder::word_piece_prefix, 0))?;
+    class.define_method("prefix", method!(RbDecoder::word_piece_get_prefix, 0))?;
     class.define_method("prefix=", method!(RbDecoder::word_piece_set_prefix, 1))?;
 
     Ok(())

data/ext/tokenizers/src/encoding.rs

@@ -1,6 +1,8 @@
-use magnus::{RArray, Ruby};
+use magnus::{method, Module, RArray, RModule, Ruby};
 use tk::{Encoding, Offsets};
 
+use super::RbResult;
+
 #[magnus::wrap(class = "Tokenizers::Encoding")]
 #[repr(transparent)]
 pub struct RbEncoding {
@@ -14,43 +16,43 @@ impl From<Encoding> for RbEncoding {
 }
 
 impl RbEncoding {
-    pub fn n_sequences(&self) -> usize {
+    pub fn get_n_sequences(&self) -> usize {
         self.encoding.n_sequences()
     }
 
-    pub fn ids(&self) -> Vec<u32> {
+    pub fn get_ids(&self) -> Vec<u32> {
         self.encoding.get_ids().to_vec()
     }
 
-    pub fn tokens(&self) -> Vec<String> {
+    pub fn get_tokens(&self) -> Vec<String> {
         self.encoding.get_tokens().to_vec()
     }
 
-    pub fn word_ids(&self) -> Vec<Option<u32>> {
+    pub fn get_word_ids(&self) -> Vec<Option<u32>> {
         self.encoding.get_word_ids().to_vec()
     }
 
-    pub fn sequence_ids(&self) -> Vec<Option<usize>> {
+    pub fn get_sequence_ids(&self) -> Vec<Option<usize>> {
         self.encoding.get_sequence_ids()
     }
 
-    pub fn type_ids(&self) -> Vec<u32> {
+    pub fn get_type_ids(&self) -> Vec<u32> {
         self.encoding.get_type_ids().to_vec()
     }
 
-    pub fn offsets(&self) -> Vec<(usize, usize)> {
+    pub fn get_offsets(&self) -> Vec<(usize, usize)> {
         self.encoding.get_offsets().to_vec()
     }
 
-    pub fn special_tokens_mask(&self) -> Vec<u32> {
+    pub fn get_special_tokens_mask(&self) -> Vec<u32> {
         self.encoding.get_special_tokens_mask().to_vec()
     }
 
-    pub fn attention_mask(&self) -> Vec<u32> {
+    pub fn get_attention_mask(&self) -> Vec<u32> {
         self.encoding.get_attention_mask().to_vec()
     }
 
-    pub fn overflowing(ruby: &Ruby, rb_self: &Self) -> RArray {
+    pub fn get_overflowing(ruby: &Ruby, rb_self: &Self) -> RArray {
         ruby.ary_from_iter(
             rb_self
                 .encoding
@@ -91,3 +93,32 @@ impl RbEncoding {
         self.encoding.char_to_word(char_pos, sequence_index)
     }
 }
+
+pub fn init_encoding(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let class = module.define_class("Encoding", ruby.class_object())?;
+    class.define_method("n_sequences", method!(RbEncoding::get_n_sequences, 0))?;
+    class.define_method("ids", method!(RbEncoding::get_ids, 0))?;
+    class.define_method("tokens", method!(RbEncoding::get_tokens, 0))?;
+    class.define_method("word_ids", method!(RbEncoding::get_word_ids, 0))?;
+    class.define_method("sequence_ids", method!(RbEncoding::get_sequence_ids, 0))?;
+    class.define_method("type_ids", method!(RbEncoding::get_type_ids, 0))?;
+    class.define_method("offsets", method!(RbEncoding::get_offsets, 0))?;
+    class.define_method(
+        "special_tokens_mask",
+        method!(RbEncoding::get_special_tokens_mask, 0),
+    )?;
+    class.define_method("attention_mask", method!(RbEncoding::get_attention_mask, 0))?;
+    class.define_method("overflowing", method!(RbEncoding::get_overflowing, 0))?;
+    class.define_method("_word_to_tokens", method!(RbEncoding::word_to_tokens, 2))?;
+    class.define_method("_word_to_chars", method!(RbEncoding::word_to_chars, 2))?;
+    class.define_method(
+        "token_to_sequence",
+        method!(RbEncoding::token_to_sequence, 1),
+    )?;
+    class.define_method("token_to_chars", method!(RbEncoding::token_to_chars, 1))?;
+    class.define_method("token_to_word", method!(RbEncoding::token_to_word, 1))?;
+    class.define_method("_char_to_token", method!(RbEncoding::char_to_token, 2))?;
+    class.define_method("_char_to_word", method!(RbEncoding::char_to_word, 2))?;
+
+    Ok(())
+}

data/ext/tokenizers/src/error.rs

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
 use magnus::{prelude::*, value::Lazy, Error, ExceptionClass, Ruby};
 
 use super::TOKENIZERS;
@@ -7,17 +9,20 @@ pub struct RbError {}
 impl RbError {
     // convert to Error instead of Self
     pub fn from(e: Box<dyn std::error::Error + Send + Sync>) -> Error {
-        Error::new(error(), e.to_string())
+        Error::new(error(&Ruby::get().unwrap()), e.to_string())
     }
 
-    pub fn new_err(s: String) -> Error {
-        Error::new(error(), s)
+    pub fn new_err<T>(s: T) -> Error
+    where
+        T: Into<Cow<'static, str>>,
+    {
+        Error::new(error(&Ruby::get().unwrap()), s)
     }
 }
 
 static ERROR: Lazy<ExceptionClass> =
     Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Error").unwrap());
 
-fn error() -> ExceptionClass {
-    Ruby::get().unwrap().get_inner(&ERROR)
+fn error(ruby: &Ruby) -> ExceptionClass {
+    ruby.get_inner(&ERROR)
 }

data/ext/tokenizers/src/lib.rs

@@ -9,16 +9,15 @@ mod models;
 mod normalizers;
 mod pre_tokenizers;
 mod processors;
+mod ruby;
 mod tokenizer;
 mod trainers;
 mod utils;
 
-use encoding::RbEncoding;
 use error::RbError;
-use tokenizer::{RbAddedToken, RbTokenizer};
 use utils::RbRegex;
 
-use magnus::{function, method, prelude::*, value::Lazy, Error, RModule, Ruby};
+use magnus::{function, prelude::*, value::Lazy, Error, RModule, Ruby};
 
 type RbResult<T> = Result<T, Error>;
 
@@ -53,97 +52,9 @@ static TRAINERS: Lazy<RModule> =
 fn init(ruby: &Ruby) -> RbResult<()> {
     let module = ruby.define_module("Tokenizers")?;
 
-    let class = module.define_class("Tokenizer", ruby.class_object())?;
-    class.define_singleton_method("new", function!(RbTokenizer::from_model, 1))?;
-    class.define_singleton_method("from_str", function!(RbTokenizer::from_str, 1))?;
-    class.define_singleton_method("from_file", function!(RbTokenizer::from_file, 1))?;
-    class.define_method(
-        "add_special_tokens",
-        method!(RbTokenizer::add_special_tokens, 1),
-    )?;
-    class.define_method("train", method!(RbTokenizer::train, 2))?;
-    class.define_method("_save", method!(RbTokenizer::save, 2))?;
-    class.define_method("add_tokens", method!(RbTokenizer::add_tokens, 1))?;
-    class.define_method("_encode", method!(RbTokenizer::encode, 4))?;
-    class.define_method("_encode_batch", method!(RbTokenizer::encode_batch, 3))?;
-    class.define_method("_decode", method!(RbTokenizer::decode, 2))?;
-    class.define_method("_decode_batch", method!(RbTokenizer::decode_batch, 2))?;
-    class.define_method("model", method!(RbTokenizer::get_model, 0))?;
-    class.define_method("model=", method!(RbTokenizer::set_model, 1))?;
-    class.define_method("decoder", method!(RbTokenizer::get_decoder, 0))?;
-    class.define_method("decoder=", method!(RbTokenizer::set_decoder, 1))?;
-    class.define_method("pre_tokenizer", method!(RbTokenizer::get_pre_tokenizer, 0))?;
-    class.define_method("pre_tokenizer=", method!(RbTokenizer::set_pre_tokenizer, 1))?;
-    class.define_method(
-        "post_processor",
-        method!(RbTokenizer::get_post_processor, 0),
-    )?;
-    class.define_method(
-        "post_processor=",
-        method!(RbTokenizer::set_post_processor, 1),
-    )?;
-    class.define_method("normalizer", method!(RbTokenizer::get_normalizer, 0))?;
-    class.define_method("normalizer=", method!(RbTokenizer::set_normalizer, 1))?;
-    class.define_method("token_to_id", method!(RbTokenizer::token_to_id, 1))?;
-    class.define_method("id_to_token", method!(RbTokenizer::id_to_token, 1))?;
-    class.define_method("_enable_padding", method!(RbTokenizer::enable_padding, 1))?;
-    class.define_method("padding", method!(RbTokenizer::padding, 0))?;
-    class.define_method("no_padding", method!(RbTokenizer::no_padding, 0))?;
-    class.define_method(
-        "_enable_truncation",
-        method!(RbTokenizer::enable_truncation, 2),
-    )?;
-    class.define_method("truncation", method!(RbTokenizer::truncation, 0))?;
-    class.define_method("no_truncation", method!(RbTokenizer::no_truncation, 0))?;
-    class.define_method(
-        "num_special_tokens_to_add",
-        method!(RbTokenizer::num_special_tokens_to_add, 1),
-    )?;
-    class.define_method("_vocab", method!(RbTokenizer::vocab, 1))?;
-    class.define_method("_vocab_size", method!(RbTokenizer::vocab_size, 1))?;
-    class.define_method(
-        "added_tokens_decoder",
-        method!(RbTokenizer::get_added_tokens_decoder, 0),
-    )?;
-    class.define_method("_to_s", method!(RbTokenizer::to_str, 1))?;
-
-    let class = module.define_class("Encoding", ruby.class_object())?;
-    class.define_method("n_sequences", method!(RbEncoding::n_sequences, 0))?;
-    class.define_method("ids", method!(RbEncoding::ids, 0))?;
-    class.define_method("tokens", method!(RbEncoding::tokens, 0))?;
-    class.define_method("word_ids", method!(RbEncoding::word_ids, 0))?;
-    class.define_method("sequence_ids", method!(RbEncoding::sequence_ids, 0))?;
-    class.define_method("type_ids", method!(RbEncoding::type_ids, 0))?;
-    class.define_method("offsets", method!(RbEncoding::offsets, 0))?;
-    class.define_method(
-        "special_tokens_mask",
-        method!(RbEncoding::special_tokens_mask, 0),
-    )?;
-    class.define_method("attention_mask", method!(RbEncoding::attention_mask, 0))?;
-    class.define_method("overflowing", method!(RbEncoding::overflowing, 0))?;
-    class.define_method("_word_to_tokens", method!(RbEncoding::word_to_tokens, 2))?;
-    class.define_method("_word_to_chars", method!(RbEncoding::word_to_chars, 2))?;
-    class.define_method(
-        "token_to_sequence",
-        method!(RbEncoding::token_to_sequence, 1),
-    )?;
-    class.define_method("token_to_chars", method!(RbEncoding::token_to_chars, 1))?;
-    class.define_method("token_to_word", method!(RbEncoding::token_to_word, 1))?;
-    class.define_method("_char_to_token", method!(RbEncoding::char_to_token, 2))?;
-    class.define_method("_char_to_word", method!(RbEncoding::char_to_word, 2))?;
-
     let class = module.define_class("Regex", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbRegex::new, 1))?;
 
-    let class = module.define_class("AddedToken", ruby.class_object())?;
-    class.define_singleton_method("_new", function!(RbAddedToken::new, 2))?;
-    class.define_method("content", method!(RbAddedToken::get_content, 0))?;
-    class.define_method("rstrip", method!(RbAddedToken::get_rstrip, 0))?;
-    class.define_method("lstrip", method!(RbAddedToken::get_lstrip, 0))?;
-    class.define_method("single_word", method!(RbAddedToken::get_single_word, 0))?;
-    class.define_method("normalized", method!(RbAddedToken::get_normalized, 0))?;
-    class.define_method("special", method!(RbAddedToken::get_special, 0))?;
-
     let models = module.define_module("Models")?;
     let pre_tokenizers = module.define_module("PreTokenizers")?;
     let decoders = module.define_module("Decoders")?;
@@ -151,6 +62,8 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     let normalizers = module.define_module("Normalizers")?;
     let trainers = module.define_module("Trainers")?;
 
+    tokenizer::init_tokenizer(ruby, &module)?;
+    encoding::init_encoding(ruby, &module)?;
     models::init_models(ruby, &models)?;
     pre_tokenizers::init_pre_tokenizers(ruby, &pre_tokenizers)?;
     decoders::init_decoders(ruby, &decoders)?;