tokenizers 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6e88ec5618e36e317434410c960603695806bb59dadb2252f2957d8dbf0525b
-  data.tar.gz: 33a04a4a5faada27e6e7246c16d836a4ff9f6793e89de3cfd4880e30c6c8ed0d
+  metadata.gz: 0677a3662d1f3af9007bdc0524a71903270cfbb85c3bf0efaa2365946df6c5b1
+  data.tar.gz: 00e49737180957c88fb8f8418de40c8983715c733275df16c1ab2fe97355f92d
 SHA512:
-  metadata.gz: 88e4f2ad57fd1d66cd5fcf0d8b7ff6b1ea902258296fb02d207a446032134189e3445a104658074e94f914331c94f46cfdd09eed7c745c0483cb3b32b09e6abf
-  data.tar.gz: e8a1721ecbd36874322477077331743b0d1ba2de6f90076e07ad5456c230f76625d7f28ed6e6026c11395c6bb27701a6b8c0feedf2050387d32d9b777baa51fe
+  metadata.gz: f1fd8f9a57be7cac938ae6bc46533f540b210ef6d46a3f46e168a257d2b3156000b9f451e1085a9c90dd54444abeaa69b5d2ae2905fdb5abfc85fb7d610f1427
+  data.tar.gz: c748fb9150431ecce025e9f98f870e8140f186a2cc796eb5df2dfef630db05ab45eb8f23f2dc5e50d39b4eadb5fc1f5e4902482f41b6dd469ff95eae4ec3d2f3

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.0 (2023-07-20)
+
+- Updated Tokenizers to 0.14.0
+- Dropped support for Ruby < 3
+
 ## 0.3.3 (2023-04-09)
 
 - Updated Tokenizers to 0.13.3

data/Cargo.lock

@@ -11,6 +11,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "aho-corasick"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -25,9 +34,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "bindgen"
-version = "0.60.1"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+checksum = "c6720a8b7b2d39dd533285ed438d458f65b31b5c257e6ac7bb3d7e82844dd722"
 dependencies = [
  "bitflags",
  "cexpr",
@@ -40,6 +49,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -352,31 +362,32 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
 
 [[package]]
 name = "magnus"
-version = "0.5.3"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8dc14463c2552e753ef562961f486ca76f17a857c121db40e9f3ade3f35ab81"
+checksum = "68e9585bfe236e88e6b10b6d8eb5349bd0e0009f3f9dff8d2e99a82601b33743"
 dependencies = [
  "magnus-macros",
  "rb-sys",
  "rb-sys-env",
+ "seq-macro",
 ]
 
 [[package]]
 name = "magnus-macros"
-version = "0.4.1"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cc17af1d45442c011aa579d727ec6cff8a69aea8a6bbad26736e7112d749bfb"
+checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.13",
 ]
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
 
 [[package]]
 name = "memoffset"
@@ -575,18 +586,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.71"
+version = "0.9.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "156bfedced1e236600bcaad538477097ff2ed5c6b474e411d15b791e1d24c0f1"
+checksum = "939fb78db3e4f26665c1d4c7b91ca66d3578335a19aba552d4a6445811d07072"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.71"
+version = "0.9.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cb2e4a32cbc290b543a74567072ad24b708aff7bb5dde5a68d5690379cd7938"
+checksum = "335a95eb0420d52fa94ef12019df3c2c250c6b19cbb3c60bd05cb7e9c362072c"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -605,20 +616,32 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
 
 [[package]]
 name = "regex"
-version = "1.7.3"
+version = "1.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
+checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
 dependencies = [
- "aho-corasick",
+ "aho-corasick 1.0.5",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
+dependencies = [
+ "aho-corasick 1.0.5",
  "memchr",
  "regex-syntax",
 ]
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.29"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
 
 [[package]]
 name = "rustc-hash"
@@ -638,6 +661,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "seq-macro"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
+
 [[package]]
 name = "serde"
 version = "1.0.159"
@@ -749,21 +778,21 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "magnus",
  "onig",
  "serde",
- "tokenizers 0.13.3",
+ "tokenizers 0.14.0",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.13.3"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5"
+checksum = "12b515a66453a4d68f03398054f7204fd0dde6b93d3f20ea90b08025ab49b499"
 dependencies = [
- "aho-corasick",
+ "aho-corasick 0.7.20",
  "derive_builder",
  "esaxx-rs",
  "getrandom",

data/ext/tokenizers/Cargo.toml

@@ -1,20 +1,21 @@
 [package]
 name = "tokenizers"
-version = "0.3.3"
+version = "0.4.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
+rust-version = "1.62.0"
 publish = false
 
 [lib]
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = "0.5"
+magnus = "0.6"
 onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.13.3" # also update in from_pretrained.rb
+version = "=0.14.0" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/decoders.rs

@@ -1,9 +1,9 @@
 use std::sync::{Arc, RwLock};
 
-use magnus::typed_data::DataTypeBuilder;
+use magnus::value::Lazy;
 use magnus::{
-    function, memoize, method, Class, DataType, DataTypeFunctions, Module, Object, RClass, RModule,
-    TypedData,
+    data_type_builder, function, method, Class, DataType, DataTypeFunctions, Module, Object, RClass, RModule,
+    Ruby, TypedData,
 };
 use serde::{Deserialize, Serialize};
 use tk::decoders::bpe::BPEDecoder;
@@ -19,7 +19,7 @@ use tk::Decoder;
 use tk::normalizers::replace::Replace;
 
 use super::utils::*;
-use super::{RbError, RbResult};
+use super::{DECODERS, RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
 pub struct RbDecoder {
@@ -260,74 +260,85 @@ impl Decoder for RbDecoderWrapper {
 }
 
 unsafe impl TypedData for RbDecoder {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::decoders().const_get("Decoder").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("Decoder").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbDecoder>::new("Tokenizers::Decoders::Decoder").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbDecoder, "Tokenizers::Decoders::Decoder").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static BPE_DECODER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("BPEDecoder").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BYTE_FALLBACK: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("ByteFallback").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static BYTE_LEVEL: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("ByteLevel").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static CTC: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("CTC").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static FUSE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("Fuse").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static METASPACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("Metaspace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static REPLACE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("Replace").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static STRIP: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("Strip").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_PIECE: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&DECODERS).const_get("WordPiece").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match &value.decoder {
             RbDecoderWrapper::Wrapped(inner) => match *inner.read().unwrap() {
-                DecoderWrapper::BPE(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("BPEDecoder").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::ByteFallback(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("ByteFallback").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::ByteLevel(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("ByteLevel").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::CTC(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("CTC").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::Fuse(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("Fuse").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::Metaspace(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("Metaspace").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::Replace(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("Replace").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::Strip(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("Strip").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
-                DecoderWrapper::WordPiece(_) => *memoize!(RClass: {
-                    let class: RClass = crate::decoders().const_get("WordPiece").unwrap();
-                    class.undef_alloc_func();
-                    class
-                }),
+                DecoderWrapper::BPE(_) => ruby.get_inner(&BPE_DECODER),
+                DecoderWrapper::ByteFallback(_) => ruby.get_inner(&BYTE_FALLBACK),
+                DecoderWrapper::ByteLevel(_) => ruby.get_inner(&BYTE_LEVEL),
+                DecoderWrapper::CTC(_) => ruby.get_inner(&CTC),
+                DecoderWrapper::Fuse(_) => ruby.get_inner(&FUSE),
+                DecoderWrapper::Metaspace(_) => ruby.get_inner(&METASPACE),
+                DecoderWrapper::Replace(_) => ruby.get_inner(&REPLACE),
+                DecoderWrapper::Strip(_) => ruby.get_inner(&STRIP),
+                DecoderWrapper::WordPiece(_) => ruby.get_inner(&WORD_PIECE),
                 _ => todo!(),
             },
         }
     }
 }
 
-pub fn decoders(module: &RModule) -> RbResult<()> {
-    let decoder = module.define_class("Decoder", Default::default())?;
+pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let decoder = module.define_class("Decoder", ruby.class_object())?;
 
     let class = module.define_class("BPEDecoder", decoder)?;
     class.define_singleton_method("_new", function!(RbBPEDecoder::new, 1))?;

data/ext/tokenizers/src/error.rs

@@ -1,6 +1,6 @@
-use magnus::{memoize, Error, ExceptionClass, Module};
+use magnus::{prelude::*, value::Lazy, Error, ExceptionClass, Ruby};
 
-use super::module;
+use super::TOKENIZERS;
 
 pub struct RbError {}
 
@@ -11,6 +11,8 @@ impl RbError {
     }
 }
 
+static ERROR: Lazy<ExceptionClass> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Error").unwrap());
+
 fn error() -> ExceptionClass {
-    *memoize!(ExceptionClass: module().const_get("Error").unwrap())
+    Ruby::get().unwrap().get_inner(&ERROR)
 }

data/ext/tokenizers/src/lib.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::new_ret_no_self)]
+
 extern crate tokenizers as tk;
 
 mod decoders;
@@ -16,43 +18,29 @@ use error::RbError;
 use tokenizer::RbTokenizer;
 use utils::RbRegex;
 
-use magnus::{define_module, function, memoize, method, prelude::*, Error, RModule};
+use magnus::{function, method, prelude::*, value::Lazy, Error, RModule, Ruby};
 
 type RbResult<T> = Result<T, Error>;
 
-fn module() -> RModule {
-    *memoize!(RModule: define_module("Tokenizers").unwrap())
-}
+static TOKENIZERS: Lazy<RModule> = Lazy::new(|ruby| ruby.class_object().const_get("Tokenizers").unwrap());
 
-fn decoders() -> RModule {
-    *memoize!(RModule: module().const_get("Decoders").unwrap())
-}
+static DECODERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Decoders").unwrap());
 
-fn models() -> RModule {
-    *memoize!(RModule: module().const_get("Models").unwrap())
-}
+static MODELS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Models").unwrap());
 
-fn normalizers() -> RModule {
-    *memoize!(RModule: module().const_get("Normalizers").unwrap())
-}
+static NORMALIZERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Normalizers").unwrap());
 
-fn pre_tokenizers() -> RModule {
-    *memoize!(RModule: module().const_get("PreTokenizers").unwrap())
-}
+static PRE_TOKENIZERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("PreTokenizers").unwrap());
 
-fn processors() -> RModule {
-    *memoize!(RModule: module().const_get("Processors").unwrap())
-}
+static PROCESSORS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Processors").unwrap());
 
-fn trainers() -> RModule {
-    *memoize!(RModule: module().const_get("Trainers").unwrap())
-}
+static TRAINERS: Lazy<RModule> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Trainers").unwrap());
 
 #[magnus::init]
-fn init() -> RbResult<()> {
-    let module = module();
+fn init(ruby: &Ruby) -> RbResult<()> {
+    let module = ruby.get_inner(&TOKENIZERS);
 
-    let class = module.define_class("Tokenizer", Default::default())?;
+    let class = module.define_class("Tokenizer", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbTokenizer::from_model, 1))?;
     class.define_singleton_method("from_file", function!(RbTokenizer::from_file, 1))?;
     class.define_method(
@@ -86,7 +74,7 @@ fn init() -> RbResult<()> {
     class.define_method("_vocab_size", method!(RbTokenizer::vocab_size, 1))?;
     class.define_method("_to_s", method!(RbTokenizer::to_str, 1))?;
 
-    let class = module.define_class("Encoding", Default::default())?;
+    let class = module.define_class("Encoding", ruby.class_object())?;
     class.define_method("n_sequences", method!(RbEncoding::n_sequences, 0))?;
     class.define_method("ids", method!(RbEncoding::ids, 0))?;
     class.define_method("tokens", method!(RbEncoding::tokens, 0))?;
@@ -111,7 +99,7 @@ fn init() -> RbResult<()> {
     class.define_method("_char_to_token", method!(RbEncoding::char_to_token, 2))?;
     class.define_method("_char_to_word", method!(RbEncoding::char_to_word, 2))?;
 
-    let class = module.define_class("Regex", Default::default())?;
+    let class = module.define_class("Regex", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbRegex::new, 1))?;
 
     let models = module.define_module("Models")?;
@@ -121,12 +109,12 @@ fn init() -> RbResult<()> {
     let normalizers = module.define_module("Normalizers")?;
     let trainers = module.define_module("Trainers")?;
 
-    models::models(&models)?;
-    pre_tokenizers::pre_tokenizers(&pre_tokenizers)?;
-    decoders::decoders(&decoders)?;
-    processors::processors(&processors)?;
-    normalizers::normalizers(&normalizers)?;
-    trainers::trainers(&trainers)?;
+    models::init_models(ruby, &models)?;
+    pre_tokenizers::init_pre_tokenizers(ruby, &pre_tokenizers)?;
+    decoders::init_decoders(ruby, &decoders)?;
+    processors::init_processors(ruby, &processors)?;
+    normalizers::init_normalizers(ruby, &normalizers)?;
+    trainers::init_trainers(ruby, &trainers)?;
 
     Ok(())
 }