tokenizers 0.3.3 → 0.4.1

data/ext/tokenizers/src/trainers.rs

@@ -3,16 +3,16 @@ use std::sync::{Arc, RwLock};
 
 use crate::models::RbModel;
 use crate::tokenizer::RbAddedToken;
-use magnus::typed_data::DataTypeBuilder;
+use magnus::prelude::*;
 use magnus::{
-    exception, function, memoize, method, Class, DataType, DataTypeFunctions, Error, Module, Object,
-    RArray, RClass, RHash, RModule, Symbol, TypedData, Value,
+    data_type_builder, exception, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error, Module, Object,
+    RArray, RClass, RHash, RModule, Ruby, Symbol, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
 use tk::models::TrainerWrapper;
 use tk::Trainer;
 
-use super::RbResult;
+use super::{RbResult, TRAINERS};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
 pub struct RbTrainer {
@@ -112,7 +112,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -144,7 +144,7 @@ impl RbTrainer {
             self,
             BpeTrainer,
             initial_alphabet,
-            alphabet.into_iter().map(|c| c).collect()
+            alphabet.into_iter().collect()
         );
     }
 
@@ -199,7 +199,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -223,7 +223,7 @@ impl RbTrainer {
             self,
             UnigramTrainer,
             initial_alphabet,
-            alphabet.into_iter().map(|c| c).collect()
+            alphabet.into_iter().collect()
         );
     }
 
@@ -270,7 +270,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -324,7 +324,7 @@ impl RbTrainer {
             special_tokens
                 .each()
                 .map(|token| {
-                    if let Ok(content) = token?.try_convert::<String>() {
+                    if let Ok(content) = String::try_convert(token?) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -356,7 +356,7 @@ impl RbTrainer {
             self,
             WordPieceTrainer,
             @set_initial_alphabet,
-            alphabet.into_iter().map(|c| c).collect()
+            alphabet.into_iter().collect()
         );
     }
 
@@ -397,11 +397,10 @@ impl RbBpeTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder = builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -413,39 +412,39 @@ impl RbBpeTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = value.try_convert::<Vec<char>>()?;
+            let arr = <Vec<char>>::try_convert(value)?;
             let set: HashSet<char> = HashSet::from_iter(arr);
             builder = builder.initial_alphabet(set);
         }
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder = builder.vocab_size(value.try_convert()?);
+            builder = builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
         if !value.is_nil() {
-            builder = builder.min_frequency(value.try_convert()?);
+            builder = builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder = builder.show_progress(value.try_convert()?);
+            builder = builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("limit_alphabet"))?;
         if !value.is_nil() {
-            builder = builder.limit_alphabet(value.try_convert()?);
+            builder = builder.limit_alphabet(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
         if !value.is_nil() {
-            builder = builder.continuing_subword_prefix(value.try_convert()?);
+            builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
         if !value.is_nil() {
-            builder = builder.end_of_word_suffix(value.try_convert()?);
+            builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -466,11 +465,10 @@ impl RbUnigramTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -482,44 +480,44 @@ impl RbUnigramTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = value.try_convert::<Vec<char>>()?;
+            let arr = <Vec<char>>::try_convert(value)?;
             let set: HashSet<char> = HashSet::from_iter(arr);
             builder.initial_alphabet(set);
         }
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder.vocab_size(value.try_convert()?);
+            builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder.show_progress(value.try_convert()?);
+            builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("n_sub_iterations"))?;
         if !value.is_nil() {
-            builder.n_sub_iterations(value.try_convert()?);
+            builder.n_sub_iterations(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
         if !value.is_nil() {
-            builder.unk_token(Some(value.try_convert()?));
+            builder.unk_token(Some(TryConvert::try_convert(value)?));
         }
 
         let value: Value = kwargs.delete(Symbol::new("max_piece_length"))?;
         if !value.is_nil() {
-            builder.max_piece_length(value.try_convert()?);
+            builder.max_piece_length(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("seed_size"))?;
         if !value.is_nil() {
-            builder.seed_size(value.try_convert()?);
+            builder.seed_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("shrinking_factor"))?;
         if !value.is_nil() {
-            builder.shrinking_factor(value.try_convert()?);
+            builder.shrinking_factor(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -541,11 +539,10 @@ impl RbWordLevelTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -557,17 +554,17 @@ impl RbWordLevelTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder.vocab_size(value.try_convert()?);
+            builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
         if !value.is_nil() {
-            builder.min_frequency(value.try_convert()?);
+            builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder.show_progress(value.try_convert()?);
+            builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         Ok(builder.build().expect("WordLevelTrainerBuilder cannot fail").into())
@@ -583,11 +580,10 @@ impl RbWordPieceTrainer {
         let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
         if !value.is_nil() {
             builder = builder.special_tokens(
-                value
-                    .try_convert::<RArray>()?
+                RArray::try_convert(value)?
                     .each()
                     .map(|token| {
-                        if let Ok(content) = token?.try_convert::<String>() {
+                        if let Ok(content) = String::try_convert(token?) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -599,39 +595,39 @@ impl RbWordPieceTrainer {
 
         let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = value.try_convert::<Vec<char>>()?;
+            let arr = <Vec<char>>::try_convert(value)?;
             let set: HashSet<char> = HashSet::from_iter(arr);
             builder = builder.initial_alphabet(set);
         }
 
         let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
         if !value.is_nil() {
-            builder = builder.vocab_size(value.try_convert()?);
+            builder = builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
         if !value.is_nil() {
-            builder = builder.min_frequency(value.try_convert()?);
+            builder = builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
         if !value.is_nil() {
-            builder = builder.show_progress(value.try_convert()?);
+            builder = builder.show_progress(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("limit_alphabet"))?;
         if !value.is_nil() {
-            builder = builder.limit_alphabet(value.try_convert()?);
+            builder = builder.limit_alphabet(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
         if !value.is_nil() {
-            builder = builder.continuing_subword_prefix(value.try_convert()?);
+            builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
         let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
         if !value.is_nil() {
-            builder = builder.end_of_word_suffix(value.try_convert()?);
+            builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
@@ -644,46 +640,52 @@ impl RbWordPieceTrainer {
 }
 
 unsafe impl TypedData for RbTrainer {
-    fn class() -> RClass {
-        *memoize!(RClass: {
-          let class: RClass = crate::trainers().const_get("Trainer").unwrap();
-          class.undef_alloc_func();
-          class
-        })
+    fn class(ruby: &Ruby) -> RClass {
+        static CLASS: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("Trainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        ruby.get_inner(&CLASS)
     }
 
     fn data_type() -> &'static DataType {
-        memoize!(DataType: DataTypeBuilder::<RbTrainer>::new("Tokenizers::Trainers::Trainer").build())
-    }
-
-    fn class_for(value: &Self) -> RClass {
+        static DATA_TYPE: DataType = data_type_builder!(RbTrainer, "Tokenizers::Trainers::Trainer").build();
+        &DATA_TYPE
+    }
+
+    fn class_for(ruby: &Ruby, value: &Self) -> RClass {
+        static BPE_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("BpeTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static UNIGRAM_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("UnigramTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_LEVEL_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("WordLevelTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
+        static WORD_PIECE_TRAINER: Lazy<RClass> = Lazy::new(|ruby| {
+            let class: RClass = ruby.get_inner(&TRAINERS).const_get("WordPieceTrainer").unwrap();
+            class.undef_default_alloc_func();
+            class
+        });
         match *value.trainer.read().unwrap() {
-            TrainerWrapper::BpeTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("BpeTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            TrainerWrapper::UnigramTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("UnigramTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            TrainerWrapper::WordLevelTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("WordLevelTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
-            TrainerWrapper::WordPieceTrainer(_) => *memoize!(RClass: {
-                let class: RClass = crate::trainers().const_get("WordPieceTrainer").unwrap();
-                class.undef_alloc_func();
-                class
-            }),
+            TrainerWrapper::BpeTrainer(_) => ruby.get_inner(&BPE_TRAINER),
+            TrainerWrapper::UnigramTrainer(_) => ruby.get_inner(&UNIGRAM_TRAINER),
+            TrainerWrapper::WordLevelTrainer(_) => ruby.get_inner(&WORD_LEVEL_TRAINER),
+            TrainerWrapper::WordPieceTrainer(_) => ruby.get_inner(&WORD_PIECE_TRAINER),
         }
     }
 }
 
-pub fn trainers(module: &RModule) -> RbResult<()> {
-    let trainer = module.define_class("Trainer", Default::default())?;
+pub fn init_trainers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
+    let trainer = module.define_class("Trainer", ruby.class_object())?;
 
     let class = module.define_class("BpeTrainer", trainer)?;
     class.define_singleton_method("_new", function!(RbBpeTrainer::new, 1))?;

data/ext/tokenizers/src/utils/normalization.rs

@@ -1,5 +1,6 @@
 use super::regex::{regex, RbRegex};
 use crate::RbResult;
+use magnus::prelude::*;
 use magnus::{exception, Error, TryConvert, Value};
 use tk::normalizer::SplitDelimiterBehavior;
 use tk::pattern::Pattern;
@@ -13,9 +14,9 @@ pub enum RbPattern<'p> {
 impl TryConvert for RbPattern<'_> {
     fn try_convert(obj: Value) -> RbResult<Self> {
         if obj.is_kind_of(regex()) {
-            Ok(RbPattern::Regex(obj.try_convert()?))
+            Ok(RbPattern::Regex(TryConvert::try_convert(obj)?))
         } else {
-            Ok(RbPattern::Str(obj.try_convert()?))
+            Ok(RbPattern::Str(TryConvert::try_convert(obj)?))
         }
     }
 }
@@ -61,7 +62,7 @@ pub struct RbSplitDelimiterBehavior(pub SplitDelimiterBehavior);
 
 impl TryConvert for RbSplitDelimiterBehavior {
     fn try_convert(obj: Value) -> RbResult<Self> {
-        let s = obj.try_convert::<String>()?;
+        let s = String::try_convert(obj)?;
 
         Ok(Self(match s.as_str() {
             "removed" => Ok(SplitDelimiterBehavior::Removed),

data/ext/tokenizers/src/utils/regex.rs

@@ -1,6 +1,6 @@
 use onig::Regex;
-use magnus::{exception, memoize, Error, Module, RClass};
-use crate::{module, RbResult};
+use magnus::{exception, prelude::*, value::Lazy, Error, RClass, Ruby};
+use crate::{RbResult, TOKENIZERS};
 
 #[magnus::wrap(class = "Tokenizers::Regex")]
 pub struct RbRegex {
@@ -17,6 +17,8 @@ impl RbRegex {
     }
 }
 
+static REGEX: Lazy<RClass> = Lazy::new(|ruby| ruby.get_inner(&TOKENIZERS).const_get("Regex").unwrap());
+
 pub fn regex() -> RClass {
-    *memoize!(RClass: module().const_get("Regex").unwrap())
+    Ruby::get().unwrap().get_inner(&REGEX)
 }

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.13.3"
+    TOKENIZERS_VERSION = "0.14.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/models/unigram.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module Models
     class Unigram
-      def self.new(vocab: nil, unk_id: nil)
-        _new(vocab, unk_id)
+      def self.new(vocab: nil, unk_id: nil, byte_fallback: nil)
+        _new(vocab, unk_id, byte_fallback)
       end
     end
   end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.3"
+  VERSION = "0.4.1"
 end

data/lib/tokenizers.rb

@@ -1,8 +1,8 @@
 # ext
 begin
-  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require_relative "tokenizers/tokenizers"
+  require "tokenizers/tokenizers"
 end
 
 # decoders

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-09 00:00:00.000000000 Z
+date: 2023-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -93,7 +93,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="