tokenizers 0.5.4 → 0.6.0

data/ext/tokenizers/src/trainers.rs

@@ -1,13 +1,11 @@
-use std::collections::HashSet;
 use std::sync::{Arc, RwLock};
 
 use crate::models::RbModel;
 use crate::tokenizer::RbAddedToken;
 use magnus::prelude::*;
 use magnus::{
-    data_type_builder, exception, function, method, value::Lazy, Class, DataType,
-    DataTypeFunctions, Error, Module, Object, RArray, RClass, RHash, RModule, Ruby, Symbol,
-    TryConvert, TypedData, Value,
+    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error,
+    Module, Object, RArray, RClass, RHash, RModule, Ruby, TryConvert, TypedData, Value,
 };
 use serde::{Deserialize, Serialize};
 use tk::models::TrainerWrapper;
@@ -391,10 +389,10 @@ where
 pub struct RbBpeTrainer {}
 
 impl RbBpeTrainer {
-    pub fn new(kwargs: RHash) -> RbResult<RbTrainer> {
+    pub fn new(ruby: &Ruby, kwargs: RHash) -> RbResult<RbTrainer> {
         let mut builder = tk::models::bpe::BpeTrainer::builder();
 
-        let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("special_tokens"))?;
         if !value.is_nil() {
             builder = builder.special_tokens(
                 RArray::try_convert(value)?
@@ -410,46 +408,50 @@ impl RbBpeTrainer {
             );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = <Vec<char>>::try_convert(value)?;
-            let set: HashSet<char> = HashSet::from_iter(arr);
-            builder = builder.initial_alphabet(set);
+            let alphabet = Vec::<String>::try_convert(value)?;
+            builder = builder.initial_alphabet(
+                alphabet
+                    .into_iter()
+                    .filter_map(|s| s.chars().next())
+                    .collect(),
+            );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("vocab_size"))?;
         if !value.is_nil() {
             builder = builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("min_frequency"))?;
         if !value.is_nil() {
             builder = builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("show_progress"))?;
         if !value.is_nil() {
             builder = builder.show_progress(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("limit_alphabet"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("limit_alphabet"))?;
         if !value.is_nil() {
             builder = builder.limit_alphabet(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("continuing_subword_prefix"))?;
         if !value.is_nil() {
             builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("end_of_word_suffix"))?;
         if !value.is_nil() {
             builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         Ok(builder.build().into())
@@ -459,10 +461,10 @@ impl RbBpeTrainer {
 pub struct RbUnigramTrainer {}
 
 impl RbUnigramTrainer {
-    pub fn new(kwargs: RHash) -> RbResult<RbTrainer> {
+    pub fn new(ruby: &Ruby, kwargs: RHash) -> RbResult<RbTrainer> {
         let mut builder = tk::models::unigram::UnigramTrainer::builder();
 
-        let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("special_tokens"))?;
         if !value.is_nil() {
             builder.special_tokens(
                 RArray::try_convert(value)?
@@ -478,56 +480,60 @@ impl RbUnigramTrainer {
             );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = <Vec<char>>::try_convert(value)?;
-            let set: HashSet<char> = HashSet::from_iter(arr);
-            builder.initial_alphabet(set);
+            let alphabet = Vec::<String>::try_convert(value)?;
+            builder.initial_alphabet(
+                alphabet
+                    .into_iter()
+                    .filter_map(|s| s.chars().next())
+                    .collect(),
+            );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("vocab_size"))?;
         if !value.is_nil() {
             builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("show_progress"))?;
         if !value.is_nil() {
             builder.show_progress(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("n_sub_iterations"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("n_sub_iterations"))?;
         if !value.is_nil() {
             builder.n_sub_iterations(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("unk_token"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("unk_token"))?;
         if !value.is_nil() {
             builder.unk_token(Some(TryConvert::try_convert(value)?));
         }
 
-        let value: Value = kwargs.delete(Symbol::new("max_piece_length"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("max_piece_length"))?;
         if !value.is_nil() {
             builder.max_piece_length(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("seed_size"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("seed_size"))?;
         if !value.is_nil() {
             builder.seed_size(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("shrinking_factor"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("shrinking_factor"))?;
         if !value.is_nil() {
             builder.shrinking_factor(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         let trainer = builder
             .build()
-            .map_err(|_| Error::new(exception::arg_error(), "Cannot build UnigramTrainer"))?;
+            .map_err(|_| Error::new(ruby.exception_arg_error(), "Cannot build UnigramTrainer"))?;
         Ok(trainer.into())
     }
 }
@@ -535,10 +541,10 @@ impl RbUnigramTrainer {
 pub struct RbWordLevelTrainer {}
 
 impl RbWordLevelTrainer {
-    pub fn new(kwargs: RHash) -> RbResult<RbTrainer> {
+    pub fn new(ruby: &Ruby, kwargs: RHash) -> RbResult<RbTrainer> {
         let mut builder = tk::models::wordlevel::WordLevelTrainer::builder();
 
-        let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("special_tokens"))?;
         if !value.is_nil() {
             builder.special_tokens(
                 RArray::try_convert(value)?
@@ -554,17 +560,17 @@ impl RbWordLevelTrainer {
             );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("vocab_size"))?;
         if !value.is_nil() {
             builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("min_frequency"))?;
         if !value.is_nil() {
             builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("show_progress"))?;
         if !value.is_nil() {
             builder.show_progress(TryConvert::try_convert(value)?);
         }
@@ -579,10 +585,10 @@ impl RbWordLevelTrainer {
 pub struct RbWordPieceTrainer {}
 
 impl RbWordPieceTrainer {
-    pub fn new(kwargs: RHash) -> RbResult<RbTrainer> {
+    pub fn new(ruby: &Ruby, kwargs: RHash) -> RbResult<RbTrainer> {
         let mut builder = tk::models::wordpiece::WordPieceTrainer::builder();
 
-        let value: Value = kwargs.delete(Symbol::new("special_tokens"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("special_tokens"))?;
         if !value.is_nil() {
             builder = builder.special_tokens(
                 RArray::try_convert(value)?
@@ -598,46 +604,50 @@ impl RbWordPieceTrainer {
             );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("initial_alphabet"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("initial_alphabet"))?;
         if !value.is_nil() {
-            let arr = <Vec<char>>::try_convert(value)?;
-            let set: HashSet<char> = HashSet::from_iter(arr);
-            builder = builder.initial_alphabet(set);
+            let alphabet = Vec::<String>::try_convert(value)?;
+            builder = builder.initial_alphabet(
+                alphabet
+                    .into_iter()
+                    .filter_map(|s| s.chars().next())
+                    .collect(),
+            );
         }
 
-        let value: Value = kwargs.delete(Symbol::new("vocab_size"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("vocab_size"))?;
         if !value.is_nil() {
             builder = builder.vocab_size(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("min_frequency"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("min_frequency"))?;
         if !value.is_nil() {
             builder = builder.min_frequency(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("show_progress"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("show_progress"))?;
         if !value.is_nil() {
             builder = builder.show_progress(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("limit_alphabet"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("limit_alphabet"))?;
         if !value.is_nil() {
             builder = builder.limit_alphabet(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("continuing_subword_prefix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("continuing_subword_prefix"))?;
         if !value.is_nil() {
             builder = builder.continuing_subword_prefix(TryConvert::try_convert(value)?);
         }
 
-        let value: Value = kwargs.delete(Symbol::new("end_of_word_suffix"))?;
+        let value: Value = kwargs.delete(ruby.to_symbol("end_of_word_suffix"))?;
         if !value.is_nil() {
             builder = builder.end_of_word_suffix(TryConvert::try_convert(value)?);
         }
 
         if !kwargs.is_empty() {
             // TODO improve message
-            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+            return Err(Error::new(ruby.exception_arg_error(), "unknown keyword"));
         }
 
         Ok(builder.build().into())

data/ext/tokenizers/src/utils/normalization.rs

@@ -1,7 +1,7 @@
 use super::regex::{regex, RbRegex};
 use crate::RbResult;
 use magnus::prelude::*;
-use magnus::{exception, Error, TryConvert, Value};
+use magnus::{Error, Ruby, TryConvert, Value};
 use tk::normalizer::SplitDelimiterBehavior;
 use tk::pattern::Pattern;
 
@@ -62,6 +62,7 @@ pub struct RbSplitDelimiterBehavior(pub SplitDelimiterBehavior);
 
 impl TryConvert for RbSplitDelimiterBehavior {
     fn try_convert(obj: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(obj);
         let s = String::try_convert(obj)?;
 
         Ok(Self(match s.as_str() {
@@ -71,7 +72,7 @@ impl TryConvert for RbSplitDelimiterBehavior {
             "merged_with_next" => Ok(SplitDelimiterBehavior::MergedWithNext),
             "contiguous" => Ok(SplitDelimiterBehavior::Contiguous),
             _ => Err(Error::new(
-                exception::arg_error(),
+                ruby.exception_arg_error(),
                 "Wrong value for SplitDelimiterBehavior, expected one of: \
                 `removed, isolated, merged_with_previous, merged_with_next, contiguous`",
             )),

data/ext/tokenizers/src/utils/regex.rs

@@ -1,5 +1,5 @@
 use crate::{RbResult, TOKENIZERS};
-use magnus::{exception, prelude::*, value::Lazy, Error, RClass, Ruby};
+use magnus::{prelude::*, value::Lazy, Error, RClass, Ruby};
 use onig::Regex;
 
 #[magnus::wrap(class = "Tokenizers::Regex")]
@@ -9,10 +9,11 @@ pub struct RbRegex {
 }
 
 impl RbRegex {
-    pub fn new(s: String) -> RbResult<Self> {
+    pub fn new(ruby: &Ruby, s: String) -> RbResult<Self> {
         Ok(Self {
-            inner: Regex::new(&s)
-                .map_err(|e| Error::new(exception::runtime_error(), e.description().to_owned()))?,
+            inner: Regex::new(&s).map_err(|e| {
+                Error::new(ruby.exception_runtime_error(), e.description().to_owned())
+            })?,
             pattern: s,
         })
     }

data/lib/tokenizers/from_pretrained.rb

@@ -1,13 +1,13 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.21.0"
+    TOKENIZERS_VERSION = "0.22.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
     # and reduces the extension size by about half
     def from_pretrained(identifier, revision: "main", auth_token: nil)
-      require "cgi"
+      require "cgi/escape"
       require "digest"
       require "fileutils"
       require "json"

data/lib/tokenizers/trainers/unigram_trainer.rb

@@ -1,15 +1,16 @@
 module Tokenizers
   module Trainers
     class UnigramTrainer
-      def self.new(vocab_size: 8000,
-                   show_progress: true,
-                   special_tokens: [],
-                   initial_alphabet: [],
-                   shrinking_factor: 0.75,
-                   unk_token: nil,
-                   max_piece_length: 16,
-                   n_sub_iterations: 2)
-
+      def self.new(
+        vocab_size: 8000,
+        show_progress: true,
+        special_tokens: [],
+        initial_alphabet: [],
+        shrinking_factor: 0.75,
+        unk_token: nil,
+        max_piece_length: 16,
+        n_sub_iterations: 2
+      )
         _new({
           vocab_size: vocab_size,
           show_progress: show_progress,

data/lib/tokenizers/trainers/word_piece_trainer.rb

@@ -1,15 +1,16 @@
 module Tokenizers
   module Trainers
     class WordPieceTrainer
-      def self.new(vocab_size: 30000,
-                   min_frequency: 0,
-                   show_progress: true,
-                   special_tokens: [],
-                   limit_alphabet: nil,
-                   initial_alphabet: [],
-                   continuing_subword_prefix: "##",
-                   end_of_word_suffix: nil)
-
+      def self.new(
+        vocab_size: 30000,
+        min_frequency: 0,
+        show_progress: true,
+        special_tokens: [],
+        limit_alphabet: nil,
+        initial_alphabet: [],
+        continuing_subword_prefix: "##",
+        end_of_word_suffix: nil
+      )
         _new({
           vocab_size: vocab_size,
           min_frequency: min_frequency,

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.4"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.6.0
 platform: ruby
 authors:
 - Andrew Kane
 bindir: bin
 cert_chain: []
-date: 2024-12-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -91,14 +91,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby
 test_files: []