tokenizers 0.4.4 → 0.5.1

data/ext/tokenizers/Cargo.toml

@@ -1,21 +1,21 @@
 [package]
 name = "tokenizers"
-version = "0.4.4"
+version = "0.5.1"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
-rust-version = "1.62.0"
+rust-version = "1.63.0"
 publish = false
 
 [lib]
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = "0.6"
+magnus = "0.7"
 onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.15.2" # also update in from_pretrained.rb
+version = "=0.20.0" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/decoders.rs

@@ -1,5 +1,6 @@
 use std::sync::{Arc, RwLock};
 
+use crate::pre_tokenizers::from_string;
 use magnus::value::Lazy;
 use magnus::{
     data_type_builder, function, method, Class, DataType, DataTypeFunctions, Module, Object, RClass, RModule,
@@ -11,7 +12,7 @@ use tk::decoders::byte_fallback::ByteFallback;
 use tk::decoders::byte_level::ByteLevel;
 use tk::decoders::ctc::CTC;
 use tk::decoders::fuse::Fuse;
-use tk::decoders::metaspace::Metaspace;
+use tk::decoders::metaspace::{Metaspace, PrependScheme};
 use tk::decoders::strip::Strip;
 use tk::decoders::wordpiece::WordPiece;
 use tk::decoders::DecoderWrapper;
@@ -126,12 +127,29 @@ impl RbDecoder {
         setter!(self, Metaspace, @set_replacement, replacement);
     }
 
-    pub fn metaspace_add_prefix_space(&self) -> bool {
-        getter!(self, Metaspace, add_prefix_space)
+    pub fn metaspace_split(&self) -> bool {
+        getter!(self, Metaspace, get_split())
     }
 
-    pub fn metaspace_set_add_prefix_space(&self, add_prefix_space: bool) {
-        setter!(self, Metaspace, add_prefix_space, add_prefix_space);
+    pub fn metaspace_set_split(&self, split: bool) {
+        setter!(self, Metaspace, @set_split, split);
+    }
+
+    pub fn metaspace_prepend_scheme(&self) -> String {
+        // Assuming Metaspace has a method to get the prepend_scheme as a string
+        let scheme: PrependScheme = getter!(self, Metaspace, get_prepend_scheme());
+        match scheme {
+            PrependScheme::First => "first",
+            PrependScheme::Never => "never",
+            PrependScheme::Always => "always",
+        }
+        .to_string()
+    }
+
+    pub fn metaspace_set_prepend_scheme(&self, prepend_scheme: String) -> RbResult<()> {
+        let scheme = from_string(prepend_scheme)?;
+        setter!(self, Metaspace, @set_prepend_scheme, scheme);
+        Ok(())
     }
 
     pub fn word_piece_cleanup(&self) -> bool {
@@ -194,8 +212,9 @@ impl RbFuse {
 pub struct RbMetaspaceDecoder {}
 
 impl RbMetaspaceDecoder {
-    pub fn new(replacement: char, add_prefix_space: bool) -> RbDecoder {
-        Metaspace::new(replacement, add_prefix_space).into()
+    pub fn new(replacement: char, prepend_scheme: String, split: bool) -> RbResult<RbDecoder> {
+        let prepend_scheme = from_string(prepend_scheme)?;
+        Ok(Metaspace::new(replacement, prepend_scheme, split).into())
     }
 }
 
@@ -364,11 +383,13 @@ pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("new", function!(RbFuse::new, 0))?;
 
     let class = module.define_class("Metaspace", decoder)?;
-    class.define_singleton_method("_new", function!(RbMetaspaceDecoder::new, 2))?;
-    class.define_method("add_prefix_space", method!(RbDecoder::metaspace_add_prefix_space, 0))?;
-    class.define_method("add_prefix_space=", method!(RbDecoder::metaspace_set_add_prefix_space, 1))?;
+    class.define_singleton_method("_new", function!(RbMetaspaceDecoder::new, 3))?;
+    class.define_method("prepend_scheme", method!(RbDecoder::metaspace_prepend_scheme, 0))?;
+    class.define_method("prepend_scheme=", method!(RbDecoder::metaspace_set_prepend_scheme, 1))?;
     class.define_method("replacement", method!(RbDecoder::metaspace_replacement, 0))?;
     class.define_method("replacement=", method!(RbDecoder::metaspace_set_replacement, 1))?;
+    class.define_method("split", method!(RbDecoder::metaspace_split, 0))?;
+    class.define_method("split=", method!(RbDecoder::metaspace_set_split, 1))?;
 
     let class = module.define_class("Replace", decoder)?;
     class.define_singleton_method("new", function!(RbReplaceDecoder::new, 2))?;

data/ext/tokenizers/src/normalizers.rs

@@ -222,8 +222,8 @@ pub struct RbSequence {}
 impl RbSequence {
     fn new(normalizers: RArray) -> RbResult<RbNormalizer> {
         let mut sequence = Vec::with_capacity(normalizers.len());
-        for n in normalizers.each() {
-            let normalizer: &RbNormalizer = TryConvert::try_convert(n?)?;
+        for n in normalizers.into_iter() {
+            let normalizer: &RbNormalizer = TryConvert::try_convert(n)?;
             match &normalizer.normalizer {
                 RbNormalizerTypeWrapper::Sequence(inner) => sequence.extend(inner.iter().cloned()),
                 RbNormalizerTypeWrapper::Single(inner) => sequence.push(inner.clone()),

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, RwLock};
 
 use magnus::{
-    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object,
+    data_type_builder, exception, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error, Module, Object,
     RArray, RClass, RModule, Ruby, TryConvert, TypedData,
 };
 
@@ -12,7 +12,7 @@ use tk::pre_tokenizers::bert::BertPreTokenizer;
 use tk::pre_tokenizers::byte_level::ByteLevel;
 use tk::pre_tokenizers::delimiter::CharDelimiterSplit;
 use tk::pre_tokenizers::digits::Digits;
-use tk::pre_tokenizers::metaspace::Metaspace;
+use tk::pre_tokenizers::metaspace::{Metaspace, PrependScheme};
 use tk::pre_tokenizers::punctuation::Punctuation;
 use tk::pre_tokenizers::split::Split;
 use tk::pre_tokenizers::unicode_scripts::UnicodeScripts;
@@ -118,14 +118,6 @@ impl RbPreTokenizer {
         setter!(self, Digits, individual_digits, individual_digits);
     }
 
-    fn metaspace_add_prefix_space(&self) -> bool {
-        getter!(self, Metaspace, add_prefix_space)
-    }
-
-    fn metaspace_set_add_prefix_space(&self, add_prefix_space: bool) {
-        setter!(self, Metaspace, add_prefix_space, add_prefix_space);
-    }
-
     fn metaspace_replacement(&self) -> String {
         getter!(self, Metaspace, get_replacement().to_string())
     }
@@ -133,6 +125,31 @@ impl RbPreTokenizer {
     fn metaspace_set_replacement(&self, replacement: char) {
         setter!(self, Metaspace, @set_replacement, replacement);
     }
+
+    fn metaspace_split(&self) -> bool {
+        getter!(self, Metaspace, get_split())
+    }
+
+    fn metaspace_set_split(&self, split: bool) {
+        setter!(self, Metaspace, @set_split, split);
+    }
+
+    fn metaspace_prepend_scheme(&self) -> String {
+        // Assuming Metaspace has a method to get the prepend_scheme as a string
+        let scheme: PrependScheme = getter!(self, Metaspace, get_prepend_scheme());
+        match scheme {
+            PrependScheme::First => "first",
+            PrependScheme::Never => "never",
+            PrependScheme::Always => "always",
+        }
+        .to_string()
+    }
+
+    fn metaspace_set_prepend_scheme(&self, prepend_scheme: String) -> RbResult<()> {
+        let scheme = from_string(prepend_scheme)?;
+        setter!(self, Metaspace, @set_prepend_scheme, scheme);
+        Ok(())
+    }
 }
 
 impl PreTokenizer for RbPreTokenizer {
@@ -180,9 +197,11 @@ pub struct RbMetaspace {}
 impl RbMetaspace {
     fn new(
         replacement: char,
-        add_prefix_space: bool,
-    ) -> RbPreTokenizer {
-        Metaspace::new(replacement, add_prefix_space).into()
+        prepend_scheme: String,
+        split: bool,
+    ) -> RbResult<RbPreTokenizer> {
+        let prepend_scheme = from_string(prepend_scheme)?;
+        Ok(Metaspace::new(replacement, prepend_scheme, split).into())
     }
 }
 
@@ -239,8 +258,8 @@ pub struct RbSequence {}
 impl RbSequence {
     fn new(pre_tokenizers: RArray) -> RbResult<RbPreTokenizer> {
         let mut sequence = Vec::with_capacity(pre_tokenizers.len());
-        for n in pre_tokenizers.each() {
-            let pretokenizer: &RbPreTokenizer = TryConvert::try_convert(n?)?;
+        for n in pre_tokenizers.into_iter() {
+            let pretokenizer: &RbPreTokenizer = TryConvert::try_convert(n)?;
             match &pretokenizer.pretok {
                 RbPreTokenizerTypeWrapper::Sequence(inner) => {
                     sequence.extend(inner.iter().cloned())
@@ -252,6 +271,21 @@ impl RbSequence {
     }
 }
 
+pub(crate) fn from_string(string: String) -> RbResult<PrependScheme> {
+    let scheme = match string.as_str() {
+        "first" => PrependScheme::First,
+        "never" => PrependScheme::Never,
+        "always" => PrependScheme::Always,
+        _ => {
+            return Err(Error::new(exception::arg_error(), format!(
+                "{} is an unknown variant, should be one of ['first', 'never', 'always']",
+                string
+            )));
+        }
+    };
+    Ok(scheme)
+}
+
 #[derive(Clone, Deserialize)]
 #[serde(untagged)]
 pub(crate) enum RbPreTokenizerWrapper {
@@ -465,11 +499,13 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_method("individual_digits=", method!(RbPreTokenizer::digits_set_individual_digits, 1))?;
 
     let class = module.define_class("Metaspace", pre_tokenizer)?;
-    class.define_singleton_method("_new", function!(RbMetaspace::new, 2))?;
-    class.define_method("add_prefix_space", method!(RbPreTokenizer::metaspace_add_prefix_space, 0))?;
-    class.define_method("add_prefix_space=", method!(RbPreTokenizer::metaspace_set_add_prefix_space, 1))?;
+    class.define_singleton_method("_new", function!(RbMetaspace::new, 3))?;
+    class.define_method("prepend_scheme", method!(RbPreTokenizer::metaspace_prepend_scheme, 0))?;
+    class.define_method("prepend_scheme=", method!(RbPreTokenizer::metaspace_set_prepend_scheme, 1))?;
     class.define_method("replacement", method!(RbPreTokenizer::metaspace_replacement, 0))?;
     class.define_method("replacement=", method!(RbPreTokenizer::metaspace_set_replacement, 1))?;
+    class.define_method("split", method!(RbPreTokenizer::metaspace_split, 0))?;
+    class.define_method("split=", method!(RbPreTokenizer::metaspace_set_split, 1))?;
 
     let class = module.define_class("Punctuation", pre_tokenizer)?;
     class.define_singleton_method("_new", function!(RbPunctuation::new, 1))?;

data/ext/tokenizers/src/tokenizer.rs

@@ -282,12 +282,12 @@ impl RbTokenizer {
         add_special_tokens: bool,
     ) -> RbResult<RArray> {
         let input: Vec<tk::EncodeInput> = input
-            .each()
+            .into_iter()
             .map(|o| {
                 let input: tk::EncodeInput = if is_pretokenized {
-                    PreTokenizedEncodeInput::try_convert(o?)?.into()
+                    PreTokenizedEncodeInput::try_convert(o)?.into()
                 } else {
-                    TextEncodeInput::try_convert(o?)?.into()
+                    TextEncodeInput::try_convert(o)?.into()
                 };
                 Ok(input)
             })
@@ -319,26 +319,26 @@ impl RbTokenizer {
             .map_err(RbError::from)
     }
 
-    pub fn set_decoder(&self, decoder: &RbDecoder) {
-        self.tokenizer.borrow_mut().with_decoder(decoder.clone());
+    pub fn set_decoder(&self, decoder: Option<&RbDecoder>) {
+        self.tokenizer.borrow_mut().with_decoder(decoder.cloned());
     }
 
-    pub fn set_pre_tokenizer(&self, pretok: &RbPreTokenizer) {
+    pub fn set_pre_tokenizer(&self, pretok: Option<&RbPreTokenizer>) {
         self.tokenizer
             .borrow_mut()
-            .with_pre_tokenizer(pretok.clone());
+            .with_pre_tokenizer(pretok.cloned());
     }
 
-    pub fn set_post_processor(&self, processor: &RbPostProcessor) {
+    pub fn set_post_processor(&self, processor: Option<&RbPostProcessor>) {
         self.tokenizer
             .borrow_mut()
-            .with_post_processor(processor.clone());
+            .with_post_processor(processor.cloned());
     }
 
-    pub fn set_normalizer(&self, normalizer: &RbNormalizer) {
+    pub fn set_normalizer(&self, normalizer: Option<&RbNormalizer>) {
         self.tokenizer
             .borrow_mut()
-            .with_normalizer(normalizer.clone());
+            .with_normalizer(normalizer.cloned());
     }
 
     pub fn token_to_id(&self, token: String) -> Option<u32> {

data/ext/tokenizers/src/trainers.rs

@@ -110,9 +110,9 @@ impl RbTrainer {
             BpeTrainer,
             special_tokens,
             special_tokens
-                .each()
+                .into_iter()
                 .map(|token| {
-                    if let Ok(content) = String::try_convert(token?) {
+                    if let Ok(content) = String::try_convert(token) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -197,9 +197,9 @@ impl RbTrainer {
             UnigramTrainer,
             special_tokens,
             special_tokens
-                .each()
+                .into_iter()
                 .map(|token| {
-                    if let Ok(content) = String::try_convert(token?) {
+                    if let Ok(content) = String::try_convert(token) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -268,9 +268,9 @@ impl RbTrainer {
             WordLevelTrainer,
             special_tokens,
             special_tokens
-                .each()
+                .into_iter()
                 .map(|token| {
-                    if let Ok(content) = String::try_convert(token?) {
+                    if let Ok(content) = String::try_convert(token) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -322,9 +322,9 @@ impl RbTrainer {
             WordPieceTrainer,
             @set_special_tokens,
             special_tokens
-                .each()
+                .into_iter()
                 .map(|token| {
-                    if let Ok(content) = String::try_convert(token?) {
+                    if let Ok(content) = String::try_convert(token) {
                         Ok(RbAddedToken::from(content, Some(true)).get_token())
                     } else {
                         todo!()
@@ -398,9 +398,9 @@ impl RbBpeTrainer {
         if !value.is_nil() {
             builder = builder.special_tokens(
                 RArray::try_convert(value)?
-                    .each()
+                    .into_iter()
                     .map(|token| {
-                        if let Ok(content) = String::try_convert(token?) {
+                        if let Ok(content) = String::try_convert(token) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -466,9 +466,9 @@ impl RbUnigramTrainer {
         if !value.is_nil() {
             builder.special_tokens(
                 RArray::try_convert(value)?
-                    .each()
+                    .into_iter()
                     .map(|token| {
-                        if let Ok(content) = String::try_convert(token?) {
+                        if let Ok(content) = String::try_convert(token) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -540,9 +540,9 @@ impl RbWordLevelTrainer {
         if !value.is_nil() {
             builder.special_tokens(
                 RArray::try_convert(value)?
-                    .each()
+                    .into_iter()
                     .map(|token| {
-                        if let Ok(content) = String::try_convert(token?) {
+                        if let Ok(content) = String::try_convert(token) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()
@@ -581,9 +581,9 @@ impl RbWordPieceTrainer {
         if !value.is_nil() {
             builder = builder.special_tokens(
                 RArray::try_convert(value)?
-                    .each()
+                    .into_iter()
                     .map(|token| {
-                        if let Ok(content) = String::try_convert(token?) {
+                        if let Ok(content) = String::try_convert(token) {
                             Ok(RbAddedToken::from(content, Some(true)).get_token())
                         } else {
                             todo!()

data/lib/tokenizers/decoders/metaspace.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module Decoders
     class Metaspace
-      def self.new(replacement: "\u2581", add_prefix_space: true)
-        _new(replacement, add_prefix_space)
+      def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
+        _new(replacement, prepend_scheme, split)
       end
     end
   end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.15.2"
+    TOKENIZERS_VERSION = "0.20.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
@@ -67,7 +67,7 @@ module Tokenizers
         end
       end
 
-      options[:content_length_proc] = -> (_) { puts "Downloading..." }
+      options[:content_length_proc] = ->(_) { puts "Downloading..." }
 
       # string options are headers
       tempfile = URI.parse(url).open(headers.merge(options))

data/lib/tokenizers/pre_tokenizers/metaspace.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module PreTokenizers
     class Metaspace
-      def self.new(replacement: "\u2581", add_prefix_space: true)
-        _new(replacement, add_prefix_space)
+      def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
+        _new(replacement, prepend_scheme, split)
       end
     end
   end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.4.4"
+  VERSION = "0.5.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-27 00:00:00.000000000 Z
+date: 2024-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -93,14 +93,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby