tokenizers 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 556d084ad69603fa0d5ff61c4a03864d56fbe4d525d706390851d1a5761d173a
-  data.tar.gz: 4dd1ab9a2de88f60135aca333c7d48557fa1ad6c8e31c61065717e77a37f0cdd
+  metadata.gz: 30c4558340092b3fe4b60adbfed6b042810e27df9a62bd8fe828c3a2c9b5cf7a
+  data.tar.gz: bc97136598b82cdb47b0d50de4ead4b5afd8500dc52b487496f3179dd48ecee8
 SHA512:
-  metadata.gz: 3d6b7209189aeec8846a50f0e65a24e1089e7e0998d1f3d07026446c18b2b0c139d5b7566ca2fc721f72b67519c50595144d6deb3603d032fb41d20c7bc8c6e7
-  data.tar.gz: a7f04aa13c7cbc3c3973408140fc9f4c3330dacd150d4edd33f16e6c218e03a949c9d44ff03b7b4b6e63eedd2623b2abf417a647a8f2260aa67d64594f06c6fc
+  metadata.gz: '0184d588343d823b0a2942828c0a496e131b5dfbae475d46ed7ebb2f3e89e5fd6d420705e88b31293331b247920c209653d0590b3aad618aab583a6a9ff49c8a'
+  data.tar.gz: a7c590677a968516ae075fb46a5153e301b93e2bd13cf372d5cf020c4bd0c9c0cde7a7118e708e853a61c42a8957fcec73afa32e8a2eebd517943254905d0621

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
 ## 0.5.2 (2024-08-26)
 
 - Added `from_str` method to `Tokenizer`

data/Cargo.lock

@@ -724,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.2"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.5.2"
+version = "0.5.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"

data/ext/tokenizers/src/lib.rs

@@ -15,7 +15,7 @@ mod utils;
 
 use encoding::RbEncoding;
 use error::RbError;
-use tokenizer::RbTokenizer;
+use tokenizer::{RbAddedToken, RbTokenizer};
 use utils::RbRegex;
 
 use magnus::{function, method, prelude::*, value::Lazy, Error, RModule, Ruby};
@@ -79,6 +79,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("num_special_tokens_to_add", method!(RbTokenizer::num_special_tokens_to_add, 1))?;
     class.define_method("_vocab", method!(RbTokenizer::vocab, 1))?;
     class.define_method("_vocab_size", method!(RbTokenizer::vocab_size, 1))?;
+    class.define_method("added_tokens_decoder", method!(RbTokenizer::get_added_tokens_decoder, 0))?;
     class.define_method("_to_s", method!(RbTokenizer::to_str, 1))?;
 
     let class = module.define_class("Encoding", ruby.class_object())?;
@@ -109,6 +110,15 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     let class = module.define_class("Regex", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbRegex::new, 1))?;
 
+    let class = module.define_class("AddedToken", ruby.class_object())?;
+    class.define_singleton_method("_new", function!(RbAddedToken::new, 2))?;
+    class.define_method("content", method!(RbAddedToken::get_content, 0))?;
+    class.define_method("rstrip", method!(RbAddedToken::get_rstrip, 0))?;
+    class.define_method("lstrip", method!(RbAddedToken::get_lstrip, 0))?;
+    class.define_method("single_word", method!(RbAddedToken::get_single_word, 0))?;
+    class.define_method("normalized", method!(RbAddedToken::get_normalized, 0))?;
+    class.define_method("special", method!(RbAddedToken::get_special, 0))?;
+
     let models = module.define_module("Models")?;
     let pre_tokenizers = module.define_module("PreTokenizers")?;
     let decoders = module.define_module("Decoders")?;

data/ext/tokenizers/src/tokenizer.rs

@@ -22,9 +22,10 @@ use super::processors::RbPostProcessor;
 use super::trainers::RbTrainer;
 use super::{RbError, RbResult};
 
+#[magnus::wrap(class = "Tokenizers::AddedToken")]
 pub struct RbAddedToken {
     pub content: String,
-    pub is_special_token: bool,
+    pub special: bool,
     pub single_word: Option<bool>,
     pub lstrip: Option<bool>,
     pub rstrip: Option<bool>,
@@ -32,10 +33,10 @@ pub struct RbAddedToken {
 }
 
 impl RbAddedToken {
-    pub fn from<S: Into<String>>(content: S, is_special_token: Option<bool>) -> Self {
+    pub fn from<S: Into<String>>(content: S, special: Option<bool>) -> Self {
         Self {
             content: content.into(),
-            is_special_token: is_special_token.unwrap_or(false),
+            special: special.unwrap_or(false),
             single_word: None,
             lstrip: None,
             rstrip: None,
@@ -44,7 +45,7 @@ impl RbAddedToken {
     }
 
     pub fn get_token(&self) -> tk::tokenizer::AddedToken {
-        let mut token = tk::AddedToken::from(&self.content, self.is_special_token);
+        let mut token = tk::AddedToken::from(&self.content, self.special);
 
         if let Some(sw) = self.single_word {
             token = token.single_word(sw);
@@ -71,11 +72,73 @@ impl From<tk::AddedToken> for RbAddedToken {
             lstrip: Some(token.lstrip),
             rstrip: Some(token.rstrip),
             normalized: Some(token.normalized),
-            is_special_token: !token.normalized,
+            special: !token.normalized,
         }
     }
 }
 
+impl RbAddedToken {
+    pub fn new(content: Option<String>, kwargs: RHash) -> RbResult<Self> {
+        let mut token = RbAddedToken::from(content.unwrap_or("".to_string()), None);
+
+        let value: Value = kwargs.delete(Symbol::new("single_word"))?;
+        if !value.is_nil() {
+            token.single_word = TryConvert::try_convert(value)?;
+        }
+
+        let value: Value = kwargs.delete(Symbol::new("lstrip"))?;
+        if !value.is_nil() {
+            token.lstrip = TryConvert::try_convert(value)?;
+        }
+
+        let value: Value = kwargs.delete(Symbol::new("rstrip"))?;
+        if !value.is_nil() {
+            token.rstrip = TryConvert::try_convert(value)?;
+        }
+
+        let value: Value = kwargs.delete(Symbol::new("normalized"))?;
+        if !value.is_nil() {
+            token.normalized = TryConvert::try_convert(value)?;
+        }
+
+        let value: Value = kwargs.delete(Symbol::new("special"))?;
+        if !value.is_nil() {
+            token.special = TryConvert::try_convert(value)?;
+        }
+
+        if !kwargs.is_empty() {
+            // TODO improve message
+            return Err(Error::new(exception::arg_error(), "unknown keyword"));
+        }
+
+        Ok(token)
+    }
+
+    pub fn get_content(&self) -> String {
+        self.content.to_string()
+    }
+
+    pub fn get_rstrip(&self) -> bool {
+        self.get_token().rstrip
+    }
+
+    pub fn get_lstrip(&self) -> bool {
+        self.get_token().lstrip
+    }
+
+    pub fn get_single_word(&self) -> bool {
+        self.get_token().single_word
+    }
+
+    pub fn get_normalized(&self) -> bool {
+        self.get_token().normalized
+    }
+
+    pub fn get_special(&self) -> bool {
+        self.get_token().special
+    }
+}
+
 struct TextInputSequence<'s>(tk::InputSequence<'s>);
 
 impl<'s> TryConvert for TextInputSequence<'s> {
@@ -536,4 +599,14 @@ impl RbTokenizer {
     pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
         self.tokenizer.borrow().get_vocab_size(with_added_tokens)
     }
+
+    pub fn get_added_tokens_decoder(&self) -> RbResult<RHash> {
+        let sorted_map = RHash::new();
+
+        for (key, value) in self.tokenizer.borrow().get_added_tokens_decoder() {
+            sorted_map.aset::<u32, RbAddedToken>(key, value.into())?;
+        }
+
+        Ok(sorted_map)
+    }
 }

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -52,6 +52,7 @@ files:
 - ext/tokenizers/src/utils/normalization.rs
 - ext/tokenizers/src/utils/regex.rs
 - lib/tokenizers.rb
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb
@@ -100,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby