tokenizers 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4846b5d3dc0fe8f5828ddffe46908b1f3812ebf6a03a939ca0395ad7748533bb
-  data.tar.gz: 259795bfa6b13a36f62ab2ffb65e9feabd460e01efe9f59e7d6017c6dcd9b9b0
+  metadata.gz: 556d084ad69603fa0d5ff61c4a03864d56fbe4d525d706390851d1a5761d173a
+  data.tar.gz: 4dd1ab9a2de88f60135aca333c7d48557fa1ad6c8e31c61065717e77a37f0cdd
 SHA512:
-  metadata.gz: 90f55feb8ceec81815bb61b7773e6f67924eb6d47869e5da67d579e2ac9df6a48fddee5e97f5a028e3fa1e39941f3bfe6ec6c04cf49fd1b87f18bade54911231
-  data.tar.gz: 9c1895b43222494b393f3fbddaa6e78216e025f3d3dddebf0c0311d2d897b282a16b8b0044aacb2466790b0a93c8d01099b25d662750b96d5798d0a4a927267b
+  metadata.gz: 3d6b7209189aeec8846a50f0e65a24e1089e7e0998d1f3d07026446c18b2b0c139d5b7566ca2fc721f72b67519c50595144d6deb3603d032fb41d20c7bc8c6e7
+  data.tar.gz: a7f04aa13c7cbc3c3973408140fc9f4c3330dacd150d4edd33f16e6c218e03a949c9d44ff03b7b4b6e63eedd2623b2abf417a647a8f2260aa67d64594f06c6fc

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.5.2 (2024-08-26)
+
+- Added `from_str` method to `Tokenizer`
+- Added `model` and `model=` methods to `Tokenizer`
+- Added `decoder`, `pre_tokenizer`, `post_processor`, and `normalizer` methods to `Tokenizer`
+- Added `decode` method to `Decoder`
+
 ## 0.5.1 (2024-08-13)
 
 - Updated Tokenizers to 0.20.0

data/Cargo.lock

@@ -57,9 +57,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.1.8"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cexpr"
@@ -301,9 +304,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libloading"
@@ -475,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -545,18 +548,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565"
+checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48"
+checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -622,18 +625,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -642,9 +645,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
 dependencies = [
  "itoa",
  "memchr",
@@ -690,9 +693,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -721,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "magnus",
  "onig",

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.5.1"
+version = "0.5.2"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"

data/ext/tokenizers/src/decoders.rs

@@ -34,6 +34,12 @@ impl Decoder for RbDecoder {
     }
 }
 
+impl RbDecoder {
+    pub fn decode(&self, tokens: Vec<String>) -> RbResult<String> {
+        self.decoder.decode(tokens).map_err(RbError::from)
+    }
+}
+
 macro_rules! getter {
     ($self: ident, $variant: ident, $($name: tt)+) => {{
         let decoder = &$self.decoder;
@@ -358,6 +364,7 @@ unsafe impl TypedData for RbDecoder {
 
 pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     let decoder = module.define_class("Decoder", ruby.class_object())?;
+    decoder.define_method("decode", method!(RbDecoder::decode, 1))?;
 
     let class = module.define_class("BPEDecoder", decoder)?;
     class.define_singleton_method("_new", function!(RbBPEDecoder::new, 1))?;

data/ext/tokenizers/src/lib.rs

@@ -42,6 +42,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
 
     let class = module.define_class("Tokenizer", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbTokenizer::from_model, 1))?;
+    class.define_singleton_method("from_str", function!(RbTokenizer::from_str, 1))?;
     class.define_singleton_method("from_file", function!(RbTokenizer::from_file, 1))?;
     class.define_method(
         "add_special_tokens",
@@ -54,12 +55,18 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("_encode_batch", method!(RbTokenizer::encode_batch, 3))?;
     class.define_method("_decode", method!(RbTokenizer::decode, 2))?;
     class.define_method("_decode_batch", method!(RbTokenizer::decode_batch, 2))?;
+    class.define_method("model", method!(RbTokenizer::get_model, 0))?;
+    class.define_method("model=", method!(RbTokenizer::set_model,1))?;
+    class.define_method("decoder", method!(RbTokenizer::get_decoder, 0))?;
     class.define_method("decoder=", method!(RbTokenizer::set_decoder, 1))?;
+    class.define_method("pre_tokenizer", method!(RbTokenizer::get_pre_tokenizer, 0))?;
     class.define_method("pre_tokenizer=", method!(RbTokenizer::set_pre_tokenizer, 1))?;
+    class.define_method("post_processor", method!(RbTokenizer::get_post_processor, 0))?;
     class.define_method(
         "post_processor=",
         method!(RbTokenizer::set_post_processor, 1),
     )?;
+    class.define_method("normalizer", method!(RbTokenizer::get_normalizer, 0))?;
     class.define_method("normalizer=", method!(RbTokenizer::set_normalizer, 1))?;
     class.define_method("token_to_id", method!(RbTokenizer::token_to_id, 1))?;
     class.define_method("id_to_token", method!(RbTokenizer::id_to_token, 1))?;

data/ext/tokenizers/src/tokenizer.rs

@@ -1,9 +1,10 @@
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::str::FromStr;
 
 use magnus::prelude::*;
-use magnus::{exception, Error, RArray, RHash, Symbol, TryConvert, Value};
+use magnus::{exception, Error, RArray, RHash, RString, Symbol, TryConvert, Value};
 use tk::tokenizer::{
     Model, PaddingDirection, PaddingParams, PaddingStrategy,
     TruncationDirection, TruncationParams, TruncationStrategy, TokenizerImpl
@@ -203,6 +204,14 @@ impl RbTokenizer {
         RbTokenizer::new(TokenizerImpl::new(model.clone()))
     }
 
+    pub fn from_str(json: RString) -> RbResult<Self> {
+        Tokenizer::from_str(unsafe { json.as_str()? })
+            .map(|v| RbTokenizer {
+                tokenizer: RefCell::new(v),
+            })
+            .map_err(RbError::from)
+    }
+
     pub fn from_file(path: PathBuf) -> RbResult<Self> {
         Tokenizer::from_file(path)
             .map(|v| RbTokenizer {
@@ -319,22 +328,46 @@ impl RbTokenizer {
             .map_err(RbError::from)
     }
 
+    pub fn get_model(&self) -> RbModel {
+        self.tokenizer.borrow().get_model().clone()
+    }
+
+    pub fn set_model(&self, model: &RbModel) {
+        self.tokenizer.borrow_mut().with_model(model.clone());
+    }
+
+    pub fn get_decoder(&self) -> Option<RbDecoder> {
+        self.tokenizer.borrow().get_decoder().cloned()
+    }
+
     pub fn set_decoder(&self, decoder: Option<&RbDecoder>) {
         self.tokenizer.borrow_mut().with_decoder(decoder.cloned());
     }
 
+    pub fn get_pre_tokenizer(&self) -> Option<RbPreTokenizer> {
+        self.tokenizer.borrow().get_pre_tokenizer().cloned()
+    }
+
     pub fn set_pre_tokenizer(&self, pretok: Option<&RbPreTokenizer>) {
         self.tokenizer
             .borrow_mut()
             .with_pre_tokenizer(pretok.cloned());
     }
 
+    pub fn get_post_processor(&self) -> Option<RbPostProcessor> {
+        self.tokenizer.borrow().get_post_processor().cloned()
+    }
+
     pub fn set_post_processor(&self, processor: Option<&RbPostProcessor>) {
         self.tokenizer
             .borrow_mut()
             .with_post_processor(processor.cloned());
     }
 
+    pub fn get_normalizer(&self) -> Option<RbNormalizer> {
+        self.tokenizer.borrow().get_normalizer().cloned()
+    }
+
     pub fn set_normalizer(&self, normalizer: Option<&RbNormalizer>) {
         self.tokenizer
             .borrow_mut()

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2024-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys