tokenizers 0.5.1-aarch64-linux-musl → 0.5.3-aarch64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8c4ff155c60f5ed4f81ec5254c88179ed786149d2fe0698326c7a0c6be86cb2
-  data.tar.gz: 9f2ec69eeb9ec75e47d54213421b7dca8b7b26c21daad4f42a4475dfda5b1588
+  metadata.gz: '08700fb9bdcad5fd8f72a201ff9fb9d4d5738d0994ed395f8870696e3d2c8420'
+  data.tar.gz: ae92e4266649e977717c38ecfa0e7435e2a4cd5439f4156ebe9d60db05251413
 SHA512:
-  metadata.gz: 92f5b14b17ef2d41d45dfcba81202103a63ce3f6e2fd5d575519c1faa845de7b31ae28971bac36108bd0d2e0d4cc0a30b694fd48cfb093ca47604041e03e8719
-  data.tar.gz: 64eef8d1a8e7d7990085b1f8558e9abb17e501bc70dc35509a2b42034a4ddf6011eb9967f5288b994a6193e3e4e4f690292296e6c90d9825b0a924d7e41c39c3
+  metadata.gz: a1bbba481e1d2f8c65ced5020d2d1b53526d9b31f7927a62fd39fafddf254dbcd87a525860f1734e91c4ac102453b449832995428c5626181c0d8e4506401d9d
+  data.tar.gz: 93d5e32e26ab8e9277b148e331d35f9673708fed1d3dd92cc0ae9d9651d4fe7aeda1c850c3326b91899d4ce65fd2efc2806a6e8fd21e0a1846d0594614a48cc0

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
+## 0.5.2 (2024-08-26)
+
+- Added `from_str` method to `Tokenizer`
+- Added `model` and `model=` methods to `Tokenizer`
+- Added `decoder`, `pre_tokenizer`, `post_processor`, and `normalizer` methods to `Tokenizer`
+- Added `decode` method to `Decoder`
+
 ## 0.5.1 (2024-08-13)
 
 - Updated Tokenizers to 0.20.0

data/Cargo.lock

@@ -57,9 +57,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.1.8"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cexpr"
@@ -301,9 +304,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libloading"
@@ -475,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -545,18 +548,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565"
+checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48"
+checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -622,18 +625,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -642,9 +645,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
 dependencies = [
  "itoa",
  "memchr",
@@ -690,9 +693,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -721,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.1"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/LICENSE-THIRD-PARTY.txt

@@ -26,7 +26,7 @@ byteorder v1.5.0
 https://github.com/BurntSushi/byteorder
 Unlicense OR MIT
 
-cc v1.1.8
+cc v1.1.15
 https://github.com/rust-lang/cc-rs
 MIT OR Apache-2.0
 
@@ -130,7 +130,7 @@ lazycell v1.3.0
 https://github.com/indiv0/lazycell
 MIT/Apache-2.0
 
-libc v0.2.155
+libc v0.2.158
 https://github.com/rust-lang/libc
 MIT OR Apache-2.0
 
@@ -214,7 +214,7 @@ proc-macro2 v1.0.86
 https://github.com/dtolnay/proc-macro2
 MIT OR Apache-2.0
 
-quote v1.0.36
+quote v1.0.37
 https://github.com/dtolnay/quote
 MIT OR Apache-2.0
 
@@ -242,11 +242,11 @@ rayon-core v1.12.1
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.100
+rb-sys v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.100
+rb-sys-build v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -278,15 +278,15 @@ seq-macro v0.3.5
 https://github.com/dtolnay/seq-macro
 MIT OR Apache-2.0
 
-serde v1.0.205
+serde v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_derive v1.0.205
+serde_derive v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_json v1.0.122
+serde_json v1.0.127
 https://github.com/serde-rs/json
 MIT OR Apache-2.0
 
@@ -310,7 +310,7 @@ strsim v0.11.1
 https://github.com/rapidfuzz/strsim-rs
 MIT
 
-syn v2.0.72
+syn v2.0.76
 https://github.com/dtolnay/syn
 MIT OR Apache-2.0
 

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.1"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: aarch64-linux-musl
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/3.3/tokenizers.so
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb