tokenizers 0.5.1-x86_64-linux-musl → 0.5.3-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33811f5d5774e9715d7730def78a304df68801d3dcb044831943db2708c33c5a
-  data.tar.gz: cec6f7c1bdf1ea43014284e1716cd5d51fb4474270782706ebdb43938748df07
+  metadata.gz: fce129f3910c54922d726c2dfa4f501eee8e41cc35374636663219a14fae5556
+  data.tar.gz: a8c7931b5eb452264cd4c5901d194bc0ba955d8857078e5216544edd15db0d5f
 SHA512:
-  metadata.gz: a68caf15b7de717546fbedd09921d5d80357dad6354a785fa2a729cdcc78f1ea3001f001947688f4669b0e85a7db07ee147714e6c5ce2d3af103f17ddf256335
-  data.tar.gz: 8f37e091a9e7d816b949d53da9c3b7711d9e1122f380daed0f0a2fa33551e0d46d78cb7e5437e0039e5e0fc3e7834440c8460e9e600a3b1bd71a69e45490afb2
+  metadata.gz: '08fa0d532d72de71ace9d071cce5c10d08a6c7f119381c6476a7eeeadd5968a0f37a30897c91e6e1dc975122d4899161ed9bff181169eed0940f1d7fa0363405'
+  data.tar.gz: 1ab0ded21ce97e9bc5de491d42c880fbb2e186a9725546db2fb86ef0fcebb64d8e92ace94ce2cbda0540cc07a6c806fae8ae359d82b71c2998c3799a59b266a1

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
+## 0.5.2 (2024-08-26)
+
+- Added `from_str` method to `Tokenizer`
+- Added `model` and `model=` methods to `Tokenizer`
+- Added `decoder`, `pre_tokenizer`, `post_processor`, and `normalizer` methods to `Tokenizer`
+- Added `decode` method to `Decoder`
+
 ## 0.5.1 (2024-08-13)
 
 - Updated Tokenizers to 0.20.0

data/Cargo.lock

@@ -57,9 +57,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.1.8"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cexpr"
@@ -301,9 +304,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libloading"
@@ -475,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -545,18 +548,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565"
+checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48"
+checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -622,18 +625,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -642,9 +645,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
 dependencies = [
  "itoa",
  "memchr",
@@ -690,9 +693,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -721,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.1"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/LICENSE-THIRD-PARTY.txt

@@ -26,7 +26,7 @@ byteorder v1.5.0
 https://github.com/BurntSushi/byteorder
 Unlicense OR MIT
 
-cc v1.1.8
+cc v1.1.15
 https://github.com/rust-lang/cc-rs
 MIT OR Apache-2.0
 
@@ -130,7 +130,7 @@ lazycell v1.3.0
 https://github.com/indiv0/lazycell
 MIT/Apache-2.0
 
-libc v0.2.155
+libc v0.2.158
 https://github.com/rust-lang/libc
 MIT OR Apache-2.0
 
@@ -214,7 +214,7 @@ proc-macro2 v1.0.86
 https://github.com/dtolnay/proc-macro2
 MIT OR Apache-2.0
 
-quote v1.0.36
+quote v1.0.37
 https://github.com/dtolnay/quote
 MIT OR Apache-2.0
 
@@ -242,11 +242,11 @@ rayon-core v1.12.1
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.100
+rb-sys v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.100
+rb-sys-build v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -278,15 +278,15 @@ seq-macro v0.3.5
 https://github.com/dtolnay/seq-macro
 MIT OR Apache-2.0
 
-serde v1.0.205
+serde v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_derive v1.0.205
+serde_derive v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_json v1.0.122
+serde_json v1.0.127
 https://github.com/serde-rs/json
 MIT OR Apache-2.0
 
@@ -310,7 +310,7 @@ strsim v0.11.1
 https://github.com/rapidfuzz/strsim-rs
 MIT
 
-syn v2.0.72
+syn v2.0.76
 https://github.com/dtolnay/syn
 MIT OR Apache-2.0
 

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.1"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: x86_64-linux-musl
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/3.3/tokenizers.so
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb