tokenizers 0.5.1-x86_64-linux → 0.5.3-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1ac97056f64913b16bd3d63a97b0dfa11daad03adabf7bc09fc1f1f59c3459d
-  data.tar.gz: 022cb74e0108763c23f26715accff37148809f564813409db81b9c85a8273fd4
+  metadata.gz: c5a23e4520e57dd74f97517733d0916919493906a2fa82cfc037a76ff203113d
+  data.tar.gz: d2cbc6265ac9d74a5724178da71d961ec81a0872c371839e22f1a3c2b097d637
 SHA512:
-  metadata.gz: f00e700813f6dffb830497943538d4d9d45fe4679b38a5132dd18564088fd644340b7c7f937a0ce9904f001d2914f3898cd63435ec3750c6b1b51198a5e0e207
-  data.tar.gz: 0c4b13af17bc407d788d5b263a024ccbd92130d1174ac1aa78412d2458923c9c6e32bcc562a8f68188867631321b6a33399fb594211e7f55f3964386d8f95700
+  metadata.gz: de6f0db2c89c0f3476ac3a1f11d4c43387e8bdb9bd3b5790ed2be0c91ed9733ff9db8009d9f51c39d786f83e8e6bdde3af72e13d406ce6e644103fa260d5e97f
+  data.tar.gz: f9ebdc7f298ff9c8f39f63acbd861515b3c7a3aa7929973498be040aae27f9235ee3256f72b66f13a3bccfe658db9e10cb922bef7b77a531dd5e0d1babebd25f

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
+## 0.5.2 (2024-08-26)
+
+- Added `from_str` method to `Tokenizer`
+- Added `model` and `model=` methods to `Tokenizer`
+- Added `decoder`, `pre_tokenizer`, `post_processor`, and `normalizer` methods to `Tokenizer`
+- Added `decode` method to `Decoder`
+
 ## 0.5.1 (2024-08-13)
 
 - Updated Tokenizers to 0.20.0

data/Cargo.lock

@@ -57,9 +57,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.1.8"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cexpr"
@@ -301,9 +304,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libloading"
@@ -475,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -545,18 +548,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565"
+checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48"
+checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -622,18 +625,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -642,9 +645,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
 dependencies = [
  "itoa",
  "memchr",
@@ -690,9 +693,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -721,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.1"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/LICENSE-THIRD-PARTY.txt

@@ -26,7 +26,7 @@ byteorder v1.5.0
 https://github.com/BurntSushi/byteorder
 Unlicense OR MIT
 
-cc v1.1.8
+cc v1.1.15
 https://github.com/rust-lang/cc-rs
 MIT OR Apache-2.0
 
@@ -130,7 +130,7 @@ lazycell v1.3.0
 https://github.com/indiv0/lazycell
 MIT/Apache-2.0
 
-libc v0.2.155
+libc v0.2.158
 https://github.com/rust-lang/libc
 MIT OR Apache-2.0
 
@@ -214,7 +214,7 @@ proc-macro2 v1.0.86
 https://github.com/dtolnay/proc-macro2
 MIT OR Apache-2.0
 
-quote v1.0.36
+quote v1.0.37
 https://github.com/dtolnay/quote
 MIT OR Apache-2.0
 
@@ -242,11 +242,11 @@ rayon-core v1.12.1
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.100
+rb-sys v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.100
+rb-sys-build v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -278,15 +278,15 @@ seq-macro v0.3.5
 https://github.com/dtolnay/seq-macro
 MIT OR Apache-2.0
 
-serde v1.0.205
+serde v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_derive v1.0.205
+serde_derive v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_json v1.0.122
+serde_json v1.0.127
 https://github.com/serde-rs/json
 MIT OR Apache-2.0
 
@@ -310,7 +310,7 @@ strsim v0.11.1
 https://github.com/rapidfuzz/strsim-rs
 MIT
 
-syn v2.0.72
+syn v2.0.76
 https://github.com/dtolnay/syn
 MIT OR Apache-2.0
 

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.1"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: x86_64-linux
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/3.3/tokenizers.so
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb