tokenizers 0.5.1-aarch64-linux → 0.5.2-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1dd5881d17992979c172a0b3b06d819d4d7b37f44143bf3e35c0d9e6098de56
-  data.tar.gz: e031c6b86c37c315ac94f645e73f5aef28ef1a20d649dd443cc63ebde00ac69a
+  metadata.gz: 3b8639c894fdb33fdb9741593827cc8c4f00df954be28c56f954b202f082b7fd
+  data.tar.gz: 86bf064a1a6b29a75d6b8d400a8bb7a8de1178bc291f62a3c7d52f11a3412127
 SHA512:
-  metadata.gz: d96f470e7c925bfd568db5d4ee094c9ed7ffd93d004a3f588d2596d1924deb8f38cc787459ba948909060a0395f40d93474c544920be0c974f9f4ead2d46ca0f
-  data.tar.gz: 30606972b0c329a0534304451b11a89e9f9fec4a35fb67700378aaec86186e0093c1d1b2fc3cb514eec30b3ca335442ff9c922ffe724390b335af6e24dd5901a
+  metadata.gz: bafc8c988420672150b4584378da5f4ebdac3ac9698dd9bc6d53e798a0c064e54aacc7d235c278f73c00e51e3d5d35d5e08232cb92db9f4e9fc50967df4fb075
+  data.tar.gz: 4afb2e39c3c6d8add55b58af240d1ff1473788cab7530c0b16dd826561520784da15bd5553313cf07574171362c41681880a622bc1910280045a2d28ec2abee4

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.5.2 (2024-08-26)
+
+- Added `from_str` method to `Tokenizer`
+- Added `model` and `model=` methods to `Tokenizer`
+- Added `decoder`, `pre_tokenizer`, `post_processor`, and `normalizer` methods to `Tokenizer`
+- Added `decode` method to `Decoder`
+
 ## 0.5.1 (2024-08-13)
 
 - Updated Tokenizers to 0.20.0

data/Cargo.lock

@@ -57,9 +57,12 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.1.8"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cexpr"
@@ -301,9 +304,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libloading"
@@ -475,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -545,18 +548,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565"
+checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.100"
+version = "0.9.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48"
+checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -622,18 +625,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.205"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -642,9 +645,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
 dependencies = [
  "itoa",
  "memchr",
@@ -690,9 +693,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -721,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "magnus",
  "onig",

data/LICENSE-THIRD-PARTY.txt

@@ -26,7 +26,7 @@ byteorder v1.5.0
 https://github.com/BurntSushi/byteorder
 Unlicense OR MIT
 
-cc v1.1.8
+cc v1.1.15
 https://github.com/rust-lang/cc-rs
 MIT OR Apache-2.0
 
@@ -130,7 +130,7 @@ lazycell v1.3.0
 https://github.com/indiv0/lazycell
 MIT/Apache-2.0
 
-libc v0.2.155
+libc v0.2.158
 https://github.com/rust-lang/libc
 MIT OR Apache-2.0
 
@@ -214,7 +214,7 @@ proc-macro2 v1.0.86
 https://github.com/dtolnay/proc-macro2
 MIT OR Apache-2.0
 
-quote v1.0.36
+quote v1.0.37
 https://github.com/dtolnay/quote
 MIT OR Apache-2.0
 
@@ -242,11 +242,11 @@ rayon-core v1.12.1
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.100
+rb-sys v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.100
+rb-sys-build v0.9.102
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -278,15 +278,15 @@ seq-macro v0.3.5
 https://github.com/dtolnay/seq-macro
 MIT OR Apache-2.0
 
-serde v1.0.205
+serde v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_derive v1.0.205
+serde_derive v1.0.209
 https://serde.rs
 MIT OR Apache-2.0
 
-serde_json v1.0.122
+serde_json v1.0.127
 https://github.com/serde-rs/json
 MIT OR Apache-2.0
 
@@ -310,7 +310,7 @@ strsim v0.11.1
 https://github.com/rapidfuzz/strsim-rs
 MIT
 
-syn v2.0.72
+syn v2.0.76
 https://github.com/dtolnay/syn
 MIT OR Apache-2.0
 

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: aarch64-linux
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2024-08-26 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org