tokenizers 0.4.2 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 954a1e29eea94d08df1d0a0c667d4554f0f979947e2c714114f5aa16db19ad11
-  data.tar.gz: 910f44e5c05115dce6ee1fe1070138ba1a826a1142a5f43ff104990687cc7814
+  metadata.gz: fe2e8b2ec97ac4dabb4e401cb426079c065be420bf8da41c9484b3d38290589c
+  data.tar.gz: bba3b9d9d94a278e3f5189fda48b3b3a7f1ff369b66ae8b85945e92a2d8b90ed
 SHA512:
-  metadata.gz: 8d88bcfaacae964414ad175f4f27f4e20bdf8131a4a97deabdc732f75c12ca2ad956ff3c3f0967d607b46748a8abe7fc1c94538ba9e9ddaa3abaff7132ec15ca
-  data.tar.gz: 95c8e8225da0070aa947ac8c6436e90f490e5dff033f532f3c2d61af63c7c5beff36398b6a78b4c6e69eca7c4d64834ea43d8c988e3ccdae8ee30e0c84f126bd
+  metadata.gz: fd5876cc95c22b917f1a5715f9b9651f714291f017c00f83316ad2c3cc48c8819d4de70288b4067570c9f729cca9d06cbfa8562996a61ac694e202268a58cdea
+  data.tar.gz: cb793a73aabafe933ef8d7259b7a36848b1f38cacdf20ef3f6f8554671b7de6ece53b3c14f46f368f540ecbab5dfca087f730e99c22caf84c7b35d539b07b633

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.4.4 (2024-02-27)
+
+- Updated Tokenizers to 0.15.2
+
+## 0.4.3 (2024-01-03)
+
+- Added support for Ruby 3.3
+
 ## 0.4.2 (2023-11-16)
 
 - Updated Tokenizers to 0.15.0

data/Cargo.lock

@@ -25,11 +25,11 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "bindgen"
-version = "0.62.0"
+version = "0.69.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6720a8b7b2d39dd533285ed438d458f65b31b5c257e6ac7bb3d7e82844dd722"
+checksum = "9ffcebc3849946a7170a05992aac39da343a90676ab392c51a4280981d6379c2"
 dependencies = [
- "bitflags",
+ "bitflags 2.4.1",
  "cexpr",
  "clang-sys",
  "lazy_static",
@@ -40,7 +40,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 1.0.109",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -49,6 +49,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -274,6 +280,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.6"
@@ -335,9 +350,9 @@ checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
 
 [[package]]
 name = "magnus"
-version = "0.6.0"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68e9585bfe236e88e6b10b6d8eb5349bd0e0009f3f9dff8d2e99a82601b33743"
+checksum = "4778544796676e8428e9c622460ebf284bea52d8b10db3aeb449d8b5e61b3a13"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -426,7 +441,7 @@ version = "6.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "libc",
  "once_cell",
  "onig_sys",
@@ -537,7 +552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
 dependencies = [
  "either",
- "itertools",
+ "itertools 0.11.0",
  "rayon",
 ]
 
@@ -553,18 +568,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.79"
+version = "0.9.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "939fb78db3e4f26665c1d4c7b91ca66d3578335a19aba552d4a6445811d07072"
+checksum = "0d197f2c03751ef006f29d593d22aa9068c9c358e04ca503afea0329c366147c"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.79"
+version = "0.9.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "335a95eb0420d52fa94ef12019df3c2c250c6b19cbb3c60bd05cb7e9c362072c"
+checksum = "2b50caf8fd028f12abe00d6debe2ae2adf6202c9ca3caa59487eda710d90fa28"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -572,7 +587,7 @@ dependencies = [
  "quote",
  "regex",
  "shell-words",
- "syn 1.0.109",
+ "syn 2.0.38",
 ]
 
 [[package]]
@@ -590,7 +605,7 @@ dependencies = [
  "aho-corasick",
  "memchr",
  "regex-automata",
- "regex-syntax",
+ "regex-syntax 0.7.5",
 ]
 
 [[package]]
@@ -601,7 +616,7 @@ checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-syntax 0.7.5",
 ]
 
 [[package]]
@@ -610,6 +625,12 @@ version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
 
+[[package]]
+name = "regex-syntax"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -745,26 +766,26 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "magnus",
  "onig",
  "serde",
- "tokenizers 0.15.0",
+ "tokenizers 0.15.2",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.15.0"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "062b8a9613d6017633b80fb55fbb33f1aff006c36225a3025630753398034b3c"
+checksum = "3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d"
 dependencies = [
  "aho-corasick",
  "derive_builder",
  "esaxx-rs",
  "getrandom",
  "indicatif",
- "itertools",
+ "itertools 0.12.1",
  "lazy_static",
  "log",
  "macro_rules_attribute",
@@ -775,7 +796,7 @@ dependencies = [
  "rayon",
  "rayon-cond",
  "regex",
- "regex-syntax",
+ "regex-syntax 0.8.2",
  "serde",
  "serde_json",
  "spm_precompiled",

data/README.md

@@ -2,7 +2,7 @@
 
 :slightly_smiling_face: Fast state-of-the-art [tokenizers](https://github.com/huggingface/tokenizers) for Ruby
 
-[![Build Status](https://github.com/ankane/tokenizers-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/tokenizers-ruby/actions)
+[![Build Status](https://github.com/ankane/tokenizers-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/tokenizers-ruby/actions)
 
 ## Installation
 
@@ -34,15 +34,51 @@ Decode
 tokenizer.decode(ids)
 ```
 
-Load a tokenizer from files
+## Training
+
+Create a tokenizer
 
 ```ruby
-tokenizer = Tokenizers::CharBPETokenizer.new("vocab.json", "merges.txt")
+tokenizer = Tokenizers::Tokenizer.new(Tokenizers::Models::BPE.new(unk_token: "[UNK]"))
 ```
 
-## Training
+Set the pre-tokenizer
+
+```ruby
+tokenizer.pre_tokenizer = Tokenizers::PreTokenizers::Whitespace.new
+```
+
+Train the tokenizer ([example data](https://huggingface.co/docs/tokenizers/quicktour#build-a-tokenizer-from-scratch))
+
+```ruby
+trainer = Tokenizers::Trainers::BpeTrainer.new(special_tokens: ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+tokenizer.train(["wiki.train.raw", "wiki.valid.raw", "wiki.test.raw"], trainer)
+```
+
+Encode
+
+```ruby
+output = tokenizer.encode("Hello, y'all! How are you 😁 ?")
+output.tokens
+```
+
+Save the tokenizer to a file
+
+```ruby
+tokenizer.save("tokenizer.json")
+```
+
+Load a tokenizer from a file
+
+```ruby
+tokenizer = Tokenizers.from_file("tokenizer.json")
+```
+
+Check out the [Quicktour](https://huggingface.co/docs/tokenizers/quicktour) and equivalent [Ruby code](https://github.com/ankane/tokenizers-ruby/blob/master/test/quicktour_test.rb#L8) for more info
+
+## API
 
-Check out the [Quicktour](https://huggingface.co/docs/tokenizers/quicktour) and equivalent [Ruby code](https://github.com/ankane/tokenizers-ruby/blob/master/test/quicktour_test.rb#L8)
+This library follows the [Tokenizers Python API](https://huggingface.co/docs/tokenizers/index). You can follow Python tutorials and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
 
 ## History
 

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.4.2"
+version = "0.4.4"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -16,6 +16,6 @@ onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.15.0" # also update in from_pretrained.rb
+version = "=0.15.2" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/trainers.rs

@@ -77,11 +77,11 @@ impl RbTrainer {
         setter!(self, BpeTrainer, vocab_size, vocab_size);
     }
 
-    fn bpe_trainer_min_frequency(&self) -> u32 {
+    fn bpe_trainer_min_frequency(&self) -> u64 {
         getter!(self, BpeTrainer, min_frequency)
     }
 
-    fn bpe_trainer_set_min_frequency(&self, freq: u32) {
+    fn bpe_trainer_set_min_frequency(&self, freq: u64) {
         setter!(self, BpeTrainer, min_frequency, freq);
     }
 
@@ -235,11 +235,11 @@ impl RbTrainer {
         setter!(self, WordLevelTrainer, vocab_size, vocab_size);
     }
 
-    fn word_level_trainer_min_frequency(&self) -> u32 {
+    fn word_level_trainer_min_frequency(&self) -> u64 {
         getter!(self, WordLevelTrainer, min_frequency)
     }
 
-    fn word_level_trainer_set_min_frequency(&self, freq: u32) {
+    fn word_level_trainer_set_min_frequency(&self, freq: u64) {
         setter!(self, WordLevelTrainer, min_frequency, freq);
     }
 
@@ -289,11 +289,11 @@ impl RbTrainer {
         setter!(self, WordPieceTrainer, @set_vocab_size, vocab_size);
     }
 
-    fn word_piece_trainer_min_frequency(&self) -> u32 {
+    fn word_piece_trainer_min_frequency(&self) -> u64 {
         getter!(self, WordPieceTrainer, min_frequency())
     }
 
-    fn word_piece_trainer_set_min_frequency(&self, freq: u32) {
+    fn word_piece_trainer_set_min_frequency(&self, freq: u64) {
         setter!(self, WordPieceTrainer, @set_min_frequency, freq);
     }
 

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.15.0"
+    TOKENIZERS_VERSION = "0.15.2"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.4.2"
+  VERSION = "0.4.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-16 00:00:00.000000000 Z
+date: 2024-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby