tokenizers 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +5 -0
  3. data/Cargo.lock +124 -53
  4. data/ext/tokenizers/Cargo.toml +4 -3
  5. data/ext/tokenizers/src/encoding.rs +10 -8
  6. data/ext/tokenizers/src/models.rs +37 -24
  7. data/ext/tokenizers/src/normalizers.rs +1 -2
  8. data/ext/tokenizers/src/pre_tokenizers.rs +5 -5
  9. data/ext/tokenizers/src/tokenizer.rs +61 -49
  10. data/ext/tokenizers/src/trainers.rs +60 -50
  11. data/ext/tokenizers/src/utils/normalization.rs +3 -2
  12. data/ext/tokenizers/src/utils/regex.rs +5 -4
  13. data/lib/tokenizers/from_pretrained.rb +2 -2
  14. data/lib/tokenizers/trainers/unigram_trainer.rb +10 -9
  15. data/lib/tokenizers/trainers/word_piece_trainer.rb +10 -9
  16. data/lib/tokenizers/version.rb +1 -1
  17. metadata +3 -3
data/lib/tokenizers/from_pretrained.rb CHANGED
@@ -1,13 +1,13 @@
1
1
  module Tokenizers
2
2
  module FromPretrained
3
3
  # for user agent
4
- TOKENIZERS_VERSION = "0.21.1"
4
+ TOKENIZERS_VERSION = "0.22.0"
5
5
 
6
6
  # use Ruby for downloads
7
7
  # this avoids the need to vendor OpenSSL on Linux
8
8
  # and reduces the extension size by about half
9
9
  def from_pretrained(identifier, revision: "main", auth_token: nil)
10
- require "cgi"
10
+ require "cgi/escape"
11
11
  require "digest"
12
12
  require "fileutils"
13
13
  require "json"
data/lib/tokenizers/trainers/unigram_trainer.rb CHANGED
@@ -1,15 +1,16 @@
1
1
  module Tokenizers
2
2
  module Trainers
3
3
  class UnigramTrainer
4
- def self.new(vocab_size: 8000,
5
- show_progress: true,
6
- special_tokens: [],
7
- initial_alphabet: [],
8
- shrinking_factor: 0.75,
9
- unk_token: nil,
10
- max_piece_length: 16,
11
- n_sub_iterations: 2)
12
-
4
+ def self.new(
5
+ vocab_size: 8000,
6
+ show_progress: true,
7
+ special_tokens: [],
8
+ initial_alphabet: [],
9
+ shrinking_factor: 0.75,
10
+ unk_token: nil,
11
+ max_piece_length: 16,
12
+ n_sub_iterations: 2
13
+ )
13
14
  _new({
14
15
  vocab_size: vocab_size,
15
16
  show_progress: show_progress,
data/lib/tokenizers/trainers/word_piece_trainer.rb CHANGED
@@ -1,15 +1,16 @@
1
1
  module Tokenizers
2
2
  module Trainers
3
3
  class WordPieceTrainer
4
- def self.new(vocab_size: 30000,
5
- min_frequency: 0,
6
- show_progress: true,
7
- special_tokens: [],
8
- limit_alphabet: nil,
9
- initial_alphabet: [],
10
- continuing_subword_prefix: "##",
11
- end_of_word_suffix: nil)
12
-
4
+ def self.new(
5
+ vocab_size: 30000,
6
+ min_frequency: 0,
7
+ show_progress: true,
8
+ special_tokens: [],
9
+ limit_alphabet: nil,
10
+ initial_alphabet: [],
11
+ continuing_subword_prefix: "##",
12
+ end_of_word_suffix: nil
13
+ )
13
14
  _new({
14
15
  vocab_size: vocab_size,
15
16
  min_frequency: min_frequency,
data/lib/tokenizers/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Tokenizers
2
- VERSION = "0.5.5"
2
+ VERSION = "0.6.0"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tokenizers
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.5
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
@@ -91,14 +91,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
91
91
  requirements:
92
92
  - - ">="
93
93
  - !ruby/object:Gem::Version
94
- version: '3.1'
94
+ version: '3.2'
95
95
  required_rubygems_version: !ruby/object:Gem::Requirement
96
96
  requirements:
97
97
  - - ">="
98
98
  - !ruby/object:Gem::Version
99
99
  version: '0'
100
100
  requirements: []
101
- rubygems_version: 3.6.7
101
+ rubygems_version: 3.6.9
102
102
  specification_version: 4
103
103
  summary: Fast state-of-the-art tokenizers for Ruby
104
104
  test_files: []