tokenizers 0.2.1-x86_64-darwin → 0.2.2-x86_64-darwin
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/Cargo.lock +124 -1252
- data/Cargo.toml +0 -5
- data/LICENSE-THIRD-PARTY.txt +6585 -23511
- data/README.md +1 -1
- data/lib/tokenizers/2.7/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.0/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.1/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.2/tokenizers.bundle +0 -0
- data/lib/tokenizers/from_pretrained.rb +119 -0
- data/lib/tokenizers/version.rb +1 -1
- data/lib/tokenizers.rb +2 -3
- metadata +3 -2
data/Cargo.toml
CHANGED
@@ -3,8 +3,3 @@ members = ["ext/tokenizers"]
|
|
3
3
|
|
4
4
|
[profile.release]
|
5
5
|
strip = true
|
6
|
-
|
7
|
-
[patch.crates-io]
|
8
|
-
number_prefix = { git = "https://github.com/ankane/rust-number-prefix", branch = "license-file" }
|
9
|
-
rb-sys-env = { git = "https://github.com/oxidize-rb/rb-sys" }
|
10
|
-
tokenizers = { git = "https://github.com/huggingface/tokenizers" }
|