tokenizers 0.2.0-x86_64-linux → 0.2.2-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -0
  3. data/Cargo.lock +105 -1104
  4. data/Cargo.toml +0 -6
  5. data/LICENSE-THIRD-PARTY.txt +5726 -19226
  6. data/README.md +1 -1
  7. data/lib/tokenizers/2.7/tokenizers.so +0 -0
  8. data/lib/tokenizers/3.0/tokenizers.so +0 -0
  9. data/lib/tokenizers/3.1/tokenizers.so +0 -0
  10. data/lib/tokenizers/3.2/tokenizers.so +0 -0
  11. data/lib/tokenizers/from_pretrained.rb +119 -0
  12. data/lib/tokenizers/version.rb +1 -1
  13. data/lib/tokenizers.rb +2 -3
  14. metadata +10 -8
data/Cargo.toml CHANGED
@@ -3,9 +3,3 @@ members = ["ext/tokenizers"]
3
3
 
4
4
  [profile.release]
5
5
  strip = true
6
-
7
- [patch.crates-io]
8
- magnus-macros = { git = "https://github.com/matsadler/magnus" }
9
- number_prefix = { git = "https://github.com/ankane/rust-number-prefix", branch = "license-file" }
10
- rb-sys-env = { git = "https://github.com/oxidize-rb/rb-sys" }
11
- tokenizers = { git = "https://github.com/huggingface/tokenizers" }