tokenizers 0.2.1-arm64-darwin → 0.2.2-arm64-darwin

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/Cargo.lock +124 -1252
  4. data/Cargo.toml +0 -5
  5. data/LICENSE-THIRD-PARTY.txt +6585 -23511
  6. data/README.md +1 -1
  7. data/lib/tokenizers/2.7/tokenizers.bundle +0 -0
  8. data/lib/tokenizers/3.0/tokenizers.bundle +0 -0
  9. data/lib/tokenizers/3.1/tokenizers.bundle +0 -0
  10. data/lib/tokenizers/3.2/tokenizers.bundle +0 -0
  11. data/lib/tokenizers/from_pretrained.rb +119 -0
  12. data/lib/tokenizers/version.rb +1 -1
  13. data/lib/tokenizers.rb +2 -3
  14. metadata +3 -2
data/Cargo.toml CHANGED
@@ -3,8 +3,3 @@ members = ["ext/tokenizers"]
3
3
 
4
4
  [profile.release]
5
5
  strip = true
6
-
7
- [patch.crates-io]
8
- number_prefix = { git = "https://github.com/ankane/rust-number-prefix", branch = "license-file" }
9
- rb-sys-env = { git = "https://github.com/oxidize-rb/rb-sys" }
10
- tokenizers = { git = "https://github.com/huggingface/tokenizers" }