tokenizers 0.2.0-arm64-darwin
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +22 -0
- data/Cargo.lock +1908 -0
- data/Cargo.toml +11 -0
- data/LICENSE-THIRD-PARTY.txt +31404 -0
- data/LICENSE.txt +202 -0
- data/README.md +65 -0
- data/lib/tokenizers/2.7/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.0/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.1/tokenizers.bundle +0 -0
- data/lib/tokenizers/char_bpe_tokenizer.rb +19 -0
- data/lib/tokenizers/version.rb +3 -0
- data/lib/tokenizers.rb +18 -0
- metadata +57 -0
data/Cargo.toml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
[workspace]
|
|
2
|
+
members = ["ext/tokenizers"]
|
|
3
|
+
|
|
4
|
+
[profile.release]
|
|
5
|
+
strip = true
|
|
6
|
+
|
|
7
|
+
[patch.crates-io]
|
|
8
|
+
magnus-macros = { git = "https://github.com/matsadler/magnus" }
|
|
9
|
+
number_prefix = { git = "https://github.com/ankane/rust-number-prefix", branch = "license-file" }
|
|
10
|
+
rb-sys-env = { git = "https://github.com/oxidize-rb/rb-sys" }
|
|
11
|
+
tokenizers = { git = "https://github.com/huggingface/tokenizers" }
|