tokenizers 0.2.0-x86_64-darwin → 0.2.1-x86_64-darwin
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/Cargo.lock +304 -175
- data/Cargo.toml +0 -1
- data/LICENSE-THIRD-PARTY.txt +9654 -6176
- data/lib/tokenizers/2.7/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.0/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.1/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.2/tokenizers.bundle +0 -0
- data/lib/tokenizers/version.rb +1 -1
- metadata +9 -8
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
data/lib/tokenizers/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tokenizers
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.1
|
|
5
5
|
platform: x86_64-darwin
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Kane
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-01-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
|
-
description:
|
|
13
|
+
description:
|
|
14
14
|
email: andrew@ankane.org
|
|
15
15
|
executables: []
|
|
16
16
|
extensions: []
|
|
@@ -26,13 +26,14 @@ files:
|
|
|
26
26
|
- lib/tokenizers/2.7/tokenizers.bundle
|
|
27
27
|
- lib/tokenizers/3.0/tokenizers.bundle
|
|
28
28
|
- lib/tokenizers/3.1/tokenizers.bundle
|
|
29
|
+
- lib/tokenizers/3.2/tokenizers.bundle
|
|
29
30
|
- lib/tokenizers/char_bpe_tokenizer.rb
|
|
30
31
|
- lib/tokenizers/version.rb
|
|
31
32
|
homepage: https://github.com/ankane/tokenizers-ruby
|
|
32
33
|
licenses:
|
|
33
34
|
- Apache-2.0
|
|
34
35
|
metadata: {}
|
|
35
|
-
post_install_message:
|
|
36
|
+
post_install_message:
|
|
36
37
|
rdoc_options: []
|
|
37
38
|
require_paths:
|
|
38
39
|
- lib
|
|
@@ -43,15 +44,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
43
44
|
version: '2.7'
|
|
44
45
|
- - "<"
|
|
45
46
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: 3.
|
|
47
|
+
version: 3.3.dev
|
|
47
48
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
48
49
|
requirements:
|
|
49
50
|
- - ">="
|
|
50
51
|
- !ruby/object:Gem::Version
|
|
51
52
|
version: '0'
|
|
52
53
|
requirements: []
|
|
53
|
-
rubygems_version: 3.3
|
|
54
|
-
signing_key:
|
|
54
|
+
rubygems_version: 3.4.3
|
|
55
|
+
signing_key:
|
|
55
56
|
specification_version: 4
|
|
56
57
|
summary: Fast state-of-the-art tokenizers for Ruby
|
|
57
58
|
test_files: []
|