tokenizers 0.3.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -5
  3. data/Cargo.lock +12 -7
  4. data/ext/tokenizers/Cargo.toml +1 -1
  5. data/lib/tokenizers/version.rb +1 -1
  6. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b298d2a912a3d7fe32b9dc60036822b39ff463ad9739fec9ca00809922c6f45c
4
- data.tar.gz: 8491e0c8532d9eefe5ae1c0940f57ae3986480fb9460897d7612e8937d97e384
3
+ metadata.gz: 11fdc53989bc2285096bc5eb4a971426e153375ada43bc8c51d2a191b42fe02d
4
+ data.tar.gz: 92d4d8bef2c4013d5cf0d55bff30bc070ca6e59430f87b96f9147075216a1c1d
5
5
  SHA512:
6
- metadata.gz: 3616d92731fe9c7e4214166392b05734b10dec0e004106c01c756cab544491523b596b5d0a44ef8a47708719fa32839b86a1038a31104fae6d3de3a2e17ccca9
7
- data.tar.gz: 88a375508473b97ae0b30dfb12a2c930a20477b003e5f2977a670904dab077ab4e2771b98dd3a446fc9b9c417e886f3b7355bd7eec527eb898251c9c60a0337c
6
+ metadata.gz: 7c91c33078b6c5b23a6080908fa184e2222922f35bd6e1c439329525f4554f939cf98ca4dcd3017d858cdb6ec2a96f6671ce5dc425ca643d482b49b04af00f4c
7
+ data.tar.gz: 74aad6c458792570ace93107aacccb8a9df7b61228dd062bd7bcaf015a53d4f6b4b1fcace09c69bc959bd9b04598d315344d4edc7e6abbe1603f3b92d29ab711
data/CHANGELOG.md CHANGED
@@ -1,8 +1,12 @@
1
- ## 0.3.1 (2022-02-08)
1
+ ## 0.3.2 (2023-03-06)
2
+
3
+ - Added precompiled gem for Linux x86-64 MUSL
4
+
5
+ ## 0.3.1 (2023-02-08)
2
6
 
3
7
  - Fixed error with Ruby 2.7
4
8
 
5
- ## 0.3.0 (2022-02-07)
9
+ ## 0.3.0 (2023-02-07)
6
10
 
7
11
  - Added support for training tokenizers
8
12
  - Added more methods to `Tokenizer`
@@ -11,20 +15,20 @@
11
15
  - Changed `encode` method to include special tokens by default
12
16
  - Changed how offsets are calculated for strings with multibyte characters
13
17
 
14
- ## 0.2.3 (2022-01-22)
18
+ ## 0.2.3 (2023-01-22)
15
19
 
16
20
  - Added `add_special_tokens` option to `encode` method
17
21
  - Added warning about `encode` method including special tokens by default in 0.3.0
18
22
  - Added more methods to `Encoding`
19
23
  - Fixed error with precompiled gem on Mac ARM
20
24
 
21
- ## 0.2.2 (2022-01-15)
25
+ ## 0.2.2 (2023-01-15)
22
26
 
23
27
  - Added precompiled gem for Linux ARM
24
28
  - Added `from_file` method
25
29
  - Fixed error with precompiled gem on Linux x86-64
26
30
 
27
- ## 0.2.1 (2022-01-12)
31
+ ## 0.2.1 (2023-01-12)
28
32
 
29
33
  - Added support for Ruby 3.2
30
34
 
data/Cargo.lock CHANGED
@@ -353,7 +353,8 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
353
353
  [[package]]
354
354
  name = "magnus"
355
355
  version = "0.5.0"
356
- source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
356
+ source = "registry+https://github.com/rust-lang/crates.io-index"
357
+ checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
357
358
  dependencies = [
358
359
  "magnus-macros",
359
360
  "rb-sys",
@@ -362,8 +363,9 @@ dependencies = [
362
363
 
363
364
  [[package]]
364
365
  name = "magnus-macros"
365
- version = "0.3.0"
366
- source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
366
+ version = "0.4.0"
367
+ source = "registry+https://github.com/rust-lang/crates.io-index"
368
+ checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
367
369
  dependencies = [
368
370
  "proc-macro2",
369
371
  "quote",
@@ -552,22 +554,25 @@ dependencies = [
552
554
 
553
555
  [[package]]
554
556
  name = "rb-sys"
555
- version = "0.9.64"
557
+ version = "0.9.65"
556
558
  source = "registry+https://github.com/rust-lang/crates.io-index"
557
- checksum = "cc8945662df8083245deda89e236647173cc7ad750f481ddcd7bbfd3afe3fa5e"
559
+ checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
558
560
  dependencies = [
559
561
  "rb-sys-build",
560
562
  ]
561
563
 
562
564
  [[package]]
563
565
  name = "rb-sys-build"
564
- version = "0.9.64"
566
+ version = "0.9.65"
565
567
  source = "registry+https://github.com/rust-lang/crates.io-index"
566
- checksum = "ae8c3cdf9edc3908ee1555b7a1bca58ee1b499439b32cd1c1ec3e66736a8df48"
568
+ checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
567
569
  dependencies = [
568
570
  "bindgen",
571
+ "lazy_static",
572
+ "quote",
569
573
  "regex",
570
574
  "shell-words",
575
+ "syn",
571
576
  ]
572
577
 
573
578
  [[package]]
data/ext/tokenizers/Cargo.toml CHANGED
@@ -10,7 +10,7 @@ publish = false
10
10
  crate-type = ["cdylib"]
11
11
 
12
12
  [dependencies]
13
- magnus = { git = "https://github.com/matsadler/magnus" }
13
+ magnus = "0.5"
14
14
  onig = { version = "6.0", default-features = false }
15
15
  serde = { version = "1.0", features = ["rc", "derive"] }
16
16
 
data/lib/tokenizers/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Tokenizers
2
- VERSION = "0.3.1"
2
+ VERSION = "0.3.2"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tokenizers
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-09 00:00:00.000000000 Z
11
+ date: 2023-03-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rb_sys