tokenizers 0.3.1-x86_64-linux → 0.3.3-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -5
- data/Cargo.lock +129 -89
- data/LICENSE-THIRD-PARTY.txt +1656 -1504
- data/lib/tokenizers/2.7/tokenizers.so +0 -0
- data/lib/tokenizers/3.0/tokenizers.so +0 -0
- data/lib/tokenizers/3.1/tokenizers.so +0 -0
- data/lib/tokenizers/3.2/tokenizers.so +0 -0
- data/lib/tokenizers/decoders/strip.rb +9 -0
- data/lib/tokenizers/from_pretrained.rb +1 -1
- data/lib/tokenizers/normalizers/prepend.rb +9 -0
- data/lib/tokenizers/version.rb +1 -1
- data/lib/tokenizers.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4041eb3f8e79ca25397620670357fce958823e6fb49c0ef7c3968f11b6ce99e1
|
|
4
|
+
data.tar.gz: 17952cf748eff3f62165bfedc304753f127cfde805e2be3e8cfd7adcfd5e878c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a3b4c35630874860ddc85b09b75c03f119dfd622dcf7ddd47fd682b547ba338a08656ec52b4bcad409ebdc6c128dbd640f78f639c4a11839ecc28dc4289a0da6
|
|
7
|
+
data.tar.gz: 8c90c7e8a1e67ce65c2b545bbb879e4d277da908f6fcbc3ddc029711f36e2ac4d4e73d0bf76d71f333eaceb6a612ee9261708d80c507b0d597d8f924562ec912
|
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,18 @@
|
|
|
1
|
-
## 0.3.
|
|
1
|
+
## 0.3.3 (2023-04-09)
|
|
2
|
+
|
|
3
|
+
- Updated Tokenizers to 0.13.3
|
|
4
|
+
- Added `ByteFallback`, `Fuse`, `Replace`, and `Strip` decoders
|
|
5
|
+
- Added `Prepend` normalizer
|
|
6
|
+
|
|
7
|
+
## 0.3.2 (2023-03-06)
|
|
8
|
+
|
|
9
|
+
- Added precompiled gem for Linux x86-64 MUSL
|
|
10
|
+
|
|
11
|
+
## 0.3.1 (2023-02-08)
|
|
2
12
|
|
|
3
13
|
- Fixed error with Ruby 2.7
|
|
4
14
|
|
|
5
|
-
## 0.3.0 (
|
|
15
|
+
## 0.3.0 (2023-02-07)
|
|
6
16
|
|
|
7
17
|
- Added support for training tokenizers
|
|
8
18
|
- Added more methods to `Tokenizer`
|
|
@@ -11,20 +21,20 @@
|
|
|
11
21
|
- Changed `encode` method to include special tokens by default
|
|
12
22
|
- Changed how offsets are calculated for strings with multibyte characters
|
|
13
23
|
|
|
14
|
-
## 0.2.3 (
|
|
24
|
+
## 0.2.3 (2023-01-22)
|
|
15
25
|
|
|
16
26
|
- Added `add_special_tokens` option to `encode` method
|
|
17
27
|
- Added warning about `encode` method including special tokens by default in 0.3.0
|
|
18
28
|
- Added more methods to `Encoding`
|
|
19
29
|
- Fixed error with precompiled gem on Mac ARM
|
|
20
30
|
|
|
21
|
-
## 0.2.2 (
|
|
31
|
+
## 0.2.2 (2023-01-15)
|
|
22
32
|
|
|
23
33
|
- Added precompiled gem for Linux ARM
|
|
24
34
|
- Added `from_file` method
|
|
25
35
|
- Fixed error with precompiled gem on Linux x86-64
|
|
26
36
|
|
|
27
|
-
## 0.2.1 (
|
|
37
|
+
## 0.2.1 (2023-01-12)
|
|
28
38
|
|
|
29
39
|
- Added support for Ruby 3.2
|
|
30
40
|
|
data/Cargo.lock
CHANGED
|
@@ -71,9 +71,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|
|
71
71
|
|
|
72
72
|
[[package]]
|
|
73
73
|
name = "clang-sys"
|
|
74
|
-
version = "1.
|
|
74
|
+
version = "1.6.1"
|
|
75
75
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
76
|
-
checksum = "
|
|
76
|
+
checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
|
|
77
77
|
dependencies = [
|
|
78
78
|
"glob",
|
|
79
79
|
"libc",
|
|
@@ -95,9 +95,9 @@ dependencies = [
|
|
|
95
95
|
|
|
96
96
|
[[package]]
|
|
97
97
|
name = "crossbeam-channel"
|
|
98
|
-
version = "0.5.
|
|
98
|
+
version = "0.5.8"
|
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
100
|
-
checksum = "
|
|
100
|
+
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
|
|
101
101
|
dependencies = [
|
|
102
102
|
"cfg-if",
|
|
103
103
|
"crossbeam-utils",
|
|
@@ -105,9 +105,9 @@ dependencies = [
|
|
|
105
105
|
|
|
106
106
|
[[package]]
|
|
107
107
|
name = "crossbeam-deque"
|
|
108
|
-
version = "0.8.
|
|
108
|
+
version = "0.8.3"
|
|
109
109
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
110
|
-
checksum = "
|
|
110
|
+
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
|
111
111
|
dependencies = [
|
|
112
112
|
"cfg-if",
|
|
113
113
|
"crossbeam-epoch",
|
|
@@ -116,9 +116,9 @@ dependencies = [
|
|
|
116
116
|
|
|
117
117
|
[[package]]
|
|
118
118
|
name = "crossbeam-epoch"
|
|
119
|
-
version = "0.9.
|
|
119
|
+
version = "0.9.14"
|
|
120
120
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
121
|
-
checksum = "
|
|
121
|
+
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
|
|
122
122
|
dependencies = [
|
|
123
123
|
"autocfg",
|
|
124
124
|
"cfg-if",
|
|
@@ -129,18 +129,18 @@ dependencies = [
|
|
|
129
129
|
|
|
130
130
|
[[package]]
|
|
131
131
|
name = "crossbeam-utils"
|
|
132
|
-
version = "0.8.
|
|
132
|
+
version = "0.8.15"
|
|
133
133
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
134
|
-
checksum = "
|
|
134
|
+
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
|
|
135
135
|
dependencies = [
|
|
136
136
|
"cfg-if",
|
|
137
137
|
]
|
|
138
138
|
|
|
139
139
|
[[package]]
|
|
140
140
|
name = "darling"
|
|
141
|
-
version = "0.14.
|
|
141
|
+
version = "0.14.4"
|
|
142
142
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
143
|
-
checksum = "
|
|
143
|
+
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
|
|
144
144
|
dependencies = [
|
|
145
145
|
"darling_core",
|
|
146
146
|
"darling_macro",
|
|
@@ -148,27 +148,27 @@ dependencies = [
|
|
|
148
148
|
|
|
149
149
|
[[package]]
|
|
150
150
|
name = "darling_core"
|
|
151
|
-
version = "0.14.
|
|
151
|
+
version = "0.14.4"
|
|
152
152
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
153
|
-
checksum = "
|
|
153
|
+
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
|
|
154
154
|
dependencies = [
|
|
155
155
|
"fnv",
|
|
156
156
|
"ident_case",
|
|
157
157
|
"proc-macro2",
|
|
158
158
|
"quote",
|
|
159
159
|
"strsim",
|
|
160
|
-
"syn",
|
|
160
|
+
"syn 1.0.109",
|
|
161
161
|
]
|
|
162
162
|
|
|
163
163
|
[[package]]
|
|
164
164
|
name = "darling_macro"
|
|
165
|
-
version = "0.14.
|
|
165
|
+
version = "0.14.4"
|
|
166
166
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
167
|
-
checksum = "
|
|
167
|
+
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
|
|
168
168
|
dependencies = [
|
|
169
169
|
"darling_core",
|
|
170
170
|
"quote",
|
|
171
|
-
"syn",
|
|
171
|
+
"syn 1.0.109",
|
|
172
172
|
]
|
|
173
173
|
|
|
174
174
|
[[package]]
|
|
@@ -189,7 +189,7 @@ dependencies = [
|
|
|
189
189
|
"darling",
|
|
190
190
|
"proc-macro2",
|
|
191
191
|
"quote",
|
|
192
|
-
"syn",
|
|
192
|
+
"syn 1.0.109",
|
|
193
193
|
]
|
|
194
194
|
|
|
195
195
|
[[package]]
|
|
@@ -199,7 +199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
|
199
199
|
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
|
|
200
200
|
dependencies = [
|
|
201
201
|
"derive_builder_core",
|
|
202
|
-
"syn",
|
|
202
|
+
"syn 1.0.109",
|
|
203
203
|
]
|
|
204
204
|
|
|
205
205
|
[[package]]
|
|
@@ -231,9 +231,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
|
|
231
231
|
|
|
232
232
|
[[package]]
|
|
233
233
|
name = "getrandom"
|
|
234
|
-
version = "0.2.
|
|
234
|
+
version = "0.2.9"
|
|
235
235
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
236
|
-
checksum = "
|
|
236
|
+
checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
|
|
237
237
|
dependencies = [
|
|
238
238
|
"cfg-if",
|
|
239
239
|
"libc",
|
|
@@ -293,9 +293,9 @@ dependencies = [
|
|
|
293
293
|
|
|
294
294
|
[[package]]
|
|
295
295
|
name = "itoa"
|
|
296
|
-
version = "1.0.
|
|
296
|
+
version = "1.0.6"
|
|
297
297
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
298
|
-
checksum = "
|
|
298
|
+
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
|
|
299
299
|
|
|
300
300
|
[[package]]
|
|
301
301
|
name = "lazy_static"
|
|
@@ -311,9 +311,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
|
311
311
|
|
|
312
312
|
[[package]]
|
|
313
313
|
name = "libc"
|
|
314
|
-
version = "0.2.
|
|
314
|
+
version = "0.2.141"
|
|
315
315
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
316
|
-
checksum = "
|
|
316
|
+
checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
|
|
317
317
|
|
|
318
318
|
[[package]]
|
|
319
319
|
name = "libloading"
|
|
@@ -352,8 +352,9 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
|
|
|
352
352
|
|
|
353
353
|
[[package]]
|
|
354
354
|
name = "magnus"
|
|
355
|
-
version = "0.5.
|
|
356
|
-
source = "
|
|
355
|
+
version = "0.5.3"
|
|
356
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
357
|
+
checksum = "c8dc14463c2552e753ef562961f486ca76f17a857c121db40e9f3ade3f35ab81"
|
|
357
358
|
dependencies = [
|
|
358
359
|
"magnus-macros",
|
|
359
360
|
"rb-sys",
|
|
@@ -362,12 +363,13 @@ dependencies = [
|
|
|
362
363
|
|
|
363
364
|
[[package]]
|
|
364
365
|
name = "magnus-macros"
|
|
365
|
-
version = "0.
|
|
366
|
-
source = "
|
|
366
|
+
version = "0.4.1"
|
|
367
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
368
|
+
checksum = "6cc17af1d45442c011aa579d727ec6cff8a69aea8a6bbad26736e7112d749bfb"
|
|
367
369
|
dependencies = [
|
|
368
370
|
"proc-macro2",
|
|
369
371
|
"quote",
|
|
370
|
-
"syn",
|
|
372
|
+
"syn 1.0.109",
|
|
371
373
|
]
|
|
372
374
|
|
|
373
375
|
[[package]]
|
|
@@ -378,9 +380,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
|
|
378
380
|
|
|
379
381
|
[[package]]
|
|
380
382
|
name = "memoffset"
|
|
381
|
-
version = "0.
|
|
383
|
+
version = "0.8.0"
|
|
382
384
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
383
|
-
checksum = "
|
|
385
|
+
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
|
|
384
386
|
dependencies = [
|
|
385
387
|
"autocfg",
|
|
386
388
|
]
|
|
@@ -391,6 +393,27 @@ version = "0.2.1"
|
|
|
391
393
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
392
394
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
|
393
395
|
|
|
396
|
+
[[package]]
|
|
397
|
+
name = "monostate"
|
|
398
|
+
version = "0.1.6"
|
|
399
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
400
|
+
checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
|
|
401
|
+
dependencies = [
|
|
402
|
+
"monostate-impl",
|
|
403
|
+
"serde",
|
|
404
|
+
]
|
|
405
|
+
|
|
406
|
+
[[package]]
|
|
407
|
+
name = "monostate-impl"
|
|
408
|
+
version = "0.1.6"
|
|
409
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
410
|
+
checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
|
|
411
|
+
dependencies = [
|
|
412
|
+
"proc-macro2",
|
|
413
|
+
"quote",
|
|
414
|
+
"syn 2.0.13",
|
|
415
|
+
]
|
|
416
|
+
|
|
394
417
|
[[package]]
|
|
395
418
|
name = "nom"
|
|
396
419
|
version = "7.1.3"
|
|
@@ -419,9 +442,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
|
|
|
419
442
|
|
|
420
443
|
[[package]]
|
|
421
444
|
name = "once_cell"
|
|
422
|
-
version = "1.17.
|
|
445
|
+
version = "1.17.1"
|
|
423
446
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
424
|
-
checksum = "
|
|
447
|
+
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
|
|
425
448
|
|
|
426
449
|
[[package]]
|
|
427
450
|
name = "onig"
|
|
@@ -447,9 +470,9 @@ dependencies = [
|
|
|
447
470
|
|
|
448
471
|
[[package]]
|
|
449
472
|
name = "paste"
|
|
450
|
-
version = "1.0.
|
|
473
|
+
version = "1.0.12"
|
|
451
474
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
452
|
-
checksum = "
|
|
475
|
+
checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
|
|
453
476
|
|
|
454
477
|
[[package]]
|
|
455
478
|
name = "peeking_take_while"
|
|
@@ -471,18 +494,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
|
|
471
494
|
|
|
472
495
|
[[package]]
|
|
473
496
|
name = "proc-macro2"
|
|
474
|
-
version = "1.0.
|
|
497
|
+
version = "1.0.56"
|
|
475
498
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
476
|
-
checksum = "
|
|
499
|
+
checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
|
|
477
500
|
dependencies = [
|
|
478
501
|
"unicode-ident",
|
|
479
502
|
]
|
|
480
503
|
|
|
481
504
|
[[package]]
|
|
482
505
|
name = "quote"
|
|
483
|
-
version = "1.0.
|
|
506
|
+
version = "1.0.26"
|
|
484
507
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
485
|
-
checksum = "
|
|
508
|
+
checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
|
|
486
509
|
dependencies = [
|
|
487
510
|
"proc-macro2",
|
|
488
511
|
]
|
|
@@ -519,9 +542,9 @@ dependencies = [
|
|
|
519
542
|
|
|
520
543
|
[[package]]
|
|
521
544
|
name = "rayon"
|
|
522
|
-
version = "1.
|
|
545
|
+
version = "1.7.0"
|
|
523
546
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
524
|
-
checksum = "
|
|
547
|
+
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
|
|
525
548
|
dependencies = [
|
|
526
549
|
"either",
|
|
527
550
|
"rayon-core",
|
|
@@ -540,9 +563,9 @@ dependencies = [
|
|
|
540
563
|
|
|
541
564
|
[[package]]
|
|
542
565
|
name = "rayon-core"
|
|
543
|
-
version = "1.
|
|
566
|
+
version = "1.11.0"
|
|
544
567
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
545
|
-
checksum = "
|
|
568
|
+
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
|
|
546
569
|
dependencies = [
|
|
547
570
|
"crossbeam-channel",
|
|
548
571
|
"crossbeam-deque",
|
|
@@ -552,22 +575,26 @@ dependencies = [
|
|
|
552
575
|
|
|
553
576
|
[[package]]
|
|
554
577
|
name = "rb-sys"
|
|
555
|
-
version = "0.9.
|
|
578
|
+
version = "0.9.71"
|
|
556
579
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
557
|
-
checksum = "
|
|
580
|
+
checksum = "156bfedced1e236600bcaad538477097ff2ed5c6b474e411d15b791e1d24c0f1"
|
|
558
581
|
dependencies = [
|
|
559
582
|
"rb-sys-build",
|
|
560
583
|
]
|
|
561
584
|
|
|
562
585
|
[[package]]
|
|
563
586
|
name = "rb-sys-build"
|
|
564
|
-
version = "0.9.
|
|
587
|
+
version = "0.9.71"
|
|
565
588
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
566
|
-
checksum = "
|
|
589
|
+
checksum = "5cb2e4a32cbc290b543a74567072ad24b708aff7bb5dde5a68d5690379cd7938"
|
|
567
590
|
dependencies = [
|
|
568
591
|
"bindgen",
|
|
592
|
+
"lazy_static",
|
|
593
|
+
"proc-macro2",
|
|
594
|
+
"quote",
|
|
569
595
|
"regex",
|
|
570
596
|
"shell-words",
|
|
597
|
+
"syn 1.0.109",
|
|
571
598
|
]
|
|
572
599
|
|
|
573
600
|
[[package]]
|
|
@@ -578,9 +605,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
|
|
|
578
605
|
|
|
579
606
|
[[package]]
|
|
580
607
|
name = "regex"
|
|
581
|
-
version = "1.7.
|
|
608
|
+
version = "1.7.3"
|
|
582
609
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
583
|
-
checksum = "
|
|
610
|
+
checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
|
|
584
611
|
dependencies = [
|
|
585
612
|
"aho-corasick",
|
|
586
613
|
"memchr",
|
|
@@ -589,9 +616,9 @@ dependencies = [
|
|
|
589
616
|
|
|
590
617
|
[[package]]
|
|
591
618
|
name = "regex-syntax"
|
|
592
|
-
version = "0.6.
|
|
619
|
+
version = "0.6.29"
|
|
593
620
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
594
|
-
checksum = "
|
|
621
|
+
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
|
595
622
|
|
|
596
623
|
[[package]]
|
|
597
624
|
name = "rustc-hash"
|
|
@@ -601,9 +628,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
|
601
628
|
|
|
602
629
|
[[package]]
|
|
603
630
|
name = "ryu"
|
|
604
|
-
version = "1.0.
|
|
631
|
+
version = "1.0.13"
|
|
605
632
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
606
|
-
checksum = "
|
|
633
|
+
checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
|
|
607
634
|
|
|
608
635
|
[[package]]
|
|
609
636
|
name = "scopeguard"
|
|
@@ -613,29 +640,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
|
|
613
640
|
|
|
614
641
|
[[package]]
|
|
615
642
|
name = "serde"
|
|
616
|
-
version = "1.0.
|
|
643
|
+
version = "1.0.159"
|
|
617
644
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
618
|
-
checksum = "
|
|
645
|
+
checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
|
|
619
646
|
dependencies = [
|
|
620
647
|
"serde_derive",
|
|
621
648
|
]
|
|
622
649
|
|
|
623
650
|
[[package]]
|
|
624
651
|
name = "serde_derive"
|
|
625
|
-
version = "1.0.
|
|
652
|
+
version = "1.0.159"
|
|
626
653
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
627
|
-
checksum = "
|
|
654
|
+
checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
|
|
628
655
|
dependencies = [
|
|
629
656
|
"proc-macro2",
|
|
630
657
|
"quote",
|
|
631
|
-
"syn",
|
|
658
|
+
"syn 2.0.13",
|
|
632
659
|
]
|
|
633
660
|
|
|
634
661
|
[[package]]
|
|
635
662
|
name = "serde_json"
|
|
636
|
-
version = "1.0.
|
|
663
|
+
version = "1.0.95"
|
|
637
664
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
638
|
-
checksum = "
|
|
665
|
+
checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
|
|
639
666
|
dependencies = [
|
|
640
667
|
"itoa",
|
|
641
668
|
"ryu",
|
|
@@ -680,9 +707,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
|
|
680
707
|
|
|
681
708
|
[[package]]
|
|
682
709
|
name = "syn"
|
|
683
|
-
version = "1.0.
|
|
710
|
+
version = "1.0.109"
|
|
684
711
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
685
|
-
checksum = "
|
|
712
|
+
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
|
713
|
+
dependencies = [
|
|
714
|
+
"proc-macro2",
|
|
715
|
+
"quote",
|
|
716
|
+
"unicode-ident",
|
|
717
|
+
]
|
|
718
|
+
|
|
719
|
+
[[package]]
|
|
720
|
+
name = "syn"
|
|
721
|
+
version = "2.0.13"
|
|
722
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
723
|
+
checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
|
|
686
724
|
dependencies = [
|
|
687
725
|
"proc-macro2",
|
|
688
726
|
"quote",
|
|
@@ -691,38 +729,39 @@ dependencies = [
|
|
|
691
729
|
|
|
692
730
|
[[package]]
|
|
693
731
|
name = "thiserror"
|
|
694
|
-
version = "1.0.
|
|
732
|
+
version = "1.0.40"
|
|
695
733
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
696
|
-
checksum = "
|
|
734
|
+
checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
|
|
697
735
|
dependencies = [
|
|
698
736
|
"thiserror-impl",
|
|
699
737
|
]
|
|
700
738
|
|
|
701
739
|
[[package]]
|
|
702
740
|
name = "thiserror-impl"
|
|
703
|
-
version = "1.0.
|
|
741
|
+
version = "1.0.40"
|
|
704
742
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
705
|
-
checksum = "
|
|
743
|
+
checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
|
|
706
744
|
dependencies = [
|
|
707
745
|
"proc-macro2",
|
|
708
746
|
"quote",
|
|
709
|
-
"syn",
|
|
747
|
+
"syn 2.0.13",
|
|
710
748
|
]
|
|
711
749
|
|
|
712
750
|
[[package]]
|
|
713
751
|
name = "tokenizers"
|
|
714
|
-
version = "0.3.
|
|
752
|
+
version = "0.3.3"
|
|
715
753
|
dependencies = [
|
|
716
754
|
"magnus",
|
|
717
755
|
"onig",
|
|
718
756
|
"serde",
|
|
719
|
-
"tokenizers 0.13.
|
|
757
|
+
"tokenizers 0.13.3",
|
|
720
758
|
]
|
|
721
759
|
|
|
722
760
|
[[package]]
|
|
723
761
|
name = "tokenizers"
|
|
724
|
-
version = "0.13.
|
|
725
|
-
source = "
|
|
762
|
+
version = "0.13.3"
|
|
763
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
764
|
+
checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5"
|
|
726
765
|
dependencies = [
|
|
727
766
|
"aho-corasick",
|
|
728
767
|
"derive_builder",
|
|
@@ -733,6 +772,7 @@ dependencies = [
|
|
|
733
772
|
"lazy_static",
|
|
734
773
|
"log",
|
|
735
774
|
"macro_rules_attribute",
|
|
775
|
+
"monostate",
|
|
736
776
|
"onig",
|
|
737
777
|
"paste",
|
|
738
778
|
"rand",
|
|
@@ -751,9 +791,9 @@ dependencies = [
|
|
|
751
791
|
|
|
752
792
|
[[package]]
|
|
753
793
|
name = "unicode-ident"
|
|
754
|
-
version = "1.0.
|
|
794
|
+
version = "1.0.8"
|
|
755
795
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
756
|
-
checksum = "
|
|
796
|
+
checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
|
|
757
797
|
|
|
758
798
|
[[package]]
|
|
759
799
|
name = "unicode-normalization-alignments"
|
|
@@ -827,42 +867,42 @@ dependencies = [
|
|
|
827
867
|
|
|
828
868
|
[[package]]
|
|
829
869
|
name = "windows_aarch64_gnullvm"
|
|
830
|
-
version = "0.42.
|
|
870
|
+
version = "0.42.2"
|
|
831
871
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
832
|
-
checksum = "
|
|
872
|
+
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
|
|
833
873
|
|
|
834
874
|
[[package]]
|
|
835
875
|
name = "windows_aarch64_msvc"
|
|
836
|
-
version = "0.42.
|
|
876
|
+
version = "0.42.2"
|
|
837
877
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
838
|
-
checksum = "
|
|
878
|
+
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
|
|
839
879
|
|
|
840
880
|
[[package]]
|
|
841
881
|
name = "windows_i686_gnu"
|
|
842
|
-
version = "0.42.
|
|
882
|
+
version = "0.42.2"
|
|
843
883
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
844
|
-
checksum = "
|
|
884
|
+
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
|
|
845
885
|
|
|
846
886
|
[[package]]
|
|
847
887
|
name = "windows_i686_msvc"
|
|
848
|
-
version = "0.42.
|
|
888
|
+
version = "0.42.2"
|
|
849
889
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
850
|
-
checksum = "
|
|
890
|
+
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
|
|
851
891
|
|
|
852
892
|
[[package]]
|
|
853
893
|
name = "windows_x86_64_gnu"
|
|
854
|
-
version = "0.42.
|
|
894
|
+
version = "0.42.2"
|
|
855
895
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
856
|
-
checksum = "
|
|
896
|
+
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
|
|
857
897
|
|
|
858
898
|
[[package]]
|
|
859
899
|
name = "windows_x86_64_gnullvm"
|
|
860
|
-
version = "0.42.
|
|
900
|
+
version = "0.42.2"
|
|
861
901
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
862
|
-
checksum = "
|
|
902
|
+
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
|
|
863
903
|
|
|
864
904
|
[[package]]
|
|
865
905
|
name = "windows_x86_64_msvc"
|
|
866
|
-
version = "0.42.
|
|
906
|
+
version = "0.42.2"
|
|
867
907
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
868
|
-
checksum = "
|
|
908
|
+
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
|