tokenizers 0.3.2-x86_64-linux-musl → 0.3.3-x86_64-linux-musl
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/Cargo.lock +125 -90
- data/LICENSE-THIRD-PARTY.txt +1669 -1491
- data/lib/tokenizers/2.7/tokenizers.so +0 -0
- data/lib/tokenizers/3.0/tokenizers.so +0 -0
- data/lib/tokenizers/3.1/tokenizers.so +0 -0
- data/lib/tokenizers/3.2/tokenizers.so +0 -0
- data/lib/tokenizers/decoders/strip.rb +9 -0
- data/lib/tokenizers/from_pretrained.rb +1 -1
- data/lib/tokenizers/normalizers/prepend.rb +9 -0
- data/lib/tokenizers/version.rb +1 -1
- data/lib/tokenizers.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 823ff10442a490f7aba2787454e57d1465a65f5ebb8bfb1459f9866a7ffcb16b
|
4
|
+
data.tar.gz: '082019f4aae17cf7b1178676bf7c9585de579219f2d3cb9f543d94b31c2c2683'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 851ac588098758a3afda1415ddecc1a2191ddda5959e136e9cc40675c677313912cba51f0e807666ed0031c4108d1a1b435fbfb07d2abca87355d3ecaa211872
|
7
|
+
data.tar.gz: ef238cb6cc1936dd62f6f28fc0ee375dcc7eb7736721199e9bb244236e775a3ec3985eadd20ac818eac0ff1616c998bf3314d572dfd344506b19fe49573e683d
|
data/CHANGELOG.md
CHANGED
data/Cargo.lock
CHANGED
@@ -71,9 +71,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|
71
71
|
|
72
72
|
[[package]]
|
73
73
|
name = "clang-sys"
|
74
|
-
version = "1.
|
74
|
+
version = "1.6.1"
|
75
75
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
76
|
-
checksum = "
|
76
|
+
checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
|
77
77
|
dependencies = [
|
78
78
|
"glob",
|
79
79
|
"libc",
|
@@ -95,9 +95,9 @@ dependencies = [
|
|
95
95
|
|
96
96
|
[[package]]
|
97
97
|
name = "crossbeam-channel"
|
98
|
-
version = "0.5.
|
98
|
+
version = "0.5.8"
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
100
|
-
checksum = "
|
100
|
+
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
|
101
101
|
dependencies = [
|
102
102
|
"cfg-if",
|
103
103
|
"crossbeam-utils",
|
@@ -105,9 +105,9 @@ dependencies = [
|
|
105
105
|
|
106
106
|
[[package]]
|
107
107
|
name = "crossbeam-deque"
|
108
|
-
version = "0.8.
|
108
|
+
version = "0.8.3"
|
109
109
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
110
|
-
checksum = "
|
110
|
+
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
111
111
|
dependencies = [
|
112
112
|
"cfg-if",
|
113
113
|
"crossbeam-epoch",
|
@@ -116,9 +116,9 @@ dependencies = [
|
|
116
116
|
|
117
117
|
[[package]]
|
118
118
|
name = "crossbeam-epoch"
|
119
|
-
version = "0.9.
|
119
|
+
version = "0.9.14"
|
120
120
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
121
|
-
checksum = "
|
121
|
+
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
|
122
122
|
dependencies = [
|
123
123
|
"autocfg",
|
124
124
|
"cfg-if",
|
@@ -129,18 +129,18 @@ dependencies = [
|
|
129
129
|
|
130
130
|
[[package]]
|
131
131
|
name = "crossbeam-utils"
|
132
|
-
version = "0.8.
|
132
|
+
version = "0.8.15"
|
133
133
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
134
|
-
checksum = "
|
134
|
+
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
|
135
135
|
dependencies = [
|
136
136
|
"cfg-if",
|
137
137
|
]
|
138
138
|
|
139
139
|
[[package]]
|
140
140
|
name = "darling"
|
141
|
-
version = "0.14.
|
141
|
+
version = "0.14.4"
|
142
142
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
143
|
-
checksum = "
|
143
|
+
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
|
144
144
|
dependencies = [
|
145
145
|
"darling_core",
|
146
146
|
"darling_macro",
|
@@ -148,27 +148,27 @@ dependencies = [
|
|
148
148
|
|
149
149
|
[[package]]
|
150
150
|
name = "darling_core"
|
151
|
-
version = "0.14.
|
151
|
+
version = "0.14.4"
|
152
152
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
153
|
-
checksum = "
|
153
|
+
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
|
154
154
|
dependencies = [
|
155
155
|
"fnv",
|
156
156
|
"ident_case",
|
157
157
|
"proc-macro2",
|
158
158
|
"quote",
|
159
159
|
"strsim",
|
160
|
-
"syn",
|
160
|
+
"syn 1.0.109",
|
161
161
|
]
|
162
162
|
|
163
163
|
[[package]]
|
164
164
|
name = "darling_macro"
|
165
|
-
version = "0.14.
|
165
|
+
version = "0.14.4"
|
166
166
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
167
|
-
checksum = "
|
167
|
+
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
|
168
168
|
dependencies = [
|
169
169
|
"darling_core",
|
170
170
|
"quote",
|
171
|
-
"syn",
|
171
|
+
"syn 1.0.109",
|
172
172
|
]
|
173
173
|
|
174
174
|
[[package]]
|
@@ -189,7 +189,7 @@ dependencies = [
|
|
189
189
|
"darling",
|
190
190
|
"proc-macro2",
|
191
191
|
"quote",
|
192
|
-
"syn",
|
192
|
+
"syn 1.0.109",
|
193
193
|
]
|
194
194
|
|
195
195
|
[[package]]
|
@@ -199,7 +199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
199
199
|
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
|
200
200
|
dependencies = [
|
201
201
|
"derive_builder_core",
|
202
|
-
"syn",
|
202
|
+
"syn 1.0.109",
|
203
203
|
]
|
204
204
|
|
205
205
|
[[package]]
|
@@ -231,9 +231,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
|
231
231
|
|
232
232
|
[[package]]
|
233
233
|
name = "getrandom"
|
234
|
-
version = "0.2.
|
234
|
+
version = "0.2.9"
|
235
235
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
236
|
-
checksum = "
|
236
|
+
checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
|
237
237
|
dependencies = [
|
238
238
|
"cfg-if",
|
239
239
|
"libc",
|
@@ -293,9 +293,9 @@ dependencies = [
|
|
293
293
|
|
294
294
|
[[package]]
|
295
295
|
name = "itoa"
|
296
|
-
version = "1.0.
|
296
|
+
version = "1.0.6"
|
297
297
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
298
|
-
checksum = "
|
298
|
+
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
|
299
299
|
|
300
300
|
[[package]]
|
301
301
|
name = "lazy_static"
|
@@ -311,9 +311,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
311
311
|
|
312
312
|
[[package]]
|
313
313
|
name = "libc"
|
314
|
-
version = "0.2.
|
314
|
+
version = "0.2.141"
|
315
315
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
316
|
-
checksum = "
|
316
|
+
checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
|
317
317
|
|
318
318
|
[[package]]
|
319
319
|
name = "libloading"
|
@@ -352,9 +352,9 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
|
|
352
352
|
|
353
353
|
[[package]]
|
354
354
|
name = "magnus"
|
355
|
-
version = "0.5.
|
355
|
+
version = "0.5.3"
|
356
356
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
357
|
-
checksum = "
|
357
|
+
checksum = "c8dc14463c2552e753ef562961f486ca76f17a857c121db40e9f3ade3f35ab81"
|
358
358
|
dependencies = [
|
359
359
|
"magnus-macros",
|
360
360
|
"rb-sys",
|
@@ -363,13 +363,13 @@ dependencies = [
|
|
363
363
|
|
364
364
|
[[package]]
|
365
365
|
name = "magnus-macros"
|
366
|
-
version = "0.4.
|
366
|
+
version = "0.4.1"
|
367
367
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
368
|
-
checksum = "
|
368
|
+
checksum = "6cc17af1d45442c011aa579d727ec6cff8a69aea8a6bbad26736e7112d749bfb"
|
369
369
|
dependencies = [
|
370
370
|
"proc-macro2",
|
371
371
|
"quote",
|
372
|
-
"syn",
|
372
|
+
"syn 1.0.109",
|
373
373
|
]
|
374
374
|
|
375
375
|
[[package]]
|
@@ -380,9 +380,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
|
380
380
|
|
381
381
|
[[package]]
|
382
382
|
name = "memoffset"
|
383
|
-
version = "0.
|
383
|
+
version = "0.8.0"
|
384
384
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
385
|
-
checksum = "
|
385
|
+
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
|
386
386
|
dependencies = [
|
387
387
|
"autocfg",
|
388
388
|
]
|
@@ -393,6 +393,27 @@ version = "0.2.1"
|
|
393
393
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
394
394
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
395
395
|
|
396
|
+
[[package]]
|
397
|
+
name = "monostate"
|
398
|
+
version = "0.1.6"
|
399
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
400
|
+
checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
|
401
|
+
dependencies = [
|
402
|
+
"monostate-impl",
|
403
|
+
"serde",
|
404
|
+
]
|
405
|
+
|
406
|
+
[[package]]
|
407
|
+
name = "monostate-impl"
|
408
|
+
version = "0.1.6"
|
409
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
410
|
+
checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
|
411
|
+
dependencies = [
|
412
|
+
"proc-macro2",
|
413
|
+
"quote",
|
414
|
+
"syn 2.0.13",
|
415
|
+
]
|
416
|
+
|
396
417
|
[[package]]
|
397
418
|
name = "nom"
|
398
419
|
version = "7.1.3"
|
@@ -421,9 +442,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
|
|
421
442
|
|
422
443
|
[[package]]
|
423
444
|
name = "once_cell"
|
424
|
-
version = "1.17.
|
445
|
+
version = "1.17.1"
|
425
446
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
426
|
-
checksum = "
|
447
|
+
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
|
427
448
|
|
428
449
|
[[package]]
|
429
450
|
name = "onig"
|
@@ -449,9 +470,9 @@ dependencies = [
|
|
449
470
|
|
450
471
|
[[package]]
|
451
472
|
name = "paste"
|
452
|
-
version = "1.0.
|
473
|
+
version = "1.0.12"
|
453
474
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
454
|
-
checksum = "
|
475
|
+
checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
|
455
476
|
|
456
477
|
[[package]]
|
457
478
|
name = "peeking_take_while"
|
@@ -473,18 +494,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
|
473
494
|
|
474
495
|
[[package]]
|
475
496
|
name = "proc-macro2"
|
476
|
-
version = "1.0.
|
497
|
+
version = "1.0.56"
|
477
498
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
478
|
-
checksum = "
|
499
|
+
checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
|
479
500
|
dependencies = [
|
480
501
|
"unicode-ident",
|
481
502
|
]
|
482
503
|
|
483
504
|
[[package]]
|
484
505
|
name = "quote"
|
485
|
-
version = "1.0.
|
506
|
+
version = "1.0.26"
|
486
507
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
487
|
-
checksum = "
|
508
|
+
checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
|
488
509
|
dependencies = [
|
489
510
|
"proc-macro2",
|
490
511
|
]
|
@@ -521,9 +542,9 @@ dependencies = [
|
|
521
542
|
|
522
543
|
[[package]]
|
523
544
|
name = "rayon"
|
524
|
-
version = "1.
|
545
|
+
version = "1.7.0"
|
525
546
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
526
|
-
checksum = "
|
547
|
+
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
|
527
548
|
dependencies = [
|
528
549
|
"either",
|
529
550
|
"rayon-core",
|
@@ -542,9 +563,9 @@ dependencies = [
|
|
542
563
|
|
543
564
|
[[package]]
|
544
565
|
name = "rayon-core"
|
545
|
-
version = "1.
|
566
|
+
version = "1.11.0"
|
546
567
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
547
|
-
checksum = "
|
568
|
+
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
|
548
569
|
dependencies = [
|
549
570
|
"crossbeam-channel",
|
550
571
|
"crossbeam-deque",
|
@@ -554,25 +575,26 @@ dependencies = [
|
|
554
575
|
|
555
576
|
[[package]]
|
556
577
|
name = "rb-sys"
|
557
|
-
version = "0.9.
|
578
|
+
version = "0.9.71"
|
558
579
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
559
|
-
checksum = "
|
580
|
+
checksum = "156bfedced1e236600bcaad538477097ff2ed5c6b474e411d15b791e1d24c0f1"
|
560
581
|
dependencies = [
|
561
582
|
"rb-sys-build",
|
562
583
|
]
|
563
584
|
|
564
585
|
[[package]]
|
565
586
|
name = "rb-sys-build"
|
566
|
-
version = "0.9.
|
587
|
+
version = "0.9.71"
|
567
588
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
568
|
-
checksum = "
|
589
|
+
checksum = "5cb2e4a32cbc290b543a74567072ad24b708aff7bb5dde5a68d5690379cd7938"
|
569
590
|
dependencies = [
|
570
591
|
"bindgen",
|
571
592
|
"lazy_static",
|
593
|
+
"proc-macro2",
|
572
594
|
"quote",
|
573
595
|
"regex",
|
574
596
|
"shell-words",
|
575
|
-
"syn",
|
597
|
+
"syn 1.0.109",
|
576
598
|
]
|
577
599
|
|
578
600
|
[[package]]
|
@@ -583,9 +605,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
|
|
583
605
|
|
584
606
|
[[package]]
|
585
607
|
name = "regex"
|
586
|
-
version = "1.7.
|
608
|
+
version = "1.7.3"
|
587
609
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
588
|
-
checksum = "
|
610
|
+
checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
|
589
611
|
dependencies = [
|
590
612
|
"aho-corasick",
|
591
613
|
"memchr",
|
@@ -594,9 +616,9 @@ dependencies = [
|
|
594
616
|
|
595
617
|
[[package]]
|
596
618
|
name = "regex-syntax"
|
597
|
-
version = "0.6.
|
619
|
+
version = "0.6.29"
|
598
620
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
599
|
-
checksum = "
|
621
|
+
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
600
622
|
|
601
623
|
[[package]]
|
602
624
|
name = "rustc-hash"
|
@@ -606,9 +628,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
606
628
|
|
607
629
|
[[package]]
|
608
630
|
name = "ryu"
|
609
|
-
version = "1.0.
|
631
|
+
version = "1.0.13"
|
610
632
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
611
|
-
checksum = "
|
633
|
+
checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
|
612
634
|
|
613
635
|
[[package]]
|
614
636
|
name = "scopeguard"
|
@@ -618,29 +640,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
|
618
640
|
|
619
641
|
[[package]]
|
620
642
|
name = "serde"
|
621
|
-
version = "1.0.
|
643
|
+
version = "1.0.159"
|
622
644
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
623
|
-
checksum = "
|
645
|
+
checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
|
624
646
|
dependencies = [
|
625
647
|
"serde_derive",
|
626
648
|
]
|
627
649
|
|
628
650
|
[[package]]
|
629
651
|
name = "serde_derive"
|
630
|
-
version = "1.0.
|
652
|
+
version = "1.0.159"
|
631
653
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
632
|
-
checksum = "
|
654
|
+
checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
|
633
655
|
dependencies = [
|
634
656
|
"proc-macro2",
|
635
657
|
"quote",
|
636
|
-
"syn",
|
658
|
+
"syn 2.0.13",
|
637
659
|
]
|
638
660
|
|
639
661
|
[[package]]
|
640
662
|
name = "serde_json"
|
641
|
-
version = "1.0.
|
663
|
+
version = "1.0.95"
|
642
664
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
643
|
-
checksum = "
|
665
|
+
checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
|
644
666
|
dependencies = [
|
645
667
|
"itoa",
|
646
668
|
"ryu",
|
@@ -685,9 +707,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
|
685
707
|
|
686
708
|
[[package]]
|
687
709
|
name = "syn"
|
688
|
-
version = "1.0.
|
710
|
+
version = "1.0.109"
|
689
711
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
690
|
-
checksum = "
|
712
|
+
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
713
|
+
dependencies = [
|
714
|
+
"proc-macro2",
|
715
|
+
"quote",
|
716
|
+
"unicode-ident",
|
717
|
+
]
|
718
|
+
|
719
|
+
[[package]]
|
720
|
+
name = "syn"
|
721
|
+
version = "2.0.13"
|
722
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
723
|
+
checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
|
691
724
|
dependencies = [
|
692
725
|
"proc-macro2",
|
693
726
|
"quote",
|
@@ -696,38 +729,39 @@ dependencies = [
|
|
696
729
|
|
697
730
|
[[package]]
|
698
731
|
name = "thiserror"
|
699
|
-
version = "1.0.
|
732
|
+
version = "1.0.40"
|
700
733
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
701
|
-
checksum = "
|
734
|
+
checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
|
702
735
|
dependencies = [
|
703
736
|
"thiserror-impl",
|
704
737
|
]
|
705
738
|
|
706
739
|
[[package]]
|
707
740
|
name = "thiserror-impl"
|
708
|
-
version = "1.0.
|
741
|
+
version = "1.0.40"
|
709
742
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
710
|
-
checksum = "
|
743
|
+
checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
|
711
744
|
dependencies = [
|
712
745
|
"proc-macro2",
|
713
746
|
"quote",
|
714
|
-
"syn",
|
747
|
+
"syn 2.0.13",
|
715
748
|
]
|
716
749
|
|
717
750
|
[[package]]
|
718
751
|
name = "tokenizers"
|
719
|
-
version = "0.3.
|
752
|
+
version = "0.3.3"
|
720
753
|
dependencies = [
|
721
754
|
"magnus",
|
722
755
|
"onig",
|
723
756
|
"serde",
|
724
|
-
"tokenizers 0.13.
|
757
|
+
"tokenizers 0.13.3",
|
725
758
|
]
|
726
759
|
|
727
760
|
[[package]]
|
728
761
|
name = "tokenizers"
|
729
|
-
version = "0.13.
|
730
|
-
source = "
|
762
|
+
version = "0.13.3"
|
763
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
764
|
+
checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5"
|
731
765
|
dependencies = [
|
732
766
|
"aho-corasick",
|
733
767
|
"derive_builder",
|
@@ -738,6 +772,7 @@ dependencies = [
|
|
738
772
|
"lazy_static",
|
739
773
|
"log",
|
740
774
|
"macro_rules_attribute",
|
775
|
+
"monostate",
|
741
776
|
"onig",
|
742
777
|
"paste",
|
743
778
|
"rand",
|
@@ -756,9 +791,9 @@ dependencies = [
|
|
756
791
|
|
757
792
|
[[package]]
|
758
793
|
name = "unicode-ident"
|
759
|
-
version = "1.0.
|
794
|
+
version = "1.0.8"
|
760
795
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
761
|
-
checksum = "
|
796
|
+
checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
|
762
797
|
|
763
798
|
[[package]]
|
764
799
|
name = "unicode-normalization-alignments"
|
@@ -832,42 +867,42 @@ dependencies = [
|
|
832
867
|
|
833
868
|
[[package]]
|
834
869
|
name = "windows_aarch64_gnullvm"
|
835
|
-
version = "0.42.
|
870
|
+
version = "0.42.2"
|
836
871
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
837
|
-
checksum = "
|
872
|
+
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
|
838
873
|
|
839
874
|
[[package]]
|
840
875
|
name = "windows_aarch64_msvc"
|
841
|
-
version = "0.42.
|
876
|
+
version = "0.42.2"
|
842
877
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
843
|
-
checksum = "
|
878
|
+
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
|
844
879
|
|
845
880
|
[[package]]
|
846
881
|
name = "windows_i686_gnu"
|
847
|
-
version = "0.42.
|
882
|
+
version = "0.42.2"
|
848
883
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
849
|
-
checksum = "
|
884
|
+
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
|
850
885
|
|
851
886
|
[[package]]
|
852
887
|
name = "windows_i686_msvc"
|
853
|
-
version = "0.42.
|
888
|
+
version = "0.42.2"
|
854
889
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
855
|
-
checksum = "
|
890
|
+
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
|
856
891
|
|
857
892
|
[[package]]
|
858
893
|
name = "windows_x86_64_gnu"
|
859
|
-
version = "0.42.
|
894
|
+
version = "0.42.2"
|
860
895
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
861
|
-
checksum = "
|
896
|
+
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
|
862
897
|
|
863
898
|
[[package]]
|
864
899
|
name = "windows_x86_64_gnullvm"
|
865
|
-
version = "0.42.
|
900
|
+
version = "0.42.2"
|
866
901
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
867
|
-
checksum = "
|
902
|
+
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
|
868
903
|
|
869
904
|
[[package]]
|
870
905
|
name = "windows_x86_64_msvc"
|
871
|
-
version = "0.42.
|
906
|
+
version = "0.42.2"
|
872
907
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
873
|
-
checksum = "
|
908
|
+
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
|