tokenizers 0.3.2-arm64-darwin → 0.3.3-arm64-darwin
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/Cargo.lock +125 -90
- data/LICENSE-THIRD-PARTY.txt +1669 -1491
- data/lib/tokenizers/2.7/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.0/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.1/tokenizers.bundle +0 -0
- data/lib/tokenizers/3.2/tokenizers.bundle +0 -0
- data/lib/tokenizers/decoders/strip.rb +9 -0
- data/lib/tokenizers/from_pretrained.rb +1 -1
- data/lib/tokenizers/normalizers/prepend.rb +9 -0
- data/lib/tokenizers/version.rb +1 -1
- data/lib/tokenizers.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1bec554a0a6be342b2c159d005e9d0fa097a22ca7219713494b97ccf7e697b41
|
4
|
+
data.tar.gz: 80eef8238eb2db824b64c30a80bd97a8babc4ec03a33003a49415269443220c0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ea0b79d006ffd25ed1a57e22241cc9ad24706372bd72651db32e48957e140de919f72d19d01188556a2149fdb91ce1193624a32de06213f91763ee1e25ad1140
|
7
|
+
data.tar.gz: bef61ac269d494834a971abfa77d589aa68f8b58b00421b4e0c7f0b4006c0369fcbd136e2562e94426ea048b720888389a2db6dde69a5f46223d893b4a79cd65
|
data/CHANGELOG.md
CHANGED
data/Cargo.lock
CHANGED
@@ -71,9 +71,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|
71
71
|
|
72
72
|
[[package]]
|
73
73
|
name = "clang-sys"
|
74
|
-
version = "1.
|
74
|
+
version = "1.6.1"
|
75
75
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
76
|
-
checksum = "
|
76
|
+
checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
|
77
77
|
dependencies = [
|
78
78
|
"glob",
|
79
79
|
"libc",
|
@@ -95,9 +95,9 @@ dependencies = [
|
|
95
95
|
|
96
96
|
[[package]]
|
97
97
|
name = "crossbeam-channel"
|
98
|
-
version = "0.5.
|
98
|
+
version = "0.5.8"
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
100
|
-
checksum = "
|
100
|
+
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
|
101
101
|
dependencies = [
|
102
102
|
"cfg-if",
|
103
103
|
"crossbeam-utils",
|
@@ -105,9 +105,9 @@ dependencies = [
|
|
105
105
|
|
106
106
|
[[package]]
|
107
107
|
name = "crossbeam-deque"
|
108
|
-
version = "0.8.
|
108
|
+
version = "0.8.3"
|
109
109
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
110
|
-
checksum = "
|
110
|
+
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
111
111
|
dependencies = [
|
112
112
|
"cfg-if",
|
113
113
|
"crossbeam-epoch",
|
@@ -116,9 +116,9 @@ dependencies = [
|
|
116
116
|
|
117
117
|
[[package]]
|
118
118
|
name = "crossbeam-epoch"
|
119
|
-
version = "0.9.
|
119
|
+
version = "0.9.14"
|
120
120
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
121
|
-
checksum = "
|
121
|
+
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
|
122
122
|
dependencies = [
|
123
123
|
"autocfg",
|
124
124
|
"cfg-if",
|
@@ -129,18 +129,18 @@ dependencies = [
|
|
129
129
|
|
130
130
|
[[package]]
|
131
131
|
name = "crossbeam-utils"
|
132
|
-
version = "0.8.
|
132
|
+
version = "0.8.15"
|
133
133
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
134
|
-
checksum = "
|
134
|
+
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
|
135
135
|
dependencies = [
|
136
136
|
"cfg-if",
|
137
137
|
]
|
138
138
|
|
139
139
|
[[package]]
|
140
140
|
name = "darling"
|
141
|
-
version = "0.14.
|
141
|
+
version = "0.14.4"
|
142
142
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
143
|
-
checksum = "
|
143
|
+
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
|
144
144
|
dependencies = [
|
145
145
|
"darling_core",
|
146
146
|
"darling_macro",
|
@@ -148,27 +148,27 @@ dependencies = [
|
|
148
148
|
|
149
149
|
[[package]]
|
150
150
|
name = "darling_core"
|
151
|
-
version = "0.14.
|
151
|
+
version = "0.14.4"
|
152
152
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
153
|
-
checksum = "
|
153
|
+
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
|
154
154
|
dependencies = [
|
155
155
|
"fnv",
|
156
156
|
"ident_case",
|
157
157
|
"proc-macro2",
|
158
158
|
"quote",
|
159
159
|
"strsim",
|
160
|
-
"syn",
|
160
|
+
"syn 1.0.109",
|
161
161
|
]
|
162
162
|
|
163
163
|
[[package]]
|
164
164
|
name = "darling_macro"
|
165
|
-
version = "0.14.
|
165
|
+
version = "0.14.4"
|
166
166
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
167
|
-
checksum = "
|
167
|
+
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
|
168
168
|
dependencies = [
|
169
169
|
"darling_core",
|
170
170
|
"quote",
|
171
|
-
"syn",
|
171
|
+
"syn 1.0.109",
|
172
172
|
]
|
173
173
|
|
174
174
|
[[package]]
|
@@ -189,7 +189,7 @@ dependencies = [
|
|
189
189
|
"darling",
|
190
190
|
"proc-macro2",
|
191
191
|
"quote",
|
192
|
-
"syn",
|
192
|
+
"syn 1.0.109",
|
193
193
|
]
|
194
194
|
|
195
195
|
[[package]]
|
@@ -199,7 +199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
199
199
|
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
|
200
200
|
dependencies = [
|
201
201
|
"derive_builder_core",
|
202
|
-
"syn",
|
202
|
+
"syn 1.0.109",
|
203
203
|
]
|
204
204
|
|
205
205
|
[[package]]
|
@@ -231,9 +231,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
|
231
231
|
|
232
232
|
[[package]]
|
233
233
|
name = "getrandom"
|
234
|
-
version = "0.2.
|
234
|
+
version = "0.2.9"
|
235
235
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
236
|
-
checksum = "
|
236
|
+
checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
|
237
237
|
dependencies = [
|
238
238
|
"cfg-if",
|
239
239
|
"libc",
|
@@ -293,9 +293,9 @@ dependencies = [
|
|
293
293
|
|
294
294
|
[[package]]
|
295
295
|
name = "itoa"
|
296
|
-
version = "1.0.
|
296
|
+
version = "1.0.6"
|
297
297
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
298
|
-
checksum = "
|
298
|
+
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
|
299
299
|
|
300
300
|
[[package]]
|
301
301
|
name = "lazy_static"
|
@@ -311,9 +311,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
311
311
|
|
312
312
|
[[package]]
|
313
313
|
name = "libc"
|
314
|
-
version = "0.2.
|
314
|
+
version = "0.2.141"
|
315
315
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
316
|
-
checksum = "
|
316
|
+
checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
|
317
317
|
|
318
318
|
[[package]]
|
319
319
|
name = "libloading"
|
@@ -352,9 +352,9 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
|
|
352
352
|
|
353
353
|
[[package]]
|
354
354
|
name = "magnus"
|
355
|
-
version = "0.5.
|
355
|
+
version = "0.5.3"
|
356
356
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
357
|
-
checksum = "
|
357
|
+
checksum = "c8dc14463c2552e753ef562961f486ca76f17a857c121db40e9f3ade3f35ab81"
|
358
358
|
dependencies = [
|
359
359
|
"magnus-macros",
|
360
360
|
"rb-sys",
|
@@ -363,13 +363,13 @@ dependencies = [
|
|
363
363
|
|
364
364
|
[[package]]
|
365
365
|
name = "magnus-macros"
|
366
|
-
version = "0.4.
|
366
|
+
version = "0.4.1"
|
367
367
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
368
|
-
checksum = "
|
368
|
+
checksum = "6cc17af1d45442c011aa579d727ec6cff8a69aea8a6bbad26736e7112d749bfb"
|
369
369
|
dependencies = [
|
370
370
|
"proc-macro2",
|
371
371
|
"quote",
|
372
|
-
"syn",
|
372
|
+
"syn 1.0.109",
|
373
373
|
]
|
374
374
|
|
375
375
|
[[package]]
|
@@ -380,9 +380,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
|
380
380
|
|
381
381
|
[[package]]
|
382
382
|
name = "memoffset"
|
383
|
-
version = "0.
|
383
|
+
version = "0.8.0"
|
384
384
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
385
|
-
checksum = "
|
385
|
+
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
|
386
386
|
dependencies = [
|
387
387
|
"autocfg",
|
388
388
|
]
|
@@ -393,6 +393,27 @@ version = "0.2.1"
|
|
393
393
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
394
394
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
395
395
|
|
396
|
+
[[package]]
|
397
|
+
name = "monostate"
|
398
|
+
version = "0.1.6"
|
399
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
400
|
+
checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
|
401
|
+
dependencies = [
|
402
|
+
"monostate-impl",
|
403
|
+
"serde",
|
404
|
+
]
|
405
|
+
|
406
|
+
[[package]]
|
407
|
+
name = "monostate-impl"
|
408
|
+
version = "0.1.6"
|
409
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
410
|
+
checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
|
411
|
+
dependencies = [
|
412
|
+
"proc-macro2",
|
413
|
+
"quote",
|
414
|
+
"syn 2.0.13",
|
415
|
+
]
|
416
|
+
|
396
417
|
[[package]]
|
397
418
|
name = "nom"
|
398
419
|
version = "7.1.3"
|
@@ -421,9 +442,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
|
|
421
442
|
|
422
443
|
[[package]]
|
423
444
|
name = "once_cell"
|
424
|
-
version = "1.17.
|
445
|
+
version = "1.17.1"
|
425
446
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
426
|
-
checksum = "
|
447
|
+
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
|
427
448
|
|
428
449
|
[[package]]
|
429
450
|
name = "onig"
|
@@ -449,9 +470,9 @@ dependencies = [
|
|
449
470
|
|
450
471
|
[[package]]
|
451
472
|
name = "paste"
|
452
|
-
version = "1.0.
|
473
|
+
version = "1.0.12"
|
453
474
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
454
|
-
checksum = "
|
475
|
+
checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
|
455
476
|
|
456
477
|
[[package]]
|
457
478
|
name = "peeking_take_while"
|
@@ -473,18 +494,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
|
473
494
|
|
474
495
|
[[package]]
|
475
496
|
name = "proc-macro2"
|
476
|
-
version = "1.0.
|
497
|
+
version = "1.0.56"
|
477
498
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
478
|
-
checksum = "
|
499
|
+
checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
|
479
500
|
dependencies = [
|
480
501
|
"unicode-ident",
|
481
502
|
]
|
482
503
|
|
483
504
|
[[package]]
|
484
505
|
name = "quote"
|
485
|
-
version = "1.0.
|
506
|
+
version = "1.0.26"
|
486
507
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
487
|
-
checksum = "
|
508
|
+
checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
|
488
509
|
dependencies = [
|
489
510
|
"proc-macro2",
|
490
511
|
]
|
@@ -521,9 +542,9 @@ dependencies = [
|
|
521
542
|
|
522
543
|
[[package]]
|
523
544
|
name = "rayon"
|
524
|
-
version = "1.
|
545
|
+
version = "1.7.0"
|
525
546
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
526
|
-
checksum = "
|
547
|
+
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
|
527
548
|
dependencies = [
|
528
549
|
"either",
|
529
550
|
"rayon-core",
|
@@ -542,9 +563,9 @@ dependencies = [
|
|
542
563
|
|
543
564
|
[[package]]
|
544
565
|
name = "rayon-core"
|
545
|
-
version = "1.
|
566
|
+
version = "1.11.0"
|
546
567
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
547
|
-
checksum = "
|
568
|
+
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
|
548
569
|
dependencies = [
|
549
570
|
"crossbeam-channel",
|
550
571
|
"crossbeam-deque",
|
@@ -554,25 +575,26 @@ dependencies = [
|
|
554
575
|
|
555
576
|
[[package]]
|
556
577
|
name = "rb-sys"
|
557
|
-
version = "0.9.
|
578
|
+
version = "0.9.71"
|
558
579
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
559
|
-
checksum = "
|
580
|
+
checksum = "156bfedced1e236600bcaad538477097ff2ed5c6b474e411d15b791e1d24c0f1"
|
560
581
|
dependencies = [
|
561
582
|
"rb-sys-build",
|
562
583
|
]
|
563
584
|
|
564
585
|
[[package]]
|
565
586
|
name = "rb-sys-build"
|
566
|
-
version = "0.9.
|
587
|
+
version = "0.9.71"
|
567
588
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
568
|
-
checksum = "
|
589
|
+
checksum = "5cb2e4a32cbc290b543a74567072ad24b708aff7bb5dde5a68d5690379cd7938"
|
569
590
|
dependencies = [
|
570
591
|
"bindgen",
|
571
592
|
"lazy_static",
|
593
|
+
"proc-macro2",
|
572
594
|
"quote",
|
573
595
|
"regex",
|
574
596
|
"shell-words",
|
575
|
-
"syn",
|
597
|
+
"syn 1.0.109",
|
576
598
|
]
|
577
599
|
|
578
600
|
[[package]]
|
@@ -583,9 +605,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
|
|
583
605
|
|
584
606
|
[[package]]
|
585
607
|
name = "regex"
|
586
|
-
version = "1.7.
|
608
|
+
version = "1.7.3"
|
587
609
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
588
|
-
checksum = "
|
610
|
+
checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
|
589
611
|
dependencies = [
|
590
612
|
"aho-corasick",
|
591
613
|
"memchr",
|
@@ -594,9 +616,9 @@ dependencies = [
|
|
594
616
|
|
595
617
|
[[package]]
|
596
618
|
name = "regex-syntax"
|
597
|
-
version = "0.6.
|
619
|
+
version = "0.6.29"
|
598
620
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
599
|
-
checksum = "
|
621
|
+
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
|
600
622
|
|
601
623
|
[[package]]
|
602
624
|
name = "rustc-hash"
|
@@ -606,9 +628,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
606
628
|
|
607
629
|
[[package]]
|
608
630
|
name = "ryu"
|
609
|
-
version = "1.0.
|
631
|
+
version = "1.0.13"
|
610
632
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
611
|
-
checksum = "
|
633
|
+
checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
|
612
634
|
|
613
635
|
[[package]]
|
614
636
|
name = "scopeguard"
|
@@ -618,29 +640,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
|
618
640
|
|
619
641
|
[[package]]
|
620
642
|
name = "serde"
|
621
|
-
version = "1.0.
|
643
|
+
version = "1.0.159"
|
622
644
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
623
|
-
checksum = "
|
645
|
+
checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
|
624
646
|
dependencies = [
|
625
647
|
"serde_derive",
|
626
648
|
]
|
627
649
|
|
628
650
|
[[package]]
|
629
651
|
name = "serde_derive"
|
630
|
-
version = "1.0.
|
652
|
+
version = "1.0.159"
|
631
653
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
632
|
-
checksum = "
|
654
|
+
checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
|
633
655
|
dependencies = [
|
634
656
|
"proc-macro2",
|
635
657
|
"quote",
|
636
|
-
"syn",
|
658
|
+
"syn 2.0.13",
|
637
659
|
]
|
638
660
|
|
639
661
|
[[package]]
|
640
662
|
name = "serde_json"
|
641
|
-
version = "1.0.
|
663
|
+
version = "1.0.95"
|
642
664
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
643
|
-
checksum = "
|
665
|
+
checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
|
644
666
|
dependencies = [
|
645
667
|
"itoa",
|
646
668
|
"ryu",
|
@@ -685,9 +707,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
|
685
707
|
|
686
708
|
[[package]]
|
687
709
|
name = "syn"
|
688
|
-
version = "1.0.
|
710
|
+
version = "1.0.109"
|
689
711
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
690
|
-
checksum = "
|
712
|
+
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
713
|
+
dependencies = [
|
714
|
+
"proc-macro2",
|
715
|
+
"quote",
|
716
|
+
"unicode-ident",
|
717
|
+
]
|
718
|
+
|
719
|
+
[[package]]
|
720
|
+
name = "syn"
|
721
|
+
version = "2.0.13"
|
722
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
723
|
+
checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
|
691
724
|
dependencies = [
|
692
725
|
"proc-macro2",
|
693
726
|
"quote",
|
@@ -696,38 +729,39 @@ dependencies = [
|
|
696
729
|
|
697
730
|
[[package]]
|
698
731
|
name = "thiserror"
|
699
|
-
version = "1.0.
|
732
|
+
version = "1.0.40"
|
700
733
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
701
|
-
checksum = "
|
734
|
+
checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
|
702
735
|
dependencies = [
|
703
736
|
"thiserror-impl",
|
704
737
|
]
|
705
738
|
|
706
739
|
[[package]]
|
707
740
|
name = "thiserror-impl"
|
708
|
-
version = "1.0.
|
741
|
+
version = "1.0.40"
|
709
742
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
710
|
-
checksum = "
|
743
|
+
checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
|
711
744
|
dependencies = [
|
712
745
|
"proc-macro2",
|
713
746
|
"quote",
|
714
|
-
"syn",
|
747
|
+
"syn 2.0.13",
|
715
748
|
]
|
716
749
|
|
717
750
|
[[package]]
|
718
751
|
name = "tokenizers"
|
719
|
-
version = "0.3.
|
752
|
+
version = "0.3.3"
|
720
753
|
dependencies = [
|
721
754
|
"magnus",
|
722
755
|
"onig",
|
723
756
|
"serde",
|
724
|
-
"tokenizers 0.13.
|
757
|
+
"tokenizers 0.13.3",
|
725
758
|
]
|
726
759
|
|
727
760
|
[[package]]
|
728
761
|
name = "tokenizers"
|
729
|
-
version = "0.13.
|
730
|
-
source = "
|
762
|
+
version = "0.13.3"
|
763
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
764
|
+
checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5"
|
731
765
|
dependencies = [
|
732
766
|
"aho-corasick",
|
733
767
|
"derive_builder",
|
@@ -738,6 +772,7 @@ dependencies = [
|
|
738
772
|
"lazy_static",
|
739
773
|
"log",
|
740
774
|
"macro_rules_attribute",
|
775
|
+
"monostate",
|
741
776
|
"onig",
|
742
777
|
"paste",
|
743
778
|
"rand",
|
@@ -756,9 +791,9 @@ dependencies = [
|
|
756
791
|
|
757
792
|
[[package]]
|
758
793
|
name = "unicode-ident"
|
759
|
-
version = "1.0.
|
794
|
+
version = "1.0.8"
|
760
795
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
761
|
-
checksum = "
|
796
|
+
checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
|
762
797
|
|
763
798
|
[[package]]
|
764
799
|
name = "unicode-normalization-alignments"
|
@@ -832,42 +867,42 @@ dependencies = [
|
|
832
867
|
|
833
868
|
[[package]]
|
834
869
|
name = "windows_aarch64_gnullvm"
|
835
|
-
version = "0.42.
|
870
|
+
version = "0.42.2"
|
836
871
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
837
|
-
checksum = "
|
872
|
+
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
|
838
873
|
|
839
874
|
[[package]]
|
840
875
|
name = "windows_aarch64_msvc"
|
841
|
-
version = "0.42.
|
876
|
+
version = "0.42.2"
|
842
877
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
843
|
-
checksum = "
|
878
|
+
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
|
844
879
|
|
845
880
|
[[package]]
|
846
881
|
name = "windows_i686_gnu"
|
847
|
-
version = "0.42.
|
882
|
+
version = "0.42.2"
|
848
883
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
849
|
-
checksum = "
|
884
|
+
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
|
850
885
|
|
851
886
|
[[package]]
|
852
887
|
name = "windows_i686_msvc"
|
853
|
-
version = "0.42.
|
888
|
+
version = "0.42.2"
|
854
889
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
855
|
-
checksum = "
|
890
|
+
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
|
856
891
|
|
857
892
|
[[package]]
|
858
893
|
name = "windows_x86_64_gnu"
|
859
|
-
version = "0.42.
|
894
|
+
version = "0.42.2"
|
860
895
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
861
|
-
checksum = "
|
896
|
+
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
|
862
897
|
|
863
898
|
[[package]]
|
864
899
|
name = "windows_x86_64_gnullvm"
|
865
|
-
version = "0.42.
|
900
|
+
version = "0.42.2"
|
866
901
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
867
|
-
checksum = "
|
902
|
+
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
|
868
903
|
|
869
904
|
[[package]]
|
870
905
|
name = "windows_x86_64_msvc"
|
871
|
-
version = "0.42.
|
906
|
+
version = "0.42.2"
|
872
907
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
873
|
-
checksum = "
|
908
|
+
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
|