tokenizers 0.3.2-aarch64-linux → 0.3.3-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cc23e990e29da0f9ce8fe5ed3cdef9d445073a0659489deae28c4a1e11cf5068
4
- data.tar.gz: de8b19d1e5bd64416a530598519cafcaf23efba30d1b3b722b58fdc488c2f4ac
3
+ metadata.gz: 40b5e60b932d72f40e24cfd440a0af4a222d1654b4ab415c9a30928fd8886822
4
+ data.tar.gz: 235286a3566f121a6a33e81db6770146bcdbb756f689183b5bf41954043a2c4b
5
5
  SHA512:
6
- metadata.gz: e6b40f9792d37c02e287ccaa8fbfb1473bbcb534af327a8b1d8a7805431c037ed52473f400bec33b924b49e0f85486b5762b3f36d9842e87fa735d73efcc5f68
7
- data.tar.gz: 972a9660b327f46de7b6e63733ce9aa602a5d7c522462a519f727e9eca8e715e373572991e55b21cc243358e3ad36cb79705e8f272b3a15e31db45b3714fe524
6
+ metadata.gz: 0cffb67f898157a97a88f2ce878b4f4af81bfbebefad870e072755cf7012066ca96958b51480f30b9da355e95e3382d0fe27c4e4c4c844a7eb3a5cd68279964c
7
+ data.tar.gz: 52944747fb528de0e1fc8c770902e3db9dd37a1b0fdd823e282af95aede1de4482b58343878276823b94c8cc2929a8b04b168c937b7176d1547b35a54185e126
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.3.3 (2023-04-09)
2
+
3
+ - Updated Tokenizers to 0.13.3
4
+ - Added `ByteFallback`, `Fuse`, `Replace`, and `Strip` decoders
5
+ - Added `Prepend` normalizer
6
+
1
7
  ## 0.3.2 (2023-03-06)
2
8
 
3
9
  - Added precompiled gem for Linux x86-64 MUSL
data/Cargo.lock CHANGED
@@ -71,9 +71,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
71
71
 
72
72
  [[package]]
73
73
  name = "clang-sys"
74
- version = "1.4.0"
74
+ version = "1.6.1"
75
75
  source = "registry+https://github.com/rust-lang/crates.io-index"
76
- checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3"
76
+ checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
77
77
  dependencies = [
78
78
  "glob",
79
79
  "libc",
@@ -95,9 +95,9 @@ dependencies = [
95
95
 
96
96
  [[package]]
97
97
  name = "crossbeam-channel"
98
- version = "0.5.6"
98
+ version = "0.5.8"
99
99
  source = "registry+https://github.com/rust-lang/crates.io-index"
100
- checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
100
+ checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
101
101
  dependencies = [
102
102
  "cfg-if",
103
103
  "crossbeam-utils",
@@ -105,9 +105,9 @@ dependencies = [
105
105
 
106
106
  [[package]]
107
107
  name = "crossbeam-deque"
108
- version = "0.8.2"
108
+ version = "0.8.3"
109
109
  source = "registry+https://github.com/rust-lang/crates.io-index"
110
- checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
110
+ checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
111
111
  dependencies = [
112
112
  "cfg-if",
113
113
  "crossbeam-epoch",
@@ -116,9 +116,9 @@ dependencies = [
116
116
 
117
117
  [[package]]
118
118
  name = "crossbeam-epoch"
119
- version = "0.9.13"
119
+ version = "0.9.14"
120
120
  source = "registry+https://github.com/rust-lang/crates.io-index"
121
- checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
121
+ checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
122
122
  dependencies = [
123
123
  "autocfg",
124
124
  "cfg-if",
@@ -129,18 +129,18 @@ dependencies = [
129
129
 
130
130
  [[package]]
131
131
  name = "crossbeam-utils"
132
- version = "0.8.14"
132
+ version = "0.8.15"
133
133
  source = "registry+https://github.com/rust-lang/crates.io-index"
134
- checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
134
+ checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
135
135
  dependencies = [
136
136
  "cfg-if",
137
137
  ]
138
138
 
139
139
  [[package]]
140
140
  name = "darling"
141
- version = "0.14.3"
141
+ version = "0.14.4"
142
142
  source = "registry+https://github.com/rust-lang/crates.io-index"
143
- checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8"
143
+ checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
144
144
  dependencies = [
145
145
  "darling_core",
146
146
  "darling_macro",
@@ -148,27 +148,27 @@ dependencies = [
148
148
 
149
149
  [[package]]
150
150
  name = "darling_core"
151
- version = "0.14.3"
151
+ version = "0.14.4"
152
152
  source = "registry+https://github.com/rust-lang/crates.io-index"
153
- checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb"
153
+ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
154
154
  dependencies = [
155
155
  "fnv",
156
156
  "ident_case",
157
157
  "proc-macro2",
158
158
  "quote",
159
159
  "strsim",
160
- "syn",
160
+ "syn 1.0.109",
161
161
  ]
162
162
 
163
163
  [[package]]
164
164
  name = "darling_macro"
165
- version = "0.14.3"
165
+ version = "0.14.4"
166
166
  source = "registry+https://github.com/rust-lang/crates.io-index"
167
- checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685"
167
+ checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
168
168
  dependencies = [
169
169
  "darling_core",
170
170
  "quote",
171
- "syn",
171
+ "syn 1.0.109",
172
172
  ]
173
173
 
174
174
  [[package]]
@@ -189,7 +189,7 @@ dependencies = [
189
189
  "darling",
190
190
  "proc-macro2",
191
191
  "quote",
192
- "syn",
192
+ "syn 1.0.109",
193
193
  ]
194
194
 
195
195
  [[package]]
@@ -199,7 +199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
199
199
  checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
200
200
  dependencies = [
201
201
  "derive_builder_core",
202
- "syn",
202
+ "syn 1.0.109",
203
203
  ]
204
204
 
205
205
  [[package]]
@@ -231,9 +231,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
231
231
 
232
232
  [[package]]
233
233
  name = "getrandom"
234
- version = "0.2.8"
234
+ version = "0.2.9"
235
235
  source = "registry+https://github.com/rust-lang/crates.io-index"
236
- checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
236
+ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
237
237
  dependencies = [
238
238
  "cfg-if",
239
239
  "libc",
@@ -293,9 +293,9 @@ dependencies = [
293
293
 
294
294
  [[package]]
295
295
  name = "itoa"
296
- version = "1.0.5"
296
+ version = "1.0.6"
297
297
  source = "registry+https://github.com/rust-lang/crates.io-index"
298
- checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
298
+ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
299
299
 
300
300
  [[package]]
301
301
  name = "lazy_static"
@@ -311,9 +311,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
311
311
 
312
312
  [[package]]
313
313
  name = "libc"
314
- version = "0.2.139"
314
+ version = "0.2.141"
315
315
  source = "registry+https://github.com/rust-lang/crates.io-index"
316
- checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
316
+ checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
317
317
 
318
318
  [[package]]
319
319
  name = "libloading"
@@ -352,9 +352,9 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
352
352
 
353
353
  [[package]]
354
354
  name = "magnus"
355
- version = "0.5.0"
355
+ version = "0.5.3"
356
356
  source = "registry+https://github.com/rust-lang/crates.io-index"
357
- checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
357
+ checksum = "c8dc14463c2552e753ef562961f486ca76f17a857c121db40e9f3ade3f35ab81"
358
358
  dependencies = [
359
359
  "magnus-macros",
360
360
  "rb-sys",
@@ -363,13 +363,13 @@ dependencies = [
363
363
 
364
364
  [[package]]
365
365
  name = "magnus-macros"
366
- version = "0.4.0"
366
+ version = "0.4.1"
367
367
  source = "registry+https://github.com/rust-lang/crates.io-index"
368
- checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
368
+ checksum = "6cc17af1d45442c011aa579d727ec6cff8a69aea8a6bbad26736e7112d749bfb"
369
369
  dependencies = [
370
370
  "proc-macro2",
371
371
  "quote",
372
- "syn",
372
+ "syn 1.0.109",
373
373
  ]
374
374
 
375
375
  [[package]]
@@ -380,9 +380,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
380
380
 
381
381
  [[package]]
382
382
  name = "memoffset"
383
- version = "0.7.1"
383
+ version = "0.8.0"
384
384
  source = "registry+https://github.com/rust-lang/crates.io-index"
385
- checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
385
+ checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
386
386
  dependencies = [
387
387
  "autocfg",
388
388
  ]
@@ -393,6 +393,27 @@ version = "0.2.1"
393
393
  source = "registry+https://github.com/rust-lang/crates.io-index"
394
394
  checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
395
395
 
396
+ [[package]]
397
+ name = "monostate"
398
+ version = "0.1.6"
399
+ source = "registry+https://github.com/rust-lang/crates.io-index"
400
+ checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
401
+ dependencies = [
402
+ "monostate-impl",
403
+ "serde",
404
+ ]
405
+
406
+ [[package]]
407
+ name = "monostate-impl"
408
+ version = "0.1.6"
409
+ source = "registry+https://github.com/rust-lang/crates.io-index"
410
+ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
411
+ dependencies = [
412
+ "proc-macro2",
413
+ "quote",
414
+ "syn 2.0.13",
415
+ ]
416
+
396
417
  [[package]]
397
418
  name = "nom"
398
419
  version = "7.1.3"
@@ -421,9 +442,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
421
442
 
422
443
  [[package]]
423
444
  name = "once_cell"
424
- version = "1.17.0"
445
+ version = "1.17.1"
425
446
  source = "registry+https://github.com/rust-lang/crates.io-index"
426
- checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
447
+ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
427
448
 
428
449
  [[package]]
429
450
  name = "onig"
@@ -449,9 +470,9 @@ dependencies = [
449
470
 
450
471
  [[package]]
451
472
  name = "paste"
452
- version = "1.0.11"
473
+ version = "1.0.12"
453
474
  source = "registry+https://github.com/rust-lang/crates.io-index"
454
- checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
475
+ checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
455
476
 
456
477
  [[package]]
457
478
  name = "peeking_take_while"
@@ -473,18 +494,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
473
494
 
474
495
  [[package]]
475
496
  name = "proc-macro2"
476
- version = "1.0.51"
497
+ version = "1.0.56"
477
498
  source = "registry+https://github.com/rust-lang/crates.io-index"
478
- checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
499
+ checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
479
500
  dependencies = [
480
501
  "unicode-ident",
481
502
  ]
482
503
 
483
504
  [[package]]
484
505
  name = "quote"
485
- version = "1.0.23"
506
+ version = "1.0.26"
486
507
  source = "registry+https://github.com/rust-lang/crates.io-index"
487
- checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
508
+ checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
488
509
  dependencies = [
489
510
  "proc-macro2",
490
511
  ]
@@ -521,9 +542,9 @@ dependencies = [
521
542
 
522
543
  [[package]]
523
544
  name = "rayon"
524
- version = "1.6.1"
545
+ version = "1.7.0"
525
546
  source = "registry+https://github.com/rust-lang/crates.io-index"
526
- checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
547
+ checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
527
548
  dependencies = [
528
549
  "either",
529
550
  "rayon-core",
@@ -542,9 +563,9 @@ dependencies = [
542
563
 
543
564
  [[package]]
544
565
  name = "rayon-core"
545
- version = "1.10.2"
566
+ version = "1.11.0"
546
567
  source = "registry+https://github.com/rust-lang/crates.io-index"
547
- checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
568
+ checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
548
569
  dependencies = [
549
570
  "crossbeam-channel",
550
571
  "crossbeam-deque",
@@ -554,25 +575,26 @@ dependencies = [
554
575
 
555
576
  [[package]]
556
577
  name = "rb-sys"
557
- version = "0.9.65"
578
+ version = "0.9.71"
558
579
  source = "registry+https://github.com/rust-lang/crates.io-index"
559
- checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
580
+ checksum = "156bfedced1e236600bcaad538477097ff2ed5c6b474e411d15b791e1d24c0f1"
560
581
  dependencies = [
561
582
  "rb-sys-build",
562
583
  ]
563
584
 
564
585
  [[package]]
565
586
  name = "rb-sys-build"
566
- version = "0.9.65"
587
+ version = "0.9.71"
567
588
  source = "registry+https://github.com/rust-lang/crates.io-index"
568
- checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
589
+ checksum = "5cb2e4a32cbc290b543a74567072ad24b708aff7bb5dde5a68d5690379cd7938"
569
590
  dependencies = [
570
591
  "bindgen",
571
592
  "lazy_static",
593
+ "proc-macro2",
572
594
  "quote",
573
595
  "regex",
574
596
  "shell-words",
575
- "syn",
597
+ "syn 1.0.109",
576
598
  ]
577
599
 
578
600
  [[package]]
@@ -583,9 +605,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
583
605
 
584
606
  [[package]]
585
607
  name = "regex"
586
- version = "1.7.1"
608
+ version = "1.7.3"
587
609
  source = "registry+https://github.com/rust-lang/crates.io-index"
588
- checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
610
+ checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
589
611
  dependencies = [
590
612
  "aho-corasick",
591
613
  "memchr",
@@ -594,9 +616,9 @@ dependencies = [
594
616
 
595
617
  [[package]]
596
618
  name = "regex-syntax"
597
- version = "0.6.28"
619
+ version = "0.6.29"
598
620
  source = "registry+https://github.com/rust-lang/crates.io-index"
599
- checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
621
+ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
600
622
 
601
623
  [[package]]
602
624
  name = "rustc-hash"
@@ -606,9 +628,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
606
628
 
607
629
  [[package]]
608
630
  name = "ryu"
609
- version = "1.0.12"
631
+ version = "1.0.13"
610
632
  source = "registry+https://github.com/rust-lang/crates.io-index"
611
- checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
633
+ checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
612
634
 
613
635
  [[package]]
614
636
  name = "scopeguard"
@@ -618,29 +640,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
618
640
 
619
641
  [[package]]
620
642
  name = "serde"
621
- version = "1.0.152"
643
+ version = "1.0.159"
622
644
  source = "registry+https://github.com/rust-lang/crates.io-index"
623
- checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
645
+ checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
624
646
  dependencies = [
625
647
  "serde_derive",
626
648
  ]
627
649
 
628
650
  [[package]]
629
651
  name = "serde_derive"
630
- version = "1.0.152"
652
+ version = "1.0.159"
631
653
  source = "registry+https://github.com/rust-lang/crates.io-index"
632
- checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
654
+ checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
633
655
  dependencies = [
634
656
  "proc-macro2",
635
657
  "quote",
636
- "syn",
658
+ "syn 2.0.13",
637
659
  ]
638
660
 
639
661
  [[package]]
640
662
  name = "serde_json"
641
- version = "1.0.92"
663
+ version = "1.0.95"
642
664
  source = "registry+https://github.com/rust-lang/crates.io-index"
643
- checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
665
+ checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
644
666
  dependencies = [
645
667
  "itoa",
646
668
  "ryu",
@@ -685,9 +707,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
685
707
 
686
708
  [[package]]
687
709
  name = "syn"
688
- version = "1.0.107"
710
+ version = "1.0.109"
689
711
  source = "registry+https://github.com/rust-lang/crates.io-index"
690
- checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
712
+ checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
713
+ dependencies = [
714
+ "proc-macro2",
715
+ "quote",
716
+ "unicode-ident",
717
+ ]
718
+
719
+ [[package]]
720
+ name = "syn"
721
+ version = "2.0.13"
722
+ source = "registry+https://github.com/rust-lang/crates.io-index"
723
+ checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
691
724
  dependencies = [
692
725
  "proc-macro2",
693
726
  "quote",
@@ -696,38 +729,39 @@ dependencies = [
696
729
 
697
730
  [[package]]
698
731
  name = "thiserror"
699
- version = "1.0.38"
732
+ version = "1.0.40"
700
733
  source = "registry+https://github.com/rust-lang/crates.io-index"
701
- checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
734
+ checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
702
735
  dependencies = [
703
736
  "thiserror-impl",
704
737
  ]
705
738
 
706
739
  [[package]]
707
740
  name = "thiserror-impl"
708
- version = "1.0.38"
741
+ version = "1.0.40"
709
742
  source = "registry+https://github.com/rust-lang/crates.io-index"
710
- checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
743
+ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
711
744
  dependencies = [
712
745
  "proc-macro2",
713
746
  "quote",
714
- "syn",
747
+ "syn 2.0.13",
715
748
  ]
716
749
 
717
750
  [[package]]
718
751
  name = "tokenizers"
719
- version = "0.3.1"
752
+ version = "0.3.3"
720
753
  dependencies = [
721
754
  "magnus",
722
755
  "onig",
723
756
  "serde",
724
- "tokenizers 0.13.2",
757
+ "tokenizers 0.13.3",
725
758
  ]
726
759
 
727
760
  [[package]]
728
761
  name = "tokenizers"
729
- version = "0.13.2"
730
- source = "git+https://github.com/huggingface/tokenizers#fa66caf0abff16bae2213658ffa3e969c5445750"
762
+ version = "0.13.3"
763
+ source = "registry+https://github.com/rust-lang/crates.io-index"
764
+ checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5"
731
765
  dependencies = [
732
766
  "aho-corasick",
733
767
  "derive_builder",
@@ -738,6 +772,7 @@ dependencies = [
738
772
  "lazy_static",
739
773
  "log",
740
774
  "macro_rules_attribute",
775
+ "monostate",
741
776
  "onig",
742
777
  "paste",
743
778
  "rand",
@@ -756,9 +791,9 @@ dependencies = [
756
791
 
757
792
  [[package]]
758
793
  name = "unicode-ident"
759
- version = "1.0.6"
794
+ version = "1.0.8"
760
795
  source = "registry+https://github.com/rust-lang/crates.io-index"
761
- checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
796
+ checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
762
797
 
763
798
  [[package]]
764
799
  name = "unicode-normalization-alignments"
@@ -832,42 +867,42 @@ dependencies = [
832
867
 
833
868
  [[package]]
834
869
  name = "windows_aarch64_gnullvm"
835
- version = "0.42.1"
870
+ version = "0.42.2"
836
871
  source = "registry+https://github.com/rust-lang/crates.io-index"
837
- checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
872
+ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
838
873
 
839
874
  [[package]]
840
875
  name = "windows_aarch64_msvc"
841
- version = "0.42.1"
876
+ version = "0.42.2"
842
877
  source = "registry+https://github.com/rust-lang/crates.io-index"
843
- checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
878
+ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
844
879
 
845
880
  [[package]]
846
881
  name = "windows_i686_gnu"
847
- version = "0.42.1"
882
+ version = "0.42.2"
848
883
  source = "registry+https://github.com/rust-lang/crates.io-index"
849
- checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
884
+ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
850
885
 
851
886
  [[package]]
852
887
  name = "windows_i686_msvc"
853
- version = "0.42.1"
888
+ version = "0.42.2"
854
889
  source = "registry+https://github.com/rust-lang/crates.io-index"
855
- checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
890
+ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
856
891
 
857
892
  [[package]]
858
893
  name = "windows_x86_64_gnu"
859
- version = "0.42.1"
894
+ version = "0.42.2"
860
895
  source = "registry+https://github.com/rust-lang/crates.io-index"
861
- checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
896
+ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
862
897
 
863
898
  [[package]]
864
899
  name = "windows_x86_64_gnullvm"
865
- version = "0.42.1"
900
+ version = "0.42.2"
866
901
  source = "registry+https://github.com/rust-lang/crates.io-index"
867
- checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
902
+ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
868
903
 
869
904
  [[package]]
870
905
  name = "windows_x86_64_msvc"
871
- version = "0.42.1"
906
+ version = "0.42.2"
872
907
  source = "registry+https://github.com/rust-lang/crates.io-index"
873
- checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
908
+ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"