tokenizers 0.3.2-x86_64-linux → 0.4.0-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5c255ccee12a9f089e4c55c80c8874d2eb236bd8e2d2bc5b07a56b69ff553415
4
- data.tar.gz: fe58ce9b34f3220aec3ae2f8090caf0cc519b3dacc187328a7b364b38f3d430f
3
+ metadata.gz: 32e0ffbced9ba6ede5acdd6ff7a1d2a39efaea0445dc7b42d23524a387c039da
4
+ data.tar.gz: 5c896851ce062d1fa4457038549f9742d16e6f1c32fe3b3f715f5989fc06ca69
5
5
  SHA512:
6
- metadata.gz: 6745dcdda9eb9aecbf7fc31755107de76110ab6ed9d1ffcedbc63e6e034c7d4276ba6de959763ba4463dcbb2ec8cd1783680f558a733977c6799cd427064b6af
7
- data.tar.gz: 959f49d956aafee7556a38b724b812b7c49ff2e22f55b0770ff069bafd7dcf63419d6dc33b65cef71da5b45b020534d6905d6770e95ce6142b9f57e59c3582bc
6
+ metadata.gz: 0b9eb472f71f49273d6b1ade1e1017f66953c35159530b1e944ba67d6575697f27c5b8bd9534fbe506562541664f193ed9e9efaf7e6937db7fb482e98cd74d79
7
+ data.tar.gz: 375f812105aa5a78688e47c7e65ad69d86e65a9112821cb63b470c51a00afd3281bf69162a47e980d7f777e5cca1ad5d2c66ce16fd064aca5cf19b14115b106c
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## 0.4.0 (2023-07-20)
2
+
3
+ - Updated Tokenizers to 0.14.0
4
+ - Dropped support for Ruby < 3
5
+
6
+ ## 0.3.3 (2023-04-09)
7
+
8
+ - Updated Tokenizers to 0.13.3
9
+ - Added `ByteFallback`, `Fuse`, `Replace`, and `Strip` decoders
10
+ - Added `Prepend` normalizer
11
+
1
12
  ## 0.3.2 (2023-03-06)
2
13
 
3
14
  - Added precompiled gem for Linux x86-64 MUSL
data/Cargo.lock CHANGED
@@ -11,6 +11,15 @@ dependencies = [
11
11
  "memchr",
12
12
  ]
13
13
 
14
+ [[package]]
15
+ name = "aho-corasick"
16
+ version = "1.0.5"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
19
+ dependencies = [
20
+ "memchr",
21
+ ]
22
+
14
23
  [[package]]
15
24
  name = "autocfg"
16
25
  version = "1.1.0"
@@ -25,9 +34,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
25
34
 
26
35
  [[package]]
27
36
  name = "bindgen"
28
- version = "0.60.1"
37
+ version = "0.62.0"
29
38
  source = "registry+https://github.com/rust-lang/crates.io-index"
30
- checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
39
+ checksum = "c6720a8b7b2d39dd533285ed438d458f65b31b5c257e6ac7bb3d7e82844dd722"
31
40
  dependencies = [
32
41
  "bitflags",
33
42
  "cexpr",
@@ -40,6 +49,7 @@ dependencies = [
40
49
  "regex",
41
50
  "rustc-hash",
42
51
  "shlex",
52
+ "syn 1.0.109",
43
53
  ]
44
54
 
45
55
  [[package]]
@@ -71,9 +81,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
71
81
 
72
82
  [[package]]
73
83
  name = "clang-sys"
74
- version = "1.4.0"
84
+ version = "1.6.1"
75
85
  source = "registry+https://github.com/rust-lang/crates.io-index"
76
- checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3"
86
+ checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
77
87
  dependencies = [
78
88
  "glob",
79
89
  "libc",
@@ -95,9 +105,9 @@ dependencies = [
95
105
 
96
106
  [[package]]
97
107
  name = "crossbeam-channel"
98
- version = "0.5.6"
108
+ version = "0.5.8"
99
109
  source = "registry+https://github.com/rust-lang/crates.io-index"
100
- checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
110
+ checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
101
111
  dependencies = [
102
112
  "cfg-if",
103
113
  "crossbeam-utils",
@@ -105,9 +115,9 @@ dependencies = [
105
115
 
106
116
  [[package]]
107
117
  name = "crossbeam-deque"
108
- version = "0.8.2"
118
+ version = "0.8.3"
109
119
  source = "registry+https://github.com/rust-lang/crates.io-index"
110
- checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
120
+ checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
111
121
  dependencies = [
112
122
  "cfg-if",
113
123
  "crossbeam-epoch",
@@ -116,9 +126,9 @@ dependencies = [
116
126
 
117
127
  [[package]]
118
128
  name = "crossbeam-epoch"
119
- version = "0.9.13"
129
+ version = "0.9.14"
120
130
  source = "registry+https://github.com/rust-lang/crates.io-index"
121
- checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
131
+ checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
122
132
  dependencies = [
123
133
  "autocfg",
124
134
  "cfg-if",
@@ -129,18 +139,18 @@ dependencies = [
129
139
 
130
140
  [[package]]
131
141
  name = "crossbeam-utils"
132
- version = "0.8.14"
142
+ version = "0.8.15"
133
143
  source = "registry+https://github.com/rust-lang/crates.io-index"
134
- checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
144
+ checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
135
145
  dependencies = [
136
146
  "cfg-if",
137
147
  ]
138
148
 
139
149
  [[package]]
140
150
  name = "darling"
141
- version = "0.14.3"
151
+ version = "0.14.4"
142
152
  source = "registry+https://github.com/rust-lang/crates.io-index"
143
- checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8"
153
+ checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
144
154
  dependencies = [
145
155
  "darling_core",
146
156
  "darling_macro",
@@ -148,27 +158,27 @@ dependencies = [
148
158
 
149
159
  [[package]]
150
160
  name = "darling_core"
151
- version = "0.14.3"
161
+ version = "0.14.4"
152
162
  source = "registry+https://github.com/rust-lang/crates.io-index"
153
- checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb"
163
+ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
154
164
  dependencies = [
155
165
  "fnv",
156
166
  "ident_case",
157
167
  "proc-macro2",
158
168
  "quote",
159
169
  "strsim",
160
- "syn",
170
+ "syn 1.0.109",
161
171
  ]
162
172
 
163
173
  [[package]]
164
174
  name = "darling_macro"
165
- version = "0.14.3"
175
+ version = "0.14.4"
166
176
  source = "registry+https://github.com/rust-lang/crates.io-index"
167
- checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685"
177
+ checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
168
178
  dependencies = [
169
179
  "darling_core",
170
180
  "quote",
171
- "syn",
181
+ "syn 1.0.109",
172
182
  ]
173
183
 
174
184
  [[package]]
@@ -189,7 +199,7 @@ dependencies = [
189
199
  "darling",
190
200
  "proc-macro2",
191
201
  "quote",
192
- "syn",
202
+ "syn 1.0.109",
193
203
  ]
194
204
 
195
205
  [[package]]
@@ -199,7 +209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
199
209
  checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
200
210
  dependencies = [
201
211
  "derive_builder_core",
202
- "syn",
212
+ "syn 1.0.109",
203
213
  ]
204
214
 
205
215
  [[package]]
@@ -231,9 +241,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
231
241
 
232
242
  [[package]]
233
243
  name = "getrandom"
234
- version = "0.2.8"
244
+ version = "0.2.9"
235
245
  source = "registry+https://github.com/rust-lang/crates.io-index"
236
- checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
246
+ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
237
247
  dependencies = [
238
248
  "cfg-if",
239
249
  "libc",
@@ -293,9 +303,9 @@ dependencies = [
293
303
 
294
304
  [[package]]
295
305
  name = "itoa"
296
- version = "1.0.5"
306
+ version = "1.0.6"
297
307
  source = "registry+https://github.com/rust-lang/crates.io-index"
298
- checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
308
+ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
299
309
 
300
310
  [[package]]
301
311
  name = "lazy_static"
@@ -311,9 +321,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
311
321
 
312
322
  [[package]]
313
323
  name = "libc"
314
- version = "0.2.139"
324
+ version = "0.2.141"
315
325
  source = "registry+https://github.com/rust-lang/crates.io-index"
316
- checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
326
+ checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
317
327
 
318
328
  [[package]]
319
329
  name = "libloading"
@@ -352,37 +362,38 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
352
362
 
353
363
  [[package]]
354
364
  name = "magnus"
355
- version = "0.5.0"
365
+ version = "0.6.0"
356
366
  source = "registry+https://github.com/rust-lang/crates.io-index"
357
- checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
367
+ checksum = "68e9585bfe236e88e6b10b6d8eb5349bd0e0009f3f9dff8d2e99a82601b33743"
358
368
  dependencies = [
359
369
  "magnus-macros",
360
370
  "rb-sys",
361
371
  "rb-sys-env",
372
+ "seq-macro",
362
373
  ]
363
374
 
364
375
  [[package]]
365
376
  name = "magnus-macros"
366
- version = "0.4.0"
377
+ version = "0.6.0"
367
378
  source = "registry+https://github.com/rust-lang/crates.io-index"
368
- checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
379
+ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
369
380
  dependencies = [
370
381
  "proc-macro2",
371
382
  "quote",
372
- "syn",
383
+ "syn 2.0.13",
373
384
  ]
374
385
 
375
386
  [[package]]
376
387
  name = "memchr"
377
- version = "2.5.0"
388
+ version = "2.6.3"
378
389
  source = "registry+https://github.com/rust-lang/crates.io-index"
379
- checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
390
+ checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
380
391
 
381
392
  [[package]]
382
393
  name = "memoffset"
383
- version = "0.7.1"
394
+ version = "0.8.0"
384
395
  source = "registry+https://github.com/rust-lang/crates.io-index"
385
- checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
396
+ checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
386
397
  dependencies = [
387
398
  "autocfg",
388
399
  ]
@@ -393,6 +404,27 @@ version = "0.2.1"
393
404
  source = "registry+https://github.com/rust-lang/crates.io-index"
394
405
  checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
395
406
 
407
+ [[package]]
408
+ name = "monostate"
409
+ version = "0.1.6"
410
+ source = "registry+https://github.com/rust-lang/crates.io-index"
411
+ checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
412
+ dependencies = [
413
+ "monostate-impl",
414
+ "serde",
415
+ ]
416
+
417
+ [[package]]
418
+ name = "monostate-impl"
419
+ version = "0.1.6"
420
+ source = "registry+https://github.com/rust-lang/crates.io-index"
421
+ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
422
+ dependencies = [
423
+ "proc-macro2",
424
+ "quote",
425
+ "syn 2.0.13",
426
+ ]
427
+
396
428
  [[package]]
397
429
  name = "nom"
398
430
  version = "7.1.3"
@@ -421,9 +453,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
421
453
 
422
454
  [[package]]
423
455
  name = "once_cell"
424
- version = "1.17.0"
456
+ version = "1.17.1"
425
457
  source = "registry+https://github.com/rust-lang/crates.io-index"
426
- checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
458
+ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
427
459
 
428
460
  [[package]]
429
461
  name = "onig"
@@ -449,9 +481,9 @@ dependencies = [
449
481
 
450
482
  [[package]]
451
483
  name = "paste"
452
- version = "1.0.11"
484
+ version = "1.0.12"
453
485
  source = "registry+https://github.com/rust-lang/crates.io-index"
454
- checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
486
+ checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
455
487
 
456
488
  [[package]]
457
489
  name = "peeking_take_while"
@@ -473,18 +505,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
473
505
 
474
506
  [[package]]
475
507
  name = "proc-macro2"
476
- version = "1.0.51"
508
+ version = "1.0.56"
477
509
  source = "registry+https://github.com/rust-lang/crates.io-index"
478
- checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
510
+ checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
479
511
  dependencies = [
480
512
  "unicode-ident",
481
513
  ]
482
514
 
483
515
  [[package]]
484
516
  name = "quote"
485
- version = "1.0.23"
517
+ version = "1.0.26"
486
518
  source = "registry+https://github.com/rust-lang/crates.io-index"
487
- checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
519
+ checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
488
520
  dependencies = [
489
521
  "proc-macro2",
490
522
  ]
@@ -521,9 +553,9 @@ dependencies = [
521
553
 
522
554
  [[package]]
523
555
  name = "rayon"
524
- version = "1.6.1"
556
+ version = "1.7.0"
525
557
  source = "registry+https://github.com/rust-lang/crates.io-index"
526
- checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
558
+ checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
527
559
  dependencies = [
528
560
  "either",
529
561
  "rayon-core",
@@ -542,9 +574,9 @@ dependencies = [
542
574
 
543
575
  [[package]]
544
576
  name = "rayon-core"
545
- version = "1.10.2"
577
+ version = "1.11.0"
546
578
  source = "registry+https://github.com/rust-lang/crates.io-index"
547
- checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
579
+ checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
548
580
  dependencies = [
549
581
  "crossbeam-channel",
550
582
  "crossbeam-deque",
@@ -554,25 +586,26 @@ dependencies = [
554
586
 
555
587
  [[package]]
556
588
  name = "rb-sys"
557
- version = "0.9.65"
589
+ version = "0.9.79"
558
590
  source = "registry+https://github.com/rust-lang/crates.io-index"
559
- checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
591
+ checksum = "939fb78db3e4f26665c1d4c7b91ca66d3578335a19aba552d4a6445811d07072"
560
592
  dependencies = [
561
593
  "rb-sys-build",
562
594
  ]
563
595
 
564
596
  [[package]]
565
597
  name = "rb-sys-build"
566
- version = "0.9.65"
598
+ version = "0.9.79"
567
599
  source = "registry+https://github.com/rust-lang/crates.io-index"
568
- checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
600
+ checksum = "335a95eb0420d52fa94ef12019df3c2c250c6b19cbb3c60bd05cb7e9c362072c"
569
601
  dependencies = [
570
602
  "bindgen",
571
603
  "lazy_static",
604
+ "proc-macro2",
572
605
  "quote",
573
606
  "regex",
574
607
  "shell-words",
575
- "syn",
608
+ "syn 1.0.109",
576
609
  ]
577
610
 
578
611
  [[package]]
@@ -583,20 +616,32 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
583
616
 
584
617
  [[package]]
585
618
  name = "regex"
586
- version = "1.7.1"
619
+ version = "1.9.5"
620
+ source = "registry+https://github.com/rust-lang/crates.io-index"
621
+ checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
622
+ dependencies = [
623
+ "aho-corasick 1.0.5",
624
+ "memchr",
625
+ "regex-automata",
626
+ "regex-syntax",
627
+ ]
628
+
629
+ [[package]]
630
+ name = "regex-automata"
631
+ version = "0.3.8"
587
632
  source = "registry+https://github.com/rust-lang/crates.io-index"
588
- checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
633
+ checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
589
634
  dependencies = [
590
- "aho-corasick",
635
+ "aho-corasick 1.0.5",
591
636
  "memchr",
592
637
  "regex-syntax",
593
638
  ]
594
639
 
595
640
  [[package]]
596
641
  name = "regex-syntax"
597
- version = "0.6.28"
642
+ version = "0.7.5"
598
643
  source = "registry+https://github.com/rust-lang/crates.io-index"
599
- checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
644
+ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
600
645
 
601
646
  [[package]]
602
647
  name = "rustc-hash"
@@ -606,9 +651,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
606
651
 
607
652
  [[package]]
608
653
  name = "ryu"
609
- version = "1.0.12"
654
+ version = "1.0.13"
610
655
  source = "registry+https://github.com/rust-lang/crates.io-index"
611
- checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
656
+ checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
612
657
 
613
658
  [[package]]
614
659
  name = "scopeguard"
@@ -616,31 +661,37 @@ version = "1.1.0"
616
661
  source = "registry+https://github.com/rust-lang/crates.io-index"
617
662
  checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
618
663
 
664
+ [[package]]
665
+ name = "seq-macro"
666
+ version = "0.3.5"
667
+ source = "registry+https://github.com/rust-lang/crates.io-index"
668
+ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
669
+
619
670
  [[package]]
620
671
  name = "serde"
621
- version = "1.0.152"
672
+ version = "1.0.159"
622
673
  source = "registry+https://github.com/rust-lang/crates.io-index"
623
- checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
674
+ checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
624
675
  dependencies = [
625
676
  "serde_derive",
626
677
  ]
627
678
 
628
679
  [[package]]
629
680
  name = "serde_derive"
630
- version = "1.0.152"
681
+ version = "1.0.159"
631
682
  source = "registry+https://github.com/rust-lang/crates.io-index"
632
- checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
683
+ checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
633
684
  dependencies = [
634
685
  "proc-macro2",
635
686
  "quote",
636
- "syn",
687
+ "syn 2.0.13",
637
688
  ]
638
689
 
639
690
  [[package]]
640
691
  name = "serde_json"
641
- version = "1.0.92"
692
+ version = "1.0.95"
642
693
  source = "registry+https://github.com/rust-lang/crates.io-index"
643
- checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
694
+ checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
644
695
  dependencies = [
645
696
  "itoa",
646
697
  "ryu",
@@ -685,9 +736,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
685
736
 
686
737
  [[package]]
687
738
  name = "syn"
688
- version = "1.0.107"
739
+ version = "1.0.109"
740
+ source = "registry+https://github.com/rust-lang/crates.io-index"
741
+ checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
742
+ dependencies = [
743
+ "proc-macro2",
744
+ "quote",
745
+ "unicode-ident",
746
+ ]
747
+
748
+ [[package]]
749
+ name = "syn"
750
+ version = "2.0.13"
689
751
  source = "registry+https://github.com/rust-lang/crates.io-index"
690
- checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
752
+ checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
691
753
  dependencies = [
692
754
  "proc-macro2",
693
755
  "quote",
@@ -696,40 +758,41 @@ dependencies = [
696
758
 
697
759
  [[package]]
698
760
  name = "thiserror"
699
- version = "1.0.38"
761
+ version = "1.0.40"
700
762
  source = "registry+https://github.com/rust-lang/crates.io-index"
701
- checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
763
+ checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
702
764
  dependencies = [
703
765
  "thiserror-impl",
704
766
  ]
705
767
 
706
768
  [[package]]
707
769
  name = "thiserror-impl"
708
- version = "1.0.38"
770
+ version = "1.0.40"
709
771
  source = "registry+https://github.com/rust-lang/crates.io-index"
710
- checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
772
+ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
711
773
  dependencies = [
712
774
  "proc-macro2",
713
775
  "quote",
714
- "syn",
776
+ "syn 2.0.13",
715
777
  ]
716
778
 
717
779
  [[package]]
718
780
  name = "tokenizers"
719
- version = "0.3.1"
781
+ version = "0.4.0"
720
782
  dependencies = [
721
783
  "magnus",
722
784
  "onig",
723
785
  "serde",
724
- "tokenizers 0.13.2",
786
+ "tokenizers 0.14.0",
725
787
  ]
726
788
 
727
789
  [[package]]
728
790
  name = "tokenizers"
729
- version = "0.13.2"
730
- source = "git+https://github.com/huggingface/tokenizers#fa66caf0abff16bae2213658ffa3e969c5445750"
791
+ version = "0.14.0"
792
+ source = "registry+https://github.com/rust-lang/crates.io-index"
793
+ checksum = "12b515a66453a4d68f03398054f7204fd0dde6b93d3f20ea90b08025ab49b499"
731
794
  dependencies = [
732
- "aho-corasick",
795
+ "aho-corasick 0.7.20",
733
796
  "derive_builder",
734
797
  "esaxx-rs",
735
798
  "getrandom",
@@ -738,6 +801,7 @@ dependencies = [
738
801
  "lazy_static",
739
802
  "log",
740
803
  "macro_rules_attribute",
804
+ "monostate",
741
805
  "onig",
742
806
  "paste",
743
807
  "rand",
@@ -756,9 +820,9 @@ dependencies = [
756
820
 
757
821
  [[package]]
758
822
  name = "unicode-ident"
759
- version = "1.0.6"
823
+ version = "1.0.8"
760
824
  source = "registry+https://github.com/rust-lang/crates.io-index"
761
- checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
825
+ checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
762
826
 
763
827
  [[package]]
764
828
  name = "unicode-normalization-alignments"
@@ -832,42 +896,42 @@ dependencies = [
832
896
 
833
897
  [[package]]
834
898
  name = "windows_aarch64_gnullvm"
835
- version = "0.42.1"
899
+ version = "0.42.2"
836
900
  source = "registry+https://github.com/rust-lang/crates.io-index"
837
- checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
901
+ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
838
902
 
839
903
  [[package]]
840
904
  name = "windows_aarch64_msvc"
841
- version = "0.42.1"
905
+ version = "0.42.2"
842
906
  source = "registry+https://github.com/rust-lang/crates.io-index"
843
- checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
907
+ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
844
908
 
845
909
  [[package]]
846
910
  name = "windows_i686_gnu"
847
- version = "0.42.1"
911
+ version = "0.42.2"
848
912
  source = "registry+https://github.com/rust-lang/crates.io-index"
849
- checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
913
+ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
850
914
 
851
915
  [[package]]
852
916
  name = "windows_i686_msvc"
853
- version = "0.42.1"
917
+ version = "0.42.2"
854
918
  source = "registry+https://github.com/rust-lang/crates.io-index"
855
- checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
919
+ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
856
920
 
857
921
  [[package]]
858
922
  name = "windows_x86_64_gnu"
859
- version = "0.42.1"
923
+ version = "0.42.2"
860
924
  source = "registry+https://github.com/rust-lang/crates.io-index"
861
- checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
925
+ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
862
926
 
863
927
  [[package]]
864
928
  name = "windows_x86_64_gnullvm"
865
- version = "0.42.1"
929
+ version = "0.42.2"
866
930
  source = "registry+https://github.com/rust-lang/crates.io-index"
867
- checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
931
+ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
868
932
 
869
933
  [[package]]
870
934
  name = "windows_x86_64_msvc"
871
- version = "0.42.1"
935
+ version = "0.42.2"
872
936
  source = "registry+https://github.com/rust-lang/crates.io-index"
873
- checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
937
+ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"