tokenizers 0.3.2-arm64-darwin → 0.4.0-arm64-darwin

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ba8379e40522e811dfd2d2082a241e4c8bb71f0d42fe0219175d65318b5d0f64
4
- data.tar.gz: 9f731c1aa940e44828276cc765adec19a1d3bad6bea84260992b491cc892bce5
3
+ metadata.gz: d6b2979406f4b3507e32e4b01979b06e1c6640a530cceb221b59f3691cf3fb23
4
+ data.tar.gz: 0dd2519b8e900e99d1c5b87bb7ec096751993cd3b25d479388c3a4c46420be91
5
5
  SHA512:
6
- metadata.gz: 322363a3f931652734fcca0a39a363fd7c1538096bc70ed6a3dde80bb88f5690f412ea82bf193b148ae95369c88cc25ecaeba9f2911ad859d93e2e2219a260ff
7
- data.tar.gz: 49061d4df62725e1224024921c2c8bb1ca9e18c60efb630c9345c25d7811508280add5223dabedbd9c1870555189c6650efb137627e31383535a54dffe12d9b1
6
+ metadata.gz: ae5134b13422550d336de9b61943487b014e860476e1b247f2af34d4b174354c5cac34b654f22d706988e1c6c751bf875012d676bac2e31e28937c55136149e0
7
+ data.tar.gz: 48fdb8ac1061cd28328266f12e764bb4c9de85fac8f5dea5eb7a33d0a733d7bc10b68b4f398094cdf259dbb6e4c59927b9afa21ab12d79aacfcda3eea42395ab
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## 0.4.0 (2023-07-20)
2
+
3
+ - Updated Tokenizers to 0.14.0
4
+ - Dropped support for Ruby < 3
5
+
6
+ ## 0.3.3 (2023-04-09)
7
+
8
+ - Updated Tokenizers to 0.13.3
9
+ - Added `ByteFallback`, `Fuse`, `Replace`, and `Strip` decoders
10
+ - Added `Prepend` normalizer
11
+
1
12
  ## 0.3.2 (2023-03-06)
2
13
 
3
14
  - Added precompiled gem for Linux x86-64 MUSL
data/Cargo.lock CHANGED
@@ -11,6 +11,15 @@ dependencies = [
11
11
  "memchr",
12
12
  ]
13
13
 
14
+ [[package]]
15
+ name = "aho-corasick"
16
+ version = "1.0.5"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783"
19
+ dependencies = [
20
+ "memchr",
21
+ ]
22
+
14
23
  [[package]]
15
24
  name = "autocfg"
16
25
  version = "1.1.0"
@@ -25,9 +34,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
25
34
 
26
35
  [[package]]
27
36
  name = "bindgen"
28
- version = "0.60.1"
37
+ version = "0.62.0"
29
38
  source = "registry+https://github.com/rust-lang/crates.io-index"
30
- checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
39
+ checksum = "c6720a8b7b2d39dd533285ed438d458f65b31b5c257e6ac7bb3d7e82844dd722"
31
40
  dependencies = [
32
41
  "bitflags",
33
42
  "cexpr",
@@ -40,6 +49,7 @@ dependencies = [
40
49
  "regex",
41
50
  "rustc-hash",
42
51
  "shlex",
52
+ "syn 1.0.109",
43
53
  ]
44
54
 
45
55
  [[package]]
@@ -71,9 +81,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
71
81
 
72
82
  [[package]]
73
83
  name = "clang-sys"
74
- version = "1.4.0"
84
+ version = "1.6.1"
75
85
  source = "registry+https://github.com/rust-lang/crates.io-index"
76
- checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3"
86
+ checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
77
87
  dependencies = [
78
88
  "glob",
79
89
  "libc",
@@ -95,9 +105,9 @@ dependencies = [
95
105
 
96
106
  [[package]]
97
107
  name = "crossbeam-channel"
98
- version = "0.5.6"
108
+ version = "0.5.8"
99
109
  source = "registry+https://github.com/rust-lang/crates.io-index"
100
- checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
110
+ checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
101
111
  dependencies = [
102
112
  "cfg-if",
103
113
  "crossbeam-utils",
@@ -105,9 +115,9 @@ dependencies = [
105
115
 
106
116
  [[package]]
107
117
  name = "crossbeam-deque"
108
- version = "0.8.2"
118
+ version = "0.8.3"
109
119
  source = "registry+https://github.com/rust-lang/crates.io-index"
110
- checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
120
+ checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
111
121
  dependencies = [
112
122
  "cfg-if",
113
123
  "crossbeam-epoch",
@@ -116,9 +126,9 @@ dependencies = [
116
126
 
117
127
  [[package]]
118
128
  name = "crossbeam-epoch"
119
- version = "0.9.13"
129
+ version = "0.9.14"
120
130
  source = "registry+https://github.com/rust-lang/crates.io-index"
121
- checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
131
+ checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
122
132
  dependencies = [
123
133
  "autocfg",
124
134
  "cfg-if",
@@ -129,18 +139,18 @@ dependencies = [
129
139
 
130
140
  [[package]]
131
141
  name = "crossbeam-utils"
132
- version = "0.8.14"
142
+ version = "0.8.15"
133
143
  source = "registry+https://github.com/rust-lang/crates.io-index"
134
- checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
144
+ checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
135
145
  dependencies = [
136
146
  "cfg-if",
137
147
  ]
138
148
 
139
149
  [[package]]
140
150
  name = "darling"
141
- version = "0.14.3"
151
+ version = "0.14.4"
142
152
  source = "registry+https://github.com/rust-lang/crates.io-index"
143
- checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8"
153
+ checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
144
154
  dependencies = [
145
155
  "darling_core",
146
156
  "darling_macro",
@@ -148,27 +158,27 @@ dependencies = [
148
158
 
149
159
  [[package]]
150
160
  name = "darling_core"
151
- version = "0.14.3"
161
+ version = "0.14.4"
152
162
  source = "registry+https://github.com/rust-lang/crates.io-index"
153
- checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb"
163
+ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
154
164
  dependencies = [
155
165
  "fnv",
156
166
  "ident_case",
157
167
  "proc-macro2",
158
168
  "quote",
159
169
  "strsim",
160
- "syn",
170
+ "syn 1.0.109",
161
171
  ]
162
172
 
163
173
  [[package]]
164
174
  name = "darling_macro"
165
- version = "0.14.3"
175
+ version = "0.14.4"
166
176
  source = "registry+https://github.com/rust-lang/crates.io-index"
167
- checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685"
177
+ checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
168
178
  dependencies = [
169
179
  "darling_core",
170
180
  "quote",
171
- "syn",
181
+ "syn 1.0.109",
172
182
  ]
173
183
 
174
184
  [[package]]
@@ -189,7 +199,7 @@ dependencies = [
189
199
  "darling",
190
200
  "proc-macro2",
191
201
  "quote",
192
- "syn",
202
+ "syn 1.0.109",
193
203
  ]
194
204
 
195
205
  [[package]]
@@ -199,7 +209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
199
209
  checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
200
210
  dependencies = [
201
211
  "derive_builder_core",
202
- "syn",
212
+ "syn 1.0.109",
203
213
  ]
204
214
 
205
215
  [[package]]
@@ -231,9 +241,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
231
241
 
232
242
  [[package]]
233
243
  name = "getrandom"
234
- version = "0.2.8"
244
+ version = "0.2.9"
235
245
  source = "registry+https://github.com/rust-lang/crates.io-index"
236
- checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
246
+ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
237
247
  dependencies = [
238
248
  "cfg-if",
239
249
  "libc",
@@ -293,9 +303,9 @@ dependencies = [
293
303
 
294
304
  [[package]]
295
305
  name = "itoa"
296
- version = "1.0.5"
306
+ version = "1.0.6"
297
307
  source = "registry+https://github.com/rust-lang/crates.io-index"
298
- checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
308
+ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
299
309
 
300
310
  [[package]]
301
311
  name = "lazy_static"
@@ -311,9 +321,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
311
321
 
312
322
  [[package]]
313
323
  name = "libc"
314
- version = "0.2.139"
324
+ version = "0.2.141"
315
325
  source = "registry+https://github.com/rust-lang/crates.io-index"
316
- checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
326
+ checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
317
327
 
318
328
  [[package]]
319
329
  name = "libloading"
@@ -352,37 +362,38 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
352
362
 
353
363
  [[package]]
354
364
  name = "magnus"
355
- version = "0.5.0"
365
+ version = "0.6.0"
356
366
  source = "registry+https://github.com/rust-lang/crates.io-index"
357
- checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
367
+ checksum = "68e9585bfe236e88e6b10b6d8eb5349bd0e0009f3f9dff8d2e99a82601b33743"
358
368
  dependencies = [
359
369
  "magnus-macros",
360
370
  "rb-sys",
361
371
  "rb-sys-env",
372
+ "seq-macro",
362
373
  ]
363
374
 
364
375
  [[package]]
365
376
  name = "magnus-macros"
366
- version = "0.4.0"
377
+ version = "0.6.0"
367
378
  source = "registry+https://github.com/rust-lang/crates.io-index"
368
- checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
379
+ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
369
380
  dependencies = [
370
381
  "proc-macro2",
371
382
  "quote",
372
- "syn",
383
+ "syn 2.0.13",
373
384
  ]
374
385
 
375
386
  [[package]]
376
387
  name = "memchr"
377
- version = "2.5.0"
388
+ version = "2.6.3"
378
389
  source = "registry+https://github.com/rust-lang/crates.io-index"
379
- checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
390
+ checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
380
391
 
381
392
  [[package]]
382
393
  name = "memoffset"
383
- version = "0.7.1"
394
+ version = "0.8.0"
384
395
  source = "registry+https://github.com/rust-lang/crates.io-index"
385
- checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
396
+ checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
386
397
  dependencies = [
387
398
  "autocfg",
388
399
  ]
@@ -393,6 +404,27 @@ version = "0.2.1"
393
404
  source = "registry+https://github.com/rust-lang/crates.io-index"
394
405
  checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
395
406
 
407
+ [[package]]
408
+ name = "monostate"
409
+ version = "0.1.6"
410
+ source = "registry+https://github.com/rust-lang/crates.io-index"
411
+ checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
412
+ dependencies = [
413
+ "monostate-impl",
414
+ "serde",
415
+ ]
416
+
417
+ [[package]]
418
+ name = "monostate-impl"
419
+ version = "0.1.6"
420
+ source = "registry+https://github.com/rust-lang/crates.io-index"
421
+ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
422
+ dependencies = [
423
+ "proc-macro2",
424
+ "quote",
425
+ "syn 2.0.13",
426
+ ]
427
+
396
428
  [[package]]
397
429
  name = "nom"
398
430
  version = "7.1.3"
@@ -421,9 +453,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
421
453
 
422
454
  [[package]]
423
455
  name = "once_cell"
424
- version = "1.17.0"
456
+ version = "1.17.1"
425
457
  source = "registry+https://github.com/rust-lang/crates.io-index"
426
- checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
458
+ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
427
459
 
428
460
  [[package]]
429
461
  name = "onig"
@@ -449,9 +481,9 @@ dependencies = [
449
481
 
450
482
  [[package]]
451
483
  name = "paste"
452
- version = "1.0.11"
484
+ version = "1.0.12"
453
485
  source = "registry+https://github.com/rust-lang/crates.io-index"
454
- checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
486
+ checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
455
487
 
456
488
  [[package]]
457
489
  name = "peeking_take_while"
@@ -473,18 +505,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
473
505
 
474
506
  [[package]]
475
507
  name = "proc-macro2"
476
- version = "1.0.51"
508
+ version = "1.0.56"
477
509
  source = "registry+https://github.com/rust-lang/crates.io-index"
478
- checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
510
+ checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
479
511
  dependencies = [
480
512
  "unicode-ident",
481
513
  ]
482
514
 
483
515
  [[package]]
484
516
  name = "quote"
485
- version = "1.0.23"
517
+ version = "1.0.26"
486
518
  source = "registry+https://github.com/rust-lang/crates.io-index"
487
- checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
519
+ checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
488
520
  dependencies = [
489
521
  "proc-macro2",
490
522
  ]
@@ -521,9 +553,9 @@ dependencies = [
521
553
 
522
554
  [[package]]
523
555
  name = "rayon"
524
- version = "1.6.1"
556
+ version = "1.7.0"
525
557
  source = "registry+https://github.com/rust-lang/crates.io-index"
526
- checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
558
+ checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
527
559
  dependencies = [
528
560
  "either",
529
561
  "rayon-core",
@@ -542,9 +574,9 @@ dependencies = [
542
574
 
543
575
  [[package]]
544
576
  name = "rayon-core"
545
- version = "1.10.2"
577
+ version = "1.11.0"
546
578
  source = "registry+https://github.com/rust-lang/crates.io-index"
547
- checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
579
+ checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
548
580
  dependencies = [
549
581
  "crossbeam-channel",
550
582
  "crossbeam-deque",
@@ -554,25 +586,26 @@ dependencies = [
554
586
 
555
587
  [[package]]
556
588
  name = "rb-sys"
557
- version = "0.9.65"
589
+ version = "0.9.79"
558
590
  source = "registry+https://github.com/rust-lang/crates.io-index"
559
- checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
591
+ checksum = "939fb78db3e4f26665c1d4c7b91ca66d3578335a19aba552d4a6445811d07072"
560
592
  dependencies = [
561
593
  "rb-sys-build",
562
594
  ]
563
595
 
564
596
  [[package]]
565
597
  name = "rb-sys-build"
566
- version = "0.9.65"
598
+ version = "0.9.79"
567
599
  source = "registry+https://github.com/rust-lang/crates.io-index"
568
- checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
600
+ checksum = "335a95eb0420d52fa94ef12019df3c2c250c6b19cbb3c60bd05cb7e9c362072c"
569
601
  dependencies = [
570
602
  "bindgen",
571
603
  "lazy_static",
604
+ "proc-macro2",
572
605
  "quote",
573
606
  "regex",
574
607
  "shell-words",
575
- "syn",
608
+ "syn 1.0.109",
576
609
  ]
577
610
 
578
611
  [[package]]
@@ -583,20 +616,32 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
583
616
 
584
617
  [[package]]
585
618
  name = "regex"
586
- version = "1.7.1"
619
+ version = "1.9.5"
620
+ source = "registry+https://github.com/rust-lang/crates.io-index"
621
+ checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
622
+ dependencies = [
623
+ "aho-corasick 1.0.5",
624
+ "memchr",
625
+ "regex-automata",
626
+ "regex-syntax",
627
+ ]
628
+
629
+ [[package]]
630
+ name = "regex-automata"
631
+ version = "0.3.8"
587
632
  source = "registry+https://github.com/rust-lang/crates.io-index"
588
- checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
633
+ checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
589
634
  dependencies = [
590
- "aho-corasick",
635
+ "aho-corasick 1.0.5",
591
636
  "memchr",
592
637
  "regex-syntax",
593
638
  ]
594
639
 
595
640
  [[package]]
596
641
  name = "regex-syntax"
597
- version = "0.6.28"
642
+ version = "0.7.5"
598
643
  source = "registry+https://github.com/rust-lang/crates.io-index"
599
- checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
644
+ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
600
645
 
601
646
  [[package]]
602
647
  name = "rustc-hash"
@@ -606,9 +651,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
606
651
 
607
652
  [[package]]
608
653
  name = "ryu"
609
- version = "1.0.12"
654
+ version = "1.0.13"
610
655
  source = "registry+https://github.com/rust-lang/crates.io-index"
611
- checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
656
+ checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
612
657
 
613
658
  [[package]]
614
659
  name = "scopeguard"
@@ -616,31 +661,37 @@ version = "1.1.0"
616
661
  source = "registry+https://github.com/rust-lang/crates.io-index"
617
662
  checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
618
663
 
664
+ [[package]]
665
+ name = "seq-macro"
666
+ version = "0.3.5"
667
+ source = "registry+https://github.com/rust-lang/crates.io-index"
668
+ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
669
+
619
670
  [[package]]
620
671
  name = "serde"
621
- version = "1.0.152"
672
+ version = "1.0.159"
622
673
  source = "registry+https://github.com/rust-lang/crates.io-index"
623
- checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
674
+ checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
624
675
  dependencies = [
625
676
  "serde_derive",
626
677
  ]
627
678
 
628
679
  [[package]]
629
680
  name = "serde_derive"
630
- version = "1.0.152"
681
+ version = "1.0.159"
631
682
  source = "registry+https://github.com/rust-lang/crates.io-index"
632
- checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
683
+ checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
633
684
  dependencies = [
634
685
  "proc-macro2",
635
686
  "quote",
636
- "syn",
687
+ "syn 2.0.13",
637
688
  ]
638
689
 
639
690
  [[package]]
640
691
  name = "serde_json"
641
- version = "1.0.92"
692
+ version = "1.0.95"
642
693
  source = "registry+https://github.com/rust-lang/crates.io-index"
643
- checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
694
+ checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
644
695
  dependencies = [
645
696
  "itoa",
646
697
  "ryu",
@@ -685,9 +736,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
685
736
 
686
737
  [[package]]
687
738
  name = "syn"
688
- version = "1.0.107"
739
+ version = "1.0.109"
740
+ source = "registry+https://github.com/rust-lang/crates.io-index"
741
+ checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
742
+ dependencies = [
743
+ "proc-macro2",
744
+ "quote",
745
+ "unicode-ident",
746
+ ]
747
+
748
+ [[package]]
749
+ name = "syn"
750
+ version = "2.0.13"
689
751
  source = "registry+https://github.com/rust-lang/crates.io-index"
690
- checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
752
+ checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
691
753
  dependencies = [
692
754
  "proc-macro2",
693
755
  "quote",
@@ -696,40 +758,41 @@ dependencies = [
696
758
 
697
759
  [[package]]
698
760
  name = "thiserror"
699
- version = "1.0.38"
761
+ version = "1.0.40"
700
762
  source = "registry+https://github.com/rust-lang/crates.io-index"
701
- checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
763
+ checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
702
764
  dependencies = [
703
765
  "thiserror-impl",
704
766
  ]
705
767
 
706
768
  [[package]]
707
769
  name = "thiserror-impl"
708
- version = "1.0.38"
770
+ version = "1.0.40"
709
771
  source = "registry+https://github.com/rust-lang/crates.io-index"
710
- checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
772
+ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
711
773
  dependencies = [
712
774
  "proc-macro2",
713
775
  "quote",
714
- "syn",
776
+ "syn 2.0.13",
715
777
  ]
716
778
 
717
779
  [[package]]
718
780
  name = "tokenizers"
719
- version = "0.3.1"
781
+ version = "0.4.0"
720
782
  dependencies = [
721
783
  "magnus",
722
784
  "onig",
723
785
  "serde",
724
- "tokenizers 0.13.2",
786
+ "tokenizers 0.14.0",
725
787
  ]
726
788
 
727
789
  [[package]]
728
790
  name = "tokenizers"
729
- version = "0.13.2"
730
- source = "git+https://github.com/huggingface/tokenizers#fa66caf0abff16bae2213658ffa3e969c5445750"
791
+ version = "0.14.0"
792
+ source = "registry+https://github.com/rust-lang/crates.io-index"
793
+ checksum = "12b515a66453a4d68f03398054f7204fd0dde6b93d3f20ea90b08025ab49b499"
731
794
  dependencies = [
732
- "aho-corasick",
795
+ "aho-corasick 0.7.20",
733
796
  "derive_builder",
734
797
  "esaxx-rs",
735
798
  "getrandom",
@@ -738,6 +801,7 @@ dependencies = [
738
801
  "lazy_static",
739
802
  "log",
740
803
  "macro_rules_attribute",
804
+ "monostate",
741
805
  "onig",
742
806
  "paste",
743
807
  "rand",
@@ -756,9 +820,9 @@ dependencies = [
756
820
 
757
821
  [[package]]
758
822
  name = "unicode-ident"
759
- version = "1.0.6"
823
+ version = "1.0.8"
760
824
  source = "registry+https://github.com/rust-lang/crates.io-index"
761
- checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
825
+ checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
762
826
 
763
827
  [[package]]
764
828
  name = "unicode-normalization-alignments"
@@ -832,42 +896,42 @@ dependencies = [
832
896
 
833
897
  [[package]]
834
898
  name = "windows_aarch64_gnullvm"
835
- version = "0.42.1"
899
+ version = "0.42.2"
836
900
  source = "registry+https://github.com/rust-lang/crates.io-index"
837
- checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
901
+ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
838
902
 
839
903
  [[package]]
840
904
  name = "windows_aarch64_msvc"
841
- version = "0.42.1"
905
+ version = "0.42.2"
842
906
  source = "registry+https://github.com/rust-lang/crates.io-index"
843
- checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
907
+ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
844
908
 
845
909
  [[package]]
846
910
  name = "windows_i686_gnu"
847
- version = "0.42.1"
911
+ version = "0.42.2"
848
912
  source = "registry+https://github.com/rust-lang/crates.io-index"
849
- checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
913
+ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
850
914
 
851
915
  [[package]]
852
916
  name = "windows_i686_msvc"
853
- version = "0.42.1"
917
+ version = "0.42.2"
854
918
  source = "registry+https://github.com/rust-lang/crates.io-index"
855
- checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
919
+ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
856
920
 
857
921
  [[package]]
858
922
  name = "windows_x86_64_gnu"
859
- version = "0.42.1"
923
+ version = "0.42.2"
860
924
  source = "registry+https://github.com/rust-lang/crates.io-index"
861
- checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
925
+ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
862
926
 
863
927
  [[package]]
864
928
  name = "windows_x86_64_gnullvm"
865
- version = "0.42.1"
929
+ version = "0.42.2"
866
930
  source = "registry+https://github.com/rust-lang/crates.io-index"
867
- checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
931
+ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
868
932
 
869
933
  [[package]]
870
934
  name = "windows_x86_64_msvc"
871
- version = "0.42.1"
935
+ version = "0.42.2"
872
936
  source = "registry+https://github.com/rust-lang/crates.io-index"
873
- checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
937
+ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"