tokenizers 0.5.3-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +107 -0
  3. data/Cargo.lock +898 -0
  4. data/Cargo.toml +6 -0
  5. data/LICENSE-THIRD-PARTY.txt +17427 -0
  6. data/LICENSE.txt +202 -0
  7. data/README.md +105 -0
  8. data/lib/tokenizers/3.1/tokenizers.so +0 -0
  9. data/lib/tokenizers/3.2/tokenizers.so +0 -0
  10. data/lib/tokenizers/3.3/tokenizers.so +0 -0
  11. data/lib/tokenizers/added_token.rb +7 -0
  12. data/lib/tokenizers/char_bpe_tokenizer.rb +22 -0
  13. data/lib/tokenizers/decoders/bpe_decoder.rb +9 -0
  14. data/lib/tokenizers/decoders/ctc.rb +9 -0
  15. data/lib/tokenizers/decoders/metaspace.rb +9 -0
  16. data/lib/tokenizers/decoders/strip.rb +9 -0
  17. data/lib/tokenizers/decoders/word_piece.rb +9 -0
  18. data/lib/tokenizers/encoding.rb +19 -0
  19. data/lib/tokenizers/from_pretrained.rb +125 -0
  20. data/lib/tokenizers/models/bpe.rb +9 -0
  21. data/lib/tokenizers/models/unigram.rb +9 -0
  22. data/lib/tokenizers/models/word_level.rb +13 -0
  23. data/lib/tokenizers/models/word_piece.rb +9 -0
  24. data/lib/tokenizers/normalizers/bert_normalizer.rb +9 -0
  25. data/lib/tokenizers/normalizers/prepend.rb +9 -0
  26. data/lib/tokenizers/normalizers/strip.rb +9 -0
  27. data/lib/tokenizers/pre_tokenizers/byte_level.rb +9 -0
  28. data/lib/tokenizers/pre_tokenizers/digits.rb +9 -0
  29. data/lib/tokenizers/pre_tokenizers/metaspace.rb +9 -0
  30. data/lib/tokenizers/pre_tokenizers/punctuation.rb +9 -0
  31. data/lib/tokenizers/pre_tokenizers/split.rb +9 -0
  32. data/lib/tokenizers/processors/byte_level.rb +9 -0
  33. data/lib/tokenizers/processors/roberta_processing.rb +9 -0
  34. data/lib/tokenizers/processors/template_processing.rb +9 -0
  35. data/lib/tokenizers/tokenizer.rb +45 -0
  36. data/lib/tokenizers/trainers/bpe_trainer.rb +9 -0
  37. data/lib/tokenizers/trainers/unigram_trainer.rb +26 -0
  38. data/lib/tokenizers/trainers/word_level_trainer.rb +9 -0
  39. data/lib/tokenizers/trainers/word_piece_trainer.rb +26 -0
  40. data/lib/tokenizers/version.rb +3 -0
  41. data/lib/tokenizers.rb +62 -0
  42. metadata +85 -0
data/Cargo.lock ADDED
@@ -0,0 +1,898 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 3
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.3"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "base64"
16
+ version = "0.13.1"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
19
+
20
+ [[package]]
21
+ name = "bindgen"
22
+ version = "0.69.4"
23
+ source = "registry+https://github.com/rust-lang/crates.io-index"
24
+ checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
25
+ dependencies = [
26
+ "bitflags 2.6.0",
27
+ "cexpr",
28
+ "clang-sys",
29
+ "itertools 0.12.1",
30
+ "lazy_static",
31
+ "lazycell",
32
+ "proc-macro2",
33
+ "quote",
34
+ "regex",
35
+ "rustc-hash",
36
+ "shlex",
37
+ "syn",
38
+ ]
39
+
40
+ [[package]]
41
+ name = "bitflags"
42
+ version = "1.3.2"
43
+ source = "registry+https://github.com/rust-lang/crates.io-index"
44
+ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
45
+
46
+ [[package]]
47
+ name = "bitflags"
48
+ version = "2.6.0"
49
+ source = "registry+https://github.com/rust-lang/crates.io-index"
50
+ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
51
+
52
+ [[package]]
53
+ name = "byteorder"
54
+ version = "1.5.0"
55
+ source = "registry+https://github.com/rust-lang/crates.io-index"
56
+ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
57
+
58
+ [[package]]
59
+ name = "cc"
60
+ version = "1.1.15"
61
+ source = "registry+https://github.com/rust-lang/crates.io-index"
62
+ checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
63
+ dependencies = [
64
+ "shlex",
65
+ ]
66
+
67
+ [[package]]
68
+ name = "cexpr"
69
+ version = "0.6.0"
70
+ source = "registry+https://github.com/rust-lang/crates.io-index"
71
+ checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
72
+ dependencies = [
73
+ "nom",
74
+ ]
75
+
76
+ [[package]]
77
+ name = "cfg-if"
78
+ version = "1.0.0"
79
+ source = "registry+https://github.com/rust-lang/crates.io-index"
80
+ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
81
+
82
+ [[package]]
83
+ name = "clang-sys"
84
+ version = "1.8.1"
85
+ source = "registry+https://github.com/rust-lang/crates.io-index"
86
+ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
87
+ dependencies = [
88
+ "glob",
89
+ "libc",
90
+ "libloading",
91
+ ]
92
+
93
+ [[package]]
94
+ name = "console"
95
+ version = "0.15.8"
96
+ source = "registry+https://github.com/rust-lang/crates.io-index"
97
+ checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
98
+ dependencies = [
99
+ "encode_unicode",
100
+ "lazy_static",
101
+ "libc",
102
+ "unicode-width",
103
+ "windows-sys",
104
+ ]
105
+
106
+ [[package]]
107
+ name = "crossbeam-deque"
108
+ version = "0.8.5"
109
+ source = "registry+https://github.com/rust-lang/crates.io-index"
110
+ checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
111
+ dependencies = [
112
+ "crossbeam-epoch",
113
+ "crossbeam-utils",
114
+ ]
115
+
116
+ [[package]]
117
+ name = "crossbeam-epoch"
118
+ version = "0.9.18"
119
+ source = "registry+https://github.com/rust-lang/crates.io-index"
120
+ checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
121
+ dependencies = [
122
+ "crossbeam-utils",
123
+ ]
124
+
125
+ [[package]]
126
+ name = "crossbeam-utils"
127
+ version = "0.8.20"
128
+ source = "registry+https://github.com/rust-lang/crates.io-index"
129
+ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
130
+
131
+ [[package]]
132
+ name = "darling"
133
+ version = "0.20.10"
134
+ source = "registry+https://github.com/rust-lang/crates.io-index"
135
+ checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
136
+ dependencies = [
137
+ "darling_core",
138
+ "darling_macro",
139
+ ]
140
+
141
+ [[package]]
142
+ name = "darling_core"
143
+ version = "0.20.10"
144
+ source = "registry+https://github.com/rust-lang/crates.io-index"
145
+ checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
146
+ dependencies = [
147
+ "fnv",
148
+ "ident_case",
149
+ "proc-macro2",
150
+ "quote",
151
+ "strsim",
152
+ "syn",
153
+ ]
154
+
155
+ [[package]]
156
+ name = "darling_macro"
157
+ version = "0.20.10"
158
+ source = "registry+https://github.com/rust-lang/crates.io-index"
159
+ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
160
+ dependencies = [
161
+ "darling_core",
162
+ "quote",
163
+ "syn",
164
+ ]
165
+
166
+ [[package]]
167
+ name = "derive_builder"
168
+ version = "0.20.0"
169
+ source = "registry+https://github.com/rust-lang/crates.io-index"
170
+ checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
171
+ dependencies = [
172
+ "derive_builder_macro",
173
+ ]
174
+
175
+ [[package]]
176
+ name = "derive_builder_core"
177
+ version = "0.20.0"
178
+ source = "registry+https://github.com/rust-lang/crates.io-index"
179
+ checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
180
+ dependencies = [
181
+ "darling",
182
+ "proc-macro2",
183
+ "quote",
184
+ "syn",
185
+ ]
186
+
187
+ [[package]]
188
+ name = "derive_builder_macro"
189
+ version = "0.20.0"
190
+ source = "registry+https://github.com/rust-lang/crates.io-index"
191
+ checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
192
+ dependencies = [
193
+ "derive_builder_core",
194
+ "syn",
195
+ ]
196
+
197
+ [[package]]
198
+ name = "either"
199
+ version = "1.13.0"
200
+ source = "registry+https://github.com/rust-lang/crates.io-index"
201
+ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
202
+
203
+ [[package]]
204
+ name = "encode_unicode"
205
+ version = "0.3.6"
206
+ source = "registry+https://github.com/rust-lang/crates.io-index"
207
+ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
208
+
209
+ [[package]]
210
+ name = "esaxx-rs"
211
+ version = "0.1.10"
212
+ source = "registry+https://github.com/rust-lang/crates.io-index"
213
+ checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
214
+ dependencies = [
215
+ "cc",
216
+ ]
217
+
218
+ [[package]]
219
+ name = "fnv"
220
+ version = "1.0.7"
221
+ source = "registry+https://github.com/rust-lang/crates.io-index"
222
+ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
223
+
224
+ [[package]]
225
+ name = "getrandom"
226
+ version = "0.2.15"
227
+ source = "registry+https://github.com/rust-lang/crates.io-index"
228
+ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
229
+ dependencies = [
230
+ "cfg-if",
231
+ "libc",
232
+ "wasi",
233
+ ]
234
+
235
+ [[package]]
236
+ name = "glob"
237
+ version = "0.3.1"
238
+ source = "registry+https://github.com/rust-lang/crates.io-index"
239
+ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
240
+
241
+ [[package]]
242
+ name = "ident_case"
243
+ version = "1.0.1"
244
+ source = "registry+https://github.com/rust-lang/crates.io-index"
245
+ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
246
+
247
+ [[package]]
248
+ name = "indicatif"
249
+ version = "0.17.8"
250
+ source = "registry+https://github.com/rust-lang/crates.io-index"
251
+ checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3"
252
+ dependencies = [
253
+ "console",
254
+ "instant",
255
+ "number_prefix",
256
+ "portable-atomic",
257
+ "unicode-width",
258
+ ]
259
+
260
+ [[package]]
261
+ name = "instant"
262
+ version = "0.1.13"
263
+ source = "registry+https://github.com/rust-lang/crates.io-index"
264
+ checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
265
+ dependencies = [
266
+ "cfg-if",
267
+ ]
268
+
269
+ [[package]]
270
+ name = "itertools"
271
+ version = "0.11.0"
272
+ source = "registry+https://github.com/rust-lang/crates.io-index"
273
+ checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
274
+ dependencies = [
275
+ "either",
276
+ ]
277
+
278
+ [[package]]
279
+ name = "itertools"
280
+ version = "0.12.1"
281
+ source = "registry+https://github.com/rust-lang/crates.io-index"
282
+ checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
283
+ dependencies = [
284
+ "either",
285
+ ]
286
+
287
+ [[package]]
288
+ name = "itoa"
289
+ version = "1.0.11"
290
+ source = "registry+https://github.com/rust-lang/crates.io-index"
291
+ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
292
+
293
+ [[package]]
294
+ name = "lazy_static"
295
+ version = "1.5.0"
296
+ source = "registry+https://github.com/rust-lang/crates.io-index"
297
+ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
298
+
299
+ [[package]]
300
+ name = "lazycell"
301
+ version = "1.3.0"
302
+ source = "registry+https://github.com/rust-lang/crates.io-index"
303
+ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
304
+
305
+ [[package]]
306
+ name = "libc"
307
+ version = "0.2.158"
308
+ source = "registry+https://github.com/rust-lang/crates.io-index"
309
+ checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
310
+
311
+ [[package]]
312
+ name = "libloading"
313
+ version = "0.8.5"
314
+ source = "registry+https://github.com/rust-lang/crates.io-index"
315
+ checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
316
+ dependencies = [
317
+ "cfg-if",
318
+ "windows-targets",
319
+ ]
320
+
321
+ [[package]]
322
+ name = "log"
323
+ version = "0.4.22"
324
+ source = "registry+https://github.com/rust-lang/crates.io-index"
325
+ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
326
+
327
+ [[package]]
328
+ name = "macro_rules_attribute"
329
+ version = "0.2.0"
330
+ source = "registry+https://github.com/rust-lang/crates.io-index"
331
+ checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13"
332
+ dependencies = [
333
+ "macro_rules_attribute-proc_macro",
334
+ "paste",
335
+ ]
336
+
337
+ [[package]]
338
+ name = "macro_rules_attribute-proc_macro"
339
+ version = "0.2.0"
340
+ source = "registry+https://github.com/rust-lang/crates.io-index"
341
+ checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
342
+
343
+ [[package]]
344
+ name = "magnus"
345
+ version = "0.7.1"
346
+ source = "registry+https://github.com/rust-lang/crates.io-index"
347
+ checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
348
+ dependencies = [
349
+ "magnus-macros",
350
+ "rb-sys",
351
+ "rb-sys-env",
352
+ "seq-macro",
353
+ ]
354
+
355
+ [[package]]
356
+ name = "magnus-macros"
357
+ version = "0.6.0"
358
+ source = "registry+https://github.com/rust-lang/crates.io-index"
359
+ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
360
+ dependencies = [
361
+ "proc-macro2",
362
+ "quote",
363
+ "syn",
364
+ ]
365
+
366
+ [[package]]
367
+ name = "memchr"
368
+ version = "2.7.4"
369
+ source = "registry+https://github.com/rust-lang/crates.io-index"
370
+ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
371
+
372
+ [[package]]
373
+ name = "minimal-lexical"
374
+ version = "0.2.1"
375
+ source = "registry+https://github.com/rust-lang/crates.io-index"
376
+ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
377
+
378
+ [[package]]
379
+ name = "monostate"
380
+ version = "0.1.13"
381
+ source = "registry+https://github.com/rust-lang/crates.io-index"
382
+ checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e"
383
+ dependencies = [
384
+ "monostate-impl",
385
+ "serde",
386
+ ]
387
+
388
+ [[package]]
389
+ name = "monostate-impl"
390
+ version = "0.1.13"
391
+ source = "registry+https://github.com/rust-lang/crates.io-index"
392
+ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0"
393
+ dependencies = [
394
+ "proc-macro2",
395
+ "quote",
396
+ "syn",
397
+ ]
398
+
399
+ [[package]]
400
+ name = "nom"
401
+ version = "7.1.3"
402
+ source = "registry+https://github.com/rust-lang/crates.io-index"
403
+ checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
404
+ dependencies = [
405
+ "memchr",
406
+ "minimal-lexical",
407
+ ]
408
+
409
+ [[package]]
410
+ name = "number_prefix"
411
+ version = "0.4.0"
412
+ source = "registry+https://github.com/rust-lang/crates.io-index"
413
+ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
414
+
415
+ [[package]]
416
+ name = "once_cell"
417
+ version = "1.19.0"
418
+ source = "registry+https://github.com/rust-lang/crates.io-index"
419
+ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
420
+
421
+ [[package]]
422
+ name = "onig"
423
+ version = "6.4.0"
424
+ source = "registry+https://github.com/rust-lang/crates.io-index"
425
+ checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
426
+ dependencies = [
427
+ "bitflags 1.3.2",
428
+ "libc",
429
+ "once_cell",
430
+ "onig_sys",
431
+ ]
432
+
433
+ [[package]]
434
+ name = "onig_sys"
435
+ version = "69.8.1"
436
+ source = "registry+https://github.com/rust-lang/crates.io-index"
437
+ checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
438
+ dependencies = [
439
+ "cc",
440
+ "pkg-config",
441
+ ]
442
+
443
+ [[package]]
444
+ name = "paste"
445
+ version = "1.0.15"
446
+ source = "registry+https://github.com/rust-lang/crates.io-index"
447
+ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
448
+
449
+ [[package]]
450
+ name = "pkg-config"
451
+ version = "0.3.30"
452
+ source = "registry+https://github.com/rust-lang/crates.io-index"
453
+ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
454
+
455
+ [[package]]
456
+ name = "portable-atomic"
457
+ version = "1.7.0"
458
+ source = "registry+https://github.com/rust-lang/crates.io-index"
459
+ checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
460
+
461
+ [[package]]
462
+ name = "ppv-lite86"
463
+ version = "0.2.20"
464
+ source = "registry+https://github.com/rust-lang/crates.io-index"
465
+ checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
466
+ dependencies = [
467
+ "zerocopy",
468
+ ]
469
+
470
+ [[package]]
471
+ name = "proc-macro2"
472
+ version = "1.0.86"
473
+ source = "registry+https://github.com/rust-lang/crates.io-index"
474
+ checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
475
+ dependencies = [
476
+ "unicode-ident",
477
+ ]
478
+
479
+ [[package]]
480
+ name = "quote"
481
+ version = "1.0.37"
482
+ source = "registry+https://github.com/rust-lang/crates.io-index"
483
+ checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
484
+ dependencies = [
485
+ "proc-macro2",
486
+ ]
487
+
488
+ [[package]]
489
+ name = "rand"
490
+ version = "0.8.5"
491
+ source = "registry+https://github.com/rust-lang/crates.io-index"
492
+ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
493
+ dependencies = [
494
+ "libc",
495
+ "rand_chacha",
496
+ "rand_core",
497
+ ]
498
+
499
+ [[package]]
500
+ name = "rand_chacha"
501
+ version = "0.3.1"
502
+ source = "registry+https://github.com/rust-lang/crates.io-index"
503
+ checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
504
+ dependencies = [
505
+ "ppv-lite86",
506
+ "rand_core",
507
+ ]
508
+
509
+ [[package]]
510
+ name = "rand_core"
511
+ version = "0.6.4"
512
+ source = "registry+https://github.com/rust-lang/crates.io-index"
513
+ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
514
+ dependencies = [
515
+ "getrandom",
516
+ ]
517
+
518
+ [[package]]
519
+ name = "rayon"
520
+ version = "1.10.0"
521
+ source = "registry+https://github.com/rust-lang/crates.io-index"
522
+ checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
523
+ dependencies = [
524
+ "either",
525
+ "rayon-core",
526
+ ]
527
+
528
+ [[package]]
529
+ name = "rayon-cond"
530
+ version = "0.3.0"
531
+ source = "registry+https://github.com/rust-lang/crates.io-index"
532
+ checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
533
+ dependencies = [
534
+ "either",
535
+ "itertools 0.11.0",
536
+ "rayon",
537
+ ]
538
+
539
+ [[package]]
540
+ name = "rayon-core"
541
+ version = "1.12.1"
542
+ source = "registry+https://github.com/rust-lang/crates.io-index"
543
+ checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
544
+ dependencies = [
545
+ "crossbeam-deque",
546
+ "crossbeam-utils",
547
+ ]
548
+
549
+ [[package]]
550
+ name = "rb-sys"
551
+ version = "0.9.102"
552
+ source = "registry+https://github.com/rust-lang/crates.io-index"
553
+ checksum = "df4dec4b1d304c3b308a2cd86b1216ea45dd4361f4e9fa056f108332d0a450c1"
554
+ dependencies = [
555
+ "rb-sys-build",
556
+ ]
557
+
558
+ [[package]]
559
+ name = "rb-sys-build"
560
+ version = "0.9.102"
561
+ source = "registry+https://github.com/rust-lang/crates.io-index"
562
+ checksum = "1d71de3e29d174b8fb17b5d4470f27d7aa2605f8a9d05fda0d3aeff30e05a570"
563
+ dependencies = [
564
+ "bindgen",
565
+ "lazy_static",
566
+ "proc-macro2",
567
+ "quote",
568
+ "regex",
569
+ "shell-words",
570
+ "syn",
571
+ ]
572
+
573
+ [[package]]
574
+ name = "rb-sys-env"
575
+ version = "0.1.2"
576
+ source = "registry+https://github.com/rust-lang/crates.io-index"
577
+ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
578
+
579
+ [[package]]
580
+ name = "regex"
581
+ version = "1.10.6"
582
+ source = "registry+https://github.com/rust-lang/crates.io-index"
583
+ checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
584
+ dependencies = [
585
+ "aho-corasick",
586
+ "memchr",
587
+ "regex-automata",
588
+ "regex-syntax",
589
+ ]
590
+
591
+ [[package]]
592
+ name = "regex-automata"
593
+ version = "0.4.7"
594
+ source = "registry+https://github.com/rust-lang/crates.io-index"
595
+ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
596
+ dependencies = [
597
+ "aho-corasick",
598
+ "memchr",
599
+ "regex-syntax",
600
+ ]
601
+
602
+ [[package]]
603
+ name = "regex-syntax"
604
+ version = "0.8.4"
605
+ source = "registry+https://github.com/rust-lang/crates.io-index"
606
+ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
607
+
608
+ [[package]]
609
+ name = "rustc-hash"
610
+ version = "1.1.0"
611
+ source = "registry+https://github.com/rust-lang/crates.io-index"
612
+ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
613
+
614
+ [[package]]
615
+ name = "ryu"
616
+ version = "1.0.18"
617
+ source = "registry+https://github.com/rust-lang/crates.io-index"
618
+ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
619
+
620
+ [[package]]
621
+ name = "seq-macro"
622
+ version = "0.3.5"
623
+ source = "registry+https://github.com/rust-lang/crates.io-index"
624
+ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
625
+
626
+ [[package]]
627
+ name = "serde"
628
+ version = "1.0.209"
629
+ source = "registry+https://github.com/rust-lang/crates.io-index"
630
+ checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
631
+ dependencies = [
632
+ "serde_derive",
633
+ ]
634
+
635
+ [[package]]
636
+ name = "serde_derive"
637
+ version = "1.0.209"
638
+ source = "registry+https://github.com/rust-lang/crates.io-index"
639
+ checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
640
+ dependencies = [
641
+ "proc-macro2",
642
+ "quote",
643
+ "syn",
644
+ ]
645
+
646
+ [[package]]
647
+ name = "serde_json"
648
+ version = "1.0.127"
649
+ source = "registry+https://github.com/rust-lang/crates.io-index"
650
+ checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
651
+ dependencies = [
652
+ "itoa",
653
+ "memchr",
654
+ "ryu",
655
+ "serde",
656
+ ]
657
+
658
+ [[package]]
659
+ name = "shell-words"
660
+ version = "1.1.0"
661
+ source = "registry+https://github.com/rust-lang/crates.io-index"
662
+ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
663
+
664
+ [[package]]
665
+ name = "shlex"
666
+ version = "1.3.0"
667
+ source = "registry+https://github.com/rust-lang/crates.io-index"
668
+ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
669
+
670
+ [[package]]
671
+ name = "smallvec"
672
+ version = "1.13.2"
673
+ source = "registry+https://github.com/rust-lang/crates.io-index"
674
+ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
675
+
676
+ [[package]]
677
+ name = "spm_precompiled"
678
+ version = "0.1.4"
679
+ source = "registry+https://github.com/rust-lang/crates.io-index"
680
+ checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
681
+ dependencies = [
682
+ "base64",
683
+ "nom",
684
+ "serde",
685
+ "unicode-segmentation",
686
+ ]
687
+
688
+ [[package]]
689
+ name = "strsim"
690
+ version = "0.11.1"
691
+ source = "registry+https://github.com/rust-lang/crates.io-index"
692
+ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
693
+
694
+ [[package]]
695
+ name = "syn"
696
+ version = "2.0.76"
697
+ source = "registry+https://github.com/rust-lang/crates.io-index"
698
+ checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
699
+ dependencies = [
700
+ "proc-macro2",
701
+ "quote",
702
+ "unicode-ident",
703
+ ]
704
+
705
+ [[package]]
706
+ name = "thiserror"
707
+ version = "1.0.63"
708
+ source = "registry+https://github.com/rust-lang/crates.io-index"
709
+ checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
710
+ dependencies = [
711
+ "thiserror-impl",
712
+ ]
713
+
714
+ [[package]]
715
+ name = "thiserror-impl"
716
+ version = "1.0.63"
717
+ source = "registry+https://github.com/rust-lang/crates.io-index"
718
+ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
719
+ dependencies = [
720
+ "proc-macro2",
721
+ "quote",
722
+ "syn",
723
+ ]
724
+
725
+ [[package]]
726
+ name = "tokenizers"
727
+ version = "0.5.3"
728
+ dependencies = [
729
+ "magnus",
730
+ "onig",
731
+ "serde",
732
+ "tokenizers 0.20.0",
733
+ ]
734
+
735
+ [[package]]
736
+ name = "tokenizers"
737
+ version = "0.20.0"
738
+ source = "registry+https://github.com/rust-lang/crates.io-index"
739
+ checksum = "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
740
+ dependencies = [
741
+ "aho-corasick",
742
+ "derive_builder",
743
+ "esaxx-rs",
744
+ "getrandom",
745
+ "indicatif",
746
+ "itertools 0.12.1",
747
+ "lazy_static",
748
+ "log",
749
+ "macro_rules_attribute",
750
+ "monostate",
751
+ "onig",
752
+ "paste",
753
+ "rand",
754
+ "rayon",
755
+ "rayon-cond",
756
+ "regex",
757
+ "regex-syntax",
758
+ "serde",
759
+ "serde_json",
760
+ "spm_precompiled",
761
+ "thiserror",
762
+ "unicode-normalization-alignments",
763
+ "unicode-segmentation",
764
+ "unicode_categories",
765
+ ]
766
+
767
+ [[package]]
768
+ name = "unicode-ident"
769
+ version = "1.0.12"
770
+ source = "registry+https://github.com/rust-lang/crates.io-index"
771
+ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
772
+
773
+ [[package]]
774
+ name = "unicode-normalization-alignments"
775
+ version = "0.1.12"
776
+ source = "registry+https://github.com/rust-lang/crates.io-index"
777
+ checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
778
+ dependencies = [
779
+ "smallvec",
780
+ ]
781
+
782
+ [[package]]
783
+ name = "unicode-segmentation"
784
+ version = "1.11.0"
785
+ source = "registry+https://github.com/rust-lang/crates.io-index"
786
+ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
787
+
788
+ [[package]]
789
+ name = "unicode-width"
790
+ version = "0.1.13"
791
+ source = "registry+https://github.com/rust-lang/crates.io-index"
792
+ checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
793
+
794
+ [[package]]
795
+ name = "unicode_categories"
796
+ version = "0.1.1"
797
+ source = "registry+https://github.com/rust-lang/crates.io-index"
798
+ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
799
+
800
+ [[package]]
801
+ name = "wasi"
802
+ version = "0.11.0+wasi-snapshot-preview1"
803
+ source = "registry+https://github.com/rust-lang/crates.io-index"
804
+ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
805
+
806
+ [[package]]
807
+ name = "windows-sys"
808
+ version = "0.52.0"
809
+ source = "registry+https://github.com/rust-lang/crates.io-index"
810
+ checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
811
+ dependencies = [
812
+ "windows-targets",
813
+ ]
814
+
815
+ [[package]]
816
+ name = "windows-targets"
817
+ version = "0.52.6"
818
+ source = "registry+https://github.com/rust-lang/crates.io-index"
819
+ checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
820
+ dependencies = [
821
+ "windows_aarch64_gnullvm",
822
+ "windows_aarch64_msvc",
823
+ "windows_i686_gnu",
824
+ "windows_i686_gnullvm",
825
+ "windows_i686_msvc",
826
+ "windows_x86_64_gnu",
827
+ "windows_x86_64_gnullvm",
828
+ "windows_x86_64_msvc",
829
+ ]
830
+
831
+ [[package]]
832
+ name = "windows_aarch64_gnullvm"
833
+ version = "0.52.6"
834
+ source = "registry+https://github.com/rust-lang/crates.io-index"
835
+ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
836
+
837
+ [[package]]
838
+ name = "windows_aarch64_msvc"
839
+ version = "0.52.6"
840
+ source = "registry+https://github.com/rust-lang/crates.io-index"
841
+ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
842
+
843
+ [[package]]
844
+ name = "windows_i686_gnu"
845
+ version = "0.52.6"
846
+ source = "registry+https://github.com/rust-lang/crates.io-index"
847
+ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
848
+
849
+ [[package]]
850
+ name = "windows_i686_gnullvm"
851
+ version = "0.52.6"
852
+ source = "registry+https://github.com/rust-lang/crates.io-index"
853
+ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
854
+
855
+ [[package]]
856
+ name = "windows_i686_msvc"
857
+ version = "0.52.6"
858
+ source = "registry+https://github.com/rust-lang/crates.io-index"
859
+ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
860
+
861
+ [[package]]
862
+ name = "windows_x86_64_gnu"
863
+ version = "0.52.6"
864
+ source = "registry+https://github.com/rust-lang/crates.io-index"
865
+ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
866
+
867
+ [[package]]
868
+ name = "windows_x86_64_gnullvm"
869
+ version = "0.52.6"
870
+ source = "registry+https://github.com/rust-lang/crates.io-index"
871
+ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
872
+
873
+ [[package]]
874
+ name = "windows_x86_64_msvc"
875
+ version = "0.52.6"
876
+ source = "registry+https://github.com/rust-lang/crates.io-index"
877
+ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
878
+
879
+ [[package]]
880
+ name = "zerocopy"
881
+ version = "0.7.35"
882
+ source = "registry+https://github.com/rust-lang/crates.io-index"
883
+ checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
884
+ dependencies = [
885
+ "byteorder",
886
+ "zerocopy-derive",
887
+ ]
888
+
889
+ [[package]]
890
+ name = "zerocopy-derive"
891
+ version = "0.7.35"
892
+ source = "registry+https://github.com/rust-lang/crates.io-index"
893
+ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
894
+ dependencies = [
895
+ "proc-macro2",
896
+ "quote",
897
+ "syn",
898
+ ]