tokenizers 0.5.1-aarch64-linux-musl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +95 -0
  3. data/Cargo.lock +895 -0
  4. data/Cargo.toml +6 -0
  5. data/LICENSE-THIRD-PARTY.txt +17104 -0
  6. data/LICENSE.txt +202 -0
  7. data/README.md +105 -0
  8. data/lib/tokenizers/3.1/tokenizers.so +0 -0
  9. data/lib/tokenizers/3.2/tokenizers.so +0 -0
  10. data/lib/tokenizers/3.3/tokenizers.so +0 -0
  11. data/lib/tokenizers/char_bpe_tokenizer.rb +22 -0
  12. data/lib/tokenizers/decoders/bpe_decoder.rb +9 -0
  13. data/lib/tokenizers/decoders/ctc.rb +9 -0
  14. data/lib/tokenizers/decoders/metaspace.rb +9 -0
  15. data/lib/tokenizers/decoders/strip.rb +9 -0
  16. data/lib/tokenizers/decoders/word_piece.rb +9 -0
  17. data/lib/tokenizers/encoding.rb +19 -0
  18. data/lib/tokenizers/from_pretrained.rb +125 -0
  19. data/lib/tokenizers/models/bpe.rb +9 -0
  20. data/lib/tokenizers/models/unigram.rb +9 -0
  21. data/lib/tokenizers/models/word_level.rb +13 -0
  22. data/lib/tokenizers/models/word_piece.rb +9 -0
  23. data/lib/tokenizers/normalizers/bert_normalizer.rb +9 -0
  24. data/lib/tokenizers/normalizers/prepend.rb +9 -0
  25. data/lib/tokenizers/normalizers/strip.rb +9 -0
  26. data/lib/tokenizers/pre_tokenizers/byte_level.rb +9 -0
  27. data/lib/tokenizers/pre_tokenizers/digits.rb +9 -0
  28. data/lib/tokenizers/pre_tokenizers/metaspace.rb +9 -0
  29. data/lib/tokenizers/pre_tokenizers/punctuation.rb +9 -0
  30. data/lib/tokenizers/pre_tokenizers/split.rb +9 -0
  31. data/lib/tokenizers/processors/byte_level.rb +9 -0
  32. data/lib/tokenizers/processors/roberta_processing.rb +9 -0
  33. data/lib/tokenizers/processors/template_processing.rb +9 -0
  34. data/lib/tokenizers/tokenizer.rb +45 -0
  35. data/lib/tokenizers/trainers/bpe_trainer.rb +9 -0
  36. data/lib/tokenizers/trainers/unigram_trainer.rb +26 -0
  37. data/lib/tokenizers/trainers/word_level_trainer.rb +9 -0
  38. data/lib/tokenizers/trainers/word_piece_trainer.rb +26 -0
  39. data/lib/tokenizers/version.rb +3 -0
  40. data/lib/tokenizers.rb +61 -0
  41. metadata +84 -0
data/Cargo.lock ADDED
@@ -0,0 +1,895 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 3
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.3"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "base64"
16
+ version = "0.13.1"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
19
+
20
+ [[package]]
21
+ name = "bindgen"
22
+ version = "0.69.4"
23
+ source = "registry+https://github.com/rust-lang/crates.io-index"
24
+ checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
25
+ dependencies = [
26
+ "bitflags 2.6.0",
27
+ "cexpr",
28
+ "clang-sys",
29
+ "itertools 0.12.1",
30
+ "lazy_static",
31
+ "lazycell",
32
+ "proc-macro2",
33
+ "quote",
34
+ "regex",
35
+ "rustc-hash",
36
+ "shlex",
37
+ "syn",
38
+ ]
39
+
40
+ [[package]]
41
+ name = "bitflags"
42
+ version = "1.3.2"
43
+ source = "registry+https://github.com/rust-lang/crates.io-index"
44
+ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
45
+
46
+ [[package]]
47
+ name = "bitflags"
48
+ version = "2.6.0"
49
+ source = "registry+https://github.com/rust-lang/crates.io-index"
50
+ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
51
+
52
+ [[package]]
53
+ name = "byteorder"
54
+ version = "1.5.0"
55
+ source = "registry+https://github.com/rust-lang/crates.io-index"
56
+ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
57
+
58
+ [[package]]
59
+ name = "cc"
60
+ version = "1.1.8"
61
+ source = "registry+https://github.com/rust-lang/crates.io-index"
62
+ checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
63
+
64
+ [[package]]
65
+ name = "cexpr"
66
+ version = "0.6.0"
67
+ source = "registry+https://github.com/rust-lang/crates.io-index"
68
+ checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
69
+ dependencies = [
70
+ "nom",
71
+ ]
72
+
73
+ [[package]]
74
+ name = "cfg-if"
75
+ version = "1.0.0"
76
+ source = "registry+https://github.com/rust-lang/crates.io-index"
77
+ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
78
+
79
+ [[package]]
80
+ name = "clang-sys"
81
+ version = "1.8.1"
82
+ source = "registry+https://github.com/rust-lang/crates.io-index"
83
+ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
84
+ dependencies = [
85
+ "glob",
86
+ "libc",
87
+ "libloading",
88
+ ]
89
+
90
+ [[package]]
91
+ name = "console"
92
+ version = "0.15.8"
93
+ source = "registry+https://github.com/rust-lang/crates.io-index"
94
+ checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
95
+ dependencies = [
96
+ "encode_unicode",
97
+ "lazy_static",
98
+ "libc",
99
+ "unicode-width",
100
+ "windows-sys",
101
+ ]
102
+
103
+ [[package]]
104
+ name = "crossbeam-deque"
105
+ version = "0.8.5"
106
+ source = "registry+https://github.com/rust-lang/crates.io-index"
107
+ checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
108
+ dependencies = [
109
+ "crossbeam-epoch",
110
+ "crossbeam-utils",
111
+ ]
112
+
113
+ [[package]]
114
+ name = "crossbeam-epoch"
115
+ version = "0.9.18"
116
+ source = "registry+https://github.com/rust-lang/crates.io-index"
117
+ checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
118
+ dependencies = [
119
+ "crossbeam-utils",
120
+ ]
121
+
122
+ [[package]]
123
+ name = "crossbeam-utils"
124
+ version = "0.8.20"
125
+ source = "registry+https://github.com/rust-lang/crates.io-index"
126
+ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
127
+
128
+ [[package]]
129
+ name = "darling"
130
+ version = "0.20.10"
131
+ source = "registry+https://github.com/rust-lang/crates.io-index"
132
+ checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
133
+ dependencies = [
134
+ "darling_core",
135
+ "darling_macro",
136
+ ]
137
+
138
+ [[package]]
139
+ name = "darling_core"
140
+ version = "0.20.10"
141
+ source = "registry+https://github.com/rust-lang/crates.io-index"
142
+ checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
143
+ dependencies = [
144
+ "fnv",
145
+ "ident_case",
146
+ "proc-macro2",
147
+ "quote",
148
+ "strsim",
149
+ "syn",
150
+ ]
151
+
152
+ [[package]]
153
+ name = "darling_macro"
154
+ version = "0.20.10"
155
+ source = "registry+https://github.com/rust-lang/crates.io-index"
156
+ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
157
+ dependencies = [
158
+ "darling_core",
159
+ "quote",
160
+ "syn",
161
+ ]
162
+
163
+ [[package]]
164
+ name = "derive_builder"
165
+ version = "0.20.0"
166
+ source = "registry+https://github.com/rust-lang/crates.io-index"
167
+ checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
168
+ dependencies = [
169
+ "derive_builder_macro",
170
+ ]
171
+
172
+ [[package]]
173
+ name = "derive_builder_core"
174
+ version = "0.20.0"
175
+ source = "registry+https://github.com/rust-lang/crates.io-index"
176
+ checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
177
+ dependencies = [
178
+ "darling",
179
+ "proc-macro2",
180
+ "quote",
181
+ "syn",
182
+ ]
183
+
184
+ [[package]]
185
+ name = "derive_builder_macro"
186
+ version = "0.20.0"
187
+ source = "registry+https://github.com/rust-lang/crates.io-index"
188
+ checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
189
+ dependencies = [
190
+ "derive_builder_core",
191
+ "syn",
192
+ ]
193
+
194
+ [[package]]
195
+ name = "either"
196
+ version = "1.13.0"
197
+ source = "registry+https://github.com/rust-lang/crates.io-index"
198
+ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
199
+
200
+ [[package]]
201
+ name = "encode_unicode"
202
+ version = "0.3.6"
203
+ source = "registry+https://github.com/rust-lang/crates.io-index"
204
+ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
205
+
206
+ [[package]]
207
+ name = "esaxx-rs"
208
+ version = "0.1.10"
209
+ source = "registry+https://github.com/rust-lang/crates.io-index"
210
+ checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
211
+ dependencies = [
212
+ "cc",
213
+ ]
214
+
215
+ [[package]]
216
+ name = "fnv"
217
+ version = "1.0.7"
218
+ source = "registry+https://github.com/rust-lang/crates.io-index"
219
+ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
220
+
221
+ [[package]]
222
+ name = "getrandom"
223
+ version = "0.2.15"
224
+ source = "registry+https://github.com/rust-lang/crates.io-index"
225
+ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
226
+ dependencies = [
227
+ "cfg-if",
228
+ "libc",
229
+ "wasi",
230
+ ]
231
+
232
+ [[package]]
233
+ name = "glob"
234
+ version = "0.3.1"
235
+ source = "registry+https://github.com/rust-lang/crates.io-index"
236
+ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
237
+
238
+ [[package]]
239
+ name = "ident_case"
240
+ version = "1.0.1"
241
+ source = "registry+https://github.com/rust-lang/crates.io-index"
242
+ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
243
+
244
+ [[package]]
245
+ name = "indicatif"
246
+ version = "0.17.8"
247
+ source = "registry+https://github.com/rust-lang/crates.io-index"
248
+ checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3"
249
+ dependencies = [
250
+ "console",
251
+ "instant",
252
+ "number_prefix",
253
+ "portable-atomic",
254
+ "unicode-width",
255
+ ]
256
+
257
+ [[package]]
258
+ name = "instant"
259
+ version = "0.1.13"
260
+ source = "registry+https://github.com/rust-lang/crates.io-index"
261
+ checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
262
+ dependencies = [
263
+ "cfg-if",
264
+ ]
265
+
266
+ [[package]]
267
+ name = "itertools"
268
+ version = "0.11.0"
269
+ source = "registry+https://github.com/rust-lang/crates.io-index"
270
+ checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
271
+ dependencies = [
272
+ "either",
273
+ ]
274
+
275
+ [[package]]
276
+ name = "itertools"
277
+ version = "0.12.1"
278
+ source = "registry+https://github.com/rust-lang/crates.io-index"
279
+ checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
280
+ dependencies = [
281
+ "either",
282
+ ]
283
+
284
+ [[package]]
285
+ name = "itoa"
286
+ version = "1.0.11"
287
+ source = "registry+https://github.com/rust-lang/crates.io-index"
288
+ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
289
+
290
+ [[package]]
291
+ name = "lazy_static"
292
+ version = "1.5.0"
293
+ source = "registry+https://github.com/rust-lang/crates.io-index"
294
+ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
295
+
296
+ [[package]]
297
+ name = "lazycell"
298
+ version = "1.3.0"
299
+ source = "registry+https://github.com/rust-lang/crates.io-index"
300
+ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
301
+
302
+ [[package]]
303
+ name = "libc"
304
+ version = "0.2.155"
305
+ source = "registry+https://github.com/rust-lang/crates.io-index"
306
+ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
307
+
308
+ [[package]]
309
+ name = "libloading"
310
+ version = "0.8.5"
311
+ source = "registry+https://github.com/rust-lang/crates.io-index"
312
+ checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
313
+ dependencies = [
314
+ "cfg-if",
315
+ "windows-targets",
316
+ ]
317
+
318
+ [[package]]
319
+ name = "log"
320
+ version = "0.4.22"
321
+ source = "registry+https://github.com/rust-lang/crates.io-index"
322
+ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
323
+
324
+ [[package]]
325
+ name = "macro_rules_attribute"
326
+ version = "0.2.0"
327
+ source = "registry+https://github.com/rust-lang/crates.io-index"
328
+ checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13"
329
+ dependencies = [
330
+ "macro_rules_attribute-proc_macro",
331
+ "paste",
332
+ ]
333
+
334
+ [[package]]
335
+ name = "macro_rules_attribute-proc_macro"
336
+ version = "0.2.0"
337
+ source = "registry+https://github.com/rust-lang/crates.io-index"
338
+ checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
339
+
340
+ [[package]]
341
+ name = "magnus"
342
+ version = "0.7.1"
343
+ source = "registry+https://github.com/rust-lang/crates.io-index"
344
+ checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
345
+ dependencies = [
346
+ "magnus-macros",
347
+ "rb-sys",
348
+ "rb-sys-env",
349
+ "seq-macro",
350
+ ]
351
+
352
+ [[package]]
353
+ name = "magnus-macros"
354
+ version = "0.6.0"
355
+ source = "registry+https://github.com/rust-lang/crates.io-index"
356
+ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
357
+ dependencies = [
358
+ "proc-macro2",
359
+ "quote",
360
+ "syn",
361
+ ]
362
+
363
+ [[package]]
364
+ name = "memchr"
365
+ version = "2.7.4"
366
+ source = "registry+https://github.com/rust-lang/crates.io-index"
367
+ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
368
+
369
+ [[package]]
370
+ name = "minimal-lexical"
371
+ version = "0.2.1"
372
+ source = "registry+https://github.com/rust-lang/crates.io-index"
373
+ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
374
+
375
+ [[package]]
376
+ name = "monostate"
377
+ version = "0.1.13"
378
+ source = "registry+https://github.com/rust-lang/crates.io-index"
379
+ checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e"
380
+ dependencies = [
381
+ "monostate-impl",
382
+ "serde",
383
+ ]
384
+
385
+ [[package]]
386
+ name = "monostate-impl"
387
+ version = "0.1.13"
388
+ source = "registry+https://github.com/rust-lang/crates.io-index"
389
+ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0"
390
+ dependencies = [
391
+ "proc-macro2",
392
+ "quote",
393
+ "syn",
394
+ ]
395
+
396
+ [[package]]
397
+ name = "nom"
398
+ version = "7.1.3"
399
+ source = "registry+https://github.com/rust-lang/crates.io-index"
400
+ checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
401
+ dependencies = [
402
+ "memchr",
403
+ "minimal-lexical",
404
+ ]
405
+
406
+ [[package]]
407
+ name = "number_prefix"
408
+ version = "0.4.0"
409
+ source = "registry+https://github.com/rust-lang/crates.io-index"
410
+ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
411
+
412
+ [[package]]
413
+ name = "once_cell"
414
+ version = "1.19.0"
415
+ source = "registry+https://github.com/rust-lang/crates.io-index"
416
+ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
417
+
418
+ [[package]]
419
+ name = "onig"
420
+ version = "6.4.0"
421
+ source = "registry+https://github.com/rust-lang/crates.io-index"
422
+ checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
423
+ dependencies = [
424
+ "bitflags 1.3.2",
425
+ "libc",
426
+ "once_cell",
427
+ "onig_sys",
428
+ ]
429
+
430
+ [[package]]
431
+ name = "onig_sys"
432
+ version = "69.8.1"
433
+ source = "registry+https://github.com/rust-lang/crates.io-index"
434
+ checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
435
+ dependencies = [
436
+ "cc",
437
+ "pkg-config",
438
+ ]
439
+
440
+ [[package]]
441
+ name = "paste"
442
+ version = "1.0.15"
443
+ source = "registry+https://github.com/rust-lang/crates.io-index"
444
+ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
445
+
446
+ [[package]]
447
+ name = "pkg-config"
448
+ version = "0.3.30"
449
+ source = "registry+https://github.com/rust-lang/crates.io-index"
450
+ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
451
+
452
+ [[package]]
453
+ name = "portable-atomic"
454
+ version = "1.7.0"
455
+ source = "registry+https://github.com/rust-lang/crates.io-index"
456
+ checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
457
+
458
+ [[package]]
459
+ name = "ppv-lite86"
460
+ version = "0.2.20"
461
+ source = "registry+https://github.com/rust-lang/crates.io-index"
462
+ checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
463
+ dependencies = [
464
+ "zerocopy",
465
+ ]
466
+
467
+ [[package]]
468
+ name = "proc-macro2"
469
+ version = "1.0.86"
470
+ source = "registry+https://github.com/rust-lang/crates.io-index"
471
+ checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
472
+ dependencies = [
473
+ "unicode-ident",
474
+ ]
475
+
476
+ [[package]]
477
+ name = "quote"
478
+ version = "1.0.36"
479
+ source = "registry+https://github.com/rust-lang/crates.io-index"
480
+ checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
481
+ dependencies = [
482
+ "proc-macro2",
483
+ ]
484
+
485
+ [[package]]
486
+ name = "rand"
487
+ version = "0.8.5"
488
+ source = "registry+https://github.com/rust-lang/crates.io-index"
489
+ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
490
+ dependencies = [
491
+ "libc",
492
+ "rand_chacha",
493
+ "rand_core",
494
+ ]
495
+
496
+ [[package]]
497
+ name = "rand_chacha"
498
+ version = "0.3.1"
499
+ source = "registry+https://github.com/rust-lang/crates.io-index"
500
+ checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
501
+ dependencies = [
502
+ "ppv-lite86",
503
+ "rand_core",
504
+ ]
505
+
506
+ [[package]]
507
+ name = "rand_core"
508
+ version = "0.6.4"
509
+ source = "registry+https://github.com/rust-lang/crates.io-index"
510
+ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
511
+ dependencies = [
512
+ "getrandom",
513
+ ]
514
+
515
+ [[package]]
516
+ name = "rayon"
517
+ version = "1.10.0"
518
+ source = "registry+https://github.com/rust-lang/crates.io-index"
519
+ checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
520
+ dependencies = [
521
+ "either",
522
+ "rayon-core",
523
+ ]
524
+
525
+ [[package]]
526
+ name = "rayon-cond"
527
+ version = "0.3.0"
528
+ source = "registry+https://github.com/rust-lang/crates.io-index"
529
+ checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
530
+ dependencies = [
531
+ "either",
532
+ "itertools 0.11.0",
533
+ "rayon",
534
+ ]
535
+
536
+ [[package]]
537
+ name = "rayon-core"
538
+ version = "1.12.1"
539
+ source = "registry+https://github.com/rust-lang/crates.io-index"
540
+ checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
541
+ dependencies = [
542
+ "crossbeam-deque",
543
+ "crossbeam-utils",
544
+ ]
545
+
546
+ [[package]]
547
+ name = "rb-sys"
548
+ version = "0.9.100"
549
+ source = "registry+https://github.com/rust-lang/crates.io-index"
550
+ checksum = "87f2ba20be84b32fad6b0ce397764bcdd0f2dca4431cf7035f6a6721e5747565"
551
+ dependencies = [
552
+ "rb-sys-build",
553
+ ]
554
+
555
+ [[package]]
556
+ name = "rb-sys-build"
557
+ version = "0.9.100"
558
+ source = "registry+https://github.com/rust-lang/crates.io-index"
559
+ checksum = "7ecae2bdcb118ee721d9a3929f89e8578237fade298dfcf8c928609aa88abc48"
560
+ dependencies = [
561
+ "bindgen",
562
+ "lazy_static",
563
+ "proc-macro2",
564
+ "quote",
565
+ "regex",
566
+ "shell-words",
567
+ "syn",
568
+ ]
569
+
570
+ [[package]]
571
+ name = "rb-sys-env"
572
+ version = "0.1.2"
573
+ source = "registry+https://github.com/rust-lang/crates.io-index"
574
+ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
575
+
576
+ [[package]]
577
+ name = "regex"
578
+ version = "1.10.6"
579
+ source = "registry+https://github.com/rust-lang/crates.io-index"
580
+ checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
581
+ dependencies = [
582
+ "aho-corasick",
583
+ "memchr",
584
+ "regex-automata",
585
+ "regex-syntax",
586
+ ]
587
+
588
+ [[package]]
589
+ name = "regex-automata"
590
+ version = "0.4.7"
591
+ source = "registry+https://github.com/rust-lang/crates.io-index"
592
+ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
593
+ dependencies = [
594
+ "aho-corasick",
595
+ "memchr",
596
+ "regex-syntax",
597
+ ]
598
+
599
+ [[package]]
600
+ name = "regex-syntax"
601
+ version = "0.8.4"
602
+ source = "registry+https://github.com/rust-lang/crates.io-index"
603
+ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
604
+
605
+ [[package]]
606
+ name = "rustc-hash"
607
+ version = "1.1.0"
608
+ source = "registry+https://github.com/rust-lang/crates.io-index"
609
+ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
610
+
611
+ [[package]]
612
+ name = "ryu"
613
+ version = "1.0.18"
614
+ source = "registry+https://github.com/rust-lang/crates.io-index"
615
+ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
616
+
617
+ [[package]]
618
+ name = "seq-macro"
619
+ version = "0.3.5"
620
+ source = "registry+https://github.com/rust-lang/crates.io-index"
621
+ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
622
+
623
+ [[package]]
624
+ name = "serde"
625
+ version = "1.0.205"
626
+ source = "registry+https://github.com/rust-lang/crates.io-index"
627
+ checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
628
+ dependencies = [
629
+ "serde_derive",
630
+ ]
631
+
632
+ [[package]]
633
+ name = "serde_derive"
634
+ version = "1.0.205"
635
+ source = "registry+https://github.com/rust-lang/crates.io-index"
636
+ checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
637
+ dependencies = [
638
+ "proc-macro2",
639
+ "quote",
640
+ "syn",
641
+ ]
642
+
643
+ [[package]]
644
+ name = "serde_json"
645
+ version = "1.0.122"
646
+ source = "registry+https://github.com/rust-lang/crates.io-index"
647
+ checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
648
+ dependencies = [
649
+ "itoa",
650
+ "memchr",
651
+ "ryu",
652
+ "serde",
653
+ ]
654
+
655
+ [[package]]
656
+ name = "shell-words"
657
+ version = "1.1.0"
658
+ source = "registry+https://github.com/rust-lang/crates.io-index"
659
+ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
660
+
661
+ [[package]]
662
+ name = "shlex"
663
+ version = "1.3.0"
664
+ source = "registry+https://github.com/rust-lang/crates.io-index"
665
+ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
666
+
667
+ [[package]]
668
+ name = "smallvec"
669
+ version = "1.13.2"
670
+ source = "registry+https://github.com/rust-lang/crates.io-index"
671
+ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
672
+
673
+ [[package]]
674
+ name = "spm_precompiled"
675
+ version = "0.1.4"
676
+ source = "registry+https://github.com/rust-lang/crates.io-index"
677
+ checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
678
+ dependencies = [
679
+ "base64",
680
+ "nom",
681
+ "serde",
682
+ "unicode-segmentation",
683
+ ]
684
+
685
+ [[package]]
686
+ name = "strsim"
687
+ version = "0.11.1"
688
+ source = "registry+https://github.com/rust-lang/crates.io-index"
689
+ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
690
+
691
+ [[package]]
692
+ name = "syn"
693
+ version = "2.0.72"
694
+ source = "registry+https://github.com/rust-lang/crates.io-index"
695
+ checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
696
+ dependencies = [
697
+ "proc-macro2",
698
+ "quote",
699
+ "unicode-ident",
700
+ ]
701
+
702
+ [[package]]
703
+ name = "thiserror"
704
+ version = "1.0.63"
705
+ source = "registry+https://github.com/rust-lang/crates.io-index"
706
+ checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
707
+ dependencies = [
708
+ "thiserror-impl",
709
+ ]
710
+
711
+ [[package]]
712
+ name = "thiserror-impl"
713
+ version = "1.0.63"
714
+ source = "registry+https://github.com/rust-lang/crates.io-index"
715
+ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
716
+ dependencies = [
717
+ "proc-macro2",
718
+ "quote",
719
+ "syn",
720
+ ]
721
+
722
+ [[package]]
723
+ name = "tokenizers"
724
+ version = "0.5.1"
725
+ dependencies = [
726
+ "magnus",
727
+ "onig",
728
+ "serde",
729
+ "tokenizers 0.20.0",
730
+ ]
731
+
732
+ [[package]]
733
+ name = "tokenizers"
734
+ version = "0.20.0"
735
+ source = "registry+https://github.com/rust-lang/crates.io-index"
736
+ checksum = "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
737
+ dependencies = [
738
+ "aho-corasick",
739
+ "derive_builder",
740
+ "esaxx-rs",
741
+ "getrandom",
742
+ "indicatif",
743
+ "itertools 0.12.1",
744
+ "lazy_static",
745
+ "log",
746
+ "macro_rules_attribute",
747
+ "monostate",
748
+ "onig",
749
+ "paste",
750
+ "rand",
751
+ "rayon",
752
+ "rayon-cond",
753
+ "regex",
754
+ "regex-syntax",
755
+ "serde",
756
+ "serde_json",
757
+ "spm_precompiled",
758
+ "thiserror",
759
+ "unicode-normalization-alignments",
760
+ "unicode-segmentation",
761
+ "unicode_categories",
762
+ ]
763
+
764
+ [[package]]
765
+ name = "unicode-ident"
766
+ version = "1.0.12"
767
+ source = "registry+https://github.com/rust-lang/crates.io-index"
768
+ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
769
+
770
+ [[package]]
771
+ name = "unicode-normalization-alignments"
772
+ version = "0.1.12"
773
+ source = "registry+https://github.com/rust-lang/crates.io-index"
774
+ checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
775
+ dependencies = [
776
+ "smallvec",
777
+ ]
778
+
779
+ [[package]]
780
+ name = "unicode-segmentation"
781
+ version = "1.11.0"
782
+ source = "registry+https://github.com/rust-lang/crates.io-index"
783
+ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
784
+
785
+ [[package]]
786
+ name = "unicode-width"
787
+ version = "0.1.13"
788
+ source = "registry+https://github.com/rust-lang/crates.io-index"
789
+ checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
790
+
791
+ [[package]]
792
+ name = "unicode_categories"
793
+ version = "0.1.1"
794
+ source = "registry+https://github.com/rust-lang/crates.io-index"
795
+ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
796
+
797
+ [[package]]
798
+ name = "wasi"
799
+ version = "0.11.0+wasi-snapshot-preview1"
800
+ source = "registry+https://github.com/rust-lang/crates.io-index"
801
+ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
802
+
803
+ [[package]]
804
+ name = "windows-sys"
805
+ version = "0.52.0"
806
+ source = "registry+https://github.com/rust-lang/crates.io-index"
807
+ checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
808
+ dependencies = [
809
+ "windows-targets",
810
+ ]
811
+
812
+ [[package]]
813
+ name = "windows-targets"
814
+ version = "0.52.6"
815
+ source = "registry+https://github.com/rust-lang/crates.io-index"
816
+ checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
817
+ dependencies = [
818
+ "windows_aarch64_gnullvm",
819
+ "windows_aarch64_msvc",
820
+ "windows_i686_gnu",
821
+ "windows_i686_gnullvm",
822
+ "windows_i686_msvc",
823
+ "windows_x86_64_gnu",
824
+ "windows_x86_64_gnullvm",
825
+ "windows_x86_64_msvc",
826
+ ]
827
+
828
+ [[package]]
829
+ name = "windows_aarch64_gnullvm"
830
+ version = "0.52.6"
831
+ source = "registry+https://github.com/rust-lang/crates.io-index"
832
+ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
833
+
834
+ [[package]]
835
+ name = "windows_aarch64_msvc"
836
+ version = "0.52.6"
837
+ source = "registry+https://github.com/rust-lang/crates.io-index"
838
+ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
839
+
840
+ [[package]]
841
+ name = "windows_i686_gnu"
842
+ version = "0.52.6"
843
+ source = "registry+https://github.com/rust-lang/crates.io-index"
844
+ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
845
+
846
+ [[package]]
847
+ name = "windows_i686_gnullvm"
848
+ version = "0.52.6"
849
+ source = "registry+https://github.com/rust-lang/crates.io-index"
850
+ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
851
+
852
+ [[package]]
853
+ name = "windows_i686_msvc"
854
+ version = "0.52.6"
855
+ source = "registry+https://github.com/rust-lang/crates.io-index"
856
+ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
857
+
858
+ [[package]]
859
+ name = "windows_x86_64_gnu"
860
+ version = "0.52.6"
861
+ source = "registry+https://github.com/rust-lang/crates.io-index"
862
+ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
863
+
864
+ [[package]]
865
+ name = "windows_x86_64_gnullvm"
866
+ version = "0.52.6"
867
+ source = "registry+https://github.com/rust-lang/crates.io-index"
868
+ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
869
+
870
+ [[package]]
871
+ name = "windows_x86_64_msvc"
872
+ version = "0.52.6"
873
+ source = "registry+https://github.com/rust-lang/crates.io-index"
874
+ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
875
+
876
+ [[package]]
877
+ name = "zerocopy"
878
+ version = "0.7.35"
879
+ source = "registry+https://github.com/rust-lang/crates.io-index"
880
+ checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
881
+ dependencies = [
882
+ "byteorder",
883
+ "zerocopy-derive",
884
+ ]
885
+
886
+ [[package]]
887
+ name = "zerocopy-derive"
888
+ version = "0.7.35"
889
+ source = "registry+https://github.com/rust-lang/crates.io-index"
890
+ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
891
+ dependencies = [
892
+ "proc-macro2",
893
+ "quote",
894
+ "syn",
895
+ ]