tokenizers 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Cargo.lock +194 -118
- data/ext/tokenizers/Cargo.toml +2 -2
- data/ext/tokenizers/src/decoders.rs +32 -14
- data/ext/tokenizers/src/error.rs +6 -1
- data/ext/tokenizers/src/lib.rs +37 -12
- data/ext/tokenizers/src/models.rs +75 -23
- data/ext/tokenizers/src/normalizers.rs +84 -24
- data/ext/tokenizers/src/pre_tokenizers.rs +121 -42
- data/ext/tokenizers/src/processors.rs +22 -10
- data/ext/tokenizers/src/tokenizer.rs +67 -38
- data/ext/tokenizers/src/trainers.rs +215 -56
- data/ext/tokenizers/src/utils/regex.rs +6 -4
- data/lib/tokenizers/from_pretrained.rb +1 -1
- data/lib/tokenizers/version.rb +1 -1
- metadata +3 -7
data/Cargo.lock
CHANGED
|
@@ -19,11 +19,11 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
|
|
|
19
19
|
|
|
20
20
|
[[package]]
|
|
21
21
|
name = "bindgen"
|
|
22
|
-
version = "0.69.
|
|
22
|
+
version = "0.69.5"
|
|
23
23
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
24
|
-
checksum = "
|
|
24
|
+
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
|
|
25
25
|
dependencies = [
|
|
26
|
-
"bitflags 2.
|
|
26
|
+
"bitflags 2.9.0",
|
|
27
27
|
"cexpr",
|
|
28
28
|
"clang-sys",
|
|
29
29
|
"itertools 0.12.1",
|
|
@@ -45,21 +45,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
|
|
45
45
|
|
|
46
46
|
[[package]]
|
|
47
47
|
name = "bitflags"
|
|
48
|
-
version = "2.
|
|
48
|
+
version = "2.9.0"
|
|
49
49
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
50
|
-
checksum = "
|
|
50
|
+
checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
|
|
51
51
|
|
|
52
52
|
[[package]]
|
|
53
|
-
name = "
|
|
54
|
-
version = "
|
|
53
|
+
name = "bumpalo"
|
|
54
|
+
version = "3.17.0"
|
|
55
55
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
56
|
-
checksum = "
|
|
56
|
+
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
|
|
57
57
|
|
|
58
58
|
[[package]]
|
|
59
59
|
name = "cc"
|
|
60
|
-
version = "1.
|
|
60
|
+
version = "1.2.21"
|
|
61
61
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
62
|
-
checksum = "
|
|
62
|
+
checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0"
|
|
63
63
|
dependencies = [
|
|
64
64
|
"shlex",
|
|
65
65
|
]
|
|
@@ -92,22 +92,22 @@ dependencies = [
|
|
|
92
92
|
|
|
93
93
|
[[package]]
|
|
94
94
|
name = "console"
|
|
95
|
-
version = "0.15.
|
|
95
|
+
version = "0.15.11"
|
|
96
96
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
97
|
-
checksum = "
|
|
97
|
+
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
|
|
98
98
|
dependencies = [
|
|
99
99
|
"encode_unicode",
|
|
100
|
-
"lazy_static",
|
|
101
100
|
"libc",
|
|
101
|
+
"once_cell",
|
|
102
102
|
"unicode-width",
|
|
103
103
|
"windows-sys",
|
|
104
104
|
]
|
|
105
105
|
|
|
106
106
|
[[package]]
|
|
107
107
|
name = "crossbeam-deque"
|
|
108
|
-
version = "0.8.
|
|
108
|
+
version = "0.8.6"
|
|
109
109
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
110
|
-
checksum = "
|
|
110
|
+
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
|
|
111
111
|
dependencies = [
|
|
112
112
|
"crossbeam-epoch",
|
|
113
113
|
"crossbeam-utils",
|
|
@@ -124,15 +124,15 @@ dependencies = [
|
|
|
124
124
|
|
|
125
125
|
[[package]]
|
|
126
126
|
name = "crossbeam-utils"
|
|
127
|
-
version = "0.8.
|
|
127
|
+
version = "0.8.21"
|
|
128
128
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
129
|
-
checksum = "
|
|
129
|
+
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
|
130
130
|
|
|
131
131
|
[[package]]
|
|
132
132
|
name = "darling"
|
|
133
|
-
version = "0.20.
|
|
133
|
+
version = "0.20.11"
|
|
134
134
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
135
|
-
checksum = "
|
|
135
|
+
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
|
|
136
136
|
dependencies = [
|
|
137
137
|
"darling_core",
|
|
138
138
|
"darling_macro",
|
|
@@ -140,9 +140,9 @@ dependencies = [
|
|
|
140
140
|
|
|
141
141
|
[[package]]
|
|
142
142
|
name = "darling_core"
|
|
143
|
-
version = "0.20.
|
|
143
|
+
version = "0.20.11"
|
|
144
144
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
145
|
-
checksum = "
|
|
145
|
+
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
|
|
146
146
|
dependencies = [
|
|
147
147
|
"fnv",
|
|
148
148
|
"ident_case",
|
|
@@ -154,9 +154,9 @@ dependencies = [
|
|
|
154
154
|
|
|
155
155
|
[[package]]
|
|
156
156
|
name = "darling_macro"
|
|
157
|
-
version = "0.20.
|
|
157
|
+
version = "0.20.11"
|
|
158
158
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
159
|
-
checksum = "
|
|
159
|
+
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
|
|
160
160
|
dependencies = [
|
|
161
161
|
"darling_core",
|
|
162
162
|
"quote",
|
|
@@ -165,18 +165,18 @@ dependencies = [
|
|
|
165
165
|
|
|
166
166
|
[[package]]
|
|
167
167
|
name = "derive_builder"
|
|
168
|
-
version = "0.20.
|
|
168
|
+
version = "0.20.2"
|
|
169
169
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
170
|
-
checksum = "
|
|
170
|
+
checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
|
|
171
171
|
dependencies = [
|
|
172
172
|
"derive_builder_macro",
|
|
173
173
|
]
|
|
174
174
|
|
|
175
175
|
[[package]]
|
|
176
176
|
name = "derive_builder_core"
|
|
177
|
-
version = "0.20.
|
|
177
|
+
version = "0.20.2"
|
|
178
178
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
179
|
-
checksum = "
|
|
179
|
+
checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
|
|
180
180
|
dependencies = [
|
|
181
181
|
"darling",
|
|
182
182
|
"proc-macro2",
|
|
@@ -186,9 +186,9 @@ dependencies = [
|
|
|
186
186
|
|
|
187
187
|
[[package]]
|
|
188
188
|
name = "derive_builder_macro"
|
|
189
|
-
version = "0.20.
|
|
189
|
+
version = "0.20.2"
|
|
190
190
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
191
|
-
checksum = "
|
|
191
|
+
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
|
|
192
192
|
dependencies = [
|
|
193
193
|
"derive_builder_core",
|
|
194
194
|
"syn",
|
|
@@ -196,15 +196,15 @@ dependencies = [
|
|
|
196
196
|
|
|
197
197
|
[[package]]
|
|
198
198
|
name = "either"
|
|
199
|
-
version = "1.
|
|
199
|
+
version = "1.15.0"
|
|
200
200
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
201
|
-
checksum = "
|
|
201
|
+
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
|
202
202
|
|
|
203
203
|
[[package]]
|
|
204
204
|
name = "encode_unicode"
|
|
205
|
-
version = "0.
|
|
205
|
+
version = "1.0.0"
|
|
206
206
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
207
|
-
checksum = "
|
|
207
|
+
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
|
208
208
|
|
|
209
209
|
[[package]]
|
|
210
210
|
name = "esaxx-rs"
|
|
@@ -223,9 +223,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
|
|
223
223
|
|
|
224
224
|
[[package]]
|
|
225
225
|
name = "getrandom"
|
|
226
|
-
version = "0.2.
|
|
226
|
+
version = "0.2.16"
|
|
227
227
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
228
|
-
checksum = "
|
|
228
|
+
checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
|
|
229
229
|
dependencies = [
|
|
230
230
|
"cfg-if",
|
|
231
231
|
"libc",
|
|
@@ -234,9 +234,9 @@ dependencies = [
|
|
|
234
234
|
|
|
235
235
|
[[package]]
|
|
236
236
|
name = "glob"
|
|
237
|
-
version = "0.3.
|
|
237
|
+
version = "0.3.2"
|
|
238
238
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
239
|
-
checksum = "
|
|
239
|
+
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
|
240
240
|
|
|
241
241
|
[[package]]
|
|
242
242
|
name = "ident_case"
|
|
@@ -246,49 +246,59 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
|
|
246
246
|
|
|
247
247
|
[[package]]
|
|
248
248
|
name = "indicatif"
|
|
249
|
-
version = "0.17.
|
|
249
|
+
version = "0.17.11"
|
|
250
250
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
251
|
-
checksum = "
|
|
251
|
+
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
|
|
252
252
|
dependencies = [
|
|
253
253
|
"console",
|
|
254
|
-
"instant",
|
|
255
254
|
"number_prefix",
|
|
256
255
|
"portable-atomic",
|
|
257
256
|
"unicode-width",
|
|
257
|
+
"web-time",
|
|
258
258
|
]
|
|
259
259
|
|
|
260
260
|
[[package]]
|
|
261
|
-
name = "
|
|
262
|
-
version = "0.
|
|
261
|
+
name = "itertools"
|
|
262
|
+
version = "0.11.0"
|
|
263
263
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
264
|
-
checksum = "
|
|
264
|
+
checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
|
|
265
265
|
dependencies = [
|
|
266
|
-
"
|
|
266
|
+
"either",
|
|
267
267
|
]
|
|
268
268
|
|
|
269
269
|
[[package]]
|
|
270
270
|
name = "itertools"
|
|
271
|
-
version = "0.
|
|
271
|
+
version = "0.12.1"
|
|
272
272
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
273
|
-
checksum = "
|
|
273
|
+
checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
|
|
274
274
|
dependencies = [
|
|
275
275
|
"either",
|
|
276
276
|
]
|
|
277
277
|
|
|
278
278
|
[[package]]
|
|
279
279
|
name = "itertools"
|
|
280
|
-
version = "0.
|
|
280
|
+
version = "0.13.0"
|
|
281
281
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
282
|
-
checksum = "
|
|
282
|
+
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
|
283
283
|
dependencies = [
|
|
284
284
|
"either",
|
|
285
285
|
]
|
|
286
286
|
|
|
287
287
|
[[package]]
|
|
288
288
|
name = "itoa"
|
|
289
|
-
version = "1.0.
|
|
289
|
+
version = "1.0.15"
|
|
290
290
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
291
|
-
checksum = "
|
|
291
|
+
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
|
292
|
+
|
|
293
|
+
[[package]]
|
|
294
|
+
name = "js-sys"
|
|
295
|
+
version = "0.3.77"
|
|
296
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
297
|
+
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
|
|
298
|
+
dependencies = [
|
|
299
|
+
"once_cell",
|
|
300
|
+
"wasm-bindgen",
|
|
301
|
+
]
|
|
292
302
|
|
|
293
303
|
[[package]]
|
|
294
304
|
name = "lazy_static"
|
|
@@ -304,15 +314,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
|
304
314
|
|
|
305
315
|
[[package]]
|
|
306
316
|
name = "libc"
|
|
307
|
-
version = "0.2.
|
|
317
|
+
version = "0.2.172"
|
|
308
318
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
309
|
-
checksum = "
|
|
319
|
+
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
|
|
310
320
|
|
|
311
321
|
[[package]]
|
|
312
322
|
name = "libloading"
|
|
313
|
-
version = "0.8.
|
|
323
|
+
version = "0.8.6"
|
|
314
324
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
315
|
-
checksum = "
|
|
325
|
+
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
|
|
316
326
|
dependencies = [
|
|
317
327
|
"cfg-if",
|
|
318
328
|
"windows-targets",
|
|
@@ -320,9 +330,9 @@ dependencies = [
|
|
|
320
330
|
|
|
321
331
|
[[package]]
|
|
322
332
|
name = "log"
|
|
323
|
-
version = "0.4.
|
|
333
|
+
version = "0.4.27"
|
|
324
334
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
325
|
-
checksum = "
|
|
335
|
+
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
|
326
336
|
|
|
327
337
|
[[package]]
|
|
328
338
|
name = "macro_rules_attribute"
|
|
@@ -377,9 +387,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
|
|
377
387
|
|
|
378
388
|
[[package]]
|
|
379
389
|
name = "monostate"
|
|
380
|
-
version = "0.1.
|
|
390
|
+
version = "0.1.14"
|
|
381
391
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
382
|
-
checksum = "
|
|
392
|
+
checksum = "aafe1be9d0c75642e3e50fedc7ecadf1ef1cbce6eb66462153fc44245343fbee"
|
|
383
393
|
dependencies = [
|
|
384
394
|
"monostate-impl",
|
|
385
395
|
"serde",
|
|
@@ -387,9 +397,9 @@ dependencies = [
|
|
|
387
397
|
|
|
388
398
|
[[package]]
|
|
389
399
|
name = "monostate-impl"
|
|
390
|
-
version = "0.1.
|
|
400
|
+
version = "0.1.14"
|
|
391
401
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
392
|
-
checksum = "
|
|
402
|
+
checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5"
|
|
393
403
|
dependencies = [
|
|
394
404
|
"proc-macro2",
|
|
395
405
|
"quote",
|
|
@@ -414,9 +424,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
|
|
414
424
|
|
|
415
425
|
[[package]]
|
|
416
426
|
name = "once_cell"
|
|
417
|
-
version = "1.
|
|
427
|
+
version = "1.21.3"
|
|
418
428
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
419
|
-
checksum = "
|
|
429
|
+
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
|
420
430
|
|
|
421
431
|
[[package]]
|
|
422
432
|
name = "onig"
|
|
@@ -448,39 +458,39 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
|
|
448
458
|
|
|
449
459
|
[[package]]
|
|
450
460
|
name = "pkg-config"
|
|
451
|
-
version = "0.3.
|
|
461
|
+
version = "0.3.32"
|
|
452
462
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
453
|
-
checksum = "
|
|
463
|
+
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
|
454
464
|
|
|
455
465
|
[[package]]
|
|
456
466
|
name = "portable-atomic"
|
|
457
|
-
version = "1.
|
|
467
|
+
version = "1.11.0"
|
|
458
468
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
459
|
-
checksum = "
|
|
469
|
+
checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
|
|
460
470
|
|
|
461
471
|
[[package]]
|
|
462
472
|
name = "ppv-lite86"
|
|
463
|
-
version = "0.2.
|
|
473
|
+
version = "0.2.21"
|
|
464
474
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
465
|
-
checksum = "
|
|
475
|
+
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
|
466
476
|
dependencies = [
|
|
467
477
|
"zerocopy",
|
|
468
478
|
]
|
|
469
479
|
|
|
470
480
|
[[package]]
|
|
471
481
|
name = "proc-macro2"
|
|
472
|
-
version = "1.0.
|
|
482
|
+
version = "1.0.95"
|
|
473
483
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
474
|
-
checksum = "
|
|
484
|
+
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
|
|
475
485
|
dependencies = [
|
|
476
486
|
"unicode-ident",
|
|
477
487
|
]
|
|
478
488
|
|
|
479
489
|
[[package]]
|
|
480
490
|
name = "quote"
|
|
481
|
-
version = "1.0.
|
|
491
|
+
version = "1.0.40"
|
|
482
492
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
483
|
-
checksum = "
|
|
493
|
+
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
|
|
484
494
|
dependencies = [
|
|
485
495
|
"proc-macro2",
|
|
486
496
|
]
|
|
@@ -548,18 +558,18 @@ dependencies = [
|
|
|
548
558
|
|
|
549
559
|
[[package]]
|
|
550
560
|
name = "rb-sys"
|
|
551
|
-
version = "0.9.
|
|
561
|
+
version = "0.9.111"
|
|
552
562
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
553
|
-
checksum = "
|
|
563
|
+
checksum = "becea799ce051c16fb140be80f5e7cf781070f99ca099332383c2b17861249af"
|
|
554
564
|
dependencies = [
|
|
555
565
|
"rb-sys-build",
|
|
556
566
|
]
|
|
557
567
|
|
|
558
568
|
[[package]]
|
|
559
569
|
name = "rb-sys-build"
|
|
560
|
-
version = "0.9.
|
|
570
|
+
version = "0.9.111"
|
|
561
571
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
562
|
-
checksum = "
|
|
572
|
+
checksum = "64691175abc704862f60a9ca8ef06174080cc50615f2bf1d4759f46db18b4d29"
|
|
563
573
|
dependencies = [
|
|
564
574
|
"bindgen",
|
|
565
575
|
"lazy_static",
|
|
@@ -578,9 +588,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
|
|
|
578
588
|
|
|
579
589
|
[[package]]
|
|
580
590
|
name = "regex"
|
|
581
|
-
version = "1.
|
|
591
|
+
version = "1.11.1"
|
|
582
592
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
583
|
-
checksum = "
|
|
593
|
+
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
|
|
584
594
|
dependencies = [
|
|
585
595
|
"aho-corasick",
|
|
586
596
|
"memchr",
|
|
@@ -590,9 +600,9 @@ dependencies = [
|
|
|
590
600
|
|
|
591
601
|
[[package]]
|
|
592
602
|
name = "regex-automata"
|
|
593
|
-
version = "0.4.
|
|
603
|
+
version = "0.4.9"
|
|
594
604
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
595
|
-
checksum = "
|
|
605
|
+
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
|
|
596
606
|
dependencies = [
|
|
597
607
|
"aho-corasick",
|
|
598
608
|
"memchr",
|
|
@@ -601,9 +611,9 @@ dependencies = [
|
|
|
601
611
|
|
|
602
612
|
[[package]]
|
|
603
613
|
name = "regex-syntax"
|
|
604
|
-
version = "0.8.
|
|
614
|
+
version = "0.8.5"
|
|
605
615
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
606
|
-
checksum = "
|
|
616
|
+
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
|
607
617
|
|
|
608
618
|
[[package]]
|
|
609
619
|
name = "rustc-hash"
|
|
@@ -613,30 +623,30 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
|
613
623
|
|
|
614
624
|
[[package]]
|
|
615
625
|
name = "ryu"
|
|
616
|
-
version = "1.0.
|
|
626
|
+
version = "1.0.20"
|
|
617
627
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
618
|
-
checksum = "
|
|
628
|
+
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
|
|
619
629
|
|
|
620
630
|
[[package]]
|
|
621
631
|
name = "seq-macro"
|
|
622
|
-
version = "0.3.
|
|
632
|
+
version = "0.3.6"
|
|
623
633
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
624
|
-
checksum = "
|
|
634
|
+
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
|
|
625
635
|
|
|
626
636
|
[[package]]
|
|
627
637
|
name = "serde"
|
|
628
|
-
version = "1.0.
|
|
638
|
+
version = "1.0.219"
|
|
629
639
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
630
|
-
checksum = "
|
|
640
|
+
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
|
|
631
641
|
dependencies = [
|
|
632
642
|
"serde_derive",
|
|
633
643
|
]
|
|
634
644
|
|
|
635
645
|
[[package]]
|
|
636
646
|
name = "serde_derive"
|
|
637
|
-
version = "1.0.
|
|
647
|
+
version = "1.0.219"
|
|
638
648
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
639
|
-
checksum = "
|
|
649
|
+
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
|
|
640
650
|
dependencies = [
|
|
641
651
|
"proc-macro2",
|
|
642
652
|
"quote",
|
|
@@ -645,9 +655,9 @@ dependencies = [
|
|
|
645
655
|
|
|
646
656
|
[[package]]
|
|
647
657
|
name = "serde_json"
|
|
648
|
-
version = "1.0.
|
|
658
|
+
version = "1.0.140"
|
|
649
659
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
650
|
-
checksum = "
|
|
660
|
+
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
|
|
651
661
|
dependencies = [
|
|
652
662
|
"itoa",
|
|
653
663
|
"memchr",
|
|
@@ -669,9 +679,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
|
669
679
|
|
|
670
680
|
[[package]]
|
|
671
681
|
name = "smallvec"
|
|
672
|
-
version = "1.
|
|
682
|
+
version = "1.15.0"
|
|
673
683
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
674
|
-
checksum = "
|
|
684
|
+
checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
|
|
675
685
|
|
|
676
686
|
[[package]]
|
|
677
687
|
name = "spm_precompiled"
|
|
@@ -693,9 +703,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|
|
693
703
|
|
|
694
704
|
[[package]]
|
|
695
705
|
name = "syn"
|
|
696
|
-
version = "2.0.
|
|
706
|
+
version = "2.0.101"
|
|
697
707
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
698
|
-
checksum = "
|
|
708
|
+
checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
|
|
699
709
|
dependencies = [
|
|
700
710
|
"proc-macro2",
|
|
701
711
|
"quote",
|
|
@@ -704,18 +714,18 @@ dependencies = [
|
|
|
704
714
|
|
|
705
715
|
[[package]]
|
|
706
716
|
name = "thiserror"
|
|
707
|
-
version = "
|
|
717
|
+
version = "2.0.12"
|
|
708
718
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
709
|
-
checksum = "
|
|
719
|
+
checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
|
|
710
720
|
dependencies = [
|
|
711
721
|
"thiserror-impl",
|
|
712
722
|
]
|
|
713
723
|
|
|
714
724
|
[[package]]
|
|
715
725
|
name = "thiserror-impl"
|
|
716
|
-
version = "
|
|
726
|
+
version = "2.0.12"
|
|
717
727
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
718
|
-
checksum = "
|
|
728
|
+
checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
|
|
719
729
|
dependencies = [
|
|
720
730
|
"proc-macro2",
|
|
721
731
|
"quote",
|
|
@@ -724,26 +734,26 @@ dependencies = [
|
|
|
724
734
|
|
|
725
735
|
[[package]]
|
|
726
736
|
name = "tokenizers"
|
|
727
|
-
version = "0.5.
|
|
737
|
+
version = "0.5.5"
|
|
728
738
|
dependencies = [
|
|
729
739
|
"magnus",
|
|
730
740
|
"onig",
|
|
731
741
|
"serde",
|
|
732
|
-
"tokenizers 0.
|
|
742
|
+
"tokenizers 0.21.1",
|
|
733
743
|
]
|
|
734
744
|
|
|
735
745
|
[[package]]
|
|
736
746
|
name = "tokenizers"
|
|
737
|
-
version = "0.
|
|
747
|
+
version = "0.21.1"
|
|
738
748
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
739
|
-
checksum = "
|
|
749
|
+
checksum = "3169b3195f925496c895caee7978a335d49218488ef22375267fba5a46a40bd7"
|
|
740
750
|
dependencies = [
|
|
741
751
|
"aho-corasick",
|
|
742
752
|
"derive_builder",
|
|
743
753
|
"esaxx-rs",
|
|
744
754
|
"getrandom",
|
|
745
755
|
"indicatif",
|
|
746
|
-
"itertools 0.
|
|
756
|
+
"itertools 0.13.0",
|
|
747
757
|
"lazy_static",
|
|
748
758
|
"log",
|
|
749
759
|
"macro_rules_attribute",
|
|
@@ -766,9 +776,9 @@ dependencies = [
|
|
|
766
776
|
|
|
767
777
|
[[package]]
|
|
768
778
|
name = "unicode-ident"
|
|
769
|
-
version = "1.0.
|
|
779
|
+
version = "1.0.18"
|
|
770
780
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
771
|
-
checksum = "
|
|
781
|
+
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
|
772
782
|
|
|
773
783
|
[[package]]
|
|
774
784
|
name = "unicode-normalization-alignments"
|
|
@@ -781,15 +791,15 @@ dependencies = [
|
|
|
781
791
|
|
|
782
792
|
[[package]]
|
|
783
793
|
name = "unicode-segmentation"
|
|
784
|
-
version = "1.
|
|
794
|
+
version = "1.12.0"
|
|
785
795
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
786
|
-
checksum = "
|
|
796
|
+
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
|
|
787
797
|
|
|
788
798
|
[[package]]
|
|
789
799
|
name = "unicode-width"
|
|
790
|
-
version = "0.
|
|
800
|
+
version = "0.2.0"
|
|
791
801
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
792
|
-
checksum = "
|
|
802
|
+
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
|
|
793
803
|
|
|
794
804
|
[[package]]
|
|
795
805
|
name = "unicode_categories"
|
|
@@ -803,11 +813,78 @@ version = "0.11.0+wasi-snapshot-preview1"
|
|
|
803
813
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
804
814
|
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
|
805
815
|
|
|
816
|
+
[[package]]
|
|
817
|
+
name = "wasm-bindgen"
|
|
818
|
+
version = "0.2.100"
|
|
819
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
820
|
+
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
|
|
821
|
+
dependencies = [
|
|
822
|
+
"cfg-if",
|
|
823
|
+
"once_cell",
|
|
824
|
+
"wasm-bindgen-macro",
|
|
825
|
+
]
|
|
826
|
+
|
|
827
|
+
[[package]]
|
|
828
|
+
name = "wasm-bindgen-backend"
|
|
829
|
+
version = "0.2.100"
|
|
830
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
831
|
+
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
|
|
832
|
+
dependencies = [
|
|
833
|
+
"bumpalo",
|
|
834
|
+
"log",
|
|
835
|
+
"proc-macro2",
|
|
836
|
+
"quote",
|
|
837
|
+
"syn",
|
|
838
|
+
"wasm-bindgen-shared",
|
|
839
|
+
]
|
|
840
|
+
|
|
841
|
+
[[package]]
|
|
842
|
+
name = "wasm-bindgen-macro"
|
|
843
|
+
version = "0.2.100"
|
|
844
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
845
|
+
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
|
|
846
|
+
dependencies = [
|
|
847
|
+
"quote",
|
|
848
|
+
"wasm-bindgen-macro-support",
|
|
849
|
+
]
|
|
850
|
+
|
|
851
|
+
[[package]]
|
|
852
|
+
name = "wasm-bindgen-macro-support"
|
|
853
|
+
version = "0.2.100"
|
|
854
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
855
|
+
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
|
|
856
|
+
dependencies = [
|
|
857
|
+
"proc-macro2",
|
|
858
|
+
"quote",
|
|
859
|
+
"syn",
|
|
860
|
+
"wasm-bindgen-backend",
|
|
861
|
+
"wasm-bindgen-shared",
|
|
862
|
+
]
|
|
863
|
+
|
|
864
|
+
[[package]]
|
|
865
|
+
name = "wasm-bindgen-shared"
|
|
866
|
+
version = "0.2.100"
|
|
867
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
868
|
+
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
|
|
869
|
+
dependencies = [
|
|
870
|
+
"unicode-ident",
|
|
871
|
+
]
|
|
872
|
+
|
|
873
|
+
[[package]]
|
|
874
|
+
name = "web-time"
|
|
875
|
+
version = "1.1.0"
|
|
876
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
877
|
+
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
|
878
|
+
dependencies = [
|
|
879
|
+
"js-sys",
|
|
880
|
+
"wasm-bindgen",
|
|
881
|
+
]
|
|
882
|
+
|
|
806
883
|
[[package]]
|
|
807
884
|
name = "windows-sys"
|
|
808
|
-
version = "0.
|
|
885
|
+
version = "0.59.0"
|
|
809
886
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
810
|
-
checksum = "
|
|
887
|
+
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
|
811
888
|
dependencies = [
|
|
812
889
|
"windows-targets",
|
|
813
890
|
]
|
|
@@ -878,19 +955,18 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
|
|
878
955
|
|
|
879
956
|
[[package]]
|
|
880
957
|
name = "zerocopy"
|
|
881
|
-
version = "0.
|
|
958
|
+
version = "0.8.25"
|
|
882
959
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
883
|
-
checksum = "
|
|
960
|
+
checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
|
|
884
961
|
dependencies = [
|
|
885
|
-
"byteorder",
|
|
886
962
|
"zerocopy-derive",
|
|
887
963
|
]
|
|
888
964
|
|
|
889
965
|
[[package]]
|
|
890
966
|
name = "zerocopy-derive"
|
|
891
|
-
version = "0.
|
|
967
|
+
version = "0.8.25"
|
|
892
968
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
893
|
-
checksum = "
|
|
969
|
+
checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
|
|
894
970
|
dependencies = [
|
|
895
971
|
"proc-macro2",
|
|
896
972
|
"quote",
|