tiktoken_ruby 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vscode/settings.json +5 -0
- data/Cargo.lock +117 -133
- data/Gemfile.lock +4 -4
- data/Rakefile +6 -0
- data/ext/tiktoken_ruby/Cargo.toml +2 -2
- data/ext/tiktoken_ruby/src/lib.rs +6 -0
- data/lib/tiktoken_ruby/version.rb +1 -1
- data/lib/tiktoken_ruby.rb +4 -1
- data/script/release +33 -0
- metadata +12 -11
- data/tiktoken_ruby.gemspec +0 -33
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: df7ca54ce72c32b1a0a0b278553f35365bf283af271f98436b4d330f6fb05735
|
|
4
|
+
data.tar.gz: 5a225b39a85ce71779bb7e58c6f6c47eb8a884b813377783d833b27af53cbb9e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d15ec5fee05da4104e7111e237e5bf5e80291d226c51afd90fa580ebed87571140db4b06a17e39a82892489a6a932cb02d594a9cbf047315c5094bb4f25bd99b
|
|
7
|
+
data.tar.gz: 444bd7e59d054395689a3f45beb3c288262e1e9e1903dc565821780ad9929de6aabbdbd9f543506b3cc662a54835892e9b34b9ece259d6f3f27d5642a8f1b75a
|
data/Cargo.lock
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
1
|
# This file is automatically @generated by Cargo.
|
|
2
2
|
# It is not intended for manual editing.
|
|
3
|
-
version =
|
|
3
|
+
version = 4
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
6
6
|
name = "aho-corasick"
|
|
7
|
-
version = "
|
|
7
|
+
version = "1.1.3"
|
|
8
8
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
-
checksum = "
|
|
9
|
+
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"memchr",
|
|
12
12
|
]
|
|
13
13
|
|
|
14
14
|
[[package]]
|
|
15
15
|
name = "anyhow"
|
|
16
|
-
version = "1.0.
|
|
16
|
+
version = "1.0.95"
|
|
17
17
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
-
checksum = "
|
|
18
|
+
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
|
|
19
19
|
|
|
20
20
|
[[package]]
|
|
21
21
|
name = "autocfg"
|
|
22
|
-
version = "1.
|
|
22
|
+
version = "1.4.0"
|
|
23
23
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
24
|
-
checksum = "
|
|
24
|
+
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
|
|
25
25
|
|
|
26
26
|
[[package]]
|
|
27
27
|
name = "base64"
|
|
28
|
-
version = "0.21.
|
|
28
|
+
version = "0.21.7"
|
|
29
29
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
|
-
checksum = "
|
|
30
|
+
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
|
|
31
31
|
|
|
32
32
|
[[package]]
|
|
33
33
|
name = "bindgen"
|
|
34
|
-
version = "0.69.
|
|
34
|
+
version = "0.69.5"
|
|
35
35
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
-
checksum = "
|
|
36
|
+
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
|
|
37
37
|
dependencies = [
|
|
38
|
-
"bitflags
|
|
38
|
+
"bitflags",
|
|
39
39
|
"cexpr",
|
|
40
40
|
"clang-sys",
|
|
41
41
|
"itertools",
|
|
@@ -66,24 +66,17 @@ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
|
|
|
66
66
|
|
|
67
67
|
[[package]]
|
|
68
68
|
name = "bitflags"
|
|
69
|
-
version = "
|
|
70
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
71
|
-
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
|
72
|
-
|
|
73
|
-
[[package]]
|
|
74
|
-
name = "bitflags"
|
|
75
|
-
version = "2.4.0"
|
|
69
|
+
version = "2.6.0"
|
|
76
70
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
77
|
-
checksum = "
|
|
71
|
+
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
|
|
78
72
|
|
|
79
73
|
[[package]]
|
|
80
74
|
name = "bstr"
|
|
81
|
-
version = "1.
|
|
75
|
+
version = "1.11.1"
|
|
82
76
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
83
|
-
checksum = "
|
|
77
|
+
checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8"
|
|
84
78
|
dependencies = [
|
|
85
79
|
"memchr",
|
|
86
|
-
"once_cell",
|
|
87
80
|
"regex-automata",
|
|
88
81
|
"serde",
|
|
89
82
|
]
|
|
@@ -105,9 +98,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|
|
105
98
|
|
|
106
99
|
[[package]]
|
|
107
100
|
name = "clang-sys"
|
|
108
|
-
version = "1.
|
|
101
|
+
version = "1.8.1"
|
|
109
102
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
110
|
-
checksum = "
|
|
103
|
+
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
|
|
111
104
|
dependencies = [
|
|
112
105
|
"glob",
|
|
113
106
|
"libc",
|
|
@@ -116,15 +109,15 @@ dependencies = [
|
|
|
116
109
|
|
|
117
110
|
[[package]]
|
|
118
111
|
name = "either"
|
|
119
|
-
version = "1.
|
|
112
|
+
version = "1.13.0"
|
|
120
113
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
121
|
-
checksum = "
|
|
114
|
+
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
|
|
122
115
|
|
|
123
116
|
[[package]]
|
|
124
117
|
name = "fancy-regex"
|
|
125
|
-
version = "0.
|
|
118
|
+
version = "0.12.0"
|
|
126
119
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
127
|
-
checksum = "
|
|
120
|
+
checksum = "7493d4c459da9f84325ad297371a6b2b8a162800873a22e3b6b6512e61d18c05"
|
|
128
121
|
dependencies = [
|
|
129
122
|
"bit-set",
|
|
130
123
|
"regex",
|
|
@@ -132,9 +125,9 @@ dependencies = [
|
|
|
132
125
|
|
|
133
126
|
[[package]]
|
|
134
127
|
name = "glob"
|
|
135
|
-
version = "0.3.
|
|
128
|
+
version = "0.3.2"
|
|
136
129
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
137
|
-
checksum = "
|
|
130
|
+
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
|
138
131
|
|
|
139
132
|
[[package]]
|
|
140
133
|
name = "itertools"
|
|
@@ -147,9 +140,9 @@ dependencies = [
|
|
|
147
140
|
|
|
148
141
|
[[package]]
|
|
149
142
|
name = "lazy_static"
|
|
150
|
-
version = "1.
|
|
143
|
+
version = "1.5.0"
|
|
151
144
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
152
|
-
checksum = "
|
|
145
|
+
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
|
153
146
|
|
|
154
147
|
[[package]]
|
|
155
148
|
name = "lazycell"
|
|
@@ -159,25 +152,25 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
|
159
152
|
|
|
160
153
|
[[package]]
|
|
161
154
|
name = "libc"
|
|
162
|
-
version = "0.2.
|
|
155
|
+
version = "0.2.169"
|
|
163
156
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
164
|
-
checksum = "
|
|
157
|
+
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
|
165
158
|
|
|
166
159
|
[[package]]
|
|
167
160
|
name = "libloading"
|
|
168
|
-
version = "0.
|
|
161
|
+
version = "0.8.6"
|
|
169
162
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
170
|
-
checksum = "
|
|
163
|
+
checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
|
|
171
164
|
dependencies = [
|
|
172
165
|
"cfg-if",
|
|
173
|
-
"
|
|
166
|
+
"windows-targets",
|
|
174
167
|
]
|
|
175
168
|
|
|
176
169
|
[[package]]
|
|
177
170
|
name = "lock_api"
|
|
178
|
-
version = "0.4.
|
|
171
|
+
version = "0.4.12"
|
|
179
172
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
180
|
-
checksum = "
|
|
173
|
+
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
|
|
181
174
|
dependencies = [
|
|
182
175
|
"autocfg",
|
|
183
176
|
"scopeguard",
|
|
@@ -185,9 +178,9 @@ dependencies = [
|
|
|
185
178
|
|
|
186
179
|
[[package]]
|
|
187
180
|
name = "magnus"
|
|
188
|
-
version = "0.6.
|
|
181
|
+
version = "0.6.4"
|
|
189
182
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
190
|
-
checksum = "
|
|
183
|
+
checksum = "b1597ef40aa8c36be098249e82c9a20cf7199278ac1c1a1a995eeead6a184479"
|
|
191
184
|
dependencies = [
|
|
192
185
|
"magnus-macros",
|
|
193
186
|
"rb-sys",
|
|
@@ -208,9 +201,9 @@ dependencies = [
|
|
|
208
201
|
|
|
209
202
|
[[package]]
|
|
210
203
|
name = "memchr"
|
|
211
|
-
version = "2.
|
|
204
|
+
version = "2.7.4"
|
|
212
205
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
213
|
-
checksum = "
|
|
206
|
+
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
|
214
207
|
|
|
215
208
|
[[package]]
|
|
216
209
|
name = "minimal-lexical"
|
|
@@ -228,17 +221,11 @@ dependencies = [
|
|
|
228
221
|
"minimal-lexical",
|
|
229
222
|
]
|
|
230
223
|
|
|
231
|
-
[[package]]
|
|
232
|
-
name = "once_cell"
|
|
233
|
-
version = "1.17.1"
|
|
234
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
235
|
-
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
|
|
236
|
-
|
|
237
224
|
[[package]]
|
|
238
225
|
name = "parking_lot"
|
|
239
|
-
version = "0.12.
|
|
226
|
+
version = "0.12.3"
|
|
240
227
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
241
|
-
checksum = "
|
|
228
|
+
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
|
|
242
229
|
dependencies = [
|
|
243
230
|
"lock_api",
|
|
244
231
|
"parking_lot_core",
|
|
@@ -246,49 +233,49 @@ dependencies = [
|
|
|
246
233
|
|
|
247
234
|
[[package]]
|
|
248
235
|
name = "parking_lot_core"
|
|
249
|
-
version = "0.9.
|
|
236
|
+
version = "0.9.10"
|
|
250
237
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
251
|
-
checksum = "
|
|
238
|
+
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
|
|
252
239
|
dependencies = [
|
|
253
240
|
"cfg-if",
|
|
254
241
|
"libc",
|
|
255
242
|
"redox_syscall",
|
|
256
243
|
"smallvec",
|
|
257
|
-
"windows-
|
|
244
|
+
"windows-targets",
|
|
258
245
|
]
|
|
259
246
|
|
|
260
247
|
[[package]]
|
|
261
248
|
name = "proc-macro2"
|
|
262
|
-
version = "1.0.
|
|
249
|
+
version = "1.0.92"
|
|
263
250
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
264
|
-
checksum = "
|
|
251
|
+
checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
|
|
265
252
|
dependencies = [
|
|
266
253
|
"unicode-ident",
|
|
267
254
|
]
|
|
268
255
|
|
|
269
256
|
[[package]]
|
|
270
257
|
name = "quote"
|
|
271
|
-
version = "1.0.
|
|
258
|
+
version = "1.0.38"
|
|
272
259
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
273
|
-
checksum = "
|
|
260
|
+
checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
|
|
274
261
|
dependencies = [
|
|
275
262
|
"proc-macro2",
|
|
276
263
|
]
|
|
277
264
|
|
|
278
265
|
[[package]]
|
|
279
266
|
name = "rb-sys"
|
|
280
|
-
version = "0.9.
|
|
267
|
+
version = "0.9.105"
|
|
281
268
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
282
|
-
checksum = "
|
|
269
|
+
checksum = "4b3a1f3ce8e7c36d777d52fe7a99039fe4fea7c8ec355a4c4f3a17f92a14029f"
|
|
283
270
|
dependencies = [
|
|
284
271
|
"rb-sys-build",
|
|
285
272
|
]
|
|
286
273
|
|
|
287
274
|
[[package]]
|
|
288
275
|
name = "rb-sys-build"
|
|
289
|
-
version = "0.9.
|
|
276
|
+
version = "0.9.105"
|
|
290
277
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
291
|
-
checksum = "
|
|
278
|
+
checksum = "3e6b246c29c0809e1cbe60a1ba9e093da72a4676d02adc68469297d1e589bbf0"
|
|
292
279
|
dependencies = [
|
|
293
280
|
"bindgen",
|
|
294
281
|
"lazy_static",
|
|
@@ -307,35 +294,41 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
|
|
|
307
294
|
|
|
308
295
|
[[package]]
|
|
309
296
|
name = "redox_syscall"
|
|
310
|
-
version = "0.
|
|
297
|
+
version = "0.5.8"
|
|
311
298
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
312
|
-
checksum = "
|
|
299
|
+
checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
|
|
313
300
|
dependencies = [
|
|
314
|
-
"bitflags
|
|
301
|
+
"bitflags",
|
|
315
302
|
]
|
|
316
303
|
|
|
317
304
|
[[package]]
|
|
318
305
|
name = "regex"
|
|
319
|
-
version = "1.
|
|
306
|
+
version = "1.11.1"
|
|
320
307
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
321
|
-
checksum = "
|
|
308
|
+
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
|
|
322
309
|
dependencies = [
|
|
323
310
|
"aho-corasick",
|
|
324
311
|
"memchr",
|
|
312
|
+
"regex-automata",
|
|
325
313
|
"regex-syntax",
|
|
326
314
|
]
|
|
327
315
|
|
|
328
316
|
[[package]]
|
|
329
317
|
name = "regex-automata"
|
|
330
|
-
version = "0.
|
|
318
|
+
version = "0.4.9"
|
|
331
319
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
332
|
-
checksum = "
|
|
320
|
+
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
|
|
321
|
+
dependencies = [
|
|
322
|
+
"aho-corasick",
|
|
323
|
+
"memchr",
|
|
324
|
+
"regex-syntax",
|
|
325
|
+
]
|
|
333
326
|
|
|
334
327
|
[[package]]
|
|
335
328
|
name = "regex-syntax"
|
|
336
|
-
version = "0.
|
|
329
|
+
version = "0.8.5"
|
|
337
330
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
338
|
-
checksum = "
|
|
331
|
+
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
|
339
332
|
|
|
340
333
|
[[package]]
|
|
341
334
|
name = "rustc-hash"
|
|
@@ -345,9 +338,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
|
345
338
|
|
|
346
339
|
[[package]]
|
|
347
340
|
name = "scopeguard"
|
|
348
|
-
version = "1.
|
|
341
|
+
version = "1.2.0"
|
|
349
342
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
350
|
-
checksum = "
|
|
343
|
+
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
|
351
344
|
|
|
352
345
|
[[package]]
|
|
353
346
|
name = "seq-macro"
|
|
@@ -357,9 +350,23 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
|
|
|
357
350
|
|
|
358
351
|
[[package]]
|
|
359
352
|
name = "serde"
|
|
360
|
-
version = "1.0.
|
|
353
|
+
version = "1.0.217"
|
|
354
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
355
|
+
checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
|
|
356
|
+
dependencies = [
|
|
357
|
+
"serde_derive",
|
|
358
|
+
]
|
|
359
|
+
|
|
360
|
+
[[package]]
|
|
361
|
+
name = "serde_derive"
|
|
362
|
+
version = "1.0.217"
|
|
361
363
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
362
|
-
checksum = "
|
|
364
|
+
checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
|
|
365
|
+
dependencies = [
|
|
366
|
+
"proc-macro2",
|
|
367
|
+
"quote",
|
|
368
|
+
"syn",
|
|
369
|
+
]
|
|
363
370
|
|
|
364
371
|
[[package]]
|
|
365
372
|
name = "shell-words"
|
|
@@ -369,21 +376,21 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
|
|
|
369
376
|
|
|
370
377
|
[[package]]
|
|
371
378
|
name = "shlex"
|
|
372
|
-
version = "1.
|
|
379
|
+
version = "1.3.0"
|
|
373
380
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
374
|
-
checksum = "
|
|
381
|
+
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
375
382
|
|
|
376
383
|
[[package]]
|
|
377
384
|
name = "smallvec"
|
|
378
|
-
version = "1.
|
|
385
|
+
version = "1.13.2"
|
|
379
386
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
380
|
-
checksum = "
|
|
387
|
+
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
|
|
381
388
|
|
|
382
389
|
[[package]]
|
|
383
390
|
name = "syn"
|
|
384
|
-
version = "2.0.
|
|
391
|
+
version = "2.0.93"
|
|
385
392
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
386
|
-
checksum = "
|
|
393
|
+
checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058"
|
|
387
394
|
dependencies = [
|
|
388
395
|
"proc-macro2",
|
|
389
396
|
"quote",
|
|
@@ -392,8 +399,9 @@ dependencies = [
|
|
|
392
399
|
|
|
393
400
|
[[package]]
|
|
394
401
|
name = "tiktoken-rs"
|
|
395
|
-
version = "0.
|
|
396
|
-
source = "
|
|
402
|
+
version = "0.5.9"
|
|
403
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
404
|
+
checksum = "c314e7ce51440f9e8f5a497394682a57b7c323d0f4d0a6b1b13c429056e0e234"
|
|
397
405
|
dependencies = [
|
|
398
406
|
"anyhow",
|
|
399
407
|
"base64",
|
|
@@ -415,50 +423,20 @@ dependencies = [
|
|
|
415
423
|
|
|
416
424
|
[[package]]
|
|
417
425
|
name = "unicode-ident"
|
|
418
|
-
version = "1.0.
|
|
426
|
+
version = "1.0.14"
|
|
419
427
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
420
|
-
checksum = "
|
|
421
|
-
|
|
422
|
-
[[package]]
|
|
423
|
-
name = "winapi"
|
|
424
|
-
version = "0.3.9"
|
|
425
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
426
|
-
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
|
427
|
-
dependencies = [
|
|
428
|
-
"winapi-i686-pc-windows-gnu",
|
|
429
|
-
"winapi-x86_64-pc-windows-gnu",
|
|
430
|
-
]
|
|
431
|
-
|
|
432
|
-
[[package]]
|
|
433
|
-
name = "winapi-i686-pc-windows-gnu"
|
|
434
|
-
version = "0.4.0"
|
|
435
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
436
|
-
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
|
437
|
-
|
|
438
|
-
[[package]]
|
|
439
|
-
name = "winapi-x86_64-pc-windows-gnu"
|
|
440
|
-
version = "0.4.0"
|
|
441
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
442
|
-
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
|
443
|
-
|
|
444
|
-
[[package]]
|
|
445
|
-
name = "windows-sys"
|
|
446
|
-
version = "0.45.0"
|
|
447
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
448
|
-
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
|
|
449
|
-
dependencies = [
|
|
450
|
-
"windows-targets",
|
|
451
|
-
]
|
|
428
|
+
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
|
|
452
429
|
|
|
453
430
|
[[package]]
|
|
454
431
|
name = "windows-targets"
|
|
455
|
-
version = "0.
|
|
432
|
+
version = "0.52.6"
|
|
456
433
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
457
|
-
checksum = "
|
|
434
|
+
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
|
458
435
|
dependencies = [
|
|
459
436
|
"windows_aarch64_gnullvm",
|
|
460
437
|
"windows_aarch64_msvc",
|
|
461
438
|
"windows_i686_gnu",
|
|
439
|
+
"windows_i686_gnullvm",
|
|
462
440
|
"windows_i686_msvc",
|
|
463
441
|
"windows_x86_64_gnu",
|
|
464
442
|
"windows_x86_64_gnullvm",
|
|
@@ -467,42 +445,48 @@ dependencies = [
|
|
|
467
445
|
|
|
468
446
|
[[package]]
|
|
469
447
|
name = "windows_aarch64_gnullvm"
|
|
470
|
-
version = "0.
|
|
448
|
+
version = "0.52.6"
|
|
471
449
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
472
|
-
checksum = "
|
|
450
|
+
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
|
473
451
|
|
|
474
452
|
[[package]]
|
|
475
453
|
name = "windows_aarch64_msvc"
|
|
476
|
-
version = "0.
|
|
454
|
+
version = "0.52.6"
|
|
477
455
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
478
|
-
checksum = "
|
|
456
|
+
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
|
479
457
|
|
|
480
458
|
[[package]]
|
|
481
459
|
name = "windows_i686_gnu"
|
|
482
|
-
version = "0.
|
|
460
|
+
version = "0.52.6"
|
|
461
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
462
|
+
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
|
463
|
+
|
|
464
|
+
[[package]]
|
|
465
|
+
name = "windows_i686_gnullvm"
|
|
466
|
+
version = "0.52.6"
|
|
483
467
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
484
|
-
checksum = "
|
|
468
|
+
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
|
485
469
|
|
|
486
470
|
[[package]]
|
|
487
471
|
name = "windows_i686_msvc"
|
|
488
|
-
version = "0.
|
|
472
|
+
version = "0.52.6"
|
|
489
473
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
490
|
-
checksum = "
|
|
474
|
+
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
|
491
475
|
|
|
492
476
|
[[package]]
|
|
493
477
|
name = "windows_x86_64_gnu"
|
|
494
|
-
version = "0.
|
|
478
|
+
version = "0.52.6"
|
|
495
479
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
496
|
-
checksum = "
|
|
480
|
+
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
|
497
481
|
|
|
498
482
|
[[package]]
|
|
499
483
|
name = "windows_x86_64_gnullvm"
|
|
500
|
-
version = "0.
|
|
484
|
+
version = "0.52.6"
|
|
501
485
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
502
|
-
checksum = "
|
|
486
|
+
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
|
503
487
|
|
|
504
488
|
[[package]]
|
|
505
489
|
name = "windows_x86_64_msvc"
|
|
506
|
-
version = "0.
|
|
490
|
+
version = "0.52.6"
|
|
507
491
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
508
|
-
checksum = "
|
|
492
|
+
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
data/Gemfile.lock
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
tiktoken_ruby (0.0.
|
|
5
|
-
rb_sys (>= 0.9.
|
|
4
|
+
tiktoken_ruby (0.0.10)
|
|
5
|
+
rb_sys (>= 0.9.87)
|
|
6
6
|
|
|
7
7
|
GEM
|
|
8
8
|
remote: https://rubygems.org/
|
|
@@ -22,7 +22,7 @@ GEM
|
|
|
22
22
|
rake (13.1.0)
|
|
23
23
|
rake-compiler (1.2.5)
|
|
24
24
|
rake
|
|
25
|
-
rb_sys (0.9.
|
|
25
|
+
rb_sys (0.9.105)
|
|
26
26
|
regexp_parser (2.9.0)
|
|
27
27
|
rexml (3.2.6)
|
|
28
28
|
rspec (3.12.0)
|
|
@@ -89,4 +89,4 @@ DEPENDENCIES
|
|
|
89
89
|
yard-doctest
|
|
90
90
|
|
|
91
91
|
BUNDLED WITH
|
|
92
|
-
2.
|
|
92
|
+
2.5.18
|
data/Rakefile
CHANGED
|
@@ -21,3 +21,9 @@ end
|
|
|
21
21
|
task build: :compile
|
|
22
22
|
|
|
23
23
|
task default: %i[compile spec standard]
|
|
24
|
+
|
|
25
|
+
# Packaging default (non-precompiled) gem
|
|
26
|
+
require "rubygems/package_task"
|
|
27
|
+
gem_path = Gem::PackageTask.new(GEMSPEC).define
|
|
28
|
+
desc "Package the Ruby gem"
|
|
29
|
+
task "package" => [gem_path]
|
|
@@ -11,5 +11,5 @@ crate-type = ["cdylib"]
|
|
|
11
11
|
|
|
12
12
|
[dependencies]
|
|
13
13
|
magnus = { version = "0.6.1" }
|
|
14
|
-
rb-sys = { version = "
|
|
15
|
-
tiktoken-rs = {
|
|
14
|
+
rb-sys = { version = "0.9.87", features = ["stable-api-compiled-fallback"] }
|
|
15
|
+
tiktoken-rs = { version = "0.5.9" }
|
|
@@ -20,6 +20,11 @@ fn cl100k_base() -> CoreBPEWrapper {
|
|
|
20
20
|
CoreBPEWrapper::new(core_bpe)
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
fn o200k_base() -> CoreBPEWrapper {
|
|
24
|
+
let core_bpe = tiktoken_rs::o200k_base().unwrap();
|
|
25
|
+
CoreBPEWrapper::new(core_bpe)
|
|
26
|
+
}
|
|
27
|
+
|
|
23
28
|
fn module() -> Result<RModule, magnus::Error> {
|
|
24
29
|
define_module("Tiktoken")
|
|
25
30
|
}
|
|
@@ -37,6 +42,7 @@ fn init() -> Result<(), Error> {
|
|
|
37
42
|
factory_module.define_singleton_method("p50k_base", function!(p50k_base, 0))?;
|
|
38
43
|
factory_module.define_singleton_method("p50k_edit", function!(p50k_edit, 0))?;
|
|
39
44
|
factory_module.define_singleton_method("cl100k_base", function!(cl100k_base, 0))?;
|
|
45
|
+
factory_module.define_singleton_method("o200k_base", function!(o200k_base, 0))?;
|
|
40
46
|
|
|
41
47
|
let ext_module = module.define_module("Ext")?;
|
|
42
48
|
let bpe_class = ext_module.define_class("CoreBPE", class::object())?;
|
data/lib/tiktoken_ruby.rb
CHANGED
|
@@ -64,13 +64,15 @@ module Tiktoken
|
|
|
64
64
|
:r50k_base,
|
|
65
65
|
:p50k_base,
|
|
66
66
|
:p50k_edit,
|
|
67
|
-
:cl100k_base
|
|
67
|
+
:cl100k_base,
|
|
68
|
+
:o200k_base
|
|
68
69
|
]
|
|
69
70
|
|
|
70
71
|
# taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
|
|
71
72
|
# that is also MIT licensed but by OpenAI
|
|
72
73
|
MODEL_TO_ENCODING_NAME = {
|
|
73
74
|
# chat
|
|
75
|
+
"gpt-4o": "o200k_base",
|
|
74
76
|
"gpt-4": "cl100k_base",
|
|
75
77
|
"gpt-3.5-turbo": "cl100k_base",
|
|
76
78
|
"gpt-35-turbo": "cl100k_base", # Azure deployment name
|
|
@@ -120,6 +122,7 @@ module Tiktoken
|
|
|
120
122
|
|
|
121
123
|
MODEL_PREFIX_TO_ENCODING = {
|
|
122
124
|
# chat
|
|
125
|
+
"gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13, etc.
|
|
123
126
|
"gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k
|
|
124
127
|
"gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc.
|
|
125
128
|
"gpt-35-turbo-": "cl100k_base", # Azure deployment name
|
data/script/release
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
|
|
3
|
+
set -e
|
|
4
|
+
|
|
5
|
+
if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
|
|
6
|
+
echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
|
|
7
|
+
exit 1
|
|
8
|
+
fi
|
|
9
|
+
|
|
10
|
+
version=$(grep VERSION lib/tiktoken_ruby/version.rb | head -n 1 | cut -d'"' -f2)
|
|
11
|
+
echo "Building tiktoken_ruby v$version"
|
|
12
|
+
|
|
13
|
+
targets=(
|
|
14
|
+
"arm64-darwin"
|
|
15
|
+
"x86_64-darwin"
|
|
16
|
+
"aarch64-linux"
|
|
17
|
+
"x86_64-linux"
|
|
18
|
+
"x86_64-linux-musl"
|
|
19
|
+
"arm-linux"
|
|
20
|
+
"x64-mingw-ucrt"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# for target in "${targets[@]}"; do
|
|
24
|
+
# bundle exec rb-sys-dock -p "$target" --ruby-versions 3.2 --build
|
|
25
|
+
# done
|
|
26
|
+
|
|
27
|
+
for gem in pkg/tiktoken_ruby-"$version"-*.gem ; do
|
|
28
|
+
GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
|
|
29
|
+
done
|
|
30
|
+
|
|
31
|
+
# last but not least, the uncompiled gem
|
|
32
|
+
bundle exec rake package
|
|
33
|
+
GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tiktoken_ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.10
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- IAPark
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-12-29 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rb_sys
|
|
@@ -16,14 +16,14 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: 0.9.
|
|
19
|
+
version: 0.9.87
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: 0.9.
|
|
26
|
+
version: 0.9.87
|
|
27
27
|
description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
|
|
28
28
|
used by OpenAI. It can be used to count the number of tokens in text before sending
|
|
29
29
|
it to OpenAI APIs.
|
|
@@ -36,6 +36,7 @@ extra_rdoc_files: []
|
|
|
36
36
|
files:
|
|
37
37
|
- ".rspec"
|
|
38
38
|
- ".standard.yml"
|
|
39
|
+
- ".vscode/settings.json"
|
|
39
40
|
- Cargo.lock
|
|
40
41
|
- Cargo.toml
|
|
41
42
|
- Gemfile
|
|
@@ -51,8 +52,8 @@ files:
|
|
|
51
52
|
- lib/tiktoken_ruby.rb
|
|
52
53
|
- lib/tiktoken_ruby/encoding.rb
|
|
53
54
|
- lib/tiktoken_ruby/version.rb
|
|
55
|
+
- script/release
|
|
54
56
|
- sig/tiktoken_ruby.rbs
|
|
55
|
-
- tiktoken_ruby.gemspec
|
|
56
57
|
homepage: https://github.com/IAPark/tiktoken_ruby
|
|
57
58
|
licenses:
|
|
58
59
|
- MIT
|
|
@@ -60,7 +61,7 @@ metadata:
|
|
|
60
61
|
homepage_uri: https://github.com/IAPark/tiktoken_ruby
|
|
61
62
|
source_code_uri: https://github.com/IAPark/tiktoken_ruby
|
|
62
63
|
documentation_uri: https://rubydoc.info/github/IAPark/tiktoken_ruby/main
|
|
63
|
-
post_install_message:
|
|
64
|
+
post_install_message:
|
|
64
65
|
rdoc_options: []
|
|
65
66
|
require_paths:
|
|
66
67
|
- lib
|
|
@@ -68,15 +69,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
68
69
|
requirements:
|
|
69
70
|
- - ">="
|
|
70
71
|
- !ruby/object:Gem::Version
|
|
71
|
-
version:
|
|
72
|
+
version: 3.1.0
|
|
72
73
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
73
74
|
requirements:
|
|
74
75
|
- - ">="
|
|
75
76
|
- !ruby/object:Gem::Version
|
|
76
|
-
version: 3.
|
|
77
|
+
version: 3.4.0
|
|
77
78
|
requirements: []
|
|
78
|
-
rubygems_version: 3.
|
|
79
|
-
signing_key:
|
|
79
|
+
rubygems_version: 3.5.23
|
|
80
|
+
signing_key:
|
|
80
81
|
specification_version: 4
|
|
81
82
|
summary: Ruby wrapper for Tiktoken
|
|
82
83
|
test_files: []
|
data/tiktoken_ruby.gemspec
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative "lib/tiktoken_ruby/version"
|
|
4
|
-
|
|
5
|
-
Gem::Specification.new do |spec|
|
|
6
|
-
spec.name = "tiktoken_ruby"
|
|
7
|
-
spec.version = Tiktoken::VERSION
|
|
8
|
-
spec.authors = ["IAPark"]
|
|
9
|
-
spec.email = ["isaac.a.park@gmail.com"]
|
|
10
|
-
spec.summary = "Ruby wrapper for Tiktoken"
|
|
11
|
-
spec.description = "An unofficial Ruby wrapper for Tiktoken, " \
|
|
12
|
-
"a BPE tokenizer written by and used by OpenAI. It can be used to " \
|
|
13
|
-
"count the number of tokens in text before sending it to OpenAI APIs."
|
|
14
|
-
spec.homepage = "https://github.com/IAPark/tiktoken_ruby"
|
|
15
|
-
spec.license = "MIT"
|
|
16
|
-
spec.required_ruby_version = ">= 2.7.0"
|
|
17
|
-
spec.required_rubygems_version = ">= 3.1.0"
|
|
18
|
-
spec.platform = Gem::Platform::RUBY
|
|
19
|
-
|
|
20
|
-
spec.metadata["homepage_uri"] = spec.homepage
|
|
21
|
-
spec.metadata["source_code_uri"] = "https://github.com/IAPark/tiktoken_ruby"
|
|
22
|
-
spec.metadata["documentation_uri"] = "https://rubydoc.info/github/IAPark/tiktoken_ruby/main"
|
|
23
|
-
spec.files = Dir.chdir(__dir__) do
|
|
24
|
-
`git ls-files -z`.split("\x0").reject do |f|
|
|
25
|
-
(f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
spec.bindir = "exe"
|
|
29
|
-
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
30
|
-
spec.require_paths = ["lib"]
|
|
31
|
-
spec.extensions = ["ext/tiktoken_ruby/extconf.rb"]
|
|
32
|
-
spec.add_dependency "rb_sys", ">= 0.9.86"
|
|
33
|
-
end
|