ru_token 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/CHANGELOG.md +30 -0
  4. data/Cargo.lock +439 -0
  5. data/Cargo.toml +7 -0
  6. data/LICENSE +674 -0
  7. data/README.md +50 -0
  8. data/Rakefile +22 -0
  9. data/ext/ru_token/.cargo/config.toml +9 -0
  10. data/ext/ru_token/Cargo.toml +13 -0
  11. data/ext/ru_token/extconf.rb +6 -0
  12. data/ext/ru_token/src/lib.rs +60 -0
  13. data/lib/ru_token/version.rb +5 -0
  14. data/lib/ru_token.rb +29 -0
  15. data/sig/ru_token.rbs +4 -0
  16. metadata +107 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: de568748c96f4bf2a10834fb2b0fe6fb8b61d08bf7346fc2b548f670c288073a
4
+ data.tar.gz: 82faf4029e6f550dcad8770ac7b9f616c0380e813983b3e61cdb6e2b6475de0c
5
+ SHA512:
6
+ metadata.gz: f4187bbc733caac2d5adf803be6907b864434fddc3e80ecf17db3ea202423e1ba6dd08afbaa5f752645af9e7ec5cc448fb756322664777d16d27fd2423322a83
7
+ data.tar.gz: f477e3522f689998fd84c6eb48869fd1caf10d7725e3cbc36cf6c70fd62b7785b88d8332cf338198eb30001d00051262faad0a1c45a70c0b51e3e447ed3a87ee
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/CHANGELOG.md ADDED
@@ -0,0 +1,30 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ---
11
+ ## [0.1.1] - 2025-07-30
12
+
13
+ ### Added
14
+ - Comprehensive RSpec test suite to validate token counts across all supported models.
15
+ - GitHub Actions workflow for continuous integration to automatically test against multiple Ruby versions and operating systems.
16
+ - Standard `rake release` task for simplified and automated gem publishing.
17
+
18
+ ### Changed
19
+ - Updated README with accurate usage examples and a complete, collapsible list of supported model aliases.
20
+ - Improved the default `rake` task to automatically compile the extension and run tests.
21
+
22
+ ### Fixed
23
+ - Corrected expected token counts in tests to match the `tiktoken-rs` library's actual output.
24
+
25
+ ---
26
+ ## [0.1.0] - 2025-07-29
27
+
28
+ ### Added
29
+ - Initial release of the `RuToken` gem.
30
+ - Core functionality to count tokens for OpenAI models using a native Rust extension.
data/Cargo.lock ADDED
@@ -0,0 +1,439 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.3"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "anyhow"
16
+ version = "1.0.98"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
19
+
20
+ [[package]]
21
+ name = "base64"
22
+ version = "0.22.1"
23
+ source = "registry+https://github.com/rust-lang/crates.io-index"
24
+ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
25
+
26
+ [[package]]
27
+ name = "bindgen"
28
+ version = "0.69.5"
29
+ source = "registry+https://github.com/rust-lang/crates.io-index"
30
+ checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
31
+ dependencies = [
32
+ "bitflags",
33
+ "cexpr",
34
+ "clang-sys",
35
+ "itertools",
36
+ "lazy_static",
37
+ "lazycell",
38
+ "proc-macro2",
39
+ "quote",
40
+ "regex",
41
+ "rustc-hash",
42
+ "shlex",
43
+ "syn",
44
+ ]
45
+
46
+ [[package]]
47
+ name = "bit-set"
48
+ version = "0.5.3"
49
+ source = "registry+https://github.com/rust-lang/crates.io-index"
50
+ checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
51
+ dependencies = [
52
+ "bit-vec",
53
+ ]
54
+
55
+ [[package]]
56
+ name = "bit-vec"
57
+ version = "0.6.3"
58
+ source = "registry+https://github.com/rust-lang/crates.io-index"
59
+ checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
60
+
61
+ [[package]]
62
+ name = "bitflags"
63
+ version = "2.9.1"
64
+ source = "registry+https://github.com/rust-lang/crates.io-index"
65
+ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
66
+
67
+ [[package]]
68
+ name = "bstr"
69
+ version = "1.12.0"
70
+ source = "registry+https://github.com/rust-lang/crates.io-index"
71
+ checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
72
+ dependencies = [
73
+ "memchr",
74
+ "regex-automata",
75
+ "serde",
76
+ ]
77
+
78
+ [[package]]
79
+ name = "cexpr"
80
+ version = "0.6.0"
81
+ source = "registry+https://github.com/rust-lang/crates.io-index"
82
+ checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
83
+ dependencies = [
84
+ "nom",
85
+ ]
86
+
87
+ [[package]]
88
+ name = "cfg-if"
89
+ version = "1.0.1"
90
+ source = "registry+https://github.com/rust-lang/crates.io-index"
91
+ checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
92
+
93
+ [[package]]
94
+ name = "clang-sys"
95
+ version = "1.8.1"
96
+ source = "registry+https://github.com/rust-lang/crates.io-index"
97
+ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
98
+ dependencies = [
99
+ "glob",
100
+ "libc",
101
+ "libloading",
102
+ ]
103
+
104
+ [[package]]
105
+ name = "either"
106
+ version = "1.15.0"
107
+ source = "registry+https://github.com/rust-lang/crates.io-index"
108
+ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
109
+
110
+ [[package]]
111
+ name = "fancy-regex"
112
+ version = "0.13.0"
113
+ source = "registry+https://github.com/rust-lang/crates.io-index"
114
+ checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
115
+ dependencies = [
116
+ "bit-set",
117
+ "regex-automata",
118
+ "regex-syntax",
119
+ ]
120
+
121
+ [[package]]
122
+ name = "glob"
123
+ version = "0.3.2"
124
+ source = "registry+https://github.com/rust-lang/crates.io-index"
125
+ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
126
+
127
+ [[package]]
128
+ name = "itertools"
129
+ version = "0.12.1"
130
+ source = "registry+https://github.com/rust-lang/crates.io-index"
131
+ checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
132
+ dependencies = [
133
+ "either",
134
+ ]
135
+
136
+ [[package]]
137
+ name = "lazy_static"
138
+ version = "1.5.0"
139
+ source = "registry+https://github.com/rust-lang/crates.io-index"
140
+ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
141
+
142
+ [[package]]
143
+ name = "lazycell"
144
+ version = "1.3.0"
145
+ source = "registry+https://github.com/rust-lang/crates.io-index"
146
+ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
147
+
148
+ [[package]]
149
+ name = "libc"
150
+ version = "0.2.174"
151
+ source = "registry+https://github.com/rust-lang/crates.io-index"
152
+ checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
153
+
154
+ [[package]]
155
+ name = "libloading"
156
+ version = "0.8.8"
157
+ source = "registry+https://github.com/rust-lang/crates.io-index"
158
+ checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
159
+ dependencies = [
160
+ "cfg-if",
161
+ "windows-targets",
162
+ ]
163
+
164
+ [[package]]
165
+ name = "magnus"
166
+ version = "0.6.4"
167
+ source = "registry+https://github.com/rust-lang/crates.io-index"
168
+ checksum = "b1597ef40aa8c36be098249e82c9a20cf7199278ac1c1a1a995eeead6a184479"
169
+ dependencies = [
170
+ "magnus-macros",
171
+ "rb-sys",
172
+ "rb-sys-env",
173
+ "seq-macro",
174
+ ]
175
+
176
+ [[package]]
177
+ name = "magnus-macros"
178
+ version = "0.6.0"
179
+ source = "registry+https://github.com/rust-lang/crates.io-index"
180
+ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
181
+ dependencies = [
182
+ "proc-macro2",
183
+ "quote",
184
+ "syn",
185
+ ]
186
+
187
+ [[package]]
188
+ name = "memchr"
189
+ version = "2.7.5"
190
+ source = "registry+https://github.com/rust-lang/crates.io-index"
191
+ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
192
+
193
+ [[package]]
194
+ name = "minimal-lexical"
195
+ version = "0.2.1"
196
+ source = "registry+https://github.com/rust-lang/crates.io-index"
197
+ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
198
+
199
+ [[package]]
200
+ name = "nom"
201
+ version = "7.1.3"
202
+ source = "registry+https://github.com/rust-lang/crates.io-index"
203
+ checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
204
+ dependencies = [
205
+ "memchr",
206
+ "minimal-lexical",
207
+ ]
208
+
209
+ [[package]]
210
+ name = "proc-macro2"
211
+ version = "1.0.95"
212
+ source = "registry+https://github.com/rust-lang/crates.io-index"
213
+ checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
214
+ dependencies = [
215
+ "unicode-ident",
216
+ ]
217
+
218
+ [[package]]
219
+ name = "quote"
220
+ version = "1.0.40"
221
+ source = "registry+https://github.com/rust-lang/crates.io-index"
222
+ checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
223
+ dependencies = [
224
+ "proc-macro2",
225
+ ]
226
+
227
+ [[package]]
228
+ name = "rb-sys"
229
+ version = "0.9.116"
230
+ source = "registry+https://github.com/rust-lang/crates.io-index"
231
+ checksum = "7059846f68396df83155779c75336ca24567741cb95256e6308c9fcc370e8dad"
232
+ dependencies = [
233
+ "rb-sys-build",
234
+ ]
235
+
236
+ [[package]]
237
+ name = "rb-sys-build"
238
+ version = "0.9.116"
239
+ source = "registry+https://github.com/rust-lang/crates.io-index"
240
+ checksum = "ac217510df41b9ffc041573e68d7a02aaff770c49943c7494441c4b224b0ecd0"
241
+ dependencies = [
242
+ "bindgen",
243
+ "lazy_static",
244
+ "proc-macro2",
245
+ "quote",
246
+ "regex",
247
+ "shell-words",
248
+ "syn",
249
+ ]
250
+
251
+ [[package]]
252
+ name = "rb-sys-env"
253
+ version = "0.1.2"
254
+ source = "registry+https://github.com/rust-lang/crates.io-index"
255
+ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
256
+
257
+ [[package]]
258
+ name = "regex"
259
+ version = "1.11.1"
260
+ source = "registry+https://github.com/rust-lang/crates.io-index"
261
+ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
262
+ dependencies = [
263
+ "aho-corasick",
264
+ "memchr",
265
+ "regex-automata",
266
+ "regex-syntax",
267
+ ]
268
+
269
+ [[package]]
270
+ name = "regex-automata"
271
+ version = "0.4.9"
272
+ source = "registry+https://github.com/rust-lang/crates.io-index"
273
+ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
274
+ dependencies = [
275
+ "aho-corasick",
276
+ "memchr",
277
+ "regex-syntax",
278
+ ]
279
+
280
+ [[package]]
281
+ name = "regex-syntax"
282
+ version = "0.8.5"
283
+ source = "registry+https://github.com/rust-lang/crates.io-index"
284
+ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
285
+
286
+ [[package]]
287
+ name = "ru_token"
288
+ version = "0.1.0"
289
+ dependencies = [
290
+ "magnus",
291
+ "tiktoken-rs",
292
+ ]
293
+
294
+ [[package]]
295
+ name = "rustc-hash"
296
+ version = "1.1.0"
297
+ source = "registry+https://github.com/rust-lang/crates.io-index"
298
+ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
299
+
300
+ [[package]]
301
+ name = "seq-macro"
302
+ version = "0.3.6"
303
+ source = "registry+https://github.com/rust-lang/crates.io-index"
304
+ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
305
+
306
+ [[package]]
307
+ name = "serde"
308
+ version = "1.0.219"
309
+ source = "registry+https://github.com/rust-lang/crates.io-index"
310
+ checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
311
+ dependencies = [
312
+ "serde_derive",
313
+ ]
314
+
315
+ [[package]]
316
+ name = "serde_derive"
317
+ version = "1.0.219"
318
+ source = "registry+https://github.com/rust-lang/crates.io-index"
319
+ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
320
+ dependencies = [
321
+ "proc-macro2",
322
+ "quote",
323
+ "syn",
324
+ ]
325
+
326
+ [[package]]
327
+ name = "shell-words"
328
+ version = "1.1.0"
329
+ source = "registry+https://github.com/rust-lang/crates.io-index"
330
+ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
331
+
332
+ [[package]]
333
+ name = "shlex"
334
+ version = "1.3.0"
335
+ source = "registry+https://github.com/rust-lang/crates.io-index"
336
+ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
337
+
338
+ [[package]]
339
+ name = "syn"
340
+ version = "2.0.104"
341
+ source = "registry+https://github.com/rust-lang/crates.io-index"
342
+ checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
343
+ dependencies = [
344
+ "proc-macro2",
345
+ "quote",
346
+ "unicode-ident",
347
+ ]
348
+
349
+ [[package]]
350
+ name = "tiktoken-rs"
351
+ version = "0.7.0"
352
+ source = "registry+https://github.com/rust-lang/crates.io-index"
353
+ checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
354
+ dependencies = [
355
+ "anyhow",
356
+ "base64",
357
+ "bstr",
358
+ "fancy-regex",
359
+ "lazy_static",
360
+ "regex",
361
+ "rustc-hash",
362
+ ]
363
+
364
+ [[package]]
365
+ name = "unicode-ident"
366
+ version = "1.0.18"
367
+ source = "registry+https://github.com/rust-lang/crates.io-index"
368
+ checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
369
+
370
+ [[package]]
371
+ name = "windows-link"
372
+ version = "0.1.3"
373
+ source = "registry+https://github.com/rust-lang/crates.io-index"
374
+ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
375
+
376
+ [[package]]
377
+ name = "windows-targets"
378
+ version = "0.53.3"
379
+ source = "registry+https://github.com/rust-lang/crates.io-index"
380
+ checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
381
+ dependencies = [
382
+ "windows-link",
383
+ "windows_aarch64_gnullvm",
384
+ "windows_aarch64_msvc",
385
+ "windows_i686_gnu",
386
+ "windows_i686_gnullvm",
387
+ "windows_i686_msvc",
388
+ "windows_x86_64_gnu",
389
+ "windows_x86_64_gnullvm",
390
+ "windows_x86_64_msvc",
391
+ ]
392
+
393
+ [[package]]
394
+ name = "windows_aarch64_gnullvm"
395
+ version = "0.53.0"
396
+ source = "registry+https://github.com/rust-lang/crates.io-index"
397
+ checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
398
+
399
+ [[package]]
400
+ name = "windows_aarch64_msvc"
401
+ version = "0.53.0"
402
+ source = "registry+https://github.com/rust-lang/crates.io-index"
403
+ checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
404
+
405
+ [[package]]
406
+ name = "windows_i686_gnu"
407
+ version = "0.53.0"
408
+ source = "registry+https://github.com/rust-lang/crates.io-index"
409
+ checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
410
+
411
+ [[package]]
412
+ name = "windows_i686_gnullvm"
413
+ version = "0.53.0"
414
+ source = "registry+https://github.com/rust-lang/crates.io-index"
415
+ checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
416
+
417
+ [[package]]
418
+ name = "windows_i686_msvc"
419
+ version = "0.53.0"
420
+ source = "registry+https://github.com/rust-lang/crates.io-index"
421
+ checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
422
+
423
+ [[package]]
424
+ name = "windows_x86_64_gnu"
425
+ version = "0.53.0"
426
+ source = "registry+https://github.com/rust-lang/crates.io-index"
427
+ checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
428
+
429
+ [[package]]
430
+ name = "windows_x86_64_gnullvm"
431
+ version = "0.53.0"
432
+ source = "registry+https://github.com/rust-lang/crates.io-index"
433
+ checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
434
+
435
+ [[package]]
436
+ name = "windows_x86_64_msvc"
437
+ version = "0.53.0"
438
+ source = "registry+https://github.com/rust-lang/crates.io-index"
439
+ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
data/Cargo.toml ADDED
@@ -0,0 +1,7 @@
1
+ # This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
2
+ # a Rust project. Your extensions dependencies should be added to the Cargo.toml
3
+ # in the ext/ directory.
4
+
5
+ [workspace]
6
+ members = ["./ext/ru_token"]
7
+ resolver = "2"