tiktoken_ruby 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/Cargo.lock +26 -28
- data/Gemfile.lock +18 -18
- data/ext/tiktoken_ruby/Cargo.toml +2 -2
- data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs +1 -1
- data/lib/tiktoken_ruby/version.rb +1 -1
- data/lib/tiktoken_ruby.rb +3 -0
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2d1b276821555850dac79a65fddb5152db1b0dfc7021cfa13e1b2435822eb6d1
|
|
4
|
+
data.tar.gz: 5c72d2286f0bcb53d128dbb1b52ad91e7c593c91730d98300ee44d21a8c5e737
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f586ac763017ffde2c483d703ebc3fd65d1029238e0a8c2cfac0cb4e4bd8f2f03c7eb6156ccc0bcb8b4c72108d7f8c83e227f866205fa3771b14861116ade096
|
|
7
|
+
data.tar.gz: 56ceeaf795a1b0817b5cde8641369e3963b5d613870069c1a62d17732922fc3a41edd66b02c725f3a373bf3697bcf1de7d9e2ed2bcebd70452942a6c102fd55c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
# [v0.0.16] - 02-05-2026
|
|
2
|
+
## What's Changed
|
|
3
|
+
* Bump rb-sys from 0.9.123 to 0.9.124 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/107
|
|
4
|
+
* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/108
|
|
5
|
+
* Bump standard from 1.53.0 to 1.54.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/109
|
|
6
|
+
* Bump the cargo group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/113
|
|
7
|
+
* Bump the bundler-dependencies group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/114
|
|
8
|
+
* bump version to 0.0.16 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/115
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.15.1...v0.0.16
|
|
12
|
+
# [v0.0.15.1] - 07-01-2026
|
|
13
|
+
## What's Changed
|
|
14
|
+
* Re-release 0.0.15 for mistakenly yanked gems by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/105
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.15...v0.0.15.1
|
|
1
18
|
# [v0.0.15] - 06-01-2026
|
|
2
19
|
## What's Changed
|
|
3
20
|
* No longer true by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/96
|
data/Cargo.lock
CHANGED
|
@@ -13,9 +13,9 @@ dependencies = [
|
|
|
13
13
|
|
|
14
14
|
[[package]]
|
|
15
15
|
name = "anyhow"
|
|
16
|
-
version = "1.0.
|
|
16
|
+
version = "1.0.102"
|
|
17
17
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
-
checksum = "
|
|
18
|
+
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
|
19
19
|
|
|
20
20
|
[[package]]
|
|
21
21
|
name = "base64"
|
|
@@ -25,38 +25,36 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
|
|
25
25
|
|
|
26
26
|
[[package]]
|
|
27
27
|
name = "bindgen"
|
|
28
|
-
version = "0.
|
|
28
|
+
version = "0.72.1"
|
|
29
29
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
|
-
checksum = "
|
|
30
|
+
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
|
|
31
31
|
dependencies = [
|
|
32
32
|
"bitflags",
|
|
33
33
|
"cexpr",
|
|
34
34
|
"clang-sys",
|
|
35
35
|
"itertools",
|
|
36
|
-
"lazy_static",
|
|
37
|
-
"lazycell",
|
|
38
36
|
"proc-macro2",
|
|
39
37
|
"quote",
|
|
40
38
|
"regex",
|
|
41
|
-
"rustc-hash",
|
|
39
|
+
"rustc-hash 2.1.2",
|
|
42
40
|
"shlex",
|
|
43
41
|
"syn",
|
|
44
42
|
]
|
|
45
43
|
|
|
46
44
|
[[package]]
|
|
47
45
|
name = "bit-set"
|
|
48
|
-
version = "0.
|
|
46
|
+
version = "0.8.0"
|
|
49
47
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
50
|
-
checksum = "
|
|
48
|
+
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
|
|
51
49
|
dependencies = [
|
|
52
50
|
"bit-vec",
|
|
53
51
|
]
|
|
54
52
|
|
|
55
53
|
[[package]]
|
|
56
54
|
name = "bit-vec"
|
|
57
|
-
version = "0.
|
|
55
|
+
version = "0.8.0"
|
|
58
56
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
59
|
-
checksum = "
|
|
57
|
+
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
|
60
58
|
|
|
61
59
|
[[package]]
|
|
62
60
|
name = "bitflags"
|
|
@@ -109,9 +107,9 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
|
|
109
107
|
|
|
110
108
|
[[package]]
|
|
111
109
|
name = "fancy-regex"
|
|
112
|
-
version = "0.
|
|
110
|
+
version = "0.17.0"
|
|
113
111
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
114
|
-
checksum = "
|
|
112
|
+
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
|
|
115
113
|
dependencies = [
|
|
116
114
|
"bit-set",
|
|
117
115
|
"regex-automata",
|
|
@@ -139,12 +137,6 @@ version = "1.5.0"
|
|
|
139
137
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
140
138
|
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
|
141
139
|
|
|
142
|
-
[[package]]
|
|
143
|
-
name = "lazycell"
|
|
144
|
-
version = "1.3.0"
|
|
145
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
146
|
-
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|
147
|
-
|
|
148
140
|
[[package]]
|
|
149
141
|
name = "libc"
|
|
150
142
|
version = "0.2.177"
|
|
@@ -226,18 +218,18 @@ dependencies = [
|
|
|
226
218
|
|
|
227
219
|
[[package]]
|
|
228
220
|
name = "rb-sys"
|
|
229
|
-
version = "0.9.
|
|
221
|
+
version = "0.9.127"
|
|
230
222
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
231
|
-
checksum = "
|
|
223
|
+
checksum = "d7d7c9560fe42dcffa576941394075f18a17dce89fcf718a2fa90b7dc2134d12"
|
|
232
224
|
dependencies = [
|
|
233
225
|
"rb-sys-build",
|
|
234
226
|
]
|
|
235
227
|
|
|
236
228
|
[[package]]
|
|
237
229
|
name = "rb-sys-build"
|
|
238
|
-
version = "0.9.
|
|
230
|
+
version = "0.9.127"
|
|
239
231
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
240
|
-
checksum = "
|
|
232
|
+
checksum = "f1688e8f32967ba48c89e4dfa283b57f901075f542fc7ee9c3d7c5f9091ca1d9"
|
|
241
233
|
dependencies = [
|
|
242
234
|
"bindgen",
|
|
243
235
|
"lazy_static",
|
|
@@ -256,9 +248,9 @@ checksum = "08f8d2924cf136a1315e2b4c7460a39f62ef11ee5d522df9b2750fab55b868b6"
|
|
|
256
248
|
|
|
257
249
|
[[package]]
|
|
258
250
|
name = "regex"
|
|
259
|
-
version = "1.12.
|
|
251
|
+
version = "1.12.3"
|
|
260
252
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
261
|
-
checksum = "
|
|
253
|
+
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
|
262
254
|
dependencies = [
|
|
263
255
|
"aho-corasick",
|
|
264
256
|
"memchr",
|
|
@@ -289,6 +281,12 @@ version = "1.1.0"
|
|
|
289
281
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
290
282
|
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
|
291
283
|
|
|
284
|
+
[[package]]
|
|
285
|
+
name = "rustc-hash"
|
|
286
|
+
version = "2.1.2"
|
|
287
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
288
|
+
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
|
289
|
+
|
|
292
290
|
[[package]]
|
|
293
291
|
name = "seq-macro"
|
|
294
292
|
version = "0.3.6"
|
|
@@ -349,9 +347,9 @@ dependencies = [
|
|
|
349
347
|
|
|
350
348
|
[[package]]
|
|
351
349
|
name = "tiktoken-rs"
|
|
352
|
-
version = "0.
|
|
350
|
+
version = "0.11.0"
|
|
353
351
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
354
|
-
checksum = "
|
|
352
|
+
checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"
|
|
355
353
|
dependencies = [
|
|
356
354
|
"anyhow",
|
|
357
355
|
"base64",
|
|
@@ -359,7 +357,7 @@ dependencies = [
|
|
|
359
357
|
"fancy-regex",
|
|
360
358
|
"lazy_static",
|
|
361
359
|
"regex",
|
|
362
|
-
"rustc-hash",
|
|
360
|
+
"rustc-hash 1.1.0",
|
|
363
361
|
]
|
|
364
362
|
|
|
365
363
|
[[package]]
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
tiktoken_ruby (0.0.
|
|
4
|
+
tiktoken_ruby (0.0.16)
|
|
5
5
|
rb_sys (~> 0.9)
|
|
6
6
|
|
|
7
7
|
GEM
|
|
@@ -9,24 +9,24 @@ GEM
|
|
|
9
9
|
specs:
|
|
10
10
|
ast (2.4.3)
|
|
11
11
|
diff-lcs (1.6.2)
|
|
12
|
-
json (2.
|
|
12
|
+
json (2.18.1)
|
|
13
13
|
language_server-protocol (3.17.0.5)
|
|
14
14
|
lint_roller (1.1.0)
|
|
15
15
|
minitest (6.0.1)
|
|
16
16
|
prism (~> 1.5)
|
|
17
17
|
parallel (1.27.0)
|
|
18
|
-
parser (3.3.10.
|
|
18
|
+
parser (3.3.10.2)
|
|
19
19
|
ast (~> 2.4.1)
|
|
20
20
|
racc
|
|
21
|
-
prism (1.
|
|
21
|
+
prism (1.9.0)
|
|
22
22
|
racc (1.8.1)
|
|
23
23
|
rainbow (3.1.1)
|
|
24
|
-
rake (13.
|
|
24
|
+
rake (13.4.2)
|
|
25
25
|
rake-compiler (1.3.1)
|
|
26
26
|
rake
|
|
27
|
-
rake-compiler-dock (1.
|
|
28
|
-
rb_sys (0.9.
|
|
29
|
-
rake-compiler-dock (= 1.
|
|
27
|
+
rake-compiler-dock (1.12.0)
|
|
28
|
+
rb_sys (0.9.127)
|
|
29
|
+
rake-compiler-dock (= 1.12.0)
|
|
30
30
|
regexp_parser (2.11.3)
|
|
31
31
|
rspec (3.13.2)
|
|
32
32
|
rspec-core (~> 3.13.0)
|
|
@@ -41,7 +41,7 @@ GEM
|
|
|
41
41
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
42
42
|
rspec-support (~> 3.13.0)
|
|
43
43
|
rspec-support (3.13.6)
|
|
44
|
-
rubocop (1.
|
|
44
|
+
rubocop (1.84.2)
|
|
45
45
|
json (~> 2.3)
|
|
46
46
|
language_server-protocol (~> 3.17.0.2)
|
|
47
47
|
lint_roller (~> 1.1.0)
|
|
@@ -49,29 +49,29 @@ GEM
|
|
|
49
49
|
parser (>= 3.3.0.2)
|
|
50
50
|
rainbow (>= 2.2.2, < 4.0)
|
|
51
51
|
regexp_parser (>= 2.9.3, < 3.0)
|
|
52
|
-
rubocop-ast (>= 1.
|
|
52
|
+
rubocop-ast (>= 1.49.0, < 2.0)
|
|
53
53
|
ruby-progressbar (~> 1.7)
|
|
54
54
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
55
|
-
rubocop-ast (1.
|
|
55
|
+
rubocop-ast (1.49.0)
|
|
56
56
|
parser (>= 3.3.7.2)
|
|
57
|
-
prism (~> 1.
|
|
58
|
-
rubocop-performance (1.
|
|
57
|
+
prism (~> 1.7)
|
|
58
|
+
rubocop-performance (1.26.1)
|
|
59
59
|
lint_roller (~> 1.1)
|
|
60
60
|
rubocop (>= 1.75.0, < 2.0)
|
|
61
|
-
rubocop-ast (>= 1.
|
|
61
|
+
rubocop-ast (>= 1.47.1, < 2.0)
|
|
62
62
|
ruby-progressbar (1.13.0)
|
|
63
|
-
standard (1.
|
|
63
|
+
standard (1.54.0)
|
|
64
64
|
language_server-protocol (~> 3.17.0.2)
|
|
65
65
|
lint_roller (~> 1.0)
|
|
66
|
-
rubocop (~> 1.
|
|
66
|
+
rubocop (~> 1.84.0)
|
|
67
67
|
standard-custom (~> 1.0.0)
|
|
68
68
|
standard-performance (~> 1.8)
|
|
69
69
|
standard-custom (1.0.2)
|
|
70
70
|
lint_roller (~> 1.0)
|
|
71
71
|
rubocop (~> 1.50)
|
|
72
|
-
standard-performance (1.
|
|
72
|
+
standard-performance (1.9.0)
|
|
73
73
|
lint_roller (~> 1.1)
|
|
74
|
-
rubocop-performance (~> 1.
|
|
74
|
+
rubocop-performance (~> 1.26.0)
|
|
75
75
|
unicode-display_width (3.2.0)
|
|
76
76
|
unicode-emoji (~> 4.1)
|
|
77
77
|
unicode-emoji (4.2.0)
|
|
@@ -11,5 +11,5 @@ crate-type = ["cdylib"]
|
|
|
11
11
|
|
|
12
12
|
[dependencies]
|
|
13
13
|
magnus = { version = "0.8.2" }
|
|
14
|
-
rb-sys = { version = "0.9.
|
|
15
|
-
tiktoken-rs = { version = "0.
|
|
14
|
+
rb-sys = { version = "0.9.127", features = ["stable-api-compiled-fallback"] }
|
|
15
|
+
tiktoken-rs = { version = "0.11.0" }
|
|
@@ -60,7 +60,7 @@ unsafe extern "C" fn encode_special_without_gvl(data: *mut c_void) -> *mut c_voi
|
|
|
60
60
|
unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
|
|
61
61
|
let data = &mut *(data as *mut DecodeData);
|
|
62
62
|
let core_bpe = &*data.core_bpe;
|
|
63
|
-
data.result = core_bpe.decode(
|
|
63
|
+
data.result = core_bpe.decode(&data.ids).map_err(|e| e.to_string());
|
|
64
64
|
std::ptr::null_mut()
|
|
65
65
|
}
|
|
66
66
|
|
data/lib/tiktoken_ruby.rb
CHANGED
|
@@ -79,6 +79,7 @@ module Tiktoken
|
|
|
79
79
|
o3: "o200k_base",
|
|
80
80
|
"o4-mini": "o200k_base",
|
|
81
81
|
# chat
|
|
82
|
+
"gpt-5": "o200k_base",
|
|
82
83
|
"gpt-4.1": "o200k_base",
|
|
83
84
|
"chatgpt-4o": "o200k_base",
|
|
84
85
|
"gpt-4o": "o200k_base",
|
|
@@ -136,7 +137,9 @@ module Tiktoken
|
|
|
136
137
|
"o3-": "o200k_base",
|
|
137
138
|
"o4-": "o200k_base",
|
|
138
139
|
# chat
|
|
140
|
+
"gpt-5.": "o200k_base", # e.g., gpt-5.4, gpt-5.4-mini, gpt-5.3-codex, gpt-5.2-pro
|
|
139
141
|
"gpt-5-": "o200k_base",
|
|
142
|
+
"codex-mini": "o200k_base", # e.g., codex-mini, codex-mini-latest
|
|
140
143
|
"gpt-4.5-": "o200k_base",
|
|
141
144
|
"gpt-4.1-": "o200k_base",
|
|
142
145
|
"chatgpt-4o-": "o200k_base",
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tiktoken_ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.16
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- IAPark
|
|
@@ -74,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
75
|
version: 3.4.0
|
|
76
76
|
requirements: []
|
|
77
|
-
rubygems_version: 4.0.
|
|
77
|
+
rubygems_version: 4.0.6
|
|
78
78
|
specification_version: 4
|
|
79
79
|
summary: Ruby wrapper for Tiktoken
|
|
80
80
|
test_files: []
|