tiktoken_ruby 0.0.14.1-x86_64-linux → 0.0.16-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +31 -0
  3. data/Gemfile.lock +22 -21
  4. data/README.md +1 -1
  5. data/lib/tiktoken_ruby/3.2/tiktoken_ruby.so +0 -0
  6. data/lib/tiktoken_ruby/3.3/tiktoken_ruby.so +0 -0
  7. data/lib/tiktoken_ruby/3.4/tiktoken_ruby.so +0 -0
  8. data/lib/tiktoken_ruby/4.0/tiktoken_ruby.so +0 -0
  9. data/lib/tiktoken_ruby/version.rb +1 -1
  10. data/lib/tiktoken_ruby.rb +3 -0
  11. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8bc9e92f57d58214f292fc24ef4f9f48980cbfb047853b69e385d7224b53b4a3
4
- data.tar.gz: 976e2a3cbc94f153d90a038e35d0272fe30c1efd1c58c86c5c4210cfdfde9532
3
+ metadata.gz: bc5115a618816ae53880f705b4099787528e9d9cb4f76894708bafe99841ff35
4
+ data.tar.gz: b273f795813c118edea38f53c60629d27fa5bd90b685a2a18fef4d7c02285636
5
5
  SHA512:
6
- metadata.gz: f72a3f4d8c0b3f3c26acbe9b050feeb82910d68411162a4e94579b456906affd6a07472120edab2a227e124c0d19c5608180eaccc129708b77918b80654dcbd9
7
- data.tar.gz: f94bafff68f9f50ae11589c62fd7aa6899836bf0ba43d070d4a7268f366392ab4736731283aae14d977ab3c53aee80eb60992c85ea81596f7cc3a36bf8c020c4
6
+ metadata.gz: 4e889ff78e4af75b578ae530223b988020fed28ff6a49ad49d6151e60bac013bd247a0664fa5375d799db9b0c4226d365240c945a3b83ffd6fb4634138644e2a
7
+ data.tar.gz: dc25d92bfcb4b3e989d05f7e6498fe25a5128a20569709d5fb710f8c9d087c4154b3e94cb2de654f2df886558c9acb92ba6547207429880c323c2b2b4a4e4cdf
data/CHANGELOG.md CHANGED
@@ -1,3 +1,34 @@
1
+ # [v0.0.16] - 02-05-2026
2
+ ## What's Changed
3
+ * Bump rb-sys from 0.9.123 to 0.9.124 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/107
4
+ * Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/108
5
+ * Bump standard from 1.53.0 to 1.54.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/109
6
+ * Bump the cargo group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/113
7
+ * Bump the bundler-dependencies group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/114
8
+ * bump version to 0.0.16 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/115
9
+
10
+
11
+ **Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.15.1...v0.0.16
12
+ # [v0.0.15.1] - 07-01-2026
13
+ ## What's Changed
14
+ * Re-release 0.0.15 for mistakenly yanked gems by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/105
15
+
16
+
17
+ **Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.15...v0.0.15.1
18
+ # [v0.0.15] - 06-01-2026
19
+ ## What's Changed
20
+ * No longer true by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/96
21
+ * Test build on version change by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/97
22
+ * Bump actions/cache from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/98
23
+ * Bump rb-sys from 0.9.117 to 0.9.123 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/99
24
+ * Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/100
25
+ * Add Ruby 4.0 to build matrix by @chubchenko in https://github.com/IAPark/tiktoken_ruby/pull/101
26
+ * Support Ruby 4 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/102
27
+
28
+ ## New Contributors
29
+ * @chubchenko made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/101
30
+
31
+ **Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.14.1...v0.0.15
1
32
  # [v0.0.14.1] - 20-12-2025
2
33
  ## What's Changed
3
34
  * Cut v0.0.12 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/78
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.14.1)
4
+ tiktoken_ruby (0.0.16)
5
5
  rb_sys (~> 0.9)
6
6
 
7
7
  GEM
@@ -9,23 +9,24 @@ GEM
9
9
  specs:
10
10
  ast (2.4.3)
11
11
  diff-lcs (1.6.2)
12
- json (2.16.0)
12
+ json (2.18.1)
13
13
  language_server-protocol (3.17.0.5)
14
14
  lint_roller (1.1.0)
15
- minitest (5.25.5)
15
+ minitest (6.0.1)
16
+ prism (~> 1.5)
16
17
  parallel (1.27.0)
17
- parser (3.3.10.0)
18
+ parser (3.3.10.2)
18
19
  ast (~> 2.4.1)
19
20
  racc
20
- prism (1.6.0)
21
+ prism (1.9.0)
21
22
  racc (1.8.1)
22
23
  rainbow (3.1.1)
23
- rake (13.3.1)
24
- rake-compiler (1.3.0)
24
+ rake (13.4.2)
25
+ rake-compiler (1.3.1)
25
26
  rake
26
- rake-compiler-dock (1.9.1)
27
- rb_sys (0.9.117)
28
- rake-compiler-dock (= 1.9.1)
27
+ rake-compiler-dock (1.12.0)
28
+ rb_sys (0.9.127)
29
+ rake-compiler-dock (= 1.12.0)
29
30
  regexp_parser (2.11.3)
30
31
  rspec (3.13.2)
31
32
  rspec-core (~> 3.13.0)
@@ -40,7 +41,7 @@ GEM
40
41
  diff-lcs (>= 1.2.0, < 2.0)
41
42
  rspec-support (~> 3.13.0)
42
43
  rspec-support (3.13.6)
43
- rubocop (1.81.7)
44
+ rubocop (1.84.2)
44
45
  json (~> 2.3)
45
46
  language_server-protocol (~> 3.17.0.2)
46
47
  lint_roller (~> 1.1.0)
@@ -48,32 +49,32 @@ GEM
48
49
  parser (>= 3.3.0.2)
49
50
  rainbow (>= 2.2.2, < 4.0)
50
51
  regexp_parser (>= 2.9.3, < 3.0)
51
- rubocop-ast (>= 1.47.1, < 2.0)
52
+ rubocop-ast (>= 1.49.0, < 2.0)
52
53
  ruby-progressbar (~> 1.7)
53
54
  unicode-display_width (>= 2.4.0, < 4.0)
54
- rubocop-ast (1.48.0)
55
+ rubocop-ast (1.49.0)
55
56
  parser (>= 3.3.7.2)
56
- prism (~> 1.4)
57
- rubocop-performance (1.25.0)
57
+ prism (~> 1.7)
58
+ rubocop-performance (1.26.1)
58
59
  lint_roller (~> 1.1)
59
60
  rubocop (>= 1.75.0, < 2.0)
60
- rubocop-ast (>= 1.38.0, < 2.0)
61
+ rubocop-ast (>= 1.47.1, < 2.0)
61
62
  ruby-progressbar (1.13.0)
62
- standard (1.52.0)
63
+ standard (1.54.0)
63
64
  language_server-protocol (~> 3.17.0.2)
64
65
  lint_roller (~> 1.0)
65
- rubocop (~> 1.81.7)
66
+ rubocop (~> 1.84.0)
66
67
  standard-custom (~> 1.0.0)
67
68
  standard-performance (~> 1.8)
68
69
  standard-custom (1.0.2)
69
70
  lint_roller (~> 1.0)
70
71
  rubocop (~> 1.50)
71
- standard-performance (1.8.0)
72
+ standard-performance (1.9.0)
72
73
  lint_roller (~> 1.1)
73
- rubocop-performance (~> 1.25.0)
74
+ rubocop-performance (~> 1.26.0)
74
75
  unicode-display_width (3.2.0)
75
76
  unicode-emoji (~> 4.1)
76
- unicode-emoji (4.1.0)
77
+ unicode-emoji (4.2.0)
77
78
  yard (0.9.37)
78
79
  yard-doctest (0.1.17)
79
80
  minitest
data/README.md CHANGED
@@ -73,7 +73,7 @@ All methods round-trip correctly through `decode`.
73
73
 
74
74
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
75
75
 
76
- To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
76
+ To install this gem onto your local machine, run `bundle exec rake install`.
77
77
 
78
78
  ## Contributing
79
79
 
data/lib/tiktoken_ruby/3.2/tiktoken_ruby.so CHANGED
Binary file
data/lib/tiktoken_ruby/3.3/tiktoken_ruby.so CHANGED
Binary file
data/lib/tiktoken_ruby/3.4/tiktoken_ruby.so CHANGED
Binary file
data/lib/tiktoken_ruby/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.14.1"
4
+ VERSION = "0.0.16"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -79,6 +79,7 @@ module Tiktoken
79
79
  o3: "o200k_base",
80
80
  "o4-mini": "o200k_base",
81
81
  # chat
82
+ "gpt-5": "o200k_base",
82
83
  "gpt-4.1": "o200k_base",
83
84
  "chatgpt-4o": "o200k_base",
84
85
  "gpt-4o": "o200k_base",
@@ -136,7 +137,9 @@ module Tiktoken
136
137
  "o3-": "o200k_base",
137
138
  "o4-": "o200k_base",
138
139
  # chat
140
+ "gpt-5.": "o200k_base", # e.g., gpt-5.4, gpt-5.4-mini, gpt-5.3-codex, gpt-5.2-pro
139
141
  "gpt-5-": "o200k_base",
142
+ "codex-mini": "o200k_base", # e.g., codex-mini, codex-mini-latest
140
143
  "gpt-4.5-": "o200k_base",
141
144
  "gpt-4.1-": "o200k_base",
142
145
  "chatgpt-4o-": "o200k_base",
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.14.1
4
+ version: 0.0.16
5
5
  platform: x86_64-linux
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2025-12-20 00:00:00.000000000 Z
11
+ date: 2026-05-02 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -33,6 +33,7 @@ files:
33
33
  - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
34
34
  - lib/tiktoken_ruby/3.3/tiktoken_ruby.so
35
35
  - lib/tiktoken_ruby/3.4/tiktoken_ruby.so
36
+ - lib/tiktoken_ruby/4.0/tiktoken_ruby.so
36
37
  - lib/tiktoken_ruby/encoding.rb
37
38
  - lib/tiktoken_ruby/version.rb
38
39
  - sig/tiktoken_ruby.rbs
@@ -54,7 +55,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
54
55
  version: '3.2'
55
56
  - - "<"
56
57
  - !ruby/object:Gem::Version
57
- version: 3.5.dev
58
+ version: 4.1.dev
58
59
  required_rubygems_version: !ruby/object:Gem::Requirement
59
60
  requirements:
60
61
  - - ">="