tiktoken_ruby 0.0.12-x86_64-darwin → 0.0.13-x86_64-darwin

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +18 -18
  3. data/lib/tiktoken_ruby/3.2/tiktoken_ruby.bundle +0 -0
  4. data/lib/tiktoken_ruby/3.3/tiktoken_ruby.bundle +0 -0
  5. data/lib/tiktoken_ruby/3.4/tiktoken_ruby.bundle +0 -0
  6. data/lib/tiktoken_ruby/version.rb +1 -1
  7. data/lib/tiktoken_ruby.rb +3 -1
  8. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9ce50dcb7cb1075e7a95ad99e31f429817d11afb3293a04f58d59a09c2d57719
4
- data.tar.gz: 36814c13c6679b793978885f65ee71e899c7b774c7f1f9c4d8a2d9f3c6e0cabd
3
+ metadata.gz: 2b80cc204881949a179e98b0bb39a71d0394c242fea04d95ea01a8c52ef0c4b8
4
+ data.tar.gz: f79cb0cbbc212b53491cb057b699bc84c6065ea5e110539e5be3f95bae9667fb
5
5
  SHA512:
6
- metadata.gz: fb863e6ca5caf9b575a0e07709b0d0d97e9360cca45c75424e31ed8f262b6e9378c4c13bbaf5b110f8b21a6023990478895a6a42fa6a4fbe9905f4d55008fcf4
7
- data.tar.gz: 07b8de2940b76c526708f187854d3b5a70365357e9ba32394e40fb3912a50d11e8732178b1c94ddf3beeeedaa28fd5b224d97678a6977e0f7d75a01e44cd2a11
6
+ metadata.gz: 80433cd9df5f9f2c60c73e8a094d3f4b1e794a71021d6fa155821a62642887699cdeb193b5856092feeaff0a4e76964f2cb27ff8ab4797c899f9208caad057a0
7
+ data.tar.gz: '091670d0eaebd26830ae3f37d48ca5d80df6eccb52f76118672edcc279d1d2c5d4d477f9ae772c1631d285413695c4770739ba064b07f7c543070e0a0dd6adef'
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.12)
4
+ tiktoken_ruby (0.0.13)
5
5
  rb_sys (~> 0.9)
6
6
 
7
7
  GEM
@@ -9,38 +9,38 @@ GEM
9
9
  specs:
10
10
  ast (2.4.3)
11
11
  diff-lcs (1.6.2)
12
- json (2.12.0)
12
+ json (2.15.0)
13
13
  language_server-protocol (3.17.0.5)
14
14
  lint_roller (1.1.0)
15
15
  minitest (5.25.5)
16
16
  parallel (1.27.0)
17
- parser (3.3.8.0)
17
+ parser (3.3.9.0)
18
18
  ast (~> 2.4.1)
19
19
  racc
20
- prism (1.4.0)
20
+ prism (1.5.1)
21
21
  racc (1.8.1)
22
22
  rainbow (3.1.1)
23
- rake (13.3.0)
23
+ rake (13.3.1)
24
24
  rake-compiler (1.3.0)
25
25
  rake
26
26
  rake-compiler-dock (1.9.1)
27
27
  rb_sys (0.9.117)
28
28
  rake-compiler-dock (= 1.9.1)
29
- regexp_parser (2.10.0)
30
- rspec (3.13.1)
29
+ regexp_parser (2.11.3)
30
+ rspec (3.13.2)
31
31
  rspec-core (~> 3.13.0)
32
32
  rspec-expectations (~> 3.13.0)
33
33
  rspec-mocks (~> 3.13.0)
34
- rspec-core (3.13.5)
34
+ rspec-core (3.13.6)
35
35
  rspec-support (~> 3.13.0)
36
36
  rspec-expectations (3.13.5)
37
37
  diff-lcs (>= 1.2.0, < 2.0)
38
38
  rspec-support (~> 3.13.0)
39
- rspec-mocks (3.13.5)
39
+ rspec-mocks (3.13.7)
40
40
  diff-lcs (>= 1.2.0, < 2.0)
41
41
  rspec-support (~> 3.13.0)
42
- rspec-support (3.13.5)
43
- rubocop (1.75.7)
42
+ rspec-support (3.13.6)
43
+ rubocop (1.80.2)
44
44
  json (~> 2.3)
45
45
  language_server-protocol (~> 3.17.0.2)
46
46
  lint_roller (~> 1.1.0)
@@ -48,10 +48,10 @@ GEM
48
48
  parser (>= 3.3.0.2)
49
49
  rainbow (>= 2.2.2, < 4.0)
50
50
  regexp_parser (>= 2.9.3, < 3.0)
51
- rubocop-ast (>= 1.44.0, < 2.0)
51
+ rubocop-ast (>= 1.46.0, < 2.0)
52
52
  ruby-progressbar (~> 1.7)
53
53
  unicode-display_width (>= 2.4.0, < 4.0)
54
- rubocop-ast (1.44.1)
54
+ rubocop-ast (1.47.1)
55
55
  parser (>= 3.3.7.2)
56
56
  prism (~> 1.4)
57
57
  rubocop-performance (1.25.0)
@@ -59,10 +59,10 @@ GEM
59
59
  rubocop (>= 1.75.0, < 2.0)
60
60
  rubocop-ast (>= 1.38.0, < 2.0)
61
61
  ruby-progressbar (1.13.0)
62
- standard (1.50.0)
62
+ standard (1.51.1)
63
63
  language_server-protocol (~> 3.17.0.2)
64
64
  lint_roller (~> 1.0)
65
- rubocop (~> 1.75.5)
65
+ rubocop (~> 1.80.2)
66
66
  standard-custom (~> 1.0.0)
67
67
  standard-performance (~> 1.8)
68
68
  standard-custom (1.0.2)
@@ -71,9 +71,9 @@ GEM
71
71
  standard-performance (1.8.0)
72
72
  lint_roller (~> 1.1)
73
73
  rubocop-performance (~> 1.25.0)
74
- unicode-display_width (3.1.4)
75
- unicode-emoji (~> 4.0, >= 4.0.4)
76
- unicode-emoji (4.0.4)
74
+ unicode-display_width (3.2.0)
75
+ unicode-emoji (~> 4.1)
76
+ unicode-emoji (4.1.0)
77
77
  yard (0.9.37)
78
78
  yard-doctest (0.1.17)
79
79
  minitest
data/lib/tiktoken_ruby/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.12"
4
+ VERSION = "0.0.13"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -65,7 +65,8 @@ module Tiktoken
65
65
  :p50k_base,
66
66
  :p50k_edit,
67
67
  :cl100k_base,
68
- :o200k_base
68
+ :o200k_base,
69
+ :o200k_harmony
69
70
  ]
70
71
 
71
72
  # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
@@ -143,6 +144,7 @@ module Tiktoken
143
144
  "gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k
144
145
  "gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc.
145
146
  "gpt-35-turbo-": "cl100k_base", # Azure deployment name
147
+ "gpt-oss-": "o200k_harmony",
146
148
  # fine-tuned
147
149
  "ft:gpt-4o": "cl100k_base",
148
150
  "ft:gpt-4": "cl100k_base",
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.12
4
+ version: 0.0.13
5
5
  platform: x86_64-darwin
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2025-08-26 00:00:00.000000000 Z
11
+ date: 2025-11-10 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending