tiktoken_ruby 0.0.8-x86_64-darwin → 0.0.9-x86_64-darwin

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +1 -1
  3. data/lib/tiktoken_ruby/3.1/tiktoken_ruby.bundle +0 -0
  4. data/lib/tiktoken_ruby/3.2/tiktoken_ruby.bundle +0 -0
  5. data/lib/tiktoken_ruby/3.3/tiktoken_ruby.bundle +0 -0
  6. data/lib/tiktoken_ruby/version.rb +1 -1
  7. data/lib/tiktoken_ruby.rb +4 -1
  8. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5952edc6d35682fd1f1e11e2e3b592a910cbf564267ec1d674ebdcac8197e6ab
4
- data.tar.gz: 456ccf90cd89bffdf2a8631a253b2bdb8ec97fe6bbd4b2d82b5198935a0b5bbb
3
+ metadata.gz: dfffd99df0ef60ddd07eabefa385c8f4e56ed814063933118d7043c06811e7f5
4
+ data.tar.gz: 40f71bfc9ffab7a3694f6f7761ba22ea33eb781a9c8ed10e335dbef7b41443ab
5
5
  SHA512:
6
- metadata.gz: 5aaa9e3debf8189797adaee23dbdaa1ca7455dea5e7cfb86b0b16dac07db2df6a3322c3541c2af88a1fc22eaf8a6dc9bf56a56db80a780ad91a82854a6e4b663
7
- data.tar.gz: 8c76c8e1220523539880a5b79d877aa3c2bd3d3ee51a0b6c6025502534cc9973c562b8aaeda291fa05376ef14cb81105449b690d06918b75839484bb811cbc8c
6
+ metadata.gz: 8013c6cd1f87ee18997962f56b2af713c207842224affaed762e5156c8c9ce37f6f01eef1fe07731cf763fe8c7cd820789968e4afb481be5b7876b73bf64c458
7
+ data.tar.gz: c8eff716ffefcfafb3fe3a456fe18709241d611ef66237aeb67c4187181e4d1b53b23cddf494127ad02b8f2d20b1ca512f97320135d2a005f76b279fdbdb17b4
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.8)
4
+ tiktoken_ruby (0.0.9)
5
5
  rb_sys (= 0.9.87)
6
6
 
7
7
  GEM
data/lib/tiktoken_ruby/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.8"
4
+ VERSION = "0.0.9"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -64,13 +64,15 @@ module Tiktoken
64
64
  :r50k_base,
65
65
  :p50k_base,
66
66
  :p50k_edit,
67
- :cl100k_base
67
+ :cl100k_base,
68
+ :o200k_base
68
69
  ]
69
70
 
70
71
  # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
71
72
  # that is also MIT licensed but by OpenAI
72
73
  MODEL_TO_ENCODING_NAME = {
73
74
  # chat
75
+ "gpt-4o": "o200k_base",
74
76
  "gpt-4": "cl100k_base",
75
77
  "gpt-3.5-turbo": "cl100k_base",
76
78
  "gpt-35-turbo": "cl100k_base", # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
120
122
 
121
123
  MODEL_PREFIX_TO_ENCODING = {
122
124
  # chat
125
+ "gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13, etc.
123
126
  "gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k
124
127
  "gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc.
125
128
  "gpt-35-turbo-": "cl100k_base", # Azure deployment name
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.8
4
+ version: 0.0.9
5
5
  platform: x86_64-darwin
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-04-04 00:00:00.000000000 Z
11
+ date: 2024-05-16 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending