tiktoken_ruby 0.0.8-x86_64-darwin → 0.0.9-x86_64-darwin

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5952edc6d35682fd1f1e11e2e3b592a910cbf564267ec1d674ebdcac8197e6ab
4
- data.tar.gz: 456ccf90cd89bffdf2a8631a253b2bdb8ec97fe6bbd4b2d82b5198935a0b5bbb
3
+ metadata.gz: dfffd99df0ef60ddd07eabefa385c8f4e56ed814063933118d7043c06811e7f5
4
+ data.tar.gz: 40f71bfc9ffab7a3694f6f7761ba22ea33eb781a9c8ed10e335dbef7b41443ab
5
5
  SHA512:
6
- metadata.gz: 5aaa9e3debf8189797adaee23dbdaa1ca7455dea5e7cfb86b0b16dac07db2df6a3322c3541c2af88a1fc22eaf8a6dc9bf56a56db80a780ad91a82854a6e4b663
7
- data.tar.gz: 8c76c8e1220523539880a5b79d877aa3c2bd3d3ee51a0b6c6025502534cc9973c562b8aaeda291fa05376ef14cb81105449b690d06918b75839484bb811cbc8c
6
+ metadata.gz: 8013c6cd1f87ee18997962f56b2af713c207842224affaed762e5156c8c9ce37f6f01eef1fe07731cf763fe8c7cd820789968e4afb481be5b7876b73bf64c458
7
+ data.tar.gz: c8eff716ffefcfafb3fe3a456fe18709241d611ef66237aeb67c4187181e4d1b53b23cddf494127ad02b8f2d20b1ca512f97320135d2a005f76b279fdbdb17b4
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.8)
4
+ tiktoken_ruby (0.0.9)
5
5
  rb_sys (= 0.9.87)
6
6
 
7
7
  GEM
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.8"
4
+ VERSION = "0.0.9"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -64,13 +64,15 @@ module Tiktoken
64
64
  :r50k_base,
65
65
  :p50k_base,
66
66
  :p50k_edit,
67
- :cl100k_base
67
+ :cl100k_base,
68
+ :o200k_base
68
69
  ]
69
70
 
70
71
  # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
71
72
  # that is also MIT licensed but by OpenAI
72
73
  MODEL_TO_ENCODING_NAME = {
73
74
  # chat
75
+ "gpt-4o": "o200k_base",
74
76
  "gpt-4": "cl100k_base",
75
77
  "gpt-3.5-turbo": "cl100k_base",
76
78
  "gpt-35-turbo": "cl100k_base", # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
120
122
 
121
123
  MODEL_PREFIX_TO_ENCODING = {
122
124
  # chat
125
+ "gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13, etc.
123
126
  "gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k
124
127
  "gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc.
125
128
  "gpt-35-turbo-": "cl100k_base", # Azure deployment name
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.8
4
+ version: 0.0.9
5
5
  platform: x86_64-darwin
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-04-04 00:00:00.000000000 Z
11
+ date: 2024-05-16 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending