tiktoken_ruby 0.0.8-arm64-darwin → 0.0.9-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e8099b11c2d6bed54d1eecdb44e1418e712a1ae5d83a42f843d9e4ff30bf210
-  data.tar.gz: 1ebab400772e48637f61a277b3568aacf1e3a74fcd733e1ffdbdeba4cd5f8302
+  metadata.gz: 30a86ddd6729c0ef973df0b6bcd8cb6bf9b85078ec11da565e001a70411ae92a
+  data.tar.gz: 25ad07cd3c974f3c35bebb93be2c973f572f81d6414aa609d319d19a5a5b1253
 SHA512:
-  metadata.gz: 4e47f06dbbd39dbffaea917a0f2734185a493c099e8dd47368c97d5fdabc206fd49ffee2f7c897917ca36f2426dc1f7fa2e7af3ad0e9aebc4840610eef9762f9
-  data.tar.gz: 034af2c7975ab6e4d055f0f41365547857b53e8cc9e3fef1cad6f1c6fda6a1a768d98f966163e88961e625e4272efc9982f15980cd29b2ffda9bb612ba4246b2
+  metadata.gz: 7b41b762c94e1c8f3bc2407ee55aea5e58e1c1eb129e5ef4050b4009805aefe994e8b241084c353cbd290979fe39f02a59a8512faf2d2e258b70759ddfd63670
+  data.tar.gz: 11d22b6898afc36935a60000089f7bdfca4f6bd3aabbce1e4c9ed473f35ec0021a6cf1919c73152b4436052f63335a1b6bf1f2f424871b7f64a58f7d391d2efd

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
+    tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
 
 GEM

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: arm64-darwin
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending