tiktoken_ruby 0.0.8-x86_64-linux-musl → 0.0.9-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34a2818779ff0903c9b7bdf5e7132440cbee1980475b83dc899ecc019c353cc6
-  data.tar.gz: 42f9069dca45fc54ae7d2ee9711dc8028a2d305f6caeb7db6024b139371bffbd
+  metadata.gz: 68ab7cf30658d5a38c8a7f8cceb40cb7a5e99da1df56cea9d2eb660fcd7d3608
+  data.tar.gz: 748226cd09b35adc810cc381404c5a6745f84c353314416c55a798e55ebbc4a9
 SHA512:
-  metadata.gz: dd60905cfe30d24ef26c00f46604da82530d58fc24262da51d0db6b81fba92c66b935cb570d84879c463a0adeb4c7e2bf1bbfad89fd951cd4fa68429af0b2a87
-  data.tar.gz: c952a88e2bce633514d77ce0f5bb380d1a8d622225beab41ea3e29053658f27f4774761224f45d8c963068583274f32f6d66e8b06dec1ef9d267e4e8b9a3c80f
+  metadata.gz: 8ffc23202ec430374c113b3aaa57b7c3ece7da8f192b0b722b6454dae52191932a4ca10ce30f2d580b23806170b5996b2c345cb00f4c8a1f71d425f69c02cc84
+  data.tar.gz: 904623d818d68385ca1c4cba8108331a85758fab54f1e2ba2597ee4fbde1fd20205c711607fc6e7cd85924acbc1e237e47868e8b58d411e01b146dac57fb538f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
+    tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
 
 GEM

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: x86_64-linux-musl
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending