tiktoken_ruby 0.0.8-aarch64-linux → 0.0.9-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e157d10a1f0f5d11875496ebac5945fe51202aa6abc7da2a1d85297f0e793e69
-  data.tar.gz: fe5ee65444abe465c70dfaf8408aea4573cff82099ed1ee5e0b2b19c04177b41
+  metadata.gz: 046704f021259f1292da499bbcf2e8707d0d191350fb0e4386ed06c6d893a1b0
+  data.tar.gz: 62906b1be45ff6ae8709a337935926d1cbc39de4edad2d41552ca0c896e71ba2
 SHA512:
-  metadata.gz: 9e0bccf87a5aa5b9b58d1910e28ec58988b1e5c82d45e3b6fcd7f48d7cb4d71066376f6e308136d8e67969dae7f177044fb3db05f48dc03adad2147302c23281
-  data.tar.gz: 514cabd057bc8524d628c667363884bead42fd9e84581b8f5ec07c521efee3b6646eb268b06adfbb06c0e8f97a1039b978de77114aa081dc9856a570f39142ab
+  metadata.gz: 1f752aac415ce552017560c5b636d39ecf7829e824f34a898c4f3db8e1fd1bb015f8fb0125182fc6486aa02ab8268e32f111379dbeb9fd34368065daca794db8
+  data.tar.gz: 3ac9fd3efc816235fed8f7edaebe68e7175346ee260ea825e0acc71221d5bb48ff9a1f1718b259d5a8cb8d2b467dcef7a194aa7648d00c73be609df20dec6908

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
+    tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
 
 GEM

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: aarch64-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending