tiktoken_ruby 0.0.8-x86_64-linux → 0.0.9-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3a7f91d1a21f0a8062c03988d79393676a8f6a727c58efe094f27d274c27f9f
-  data.tar.gz: 15a0056ddedcf37c6a907445027411a9a43d41a4f067fa1fab81df7ba4432adb
+  metadata.gz: b2981777cb81fd3fade2688e66f0f6711c19956a3737b834fa1db1a1c0e2aabe
+  data.tar.gz: fbbac592954875ceb1c469f8266e3b69a845a3a960c8f1de7e45c2b0cf2dd74c
 SHA512:
-  metadata.gz: 59eb286c3a2084951ee7ca6a9ba7ea585e27747fcfbd9b79271a81cfe5c978168915d48486829f6d25d43a567136b422db6da8a1b925327747fbec1a013fbbb0
-  data.tar.gz: 310063bff0976e90055dd3685a64d6c27e0cde1e5ff21ca7891f875065b3b1e7984c8adcf0f7fde3d854674fe6e85d53377b7cca634206c9675163339bb2a87a
+  metadata.gz: 603c244d51c2254f15ab748ccade9f2dd3a5ebaa04b4e02119636a23901d3b24103231473ee97ccab36b82a19234a111312cad8919a5b1477638e592b6aa059b
+  data.tar.gz: cdf87538056946d3799027098b2ce43a3134d419513c0ad7f69db5ee6014c4bade22ab6e5d006537a1b6fb0e6ab86d88b1be39ef16babdd18d4e0e37545f71b2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
+    tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
 
 GEM

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: x86_64-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending