tiktoken_ruby 0.0.8-x64-mingw-ucrt → 0.0.9-x64-mingw-ucrt

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7aadcd951231ad7e517669fcea63c342e452cdf91b91a8702886b23503eafe84
-  data.tar.gz: 237430fb0dac4f90e7b9f2e8b09386bf8bd7a05ec98dc8ead0f604aca442f991
+  metadata.gz: e75ab639a2a0830445d3b1b2a4070b9b7df74ff31479e6d63a89d9bc17419898
+  data.tar.gz: 16db25116ffa07c97d4fe4e1bf59a3697c35b2e8bcda8886e6218fcda5e771ac
 SHA512:
-  metadata.gz: e46b673a18b037435bd99581bf2db9a94f57e8fc5638c842934f91f0f001884f888af686839c6f1814d45f7f91a4b53af67a836a31995467c427246f55e0c044
-  data.tar.gz: b7ee3cd8222cdc228e1c5392a551817259acdc0c6a27bf2c877dded7daea48abb237c9e1de860f9d3d1761d80916f3c31f56ede186a47db98509b6456544bb4c
+  metadata.gz: 8614d39d1982e6f12b465b2f972e1330a64d41f3e5e49678aefb4e21a6dbfa35721274cfb2409505779a8c699de22e3b737311da27d08908eca0cb1177327d7d
+  data.tar.gz: 6998d454684a86eca1ca5064aafb16b9b49c7f6bd8d2101f501bcc7aae60912a50463b643532b9a68061cf1d3184e8936a7951597474ef0439d228248a219145

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
+    tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
 
 GEM

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: x64-mingw-ucrt
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending