tiktoken_ruby 0.0.8-arm-linux → 0.0.9-arm-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c8bb1d2e78442a762f320c0afe73b3fb0f9caa696a68d38710f4fab16aa15ad
-  data.tar.gz: fe7e0b30a686934ea9d08e023ed5153e9137dae356b43afbf00af1c61993b608
+  metadata.gz: da4ac893803fed2e1f7feac48216d4f6b5e26a2b1f82ac390140e5b6ad713da9
+  data.tar.gz: 75ead5e389c6f94477cb010d1e128985f27419ca544909ab3bec2ae125c41a45
 SHA512:
-  metadata.gz: 53baa4247f588e925697fad52fc6ab2e950d5f84c6032a1dd02cf0086ffdf0475d31a6f8965f7d32603b99d6808867802b87ba5d814ac7986f7af4993c49c06e
-  data.tar.gz: dd96bec21478a1333af3b0a17e62da65aee4865e434bb5f75c30954a8393cbab65df0ad1d7b49e335a487dd4217abacc63f70bbef7289558f9fe20bf19e37d8d
+  metadata.gz: e7c804a4af5e895f86d5ef485768f6a94f8411447e0015f7f796cbd83b1ccb18f2fa2a58a627dc000a8c01724c57ad4ef8f00524e39e99f245d3f934027c1051
+  data.tar.gz: a498b685977f163fe6892fac2d429997ba76659df18c829fa688ea60258d5648c7bb98a86f2ab42959e9d9c2914dcaec143bfcc06dbf79ad0f38756b25259ddb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
+    tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
 
 GEM

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: arm-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending