tiktoken_ruby 0.0.7-x86_64-darwin → 0.0.9-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bbdd6164228f0fa953d7988326bec5899f29b87cbe4c6b19e3146b6a98a6f35
-  data.tar.gz: 5a53fdc3cd7eb9289cffcf2dbe88328d1387c777bdf8312571c9e7c15d8ec2fc
+  metadata.gz: dfffd99df0ef60ddd07eabefa385c8f4e56ed814063933118d7043c06811e7f5
+  data.tar.gz: 40f71bfc9ffab7a3694f6f7761ba22ea33eb781a9c8ed10e335dbef7b41443ab
 SHA512:
-  metadata.gz: d895625940b103633790071183b00db0a0ee4c3df83a047ed34383de1ce54803c5b7caa7e2f71badd3d099197961c050abb425f40da9b1a57a0284e5e26e4d77
-  data.tar.gz: fddc06e751328c8cc773df5da02949ae021ad2f9cc1039e6be6affd7f954e5ca96284e3981c25a923455ef106c4f0d836076b58e0e41f55f2b2f265eaae13162
+  metadata.gz: 8013c6cd1f87ee18997962f56b2af713c207842224affaed762e5156c8c9ce37f6f01eef1fe07731cf763fe8c7cd820789968e4afb481be5b7876b73bf64c458
+  data.tar.gz: c8eff716ffefcfafb3fe3a456fe18709241d611ef66237aeb67c4187181e4d1b53b23cddf494127ad02b8f2d20b1ca512f97320135d2a005f76b279fdbdb17b4

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.7)
-      rb_sys (>= 0.9.86)
+    tiktoken_ruby (0.0.9)
+      rb_sys (= 0.9.87)
 
 GEM
   remote: https://rubygems.org/
@@ -22,7 +22,7 @@ GEM
     rake (13.1.0)
     rake-compiler (1.2.5)
       rake
-    rb_sys (0.9.86)
+    rb_sys (0.9.87)
     regexp_parser (2.9.0)
     rexml (3.2.6)
     rspec (3.12.0)
@@ -89,4 +89,4 @@ DEPENDENCIES
   yard-doctest
 
 BUNDLED WITH
-   2.4.6
+   2.4.4

data/README.md

@@ -1,8 +1,14 @@
 [![Gem Version](https://badge.fury.io/rb/tiktoken_ruby.svg)](https://badge.fury.io/rb/tiktoken_ruby)
+
 # tiktoken_ruby
 
 [Tiktoken](https://github.com/openai/tiktoken) is BPE tokenizer from OpenAI used with their GPT models.
-This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used. 
+This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used.
+
+## Request for maintainers
+
+I can't really put substantial time into maintaining this. Probably nothing more than a couple hours every few months. If you have experience maintaining ruby gems and would like to
+lend a hand please send me an email or reply to this [issue](https://github.com/IAPark/tiktoken_ruby/issues/26)
 
 ## Installation
 
@@ -15,17 +21,19 @@ If bundler is not being used to manage dependencies, install the gem by executin
     $ gem install tiktoken_ruby
 
 ## Usage
+
 Usage should be very similar to the python library. Here's a simple example
 
 Encode and decode text
+
 ```ruby
 require 'tiktoken_ruby'
-
 enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
 Encoders can also be retrieved by model name
+
 ```ruby
 require 'tiktoken_ruby'
 
@@ -53,7 +61,6 @@ bundle exec rake compile
 bundle exec rake spec
 ```
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/tiktoken_ruby/encoding.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Tiktoken::Encoding
+  CACHE_MUTEX = Mutex.new
+
   attr_reader :name
 
   # This returns a new Tiktoken::Encoding instance for the requested encoding
@@ -15,8 +17,10 @@ class Tiktoken::Encoding
   # @param encoding [Symbol] The name of the encoding to load
   # @return [Tiktoken::Encoding] The encoding instance
   def self.for_name_cached(encoding)
-    @encodings ||= {}
-    @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    CACHE_MUTEX.synchronize do
+      @encodings ||= {}
+      @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    end
   end
 
   # Encodes the text as a list of integer tokens. This encoding will encode special non text tokens

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.7"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -28,7 +28,7 @@ module Tiktoken
 
     # Gets the encoding for an OpenAI model
     # @param model_name [Symbol|String] The name of the model to get the encoding for
-    # @return [Tiktoken::Encoding] The encoding instance
+    # @return [Tiktoken::Encoding, nil] The encoding instance, or nil if no encoding is found
     # @example Count tokens for text
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
@@ -37,10 +37,12 @@ module Tiktoken
         return get_encoding(MODEL_TO_ENCODING_NAME[model_name.to_sym])
       end
 
-      MODEL_PREFIX_TO_ENCODING.each do |prefix, encoding|
-        if model_name.start_with?(prefix.to_s)
-          return get_encoding(encoding)
-        end
+      _prefix, encoding = MODEL_PREFIX_TO_ENCODING.find do |prefix, _encoding|
+        model_name.start_with?(prefix.to_s)
+      end
+
+      if encoding
+        get_encoding(encoding)
       end
     end
 
@@ -62,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -118,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.9
 platform: x86_64-darwin
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-14 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -28,7 +28,6 @@ files:
 - Rakefile
 - doctest_helper.rb
 - lib/tiktoken_ruby.rb
-- lib/tiktoken_ruby/3.0/tiktoken_ruby.bundle
 - lib/tiktoken_ruby/3.1/tiktoken_ruby.bundle
 - lib/tiktoken_ruby/3.2/tiktoken_ruby.bundle
 - lib/tiktoken_ruby/3.3/tiktoken_ruby.bundle
@@ -50,7 +49,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
   - - "<"
     - !ruby/object:Gem::Version
       version: 3.4.dev