tiktoken_ruby 0.0.7-arm64-darwin → 0.0.9-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97f483266d9dd175af4d270716a3481a157db746d4be804504d20870c11e4fe0
-  data.tar.gz: 76468fd689d5b71bdaaaee7cc6141d51fe5a7ff2af70a334d1bd538a7b3e8455
+  metadata.gz: 30a86ddd6729c0ef973df0b6bcd8cb6bf9b85078ec11da565e001a70411ae92a
+  data.tar.gz: 25ad07cd3c974f3c35bebb93be2c973f572f81d6414aa609d319d19a5a5b1253
 SHA512:
-  metadata.gz: 3501683955f9c817c90ba800ec7a4bb160b33bd3e40aa5efd2a0e482812d76a6fb3f0eac7e7aed3b02d5b72701b353090ec199f8b2929c0d77d8ac28b20462f2
-  data.tar.gz: 3ecc8a2f8fbdb30c3b0e8dcf54e1a4c6458cf994d2d86dc64c03264d81c04444968da3659ba07ca499d8ba113915c171553df147d3b98086f88325ab01d24265
+  metadata.gz: 7b41b762c94e1c8f3bc2407ee55aea5e58e1c1eb129e5ef4050b4009805aefe994e8b241084c353cbd290979fe39f02a59a8512faf2d2e258b70759ddfd63670
+  data.tar.gz: 11d22b6898afc36935a60000089f7bdfca4f6bd3aabbce1e4c9ed473f35ec0021a6cf1919c73152b4436052f63335a1b6bf1f2f424871b7f64a58f7d391d2efd

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.7)
-      rb_sys (>= 0.9.86)
+    tiktoken_ruby (0.0.9)
+      rb_sys (= 0.9.87)
 
 GEM
   remote: https://rubygems.org/
@@ -22,7 +22,7 @@ GEM
     rake (13.1.0)
     rake-compiler (1.2.5)
       rake
-    rb_sys (0.9.86)
+    rb_sys (0.9.87)
     regexp_parser (2.9.0)
     rexml (3.2.6)
     rspec (3.12.0)
@@ -89,4 +89,4 @@ DEPENDENCIES
   yard-doctest
 
 BUNDLED WITH
-   2.4.6
+   2.4.4

data/README.md

@@ -1,8 +1,14 @@
 [![Gem Version](https://badge.fury.io/rb/tiktoken_ruby.svg)](https://badge.fury.io/rb/tiktoken_ruby)
+
 # tiktoken_ruby
 
 [Tiktoken](https://github.com/openai/tiktoken) is BPE tokenizer from OpenAI used with their GPT models.
-This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used. 
+This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used.
+
+## Request for maintainers
+
+I can't really put substantial time into maintaining this. Probably nothing more than a couple hours every few months. If you have experience maintaining ruby gems and would like to
+lend a hand please send me an email or reply to this [issue](https://github.com/IAPark/tiktoken_ruby/issues/26)
 
 ## Installation
 
@@ -15,17 +21,19 @@ If bundler is not being used to manage dependencies, install the gem by executin
     $ gem install tiktoken_ruby
 
 ## Usage
+
 Usage should be very similar to the python library. Here's a simple example
 
 Encode and decode text
+
 ```ruby
 require 'tiktoken_ruby'
-
 enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
 Encoders can also be retrieved by model name
+
 ```ruby
 require 'tiktoken_ruby'
 
@@ -53,7 +61,6 @@ bundle exec rake compile
 bundle exec rake spec
 ```
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/tiktoken_ruby/encoding.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Tiktoken::Encoding
+  CACHE_MUTEX = Mutex.new
+
   attr_reader :name
 
   # This returns a new Tiktoken::Encoding instance for the requested encoding
@@ -15,8 +17,10 @@ class Tiktoken::Encoding
   # @param encoding [Symbol] The name of the encoding to load
   # @return [Tiktoken::Encoding] The encoding instance
   def self.for_name_cached(encoding)
-    @encodings ||= {}
-    @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    CACHE_MUTEX.synchronize do
+      @encodings ||= {}
+      @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    end
   end
 
   # Encodes the text as a list of integer tokens. This encoding will encode special non text tokens

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.7"
+  VERSION = "0.0.9"
 end

data/lib/tiktoken_ruby.rb

@@ -28,7 +28,7 @@ module Tiktoken
 
     # Gets the encoding for an OpenAI model
     # @param model_name [Symbol|String] The name of the model to get the encoding for
-    # @return [Tiktoken::Encoding] The encoding instance
+    # @return [Tiktoken::Encoding, nil] The encoding instance, or nil if no encoding is found
     # @example Count tokens for text
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
@@ -37,10 +37,12 @@ module Tiktoken
         return get_encoding(MODEL_TO_ENCODING_NAME[model_name.to_sym])
       end
 
-      MODEL_PREFIX_TO_ENCODING.each do |prefix, encoding|
-        if model_name.start_with?(prefix.to_s)
-          return get_encoding(encoding)
-        end
+      _prefix, encoding = MODEL_PREFIX_TO_ENCODING.find do |prefix, _encoding|
+        model_name.start_with?(prefix.to_s)
+      end
+
+      if encoding
+        get_encoding(encoding)
       end
     end
 
@@ -62,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -118,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.9
 platform: arm64-darwin
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-14 00:00:00.000000000 Z
+date: 2024-05-16 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -28,7 +28,6 @@ files:
 - Rakefile
 - doctest_helper.rb
 - lib/tiktoken_ruby.rb
-- lib/tiktoken_ruby/3.0/tiktoken_ruby.bundle
 - lib/tiktoken_ruby/3.1/tiktoken_ruby.bundle
 - lib/tiktoken_ruby/3.2/tiktoken_ruby.bundle
 - lib/tiktoken_ruby/3.3/tiktoken_ruby.bundle
@@ -50,7 +49,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
   - - "<"
     - !ruby/object:Gem::Version
       version: 3.4.dev