tiktoken_ruby 0.0.7-arm-linux → 0.0.8-arm-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9163ecb86a1d8103874f493431d81c2dffdc4c5a39143300d37d9a25589e6572
-  data.tar.gz: 19261e6a4008d216b243084cea404ded4d797851b054648e8a65af981f0f8eeb
+  metadata.gz: 6c8bb1d2e78442a762f320c0afe73b3fb0f9caa696a68d38710f4fab16aa15ad
+  data.tar.gz: fe7e0b30a686934ea9d08e023ed5153e9137dae356b43afbf00af1c61993b608
 SHA512:
-  metadata.gz: 159bfd6c3478ee9ba45a67511f5026ad2d17d8c6ecb2f1bc0789a744dcf7def00b60a3dd0a216cab5eb9cfa8301afb7a36f0c37e231de0c538a7d7d792f503a8
-  data.tar.gz: 85d02e82dc8104a87391bccb676e2be09fc927b06421fcb4a00719a17e7838d39d310627e7b62a5de875cc84fdf93290e8957aa9faf62b955bdc13485d0958fb
+  metadata.gz: 53baa4247f588e925697fad52fc6ab2e950d5f84c6032a1dd02cf0086ffdf0475d31a6f8965f7d32603b99d6808867802b87ba5d814ac7986f7af4993c49c06e
+  data.tar.gz: dd96bec21478a1333af3b0a17e62da65aee4865e434bb5f75c30954a8393cbab65df0ad1d7b49e335a487dd4217abacc63f70bbef7289558f9fe20bf19e37d8d

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.7)
-      rb_sys (>= 0.9.86)
+    tiktoken_ruby (0.0.8)
+      rb_sys (= 0.9.87)
 
 GEM
   remote: https://rubygems.org/
@@ -22,7 +22,7 @@ GEM
     rake (13.1.0)
     rake-compiler (1.2.5)
       rake
-    rb_sys (0.9.86)
+    rb_sys (0.9.87)
     regexp_parser (2.9.0)
     rexml (3.2.6)
     rspec (3.12.0)
@@ -89,4 +89,4 @@ DEPENDENCIES
   yard-doctest
 
 BUNDLED WITH
-   2.4.6
+   2.4.4

data/README.md

@@ -1,8 +1,14 @@
 [![Gem Version](https://badge.fury.io/rb/tiktoken_ruby.svg)](https://badge.fury.io/rb/tiktoken_ruby)
+
 # tiktoken_ruby
 
 [Tiktoken](https://github.com/openai/tiktoken) is BPE tokenizer from OpenAI used with their GPT models.
-This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used. 
+This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used.
+
+## Request for maintainers
+
+I can't really put substantial time into maintaining this. Probably nothing more than a couple hours every few months. If you have experience maintaining ruby gems and would like to
+lend a hand please send me an email or reply to this [issue](https://github.com/IAPark/tiktoken_ruby/issues/26)
 
 ## Installation
 
@@ -15,17 +21,19 @@ If bundler is not being used to manage dependencies, install the gem by executin
     $ gem install tiktoken_ruby
 
 ## Usage
+
 Usage should be very similar to the python library. Here's a simple example
 
 Encode and decode text
+
 ```ruby
 require 'tiktoken_ruby'
-
 enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
 Encoders can also be retrieved by model name
+
 ```ruby
 require 'tiktoken_ruby'
 
@@ -53,7 +61,6 @@ bundle exec rake compile
 bundle exec rake spec
 ```
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/tiktoken_ruby/encoding.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Tiktoken::Encoding
+  CACHE_MUTEX = Mutex.new
+
   attr_reader :name
 
   # This returns a new Tiktoken::Encoding instance for the requested encoding
@@ -15,8 +17,10 @@ class Tiktoken::Encoding
   # @param encoding [Symbol] The name of the encoding to load
   # @return [Tiktoken::Encoding] The encoding instance
   def self.for_name_cached(encoding)
-    @encodings ||= {}
-    @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    CACHE_MUTEX.synchronize do
+      @encodings ||= {}
+      @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    end
   end
 
   # Encodes the text as a list of integer tokens. This encoding will encode special non text tokens

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/tiktoken_ruby.rb

@@ -28,7 +28,7 @@ module Tiktoken
 
     # Gets the encoding for an OpenAI model
     # @param model_name [Symbol|String] The name of the model to get the encoding for
-    # @return [Tiktoken::Encoding] The encoding instance
+    # @return [Tiktoken::Encoding, nil] The encoding instance, or nil if no encoding is found
     # @example Count tokens for text
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
@@ -37,10 +37,12 @@ module Tiktoken
         return get_encoding(MODEL_TO_ENCODING_NAME[model_name.to_sym])
       end
 
-      MODEL_PREFIX_TO_ENCODING.each do |prefix, encoding|
-        if model_name.start_with?(prefix.to_s)
-          return get_encoding(encoding)
-        end
+      _prefix, encoding = MODEL_PREFIX_TO_ENCODING.find do |prefix, _encoding|
+        model_name.start_with?(prefix.to_s)
+      end
+
+      if encoding
+        get_encoding(encoding)
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: arm-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-14 00:00:00.000000000 Z
+date: 2024-04-04 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -28,7 +28,6 @@ files:
 - Rakefile
 - doctest_helper.rb
 - lib/tiktoken_ruby.rb
-- lib/tiktoken_ruby/3.0/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.1/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.3/tiktoken_ruby.so
@@ -50,7 +49,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
   - - "<"
     - !ruby/object:Gem::Version
       version: 3.4.dev