tiktoken_ruby 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de2d6e8e83771f2ef51351e019e9cebc7163a1775bfc0e812da58371574b9b63
-  data.tar.gz: fe2c629e8b435a181bfa4524655bfa137f1660061ab4f780e0edc15a11a7538d
+  metadata.gz: 347a10d045e27fca4cdfec03c4d2eac0150448b8f2125d5bcbcd1b92db83499a
+  data.tar.gz: de048e8320daa15b27ffa7ccdd9f7ec618cb2a3ad96fe2bedf71e6f780fc8b6f
 SHA512:
-  metadata.gz: bbf721963e873464fae055d23308068fcdac8db4e27dea28653c3fc017f0803da59b4c549a0fb6b6339f79bc4379e2913e1c47fa25f9894ebde840b23aa81edb
-  data.tar.gz: 1275369f56a2498ce39c5a2b259efdab0684ae834c6d769bfe23d7190c69917a22d46dc7b7675cd15c8a44dcfa8f7c3be2df850035236fa33f8e55afd31db42b
+  metadata.gz: 7c587d4b18e777a0f7692aab857fec1fc3b3bcceceab158439197123786475c65d51f5356351bcf0c9c327b94e2ff15a83b2144c2db329aedea0456c1d763ff9
+  data.tar.gz: abfdeb836d81555effa5ef6647a77b360dae2b975bb3dc23a22c779f6dd82256630942ddfef01a837ff5195896fa4251925e87dec0d245dce1c8e83719a488f1

data/Cargo.lock

@@ -31,16 +31,16 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
 
 [[package]]
 name = "bindgen"
-version = "0.66.1"
+version = "0.69.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7"
+checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
 dependencies = [
  "bitflags 2.4.0",
  "cexpr",
  "clang-sys",
+ "itertools",
  "lazy_static",
  "lazycell",
- "peeking_take_while",
  "proc-macro2",
  "quote",
  "regex",
@@ -114,6 +114,12 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "either"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
+
 [[package]]
 name = "fancy-regex"
 version = "0.11.0"
@@ -130,6 +136,15 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -242,12 +257,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "peeking_take_while"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.66"
@@ -268,18 +277,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.81"
+version = "0.9.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a57240b308b155b09dce81e32829966a99f52d1088b45957e4283e526c5317a1"
+checksum = "225103e3d69bbfe8831f9fd0d2461335f3a9dd06aa6e88bcb6d6970383494d06"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.81"
+version = "0.9.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f24ce877a4c5d07f06f6aa6fec3ac95e4b357b9f73b0f5445d8cbb7266d410e8"
+checksum = "bacce8095a5167d5ede618bbd9353e9d9e2f32ddaf54be911106f0ee6baacf09"
 dependencies = [
  "bindgen",
  "lazy_static",

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.7)
+    tiktoken_ruby (0.0.8)
       rb_sys (>= 0.9.86)
 
 GEM

data/README.md

@@ -1,8 +1,14 @@
 [![Gem Version](https://badge.fury.io/rb/tiktoken_ruby.svg)](https://badge.fury.io/rb/tiktoken_ruby)
+
 # tiktoken_ruby
 
 [Tiktoken](https://github.com/openai/tiktoken) is BPE tokenizer from OpenAI used with their GPT models.
-This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used. 
+This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used.
+
+## Request for maintainers
+
+I can't really put substantial time into maintaining this. Probably nothing more than a couple hours every few months. If you have experience maintaining ruby gems and would like to
+lend a hand please send me an email or reply to this [issue](https://github.com/IAPark/tiktoken_ruby/issues/26)
 
 ## Installation
 
@@ -15,17 +21,19 @@ If bundler is not being used to manage dependencies, install the gem by executin
     $ gem install tiktoken_ruby
 
 ## Usage
+
 Usage should be very similar to the python library. Here's a simple example
 
 Encode and decode text
+
 ```ruby
 require 'tiktoken_ruby'
-
 enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
 Encoders can also be retrieved by model name
+
 ```ruby
 require 'tiktoken_ruby'
 
@@ -53,7 +61,6 @@ bundle exec rake compile
 bundle exec rake spec
 ```
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/tiktoken_ruby/encoding.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Tiktoken::Encoding
+  CACHE_MUTEX = Mutex.new
+
   attr_reader :name
 
   # This returns a new Tiktoken::Encoding instance for the requested encoding
@@ -15,8 +17,10 @@ class Tiktoken::Encoding
   # @param encoding [Symbol] The name of the encoding to load
   # @return [Tiktoken::Encoding] The encoding instance
   def self.for_name_cached(encoding)
-    @encodings ||= {}
-    @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    CACHE_MUTEX.synchronize do
+      @encodings ||= {}
+      @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+    end
   end
 
   # Encodes the text as a list of integer tokens. This encoding will encode special non text tokens

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/tiktoken_ruby.rb

@@ -28,7 +28,7 @@ module Tiktoken
 
     # Gets the encoding for an OpenAI model
     # @param model_name [Symbol|String] The name of the model to get the encoding for
-    # @return [Tiktoken::Encoding] The encoding instance
+    # @return [Tiktoken::Encoding, nil] The encoding instance, or nil if no encoding is found
     # @example Count tokens for text
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
@@ -37,10 +37,12 @@ module Tiktoken
         return get_encoding(MODEL_TO_ENCODING_NAME[model_name.to_sym])
       end
 
-      MODEL_PREFIX_TO_ENCODING.each do |prefix, encoding|
-        if model_name.start_with?(prefix.to_s)
-          return get_encoding(encoding)
-        end
+      _prefix, encoding = MODEL_PREFIX_TO_ENCODING.find do |prefix, _encoding|
+        model_name.start_with?(prefix.to_s)
+      end
+
+      if encoding
+        get_encoding(encoding)
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - IAPark
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-12 00:00:00.000000000 Z
+date: 2024-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys