roseflow-tiktoken 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3b272bb8e3b5a4805fe59670894340347a0310ada891f29abb03235b4aca243
-  data.tar.gz: 3135181d05c8ee397d57d88f99f100f8db45dc5cd7295dff27db3641de7bcc0f
+  metadata.gz: a7f3ff6fdcfd71f07e2319202fb56ac40e462c98e4a897b239bb4e1150cf1e13
+  data.tar.gz: 0cca81f6bd18edc889124e21598109a546ac0dfdac49c84b38f615eeae3f9b9c
 SHA512:
-  metadata.gz: a21a9d867e21ea71d36f1605e894fd68933088c25df3077381bdecd1d20b7d12e13826577fe649832dc9294d6aa9e14ade0591cdf2327fe1883d60233005d342
-  data.tar.gz: 6f94d37d4f661d5d672e8a5c8a9952544458d815f7997ecd01db82f6e2b17cbb8d3a9b7838993b5567e883d744abb08cf3f51a38030437b5823a79bf7ee0c8ff
+  metadata.gz: 8a737814488c6fcd78d66fd66f620da0eb2808203be94e3bfecacb0ab71ce432a071140cea65cce8b49db8227096e1930fd8c47437fe051e7fe144b8d4272f2c
+  data.tar.gz: bdc94e15b3d375e100e682a46b978abb81eb561aed3bf52bd206123af85f2dad279f9df91f2ada274e7ca1fca04e250cec3e0bbd35ad368f6634c5c631804541

data/CHANGELOG.md

@@ -1,4 +1,6 @@
-## [Unreleased]
+## [0.2.0] - 2023-07-19
+
+- Replaces PyCall with tiktoken_ruby
 
 ## [0.1.0] - 2023-05-02
 

data/README.md

@@ -2,7 +2,7 @@
 
 [tiktoken](https://github.com/openai/tiktoken) is a fast BPE tokenizer for use with OpenAI's models. `roseflow-tiktoken` gem helps you use the tokenizer in Ruby, especially with (Roseflow)[https://github.com/ljuti/roseflow].
 
-Currently, this gem wraps the (`tiktoken` Python module)[https://github.com/openai/tiktoken] for convenient use in Roseflow.
+This gem wraps the (`tiktoken_ruby` gem)[https://github.com/IAPark/tiktoken_ruby] for convenient use in Roseflow.
 
 ## Installation
 
@@ -45,6 +45,8 @@ tokenizer.decode([19952, 420, 925, 1139, 11460, 13]) # => "Turn this string into
 | `p50k_edit`             | Use for edit models like `text-davinci-edit-001`, `code-davinci-edit-001` |
 | `r50k_base` (or `gpt2`) | GPT-3 models like `davinci`                                               |
 
+If a model is not provided or is unknown to the library, it will default to `cl100k_base` encoding.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/roseflow/tiktoken/tokenizer.rb

@@ -1,12 +1,11 @@
-require "pycall"
+require "tiktoken_ruby"
 
 module Roseflow
   module Tiktoken
     class Tokenizer
       def initialize(model: nil)
-        @tokenizer = PyCall.import_module("tiktoken")
         @model = model
-        @encoding = @tokenizer.encoding_for_model(@model) if @model
+        @encoding = determine_encoding(model)
       end
 
       def encode(input)
@@ -41,6 +40,11 @@ module Roseflow
 
       private
 
+      def determine_encoding(model)
+        encoding = model ? ::Tiktoken.encoding_for_model(model) : ::Tiktoken.get_encoding("cl100k_base")
+        encoding.is_a?(::Tiktoken::Encoding) ? encoding : ::Tiktoken.get_encoding("cl100k_base")
+      end
+
       def tokens_per_message_for_model(model)
         case model
         when "gpt-4"

data/lib/roseflow/tiktoken/version.rb

@@ -8,7 +8,7 @@ module Roseflow
 
     module VERSION
       MAJOR = 0
-      MINOR = 1
+      MINOR = 2
       PATCH = 0
       PRE = nil
 

data/roseflow-tiktoken.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |spec|
   spec.name = "roseflow-tiktoken"
   spec.version = Roseflow::Tiktoken.gem_version
   spec.authors = ["Lauri Jutila"]
-  spec.email = ["git@laurijutila.com"]
+  spec.email = ["ljuti@users.noreply.github.com"]
 
   spec.summary = "Tiktoken tokenizer for Roseflow."
   spec.description = "Tiktoken tokenizer for Roseflow."
@@ -29,5 +29,5 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "pycall", "~> 1.4"
+  spec.add_dependency "tiktoken_ruby"
 end

metadata

@@ -1,32 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: roseflow-tiktoken
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Lauri Jutila
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-10 00:00:00.000000000 Z
+date: 2023-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: pycall
+  name: tiktoken_ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '0'
 description: Tiktoken tokenizer for Roseflow.
 email:
-- git@laurijutila.com
+- ljuti@users.noreply.github.com
 executables: []
 extensions: []
 extra_rdoc_files: []