tiktoken_ruby 0.0.3-x64-mingw-ucrt → 0.0.5-x64-mingw-ucrt

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fc02070b46f894e41faf76d01edb3448bcf9fec33d8ce9c84a066f3b6a3c42c
-  data.tar.gz: d3059efb8fb610aabf5a32427d6d82040882b1384c046cdbd8c8f556babbfd54
+  metadata.gz: 5e4b7ae7f602054eaa94a4369c445e2887d3b4bb2760529de4ff810e0c914bd9
+  data.tar.gz: 19db106cdacf6b3968213edd15a1d4f3e7c13f6bbc40119afb345e39dc672b7e
 SHA512:
-  metadata.gz: 1b5826fa7ca9377abc35a5c4dd985afa2960110def15b992d350363b9fe66340d27e3ee9845abd4ba2571e9bec6777eb0d46902cb4b88c5e1ff3fc9ef0ad2346
-  data.tar.gz: f4613b799248700b53e24eaa7e87f3cca774921955b3db8fd5cddecc027b6df37924daa5dabed55943e89bb0bc6d78d7528e6c33eb46dac5ecaa2f1a868af019
+  metadata.gz: 6ad3a6ca773ba533475465d6e9c0bff75ccf7dbeb26b5ea2e2ab162f52c84aba930eb78bdb59b163e72ccc85563283caefa2a48a4f99dd1f3b610d635fb42b73
+  data.tar.gz: 51a2547eec264e17417fe28e279eec358c5f82b2f835605d945cf492323b3215f109eb0f9a20ab9870803cfe64d5fe90be33bb845c447018b86e3a5be6669cba

data/Gemfile

@@ -8,11 +8,9 @@ gemspec
 gem "rake", "~> 13.0"
 
 gem "rake-compiler"
-gem "rb_sys"
 
 gem "rspec", "~> 3.0"
 
 gem "standard", "~> 1.3"
-gem 'pry', '~> 0.14.2'
 
 gem "yard-doctest", "~> 0.1.17"

data/Gemfile.lock

@@ -1,24 +1,20 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.3)
+    tiktoken_ruby (0.0.5)
+      rb_sys (~> 0.9.68)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    coderay (1.1.3)
     diff-lcs (1.5.0)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
-    method_source (1.0.0)
     minitest (5.18.0)
     parallel (1.22.1)
     parser (3.2.1.1)
       ast (~> 2.4.1)
-    pry (0.14.2)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
     rainbow (3.1.1)
     rake (13.0.6)
     rake-compiler (1.2.1)
@@ -69,13 +65,13 @@ GEM
 
 PLATFORMS
   arm64-darwin-22
+  ruby
+  x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES
-  pry (~> 0.14.2)
   rake (~> 13.0)
   rake-compiler
-  rb_sys
   rspec (~> 3.0)
   standard (~> 1.3)
   tiktoken_ruby!

data/README.md

@@ -15,7 +15,7 @@ If bundler is not being used to manage dependencies, install the gem by executin
     $ gem install tiktoken_ruby
 
 ## Usage
-Usage should be very similar to the python library. here's a simple example
+Usage should be very similar to the python library. Here's a simple example
 
 Encode and decode text
 ```ruby
@@ -43,6 +43,17 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/iapark/tiktoken_ruby.
 
+To get started with development:
+
+```sh
+git clone https://github.com/IAPark/tiktoken_ruby.git
+cd tiktoken_ruby
+bundle install
+bundle exec rake compile
+bundle exec rake spec
+```
+
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -14,7 +14,6 @@ end
 
 RSpec::Core::RakeTask.new(:spec)
 
-
 task :native, [:platform] do |_t, platform:|
   sh "bundle", "exec", "rb-sys-dock", "--platform", platform, "--build"
 end

data/doctest_helper.rb

@@ -1 +1 @@
-require 'lib/tiktoken_ruby.rb'
+require "lib/tiktoken_ruby"

data/lib/tiktoken_ruby/encoding.rb

@@ -1,51 +1,52 @@
 # frozen_string_literal: true
 
 class Tiktoken::Encoding
-    attr_reader :name
-
-    # This returns a new Tiktoken::Encoding instance for the requested encoding
-    # @param encoding [Symbol] The name of the encoding to load
-    # @return [Tiktoken::Encoding] The encoding instance
-    def self.for_name(encoding)
-        Tiktoken::Encoding.new(Tiktoken::BpeFactory.send(encoding.to_sym), encoding.to_sym)
-    end
-
-    # This returns a Tiktoken::Encoding instance for the requested encoding
-    # It will reuse an existing encoding if it's already been loaded
-    # @param encoding [Symbol] The name of the encoding to load
-    # @return [Tiktoken::Encoding] The encoding instance
-    def self.for_name_cached(encoding)
-        @encodings ||= {}
-        @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
-    end
-
-    # Encodes the text as a list of integer tokens. This encoding will encode special non text tokens
-    # basically it's unescaped
-    # @param text [String] The text to encode
-    # @return [Array<Integer>] The encoded tokens
-    def encode_ordinary(text)
-        @ext_base_bpe.encode_ordinary(text)
-    end
-
-    # Encodes the text as a list of integer tokens. This encoding will treat special non text tokens
-    # as text unless they're in the allowed_special array. It's basically like the text was escaped
-    # @param text [String] The text to encode
-    # @param allowed_special [Array<String>] An array of special tokens to allow
-    # @return [Array<Integer>] The encoded tokens
-    def encode(text, allowed_special: [])
-        @ext_base_bpe.encode(text, allowed_special)
-    end
-
-    # Decodes the tokens back into text
-    # @param tokens [Array<Integer>] The tokens to decode
-    # @return [String] The decoded text
-    def decode(tokens)
-        @ext_base_bpe.decode(tokens)
-    end
-
-    private
-    def initialize(ext_base_bpe, name)
-        @ext_base_bpe = ext_base_bpe
-        @name = name
-    end
+  attr_reader :name
+
+  # This returns a new Tiktoken::Encoding instance for the requested encoding
+  # @param encoding [Symbol] The name of the encoding to load
+  # @return [Tiktoken::Encoding] The encoding instance
+  def self.for_name(encoding)
+    Tiktoken::Encoding.new(Tiktoken::BpeFactory.send(encoding.to_sym), encoding.to_sym)
+  end
+
+  # This returns a Tiktoken::Encoding instance for the requested encoding
+  # It will reuse an existing encoding if it's already been loaded
+  # @param encoding [Symbol] The name of the encoding to load
+  # @return [Tiktoken::Encoding] The encoding instance
+  def self.for_name_cached(encoding)
+    @encodings ||= {}
+    @encodings[encoding.to_sym] ||= Tiktoken::Encoding.for_name(encoding)
+  end
+
+  # Encodes the text as a list of integer tokens. This encoding will encode special non text tokens
+  # basically it's unescaped
+  # @param text [String] The text to encode
+  # @return [Array<Integer>] The encoded tokens
+  def encode_ordinary(text)
+    @ext_base_bpe.encode_ordinary(text)
+  end
+
+  # Encodes the text as a list of integer tokens. This encoding will treat special non text tokens
+  # as text unless they're in the allowed_special array. It's basically like the text was escaped
+  # @param text [String] The text to encode
+  # @param allowed_special [Array<String>] An array of special tokens to allow
+  # @return [Array<Integer>] The encoded tokens
+  def encode(text, allowed_special: [])
+    @ext_base_bpe.encode(text, allowed_special)
+  end
+
+  # Decodes the tokens back into text
+  # @param tokens [Array<Integer>] The tokens to decode
+  # @return [String] The decoded text
+  def decode(tokens)
+    @ext_base_bpe.decode(tokens)
+  end
+
+  private
+
+  def initialize(ext_base_bpe, name)
+    @ext_base_bpe = ext_base_bpe
+    @name = name
+  end
 end

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.3"
+  VERSION = "0.0.5"
 end

data/lib/tiktoken_ruby.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "tiktoken_ruby/version"
-require_relative "tiktoken_ruby/encoding.rb"
+require_relative "tiktoken_ruby/encoding"
 
 begin
   RUBY_VERSION =~ /(\d+\.\d+)/
@@ -12,7 +12,6 @@ end
 
 module Tiktoken
   class << self
-
     # Returns an encoding by name. If the encoding is not already loaded it will be loaded, but otherwise
     # it will reuse the instance of that type that was previous loaded
     # @param name [Symbol|String] The name of the encoding to load
@@ -34,7 +33,7 @@ module Tiktoken
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
     def encoding_for_model(model_name)
-      for prefix in PREFIX_MODELS
+      PREFIX_MODELS.each do |prefix|
         if model_name.to_s.start_with?("#{prefix}-")
           model_name = prefix
           break
@@ -65,7 +64,7 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base,
+      :cl100k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py#L13-L53
@@ -80,10 +79,10 @@ module Tiktoken
       "text-curie-001": "r50k_base",
       "text-babbage-001": "r50k_base",
       "text-ada-001": "r50k_base",
-      "davinci": "r50k_base",
-      "curie": "r50k_base",
-      "babbage": "r50k_base",
-      "ada": "r50k_base",
+      davinci: "r50k_base",
+      curie: "r50k_base",
+      babbage: "r50k_base",
+      ada: "r50k_base",
       # code
       "code-davinci-002": "p50k_base",
       "code-davinci-001": "p50k_base",
@@ -106,7 +105,7 @@ module Tiktoken
       "text-search-babbage-doc-001": "r50k_base",
       "text-search-ada-doc-001": "r50k_base",
       "code-search-babbage-code-001": "r50k_base",
-      "code-search-ada-code-001": "r50k_base",
+      "code-search-ada-code-001": "r50k_base"
     }
 
     # these are models that have a versioned models that are otherwise identical

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: x64-mingw-ucrt
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-03-21 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending