tiktoken_ruby 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee8bc941d0d8de83fb99c33acf9c831d2bd7542f54726a2ff3ec8acf41fdaab8
-  data.tar.gz: ca70753b8823328f08123328ad4409b351e99a2d564e0fc8c8e779a4b378a2b3
+  metadata.gz: de2d6e8e83771f2ef51351e019e9cebc7163a1775bfc0e812da58371574b9b63
+  data.tar.gz: fe2c629e8b435a181bfa4524655bfa137f1660061ab4f780e0edc15a11a7538d
 SHA512:
-  metadata.gz: '031225799558c1bd563ca6d40b7525f956155e7dabe899c0a5c440eedc29727fb7e8504136251623ca63ad0d2877aadf4673ceb6cda5c440aa8f9362d119d119'
-  data.tar.gz: 9df34194052e5bdf958d214e6a0d5a9cdda9341401e8226956dc2de05406d22b67a3c7a67fd998af95bf981f1ced34daf8084aeaf9afec0f6e409f416eaa362a
+  metadata.gz: bbf721963e873464fae055d23308068fcdac8db4e27dea28653c3fc017f0803da59b4c549a0fb6b6339f79bc4379e2913e1c47fa25f9894ebde840b23aa81edb
+  data.tar.gz: 1275369f56a2498ce39c5a2b259efdab0684ae834c6d769bfe23d7190c69917a22d46dc7b7675cd15c8a44dcfa8f7c3be2df850035236fa33f8e55afd31db42b

data/Cargo.lock

@@ -31,11 +31,11 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
 
 [[package]]
 name = "bindgen"
-version = "0.60.1"
+version = "0.66.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7"
 dependencies = [
- "bitflags",
+ "bitflags 2.4.0",
  "cexpr",
  "clang-sys",
  "lazy_static",
@@ -46,6 +46,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
+ "syn",
 ]
 
 [[package]]
@@ -69,6 +70,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+
 [[package]]
 name = "bstr"
 version = "1.4.0"
@@ -163,20 +170,21 @@ dependencies = [
 
 [[package]]
 name = "magnus"
-version = "0.4.4"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc87660cd7daa49fddbfd524c836de54d5c927d520cd163f43700c5087c57d6c"
+checksum = "0516897a45f8ce8270a8910bcb94cd83538b19b6ae3a0c281a765df170b64695"
 dependencies = [
  "magnus-macros",
  "rb-sys",
  "rb-sys-env",
+ "seq-macro",
 ]
 
 [[package]]
 name = "magnus-macros"
-version = "0.3.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206cb23bfeea05180c97522ef6a3e52a4eb17b0ed2f30ee3ca9c4f994d2378ae"
+checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -242,36 +250,36 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.52"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.26"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
 name = "rb-sys"
-version = "0.9.68"
+version = "0.9.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4783528e031c3902524cfe685e0008d644e8b34bad4d1a19c1f39f4394d0777b"
+checksum = "a57240b308b155b09dce81e32829966a99f52d1088b45957e4283e526c5317a1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.68"
+version = "0.9.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c04474335dd597126d20fe90d94504b3582a30cbc36bc72c7d6ed3ef8cf168"
+checksum = "f24ce877a4c5d07f06f6aa6fec3ac95e4b357b9f73b0f5445d8cbb7266d410e8"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -294,7 +302,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -332,6 +340,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "seq-macro"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
+
 [[package]]
 name = "serde"
 version = "1.0.157"
@@ -358,9 +372,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "syn"
-version = "1.0.109"
+version = "2.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -386,6 +400,7 @@ name = "tiktoken_ruby"
 version = "0.1.0"
 dependencies = [
  "magnus",
+ "rb-sys",
  "tiktoken-rs",
 ]
 

data/Gemfile

@@ -2,15 +2,11 @@
 
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in tiktoken_ruby.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
-
+gem "rake"
 gem "rake-compiler"
-
-gem "rspec", "~> 3.0"
-
-gem "standard", "~> 1.3"
-
-gem "yard-doctest", "~> 0.1.17"
+gem "rspec"
+gem "standard"
+gem "yard-doctest"
+gem "racc"

data/Gemfile.lock

@@ -1,64 +1,74 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.5)
-      rb_sys (~> 0.9.68)
+    tiktoken_ruby (0.0.7)
+      rb_sys (>= 0.9.86)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
     diff-lcs (1.5.0)
-    json (2.6.3)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
-    minitest (5.18.0)
-    parallel (1.22.1)
-    parser (3.2.1.1)
+    lint_roller (1.1.0)
+    minitest (5.21.2)
+    parallel (1.24.0)
+    parser (3.3.0.4)
       ast (~> 2.4.1)
+      racc
+    racc (1.7.3)
     rainbow (3.1.1)
-    rake (13.0.6)
-    rake-compiler (1.2.1)
+    rake (13.1.0)
+    rake-compiler (1.2.5)
       rake
-    rb_sys (0.9.68)
-    regexp_parser (2.7.0)
-    rexml (3.2.5)
+    rb_sys (0.9.86)
+    regexp_parser (2.9.0)
+    rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.4)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.48.1)
+    rspec-support (3.12.1)
+    rubocop (1.59.0)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.27.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
-    rubocop-performance (1.16.0)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.25.1)
+    standard (1.33.0)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (= 1.48.1)
-      rubocop-performance (= 1.16.0)
-    unicode-display_width (2.4.2)
-    webrick (1.7.0)
-    yard (0.9.28)
-      webrick (~> 1.7.0)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.59.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.3)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.3.1)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.20.2)
+    unicode-display_width (2.5.0)
+    yard (0.9.34)
     yard-doctest (0.1.17)
       minitest
       yard
@@ -70,12 +80,13 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  rake (~> 13.0)
+  racc
+  rake
   rake-compiler
-  rspec (~> 3.0)
-  standard (~> 1.3)
+  rspec
+  standard
   tiktoken_ruby!
-  yard-doctest (~> 0.1.17)
+  yard-doctest
 
 BUNDLED WITH
    2.4.6

data/ext/tiktoken_ruby/Cargo.toml

@@ -10,5 +10,6 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.4" }
+magnus = { version = "0.6.1" }
+rb-sys = { version = "*", features = ["stable-api-compiled-fallback"] }
 tiktoken-rs = { git = "https://github.com/IAPark/tiktoken-rs.git" }

data/ext/tiktoken_ruby/src/lib.rs

@@ -1,7 +1,7 @@
 mod core_bpe_wrapper;
 
 use core_bpe_wrapper::CoreBPEWrapper;
-use magnus::{define_module, function, prelude::*, Error, method, class, RModule, ExceptionClass};
+use magnus::{class, define_module, function, method, prelude::*, Error, ExceptionClass, RModule};
 
 fn r50k_base() -> CoreBPEWrapper {
     let core_bpe = tiktoken_rs::r50k_base().unwrap();
@@ -38,14 +38,18 @@ fn init() -> Result<(), Error> {
     factory_module.define_singleton_method("p50k_edit", function!(p50k_edit, 0))?;
     factory_module.define_singleton_method("cl100k_base", function!(cl100k_base, 0))?;
 
-
     let ext_module = module.define_module("Ext")?;
     let bpe_class = ext_module.define_class("CoreBPE", class::object())?;
 
-    bpe_class.define_method("encode_ordinary", method!(CoreBPEWrapper::encode_ordinary, 1))?;
+    bpe_class.define_method(
+        "encode_ordinary",
+        method!(CoreBPEWrapper::encode_ordinary, 1),
+    )?;
     bpe_class.define_method("encode", method!(CoreBPEWrapper::encode, 2))?;
-    bpe_class.define_method("encode_with_special_tokens", method!(CoreBPEWrapper::encode_with_special_tokens, 1))?;
-
+    bpe_class.define_method(
+        "encode_with_special_tokens",
+        method!(CoreBPEWrapper::encode_with_special_tokens, 1),
+    )?;
 
     bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?;
     Ok(())

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.5"
+  VERSION = "0.0.7"
 end

data/lib/tiktoken_ruby.rb

@@ -33,17 +33,15 @@ module Tiktoken
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
     def encoding_for_model(model_name)
-      PREFIX_MODELS.each do |prefix|
-        if model_name.to_s.start_with?("#{prefix}-")
-          model_name = prefix
-          break
-        end
+      if MODEL_TO_ENCODING_NAME.key?(model_name.to_sym)
+        return get_encoding(MODEL_TO_ENCODING_NAME[model_name.to_sym])
       end
 
-      encoding_name = MODEL_TO_ENCODING_NAME[model_name.to_sym]
-      return nil unless encoding_name
-
-      get_encoding(encoding_name)
+      MODEL_PREFIX_TO_ENCODING.each do |prefix, encoding|
+        if model_name.start_with?(prefix.to_s)
+          return get_encoding(encoding)
+        end
+      end
     end
 
     # Lists all the encodings that are supported
@@ -67,12 +65,22 @@ module Tiktoken
       :cl100k_base
     ]
 
-    # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py#L13-L53
+    # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
+      # chat
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
-      # text
+      "gpt-35-turbo": "cl100k_base",  # Azure deployment name
+      # base
+      "davinci-002": "cl100k_base",
+      "babbage-002": "cl100k_base",
+      # embeddings
+      "text-embedding-ada-002": "cl100k_base",
+      "text-embedding-3-small": "cl100k_base",
+      "text-embedding-3-large": "cl100k_base",
+      # DEPRECATED MODELS
+      # text (DEPRECATED)
       "text-davinci-003": "p50k_base",
       "text-davinci-002": "p50k_base",
       "text-davinci-001": "r50k_base",
@@ -83,19 +91,17 @@ module Tiktoken
       curie: "r50k_base",
       babbage: "r50k_base",
       ada: "r50k_base",
-      # code
+      # code (DEPRECATED)
       "code-davinci-002": "p50k_base",
       "code-davinci-001": "p50k_base",
       "code-cushman-002": "p50k_base",
       "code-cushman-001": "p50k_base",
       "davinci-codex": "p50k_base",
       "cushman-codex": "p50k_base",
-      # edit
+      # edit (DEPRECATED)
       "text-davinci-edit-001": "p50k_edit",
       "code-davinci-edit-001": "p50k_edit",
-      # embeddings
-      "text-embedding-ada-002": "cl100k_base",
-      # old embeddings
+      # old embeddings (DEPRECATED)
       "text-similarity-davinci-001": "r50k_base",
       "text-similarity-curie-001": "r50k_base",
       "text-similarity-babbage-001": "r50k_base",
@@ -105,10 +111,21 @@ module Tiktoken
       "text-search-babbage-doc-001": "r50k_base",
       "text-search-ada-doc-001": "r50k_base",
       "code-search-babbage-code-001": "r50k_base",
-      "code-search-ada-code-001": "r50k_base"
+      "code-search-ada-code-001": "r50k_base",
+      # open source
+      gpt2: "gpt2"
     }
 
-    # these are models that have a versioned models that are otherwise identical
-    PREFIX_MODELS = ["gpt-4", "gpt-3.5-turbo"]
+    MODEL_PREFIX_TO_ENCODING = {
+      # chat
+      "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
+      "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
+      "gpt-35-turbo-": "cl100k_base",  # Azure deployment name
+      # fine-tuned
+      "ft:gpt-4": "cl100k_base",
+      "ft:gpt-3.5-turbo": "cl100k_base",
+      "ft:davinci-002": "cl100k_base",
+      "ft:babbage-002": "cl100k_base"
+    }
   end
 end

data/tiktoken_ruby.gemspec

@@ -7,12 +7,10 @@ Gem::Specification.new do |spec|
   spec.version = Tiktoken::VERSION
   spec.authors = ["IAPark"]
   spec.email = ["isaac.a.park@gmail.com"]
-
   spec.summary = "Ruby wrapper for Tiktoken"
   spec.description = "An unofficial Ruby wrapper for Tiktoken, " \
     "a BPE tokenizer written by and used by OpenAI. It can be used to " \
     "count the number of tokens in text before sending it to OpenAI APIs."
-
   spec.homepage = "https://github.com/IAPark/tiktoken_ruby"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 2.7.0"
@@ -22,11 +20,6 @@ Gem::Specification.new do |spec|
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/IAPark/tiktoken_ruby"
   spec.metadata["documentation_uri"] = "https://rubydoc.info/github/IAPark/tiktoken_ruby/main"
-
-  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(__dir__) do
     `git ls-files -z`.split("\x0").reject do |f|
       (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
@@ -36,9 +29,5 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   spec.extensions = ["ext/tiktoken_ruby/extconf.rb"]
-
-  spec.add_dependency "rb_sys", "~> 0.9.68"
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
+  spec.add_dependency "rb_sys", ">= 0.9.86"
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - IAPark
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-11 00:00:00.000000000 Z
+date: 2024-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.68
+        version: 0.9.86
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.68
+        version: 0.9.86
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
   it to OpenAI APIs.