tiktoken_ruby 0.0.5-aarch64-linux → 0.0.7-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2007063b266c8a32f1c85a9413e99fc14eafff7bdd63684484fa003603d58bdd
-  data.tar.gz: 46141cccd6f7cb2ec67a485cb855b4d43558322b936cf3f73ab60498dd7daa7b
+  metadata.gz: f68fc30eaa385a8ac17a69977db4451eab822dafa100b9e88da7034626cbab69
+  data.tar.gz: 64f23c9545e14d338fd1618833cb32ece48791b7e4ef01409b51ee2f5f0e7783
 SHA512:
-  metadata.gz: 4d638035d0ea7943e8dafb72bd2482584a75e0c24a041b06799e6f332a20cad745e70a4e2635f26821bdbfe5f616eae4d42f137bd5e245fb75099081192f66eb
-  data.tar.gz: 750159ebc1fa648d743dceefda9100dd8557f5e7a679bd68232c06875146bfcb2d1c3af5d42f1ac0aa450bba4f42dacf75026ed1e6f08727e492d1b77ec791ae
+  metadata.gz: aacf9f703723a8e3c801d6f28709cce01cd370f546c6f25643d5f07119e4863db5d47b8005f91f916d32f4fd7403693ce01a08db85feb0cc3a3254271364ab28
+  data.tar.gz: ecd21ef829bc8482f8d9f100e23bf6903869983b9f7f949d9b7b2fb88148d16341b4a06c33698e3ea24901c6978d89c9f3d373cc558a9ef6b1c820e81d229f65

data/Gemfile

@@ -2,15 +2,11 @@
 
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in tiktoken_ruby.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
-
+gem "rake"
 gem "rake-compiler"
-
-gem "rspec", "~> 3.0"
-
-gem "standard", "~> 1.3"
-
-gem "yard-doctest", "~> 0.1.17"
+gem "rspec"
+gem "standard"
+gem "yard-doctest"
+gem "racc"

data/Gemfile.lock

@@ -1,64 +1,74 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.5)
-      rb_sys (~> 0.9.68)
+    tiktoken_ruby (0.0.7)
+      rb_sys (>= 0.9.86)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
     diff-lcs (1.5.0)
-    json (2.6.3)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
-    minitest (5.18.0)
-    parallel (1.22.1)
-    parser (3.2.1.1)
+    lint_roller (1.1.0)
+    minitest (5.21.2)
+    parallel (1.24.0)
+    parser (3.3.0.4)
       ast (~> 2.4.1)
+      racc
+    racc (1.7.3)
     rainbow (3.1.1)
-    rake (13.0.6)
-    rake-compiler (1.2.1)
+    rake (13.1.0)
+    rake-compiler (1.2.5)
       rake
-    rb_sys (0.9.68)
-    regexp_parser (2.7.0)
-    rexml (3.2.5)
+    rb_sys (0.9.86)
+    regexp_parser (2.9.0)
+    rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.4)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.48.1)
+    rspec-support (3.12.1)
+    rubocop (1.59.0)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.27.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
-    rubocop-performance (1.16.0)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.25.1)
+    standard (1.33.0)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (= 1.48.1)
-      rubocop-performance (= 1.16.0)
-    unicode-display_width (2.4.2)
-    webrick (1.7.0)
-    yard (0.9.28)
-      webrick (~> 1.7.0)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.59.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.3)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.3.1)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.20.2)
+    unicode-display_width (2.5.0)
+    yard (0.9.34)
     yard-doctest (0.1.17)
       minitest
       yard
@@ -70,12 +80,13 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  rake (~> 13.0)
+  racc
+  rake
   rake-compiler
-  rspec (~> 3.0)
-  standard (~> 1.3)
+  rspec
+  standard
   tiktoken_ruby!
-  yard-doctest (~> 0.1.17)
+  yard-doctest
 
 BUNDLED WITH
    2.4.6

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.5"
+  VERSION = "0.0.7"
 end

data/lib/tiktoken_ruby.rb

@@ -33,17 +33,15 @@ module Tiktoken
     #   enc = Tiktoken.encoding_for_model("gpt-4")
     #   enc.encode("hello world").length #=> 2
     def encoding_for_model(model_name)
-      PREFIX_MODELS.each do |prefix|
-        if model_name.to_s.start_with?("#{prefix}-")
-          model_name = prefix
-          break
-        end
+      if MODEL_TO_ENCODING_NAME.key?(model_name.to_sym)
+        return get_encoding(MODEL_TO_ENCODING_NAME[model_name.to_sym])
       end
 
-      encoding_name = MODEL_TO_ENCODING_NAME[model_name.to_sym]
-      return nil unless encoding_name
-
-      get_encoding(encoding_name)
+      MODEL_PREFIX_TO_ENCODING.each do |prefix, encoding|
+        if model_name.start_with?(prefix.to_s)
+          return get_encoding(encoding)
+        end
+      end
     end
 
     # Lists all the encodings that are supported
@@ -67,12 +65,22 @@ module Tiktoken
       :cl100k_base
     ]
 
-    # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py#L13-L53
+    # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
+      # chat
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
-      # text
+      "gpt-35-turbo": "cl100k_base",  # Azure deployment name
+      # base
+      "davinci-002": "cl100k_base",
+      "babbage-002": "cl100k_base",
+      # embeddings
+      "text-embedding-ada-002": "cl100k_base",
+      "text-embedding-3-small": "cl100k_base",
+      "text-embedding-3-large": "cl100k_base",
+      # DEPRECATED MODELS
+      # text (DEPRECATED)
       "text-davinci-003": "p50k_base",
       "text-davinci-002": "p50k_base",
       "text-davinci-001": "r50k_base",
@@ -83,19 +91,17 @@ module Tiktoken
       curie: "r50k_base",
       babbage: "r50k_base",
       ada: "r50k_base",
-      # code
+      # code (DEPRECATED)
       "code-davinci-002": "p50k_base",
       "code-davinci-001": "p50k_base",
       "code-cushman-002": "p50k_base",
       "code-cushman-001": "p50k_base",
       "davinci-codex": "p50k_base",
       "cushman-codex": "p50k_base",
-      # edit
+      # edit (DEPRECATED)
       "text-davinci-edit-001": "p50k_edit",
       "code-davinci-edit-001": "p50k_edit",
-      # embeddings
-      "text-embedding-ada-002": "cl100k_base",
-      # old embeddings
+      # old embeddings (DEPRECATED)
       "text-similarity-davinci-001": "r50k_base",
       "text-similarity-curie-001": "r50k_base",
       "text-similarity-babbage-001": "r50k_base",
@@ -105,10 +111,21 @@ module Tiktoken
       "text-search-babbage-doc-001": "r50k_base",
       "text-search-ada-doc-001": "r50k_base",
       "code-search-babbage-code-001": "r50k_base",
-      "code-search-ada-code-001": "r50k_base"
+      "code-search-ada-code-001": "r50k_base",
+      # open source
+      gpt2: "gpt2"
     }
 
-    # these are models that have a versioned models that are otherwise identical
-    PREFIX_MODELS = ["gpt-4", "gpt-3.5-turbo"]
+    MODEL_PREFIX_TO_ENCODING = {
+      # chat
+      "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
+      "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
+      "gpt-35-turbo-": "cl100k_base",  # Azure deployment name
+      # fine-tuned
+      "ft:gpt-4": "cl100k_base",
+      "ft:gpt-3.5-turbo": "cl100k_base",
+      "ft:davinci-002": "cl100k_base",
+      "ft:babbage-002": "cl100k_base"
+    }
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: aarch64-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-11 00:00:00.000000000 Z
+date: 2024-02-14 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -31,6 +31,7 @@ files:
 - lib/tiktoken_ruby/3.0/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.1/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
+- lib/tiktoken_ruby/3.3/tiktoken_ruby.so
 - lib/tiktoken_ruby/encoding.rb
 - lib/tiktoken_ruby/version.rb
 - sig/tiktoken_ruby.rbs
@@ -52,7 +53,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.0'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.3.dev
+      version: 3.4.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="