tiktoken_ruby 0.0.8-x64-mingw-ucrt → 0.0.11-x64-mingw-ucrt

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7aadcd951231ad7e517669fcea63c342e452cdf91b91a8702886b23503eafe84
-  data.tar.gz: 237430fb0dac4f90e7b9f2e8b09386bf8bd7a05ec98dc8ead0f604aca442f991
+  metadata.gz: 51b8e677f974f23d737f5c5887f3b4f93668d67d71cddd7dcea191e371df0a36
+  data.tar.gz: c2e1f275c3bd23f68a429b7d740406c97db332919bf677ab082215b68374147c
 SHA512:
-  metadata.gz: e46b673a18b037435bd99581bf2db9a94f57e8fc5638c842934f91f0f001884f888af686839c6f1814d45f7f91a4b53af67a836a31995467c427246f55e0c044
-  data.tar.gz: b7ee3cd8222cdc228e1c5392a551817259acdc0c6a27bf2c877dded7daea48abb237c9e1de860f9d3d1761d80916f3c31f56ede186a47db98509b6456544bb4c
+  metadata.gz: 84b43a46394129ecd43eef04210e2c9b4625a11b5e34f8dadb95eb8294b2ecb4f97d2d78af1f7a18cac2a9515c3d44be88e8ee682660ccff060296d7663aec40
+  data.tar.gz: cc97ec8e88052530033d883f6012b7ed585344d6355d13278fd3cb03974657e792a0d9afc0bc85a8de97bdebd242aa36dc99a272ad953dae7c272d48f10a9e15

data/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "[ruby]": {
+    "editor.defaultFormatter": "Shopify.ruby-lsp"
+  }
+}

data/Gemfile.lock

@@ -1,73 +1,73 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
-      rb_sys (= 0.9.87)
+    tiktoken_ruby (0.0.11)
+      rb_sys (= 0.9.106)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    diff-lcs (1.5.0)
-    json (2.7.1)
+    diff-lcs (1.5.1)
+    json (2.9.1)
     language_server-protocol (3.17.0.3)
     lint_roller (1.1.0)
     minitest (5.21.2)
-    parallel (1.24.0)
-    parser (3.3.0.4)
+    parallel (1.26.3)
+    parser (3.3.6.0)
       ast (~> 2.4.1)
       racc
-    racc (1.7.3)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.1.0)
-    rake-compiler (1.2.5)
+    rake (13.2.1)
+    rake-compiler (1.2.9)
       rake
-    rb_sys (0.9.87)
-    regexp_parser (2.9.0)
-    rexml (3.2.6)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rb_sys (0.9.106)
+    regexp_parser (2.10.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
-    rubocop (1.59.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
+    rubocop (1.69.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.4)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.36.2, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
-    rubocop-performance (1.20.2)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.37.0)
+      parser (>= 3.3.1.0)
+    rubocop-performance (1.23.0)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.33.0)
+    standard (1.43.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.59.0)
+      rubocop (~> 1.69.1)
       standard-custom (~> 1.0.0)
-      standard-performance (~> 1.3)
+      standard-performance (~> 1.6)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.3.1)
+    standard-performance (1.6.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.20.2)
-    unicode-display_width (2.5.0)
+      rubocop-performance (~> 1.23.0)
+    unicode-display_width (3.1.3)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
     yard (0.9.34)
     yard-doctest (0.1.17)
       minitest
@@ -89,4 +89,4 @@ DEPENDENCIES
   yard-doctest
 
 BUNDLED WITH
-   2.4.4
+   2.5.18

data/Rakefile

@@ -21,3 +21,9 @@ end
 task build: :compile
 
 task default: %i[compile spec standard]
+
+# Packaging default (non-precompiled) gem
+require "rubygems/package_task"
+gem_path = Gem::PackageTask.new(GEMSPEC).define
+desc "Package the Ruby gem"
+task "package" => [gem_path]

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.11"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,18 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
-    # that is also MIT licensed but by OpenAI
+    # that is also MIT licensed but by OpenAI;
+    # https://github.com/Congyuwang/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs#L50
+    # is the source of the mapping for the Rust library
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "chatgpt-4o-latest": "o200k_base",
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +125,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

data/script/release

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
+  echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
+  exit 1
+fi
+
+run_id=""
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    --run-id)
+      run_id="$2"
+      shift 2
+      ;;
+    *)
+      echo "Unknown parameter passed: $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "${run_id}" ]; then
+  echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
+  exit 1
+fi
+
+version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
+echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
+
+rm -rf pkg/cross-compiled
+gh run download "$run_id" -D pkg/cross-compiled
+
+for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
+  echo "Publishing $gem"
+  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
+done
+
+# last but not least, the uncompiled gem
+bundle exec rake package
+GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.11
 platform: x64-mingw-ucrt
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2025-01-08 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -21,6 +21,7 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".standard.yml"
+- ".vscode/settings.json"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -30,9 +31,10 @@ files:
 - lib/tiktoken_ruby.rb
 - lib/tiktoken_ruby/3.1/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
-- lib/tiktoken_ruby/3.3/tiktoken_ruby.so
+- lib/tiktoken_ruby/3.4/tiktoken_ruby.so
 - lib/tiktoken_ruby/encoding.rb
 - lib/tiktoken_ruby/version.rb
+- script/release
 - sig/tiktoken_ruby.rbs
 homepage: https://github.com/IAPark/tiktoken_ruby
 licenses:
@@ -52,14 +54,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.1'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.4.dev
+      version: 3.5.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.4.0
 requirements: []
-rubygems_version: 3.4.4
+rubygems_version: 3.5.23
 signing_key: 
 specification_version: 4
 summary: Ruby wrapper for Tiktoken