tiktoken_ruby 0.0.8-x86_64-linux-musl → 0.0.10-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34a2818779ff0903c9b7bdf5e7132440cbee1980475b83dc899ecc019c353cc6
-  data.tar.gz: 42f9069dca45fc54ae7d2ee9711dc8028a2d305f6caeb7db6024b139371bffbd
+  metadata.gz: 2e467471e3d4c0369d1d335fb546a6dee2501cdca88f6108fe5bf5be9ddbe90a
+  data.tar.gz: 652a4b9020ba88197f0fb2fd8c5d29c132196ecff5b22f6332463702e9960cf2
 SHA512:
-  metadata.gz: dd60905cfe30d24ef26c00f46604da82530d58fc24262da51d0db6b81fba92c66b935cb570d84879c463a0adeb4c7e2bf1bbfad89fd951cd4fa68429af0b2a87
-  data.tar.gz: c952a88e2bce633514d77ce0f5bb380d1a8d622225beab41ea3e29053658f27f4774761224f45d8c963068583274f32f6d66e8b06dec1ef9d267e4e8b9a3c80f
+  metadata.gz: ef153b261cdfd66d8e2aaa8f3d6d3b47e3796f9987a6a9d0c6e34ff7326c7d9f2cd89513deda61aeaa6348f3aa127e85a0ccbef4f64433b6f07fd2689ffd8d6b
+  data.tar.gz: fa22fd4dc60675eb474c78809068d18dcc20796e2345af0228b557aea57ea1913406446ebf1a5024a3e5f5d6aaa3e644638349ea9bed308b888e3a26a54c8a2b

data/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "[ruby]": {
+    "editor.defaultFormatter": "Shopify.ruby-lsp"
+  }
+}

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.8)
-      rb_sys (= 0.9.87)
+    tiktoken_ruby (0.0.10)
+      rb_sys (>= 0.9.87)
 
 GEM
   remote: https://rubygems.org/
@@ -22,7 +22,7 @@ GEM
     rake (13.1.0)
     rake-compiler (1.2.5)
       rake
-    rb_sys (0.9.87)
+    rb_sys (0.9.105)
     regexp_parser (2.9.0)
     rexml (3.2.6)
     rspec (3.12.0)
@@ -89,4 +89,4 @@ DEPENDENCIES
   yard-doctest
 
 BUNDLED WITH
-   2.4.4
+   2.5.18

data/Rakefile

@@ -21,3 +21,9 @@ end
 task build: :compile
 
 task default: %i[compile spec standard]
+
+# Packaging default (non-precompiled) gem
+require "rubygems/package_task"
+gem_path = Gem::PackageTask.new(GEMSPEC).define
+desc "Package the Ruby gem"
+task "package" => [gem_path]

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.8"
+  VERSION = "0.0.10"
 end

data/lib/tiktoken_ruby.rb

@@ -64,13 +64,15 @@ module Tiktoken
       :r50k_base,
       :p50k_base,
       :p50k_edit,
-      :cl100k_base
+      :cl100k_base,
+      :o200k_base
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
     # that is also MIT licensed but by OpenAI
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
@@ -120,6 +122,7 @@ module Tiktoken
 
     MODEL_PREFIX_TO_ENCODING = {
       # chat
+      "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name

data/script/release

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
+  echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
+  exit 1
+fi
+
+version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
+echo "Building tiktoken_ruby v$version"
+
+targets=(
+  "arm64-darwin"
+  "x86_64-darwin"
+  "aarch64-linux"
+  "x86_64-linux"
+  "x86_64-linux-musl"
+  "arm-linux"
+  "x64-mingw-ucrt"
+)
+
+for target in "${targets[@]}"; do
+  bundle exec rb-sys-dock -p "$target" --ruby-versions 3.2 --build
+done
+
+for gem in pkg/tiktoken_ruby-"$version"-*.gem ; do
+    GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
+done
+
+# last but not least
+bundle exec rake package
+GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.10
 platform: x86_64-linux-musl
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-04 00:00:00.000000000 Z
+date: 2024-12-29 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -21,6 +21,7 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".standard.yml"
+- ".vscode/settings.json"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -28,11 +29,10 @@ files:
 - Rakefile
 - doctest_helper.rb
 - lib/tiktoken_ruby.rb
-- lib/tiktoken_ruby/3.1/tiktoken_ruby.so
-- lib/tiktoken_ruby/3.2/tiktoken_ruby.so
-- lib/tiktoken_ruby/3.3/tiktoken_ruby.so
 - lib/tiktoken_ruby/encoding.rb
+- lib/tiktoken_ruby/tiktoken_ruby.so
 - lib/tiktoken_ruby/version.rb
+- script/release
 - sig/tiktoken_ruby.rbs
 homepage: https://github.com/IAPark/tiktoken_ruby
 licenses:
@@ -49,17 +49,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.4.dev
+      version: 3.3.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.4.0
 requirements: []
-rubygems_version: 3.4.4
+rubygems_version: 3.5.23
 signing_key: 
 specification_version: 4
 summary: Ruby wrapper for Tiktoken