tiktoken_ruby 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df7ca54ce72c32b1a0a0b278553f35365bf283af271f98436b4d330f6fb05735
-  data.tar.gz: 5a225b39a85ce71779bb7e58c6f6c47eb8a884b813377783d833b27af53cbb9e
+  metadata.gz: 9997f4334fdaff90a631036be451cad90eb58ce6919a9592de2ca09d7f8baf9e
+  data.tar.gz: 160c540bf8a76278ebcabfe80c50e9fdb5802a0742d4692295e659ba8626b492
 SHA512:
-  metadata.gz: d15ec5fee05da4104e7111e237e5bf5e80291d226c51afd90fa580ebed87571140db4b06a17e39a82892489a6a932cb02d594a9cbf047315c5094bb4f25bd99b
-  data.tar.gz: 444bd7e59d054395689a3f45beb3c288262e1e9e1903dc565821780ad9929de6aabbdbd9f543506b3cc662a54835892e9b34b9ece259d6f3f27d5642a8f1b75a
+  metadata.gz: ea271891e7ca2fbfb637da4945c5f6da55f72b8c39efe20c4f83d5d059d2b9997f4344feadbd6109a29859729e4bba245fe51338e77c548e9e9a24bd1981a6e9
+  data.tar.gz: 3b6eeda82acaa6b6abb324e911196bce53d20a2bbec7750e07cb3b9d7a8381ed9ebd90b68a4e770e8bb1b4be332748216bc653d31389591348bc5192656e417f

data/Cargo.lock

@@ -115,12 +115,13 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "fancy-regex"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7493d4c459da9f84325ad297371a6b2b8a162800873a22e3b6b6512e61d18c05"
+checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
 dependencies = [
  "bit-set",
- "regex",
+ "regex-automata",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -178,9 +179,9 @@ dependencies = [
 
 [[package]]
 name = "magnus"
-version = "0.6.4"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1597ef40aa8c36be098249e82c9a20cf7199278ac1c1a1a995eeead6a184479"
+checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -264,18 +265,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.105"
+version = "0.9.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b3a1f3ce8e7c36d777d52fe7a99039fe4fea7c8ec355a4c4f3a17f92a14029f"
+checksum = "17b6efdbc8c1a22cb8b5d7ead0237c16c362c9ef6fbdc09e2d1040615b0f4cd0"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.105"
+version = "0.9.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e6b246c29c0809e1cbe60a1ba9e093da72a4676d02adc68469297d1e589bbf0"
+checksum = "e1d88c51e52f8636a5efc24ec5987056e64e48a91ed2a1af96cb5564686cc10f"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -399,9 +400,9 @@ dependencies = [
 
 [[package]]
 name = "tiktoken-rs"
-version = "0.5.9"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c314e7ce51440f9e8f5a497394682a57b7c323d0f4d0a6b1b13c429056e0e234"
+checksum = "44075987ee2486402f0808505dd65692163d243a337fc54363d49afac41087f6"
 dependencies = [
  "anyhow",
  "base64",
@@ -409,6 +410,7 @@ dependencies = [
  "fancy-regex",
  "lazy_static",
  "parking_lot",
+ "regex",
  "rustc-hash",
 ]
 

data/Gemfile.lock

@@ -1,73 +1,73 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.10)
-      rb_sys (>= 0.9.87)
+    tiktoken_ruby (0.0.11)
+      rb_sys (= 0.9.106)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    diff-lcs (1.5.0)
-    json (2.7.1)
+    diff-lcs (1.5.1)
+    json (2.9.1)
     language_server-protocol (3.17.0.3)
     lint_roller (1.1.0)
     minitest (5.21.2)
-    parallel (1.24.0)
-    parser (3.3.0.4)
+    parallel (1.26.3)
+    parser (3.3.6.0)
       ast (~> 2.4.1)
       racc
-    racc (1.7.3)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.1.0)
-    rake-compiler (1.2.5)
+    rake (13.2.1)
+    rake-compiler (1.2.9)
       rake
-    rb_sys (0.9.105)
-    regexp_parser (2.9.0)
-    rexml (3.2.6)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rb_sys (0.9.106)
+    regexp_parser (2.10.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
-    rubocop (1.59.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
+    rubocop (1.69.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.4)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.36.2, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
-    rubocop-performance (1.20.2)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.37.0)
+      parser (>= 3.3.1.0)
+    rubocop-performance (1.23.0)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.33.0)
+    standard (1.43.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.59.0)
+      rubocop (~> 1.69.1)
       standard-custom (~> 1.0.0)
-      standard-performance (~> 1.3)
+      standard-performance (~> 1.6)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.3.1)
+    standard-performance (1.6.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.20.2)
-    unicode-display_width (2.5.0)
+      rubocop-performance (~> 1.23.0)
+    unicode-display_width (3.1.3)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
     yard (0.9.34)
     yard-doctest (0.1.17)
       minitest

data/ext/tiktoken_ruby/Cargo.toml

@@ -10,6 +10,6 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.6.1" }
-rb-sys = { version = "0.9.87", features = ["stable-api-compiled-fallback"] }
-tiktoken-rs = { version = "0.5.9" }
+magnus = { version = "0.7.1" }
+rb-sys = { version = "0.9.106", features = ["stable-api-compiled-fallback"] }
+tiktoken-rs = { version = "0.6.0" }

data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -1,7 +1,8 @@
 use std::collections::HashSet;
 
-use crate::uncicode_error;
+use tiktoken_rs::Rank;
 
+use crate::uncicode_error;
 
 #[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")]
 pub struct CoreBPEWrapper {
@@ -13,11 +14,15 @@ impl CoreBPEWrapper {
         Self { core_bpe }
     }
 
-    pub fn encode_ordinary(&self, text: String) -> Vec<usize> {
+    pub fn encode_ordinary(&self, text: String) -> Vec<Rank> {
         self.core_bpe.encode_ordinary(text.as_str())
     }
 
-    pub fn encode(&self, text: String, allowed_special: magnus::RArray) -> Result<Vec<usize>, magnus::Error> {
+    pub fn encode(
+        &self,
+        text: String,
+        allowed_special: magnus::RArray,
+    ) -> Result<Vec<Rank>, magnus::Error> {
         let allowed_special: Vec<String> = allowed_special.to_vec()?;
         let allowed_special: Vec<&str> = allowed_special.iter().map(|s| s.as_str()).collect();
         let allowed_special: HashSet<&str> = HashSet::from_iter(allowed_special.iter().cloned());
@@ -25,20 +30,18 @@ impl CoreBPEWrapper {
         Ok(self.core_bpe.encode(text.as_str(), allowed_special))
     }
 
-    pub fn encode_with_special_tokens(&self, text: String) -> Vec<usize> {
+    pub fn encode_with_special_tokens(&self, text: String) -> Vec<Rank> {
         self.core_bpe.encode_with_special_tokens(text.as_str())
     }
 
-    pub fn decode(&self, ids: Vec<usize>) -> Result<String, magnus::Error> {
-        self.core_bpe.decode(ids)
-            .map_err(|e| {
-                let error = match uncicode_error() {
-                    Ok(error) => error,
-                    Err(e) => return e
-                };
-                
-                magnus::Error::new(error, e.to_string())
-        })
+    pub fn decode(&self, ids: Vec<Rank>) -> Result<String, magnus::Error> {
+        self.core_bpe.decode(ids).map_err(|e| {
+            let error = match uncicode_error() {
+                Ok(error) => error,
+                Err(e) => return e,
+            };
 
+            magnus::Error::new(error, e.to_string())
+        })
     }
 }

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.10"
+  VERSION = "0.0.11"
 end

data/lib/tiktoken_ruby.rb

@@ -69,9 +69,12 @@ module Tiktoken
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
-    # that is also MIT licensed but by OpenAI
+    # that is also MIT licensed but by OpenAI;
+    # https://github.com/Congyuwang/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs#L50
+    # is the source of the mapping for the Rust library
     MODEL_TO_ENCODING_NAME = {
       # chat
+      "chatgpt-4o-latest": "o200k_base",
       "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",

data/script/release

@@ -7,25 +7,35 @@ if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
   exit 1
 fi
 
+run_id=""
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    --run-id)
+      run_id="$2"
+      shift 2
+      ;;
+    *)
+      echo "Unknown parameter passed: $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "${run_id}" ]; then
+  echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
+  exit 1
+fi
+
 version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
-echo "Building tiktoken_ruby v$version"
-
-targets=(
-  "arm64-darwin"
-  "x86_64-darwin"
-  "aarch64-linux"
-  "x86_64-linux"
-  "x86_64-linux-musl"
-  "arm-linux"
-  "x64-mingw-ucrt"
-)
-
-# for target in "${targets[@]}"; do
-#   bundle exec rb-sys-dock -p "$target" --ruby-versions 3.2 --build
-# done
-
-for gem in pkg/tiktoken_ruby-"$version"-*.gem ; do
-    GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
+echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
+
+rm -rf pkg/cross-compiled
+gh run download "$run_id" -D pkg/cross-compiled
+
+for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
+  echo "Publishing $gem"
+  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
 done
 
 # last but not least, the uncompiled gem

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - IAPark
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-29 00:00:00.000000000 Z
+date: 2025-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.87
+        version: 0.9.106
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.87
+        version: 0.9.106
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
   it to OpenAI APIs.
@@ -61,7 +61,7 @@ metadata:
   homepage_uri: https://github.com/IAPark/tiktoken_ruby
   source_code_uri: https://github.com/IAPark/tiktoken_ruby
   documentation_uri: https://rubydoc.info/github/IAPark/tiktoken_ruby/main
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -76,8 +76,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 3.4.0
 requirements: []
-rubygems_version: 3.5.23
-signing_key: 
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Ruby wrapper for Tiktoken
 test_files: []