tiktoken_ruby 0.0.13 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb1bf0a15715fcd42136ff3378a0b2b5676a06dd817fbbe59e83287c9a7e996
-  data.tar.gz: 30172f2328b4c98c62963c5ddc5afda840eb0f938e632d40e5aafc1eb7521693
+  metadata.gz: 6ad1b0b7629c92e355ac26b13f4c9ecb00cf96a047e8a50ca88bac5b2efc05c8
+  data.tar.gz: 99aa54177dfcfa435c9539b2210eb3b1af3c84ba76c1b5e1e31728f1b4ca041b
 SHA512:
-  metadata.gz: 2db1488ee96bcfc94cbe48617c1f37f54ef74e532dafd50d4c89fc56940dcd3ca83cc3bd4c49ebc8edae71998e37046af62c8e67a8c2a97b3d316ec534acec62
-  data.tar.gz: 9ce2ca75ba2ba65c375c562717cdd19c6a3e41c127964775ce49eb1e3664954c530348eba71888b037d7b3ae2568f6fe1b52bd887b27e72294e5530ab54f8f8f
+  metadata.gz: 5cfaaa543821c6bc80cc39b19bb71d4e3b0dbec058de5027b47ecbcaacbeef1436532a52fe63a4407436dfa7d506252150ae4b07656fbdb969ce7eee694f50c2
+  data.tar.gz: 15c5f6aa872daf9900ce5c10b049292ae80d48e44ace0251643749a8b925b9dc89cfa34ccc1706f36db7bf6a832466fc530e502e280adaacd234044849432ff7

data/.vscode/settings.json

@@ -4,5 +4,8 @@
   },
   "[markdown]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[github-actions-workflow]": {
+    "editor.defaultFormatter": "redhat.vscode-yaml"
   }
 }

data/CHANGELOG.md

@@ -0,0 +1,35 @@
+# [v0.0.15] - 06-01-2026
+## What's Changed
+* No longer true by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/96
+* Test build on version change by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/97
+* Bump actions/cache from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/98
+* Bump rb-sys from 0.9.117 to 0.9.123 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/99
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/100
+* Add Ruby 4.0 to build matrix by @chubchenko in https://github.com/IAPark/tiktoken_ruby/pull/101
+* Support Ruby 4 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/102
+
+## New Contributors
+* @chubchenko made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/101
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.14.1...v0.0.15
+# [v0.0.14.1] - 20-12-2025
+## What's Changed
+* Cut v0.0.12 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/78
+* Bump magnus from 0.8.0 to 0.8.1 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/83
+* Bump actions/checkout from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/82
+* Bump standard from 1.50.0 to 1.51.1 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/81
+* Bump actions/upload-artifact from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/84
+* Bump magnus from 0.8.1 to 0.8.2 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/85
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/86
+* Support  by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/87
+* Bump actions/checkout from 5 to 6 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/89
+* Bump standard from 1.51.1 to 1.52.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/88
+* release GVL while encoding / decoding tokens by @tenderworks in https://github.com/IAPark/tiktoken_ruby/pull/90
+* Drop Ruby 3.1 support; automate release process by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/92
+* Rewrite history by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/93
+* Force workflow rebuild by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/94
+
+## New Contributors
+* @tenderworks made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/90
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.12...v0.0.14.1

data/Cargo.lock

@@ -226,18 +226,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.117"
+version = "0.9.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f900d1ce4629a2ebffaf5de74bd8f9c1188d4c5ed406df02f97e22f77a006f44"
+checksum = "45fb1a185af97ee456f1c9e56dbe6e2e662bec4fdeaf83c4c28e0e6adfb18816"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.117"
+version = "0.9.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef1e9c857028f631056bcd6d88cec390c751e343ce2223ddb26d23eb4a151d59"
+checksum = "a58ebd02d7a6033e6a5f6f8d150c1e9f16506039092b84a73e6bedce6d3adf41"
 dependencies = [
  "bindgen",
  "lazy_static",

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.13)
+    tiktoken_ruby (0.0.15)
       rb_sys (~> 0.9)
 
 GEM
@@ -9,23 +9,24 @@ GEM
   specs:
     ast (2.4.3)
     diff-lcs (1.6.2)
-    json (2.15.0)
+    json (2.16.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
-    minitest (5.25.5)
+    minitest (6.0.1)
+      prism (~> 1.5)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.5.1)
+    prism (1.6.0)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.3.1)
-    rake-compiler (1.3.0)
+    rake-compiler (1.3.1)
       rake
-    rake-compiler-dock (1.9.1)
-    rb_sys (0.9.117)
-      rake-compiler-dock (= 1.9.1)
+    rake-compiler-dock (1.10.0)
+    rb_sys (0.9.123)
+      rake-compiler-dock (= 1.10.0)
     regexp_parser (2.11.3)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)
@@ -40,7 +41,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    rubocop (1.80.2)
+    rubocop (1.81.7)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -48,10 +49,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.48.0)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     rubocop-performance (1.25.0)
@@ -59,10 +60,10 @@ GEM
       rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.38.0, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.51.1)
+    standard (1.52.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.80.2)
+      rubocop (~> 1.81.7)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)
@@ -73,7 +74,7 @@ GEM
       rubocop-performance (~> 1.25.0)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
-    unicode-emoji (4.1.0)
+    unicode-emoji (4.2.0)
     yard (0.9.37)
     yard-doctest (0.1.17)
       minitest

data/README.md

@@ -36,11 +36,44 @@ enc = Tiktoken.encoding_for_model("gpt-4")
 enc.encode("hello world").length #=> 2
 ```
 
+### Encoding methods
+
+There are three methods for encoding text:
+
+- `encode_ordinary(text)` - Encodes text, always treating special tokens as ordinary text
+- `encode(text, allowed_special: [])` - Encodes text, treating special tokens as text unless listed in `allowed_special`
+- `encode_with_special_tokens(text)` - Encodes text, recognizing and parsing all special tokens
+
+**Special tokens** are control sequences used by OpenAI models, such as `<|endoftext|>`, `<|fim_prefix|>`, `<|fim_middle|>`, and `<|fim_suffix|>`. The encoding methods differ in how they handle these sequences:
+
+```ruby
+enc = Tiktoken.get_encoding("cl100k_base")
+text = "Hello<|endoftext|>World"
+
+# encode_ordinary: treats <|endoftext|> as literal characters (9 tokens)
+enc.encode_ordinary(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode: same as encode_ordinary by default
+enc.encode(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode with allowed_special: recognizes the specified special token (3 tokens)
+enc.encode(text, allowed_special: ["<|endoftext|>"])
+#=> [9906, 100257, 10343]
+
+# encode_with_special_tokens: recognizes ALL special tokens (3 tokens)
+enc.encode_with_special_tokens(text)
+#=> [9906, 100257, 10343]
+```
+
+All methods round-trip correctly through `decode`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Contributing
 

data/ext/tiktoken_ruby/Cargo.toml

@@ -11,5 +11,5 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8.2" }
-rb-sys = { version = "0.9.117", features = ["stable-api-compiled-fallback"] }
+rb-sys = { version = "0.9.123", features = ["stable-api-compiled-fallback"] }
 tiktoken-rs = { version = "0.9.0" }

data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::ffi::c_void;
 
 use tiktoken_rs::Rank;
 
@@ -9,13 +10,82 @@ pub struct CoreBPEWrapper {
     core_bpe: tiktoken_rs::CoreBPE,
 }
 
+struct EncodeOrdinaryData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    text: String,
+    result: Vec<Rank>,
+}
+
+struct EncodeData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    text: String,
+    allowed_special: HashSet<String>,
+    result: Vec<Rank>,
+}
+
+struct EncodeSpecialData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    text: String,
+    result: Vec<Rank>,
+}
+
+struct DecodeData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    ids: Vec<Rank>,
+    result: Result<String, String>,
+}
+
+unsafe extern "C" fn encode_ordinary_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut EncodeOrdinaryData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.encode_ordinary(&data.text);
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut EncodeData);
+    let core_bpe = &*data.core_bpe;
+    let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect();
+    data.result = core_bpe.encode(&data.text, &allowed_special).0;
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn encode_special_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut EncodeSpecialData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.encode_with_special_tokens(&data.text);
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut DecodeData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.decode(std::mem::take(&mut data.ids)).map_err(|e| e.to_string());
+    std::ptr::null_mut()
+}
+
 impl CoreBPEWrapper {
     pub fn new(core_bpe: tiktoken_rs::CoreBPE) -> Self {
         Self { core_bpe }
     }
 
     pub fn encode_ordinary(&self, text: String) -> Vec<Rank> {
-        self.core_bpe.encode_ordinary(text.as_str())
+        let mut data = EncodeOrdinaryData {
+            core_bpe: &self.core_bpe as *const _,
+            text,
+            result: Vec::new(),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(encode_ordinary_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        data.result
     }
 
     pub fn encode(
@@ -24,24 +94,68 @@ impl CoreBPEWrapper {
         allowed_special: magnus::RArray,
     ) -> Result<Vec<Rank>, magnus::Error> {
         let allowed_special: Vec<String> = allowed_special.to_vec()?;
-        let allowed_special: Vec<&str> = allowed_special.iter().map(|s| s.as_str()).collect();
-        let allowed_special: HashSet<&str> = HashSet::from_iter(allowed_special.iter().cloned());
 
-        Ok(self.core_bpe.encode(text.as_str(), &allowed_special).0)
+        let mut data = EncodeData {
+            core_bpe: &self.core_bpe as *const _,
+            text,
+            allowed_special: HashSet::from_iter(allowed_special),
+            result: Vec::new(),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(encode_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        Ok(data.result)
     }
 
     pub fn encode_with_special_tokens(&self, text: String) -> Vec<Rank> {
-        self.core_bpe.encode_with_special_tokens(text.as_str())
+        let mut data = EncodeSpecialData {
+            core_bpe: &self.core_bpe as *const _,
+            text,
+            result: Vec::new(),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(encode_special_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        data.result
     }
 
     pub fn decode(&self, ids: Vec<Rank>) -> Result<String, magnus::Error> {
-        self.core_bpe.decode(ids).map_err(|e| {
+        let mut data = DecodeData {
+            core_bpe: &self.core_bpe as *const _,
+            ids,
+            result: Err(String::new()),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(decode_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        data.result.map_err(|e| {
             let error = match uncicode_error() {
                 Ok(error) => error,
                 Err(e) => return e,
             };
 
-            magnus::Error::new(error, e.to_string())
+            magnus::Error::new(error, e)
         })
     }
 }

data/lib/tiktoken_ruby/encoding.rb

@@ -40,6 +40,13 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode(text, allowed_special)
   end
 
+  # Encodes the text as a list of integer tokens, including special tokens.
+  # @param text [String] The text to encode
+  # @return [Array<Integer>] The encoded tokens
+  def encode_with_special_tokens(text)
+    @ext_base_bpe.encode_with_special_tokens(text)
+  end
+
   # Decodes the tokens back into text
   # @param tokens [Array<Integer>] The tokens to decode
   # @return [String] The decoded text

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.13"
+  VERSION = "0.0.15"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.15
 platform: ruby
 authors:
 - IAPark
@@ -36,6 +36,7 @@ files:
 - ".rspec"
 - ".standard.yml"
 - ".vscode/settings.json"
+- CHANGELOG.md
 - Cargo.lock
 - Cargo.toml
 - Gemfile
@@ -51,7 +52,6 @@ files:
 - lib/tiktoken_ruby.rb
 - lib/tiktoken_ruby/encoding.rb
 - lib/tiktoken_ruby/version.rb
-- script/release
 - sig/tiktoken_ruby.rbs
 homepage: https://github.com/IAPark/tiktoken_ruby
 licenses:
@@ -67,14 +67,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: 3.4.0
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Ruby wrapper for Tiktoken
 test_files: []

data/script/release

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
-  echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
-  exit 1
-fi
-
-run_id=""
-# Parse arguments
-while [[ "$#" -gt 0 ]]; do
-  case $1 in
-    --run-id)
-      run_id="$2"
-      shift 2
-      ;;
-    *)
-      echo "Unknown parameter passed: $1"
-      exit 1
-      ;;
-  esac
-done
-
-if [ -z "${run_id}" ]; then
-  echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
-  exit 1
-fi
-
-version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
-echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
-
-rm -rf pkg/cross-compiled
-gh run download "$run_id" -D pkg/cross-compiled
-
-for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
-  echo "Publishing $gem"
-  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
-done
-
-# last but not least, the uncompiled gem
-bundle exec rake package
-GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org