tiktoken_ruby 0.0.13 → 0.0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb1bf0a15715fcd42136ff3378a0b2b5676a06dd817fbbe59e83287c9a7e996
-  data.tar.gz: 30172f2328b4c98c62963c5ddc5afda840eb0f938e632d40e5aafc1eb7521693
+  metadata.gz: 71cce652b2f6a2ca962823d1947603a5224f305901cb4d8c822ca32b58192d47
+  data.tar.gz: d2cd0525f5f784a5904e1b7bc05cb3dc0c00f1f10561d52a99c6ef12b1351e89
 SHA512:
-  metadata.gz: 2db1488ee96bcfc94cbe48617c1f37f54ef74e532dafd50d4c89fc56940dcd3ca83cc3bd4c49ebc8edae71998e37046af62c8e67a8c2a97b3d316ec534acec62
-  data.tar.gz: 9ce2ca75ba2ba65c375c562717cdd19c6a3e41c127964775ce49eb1e3664954c530348eba71888b037d7b3ae2568f6fe1b52bd887b27e72294e5530ab54f8f8f
+  metadata.gz: fe7572bf7a82f77441335a273e90e6b4bce92be2b6fb6073c1409de635af8c3521b0181ba8d3422691f8d5242b6a3b08ec899fad465084bb7fc570d153e44b00
+  data.tar.gz: 4f95fa2c39ed53c1d40cb6928f294f1617b35042abd4b8cdcf641dac016b4dba9a15b3f46835cd85600af8a73152c5f3488694db9cdf9d3dd22707ab109a133d

data/.vscode/settings.json

@@ -4,5 +4,8 @@
   },
   "[markdown]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[github-actions-workflow]": {
+    "editor.defaultFormatter": "redhat.vscode-yaml"
   }
 }

data/CHANGELOG.md

@@ -0,0 +1,21 @@
+# [v0.0.14.1] - 20-12-2025
+## What's Changed
+* Cut v0.0.12 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/78
+* Bump magnus from 0.8.0 to 0.8.1 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/83
+* Bump actions/checkout from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/82
+* Bump standard from 1.50.0 to 1.51.1 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/81
+* Bump actions/upload-artifact from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/84
+* Bump magnus from 0.8.1 to 0.8.2 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/85
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/86
+* Support  by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/87
+* Bump actions/checkout from 5 to 6 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/89
+* Bump standard from 1.51.1 to 1.52.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/88
+* release GVL while encoding / decoding tokens by @tenderworks in https://github.com/IAPark/tiktoken_ruby/pull/90
+* Drop Ruby 3.1 support; automate release process by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/92
+* Rewrite history by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/93
+* Force workflow rebuild by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/94
+
+## New Contributors
+* @tenderworks made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/90
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.12...v0.0.14.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.13)
+    tiktoken_ruby (0.0.14.1)
       rb_sys (~> 0.9)
 
 GEM
@@ -9,15 +9,15 @@ GEM
   specs:
     ast (2.4.3)
     diff-lcs (1.6.2)
-    json (2.15.0)
+    json (2.16.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     minitest (5.25.5)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.5.1)
+    prism (1.6.0)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.3.1)
@@ -40,7 +40,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    rubocop (1.80.2)
+    rubocop (1.81.7)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -48,10 +48,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.48.0)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     rubocop-performance (1.25.0)
@@ -59,10 +59,10 @@ GEM
       rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.38.0, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.51.1)
+    standard (1.52.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.80.2)
+      rubocop (~> 1.81.7)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)

data/README.md

@@ -36,6 +36,39 @@ enc = Tiktoken.encoding_for_model("gpt-4")
 enc.encode("hello world").length #=> 2
 ```
 
+### Encoding methods
+
+There are three methods for encoding text:
+
+- `encode_ordinary(text)` - Encodes text, always treating special tokens as ordinary text
+- `encode(text, allowed_special: [])` - Encodes text, treating special tokens as text unless listed in `allowed_special`
+- `encode_with_special_tokens(text)` - Encodes text, recognizing and parsing all special tokens
+
+**Special tokens** are control sequences used by OpenAI models, such as `<|endoftext|>`, `<|fim_prefix|>`, `<|fim_middle|>`, and `<|fim_suffix|>`. The encoding methods differ in how they handle these sequences:
+
+```ruby
+enc = Tiktoken.get_encoding("cl100k_base")
+text = "Hello<|endoftext|>World"
+
+# encode_ordinary: treats <|endoftext|> as literal characters (9 tokens)
+enc.encode_ordinary(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode: same as encode_ordinary by default
+enc.encode(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode with allowed_special: recognizes the specified special token (3 tokens)
+enc.encode(text, allowed_special: ["<|endoftext|>"])
+#=> [9906, 100257, 10343]
+
+# encode_with_special_tokens: recognizes ALL special tokens (3 tokens)
+enc.encode_with_special_tokens(text)
+#=> [9906, 100257, 10343]
+```
+
+All methods round-trip correctly through `decode`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::ffi::c_void;
 
 use tiktoken_rs::Rank;
 
@@ -9,13 +10,82 @@ pub struct CoreBPEWrapper {
     core_bpe: tiktoken_rs::CoreBPE,
 }
 
+struct EncodeOrdinaryData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    text: String,
+    result: Vec<Rank>,
+}
+
+struct EncodeData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    text: String,
+    allowed_special: HashSet<String>,
+    result: Vec<Rank>,
+}
+
+struct EncodeSpecialData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    text: String,
+    result: Vec<Rank>,
+}
+
+struct DecodeData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    ids: Vec<Rank>,
+    result: Result<String, String>,
+}
+
+unsafe extern "C" fn encode_ordinary_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut EncodeOrdinaryData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.encode_ordinary(&data.text);
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut EncodeData);
+    let core_bpe = &*data.core_bpe;
+    let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect();
+    data.result = core_bpe.encode(&data.text, &allowed_special).0;
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn encode_special_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut EncodeSpecialData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.encode_with_special_tokens(&data.text);
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut DecodeData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.decode(std::mem::take(&mut data.ids)).map_err(|e| e.to_string());
+    std::ptr::null_mut()
+}
+
 impl CoreBPEWrapper {
     pub fn new(core_bpe: tiktoken_rs::CoreBPE) -> Self {
         Self { core_bpe }
     }
 
     pub fn encode_ordinary(&self, text: String) -> Vec<Rank> {
-        self.core_bpe.encode_ordinary(text.as_str())
+        let mut data = EncodeOrdinaryData {
+            core_bpe: &self.core_bpe as *const _,
+            text,
+            result: Vec::new(),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(encode_ordinary_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        data.result
     }
 
     pub fn encode(
@@ -24,24 +94,68 @@ impl CoreBPEWrapper {
         allowed_special: magnus::RArray,
     ) -> Result<Vec<Rank>, magnus::Error> {
         let allowed_special: Vec<String> = allowed_special.to_vec()?;
-        let allowed_special: Vec<&str> = allowed_special.iter().map(|s| s.as_str()).collect();
-        let allowed_special: HashSet<&str> = HashSet::from_iter(allowed_special.iter().cloned());
 
-        Ok(self.core_bpe.encode(text.as_str(), &allowed_special).0)
+        let mut data = EncodeData {
+            core_bpe: &self.core_bpe as *const _,
+            text,
+            allowed_special: HashSet::from_iter(allowed_special),
+            result: Vec::new(),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(encode_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        Ok(data.result)
     }
 
     pub fn encode_with_special_tokens(&self, text: String) -> Vec<Rank> {
-        self.core_bpe.encode_with_special_tokens(text.as_str())
+        let mut data = EncodeSpecialData {
+            core_bpe: &self.core_bpe as *const _,
+            text,
+            result: Vec::new(),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(encode_special_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        data.result
     }
 
     pub fn decode(&self, ids: Vec<Rank>) -> Result<String, magnus::Error> {
-        self.core_bpe.decode(ids).map_err(|e| {
+        let mut data = DecodeData {
+            core_bpe: &self.core_bpe as *const _,
+            ids,
+            result: Err(String::new()),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(decode_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        data.result.map_err(|e| {
             let error = match uncicode_error() {
                 Ok(error) => error,
                 Err(e) => return e,
             };
 
-            magnus::Error::new(error, e.to_string())
+            magnus::Error::new(error, e)
         })
     }
 }

data/lib/tiktoken_ruby/encoding.rb

@@ -40,6 +40,13 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode(text, allowed_special)
   end
 
+  # Encodes the text as a list of integer tokens, including special tokens.
+  # @param text [String] The text to encode
+  # @return [Array<Integer>] The encoded tokens
+  def encode_with_special_tokens(text)
+    @ext_base_bpe.encode_with_special_tokens(text)
+  end
+
   # Decodes the tokens back into text
   # @param tokens [Array<Integer>] The tokens to decode
   # @return [String] The decoded text

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.13"
+  VERSION = "0.0.14.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14.1
 platform: ruby
 authors:
 - IAPark
@@ -36,6 +36,7 @@ files:
 - ".rspec"
 - ".standard.yml"
 - ".vscode/settings.json"
+- CHANGELOG.md
 - Cargo.lock
 - Cargo.toml
 - Gemfile
@@ -51,7 +52,6 @@ files:
 - lib/tiktoken_ruby.rb
 - lib/tiktoken_ruby/encoding.rb
 - lib/tiktoken_ruby/version.rb
-- script/release
 - sig/tiktoken_ruby.rbs
 homepage: https://github.com/IAPark/tiktoken_ruby
 licenses:
@@ -67,7 +67,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/script/release

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
-  echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
-  exit 1
-fi
-
-run_id=""
-# Parse arguments
-while [[ "$#" -gt 0 ]]; do
-  case $1 in
-    --run-id)
-      run_id="$2"
-      shift 2
-      ;;
-    *)
-      echo "Unknown parameter passed: $1"
-      exit 1
-      ;;
-  esac
-done
-
-if [ -z "${run_id}" ]; then
-  echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
-  exit 1
-fi
-
-version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
-echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
-
-rm -rf pkg/cross-compiled
-gh run download "$run_id" -D pkg/cross-compiled
-
-for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
-  echo "Publishing $gem"
-  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
-done
-
-# last but not least, the uncompiled gem
-bundle exec rake package
-GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org