tiktoken_ruby 0.0.16 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d1b276821555850dac79a65fddb5152db1b0dfc7021cfa13e1b2435822eb6d1
-  data.tar.gz: 5c72d2286f0bcb53d128dbb1b52ad91e7c593c91730d98300ee44d21a8c5e737
+  metadata.gz: b04168b5f23957815c4f9a7139d0629c29236befaaeab12deaa1e0fee2c3b814
+  data.tar.gz: aa8af07c52ba013e0e7d7ad0aa569db6419a3330eaace2d3401963c7f4f9d2cf
 SHA512:
-  metadata.gz: f586ac763017ffde2c483d703ebc3fd65d1029238e0a8c2cfac0cb4e4bd8f2f03c7eb6156ccc0bcb8b4c72108d7f8c83e227f866205fa3771b14861116ade096
-  data.tar.gz: 56ceeaf795a1b0817b5cde8641369e3963b5d613870069c1a62d17732922fc3a41edd66b02c725f3a373bf3697bcf1de7d9e2ed2bcebd70452942a6c102fd55c
+  metadata.gz: 0bf109e1ab9191f06bf7450cfa32589c5dc4fcff41084c3682694b9ddd03f5595151e3ba2e376916c85f5ea1ce8e0866743556ec2c0f778309ac3aad216caf8a
+  data.tar.gz: b105c5871dde75b05274e830f1b40c0a4a05364c2cb6947f1223c128af4c1564dfc573c8c6e3b521ccf11b4eb6eb0592882af734deeba39251f8a1eab619e4a5

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# [v0.0.17] - 18-07-2026
+## What's Changed
+* Bump the github-actions group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/119
+* Bump the bundler-dependencies group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/121
+* Bump the cargo group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/120
+* feat(decode): Add opt-in lossy mode and decode_bytes by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/123
+* Bump version to 0.0.17 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/124
+
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.16...v0.0.17
 # [v0.0.16] - 02-05-2026
 ## What's Changed
 * Bump rb-sys from 0.9.123 to 0.9.124 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/107

data/Cargo.lock

@@ -36,7 +36,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "shlex",
  "syn",
 ]
@@ -218,18 +218,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.127"
+version = "0.9.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7d7c9560fe42dcffa576941394075f18a17dce89fcf718a2fa90b7dc2134d12"
+checksum = "45ca28513560e56cfb79a62b1fce363c73af170a182024ce880c77ee9429920a"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.127"
+version = "0.9.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1688e8f32967ba48c89e4dfa283b57f901075f542fc7ee9c3d7c5f9091ca1d9"
+checksum = "ce04b2c55eff3a21aaa623fcc655d94373238e72cac6b3e1a3641ff31649f99a"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -275,12 +275,6 @@ version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.2"
@@ -347,9 +341,9 @@ dependencies = [
 
 [[package]]
 name = "tiktoken-rs"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"
+checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
 dependencies = [
  "anyhow",
  "base64",
@@ -357,7 +351,7 @@ dependencies = [
  "fancy-regex",
  "lazy_static",
  "regex",
- "rustc-hash 1.1.0",
+ "rustc-hash",
 ]
 
 [[package]]

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.16)
+    tiktoken_ruby (0.0.17)
       rb_sys (~> 0.9)
 
 GEM
@@ -9,13 +9,13 @@ GEM
   specs:
     ast (2.4.3)
     diff-lcs (1.6.2)
-    json (2.18.1)
+    json (2.20.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     minitest (6.0.1)
       prism (~> 1.5)
-    parallel (1.27.0)
-    parser (3.3.10.2)
+    parallel (1.28.0)
+    parser (3.3.11.1)
       ast (~> 2.4.1)
       racc
     prism (1.9.0)
@@ -25,9 +25,9 @@ GEM
     rake-compiler (1.3.1)
       rake
     rake-compiler-dock (1.12.0)
-    rb_sys (0.9.127)
+    rb_sys (0.9.128)
       rake-compiler-dock (= 1.12.0)
-    regexp_parser (2.11.3)
+    regexp_parser (2.12.0)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -41,18 +41,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    rubocop (1.84.2)
+    rubocop (1.87.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
-      parallel (~> 1.10)
+      parallel (>= 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
       rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.49.0)
+    rubocop-ast (1.49.1)
       parser (>= 3.3.7.2)
       prism (~> 1.7)
     rubocop-performance (1.26.1)
@@ -60,10 +60,10 @@ GEM
       rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.47.1, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.54.0)
+    standard (1.55.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.84.0)
+      rubocop (~> 1.87.0)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)

data/README.md

@@ -17,9 +17,7 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-Usage should be very similar to the python library. Here's a simple example
-
-Encode and decode text
+Encode and decode text:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -27,7 +25,7 @@ enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
-Encoders can also be retrieved by model name
+Encoders can also be retrieved by model name:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -69,6 +67,27 @@ enc.encode_with_special_tokens(text)
 
 All methods round-trip correctly through `decode`.
 
+### Decoding methods
+
+- `decode(tokens, errors: :strict)` - Decodes tokens back into a UTF-8 string
+- `decode_bytes(tokens)` - Decodes tokens into their raw bytes (an `ASCII-8BIT` string), without UTF-8 validation
+
+Because BPE tokens are byte-level, a single character (an emoji, or non-Latin scripts) can span multiple tokens. Truncating a token array (like, "trim text to N tokens") can leave a prefix that is not valid UTF-8. The `errors:` option controls how those invalid sequences are handled.
+
+```ruby
+enc = Tiktoken.encoding_for_model("gpt-4o")
+tokens = enc.encode("🦄") # the emoji spans multiple tokens
+
+# :strict (default) - raise Tiktoken::UnicodeError on invalid UTF-8
+enc.decode(tokens.first(2))                    #=> raises Tiktoken::UnicodeError
+
+# :replace - substitute invalid sequences with "�" (matches Python tiktoken's default)
+enc.decode(tokens.first(2), errors: :replace)  #=> "�"
+
+# decode_bytes - get the raw bytes and handle them yourself
+enc.decode_bytes(tokens.first(2))              #=> "\xF0\x9F\xA6" (ASCII-8BIT)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/ext/tiktoken_ruby/Cargo.toml

@@ -12,4 +12,4 @@ crate-type = ["cdylib"]
 [dependencies]
 magnus = { version = "0.8.2" }
 rb-sys = { version = "0.9.127", features = ["stable-api-compiled-fallback"] }
-tiktoken-rs = { version = "0.11.0" }
+tiktoken-rs = { version = "0.12.0" }

data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -3,7 +3,7 @@ use std::ffi::c_void;
 
 use tiktoken_rs::Rank;
 
-use crate::uncicode_error;
+use crate::unicode_error;
 
 #[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")]
 pub struct CoreBPEWrapper {
@@ -20,7 +20,7 @@ struct EncodeData {
     core_bpe: *const tiktoken_rs::CoreBPE,
     text: String,
     allowed_special: HashSet<String>,
-    result: Vec<Rank>,
+    result: Result<Vec<Rank>, String>,
 }
 
 struct EncodeSpecialData {
@@ -35,6 +35,12 @@ struct DecodeData {
     result: Result<String, String>,
 }
 
+struct DecodeBytesData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    ids: Vec<Rank>,
+    result: Result<Vec<u8>, String>,
+}
+
 unsafe extern "C" fn encode_ordinary_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut EncodeOrdinaryData);
     let core_bpe = &*data.core_bpe;
@@ -46,7 +52,11 @@ unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut EncodeData);
     let core_bpe = &*data.core_bpe;
     let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect();
-    data.result = core_bpe.encode(&data.text, &allowed_special).0;
+    // Keep only the token vector and surface any tokenization failure to Ruby.
+    data.result = core_bpe
+        .encode(&data.text, &allowed_special)
+        .map(|(tokens, _last_piece_token_len)| tokens)
+        .map_err(|e| e.to_string());
     std::ptr::null_mut()
 }
 
@@ -64,6 +74,13 @@ unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
     std::ptr::null_mut()
 }
 
+unsafe extern "C" fn decode_bytes_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut DecodeBytesData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.decode_bytes(&data.ids).map_err(|e| e.to_string());
+    std::ptr::null_mut()
+}
+
 impl CoreBPEWrapper {
     pub fn new(core_bpe: tiktoken_rs::CoreBPE) -> Self {
         Self { core_bpe }
@@ -99,7 +116,7 @@ impl CoreBPEWrapper {
             core_bpe: &self.core_bpe as *const _,
             text,
             allowed_special: HashSet::from_iter(allowed_special),
-            result: Vec::new(),
+            result: Ok(Vec::new()),
         };
 
         unsafe {
@@ -111,7 +128,9 @@ impl CoreBPEWrapper {
             );
         }
 
-        Ok(data.result)
+        data.result.map_err(|e| {
+            magnus::Error::new(magnus::Ruby::get().unwrap().exception_runtime_error(), e)
+        })
     }
 
     pub fn encode_with_special_tokens(&self, text: String) -> Vec<Rank> {
@@ -150,7 +169,7 @@ impl CoreBPEWrapper {
         }
 
         data.result.map_err(|e| {
-            let error = match uncicode_error() {
+            let error = match unicode_error() {
                 Ok(error) => error,
                 Err(e) => return e,
             };
@@ -158,4 +177,37 @@ impl CoreBPEWrapper {
             magnus::Error::new(error, e)
         })
     }
+
+    pub fn decode_bytes(&self, ids: Vec<Rank>) -> Result<magnus::RString, magnus::Error> {
+        let mut data = DecodeBytesData {
+            core_bpe: &self.core_bpe as *const _,
+            ids,
+            result: Err(String::new()),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(decode_bytes_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        // Build the Ruby string only after the GVL has been re-acquired.
+        // `str_from_slice` yields an ASCII-8BIT (binary) string, so the bytes
+        // are handed back verbatim without any UTF-8 validation. We hold the
+        // GVL here (this is called from Ruby), so `Ruby::get` cannot fail.
+        let ruby = magnus::Ruby::get().unwrap();
+        data.result
+            .map(|bytes| ruby.str_from_slice(&bytes))
+            .map_err(|e| {
+                let error = match unicode_error() {
+                    Ok(error) => error,
+                    Err(e) => return e,
+                };
+
+                magnus::Error::new(error, e)
+            })
+    }
 }

data/ext/tiktoken_ruby/src/lib.rs

@@ -34,7 +34,7 @@ fn module() -> Result<RModule, magnus::Error> {
     Ruby::get().unwrap().define_module("Tiktoken")
 }
 
-fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
+fn unicode_error() -> Result<ExceptionClass, magnus::Error> {
     module()?.define_error(
         "UnicodeError",
         Ruby::get().unwrap().exception_standard_error(),
@@ -45,6 +45,10 @@ fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
 fn init() -> Result<(), Error> {
     let module = module()?;
 
+    // Define Tiktoken::UnicodeError eagerly so the constant exists as soon as
+    // the extension is loaded, rather than only after the first decode error.
+    unicode_error()?;
+
     let factory_module = module.define_module("BpeFactory")?;
     factory_module.define_singleton_method("r50k_base", function!(r50k_base, 0))?;
     factory_module.define_singleton_method("p50k_base", function!(p50k_base, 0))?;
@@ -67,5 +71,6 @@ fn init() -> Result<(), Error> {
     )?;
 
     bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?;
+    bpe_class.define_method("decode_bytes", method!(CoreBPEWrapper::decode_bytes, 1))?;
     Ok(())
 }

data/lib/tiktoken_ruby/encoding.rb

@@ -47,11 +47,41 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode_with_special_tokens(text)
   end
 
-  # Decodes the tokens back into text
+  # Decodes the tokens back into text.
+  #
+  # BPE tokens are byte-level, so a single character (an emoji, non-Latin
+  # scripts) can span multiple tokens. Truncating a token array to a limit can
+  # therefore leave a prefix that is not valid UTF-8. The +errors+ option
+  # controls how those invalid byte sequences are handled:
+  #
+  # * +:strict+ (default) - raise Tiktoken::UnicodeError on invalid UTF-8.
+  # * +:replace+ - substitute each invalid sequence with the Unicode replacement
+  #   character (U+FFFD, +�+), matching Python tiktoken's default behavior.
+  #
   # @param tokens [Array<Integer>] The tokens to decode
-  # @return [String] The decoded text
-  def decode(tokens)
-    @ext_base_bpe.decode(tokens)
+  # @param errors [Symbol] How to handle invalid UTF-8 (:strict or :replace)
+  # @return [String] The decoded text (UTF-8)
+  def decode(tokens, errors: :strict)
+    case errors
+    when :strict
+      @ext_base_bpe.decode(tokens)
+    when :replace
+      decode_bytes(tokens).force_encoding(Encoding::UTF_8).scrub("\u{FFFD}")
+    else
+      raise ArgumentError, "errors must be :strict or :replace, got #{errors.inspect}"
+    end
+  end
+
+  # Decodes the tokens back into their raw bytes, without any UTF-8 validation.
+  #
+  # The returned string has ASCII-8BIT (binary) encoding and is not guaranteed
+  # to be valid UTF-8 — useful when you want to handle invalid sequences
+  # yourself. Matches Python tiktoken's +decode_bytes+.
+  #
+  # @param tokens [Array<Integer>] The tokens to decode
+  # @return [String] The decoded bytes (ASCII-8BIT)
+  def decode_bytes(tokens)
+    @ext_base_bpe.decode_bytes(tokens)
   end
 
   private

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.16"
+  VERSION = "0.0.17"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.17
 platform: ruby
 authors:
 - IAPark
@@ -74,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 3.4.0
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Ruby wrapper for Tiktoken
 test_files: []