tiktoken_ruby 0.0.15.1 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca9715653ab21761d567bd9f7d95d23752d62efab79e6fcef888769a1b9486a5
-  data.tar.gz: 706aa0a3920fc6505d76ccc0fae74a92e874161ed249c566b78fceb57b3cdb15
+  metadata.gz: b04168b5f23957815c4f9a7139d0629c29236befaaeab12deaa1e0fee2c3b814
+  data.tar.gz: aa8af07c52ba013e0e7d7ad0aa569db6419a3330eaace2d3401963c7f4f9d2cf
 SHA512:
-  metadata.gz: 1c989aebcac3182851ecf17d956e13f10cb2d4d48cfb0a7e101bd2ecef76686584d0146e8858b85c57ecbe7f1757d6ebd3aa84c6dc8aecace051581ae0d56276
-  data.tar.gz: 9799d0ce8bf67b5b82d78c1883226907ab8f5da984615b2d56b89bb832941328fd58f4f2b928590040696357a4bd1549b02d2eb51c1418381cac1a848ad9cc51
+  metadata.gz: 0bf109e1ab9191f06bf7450cfa32589c5dc4fcff41084c3682694b9ddd03f5595151e3ba2e376916c85f5ea1ce8e0866743556ec2c0f778309ac3aad216caf8a
+  data.tar.gz: b105c5871dde75b05274e830f1b40c0a4a05364c2cb6947f1223c128af4c1564dfc573c8c6e3b521ccf11b4eb6eb0592882af734deeba39251f8a1eab619e4a5

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+# [v0.0.17] - 18-07-2026
+## What's Changed
+* Bump the github-actions group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/119
+* Bump the bundler-dependencies group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/121
+* Bump the cargo group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/120
+* feat(decode): Add opt-in lossy mode and decode_bytes by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/123
+* Bump version to 0.0.17 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/124
+
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.16...v0.0.17
+# [v0.0.16] - 02-05-2026
+## What's Changed
+* Bump rb-sys from 0.9.123 to 0.9.124 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/107
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/108
+* Bump standard from 1.53.0 to 1.54.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/109
+* Bump the cargo group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/113
+* Bump the bundler-dependencies group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/114
+* bump version to 0.0.16 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/115
+
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.15.1...v0.0.16
 # [v0.0.15.1] - 07-01-2026
 ## What's Changed
 * Re-release 0.0.15 for mistakenly yanked gems by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/105

data/Cargo.lock

@@ -13,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.100"
+version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
+checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
 [[package]]
 name = "base64"
@@ -25,16 +25,14 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "bindgen"
-version = "0.69.5"
+version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
  "bitflags",
  "cexpr",
  "clang-sys",
  "itertools",
- "lazy_static",
- "lazycell",
  "proc-macro2",
  "quote",
  "regex",
@@ -45,18 +43,18 @@ dependencies = [
 
 [[package]]
 name = "bit-set"
-version = "0.5.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
 dependencies = [
  "bit-vec",
 ]
 
 [[package]]
 name = "bit-vec"
-version = "0.6.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
 
 [[package]]
 name = "bitflags"
@@ -109,9 +107,9 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "fancy-regex"
-version = "0.13.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
 dependencies = [
  "bit-set",
  "regex-automata",
@@ -139,12 +137,6 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.177"
@@ -226,18 +218,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.123"
+version = "0.9.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45fb1a185af97ee456f1c9e56dbe6e2e662bec4fdeaf83c4c28e0e6adfb18816"
+checksum = "45ca28513560e56cfb79a62b1fce363c73af170a182024ce880c77ee9429920a"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.123"
+version = "0.9.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a58ebd02d7a6033e6a5f6f8d150c1e9f16506039092b84a73e6bedce6d3adf41"
+checksum = "ce04b2c55eff3a21aaa623fcc655d94373238e72cac6b3e1a3641ff31649f99a"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -256,9 +248,9 @@ checksum = "08f8d2924cf136a1315e2b4c7460a39f62ef11ee5d522df9b2750fab55b868b6"
 
 [[package]]
 name = "regex"
-version = "1.12.2"
+version = "1.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -285,9 +277,9 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
 [[package]]
 name = "rustc-hash"
-version = "1.1.0"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
 
 [[package]]
 name = "seq-macro"
@@ -349,9 +341,9 @@ dependencies = [
 
 [[package]]
 name = "tiktoken-rs"
-version = "0.9.1"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d"
+checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
 dependencies = [
  "anyhow",
  "base64",

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.15.1)
+    tiktoken_ruby (0.0.17)
       rb_sys (~> 0.9)
 
 GEM
@@ -9,25 +9,25 @@ GEM
   specs:
     ast (2.4.3)
     diff-lcs (1.6.2)
-    json (2.16.0)
+    json (2.20.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     minitest (6.0.1)
       prism (~> 1.5)
-    parallel (1.27.0)
-    parser (3.3.10.0)
+    parallel (1.28.0)
+    parser (3.3.11.1)
       ast (~> 2.4.1)
       racc
-    prism (1.6.0)
+    prism (1.9.0)
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.3.1)
+    rake (13.4.2)
     rake-compiler (1.3.1)
       rake
-    rake-compiler-dock (1.10.0)
-    rb_sys (0.9.123)
-      rake-compiler-dock (= 1.10.0)
-    regexp_parser (2.11.3)
+    rake-compiler-dock (1.12.0)
+    rb_sys (0.9.128)
+      rake-compiler-dock (= 1.12.0)
+    regexp_parser (2.12.0)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -41,37 +41,37 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    rubocop (1.81.7)
+    rubocop (1.87.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
-      parallel (~> 1.10)
+      parallel (>= 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.47.1, < 2.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.48.0)
+    rubocop-ast (1.49.1)
       parser (>= 3.3.7.2)
-      prism (~> 1.4)
-    rubocop-performance (1.25.0)
+      prism (~> 1.7)
+    rubocop-performance (1.26.1)
       lint_roller (~> 1.1)
       rubocop (>= 1.75.0, < 2.0)
-      rubocop-ast (>= 1.38.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.52.0)
+    standard (1.55.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.81.7)
+      rubocop (~> 1.87.0)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.8.0)
+    standard-performance (1.9.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.25.0)
+      rubocop-performance (~> 1.26.0)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.2.0)

data/README.md

@@ -17,9 +17,7 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-Usage should be very similar to the python library. Here's a simple example
-
-Encode and decode text
+Encode and decode text:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -27,7 +25,7 @@ enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
-Encoders can also be retrieved by model name
+Encoders can also be retrieved by model name:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -69,6 +67,27 @@ enc.encode_with_special_tokens(text)
 
 All methods round-trip correctly through `decode`.
 
+### Decoding methods
+
+- `decode(tokens, errors: :strict)` - Decodes tokens back into a UTF-8 string
+- `decode_bytes(tokens)` - Decodes tokens into their raw bytes (an `ASCII-8BIT` string), without UTF-8 validation
+
+Because BPE tokens are byte-level, a single character (an emoji, or non-Latin scripts) can span multiple tokens. Truncating a token array (like, "trim text to N tokens") can leave a prefix that is not valid UTF-8. The `errors:` option controls how those invalid sequences are handled.
+
+```ruby
+enc = Tiktoken.encoding_for_model("gpt-4o")
+tokens = enc.encode("🦄") # the emoji spans multiple tokens
+
+# :strict (default) - raise Tiktoken::UnicodeError on invalid UTF-8
+enc.decode(tokens.first(2))                    #=> raises Tiktoken::UnicodeError
+
+# :replace - substitute invalid sequences with "�" (matches Python tiktoken's default)
+enc.decode(tokens.first(2), errors: :replace)  #=> "�"
+
+# decode_bytes - get the raw bytes and handle them yourself
+enc.decode_bytes(tokens.first(2))              #=> "\xF0\x9F\xA6" (ASCII-8BIT)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/ext/tiktoken_ruby/Cargo.toml

@@ -11,5 +11,5 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8.2" }
-rb-sys = { version = "0.9.123", features = ["stable-api-compiled-fallback"] }
-tiktoken-rs = { version = "0.9.0" }
+rb-sys = { version = "0.9.127", features = ["stable-api-compiled-fallback"] }
+tiktoken-rs = { version = "0.12.0" }

data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -3,7 +3,7 @@ use std::ffi::c_void;
 
 use tiktoken_rs::Rank;
 
-use crate::uncicode_error;
+use crate::unicode_error;
 
 #[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")]
 pub struct CoreBPEWrapper {
@@ -20,7 +20,7 @@ struct EncodeData {
     core_bpe: *const tiktoken_rs::CoreBPE,
     text: String,
     allowed_special: HashSet<String>,
-    result: Vec<Rank>,
+    result: Result<Vec<Rank>, String>,
 }
 
 struct EncodeSpecialData {
@@ -35,6 +35,12 @@ struct DecodeData {
     result: Result<String, String>,
 }
 
+struct DecodeBytesData {
+    core_bpe: *const tiktoken_rs::CoreBPE,
+    ids: Vec<Rank>,
+    result: Result<Vec<u8>, String>,
+}
+
 unsafe extern "C" fn encode_ordinary_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut EncodeOrdinaryData);
     let core_bpe = &*data.core_bpe;
@@ -46,7 +52,11 @@ unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut EncodeData);
     let core_bpe = &*data.core_bpe;
     let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect();
-    data.result = core_bpe.encode(&data.text, &allowed_special).0;
+    // Keep only the token vector and surface any tokenization failure to Ruby.
+    data.result = core_bpe
+        .encode(&data.text, &allowed_special)
+        .map(|(tokens, _last_piece_token_len)| tokens)
+        .map_err(|e| e.to_string());
     std::ptr::null_mut()
 }
 
@@ -60,7 +70,14 @@ unsafe extern "C" fn encode_special_without_gvl(data: *mut c_void) -> *mut c_voi
 unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut DecodeData);
     let core_bpe = &*data.core_bpe;
-    data.result = core_bpe.decode(std::mem::take(&mut data.ids)).map_err(|e| e.to_string());
+    data.result = core_bpe.decode(&data.ids).map_err(|e| e.to_string());
+    std::ptr::null_mut()
+}
+
+unsafe extern "C" fn decode_bytes_without_gvl(data: *mut c_void) -> *mut c_void {
+    let data = &mut *(data as *mut DecodeBytesData);
+    let core_bpe = &*data.core_bpe;
+    data.result = core_bpe.decode_bytes(&data.ids).map_err(|e| e.to_string());
     std::ptr::null_mut()
 }
 
@@ -99,7 +116,7 @@ impl CoreBPEWrapper {
             core_bpe: &self.core_bpe as *const _,
             text,
             allowed_special: HashSet::from_iter(allowed_special),
-            result: Vec::new(),
+            result: Ok(Vec::new()),
         };
 
         unsafe {
@@ -111,7 +128,9 @@ impl CoreBPEWrapper {
             );
         }
 
-        Ok(data.result)
+        data.result.map_err(|e| {
+            magnus::Error::new(magnus::Ruby::get().unwrap().exception_runtime_error(), e)
+        })
     }
 
     pub fn encode_with_special_tokens(&self, text: String) -> Vec<Rank> {
@@ -150,7 +169,7 @@ impl CoreBPEWrapper {
         }
 
         data.result.map_err(|e| {
-            let error = match uncicode_error() {
+            let error = match unicode_error() {
                 Ok(error) => error,
                 Err(e) => return e,
             };
@@ -158,4 +177,37 @@ impl CoreBPEWrapper {
             magnus::Error::new(error, e)
         })
     }
+
+    pub fn decode_bytes(&self, ids: Vec<Rank>) -> Result<magnus::RString, magnus::Error> {
+        let mut data = DecodeBytesData {
+            core_bpe: &self.core_bpe as *const _,
+            ids,
+            result: Err(String::new()),
+        };
+
+        unsafe {
+            rb_sys::rb_thread_call_without_gvl(
+                Some(decode_bytes_without_gvl),
+                &mut data as *mut _ as *mut c_void,
+                None,
+                std::ptr::null_mut(),
+            );
+        }
+
+        // Build the Ruby string only after the GVL has been re-acquired.
+        // `str_from_slice` yields an ASCII-8BIT (binary) string, so the bytes
+        // are handed back verbatim without any UTF-8 validation. We hold the
+        // GVL here (this is called from Ruby), so `Ruby::get` cannot fail.
+        let ruby = magnus::Ruby::get().unwrap();
+        data.result
+            .map(|bytes| ruby.str_from_slice(&bytes))
+            .map_err(|e| {
+                let error = match unicode_error() {
+                    Ok(error) => error,
+                    Err(e) => return e,
+                };
+
+                magnus::Error::new(error, e)
+            })
+    }
 }

data/ext/tiktoken_ruby/src/lib.rs

@@ -34,7 +34,7 @@ fn module() -> Result<RModule, magnus::Error> {
     Ruby::get().unwrap().define_module("Tiktoken")
 }
 
-fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
+fn unicode_error() -> Result<ExceptionClass, magnus::Error> {
     module()?.define_error(
         "UnicodeError",
         Ruby::get().unwrap().exception_standard_error(),
@@ -45,6 +45,10 @@ fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
 fn init() -> Result<(), Error> {
     let module = module()?;
 
+    // Define Tiktoken::UnicodeError eagerly so the constant exists as soon as
+    // the extension is loaded, rather than only after the first decode error.
+    unicode_error()?;
+
     let factory_module = module.define_module("BpeFactory")?;
     factory_module.define_singleton_method("r50k_base", function!(r50k_base, 0))?;
     factory_module.define_singleton_method("p50k_base", function!(p50k_base, 0))?;
@@ -67,5 +71,6 @@ fn init() -> Result<(), Error> {
     )?;
 
     bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?;
+    bpe_class.define_method("decode_bytes", method!(CoreBPEWrapper::decode_bytes, 1))?;
     Ok(())
 }

data/lib/tiktoken_ruby/encoding.rb

@@ -47,11 +47,41 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode_with_special_tokens(text)
   end
 
-  # Decodes the tokens back into text
+  # Decodes the tokens back into text.
+  #
+  # BPE tokens are byte-level, so a single character (an emoji, non-Latin
+  # scripts) can span multiple tokens. Truncating a token array to a limit can
+  # therefore leave a prefix that is not valid UTF-8. The +errors+ option
+  # controls how those invalid byte sequences are handled:
+  #
+  # * +:strict+ (default) - raise Tiktoken::UnicodeError on invalid UTF-8.
+  # * +:replace+ - substitute each invalid sequence with the Unicode replacement
+  #   character (U+FFFD, +�+), matching Python tiktoken's default behavior.
+  #
   # @param tokens [Array<Integer>] The tokens to decode
-  # @return [String] The decoded text
-  def decode(tokens)
-    @ext_base_bpe.decode(tokens)
+  # @param errors [Symbol] How to handle invalid UTF-8 (:strict or :replace)
+  # @return [String] The decoded text (UTF-8)
+  def decode(tokens, errors: :strict)
+    case errors
+    when :strict
+      @ext_base_bpe.decode(tokens)
+    when :replace
+      decode_bytes(tokens).force_encoding(Encoding::UTF_8).scrub("\u{FFFD}")
+    else
+      raise ArgumentError, "errors must be :strict or :replace, got #{errors.inspect}"
+    end
+  end
+
+  # Decodes the tokens back into their raw bytes, without any UTF-8 validation.
+  #
+  # The returned string has ASCII-8BIT (binary) encoding and is not guaranteed
+  # to be valid UTF-8 — useful when you want to handle invalid sequences
+  # yourself. Matches Python tiktoken's +decode_bytes+.
+  #
+  # @param tokens [Array<Integer>] The tokens to decode
+  # @return [String] The decoded bytes (ASCII-8BIT)
+  def decode_bytes(tokens)
+    @ext_base_bpe.decode_bytes(tokens)
   end
 
   private

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.15.1"
+  VERSION = "0.0.17"
 end

data/lib/tiktoken_ruby.rb

@@ -79,6 +79,7 @@ module Tiktoken
       o3: "o200k_base",
       "o4-mini": "o200k_base",
       # chat
+      "gpt-5": "o200k_base",
       "gpt-4.1": "o200k_base",
       "chatgpt-4o": "o200k_base",
       "gpt-4o": "o200k_base",
@@ -136,7 +137,9 @@ module Tiktoken
       "o3-": "o200k_base",
       "o4-": "o200k_base",
       # chat
+      "gpt-5.": "o200k_base", # e.g., gpt-5.4, gpt-5.4-mini, gpt-5.3-codex, gpt-5.2-pro
       "gpt-5-": "o200k_base",
+      "codex-mini": "o200k_base", # e.g., codex-mini, codex-mini-latest
       "gpt-4.5-": "o200k_base",
       "gpt-4.1-": "o200k_base",
       "chatgpt-4o-": "o200k_base",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.15.1
+  version: 0.0.17
 platform: ruby
 authors:
 - IAPark
@@ -74,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 3.4.0
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Ruby wrapper for Tiktoken
 test_files: []