j_r_r_token 1.1.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52cec2e717280a266a6c1a83c629dbb2944b781fb54401f6729e8cfc947d9a03
-  data.tar.gz: 6a6e3b673858c11cb5b13cef84e995d63eefbdca5a3af032b8b4d4eff016152a
+  metadata.gz: 2ec532950d7c7718d14e303c9c43d18df4eb6d191bb4dd56cfc1b9dbc62cd673
+  data.tar.gz: c9bff0c4a3cd9c815b45ee7bd1362d71c12f198241bbbee605050d5addfb9feb
 SHA512:
-  metadata.gz: dec556e6d7428f0950761653555ce6f8b6a92adf461ea333693d3b2132bff7d6d71bd38fa553307cd9c6bb762f4e1041c27ed2a3619636979675457b711a440b
-  data.tar.gz: e00f149423a9288b835628c8204ef777551a45365b3176e1648235f24c50d071c37622cd26a56993cf619fa8ea8d303201e76c25e6295c11e19e1a1310742a24
+  metadata.gz: ef83e1a569ff920d8cbd55d7f3109c883b2e1b7e5fe561db7f2ac3e67fe3de8d1ad4c552b9bd10c9bbfb9af783331dac0c65d7b45f49af8bc0efd7c8849f747f
+  data.tar.gz: c990330066e627ca909479fefb397480d05da12401aa23383c96e53c58bae351c2bc4e9ad535d476153507a67333e90cf6cd607e6dfd5590f716a02ce26d1acd

data/CHANGELOG.md

@@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+---
+## [1.2.1] - 2026-05-13
+### Breaking Change
+- Minimum Ruby version raised from 2.7.0 to 3.0.0. Ruby 2.7 has been EOL since March 2023 and is no longer supported by upstream `rake-compiler-dock` (1.11.0+) which `rb_sys` uses for cross-compiling native gems. Production environments on Ruby 3.0+ are unaffected.
+
+### New Functionality
+- Added `o200k_harmony` tokenizer with support for `gpt-oss-20b` and `gpt-oss-120b`.
+- Added support for additional model aliases now recognized by `tiktoken-rs` 0.11: `gpt-5-mini`, `gpt-5-nano`, `gpt-5.x` decimal variants (e.g. `gpt-5.4`, `gpt-5.4-pro`, `gpt-5.2-codex`), `gpt-4.5-*`, `o1`, `o3`, `o4-mini`, `codex-mini`, `codex-mini-latest`, and `ft:gpt-5` fine-tunes.
+- Fine-tuned model resolution (`ft:` prefix) is now handled natively by the upstream `get_tokenizer` mapping, so any future ft: bases are picked up automatically.
+
+### Change
+- Bumped `tiktoken-rs` from 0.7 to 0.11.
+- Bumped `magnus` from 0.6 to 0.8 (with the `old-api` feature enabled to preserve current bindings). Still supports Ruby >= 3.0.
+- Refactored Rust model dispatch to delegate to `tiktoken_rs::tokenizer::get_tokenizer` instead of maintaining a parallel alias list. The gem now stays in sync with upstream OpenAI tiktoken automatically on each `tiktoken-rs` bump.
+- CI matrix trimmed: dropped deprecated `macos-13`; consolidated to `ubuntu-latest`, `ubuntu-24.04-arm`, `macos-latest`, `macos-15-intel`. Added Ruby 4.0 to both test and cross-gem builds.
+
+---
+## [1.2.0] - 2026-05-13
+### Note
+- Partial release. The `arm-linux` cross-compile leg failed because upstream `rake-compiler-dock` 1.11.0+ dropped Ruby 2.7, so the published artifact set is incomplete. Please use 1.2.1 instead, which drops Ruby 2.7 from the support matrix and ships a complete set of platform gems.
+
 ---
 ## [1.1.0] - 2025-08-26
 ### New Functionality

data/Cargo.lock

@@ -13,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.98"
+version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
 [[package]]
 name = "base64"
@@ -25,38 +25,36 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "bindgen"
-version = "0.69.5"
+version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
  "bitflags",
  "cexpr",
  "clang-sys",
  "itertools",
- "lazy_static",
- "lazycell",
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash",
+ "rustc-hash 2.1.2",
  "shlex",
  "syn",
 ]
 
 [[package]]
 name = "bit-set"
-version = "0.5.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
 dependencies = [
  "bit-vec",
 ]
 
 [[package]]
 name = "bit-vec"
-version = "0.6.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
 
 [[package]]
 name = "bitflags"
@@ -66,9 +64,9 @@ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
 
 [[package]]
 name = "bstr"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"
 dependencies = [
  "memchr",
  "regex-automata",
@@ -109,9 +107,9 @@ checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
 [[package]]
 name = "fancy-regex"
-version = "0.13.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
 dependencies = [
  "bit-set",
  "regex-automata",
@@ -147,12 +145,6 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.174"
@@ -171,9 +163,9 @@ dependencies = [
 
 [[package]]
 name = "magnus"
-version = "0.6.4"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1597ef40aa8c36be098249e82c9a20cf7199278ac1c1a1a995eeead6a184479"
+checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -183,9 +175,9 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
+checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -234,18 +226,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.116"
+version = "0.9.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7059846f68396df83155779c75336ca24567741cb95256e6308c9fcc370e8dad"
+checksum = "45ca28513560e56cfb79a62b1fce363c73af170a182024ce880c77ee9429920a"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.116"
+version = "0.9.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac217510df41b9ffc041573e68d7a02aaff770c49943c7494441c4b224b0ecd0"
+checksum = "ce04b2c55eff3a21aaa623fcc655d94373238e72cac6b3e1a3641ff31649f99a"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -258,15 +250,15 @@ dependencies = [
 
 [[package]]
 name = "rb-sys-env"
-version = "0.1.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
+checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
 
 [[package]]
 name = "regex"
-version = "1.11.1"
+version = "1.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -276,9 +268,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.9"
+version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -297,6 +289,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
 [[package]]
 name = "seq-macro"
 version = "0.3.6"
@@ -348,9 +346,9 @@ dependencies = [
 
 [[package]]
 name = "tiktoken-rs"
-version = "0.7.0"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
+checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"
 dependencies = [
  "anyhow",
  "base64",
@@ -358,7 +356,7 @@ dependencies = [
  "fancy-regex",
  "lazy_static",
  "regex",
- "rustc-hash",
+ "rustc-hash 1.1.0",
 ]
 
 [[package]]

data/README.md

@@ -6,7 +6,7 @@ JRRToken provides a high-performance, native Ruby interface for counting tokens
 
 The gem ships with pre-compiled native extensions for Linux and macOS, so your end-users don't need a Rust toolchain installed.
 
-Supports Ruby >= 2.7.0
+Supports Ruby >= 3.0.0
 
 Extensive Model Support: Includes tokenizers for all modern and legacy OpenAI models, recognizing dozens of model aliases automatically.
 
@@ -15,16 +15,16 @@ Extensive Model Support: Includes tokenizers for all modern and legacy OpenAI mo
 ## Install
 
 #### Gemfile
-`gem 'j_r_r_token', '~> 1.1'`
+`gem 'j_r_r_token', '~> 1.2'`
 
 #### Command Line
-`gem install j_r_r_token -v '~> 1.1'`
+`gem install j_r_r_token -v '~> 1.2'`
 
 ## Use
 
 ### Count tokens for a specific model (the model keyword is required)
 ```Ruby
-count = JRRToken::Tokenizer.count("hello world!", model: "gpt-5")
+count = JRRToken::Tokenizer.count("hello world!", model: "gpt-5.4")
 ```
 
 ### The gem recognizes many aliases, including older models
@@ -40,18 +40,21 @@ count = JRRToken::Tokenizer.count(1234, model: "o200k_base")
 If you provide an unsupported model name, the gem will raise an `ArgumentError`.
 
 ## Supported Models
-The gem automatically maps dozens of model names and prefixes to the correct underlying tokenizer. You don't need to know the tokenizer's base name (e.g., cl100k_base); just use the model name you're working with.
+The gem automatically maps dozens of model names and prefixes to the correct underlying tokenizer. You don't need to know the tokenizer's base name (e.g., cl100k_base); just use the model name you're working with. Model resolution is delegated to [`tiktoken-rs`](https://github.com/zurawiki/tiktoken-rs), so the alias list stays in sync with upstream.
 
-- o200k_base Models (e.g., gpt-5, gpt-4.1, gpt-4o)
-- cl100k_base Models (e.g., gpt-4, gpt-3.5)
+- o200k_harmony Models (e.g., gpt-oss-20b, gpt-oss-120b)
+- o200k_base Models (e.g., gpt-5.4, gpt-5-mini, gpt-4.5, gpt-4.1, gpt-4o, o1, o3, o4-mini, codex-mini)
+- cl100k_base Models (e.g., gpt-4, gpt-3.5-turbo, text-embedding-3-small)
 - p50k_base Models (e.g., text-davinci-003)
 - r50k_base Models (e.g., gpt-2)
 - p50k_edit Models
 
+Fine-tuned models (`ft:<base>:org:name:id`) are automatically resolved to the same tokenizer as their base model.
+
 ## Developing JRRToken Locally
 
 ### Install Ruby and Rust dependencies
-Ruby >= 2.7.0
+Ruby >= 3.0.0
 
 RubyGems >= 3.0.0
 

data/ext/j_r_r_token/Cargo.toml

@@ -9,5 +9,5 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.6" }
-tiktoken-rs = { version= "0.7" }
+magnus = { version = "0.8", features = ["old-api"] }
+tiktoken-rs = { version = "0.11" }

data/ext/j_r_r_token/src/lib.rs

@@ -1,53 +1,40 @@
 use magnus::{define_module, exception, function, Error};
-use tiktoken_rs::{cl100k_base, o200k_base, p50k_base, p50k_edit, r50k_base, CoreBPE};
+use tiktoken_rs::{
+    cl100k_base, o200k_base, o200k_harmony, p50k_base, p50k_edit, r50k_base,
+    tokenizer::{get_tokenizer, Tokenizer},
+    CoreBPE,
+};
 
-// Helper function to get the correct tokenizer based on the model name.
-// This logic is adapted from the tiktoken-rs library.
+// Resolves a model name (or bare encoding name) to a CoreBPE instance.
+// Model -> tokenizer dispatch is delegated to `tiktoken_rs::tokenizer::get_tokenizer`,
+// which is kept in sync with upstream OpenAI tiktoken. It handles `ft:` prefix
+// stripping, GPT-5/4.5/4.1/4o families, o1/o3/o4 reasoning models, gpt-oss
+// (harmony), codex-mini, and all legacy aliases.
 fn get_bpe_from_model(model: &str) -> Result<CoreBPE, Error> {
-    let tokenizer = match model {
-        // --- O200k Base Models ---
-        "o200k_base" | "gpt-5" | "gpt-4.1" | "chatgpt-4o-latest" | "gpt-4o" => o200k_base(),
-        // --- Cl100k Base Models ---
-        "cl100k_base" | "gpt-4" | "ft:gpt-4" | "gpt-3.5-turbo" | "gpt-3.5" | "gpt-35-turbo"
-        | "davinci-002" | "babbage-002" | "text-embedding-ada-002"
-        | "text-embedding-3-small" | "text-embedding-3-large" => cl100k_base(),
-        // --- P50k Base Models ---
-        "p50k_base" | "text-davinci-003" | "text-davinci-002" | "code-davinci-002"
-        | "code-davinci-001" | "code-cushman-002" | "code-cushman-001"
-        | "davinci-codex" | "cushman-codex" => p50k_base(),
-        // --- R50k Base (GPT-2) Models ---
-        "r50k_base" | "gpt2" | "gpt-2" | "text-davinci-001" | "text-curie-001"
-        | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada"
-        | "text-similarity-davinci-001" | "text-similarity-curie-001"
-        | "text-similarity-babbage-001" | "text-similarity-ada-001"
-        | "text-search-davinci-doc-001" | "text-search-curie-doc-001"
-        | "text-search-babbage-doc-001" | "text-search-ada-doc-001"
-        | "code-search-babbage-code-001" | "code-search-ada-code-001" => r50k_base(),
-        // --- P50k Edit Models ---
-        "p50k_edit" | "text-davinci-edit-001" | "code-davinci-edit-001" => p50k_edit(),
-        // --- Fallback for Prefixes ---
-        _ => {
-            let o200k_prefixes = [
-                "o1-", "o3-", "o4-", "gpt-5-", "gpt-4.1-", "chatgpt-4o-", "gpt-4o-", "ft:gpt-4o",
-            ];
-
-            let cl100k_prefixes = [
-                "gpt-4-", "gpt-3.5-turbo-", "gpt-35-turbo-", "ft:gpt-4:",
-                "ft:gpt-4-", "ft:gpt-3.5-turbo", "ft:davinci-002", "ft:babbage-002",
-            ];
-
-
-            if o200k_prefixes.iter().any(|prefix| model.starts_with(prefix)) {
-                o200k_base()
-            } else if cl100k_prefixes.iter().any(|prefix| model.starts_with(prefix)) {
-                cl100k_base()
-            } else {
-                let err_msg = format!("Model '{}' not supported.", model);
-                return Err(Error::new(exception::arg_error(), err_msg));
+    let tokenizer_result = match model {
+        // Bare encoding names — callers may pass these directly.
+        "o200k_harmony" => o200k_harmony(),
+        "o200k_base" => o200k_base(),
+        "cl100k_base" => cl100k_base(),
+        "p50k_base" => p50k_base(),
+        "p50k_edit" => p50k_edit(),
+        "r50k_base" => r50k_base(),
+        _ => match get_tokenizer(model) {
+            Some(Tokenizer::O200kHarmony) => o200k_harmony(),
+            Some(Tokenizer::O200kBase) => o200k_base(),
+            Some(Tokenizer::Cl100kBase) => cl100k_base(),
+            Some(Tokenizer::P50kBase) => p50k_base(),
+            Some(Tokenizer::P50kEdit) => p50k_edit(),
+            Some(Tokenizer::R50kBase) | Some(Tokenizer::Gpt2) => r50k_base(),
+            None => {
+                return Err(Error::new(
+                    exception::arg_error(),
+                    format!("Model '{}' not supported.", model),
+                ));
             }
-        }
+        },
     };
-    tokenizer.map_err(|e| Error::new(exception::runtime_error(), e.to_string()))
+    tokenizer_result.map_err(|e| Error::new(exception::runtime_error(), e.to_string()))
 }
 
 // This function is exposed to Ruby.

data/j_r_r_token.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.description = "JRRToken is a Ruby gem that wraps the tiktoken Rust library, enabling fast and efficient tokenization for OpenAI models. It supports multiple models including o200k_base, cl100k_base, p50k_base, p50k_edit, and r50k_base."
   spec.homepage = "https://github.com/LoganBresnahan/j_r_r_token"
   spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.7.0"
+  spec.required_ruby_version = ">= 3.0.0"
   spec.required_rubygems_version = ">= 3.0.0"
 
   spec.metadata["allowed_push_host"] = "https://rubygems.org"

data/lib/j_r_r_token/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JRRToken
-  VERSION = "1.1.0"
+  VERSION = "1.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: j_r_r_token
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Logan Bresnahan
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-27 00:00:00.000000000 Z
+date: 2026-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,7 +90,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="