j_r_r_token 1.0.0-arm64-darwin → 1.1.0-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ceb33d317de17949aebb422787424e688042bb329d7d79234a47839419863d11
-  data.tar.gz: 4e83c6357d57fa09ff04187bd942649692e5d14316cfbea346139cae952b5b27
+  metadata.gz: 00ad7ba724d46812408a7b8a85d9386b102de6ed1efb95ba21b33f66f0b732a7
+  data.tar.gz: 953b06ce59d00dc7b4351ea3deebd479da6451c08095343b4cc38128a48f2a45
 SHA512:
-  metadata.gz: b8f2edcc0c534704248e58cd83606756dfbc9e4a92d1ff61f115e2ba770fe251e90e4b45bb7dcb9dfb6476b9494e0ea60596ed2387a723f0be08ed3a5a722c0b
-  data.tar.gz: 38136e2d54699ba0b8f4e16adc78684cd54809b8e13062226b5914464a6e08903c5494a93c0e2dcd9f50cda12b6e943b464197bda110546d28788b52b21bcef2
+  metadata.gz: f337b60a007c53b8ed640db0c5874cbdd3dc3a5e3953680b84023554a64cd6c4be570fb7d028f0ced9d1987b17e0c8faffacda08a6826aea3d7d72f1b7e057d0
+  data.tar.gz: 8d32c0f4efb44ea80c227159b039e441a22023751801da14cfe0bb8b7eb651fc3fe6ead495c1bb07111036bd543fd9f527b83387028e03a0a4fc7a5a505aac01

data/CHANGELOG.md

@@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+---
+## [1.1.0] - 2025-08-26
+### New Functionality
+- Added support for the `gpt-5` model name, which maps to the `o200k_base` tokenizer model.
+
+#### New contributors
+- [8edaf40](https://github.com/LoganBresnahan/j_r_r_token/commit/8edaf4021dae4ae41dcfa53fa5fb0fe61d9b0b9c) by [jordanmoore753](https://github.com/jordanmoore753)
+- [a9738c7](https://github.com/LoganBresnahan/j_r_r_token/commit/a9738c7f0d4e7e691af8b8219e89d09b4a3367c0) by [lsysophia](https://github.com/lsysophia)
+
+---
+## [1.0.4] - 2025-08-01
+### Change
+- Bump version for RubyGems.
+
+---
+## [1.0.3] - 2025-08-01
+### Fix
+- Add Cargo files back to the gemspec.
+### Change
+- Adjust specs.
+
+---
+## [1.0.2] - 2025-08-01
+### Change
+- Refine Model Prefix Matching so 4 and 4* match to the correct Core BPE.
+
+---
+## [1.0.1] - 2025-08-01
+### Change
+- Slim down gem by removing unnecessary files defined in the gemspec.
+- Refine Model Prefix Matching so 4 and 4* match to the correct Core BPE.
+
 ---
 ## [1.0.0] - 2025-08-01
 ### Breaking Change

data/README.md

@@ -1,6 +1,6 @@
-# JRRToken [<img src="ring.png" width="32px" alt="One ring to rule them all, one ring to find them, One ring to bring them all and in the darkness bind them.">](https://youtu.be/lemgdzLDYqA?si=Z_Ls3hbJ-Xv38oJN)
+# JRRToken [<img src="https://github.com/LoganBresnahan/j_r_r_token/raw/main/.github/assets/ring.png" width="32px" alt="One ring to rule them all, one ring to find them, One ring to bring them all and in the darkness bind them.">](https://www.youtube.com/watch?v=lemgdzLDYqA)
 
-(Just Ruby Rust Tokens)
+(Just Ruby, Rust, & Tokens)
 
 JRRToken provides a high-performance, native Ruby interface for counting tokens using the powerful [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs) library. It leverages the speed of Rust to offer a fast and efficient way to calculate token counts for various OpenAI models.
 
@@ -15,16 +15,16 @@ Extensive Model Support: Includes tokenizers for all modern and legacy OpenAI mo
 ## Install
 
 #### Gemfile
-`gem 'j_r_r_token', '~> 1.0.0'`
+`gem 'j_r_r_token', '~> 1.1'`
 
 #### Command Line
-`gem install j_r_r_token -v '~> 1.0.0'`
+`gem install j_r_r_token -v '~> 1.1'`
 
 ## Use
 
 ### Count tokens for a specific model (the model keyword is required)
 ```Ruby
-count = JRRToken::Tokenizer.count("hello world!", model: "gpt-4.1")
+count = JRRToken::Tokenizer.count("hello world!", model: "gpt-5")
 ```
 
 ### The gem recognizes many aliases, including older models
@@ -42,10 +42,10 @@ If you provide an unsupported model name, the gem will raise an `ArgumentError`.
 ## Supported Models
 The gem automatically maps dozens of model names and prefixes to the correct underlying tokenizer. You don't need to know the tokenizer's base name (e.g., cl100k_base); just use the model name you're working with.
 
-- o200k_base Models (e.g., GPT-4o)
-- cl100k_base Models (e.g., GPT-4, GPT-3.5)
+- o200k_base Models (e.g., gpt-5, gpt-4.1, gpt-4o)
+- cl100k_base Models (e.g., gpt-4, gpt-3.5)
 - p50k_base Models (e.g., text-davinci-003)
-- r50k_base Models (e.g., GPT-2)
+- r50k_base Models (e.g., gpt-2)
 - p50k_edit Models
 
 ## Developing JRRToken Locally

data/j_r_r_token.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.files = Dir.chdir(__dir__) do
     `git ls-files -z`.split("\x0").reject do |f|
       (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+        f.start_with?(*%w[bin/ test/ spec/ features/ .github/ .git .circleci .rspec appveyor Gemfile Rakefile])
     end
   end
   spec.bindir = "exe"

data/lib/j_r_r_token/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JRRToken
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: j_r_r_token
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: arm64-darwin
 authors:
 - Logan Bresnahan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2025-08-01 00:00:00.000000000 Z
+date: 2025-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -61,11 +61,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rspec"
 - CHANGELOG.md
 - LICENSE
 - README.md
-- Rakefile
 - j_r_r_token.gemspec
 - lib/j_r_r_token.rb
 - lib/j_r_r_token/2.7/j_r_r_token.bundle
@@ -75,7 +73,6 @@ files:
 - lib/j_r_r_token/3.3/j_r_r_token.bundle
 - lib/j_r_r_token/3.4/j_r_r_token.bundle
 - lib/j_r_r_token/version.rb
-- ring.png
 - sig/j_r_r_token.rbs
 homepage: https://github.com/LoganBresnahan/j_r_r_token
 licenses:

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper

data/Rakefile

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/gem_tasks"
-require "rake/extensiontask"
-require "rb_sys/extensiontask"
-
-GEMSPEC = Gem::Specification.load("j_r_r_token.gemspec")
-
-RbSys::ExtensionTask.new("j_r_r_token", GEMSPEC) do |ext|
-  ext.lib_dir = "lib/j_r_r_token"
-end
-
-# Only require rspec if it's available
-begin
-  require "rspec/core/rake_task"
-  RSpec::Core::RakeTask.new(:spec)
-  task default: %i[compile spec]
-rescue LoadError
-  puts "RSpec not available"
-  task default: :compile
-end
-
-task :native, [:platform] do |_t, platform:|
-  sh "bundle", "exec", "rb-sys-dock", "--platform", platform, "--build"
-end
-
-task build: :compile