j_r_r_token 1.0.0-x86_64-linux-musl → 1.1.0-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc09b1151f597c852f9573052001af91040f8aa35be021ef77a8ffb6af719a55
-  data.tar.gz: dfbac04ea1064878bd2270904c46955e7943bfb85e8360198025c8caabeb166d
+  metadata.gz: b0dbe3f695b827041ff28504659a75dfb3ec892f21c22a14ecddb6f0ebe92d6a
+  data.tar.gz: 67932c953a2a92f120e81f7e92ac58ce9b15f208bae1ba41620f204d5342bbd2
 SHA512:
-  metadata.gz: ed1b9ce5ebcc1a913f6f879620fb2a3f9f128f2bc0ff801ab0883e53a9f0c8461d82c8f3537532ab1d43b62d16fa4298490563bc0a5c43489b4207ee240bf1ed
-  data.tar.gz: d1ea9fbe37a82eb2e7d7881b4b82b9d3eac764f150e17be380d436d23d2ddee9c24fb1610901d8d5d4dd2d609f94bd885bd2801c59001ac598932138be713139
+  metadata.gz: a217698950589dbaf8323e3f62f5b9ba3dc213a6e04201be9c31f26031abf9cce19d9c56b6b64ec87f6166af3feb7bc220b44070262268ea7d1456fbf9c0899d
+  data.tar.gz: 1ea69bb4dfacdb2dcf36064a945f1eab128df00531376098ebc5fce105d4a4d581cfa9c8260cf506f402091c8a39e6abac58eb56df329f55d42c8365748511ea

data/CHANGELOG.md

@@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+---
+## [1.1.0] - 2025-08-26
+### New Functionality
+- Added support for the `gpt-5` model name, which maps to the `o200k_base` tokenizer model.
+
+#### New contributors
+- [8edaf40](https://github.com/LoganBresnahan/j_r_r_token/commit/8edaf4021dae4ae41dcfa53fa5fb0fe61d9b0b9c) by [jordanmoore753](https://github.com/jordanmoore753)
+- [a9738c7](https://github.com/LoganBresnahan/j_r_r_token/commit/a9738c7f0d4e7e691af8b8219e89d09b4a3367c0) by [lsysophia](https://github.com/lsysophia)
+
+---
+## [1.0.4] - 2025-08-01
+### Change
+- Bump version for RubyGems.
+
+---
+## [1.0.3] - 2025-08-01
+### Fix
+- Add Cargo files back to the gemspec.
+### Change
+- Adjust specs.
+
+---
+## [1.0.2] - 2025-08-01
+### Change
+- Refine Model Prefix Matching so 4 and 4* match to the correct Core BPE.
+
+---
+## [1.0.1] - 2025-08-01
+### Change
+- Slim down gem by removing unnecessary files defined in the gemspec.
+- Refine Model Prefix Matching so 4 and 4* match to the correct Core BPE.
+
 ---
 ## [1.0.0] - 2025-08-01
 ### Breaking Change

data/README.md

@@ -1,6 +1,6 @@
-# JRRToken [<img src="ring.png" width="32px" alt="One ring to rule them all, one ring to find them, One ring to bring them all and in the darkness bind them.">](https://youtu.be/lemgdzLDYqA?si=Z_Ls3hbJ-Xv38oJN)
+# JRRToken [<img src="https://github.com/LoganBresnahan/j_r_r_token/raw/main/.github/assets/ring.png" width="32px" alt="One ring to rule them all, one ring to find them, One ring to bring them all and in the darkness bind them.">](https://www.youtube.com/watch?v=lemgdzLDYqA)
 
-(Just Ruby Rust Tokens)
+(Just Ruby, Rust, & Tokens)
 
 JRRToken provides a high-performance, native Ruby interface for counting tokens using the powerful [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs) library. It leverages the speed of Rust to offer a fast and efficient way to calculate token counts for various OpenAI models.
 
@@ -15,16 +15,16 @@ Extensive Model Support: Includes tokenizers for all modern and legacy OpenAI mo
 ## Install
 
 #### Gemfile
-`gem 'j_r_r_token', '~> 1.0.0'`
+`gem 'j_r_r_token', '~> 1.1'`
 
 #### Command Line
-`gem install j_r_r_token -v '~> 1.0.0'`
+`gem install j_r_r_token -v '~> 1.1'`
 
 ## Use
 
 ### Count tokens for a specific model (the model keyword is required)
 ```Ruby
-count = JRRToken::Tokenizer.count("hello world!", model: "gpt-4.1")
+count = JRRToken::Tokenizer.count("hello world!", model: "gpt-5")
 ```
 
 ### The gem recognizes many aliases, including older models
@@ -42,10 +42,10 @@ If you provide an unsupported model name, the gem will raise an `ArgumentError`.
 ## Supported Models
 The gem automatically maps dozens of model names and prefixes to the correct underlying tokenizer. You don't need to know the tokenizer's base name (e.g., cl100k_base); just use the model name you're working with.
 
-- o200k_base Models (e.g., GPT-4o)
-- cl100k_base Models (e.g., GPT-4, GPT-3.5)
+- o200k_base Models (e.g., gpt-5, gpt-4.1, gpt-4o)
+- cl100k_base Models (e.g., gpt-4, gpt-3.5)
 - p50k_base Models (e.g., text-davinci-003)
-- r50k_base Models (e.g., GPT-2)
+- r50k_base Models (e.g., gpt-2)
 - p50k_edit Models
 
 ## Developing JRRToken Locally

data/j_r_r_token.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.files = Dir.chdir(__dir__) do
     `git ls-files -z`.split("\x0").reject do |f|
       (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+        f.start_with?(*%w[bin/ test/ spec/ features/ .github/ .git .circleci .rspec appveyor Gemfile Rakefile])
     end
   end
   spec.bindir = "exe"

data/lib/j_r_r_token/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JRRToken
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: j_r_r_token
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: x86_64-linux-musl
 authors:
 - Logan Bresnahan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2025-08-01 00:00:00.000000000 Z
+date: 2025-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -61,11 +61,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rspec"
 - CHANGELOG.md
 - LICENSE
 - README.md
-- Rakefile
 - j_r_r_token.gemspec
 - lib/j_r_r_token.rb
 - lib/j_r_r_token/2.7/j_r_r_token.so
@@ -75,7 +73,6 @@ files:
 - lib/j_r_r_token/3.3/j_r_r_token.so
 - lib/j_r_r_token/3.4/j_r_r_token.so
 - lib/j_r_r_token/version.rb
-- ring.png
 - sig/j_r_r_token.rbs
 homepage: https://github.com/LoganBresnahan/j_r_r_token
 licenses:

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper

data/Rakefile

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/gem_tasks"
-require "rake/extensiontask"
-require "rb_sys/extensiontask"
-
-GEMSPEC = Gem::Specification.load("j_r_r_token.gemspec")
-
-RbSys::ExtensionTask.new("j_r_r_token", GEMSPEC) do |ext|
-  ext.lib_dir = "lib/j_r_r_token"
-end
-
-# Only require rspec if it's available
-begin
-  require "rspec/core/rake_task"
-  RSpec::Core::RakeTask.new(:spec)
-  task default: %i[compile spec]
-rescue LoadError
-  puts "RSpec not available"
-  task default: :compile
-end
-
-task :native, [:platform] do |_t, platform:|
-  sh "bundle", "exec", "rb-sys-dock", "--platform", platform, "--build"
-end
-
-task build: :compile