ru_token 0.1.1

data/README.md

@@ -0,0 +1,50 @@
+# RuToken 🚀
+RuToken provides a high-performance, native Ruby interface for counting tokens using the powerful tiktoken-rs library. It leverages the speed of Rust to offer a fast and efficient way to calculate token counts for various OpenAI models.
+
+The gem ships with pre-compiled native extensions (except Windows), so your end-users don't need a Rust toolchain installed.
+
+Features
+High Performance ⚡️: Uses a native Rust implementation for blazing-fast tokenization.
+
+Simple API: A clean and straightforward interface for counting tokens.
+
+Extensive Model Support: Includes tokenizers for all modern and legacy OpenAI models, recognizing dozens of model aliases automatically.
+
+Pre-compiled: Ships with binaries for major platforms (Linux, macOS, Windows) and Ruby versions, removing the need for local compilation in production.
+
+## Count tokens for a specific model (the model keyword is required)
+`count = RuToken::Tokenizer.count(text, model: "gpt-4o")`
+#### => 13
+
+## The gem recognizes many aliases, including older models
+`count = RuToken::Tokenizer.count(text, model: "text-davinci-003")`
+#### => 13
+If you provide an unsupported model name, the gem will raise an ArgumentError.
+
+Supported Models
+The gem automatically maps dozens of model names and prefixes to the correct underlying tokenizer. You don't need to know the tokenizer's base name (e.g., cl100k_base); just use the model name you're working with.
+
+o200k_base Models (e.g., GPT-4o)
+cl100k_base Models (e.g., GPT-4, GPT-3.5)
+p50k_base Models (e.g., text-davinci-003)
+r50k_base Models (e.g., GPT-2)
+p50k_edit Models
+Development
+After checking out the repo, set up your environment:
+
+Bash
+
+## Install Ruby and Rust dependencies
+Ruby >= 2.7 required
+
+`bundle install`
+
+## Compile the Rust extension and run tests
+bundle exec rake
+To just compile the extension without running tests, use bundle exec rake compile. You can also open an interactive console for experimentation with bin/console.
+
+Contributing
+Bug reports and pull requests are welcome on GitHub at https://github.com/LoganBresnahan/ru_token. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the code of conduct.
+
+License
+The gem is available as open source under the terms of the GNU General Public License v3.0.

data/Rakefile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "bundler/gem_tasks"
+require "rake/clean"
+require "rake/extensiontask"
+require "rspec/core/rake_task"
+
+spec = Gem::Specification.load("ru_token.gemspec")
+
+Rake::ExtensionTask.new("ru_token", spec) do |ext|
+  ext.lib_dir = "lib/ru_token"
+end
+
+# Define a task to run your specs
+RSpec::Core::RakeTask.new(:spec)
+
+# Make the 'spec' task depend on the 'compile' task
+task spec: :compile
+
+# Set the default task to run specs
+task default: :spec

data/ext/ru_token/.cargo/config.toml

@@ -0,0 +1,9 @@
+# This file tells Rust's build system (Cargo) how to cross-compile.
+
+# Configuration for the aarch64-linux target
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+
+# Configuration for the x86_64-linux target
+[target.x86_64-unknown-linux-gnu]
+linker = "gcc"

data/ext/ru_token/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "ru_token"
+version = "0.1.0"
+edition = "2021"
+authors = ["Logan Bresnahan <loganbbres@gmail.com>"]
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.6" }
+tiktoken-rs = { version= "0.7" }

data/ext/ru_token/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("ru_token/ru_token")

data/ext/ru_token/src/lib.rs

@@ -0,0 +1,60 @@
+use magnus::{define_module, exception, function, Error};
+use tiktoken_rs::{cl100k_base, o200k_base, p50k_base, p50k_edit, r50k_base, CoreBPE};
+
+// Helper function to get the correct tokenizer based on the model name.
+// This logic is adapted from the tiktoken-rs library.
+fn get_bpe_from_model(model: &str) -> Result<CoreBPE, Error> {
+    let tokenizer = match model {
+        // --- O200k Base Models ---
+        "o200k_base" | "gpt-4.1" | "chatgpt-4o-latest" | "gpt-4o" => o200k_base(),
+        // --- Cl100k Base Models ---
+        "cl100k_base" | "gpt-4" | "gpt-3.5-turbo" | "gpt-3.5" | "gpt-35-turbo"
+        | "davinci-002" | "babbage-002" | "text-embedding-ada-002"
+        | "text-embedding-3-small" | "text-embedding-3-large" => cl100k_base(),
+        // --- P50k Base Models ---
+        "p50k_base" | "text-davinci-003" | "text-davinci-002" | "code-davinci-002"
+        | "code-davinci-001" | "code-cushman-002" | "code-cushman-001"
+        | "davinci-codex" | "cushman-codex" => p50k_base(),
+        // --- R50k Base (GPT-2) Models ---
+        "r50k_base" | "gpt2" | "gpt-2" | "text-davinci-001" | "text-curie-001"
+        | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada"
+        | "text-similarity-davinci-001" | "text-similarity-curie-001"
+        | "text-similarity-babbage-001" | "text-similarity-ada-001"
+        | "text-search-davinci-doc-001" | "text-search-curie-doc-001"
+        | "text-search-babbage-doc-001" | "text-search-ada-doc-001"
+        | "code-search-babbage-code-001" | "code-search-ada-code-001" => r50k_base(),
+        // --- P50k Edit Models ---
+        "p50k_edit" | "text-davinci-edit-001" | "code-davinci-edit-001" => p50k_edit(),
+        // --- Fallback for Prefixes ---
+        _ => {
+            if model.starts_with("o1-") || model.starts_with("o3-") || model.starts_with("o4-")
+            || model.starts_with("gpt-4.1-") || model.starts_with("chatgpt-4o-")
+            || model.starts_with("gpt-4o-") || model.starts_with("ft:gpt-4o") {
+                o200k_base()
+            } else if model.starts_with("gpt-4-") || model.starts_with("gpt-3.5-turbo-")
+            || model.starts_with("gpt-35-turbo-") || model.starts_with("ft:gpt-4")
+            || model.starts_with("ft:gpt-3.5-turbo") || model.starts_with("ft:davinci-002")
+            || model.starts_with("ft:babbage-002") {
+                cl100k_base()
+            } else {
+                let err_msg = format!("Model '{}' not supported.", model);
+                return Err(Error::new(exception::arg_error(), err_msg));
+            }
+        }
+    };
+    tokenizer.map_err(|e| Error::new(exception::runtime_error(), e.to_string()))
+}
+
+// This function is exposed to Ruby.
+fn count_tokens(model: String, text: String) -> Result<usize, Error> {
+    let bpe = get_bpe_from_model(&model)?;
+    let tokens = bpe.encode_with_special_tokens(&text);
+    Ok(tokens.len())
+}
+
+#[magnus::init]
+fn init() -> Result<(), Error> {
+    let module = define_module("RuToken")?;
+    module.define_module_function("count_tokens", function!(count_tokens, 2))?;
+    Ok(())
+}

data/lib/ru_token/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RuToken
+  VERSION = "0.1.1"
+end

data/lib/ru_token.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "ru_token/version"
+
+begin
+  require "ru_token/ru_token"
+rescue LoadError
+  warn "Failed to load ru_token native extension."
+end
+
+module RuToken
+  class Error < StandardError; end
+
+  # This is the main public interface for the gem.
+  class Tokenizer
+    # Counts tokens for a given text using a specified model name.
+    # The underlying Rust extension will handle mapping the model name
+    # to the correct tokenizer.
+    #
+    # @param text [String] The text to tokenize.
+    # @param model [String] The name of the model (e.g., "gpt-4o", "gpt-3.5-turbo").
+    # @return [Integer] The number of tokens.
+    def self.count(text, model:)
+      # This calls the `count_tokens` function from our Rust code,
+      # passing both the model and the text.
+      RuToken.count_tokens(model, text.to_s)
+    end
+  end
+end

data/sig/ru_token.rbs

@@ -0,0 +1,4 @@
+module RuToken
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: ru_token
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Logan Bresnahan
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: RuToken is a Ruby gem that wraps the tiktoken Rust library, enabling
+  fast and efficient tokenization for OpenAI models. It supports multiple models including
+  o200k_base, cl100k_base, p50k_base, and r50k_base.
+email:
+- loganbbres@gmail.com
+executables: []
+extensions:
+- ext/ru_token/Cargo.toml
+extra_rdoc_files: []
+files:
+- ".rspec"
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE
+- README.md
+- Rakefile
+- ext/ru_token/.cargo/config.toml
+- ext/ru_token/Cargo.toml
+- ext/ru_token/extconf.rb
+- ext/ru_token/src/lib.rs
+- lib/ru_token.rb
+- lib/ru_token/version.rb
+- sig/ru_token.rbs
+homepage: https://github.com/LoganBresnahan/ru_token
+licenses:
+- GPL-3.0-or-later
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/LoganBresnahan/ru_token
+  source_code_uri: https://github.com/LoganBresnahan/ru_token
+  changelog_uri: https://github.com/LoganBresnahan/ru_token/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Ruby wrapper for the tiktoken Rust library, providing fast tokenization for
+  OpenAI models.
+test_files: []