tiktoken_ruby 0.0.11.1-arm-linux → 0.0.15.1-arm-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3077759be498ef740196b12c300d1111f9a38a7c9f0d7d63d7a6b6e785a789ea
-  data.tar.gz: 6441e04eeb2c638dc663cc456d1db9decfb56ef14e143adb0731b4c39369b8b0
+  metadata.gz: 494d7edbd43e3f178cec789d7cd89627aa442a72085a7f34ad99ea04b89ef0a6
+  data.tar.gz: ae9622698383d18f32894b887423aaab2a3b79b9dc0f79f13fe9eedf0fdd49f1
 SHA512:
-  metadata.gz: b065e21826de85f30c825aa59535ed04d16254b82d8c3fff1344cbfba7a4619974b025444b1f14bc10e470b09629f0db416b1f04ae33ab244e9feae6a7c1e847
-  data.tar.gz: de67e9dae497859fec83f691375dfac1872ad395eb5188ea1595f99a3a9f0d1bd468d4e9f500544df1415bbfa9bd6124b33333bf477c4f489f6c9dc7b3dcaba5
+  metadata.gz: 6d96928e64a8d4a5306ee874707af9aae303803f690aa64f9639ab19bb3191e30316624c54e6dc8b57519c17452b7790b66eeb391505e3fa203a14e0255bd8ae
+  data.tar.gz: a365c641b38fb87dfa95e2045bb807954822da9a7cba8e1bece12de40de4acf6f9cf93883d8e2dc7c7489f585a546b0607d983bdf3076de219adcd6a848d4fa7

data/.vscode/settings.json

@@ -1,5 +1,11 @@
 {
   "[ruby]": {
     "editor.defaultFormatter": "Shopify.ruby-lsp"
+  },
+  "[markdown]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[github-actions-workflow]": {
+    "editor.defaultFormatter": "redhat.vscode-yaml"
   }
 }

data/CHANGELOG.md

@@ -0,0 +1,41 @@
+# [v0.0.15.1] - 07-01-2026
+## What's Changed
+* Re-release 0.0.15 for mistakenly yanked gems by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/105
+
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.15...v0.0.15.1
+# [v0.0.15] - 06-01-2026
+## What's Changed
+* No longer true by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/96
+* Test build on version change by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/97
+* Bump actions/cache from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/98
+* Bump rb-sys from 0.9.117 to 0.9.123 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/99
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/100
+* Add Ruby 4.0 to build matrix by @chubchenko in https://github.com/IAPark/tiktoken_ruby/pull/101
+* Support Ruby 4 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/102
+
+## New Contributors
+* @chubchenko made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/101
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.14.1...v0.0.15
+# [v0.0.14.1] - 20-12-2025
+## What's Changed
+* Cut v0.0.12 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/78
+* Bump magnus from 0.8.0 to 0.8.1 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/83
+* Bump actions/checkout from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/82
+* Bump standard from 1.50.0 to 1.51.1 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/81
+* Bump actions/upload-artifact from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/84
+* Bump magnus from 0.8.1 to 0.8.2 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/85
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/86
+* Support  by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/87
+* Bump actions/checkout from 5 to 6 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/89
+* Bump standard from 1.51.1 to 1.52.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/88
+* release GVL while encoding / decoding tokens by @tenderworks in https://github.com/IAPark/tiktoken_ruby/pull/90
+* Drop Ruby 3.1 support; automate release process by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/92
+* Rewrite history by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/93
+* Force workflow rebuild by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/94
+
+## New Contributors
+* @tenderworks made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/90
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.12...v0.0.14.1

data/Gemfile.lock

@@ -1,74 +1,81 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.11.1)
-      rb_sys (= 0.9.106)
+    tiktoken_ruby (0.0.15.1)
+      rb_sys (~> 0.9)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.2)
-    diff-lcs (1.5.1)
-    json (2.9.1)
-    language_server-protocol (3.17.0.3)
+    ast (2.4.3)
+    diff-lcs (1.6.2)
+    json (2.16.0)
+    language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
-    minitest (5.21.2)
-    parallel (1.26.3)
-    parser (3.3.6.0)
+    minitest (6.0.1)
+      prism (~> 1.5)
+    parallel (1.27.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
+    prism (1.6.0)
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.2.1)
-    rake-compiler (1.2.9)
+    rake (13.3.1)
+    rake-compiler (1.3.1)
       rake
-    rb_sys (0.9.106)
-    regexp_parser (2.10.0)
-    rspec (3.13.0)
+    rake-compiler-dock (1.10.0)
+    rb_sys (0.9.123)
+      rake-compiler-dock (= 1.10.0)
+    regexp_parser (2.11.3)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.2)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.3)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.2)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.2)
-    rubocop (1.69.2)
+    rspec-support (3.13.6)
+    rubocop (1.81.7)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.36.2, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.37.0)
-      parser (>= 3.3.1.0)
-    rubocop-performance (1.23.0)
-      rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-ast (1.48.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.43.0)
+    standard (1.52.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.69.1)
+      rubocop (~> 1.81.7)
       standard-custom (~> 1.0.0)
-      standard-performance (~> 1.6)
+      standard-performance (~> 1.8)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.6.0)
+    standard-performance (1.8.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.23.0)
-    unicode-display_width (3.1.3)
-      unicode-emoji (~> 4.0, >= 4.0.4)
-    unicode-emoji (4.0.4)
-    yard (0.9.34)
+      rubocop-performance (~> 1.25.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+    yard (0.9.37)
     yard-doctest (0.1.17)
       minitest
       yard
@@ -89,4 +96,4 @@ DEPENDENCIES
   yard-doctest
 
 BUNDLED WITH
-   2.5.18
+   2.6.9

data/README.md

@@ -5,11 +5,6 @@
 [Tiktoken](https://github.com/openai/tiktoken) is BPE tokenizer from OpenAI used with their GPT models.
 This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used.
 
-## Request for maintainers
-
-I can't really put substantial time into maintaining this. Probably nothing more than a couple hours every few months. If you have experience maintaining ruby gems and would like to
-lend a hand please send me an email or reply to this [issue](https://github.com/IAPark/tiktoken_ruby/issues/26)
-
 ## Installation
 
 Install the gem and add to the application's Gemfile by executing:
@@ -41,11 +36,44 @@ enc = Tiktoken.encoding_for_model("gpt-4")
 enc.encode("hello world").length #=> 2
 ```
 
+### Encoding methods
+
+There are three methods for encoding text:
+
+- `encode_ordinary(text)` - Encodes text, always treating special tokens as ordinary text
+- `encode(text, allowed_special: [])` - Encodes text, treating special tokens as text unless listed in `allowed_special`
+- `encode_with_special_tokens(text)` - Encodes text, recognizing and parsing all special tokens
+
+**Special tokens** are control sequences used by OpenAI models, such as `<|endoftext|>`, `<|fim_prefix|>`, `<|fim_middle|>`, and `<|fim_suffix|>`. The encoding methods differ in how they handle these sequences:
+
+```ruby
+enc = Tiktoken.get_encoding("cl100k_base")
+text = "Hello<|endoftext|>World"
+
+# encode_ordinary: treats <|endoftext|> as literal characters (9 tokens)
+enc.encode_ordinary(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode: same as encode_ordinary by default
+enc.encode(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode with allowed_special: recognizes the specified special token (3 tokens)
+enc.encode(text, allowed_special: ["<|endoftext|>"])
+#=> [9906, 100257, 10343]
+
+# encode_with_special_tokens: recognizes ALL special tokens (3 tokens)
+enc.encode_with_special_tokens(text)
+#=> [9906, 100257, 10343]
+```
+
+All methods round-trip correctly through `decode`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Contributing
 

data/lib/tiktoken_ruby/encoding.rb

@@ -40,6 +40,13 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode(text, allowed_special)
   end
 
+  # Encodes the text as a list of integer tokens, including special tokens.
+  # @param text [String] The text to encode
+  # @return [Array<Integer>] The encoded tokens
+  def encode_with_special_tokens(text)
+    @ext_base_bpe.encode_with_special_tokens(text)
+  end
+
   # Decodes the tokens back into text
   # @param tokens [Array<Integer>] The tokens to decode
   # @return [String] The decoded text

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.11.1"
+  VERSION = "0.0.15.1"
 end

data/lib/tiktoken_ruby.rb

@@ -65,7 +65,8 @@ module Tiktoken
       :p50k_base,
       :p50k_edit,
       :cl100k_base,
-      :o200k_base
+      :o200k_base,
+      :o200k_harmony
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
@@ -73,11 +74,17 @@ module Tiktoken
     # https://github.com/Congyuwang/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs#L50
     # is the source of the mapping for the Rust library
     MODEL_TO_ENCODING_NAME = {
+      # reasoning
+      o1: "o200k_base",
+      o3: "o200k_base",
+      "o4-mini": "o200k_base",
       # chat
-      "chatgpt-4o-latest": "o200k_base",
+      "gpt-4.1": "o200k_base",
+      "chatgpt-4o": "o200k_base",
       "gpt-4o": "o200k_base",
       "gpt-4": "cl100k_base",
       "gpt-3.5-turbo": "cl100k_base",
+      "gpt-3.5": "cl100k_base",  # Common shorthand
       "gpt-35-turbo": "cl100k_base",  # Azure deployment name
       # base
       "davinci-002": "cl100k_base",
@@ -124,12 +131,22 @@ module Tiktoken
     }
 
     MODEL_PREFIX_TO_ENCODING = {
+      # reasoning
+      "o1-": "o200k_base",
+      "o3-": "o200k_base",
+      "o4-": "o200k_base",
       # chat
+      "gpt-5-": "o200k_base",
+      "gpt-4.5-": "o200k_base",
+      "gpt-4.1-": "o200k_base",
+      "chatgpt-4o-": "o200k_base",
       "gpt-4o-": "o200k_base",  # e.g., gpt-4o-2024-05-13, etc.
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name
+      "gpt-oss-": "o200k_harmony",
       # fine-tuned
+      "ft:gpt-4o": "cl100k_base",
       "ft:gpt-4": "cl100k_base",
       "ft:gpt-3.5-turbo": "cl100k_base",
       "ft:davinci-002": "cl100k_base",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.11.1
+  version: 0.0.15.1
 platform: arm-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2025-01-08 00:00:00.000000000 Z
+date: 2026-01-07 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -22,6 +22,7 @@ files:
 - ".rspec"
 - ".standard.yml"
 - ".vscode/settings.json"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -29,13 +30,12 @@ files:
 - Rakefile
 - doctest_helper.rb
 - lib/tiktoken_ruby.rb
-- lib/tiktoken_ruby/3.1/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.3/tiktoken_ruby.so
 - lib/tiktoken_ruby/3.4/tiktoken_ruby.so
+- lib/tiktoken_ruby/4.0/tiktoken_ruby.so
 - lib/tiktoken_ruby/encoding.rb
 - lib/tiktoken_ruby/version.rb
-- script/release
 - sig/tiktoken_ruby.rbs
 homepage: https://github.com/IAPark/tiktoken_ruby
 licenses:
@@ -52,10 +52,10 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.5.dev
+      version: 4.1.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/script/release

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
-  echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
-  exit 1
-fi
-
-run_id=""
-# Parse arguments
-while [[ "$#" -gt 0 ]]; do
-  case $1 in
-    --run-id)
-      run_id="$2"
-      shift 2
-      ;;
-    *)
-      echo "Unknown parameter passed: $1"
-      exit 1
-      ;;
-  esac
-done
-
-if [ -z "${run_id}" ]; then
-  echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
-  exit 1
-fi
-
-version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
-echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
-
-rm -rf pkg/cross-compiled
-gh run download "$run_id" -D pkg/cross-compiled
-
-for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
-  echo "Publishing $gem"
-  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
-done
-
-# last but not least, the uncompiled gem
-bundle exec rake package
-GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org