tiktoken_ruby 0.0.13-x86_64-linux → 0.0.14.1-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 396dfc370a282d775b1b19e168a68ee061851022b747a40e6295e0f2e129fe7e
-  data.tar.gz: bae3d61446f4ec11d54088779b6dd497ba7015375e4b904076f7d2eb1885068e
+  metadata.gz: 8bc9e92f57d58214f292fc24ef4f9f48980cbfb047853b69e385d7224b53b4a3
+  data.tar.gz: 976e2a3cbc94f153d90a038e35d0272fe30c1efd1c58c86c5c4210cfdfde9532
 SHA512:
-  metadata.gz: 8ab17cc7f4c8c48a1831fd26d1ed73e29fdec920bb4793a2d89cea23ebf46f91c82a0c3752b21bb35fdba8b62b217ff5346576012075b9038c4e6b1b07ef623c
-  data.tar.gz: d3655dda73c30fdeca86621332e5d2583578f2b508a75b343db787dff44ea3489217a53e18ef552157ef8d7e5dbcd95c8663f56c87e5e0edffc24e42eacb15ba
+  metadata.gz: f72a3f4d8c0b3f3c26acbe9b050feeb82910d68411162a4e94579b456906affd6a07472120edab2a227e124c0d19c5608180eaccc129708b77918b80654dcbd9
+  data.tar.gz: f94bafff68f9f50ae11589c62fd7aa6899836bf0ba43d070d4a7268f366392ab4736731283aae14d977ab3c53aee80eb60992c85ea81596f7cc3a36bf8c020c4

data/.vscode/settings.json

@@ -4,5 +4,8 @@
   },
   "[markdown]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[github-actions-workflow]": {
+    "editor.defaultFormatter": "redhat.vscode-yaml"
   }
 }

data/CHANGELOG.md

@@ -0,0 +1,21 @@
+# [v0.0.14.1] - 20-12-2025
+## What's Changed
+* Cut v0.0.12 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/78
+* Bump magnus from 0.8.0 to 0.8.1 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/83
+* Bump actions/checkout from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/82
+* Bump standard from 1.50.0 to 1.51.1 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/81
+* Bump actions/upload-artifact from 4 to 5 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/84
+* Bump magnus from 0.8.1 to 0.8.2 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/85
+* Bump the bundler-dependencies group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/86
+* Support  by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/87
+* Bump actions/checkout from 5 to 6 in the github-actions group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/89
+* Bump standard from 1.51.1 to 1.52.0 in the bundler-dependencies group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/88
+* release GVL while encoding / decoding tokens by @tenderworks in https://github.com/IAPark/tiktoken_ruby/pull/90
+* Drop Ruby 3.1 support; automate release process by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/92
+* Rewrite history by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/93
+* Force workflow rebuild by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/94
+
+## New Contributors
+* @tenderworks made their first contribution in https://github.com/IAPark/tiktoken_ruby/pull/90
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.12...v0.0.14.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.13)
+    tiktoken_ruby (0.0.14.1)
       rb_sys (~> 0.9)
 
 GEM
@@ -9,15 +9,15 @@ GEM
   specs:
     ast (2.4.3)
     diff-lcs (1.6.2)
-    json (2.15.0)
+    json (2.16.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     minitest (5.25.5)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.5.1)
+    prism (1.6.0)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.3.1)
@@ -40,7 +40,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    rubocop (1.80.2)
+    rubocop (1.81.7)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -48,10 +48,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.48.0)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     rubocop-performance (1.25.0)
@@ -59,10 +59,10 @@ GEM
       rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.38.0, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.51.1)
+    standard (1.52.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.80.2)
+      rubocop (~> 1.81.7)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)

data/README.md

@@ -36,6 +36,39 @@ enc = Tiktoken.encoding_for_model("gpt-4")
 enc.encode("hello world").length #=> 2
 ```
 
+### Encoding methods
+
+There are three methods for encoding text:
+
+- `encode_ordinary(text)` - Encodes text, always treating special tokens as ordinary text
+- `encode(text, allowed_special: [])` - Encodes text, treating special tokens as text unless listed in `allowed_special`
+- `encode_with_special_tokens(text)` - Encodes text, recognizing and parsing all special tokens
+
+**Special tokens** are control sequences used by OpenAI models, such as `<|endoftext|>`, `<|fim_prefix|>`, `<|fim_middle|>`, and `<|fim_suffix|>`. The encoding methods differ in how they handle these sequences:
+
+```ruby
+enc = Tiktoken.get_encoding("cl100k_base")
+text = "Hello<|endoftext|>World"
+
+# encode_ordinary: treats <|endoftext|> as literal characters (9 tokens)
+enc.encode_ordinary(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode: same as encode_ordinary by default
+enc.encode(text)
+#=> [9906, 27, 91, 8862, 728, 428, 91, 29, 10343]
+
+# encode with allowed_special: recognizes the specified special token (3 tokens)
+enc.encode(text, allowed_special: ["<|endoftext|>"])
+#=> [9906, 100257, 10343]
+
+# encode_with_special_tokens: recognizes ALL special tokens (3 tokens)
+enc.encode_with_special_tokens(text)
+#=> [9906, 100257, 10343]
+```
+
+All methods round-trip correctly through `decode`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/tiktoken_ruby/encoding.rb

@@ -40,6 +40,13 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode(text, allowed_special)
   end
 
+  # Encodes the text as a list of integer tokens, including special tokens.
+  # @param text [String] The text to encode
+  # @return [Array<Integer>] The encoded tokens
+  def encode_with_special_tokens(text)
+    @ext_base_bpe.encode_with_special_tokens(text)
+  end
+
   # Decodes the tokens back into text
   # @param tokens [Array<Integer>] The tokens to decode
   # @return [String] The decoded text

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.13"
+  VERSION = "0.0.14.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14.1
 platform: x86_64-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2025-11-10 00:00:00.000000000 Z
+date: 2025-12-20 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -22,6 +22,7 @@ files:
 - ".rspec"
 - ".standard.yml"
 - ".vscode/settings.json"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -34,7 +35,6 @@ files:
 - lib/tiktoken_ruby/3.4/tiktoken_ruby.so
 - lib/tiktoken_ruby/encoding.rb
 - lib/tiktoken_ruby/version.rb
-- script/release
 - sig/tiktoken_ruby.rbs
 homepage: https://github.com/IAPark/tiktoken_ruby
 licenses:

data/script/release

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
-  echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
-  exit 1
-fi
-
-run_id=""
-# Parse arguments
-while [[ "$#" -gt 0 ]]; do
-  case $1 in
-    --run-id)
-      run_id="$2"
-      shift 2
-      ;;
-    *)
-      echo "Unknown parameter passed: $1"
-      exit 1
-      ;;
-  esac
-done
-
-if [ -z "${run_id}" ]; then
-  echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
-  exit 1
-fi
-
-version=$(grep VERSION lib/tiktoken_ruby/version.rb  | head -n 1 | cut -d'"' -f2)
-echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
-
-rm -rf pkg/cross-compiled
-gh run download "$run_id" -D pkg/cross-compiled
-
-for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
-  echo "Publishing $gem"
-  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
-done
-
-# last but not least, the uncompiled gem
-bundle exec rake package
-GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org