tiktoken_ruby 0.0.16-arm-linux → 0.0.17-arm-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcf37e12602d4fd40d0e772491ebd3381d1bdd6118ba11a4da0710c303315bcf
-  data.tar.gz: 87fc8e242d5597f9f1aa60ea6b8b976377b208c3dbc7bc767d1a50780c48e7b4
+  metadata.gz: ebd4c57be399d89e76b2777068873199c52172948884a8db62a83df24ad8d272
+  data.tar.gz: b30cc6f157197e24eb1ccf65f42fbccc524e22bc2587e34cdc2a1e2de085f366
 SHA512:
-  metadata.gz: a03f0c062e93f1ff3955d5466fa66721ca7dad6a319c764a1df7b3ab14efe29b31a075de68f9c64c52d49d74ce4dfd8ff09a758e6ccb0d3d4b2622e928077f40
-  data.tar.gz: e36c0a9c9d0b78584131afe6755d7c2b5cd28eafc4e5561c4e2d35afc5b1a357cb227275b173894648c15752402505baeb1ed8e74af9dbeccce5a322e6bec7e9
+  metadata.gz: e4e528f55eeb58501416eaf5b000b9ef02bc64bd2607dff76639379fe96227a50b99af3749077a8c6e9c19933a2416bb81cdb6f2224a2498712b84e345f9646c
+  data.tar.gz: cefd97d8943197f5ca84c3469df4b99bc72716320057c9d3c392658c6315aed88260200fdb1f5fab808652871dccd6b6e096241d39d5c9790f970da6bfc0dd3c

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# [v0.0.17] - 18-07-2026
+## What's Changed
+* Bump the github-actions group with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/119
+* Bump the bundler-dependencies group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/121
+* Bump the cargo group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/120
+* feat(decode): Add opt-in lossy mode and decode_bytes by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/123
+* Bump version to 0.0.17 by @gjtorikian in https://github.com/IAPark/tiktoken_ruby/pull/124
+
+
+**Full Changelog**: https://github.com/IAPark/tiktoken_ruby/compare/v0.0.16...v0.0.17
 # [v0.0.16] - 02-05-2026
 ## What's Changed
 * Bump rb-sys from 0.9.123 to 0.9.124 in the cargo group by @dependabot[bot] in https://github.com/IAPark/tiktoken_ruby/pull/107

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.16)
+    tiktoken_ruby (0.0.17)
       rb_sys (~> 0.9)
 
 GEM
@@ -9,13 +9,13 @@ GEM
   specs:
     ast (2.4.3)
     diff-lcs (1.6.2)
-    json (2.18.1)
+    json (2.20.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     minitest (6.0.1)
       prism (~> 1.5)
-    parallel (1.27.0)
-    parser (3.3.10.2)
+    parallel (1.28.0)
+    parser (3.3.11.1)
       ast (~> 2.4.1)
       racc
     prism (1.9.0)
@@ -25,9 +25,9 @@ GEM
     rake-compiler (1.3.1)
       rake
     rake-compiler-dock (1.12.0)
-    rb_sys (0.9.127)
+    rb_sys (0.9.128)
       rake-compiler-dock (= 1.12.0)
-    regexp_parser (2.11.3)
+    regexp_parser (2.12.0)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -41,18 +41,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    rubocop (1.84.2)
+    rubocop (1.87.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
-      parallel (~> 1.10)
+      parallel (>= 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
       rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.49.0)
+    rubocop-ast (1.49.1)
       parser (>= 3.3.7.2)
       prism (~> 1.7)
     rubocop-performance (1.26.1)
@@ -60,10 +60,10 @@ GEM
       rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.47.1, < 2.0)
     ruby-progressbar (1.13.0)
-    standard (1.54.0)
+    standard (1.55.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.84.0)
+      rubocop (~> 1.87.0)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)

data/README.md

@@ -17,9 +17,7 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
-Usage should be very similar to the python library. Here's a simple example
-
-Encode and decode text
+Encode and decode text:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -27,7 +25,7 @@ enc = Tiktoken.get_encoding("cl100k_base")
 enc.decode(enc.encode("hello world")) #=> "hello world"
 ```
 
-Encoders can also be retrieved by model name
+Encoders can also be retrieved by model name:
 
 ```ruby
 require 'tiktoken_ruby'
@@ -69,6 +67,27 @@ enc.encode_with_special_tokens(text)
 
 All methods round-trip correctly through `decode`.
 
+### Decoding methods
+
+- `decode(tokens, errors: :strict)` - Decodes tokens back into a UTF-8 string
+- `decode_bytes(tokens)` - Decodes tokens into their raw bytes (an `ASCII-8BIT` string), without UTF-8 validation
+
+Because BPE tokens are byte-level, a single character (an emoji, or non-Latin scripts) can span multiple tokens. Truncating a token array (like, "trim text to N tokens") can leave a prefix that is not valid UTF-8. The `errors:` option controls how those invalid sequences are handled.
+
+```ruby
+enc = Tiktoken.encoding_for_model("gpt-4o")
+tokens = enc.encode("🦄") # the emoji spans multiple tokens
+
+# :strict (default) - raise Tiktoken::UnicodeError on invalid UTF-8
+enc.decode(tokens.first(2))                    #=> raises Tiktoken::UnicodeError
+
+# :replace - substitute invalid sequences with "�" (matches Python tiktoken's default)
+enc.decode(tokens.first(2), errors: :replace)  #=> "�"
+
+# decode_bytes - get the raw bytes and handle them yourself
+enc.decode_bytes(tokens.first(2))              #=> "\xF0\x9F\xA6" (ASCII-8BIT)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/tiktoken_ruby/encoding.rb

@@ -47,11 +47,41 @@ class Tiktoken::Encoding
     @ext_base_bpe.encode_with_special_tokens(text)
   end
 
-  # Decodes the tokens back into text
+  # Decodes the tokens back into text.
+  #
+  # BPE tokens are byte-level, so a single character (an emoji, non-Latin
+  # scripts) can span multiple tokens. Truncating a token array to a limit can
+  # therefore leave a prefix that is not valid UTF-8. The +errors+ option
+  # controls how those invalid byte sequences are handled:
+  #
+  # * +:strict+ (default) - raise Tiktoken::UnicodeError on invalid UTF-8.
+  # * +:replace+ - substitute each invalid sequence with the Unicode replacement
+  #   character (U+FFFD, +�+), matching Python tiktoken's default behavior.
+  #
   # @param tokens [Array<Integer>] The tokens to decode
-  # @return [String] The decoded text
-  def decode(tokens)
-    @ext_base_bpe.decode(tokens)
+  # @param errors [Symbol] How to handle invalid UTF-8 (:strict or :replace)
+  # @return [String] The decoded text (UTF-8)
+  def decode(tokens, errors: :strict)
+    case errors
+    when :strict
+      @ext_base_bpe.decode(tokens)
+    when :replace
+      decode_bytes(tokens).force_encoding(Encoding::UTF_8).scrub("\u{FFFD}")
+    else
+      raise ArgumentError, "errors must be :strict or :replace, got #{errors.inspect}"
+    end
+  end
+
+  # Decodes the tokens back into their raw bytes, without any UTF-8 validation.
+  #
+  # The returned string has ASCII-8BIT (binary) encoding and is not guaranteed
+  # to be valid UTF-8 — useful when you want to handle invalid sequences
+  # yourself. Matches Python tiktoken's +decode_bytes+.
+  #
+  # @param tokens [Array<Integer>] The tokens to decode
+  # @return [String] The decoded bytes (ASCII-8BIT)
+  def decode_bytes(tokens)
+    @ext_base_bpe.decode_bytes(tokens)
   end
 
   private

data/lib/tiktoken_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.16"
+  VERSION = "0.0.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tiktoken_ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.17
 platform: arm-linux
 authors:
 - IAPark
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-05-02 00:00:00.000000000 Z
+date: 2026-07-19 00:00:00.000000000 Z
 dependencies: []
 description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
   used by OpenAI. It can be used to count the number of tokens in text before sending