discourse_ai-tokenizers 0.3.2 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5691c266deeffc5e632d111fdbf6fa9b54797d1f1fc6f030d53418e9a7a50394
-  data.tar.gz: 23c28ddeed6956dd051741e153b044fd2fef28c882b3efb16de14633ceca64a0
+  metadata.gz: 476e96609f0c9bd5ccbb94f5dd464df65cb125d047941689834d8ff994094d80
+  data.tar.gz: 1c987d4572105aab891e91ba86229a409d4932148521c32e180eaccbbe4fbc35
 SHA512:
-  metadata.gz: de190053755df5292b99c99fe5f758cbacd190b3c7da16379e702dd097a572ceee4b1da79e6d23e2a31e06effd2a75fd492e031139ddee6fe030b18c1267f01b
-  data.tar.gz: a3fa6e00c7e4e49244944e75b978a89f0b0ec44217d6168f1b87ea1862cb34416f571d554e5fed6995be4a96f614f72458ffc12387ddcfcbba0b62e4dfc7df4f
+  metadata.gz: 4cb19dbf675d42b4360f5beecb384a92835ffcd891f3010fcca589c1c90ba188253939aa3bb3b7b0506c72422554e69968a00a2389794febf77ab29b1b188546
+  data.tar.gz: 4bd2db8132f668ed0a8f5a71060ac5a5d254cb4020a55d2fcd629dcc0775a29616e443af71c04d007ec629ee7dfa062eeddf7f0e86a26abd3e67c5b24cac7c09

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.4.1] - 2026-02-26
+
+- Fix tiktoken-rs stack overflow crash by chunking large inputs at whitespace boundaries before encoding
+
+## [0.4.0] - 2026-01-06
+
+- Add Ruby 4.0 compatibility
+
 ## [0.3.2] - 2025-12-10
 
 - Fix truncation logic in OpenAiTokenizer could lead to string parsing fails

data/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb

@@ -4,17 +4,22 @@ module DiscourseAi
   module Tokenizer
     # Wrapper for OpenAI tokenizer library for compatibility with Discourse AI API
     class OpenAiTokenizer < BasicTokenizer
+      # tiktoken-rs uses fancy-regex which can stack overflow on large inputs
+      # due to catastrophic backtracking (github.com/openai/tiktoken/issues/245).
+      # Chunking at whitespace boundaries prevents this while preserving accuracy.
+      SAFE_CHUNK_SIZE = 50_000
+
       class << self
         def tokenizer
           @tokenizer ||= Tiktoken.get_encoding("o200k_base")
         end
 
         def tokenize(text)
-          tokenizer.encode(text)
+          safe_encode(text)
         end
 
         def encode(text)
-          tokenizer.encode(text)
+          safe_encode(text)
         end
 
         def decode(token_ids)
@@ -72,7 +77,34 @@ module DiscourseAi
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
 
-          tokenizer.encode(text).length < limit
+          safe_encode(text).length < limit
+        end
+
+        private
+
+        def safe_encode(text)
+          if !text.is_a?(String) || text.size <= SAFE_CHUNK_SIZE
+            return tokenizer.encode(text)
+          end
+
+          tokens = []
+          offset = 0
+          while offset < text.size
+            chunk_end = offset + SAFE_CHUNK_SIZE
+
+            if chunk_end < text.size
+              # Split at a whitespace boundary to preserve tokenization accuracy
+              break_point = text.rindex(/\s/, chunk_end)
+              chunk_end = break_point if break_point && break_point > offset
+            else
+              chunk_end = text.size
+            end
+
+            tokens.concat(tokenizer.encode(text[offset...chunk_end]))
+            offset = chunk_end
+          end
+
+          tokens
         end
       end
     end

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.3.2"
+    VERSION = "0.4.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.1
 platform: ruby
 authors:
 - Rafael Silva
@@ -29,42 +29,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.11.1
+        version: 0.0.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.11.1
+        version: 0.0.15
 - !ruby/object:Gem::Dependency
   name: tokenizers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.6.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.6.3
 - !ruby/object:Gem::Dependency
   name: rubocop-discourse
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.8.1
+        version: '3.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.8.1
+        version: '3.8'
 - !ruby/object:Gem::Dependency
   name: syntax_tree
   requirement: !ruby/object:Gem::Requirement