discourse_ai-tokenizers 0.4 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3868dc8e228ff41a7319ceb5c97b1443646bc715654ff6872f6a107680d1240
-  data.tar.gz: 1cd60cd362995d4b3a7be7495768e9ee0885a532eafaed9b482a7a1dab36e391
+  metadata.gz: c475199d403a36cf33668b1f64e030d2166034656e1c48760dfe3a6f18c3a394
+  data.tar.gz: 48e9daf8943ba37ad8ea14cd81c275c29352459dfd226b95b605dde403dee748
 SHA512:
-  metadata.gz: 632ada3172909b8e230e20742da68d98ffd0f48ccfdb7d5b79d7055dd1579e8b9ec8d163c30221a63f6581a46ae8dcf376be97e3db9c6677a8a06b247eabfa1a
-  data.tar.gz: 436f944b815606c911f98b465284860ebdd628a1a8b70deab0b37a3050d72dc18947caddbd4e47b0ad50ddfe997454f44454d91ac481a27b99cb9fe413c1d7be
+  metadata.gz: fcf2e4fa7d73348f55bfc13a6de3eded0391fdca58d6629fc672dc6856eade5adf3f196902d6ffc5b9ef01079783c651f74fe89aad41fdf26573e866a5110120
+  data.tar.gz: 98e9dcdffa14456a090ff2f3a06189f79585d8c97c79714710a17fd62491416fe63f0c6204fd4be76c5b3348d591ab4c978fade5ae5a45870de03787d014078d

data/CHANGELOG.md

@@ -1,4 +1,10 @@
-## [Unreleased]
+## [0.4.2] - 2026-02-27
+
+- Normalize `ASCII-8BIT`/non-UTF-8 string inputs before tokenization to prevent `EncodingError` in `truncate`, `encode`, and `below_limit?`
+
+## [0.4.1] - 2026-02-26
+
+- Fix tiktoken-rs stack overflow crash by chunking large inputs at whitespace boundaries before encoding
 
 ## [0.4.0] - 2026-01-06
 

data/lib/discourse_ai/tokenizer/basic_tokenizer.rb

@@ -21,7 +21,7 @@ module DiscourseAi
         end
 
         def tokenize(text)
-          tokenizer.encode(text).tokens
+          tokenizer.encode(normalize_text(text)).tokens
         end
 
         def size(text)
@@ -32,38 +32,67 @@ module DiscourseAi
           tokenizer.decode(token_ids)
         end
 
-        def encode(tokens)
-          tokenizer.encode(tokens).ids
+        def encode(text)
+          tokenizer.encode(normalize_text(text)).ids
         end
 
         def truncate(text, max_length, strict: false)
           return "" if max_length <= 0
 
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return text if !strict && text.size < max_length / 2
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenizer.encode(text).ids.take(max_length)
-          truncated_text = tokenizer.decode(truncated_tokens)
+          truncated_text = normalize_text(tokenizer.decode(truncated_tokens))
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenizer.encode(truncated_text).ids.length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = tokenizer.decode(truncated_tokens)
+            truncated_text = normalize_text(tokenizer.decode(truncated_tokens))
             break if truncated_tokens.empty?
           end
 
-          truncated_text
+          normalize_text(truncated_text)
         end
 
         def below_limit?(text, limit, strict: false)
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
 
           tokenizer.encode(text).ids.length < limit
         end
+
+        private
+
+        def normalize_text(text)
+          return text unless text.is_a?(String)
+
+          # Fast path: avoid allocations for the common valid UTF-8 case.
+          if text.encoding == Encoding::UTF_8 && text.valid_encoding?
+            return text
+          end
+
+          if text.encoding == Encoding::ASCII_8BIT
+            normalized = text.dup
+            normalized.force_encoding(Encoding::UTF_8)
+          elsif text.encoding != Encoding::UTF_8
+            normalized = text.encode(Encoding::UTF_8)
+          else
+            normalized = text
+          end
+
+          normalized.valid_encoding? ? normalized : normalized.scrub
+        rescue Encoding::UndefinedConversionError,
+               Encoding::InvalidByteSequenceError
+          text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        end
       end
     end
   end

data/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb

@@ -4,17 +4,22 @@ module DiscourseAi
   module Tokenizer
     # Wrapper for OpenAI tokenizer library for compatibility with Discourse AI API
     class OpenAiTokenizer < BasicTokenizer
+      # tiktoken-rs uses fancy-regex which can stack overflow on large inputs
+      # due to catastrophic backtracking (github.com/openai/tiktoken/issues/245).
+      # Chunking at whitespace boundaries prevents this while preserving accuracy.
+      SAFE_CHUNK_SIZE = 50_000
+
       class << self
         def tokenizer
           @tokenizer ||= Tiktoken.get_encoding("o200k_base")
         end
 
         def tokenize(text)
-          tokenizer.encode(text)
+          safe_encode(text)
         end
 
         def encode(text)
-          tokenizer.encode(text)
+          safe_encode(text)
         end
 
         def decode(token_ids)
@@ -49,30 +54,63 @@ module DiscourseAi
         def truncate(text, max_length, strict: false)
           return "" if max_length <= 0
 
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return text if !strict && text.size < max_length / 2
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenize(text).take(max_length)
-          truncated_text = decode(truncated_tokens)
+          truncated_text = normalize_text(decode(truncated_tokens))
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenize(truncated_text).length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = decode(truncated_tokens)
+            truncated_text = normalize_text(decode(truncated_tokens))
             break if truncated_tokens.empty?
           end
 
-          truncated_text
+          normalize_text(truncated_text)
         end
 
         def below_limit?(text, limit, strict: false)
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
 
-          tokenizer.encode(text).length < limit
+          safe_encode(text).length < limit
+        end
+
+        private
+
+        def safe_encode(text)
+          text = normalize_text(text)
+
+          if !text.is_a?(String) || text.size <= SAFE_CHUNK_SIZE
+            return tokenizer.encode(text)
+          end
+
+          tokens = []
+          offset = 0
+          while offset < text.size
+            chunk_end = offset + SAFE_CHUNK_SIZE
+
+            if chunk_end < text.size
+              # Split at a whitespace boundary to preserve tokenization accuracy
+              break_point = text.rindex(/\s/, chunk_end)
+              chunk_end = break_point if break_point && break_point > offset
+            else
+              chunk_end = text.size
+            end
+
+            tokens.concat(tokenizer.encode(text[offset...chunk_end]))
+            offset = chunk_end
+          end
+
+          tokens
         end
       end
     end

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.4"
+    VERSION = "0.4.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: 0.4.2
 platform: ruby
 authors:
 - Rafael Silva