discourse_ai-tokenizers 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 476e96609f0c9bd5ccbb94f5dd464df65cb125d047941689834d8ff994094d80
-  data.tar.gz: 1c987d4572105aab891e91ba86229a409d4932148521c32e180eaccbbe4fbc35
+  metadata.gz: c475199d403a36cf33668b1f64e030d2166034656e1c48760dfe3a6f18c3a394
+  data.tar.gz: 48e9daf8943ba37ad8ea14cd81c275c29352459dfd226b95b605dde403dee748
 SHA512:
-  metadata.gz: 4cb19dbf675d42b4360f5beecb384a92835ffcd891f3010fcca589c1c90ba188253939aa3bb3b7b0506c72422554e69968a00a2389794febf77ab29b1b188546
-  data.tar.gz: 4bd2db8132f668ed0a8f5a71060ac5a5d254cb4020a55d2fcd629dcc0775a29616e443af71c04d007ec629ee7dfa062eeddf7f0e86a26abd3e67c5b24cac7c09
+  metadata.gz: fcf2e4fa7d73348f55bfc13a6de3eded0391fdca58d6629fc672dc6856eade5adf3f196902d6ffc5b9ef01079783c651f74fe89aad41fdf26573e866a5110120
+  data.tar.gz: 98e9dcdffa14456a090ff2f3a06189f79585d8c97c79714710a17fd62491416fe63f0c6204fd4be76c5b3348d591ab4c978fade5ae5a45870de03787d014078d

data/CHANGELOG.md

@@ -1,4 +1,6 @@
-## [Unreleased]
+## [0.4.2] - 2026-02-27
+
+- Normalize `ASCII-8BIT`/non-UTF-8 string inputs before tokenization to prevent `EncodingError` in `truncate`, `encode`, and `below_limit?`
 
 ## [0.4.1] - 2026-02-26
 

data/lib/discourse_ai/tokenizer/basic_tokenizer.rb

@@ -21,7 +21,7 @@ module DiscourseAi
         end
 
         def tokenize(text)
-          tokenizer.encode(text).tokens
+          tokenizer.encode(normalize_text(text)).tokens
         end
 
         def size(text)
@@ -32,38 +32,67 @@ module DiscourseAi
           tokenizer.decode(token_ids)
         end
 
-        def encode(tokens)
-          tokenizer.encode(tokens).ids
+        def encode(text)
+          tokenizer.encode(normalize_text(text)).ids
         end
 
         def truncate(text, max_length, strict: false)
           return "" if max_length <= 0
 
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return text if !strict && text.size < max_length / 2
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenizer.encode(text).ids.take(max_length)
-          truncated_text = tokenizer.decode(truncated_tokens)
+          truncated_text = normalize_text(tokenizer.decode(truncated_tokens))
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenizer.encode(truncated_text).ids.length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = tokenizer.decode(truncated_tokens)
+            truncated_text = normalize_text(tokenizer.decode(truncated_tokens))
             break if truncated_tokens.empty?
           end
 
-          truncated_text
+          normalize_text(truncated_text)
         end
 
         def below_limit?(text, limit, strict: false)
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
 
           tokenizer.encode(text).ids.length < limit
         end
+
+        private
+
+        def normalize_text(text)
+          return text unless text.is_a?(String)
+
+          # Fast path: avoid allocations for the common valid UTF-8 case.
+          if text.encoding == Encoding::UTF_8 && text.valid_encoding?
+            return text
+          end
+
+          if text.encoding == Encoding::ASCII_8BIT
+            normalized = text.dup
+            normalized.force_encoding(Encoding::UTF_8)
+          elsif text.encoding != Encoding::UTF_8
+            normalized = text.encode(Encoding::UTF_8)
+          else
+            normalized = text
+          end
+
+          normalized.valid_encoding? ? normalized : normalized.scrub
+        rescue Encoding::UndefinedConversionError,
+               Encoding::InvalidByteSequenceError
+          text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        end
       end
     end
   end

data/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb

@@ -54,25 +54,29 @@ module DiscourseAi
         def truncate(text, max_length, strict: false)
           return "" if max_length <= 0
 
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return text if !strict && text.size < max_length / 2
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenize(text).take(max_length)
-          truncated_text = decode(truncated_tokens)
+          truncated_text = normalize_text(decode(truncated_tokens))
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenize(truncated_text).length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = decode(truncated_tokens)
+            truncated_text = normalize_text(decode(truncated_tokens))
             break if truncated_tokens.empty?
           end
 
-          truncated_text
+          normalize_text(truncated_text)
         end
 
         def below_limit?(text, limit, strict: false)
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
@@ -83,6 +87,8 @@ module DiscourseAi
         private
 
         def safe_encode(text)
+          text = normalize_text(text)
+
           if !text.is_a?(String) || text.size <= SAFE_CHUNK_SIZE
             return tokenizer.encode(text)
           end

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.4.1"
+    VERSION = "0.4.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Rafael Silva