discourse_ai-tokenizers 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36b5e98f002fe493df0c192a5ba86cf1a65d7c5d58207a3ee51a151c71d25002
-  data.tar.gz: c20fdaa5692731610370d9c8bf790a12ace12a5b3513d95f238e64369396dfcf
+  metadata.gz: 5691c266deeffc5e632d111fdbf6fa9b54797d1f1fc6f030d53418e9a7a50394
+  data.tar.gz: 23c28ddeed6956dd051741e153b044fd2fef28c882b3efb16de14633ceca64a0
 SHA512:
-  metadata.gz: f83d3e648f680f40c099add8596111d25e74cadb21109bcc7eb1914b12c19b42b118435cd0c99e2781002ea5090325a59b69ded692a05fc8ea98c86a6f13bd5e
-  data.tar.gz: 0bc123e4127d01bb85650147b4c56134c2789b12ec3edffa0377512482e731e1718b0811f7df18b9b8978e0f4556a10de030de1d4548496e11bc180979d9cf4b
+  metadata.gz: de190053755df5292b99c99fe5f758cbacd190b3c7da16379e702dd097a572ceee4b1da79e6d23e2a31e06effd2a75fd492e031139ddee6fe030b18c1267f01b
+  data.tar.gz: a3fa6e00c7e4e49244944e75b978a89f0b0ec44217d6168f1b87ea1862cb34416f571d554e5fed6995be4a96f614f72458ffc12387ddcfcbba0b62e4dfc7df4f

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.3.2] - 2025-12-10
+
+- Fix truncation logic in OpenAiTokenizer could lead to string parsing fails
+
 ## [0.3.1] - 2025-07-07
 
 - Refactor OpenAiO200kTokenizer class to OpenAiTokenizer as primary class name

data/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb

@@ -19,9 +19,31 @@ module DiscourseAi
 
         def decode(token_ids)
           tokenizer.decode(token_ids)
-        rescue Tiktoken::UnicodeError => e
-          # Handle invalid token IDs gracefully by returning empty string
-          ""
+        rescue Tiktoken::UnicodeError
+          token_ids = token_ids.dup
+
+          # this easy case, we started with a valid sequnce but truncated it on an invalid boundary
+          # work backwards removing tokens until we can decode again
+          tries = 4
+          while tries > 0
+            begin
+              token_ids.pop
+              return tokenizer.decode(token_ids)
+            rescue Tiktoken::UnicodeError
+              tries -= 1
+            end
+          end
+
+          # at this point we may have a corrupted sequence so just decode what we can
+          token_ids
+            .map do |id|
+              begin
+                tokenizer.decode([id])
+              rescue Tiktoken::UnicodeError
+                ""
+              end
+            end
+            .join
         end
 
         def truncate(text, max_length, strict: false)
@@ -33,12 +55,12 @@ module DiscourseAi
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenize(text).take(max_length)
-          truncated_text = tokenizer.decode(truncated_tokens)
+          truncated_text = decode(truncated_tokens)
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenize(truncated_text).length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = tokenizer.decode(truncated_tokens)
+            truncated_text = decode(truncated_tokens)
             break if truncated_tokens.empty?
           end
 

data/lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.3.1"
+    VERSION = "0.3.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: discourse_ai-tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Rafael Silva
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Unified tokenizer interface for AI/ML models supporting OpenAI, Anthropic,
   Gemini, Llama, and embedding models