json_mend 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0db6ff9aaecf22de21a04b136e235fa04d12d43f3c1db614f96eec2c881ce7b9
-  data.tar.gz: 00aa675a9392fdcd9d528f788144ad7a3064c689e9a6a698132c15ff0a9b4130
+  metadata.gz: 908b32104f5db7f667ee3d7f1273b43b5ae8d7e79193d40953a5d70c0e2c4afd
+  data.tar.gz: 8180522b11afd6b43414f67118361b6fc11b83d7afa0b8d8ef189d003a3995c2
 SHA512:
-  metadata.gz: 22e7132bcfe8f2109be0684afc22bcd21ac8d358dfb0f3b04ad41b14bba5853d82a9d6f1a54d996f8c4ecd9ab0fded3675c147b98fa218513336d7f1856c2a6b
-  data.tar.gz: f347eb625959326c68c2ffb283ba0d7a170a981927540d16c1b9aa526cde70fb1755cd40e8587af308ed17c83adccddca4a0ea49f986c8536419cfaf99af88bb
+  metadata.gz: b70fe8447456214a9fe13d7f2129b3d667687b652c3a7513cfb2385c90b86cffd3e2082a0bb2355610721ec5e688a83ad2ed83a7dd281cd9a9ad034c5c5ca4b1
+  data.tar.gz: 88dc8bdc36dedd7869223e5270f3c9e6e7cdfb8359700711ae1e7329355326f220d16a39b919565a5c50f9ab885dc79684a9d0c8dfb110abb329a4a7375dfd4e

data/.tool-versions

@@ -1 +1 @@
-ruby 4.0.6
+ruby 4.0.7

data/benchmark_comparison.rb

@@ -33,6 +33,16 @@ TEST_CASES = {
     label: 'Valid Single JSON',
     input: json_object
   },
+  return_objects: {
+    label: 'Valid JSON (Return Ruby Objects)',
+    input: json_object,
+    return_objects: true
+  },
+  generator_error: {
+    label: 'Valid Syntax but Invalid UTF-8 (GeneratorError)',
+    input: "{\"status\": \"ok\", \"data\": \"bad\xFFbyte\"}",
+    return_objects: true
+  },
   concatenated: {
     label: 'Concatenated JSON (x10)',
     input: json_object * 10
@@ -116,13 +126,18 @@ TEST_CASES.each_value do |data|
   puts "\n\n🔸 Scenario: #{data[:label]}"
   puts '-' * 40
 
+  # Check if this specific test requires returning objects instead of a string
+  return_objs = data.fetch(:return_objects, false)
+
   Benchmark.ips do |x|
     x.config(time: 2, warmup: 1) # Short duration for quick checks
 
     # 1. JsonMend
-    if supported?(->(i) { JsonMend.repair(i) }, data[:input])
+    mend_proc = ->(i) { JsonMend.repair(i, return_objects: return_objs) }
+
+    if supported?(mend_proc, data[:input])
       x.report('JsonMend') do
-        JsonMend.repair(data[:input])
+        JsonMend.repair(data[:input], return_objects: return_objs)
       end
     else
       puts '   JsonMend: ❌ Not Supported'
@@ -130,9 +145,12 @@ TEST_CASES.each_value do |data|
 
     # 2. json-repair
     if defined?(JSON::Repair)
-      if supported?(->(i) { JSON.repair(i) }, data[:input])
+      # For a fair comparison, if return_objects is true, json-repair must also parse its own output
+      repair_proc = ->(i) { return_objs ? JSON.parse(JSON.repair(i)) : JSON.repair(i) }
+
+      if supported?(repair_proc, data[:input])
         x.report('json-repair') do
-          JSON.repair(data[:input])
+          return_objs ? JSON.parse(JSON.repair(data[:input])) : JSON.repair(data[:input])
         end
       else
         puts '   json-repair: ❌ Not Supported'

data/lib/json_mend/parser.rb

@@ -7,9 +7,9 @@ module JsonMend
   # The core parser that does the heavy lifting of fixing the JSON
   class Parser
     MAX_ALLOWED_DEPTH = 100
-    COMMENT_DELIMETERS = ['#', '/'].freeze
+    COMMENT_DELIMETERS = '#/'
     NUMBER_CHARS = Set.new('0123456789-.eE/,_'.chars).freeze
-    STRING_DELIMITERS = ['"', "'", '“', '”'].freeze
+    STRING_DELIMITERS = "\"'“”"
     SKIP_CHARS_REGEX_CACHE = {
       '"' => /"/,
       "'" => /'/,
@@ -29,12 +29,12 @@ module JsonMend
 
     # Optimized constants for performance (CollectionLiteralInLoop)
     TERMINATORS_ARRAY = [']', '}'].freeze
-    TERMINATORS_OBJECT_KEY = [':', '}'].freeze
-    TERMINATORS_OBJECT_VALUE = [',', '}'].freeze
+    TERMINATORS_OBJECT_KEY = ':}'
+    TERMINATORS_OBJECT_VALUE = ',}'
     TERMINATORS_ARRAY_ITEM = [',', ']'].freeze
-    TERMINATORS_STRING_GUESSED = ['{', '}', '[', ']', ':', ','].freeze
-    TERMINATORS_VALUE = [',', ']', '}'].freeze
-    STRING_OR_OBJECT_START = (STRING_DELIMITERS + ['{', '[']).freeze
+    TERMINATORS_STRING_GUESSED = '{}[],:'
+    TERMINATORS_VALUE = ',]}'
+    STRING_OR_OBJECT_START = "#{STRING_DELIMITERS}{[".freeze
     SKIPPED_KEYS = %i[merged_array stray_colon].freeze
     BOOLEAN_OR_NULL_CHARS = %w[t f n].freeze
     ESCAPE_START_CHARS = %w[t n r b \\].freeze
@@ -54,6 +54,7 @@ module JsonMend
     def initialize(json_string)
       @scanner = StringScanner.new(json_string)
       @context = []
+      @context_counts = Hash.new(0)
       @current_context = nil
       @depth = 0
     end
@@ -140,12 +141,12 @@ module JsonMend
         when '['
           @scanner.getch # consume '['
           return parse_array
-        when *COMMENT_DELIMETERS
+        when '#', '/' # sync with COMMENT_DELIMETERS, not used *COMMENT_DELIMETERS for branch speed optimization
           # Avoid recursion: consume comment and continue loop
           parse_comment
         else
           if string_start?(char)
-            if @context.empty? && !STRING_DELIMITERS.include?(char)
+            if @context.empty? && char && !STRING_DELIMITERS.include?(char)
               # Top level unquoted string strictness:
               # Only allow literals (true/false/null), ignore other text as garbage
               val = parse_literal
@@ -163,7 +164,7 @@ module JsonMend
           else
             # Stop if we hit a terminator for the current context to avoid consuming it as garbage
             if (current_context?(:array) && char == ']') ||
-               (current_context?(:object_value) && TERMINATORS_OBJECT_VALUE.include?(char)) ||
+               (current_context?(:object_value) && char && TERMINATORS_OBJECT_VALUE.include?(char)) ||
                (current_context?(:object_key) && char == '}')
               return JSON_STOP_TOKEN
             end
@@ -187,7 +188,7 @@ module JsonMend
 
           # Explicitly consume comments to ensure they don't hide separators (like commas)
           # or get parsed as part of the next key.
-          if COMMENT_DELIMETERS.include?(peek_char)
+          if peek_char && COMMENT_DELIMETERS.include?(peek_char)
             parse_comment
             next
           end
@@ -252,17 +253,6 @@ module JsonMend
       # If we get an empty key and the next character is a closing brace, we're done.
       return [nil, nil, false] if key.empty? && (peek_char.nil? || peek_char == '}' || @scanner.pos == pos_before_key)
 
-      # Handle Duplicate Keys (Safer Method)
-      # This is a critical repair for lists of objects missing a comma separator.
-      if object.key?(key)
-        # Instead of rewriting the string, we safely rewind the scanner to the
-        # position before the duplicate key. This ends the parsing of the current
-        # object, allowing the top-level parser to see the duplicate key as the
-        # start of a new JSON object.
-        @scanner.pos = pos_before_key
-        return [nil, nil, false] # Signal to stop parsing this object.
-      end
-
       # Parse the Separator (:)
       skip_whitespaces
       colon_found = @scanner.skip(/:/) # Leniently skip the colon if it exists.
@@ -375,14 +365,14 @@ module JsonMend
           char = peek_char
 
           # Check for comments explicitly inside array to avoid recursion or garbage consumption issues
-          if COMMENT_DELIMETERS.include?(char)
+          if char && COMMENT_DELIMETERS.include?(char)
             parse_comment
             char = peek_char
             next
           end
 
           value = ''
-          if STRING_DELIMITERS.include?(char)
+          if char && STRING_DELIMITERS.include?(char)
             # Sometimes it can happen that LLMs forget to start an object and then you think it's a string in an array
             # So we are going to check if this string is followed by a : or not
             # And either parse the string or parse the object
@@ -424,7 +414,7 @@ module JsonMend
       char = peek_char
 
       # A valid string can only start with a valid quote or, in our case, with a literal
-      while !@scanner.eos? && !STRING_DELIMITERS.include?(char) && !char&.match?(/[\p{L}0-9$_-]/)
+      while !@scanner.eos? && char && !STRING_DELIMITERS.include?(char) && !char&.match?(/[\p{L}0-9$_-]/)
         return '' if TERMINATORS_STRING_GUESSED.include?(char)
 
         @scanner.getch
@@ -512,13 +502,13 @@ module JsonMend
       doubled_quotes = false
 
       # There is sometimes a weird case of doubled quotes, we manage this also later in the while loop
-      if STRING_DELIMITERS.include?(peek_char) && peek_char == lstring_delimiter
+      if peek_char && STRING_DELIMITERS.include?(peek_char) && peek_char == lstring_delimiter
         next_value = peek_char(1)
 
         if (
           current_context?(:object_key) && next_value == ':'
         ) || (
-          current_context?(:object_value) && TERMINATORS_OBJECT_VALUE.include?(next_value)
+          current_context?(:object_value) && next_value && TERMINATORS_OBJECT_VALUE.include?(next_value)
         )
           @scanner.getch
           return [true, '']
@@ -537,11 +527,13 @@ module JsonMend
           # Ok this is not a doubled quote, check if this is an empty string or not
           i = skip_whitespaces_at(start_idx: 1)
           next_c = peek_char(i)
-          if STRING_OR_OBJECT_START.include?(next_c)
-            @scanner.getch
-            return [true, '']
-          elsif !TERMINATORS_VALUE.include?(next_c)
-            @scanner.getch
+          if next_c
+            if STRING_OR_OBJECT_START.include?(next_c)
+              @scanner.getch
+              return [true, '']
+            elsif !TERMINATORS_VALUE.include?(next_c)
+              @scanner.getch
+            end
           end
         end
       end
@@ -582,7 +574,7 @@ module JsonMend
           missing_quotes
         )
 
-        if current_context?(:object_value) && TERMINATORS_OBJECT_VALUE.include?(char) &&
+        if current_context?(:object_value) && char && TERMINATORS_OBJECT_VALUE.include?(char) &&
            (string_parts.empty? || string_parts[-1] != rstring_delimiter)
 
           is_break = check_rstring_delimiter_missing(
@@ -712,7 +704,7 @@ module JsonMend
         check_comma_in_object_value = false if check_comma_in_object_value && next_c.match?(/\p{L}/)
 
         # If we are in an object context, let's check for the right delimiters
-        if (context_contain?(:object) && TERMINATORS_OBJECT_KEY.include?(next_c)) ||
+        if (context_contain?(:object) && next_c && TERMINATORS_OBJECT_KEY.include?(next_c)) ||
            (context_contain?(:array) && TERMINATORS_ARRAY_ITEM.include?(next_c)) ||
            (
              check_comma_in_object_value &&
@@ -737,7 +729,7 @@ module JsonMend
         i += 1
         i = skip_whitespaces_at(start_idx: i)
         next_c = peek_char(i)
-        return [true, false] if TERMINATORS_OBJECT_VALUE.include?(next_c)
+        return [true, false] if next_c && TERMINATORS_OBJECT_VALUE.include?(next_c)
       elsif next_c == rstring_delimiter && peek_char(i - 1) != '\\'
         # Check if self.index:self.index+i is only whitespaces
         return [false, false] if skip_whitespaces_at(start_idx: 1) >= i
@@ -783,7 +775,7 @@ module JsonMend
         prev_byte_idx = @scanner.pos - next_c.bytesize - 1
         is_escaped = prev_byte_idx >= 0 && @scanner.string.getbyte(prev_byte_idx) == 92 # 92 is backslash
 
-        break if TERMINATORS_VALUE.include?(next_c) || (next_c == rstring_delimiter && !is_escaped)
+        break if (next_c && TERMINATORS_VALUE.include?(next_c)) || (next_c == rstring_delimiter && !is_escaped)
 
         index += 1
       end
@@ -807,19 +799,7 @@ module JsonMend
 
       # Scan forward linearly
       while (c = @scanner.getch)
-        next if c != rstring_delimiter
-
-        # Check if escaped (count preceding backslashes)
-        bk = 1
-        slashes = 0
-        while (@scanner.pos - 1 - bk >= 0) &&
-              (char_code = @scanner.string.getbyte(@scanner.pos - 1 - bk)) &&
-              char_code == 92 # 92 is backslash
-          slashes += 1
-          bk += 1
-        end
-
-        if slashes.even?
+        if c == rstring_delimiter
           found_next = true
           break
         end
@@ -846,7 +826,11 @@ module JsonMend
         # Jump directly to the exact byte offset after the second quote!
         @scanner.pos = pos_after_second_quote
         @scanner.skip(/\s+/)
-        is_next_closer = TERMINATORS_VALUE.include?(@scanner.check(/./))
+
+        # Safely check the next character using the nil guard
+        next_char = @scanner.check(/./)
+        is_next_closer = next_char && TERMINATORS_VALUE.include?(next_char)
+
         @scanner.pos = saved_pos
       end
 
@@ -987,11 +971,7 @@ module JsonMend
               string_parts << "\uFFFD"
             else
               # Regular code point or hex escape
-              begin
-                string_parts << hex_val.chr('UTF-8')
-              rescue RangeError
-                string_parts << "\uFFFD"
-              end
+              string_parts << hex_val.chr('UTF-8')
             end
 
             # Scanner is already advanced past digits
@@ -1107,7 +1087,7 @@ module JsonMend
       end
 
       # Handle cases where the number ends with one or more invalid characters.
-      if !scanned_str.empty? && scanned_str.match?(INVALID_NUMBER_TRAILERS_REGEX)
+      if !scanned_str.empty? && scanned_str.end_with?(*INVALID_NUMBER_TRAILERS)
         # Do not rewind scanner, simply discard the invalid trailing chars (garbage)
         scanned_str.sub!(INVALID_NUMBER_TRAILERS_REGEX, '')
       end
@@ -1221,13 +1201,7 @@ module JsonMend
     # It quickly iterates to find a character, handling escaped characters, and
     # returns the index (offset) from the scanner
     def skip_to_character(characters, start_idx: 0)
-      pattern = SKIP_CHARS_REGEX_CACHE.fetch(characters, nil)
-      # :nocov:
-      if pattern.nil?
-        chars = Array(characters).map { |c| Regexp.escape(c.to_s) }
-        pattern = Regexp.new(chars.join('|'))
-      end
-      # :nocov:
+      pattern = SKIP_CHARS_REGEX_CACHE.fetch(characters)
 
       saved_pos = @scanner.pos
       # Skip start_idx
@@ -1329,11 +1303,13 @@ module JsonMend
 
     def push_context(value)
       @context.push(value)
+      @context_counts[value] += 1
       @current_context = value
     end
 
     def pop_context
-      @context.pop
+      popped_value = @context.pop
+      @context_counts[popped_value] -= 1
       @current_context = @context.last
     end
 
@@ -1342,17 +1318,17 @@ module JsonMend
     end
 
     def context_contain?(value)
-      @context.include?(value)
+      @context_counts[value].positive?
     end
 
     # Checks if the character signifies the start of a string or literal
     def string_start?(char)
-      STRING_DELIMITERS.include?(char) || (char && STRING_START_REGEX.match?(char))
+      (char && STRING_DELIMITERS.include?(char)) || (char && STRING_START_REGEX.match?(char))
     end
 
     # Checks if the character signifies the start of a number
     def number_start?(char)
-      char&.match?(/\d/) || char == '-' || char == '.'
+      (char && char >= '0' && char <= '9') || char == '-' || char == '.'
     end
   end
 end

data/lib/json_mend/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JsonMend
-  VERSION = '0.3.6'
+  VERSION = '0.3.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: json_mend
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Oleksii Vasyliev
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Repair broken JSON
 test_files: []