json-repair 0.11.3 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69085d74f416811c4ac11ca7cfe2e9545a6cecdaeb32de96532932c99ab4aaf3
-  data.tar.gz: 4deee8e6715200ae693144a2c8cab914b9e8c78c3b539671f7632ce39d7b77f3
+  metadata.gz: cf5d06053ce264da6b60de4beef4a5718fd3482c81d295deea6dbf90fc63c098
+  data.tar.gz: 9d343f549cf34414e0de45618cb19bd4a9b58b12e53c2f52da0380fc73b4d95a
 SHA512:
-  metadata.gz: 31242bd165c070b1836d85a3fca120b5853a6e1ed715dbaeb75aa026563e033a7af491e318c119f3f3f899c4a7bb55382fc998372d7b55953358dc95d9c526be
-  data.tar.gz: b8a5a58a36d1c2b36922205f2b3b8d33e89e570fc215ed02ef288577a8d1325d96fa21cda9267ca78f44754b00a3150e5f6d149d6087c9f020e23090d47a13f2
+  metadata.gz: 269f7de75ec6cadde857d2f7c10186b1d0c63a19aa3e863744957c95b98db8d0cef790deef8814d532ca62c0405d97cfc5f441c37be9f1c4e6c7f5cfe07eaa67
+  data.tar.gz: 9cbc099462defe41b8bf96f590833d61de90f79c8e5b0f2d8a7d0ae04ee72dcc7e512c6f3886f67cb0e700dfb1046abd143b2adbde560c502d25700035e69834

data/CHANGELOG.md

@@ -1,5 +1,58 @@
 # Changes
 
+### 2026-06-12 (0.13.0)
+
+* Repair `#` hash line comments, like in Python, YAML, or Hjson:
+  `{"a": 1 # comment\n}` → `{"a":1}`, `{ # note\n "a": 1}` →
+  `{"a":1}`, `# lead\n{"a": 1}` → `{"a":1}`. Divergence from upstream
+  [jsonrepair](https://github.com/josdejong/jsonrepair) (v3.14.0
+  raises on all of these), commented at the site. Recognition is
+  context-aware so unquoted values starting with `#` keep repairing
+  into strings — `{"color": #ff0000}` → `{"color":"#ff0000"}`,
+  `{#tag: 1}` → `{"#tag":1}`, `#standalone` → `"#standalone"` — where
+  Python's `json_repair` silently loses them (`{"color": ""}`).
+  Where a value or key is expected, a `#` token reaching a structural
+  delimiter (`,` `}` `]` `:`) before any whitespace, or running to
+  end-of-input without a newline, stays a value; anything else is a
+  comment stripped to the end of the line. The tradeoff: a `#` token
+  followed by whitespace or a newline at a value position now reads
+  as a comment — `{"a": #b c}` → `{"a":null}` and `{"a": #tag\n}` →
+  `{"a":null}`, where 0.12.0 kept them as strings (Python drops them
+  too), and a comment-only document now raises like `// only a
+  comment` always has. Pinned in the spec suite as conscious
+  decisions.
+
+### 2026-06-12 (0.12.0)
+
+* Repair the three known input families that raised `Internal error:
+  repaired output is not valid JSON` — cases where upstream
+  [jsonrepair](https://github.com/josdejong/jsonrepair) (v3.14.0, still
+  its latest release) emits invalid JSON and this gem's canonical
+  re-serialize guard caught it but blamed the Repairer. All three are
+  deliberate divergences from upstream, commented at each site:
+  * A stray `e`/`E` with no mantissa is now an unquoted string instead
+    of an empty-mantissa exponent: `[e]` → `["e"]`, `[e5]` → `["e5"]`,
+    `[truee]` → `[true,"e"]`, `{"k": e}` → `{"k":"e"}` (upstream emits
+    `e0` / raw `e5`). Numbers truncated at a real exponent (`[2e]` →
+    `[2.0]`) are unchanged.
+  * Negative leading-zero numbers are quoted like positive ones:
+    `{"n": -05}` → `{"n":"-05"}`, matching the existing `{"n": 05}` →
+    `{"n":"05"}` (upstream emits `-05` unrepaired). The same rule now
+    also covers the truncated-number repair, which bypassed it:
+    `[05e]` → `["05e0"]`, `00.` → `"00.0"` (upstream emits `05e0` /
+    `00.0` unrepaired). Valid `-0` / `-0.5` / `0e` / `0.` are
+    unchanged.
+  * The trailing-comma repair no longer strips a comma belonging to the
+    enclosing container when an inner object/array fails on its first
+    key or value: `[{{]` → `[{},{}]`, `[1,[}]` → `[1,[]]`,
+    `{"a": 1, "b": [}` → `{"a":1,"b":[]}` (upstream emits `[{}{}]`,
+    `[1[]]`, `{"a": 1 "b": []}`).
+  Validated by differential testing against upstream over a 270-input
+  grid of these shapes in every container context: the only behavior
+  changes vs 0.11.3 are the 123 previously-`Internal error` inputs now
+  repairing (or, for `e+` shapes where upstream emits invalid `e+0`,
+  raising a clean position-bearing error). Benchmarks flat.
+
 ### 2026-06-12 (0.11.3)
 
 * Fix infinite recursion (`SystemStackError`) on a quoted string

data/lib/json/repair/version.rb

@@ -2,6 +2,6 @@
 
 module JSON
   module Repair
-    VERSION = '0.11.3'
+    VERSION = '0.13.0'
   end
 end

data/lib/json/repairer.rb

@@ -71,7 +71,7 @@ module JSON
       # repair redundant end quotes
       while [CLOSING_BRACE, CLOSING_BRACKET].include?(@json[@index])
         @index += 1
-        parse_whitespace_and_skip_comments
+        parse_whitespace_and_skip_comments(value_expected: false)
       end
 
       if @index >= @json.length
@@ -93,17 +93,17 @@ module JSON
                 parse_keywords ||
                 parse_unquoted_string(false) ||
                 parse_regex
-      parse_whitespace_and_skip_comments
+      parse_whitespace_and_skip_comments(value_expected: false)
 
       process
     end
 
-    def parse_whitespace_and_skip_comments(skip_newline: true)
+    def parse_whitespace_and_skip_comments(skip_newline: true, value_expected: true)
       start = @index
 
       changed = parse_whitespace(skip_newline: skip_newline)
       loop do
-        changed = parse_comment
+        changed = parse_comment(value_expected: value_expected)
         changed = parse_whitespace(skip_newline: skip_newline) if changed
         break unless changed
       end
@@ -131,7 +131,7 @@ module JSON
       false
     end
 
-    def parse_comment
+    def parse_comment(value_expected: true)
       if @json[@index] == '/' && @json[@index + 1] == '*'
         # Block comment
         @index += 2
@@ -143,11 +143,42 @@ module JSON
         @index += 2
         @index += 1 until @json[@index].nil? || @json[@index] == "\n"
         true
+      elsif @json[@index] == '#' && hash_comment?(value_expected)
+        # Hash line comment, like in Python, YAML, or Hjson (divergence
+        # from upstream, which raises on `#` as of v3.14.0)
+        @index += 1
+        @index += 1 until @json[@index].nil? || @json[@index] == "\n"
+        true
       else
         false
       end
     end
 
+    # Decide whether the `#` at @index starts a line comment or an
+    # unquoted value like {"color": #ff0000}, {#tag: 1}, or a root
+    # #hashtag (which Python's json_repair eats as comments, losing
+    # data). Where no value or key is expected an unquoted token would
+    # be junk anyway, so `#` is always a comment, exactly like `//`.
+    # Where one is expected, scan the rest of the line: a structural
+    # delimiter (`,` `}` `]` `:`) before any whitespace means the token
+    # reads as a value in context; whitespace (including the newline
+    # itself) first means comment prose; reaching EOF without a newline
+    # keeps the token a value, so truncated input like {"a": #tag is
+    # repaired, not dropped. Divergence from upstream, as above.
+    def hash_comment?(value_expected)
+      return true unless value_expected
+
+      i = @index + 1
+      while (char = @json[i])
+        return true if whitespace_or_special?(char)
+        return false if [COMMA, COLON, CLOSING_BRACE, CLOSING_BRACKET].include?(char)
+
+        i += 1
+      end
+
+      false
+    end
+
     # Find and skip over a Markdown fenced code block:
     #     ``` ... ```
     # or
@@ -237,6 +268,7 @@ module JSON
 
       initial = true
       while @index < @json.length && @json[@index] != CLOSING_BRACE
+        first_pair = initial
         if initial
           initial = false
         else
@@ -255,15 +287,19 @@ module JSON
           if @json[@index] == CLOSING_BRACE || @json[@index] == OPENING_BRACE ||
              @json[@index] == CLOSING_BRACKET || @json[@index] == OPENING_BRACKET ||
              @json[@index].nil?
-            # repair trailing comma
-            @output = strip_last_occurrence(@output, ',')
+            # repair trailing comma — but only the one this object's own loop
+            # emitted or inserted; on the first pair the buffer's last
+            # comma belongs to the enclosing container, like in [{{] or
+            # {"a": 1, "b": {] (divergence from upstream, which strips
+            # the parent's comma and emits invalid JSON like [{}{}])
+            @output = strip_last_occurrence(@output, ',') unless first_pair
           else
             throw_object_key_expected
           end
           break
         end
 
-        parse_whitespace_and_skip_comments
+        parse_whitespace_and_skip_comments(value_expected: false)
         processed_colon = parse_character(COLON)
         truncated_text = @index >= @json.length
         unless processed_colon
@@ -486,7 +522,7 @@ module JSON
           @index += 1
           @output << str
 
-          parse_whitespace_and_skip_comments(skip_newline: false)
+          parse_whitespace_and_skip_comments(skip_newline: false, value_expected: false)
 
           if stop_at_delimiter ||
              @index >= @json.length ||
@@ -738,7 +774,13 @@ module JSON
         @index += 1 while digit?(@json[@index])
       end
 
-      if @json[@index] && @json[@index].downcase == 'e'
+      # Divergence from upstream: only enter the exponent branch when a
+      # mantissa was consumed — at this point @index > start implies at
+      # least one digit (the '-' and '.' paths reset otherwise). Upstream
+      # accepts a bare "e"/"E" here and emits invalid JSON like `e0` or
+      # raw `e5`; declining lets the token fall through to
+      # parse_unquoted_string, matching how "-e5" already becomes "-e5".
+      if @index > start && @json[@index] && @json[@index].downcase == 'e'
         @index += 1
         @index += 1 if ['-', '+'].include?(@json[@index])
         if at_end_of_number?
@@ -761,7 +803,9 @@ module JSON
       if @index > start
         # repair a number with leading zeros like "00789"
         num = @json[start...@index]
-        has_invalid_leading_zero = num.match?(/^0\d/)
+        # the optional sign quotes "-05" like "05" (divergence from
+        # upstream, whose unsigned check lets "-05" through unrepaired)
+        has_invalid_leading_zero = num.match?(/^-?0\d/)
 
         @output << (has_invalid_leading_zero ? "\"#{num}\"" : repair_leading_dot_number(num))
         return true
@@ -786,6 +830,7 @@ module JSON
 
         initial = true
         while @index < @json.length && @json[@index] != CLOSING_BRACKET
+          first_item = initial
           if initial
             initial = false
           else
@@ -799,8 +844,12 @@ module JSON
           processed_value = parse_value
           next if processed_value
 
-          # repair trailing comma
-          @output = strip_last_occurrence(@output, ',')
+          # repair trailing comma — but only the one this array's own loop
+          # emitted or inserted; on the first item the buffer's last
+          # comma belongs to the enclosing container, like in [1,[}] or
+          # {"a": 1, "b": [} (divergence from upstream, which strips
+          # the parent's comma and emits invalid JSON like [1[]])
+          @output = strip_last_occurrence(@output, ',') unless first_item
           break
         end
 
@@ -828,11 +877,11 @@ module JSON
     def parse_concatenated_string
       processed = false
 
-      parse_whitespace_and_skip_comments
+      parse_whitespace_and_skip_comments(value_expected: false)
       while @json[@index] == PLUS
         processed = true
         @index += 1
-        parse_whitespace_and_skip_comments
+        parse_whitespace_and_skip_comments(value_expected: false)
 
         # repair: remove the end quote of the first string
         @output = strip_last_occurrence(@output, '"', strip_remaining_text: true)
@@ -859,7 +908,11 @@ module JSON
       # repair numbers cut off at the end
       # this will only be called when we end after a '.', '-', or 'e' and does not
       # change the number more than it needs to make it valid JSON
-      @output << repair_leading_dot_number("#{@json[start...@index]}0")
+      num = "#{@json[start...@index]}0"
+      # quote a padded token that has an invalid leading zero, like "05e" ->
+      # "05e0", applying the same rule as the end of parse_number (divergence
+      # from upstream, which emits the invalid number raw)
+      @output << (num.match?(/^-?0\d/) ? "\"#{num}\"" : repair_leading_dot_number(num))
     end
 
     # Repair a number missing its digit before the decimal point, like ".5"

data/sig/json/repairer.rbs

@@ -37,7 +37,9 @@ module JSON
 
     def parse_whitespace: (?skip_newline: bool) -> bool
 
-    def parse_comment: () -> bool
+    def parse_comment: (?value_expected: bool) -> bool
+
+    def hash_comment?: (bool value_expected) -> bool
 
     # Find and skip over a Markdown fenced code block
     def parse_markdown_code_block: (::Array[::String] blocks) -> bool
@@ -87,7 +89,7 @@ module JSON
 
     def parse_character: (::String char) -> bool
 
-    def parse_whitespace_and_skip_comments: (?skip_newline: bool) -> bool
+    def parse_whitespace_and_skip_comments: (?skip_newline: bool, ?value_expected: bool) -> bool
 
     # Parse a number like 2.4 or 2.4e6
     def parse_number: () -> bool

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: json-repair
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 0.13.0
 platform: ruby
 authors:
 - Aleksandr Zykov