json_mend 0.3.5 ā 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +9 -0
- data/.tool-versions +1 -1
- data/benchmark_comparison.rb +22 -4
- data/lib/json_mend/parser.rb +88 -99
- data/lib/json_mend/version.rb +1 -1
- data/lib/json_mend.rb +7 -6
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 908b32104f5db7f667ee3d7f1273b43b5ae8d7e79193d40953a5d70c0e2c4afd
|
|
4
|
+
data.tar.gz: 8180522b11afd6b43414f67118361b6fc11b83d7afa0b8d8ef189d003a3995c2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b70fe8447456214a9fe13d7f2129b3d667687b652c3a7513cfb2385c90b86cffd3e2082a0bb2355610721ec5e688a83ad2ed83a7dd281cd9a9ad034c5c5ca4b1
|
|
7
|
+
data.tar.gz: 88dc8bdc36dedd7869223e5270f3c9e6e7cdfb8359700711ae1e7329355326f220d16a39b919565a5c50f9ab885dc79684a9d0c8dfb110abb329a4a7375dfd4e
|
data/.rubocop.yml
CHANGED
|
@@ -1,9 +1,18 @@
|
|
|
1
|
+
inherit_mode:
|
|
2
|
+
merge:
|
|
3
|
+
- Exclude
|
|
4
|
+
|
|
1
5
|
plugins:
|
|
2
6
|
- rubocop-performance
|
|
3
7
|
- rubocop-rspec
|
|
8
|
+
- rubocop-rake
|
|
4
9
|
|
|
5
10
|
AllCops:
|
|
6
11
|
NewCops: enable
|
|
12
|
+
ParserEngine: parser_prism
|
|
13
|
+
CacheRootDirectory: .rubocop_cache
|
|
14
|
+
MaxFilesInCache: 8000
|
|
15
|
+
UseCache: false
|
|
7
16
|
SuggestExtensions: false
|
|
8
17
|
|
|
9
18
|
Metrics/AbcSize:
|
data/.tool-versions
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
ruby 4.0.
|
|
1
|
+
ruby 4.0.7
|
data/benchmark_comparison.rb
CHANGED
|
@@ -33,6 +33,16 @@ TEST_CASES = {
|
|
|
33
33
|
label: 'Valid Single JSON',
|
|
34
34
|
input: json_object
|
|
35
35
|
},
|
|
36
|
+
return_objects: {
|
|
37
|
+
label: 'Valid JSON (Return Ruby Objects)',
|
|
38
|
+
input: json_object,
|
|
39
|
+
return_objects: true
|
|
40
|
+
},
|
|
41
|
+
generator_error: {
|
|
42
|
+
label: 'Valid Syntax but Invalid UTF-8 (GeneratorError)',
|
|
43
|
+
input: "{\"status\": \"ok\", \"data\": \"bad\xFFbyte\"}",
|
|
44
|
+
return_objects: true
|
|
45
|
+
},
|
|
36
46
|
concatenated: {
|
|
37
47
|
label: 'Concatenated JSON (x10)',
|
|
38
48
|
input: json_object * 10
|
|
@@ -116,13 +126,18 @@ TEST_CASES.each_value do |data|
|
|
|
116
126
|
puts "\n\nšø Scenario: #{data[:label]}"
|
|
117
127
|
puts '-' * 40
|
|
118
128
|
|
|
129
|
+
# Check if this specific test requires returning objects instead of a string
|
|
130
|
+
return_objs = data.fetch(:return_objects, false)
|
|
131
|
+
|
|
119
132
|
Benchmark.ips do |x|
|
|
120
133
|
x.config(time: 2, warmup: 1) # Short duration for quick checks
|
|
121
134
|
|
|
122
135
|
# 1. JsonMend
|
|
123
|
-
|
|
136
|
+
mend_proc = ->(i) { JsonMend.repair(i, return_objects: return_objs) }
|
|
137
|
+
|
|
138
|
+
if supported?(mend_proc, data[:input])
|
|
124
139
|
x.report('JsonMend') do
|
|
125
|
-
JsonMend.repair(data[:input])
|
|
140
|
+
JsonMend.repair(data[:input], return_objects: return_objs)
|
|
126
141
|
end
|
|
127
142
|
else
|
|
128
143
|
puts ' JsonMend: ā Not Supported'
|
|
@@ -130,9 +145,12 @@ TEST_CASES.each_value do |data|
|
|
|
130
145
|
|
|
131
146
|
# 2. json-repair
|
|
132
147
|
if defined?(JSON::Repair)
|
|
133
|
-
if
|
|
148
|
+
# For a fair comparison, if return_objects is true, json-repair must also parse its own output
|
|
149
|
+
repair_proc = ->(i) { return_objs ? JSON.parse(JSON.repair(i)) : JSON.repair(i) }
|
|
150
|
+
|
|
151
|
+
if supported?(repair_proc, data[:input])
|
|
134
152
|
x.report('json-repair') do
|
|
135
|
-
JSON.repair(data[:input])
|
|
153
|
+
return_objs ? JSON.parse(JSON.repair(data[:input])) : JSON.repair(data[:input])
|
|
136
154
|
end
|
|
137
155
|
else
|
|
138
156
|
puts ' json-repair: ā Not Supported'
|
data/lib/json_mend/parser.rb
CHANGED
|
@@ -7,9 +7,9 @@ module JsonMend
|
|
|
7
7
|
# The core parser that does the heavy lifting of fixing the JSON
|
|
8
8
|
class Parser
|
|
9
9
|
MAX_ALLOWED_DEPTH = 100
|
|
10
|
-
COMMENT_DELIMETERS =
|
|
10
|
+
COMMENT_DELIMETERS = '#/'
|
|
11
11
|
NUMBER_CHARS = Set.new('0123456789-.eE/,_'.chars).freeze
|
|
12
|
-
STRING_DELIMITERS =
|
|
12
|
+
STRING_DELIMITERS = "\"'āā"
|
|
13
13
|
SKIP_CHARS_REGEX_CACHE = {
|
|
14
14
|
'"' => /"/,
|
|
15
15
|
"'" => /'/,
|
|
@@ -29,12 +29,12 @@ module JsonMend
|
|
|
29
29
|
|
|
30
30
|
# Optimized constants for performance (CollectionLiteralInLoop)
|
|
31
31
|
TERMINATORS_ARRAY = [']', '}'].freeze
|
|
32
|
-
TERMINATORS_OBJECT_KEY =
|
|
33
|
-
TERMINATORS_OBJECT_VALUE =
|
|
32
|
+
TERMINATORS_OBJECT_KEY = ':}'
|
|
33
|
+
TERMINATORS_OBJECT_VALUE = ',}'
|
|
34
34
|
TERMINATORS_ARRAY_ITEM = [',', ']'].freeze
|
|
35
|
-
TERMINATORS_STRING_GUESSED =
|
|
36
|
-
TERMINATORS_VALUE =
|
|
37
|
-
STRING_OR_OBJECT_START =
|
|
35
|
+
TERMINATORS_STRING_GUESSED = '{}[],:'
|
|
36
|
+
TERMINATORS_VALUE = ',]}'
|
|
37
|
+
STRING_OR_OBJECT_START = "#{STRING_DELIMITERS}{[".freeze
|
|
38
38
|
SKIPPED_KEYS = %i[merged_array stray_colon].freeze
|
|
39
39
|
BOOLEAN_OR_NULL_CHARS = %w[t f n].freeze
|
|
40
40
|
ESCAPE_START_CHARS = %w[t n r b \\].freeze
|
|
@@ -42,6 +42,7 @@ module JsonMend
|
|
|
42
42
|
INVALID_NUMBER_TRAILERS = ['-', 'e', 'E', ','].freeze
|
|
43
43
|
|
|
44
44
|
# Pre-compile regexes for performance
|
|
45
|
+
STRING_START_REGEX = /[\p{L}$_]/
|
|
45
46
|
NUMBER_REGEX = /[#{Regexp.escape(NUMBER_CHARS.to_a.join)}]+/
|
|
46
47
|
NUMBER_NO_COMMA_REGEX = /[#{Regexp.escape(NUMBER_CHARS.dup.tap { |s| s.delete(',') }.to_a.join)}]+/
|
|
47
48
|
INVALID_NUMBER_TRAILERS_REGEX = /[#{Regexp.union(*INVALID_NUMBER_TRAILERS)}]+\z/
|
|
@@ -53,6 +54,8 @@ module JsonMend
|
|
|
53
54
|
def initialize(json_string)
|
|
54
55
|
@scanner = StringScanner.new(json_string)
|
|
55
56
|
@context = []
|
|
57
|
+
@context_counts = Hash.new(0)
|
|
58
|
+
@current_context = nil
|
|
56
59
|
@depth = 0
|
|
57
60
|
end
|
|
58
61
|
|
|
@@ -138,12 +141,12 @@ module JsonMend
|
|
|
138
141
|
when '['
|
|
139
142
|
@scanner.getch # consume '['
|
|
140
143
|
return parse_array
|
|
141
|
-
when *COMMENT_DELIMETERS
|
|
144
|
+
when '#', '/' # sync with COMMENT_DELIMETERS, not used *COMMENT_DELIMETERS for branch speed optimization
|
|
142
145
|
# Avoid recursion: consume comment and continue loop
|
|
143
146
|
parse_comment
|
|
144
147
|
else
|
|
145
148
|
if string_start?(char)
|
|
146
|
-
if @context.empty? && !STRING_DELIMITERS.include?(char)
|
|
149
|
+
if @context.empty? && char && !STRING_DELIMITERS.include?(char)
|
|
147
150
|
# Top level unquoted string strictness:
|
|
148
151
|
# Only allow literals (true/false/null), ignore other text as garbage
|
|
149
152
|
val = parse_literal
|
|
@@ -161,7 +164,7 @@ module JsonMend
|
|
|
161
164
|
else
|
|
162
165
|
# Stop if we hit a terminator for the current context to avoid consuming it as garbage
|
|
163
166
|
if (current_context?(:array) && char == ']') ||
|
|
164
|
-
(current_context?(:object_value) && TERMINATORS_OBJECT_VALUE.include?(char)) ||
|
|
167
|
+
(current_context?(:object_value) && char && TERMINATORS_OBJECT_VALUE.include?(char)) ||
|
|
165
168
|
(current_context?(:object_key) && char == '}')
|
|
166
169
|
return JSON_STOP_TOKEN
|
|
167
170
|
end
|
|
@@ -178,14 +181,14 @@ module JsonMend
|
|
|
178
181
|
with_depth_check do
|
|
179
182
|
object = {}
|
|
180
183
|
|
|
181
|
-
|
|
184
|
+
push_context(:object)
|
|
182
185
|
|
|
183
186
|
loop do
|
|
184
187
|
skip_whitespaces
|
|
185
188
|
|
|
186
189
|
# Explicitly consume comments to ensure they don't hide separators (like commas)
|
|
187
190
|
# or get parsed as part of the next key.
|
|
188
|
-
if COMMENT_DELIMETERS.include?(peek_char)
|
|
191
|
+
if peek_char && COMMENT_DELIMETERS.include?(peek_char)
|
|
189
192
|
parse_comment
|
|
190
193
|
next
|
|
191
194
|
end
|
|
@@ -222,7 +225,7 @@ module JsonMend
|
|
|
222
225
|
object[key] = value
|
|
223
226
|
end
|
|
224
227
|
|
|
225
|
-
|
|
228
|
+
pop_context
|
|
226
229
|
|
|
227
230
|
object
|
|
228
231
|
end
|
|
@@ -250,17 +253,6 @@ module JsonMend
|
|
|
250
253
|
# If we get an empty key and the next character is a closing brace, we're done.
|
|
251
254
|
return [nil, nil, false] if key.empty? && (peek_char.nil? || peek_char == '}' || @scanner.pos == pos_before_key)
|
|
252
255
|
|
|
253
|
-
# Handle Duplicate Keys (Safer Method)
|
|
254
|
-
# This is a critical repair for lists of objects missing a comma separator.
|
|
255
|
-
if object.key?(key)
|
|
256
|
-
# Instead of rewriting the string, we safely rewind the scanner to the
|
|
257
|
-
# position before the duplicate key. This ends the parsing of the current
|
|
258
|
-
# object, allowing the top-level parser to see the duplicate key as the
|
|
259
|
-
# start of a new JSON object.
|
|
260
|
-
@scanner.pos = pos_before_key
|
|
261
|
-
return [nil, nil, false] # Signal to stop parsing this object.
|
|
262
|
-
end
|
|
263
|
-
|
|
264
256
|
# Parse the Separator (:)
|
|
265
257
|
skip_whitespaces
|
|
266
258
|
colon_found = @scanner.skip(/:/) # Leniently skip the colon if it exists.
|
|
@@ -299,7 +291,7 @@ module JsonMend
|
|
|
299
291
|
end
|
|
300
292
|
|
|
301
293
|
# If no merge happened, proceed with standard key parsing.
|
|
302
|
-
|
|
294
|
+
push_context(:object_key)
|
|
303
295
|
is_bracketed = false
|
|
304
296
|
|
|
305
297
|
if char == '['
|
|
@@ -310,7 +302,7 @@ module JsonMend
|
|
|
310
302
|
else
|
|
311
303
|
key = parse_string.to_s
|
|
312
304
|
end
|
|
313
|
-
|
|
305
|
+
pop_context
|
|
314
306
|
|
|
315
307
|
# If the key is empty, consume any stray characters to prevent infinite loops.
|
|
316
308
|
@scanner.getch if key.empty? && !@scanner.check(/[:{\[}\]]/) && !@scanner.eos?
|
|
@@ -320,18 +312,18 @@ module JsonMend
|
|
|
320
312
|
|
|
321
313
|
# Parses the value part of a key-value pair.
|
|
322
314
|
def parse_object_value(colon_found: true)
|
|
323
|
-
|
|
315
|
+
push_context(:object_value)
|
|
324
316
|
skip_whitespaces
|
|
325
317
|
|
|
326
318
|
# Handle cases where the value is missing (e.g. "key": } or "key": ,)
|
|
327
319
|
if @scanner.eos? || @scanner.check(/[,}]/)
|
|
328
|
-
|
|
320
|
+
pop_context
|
|
329
321
|
return colon_found ? '' : :inferred_true
|
|
330
322
|
end
|
|
331
323
|
|
|
332
324
|
# Delegate to the main JSON value parser.
|
|
333
325
|
value = parse_json
|
|
334
|
-
|
|
326
|
+
pop_context
|
|
335
327
|
|
|
336
328
|
# If parse_json returned JSON_STOP_TOKEN (nothing found due to garbage->terminator),
|
|
337
329
|
# treat it as empty string for object values to be safe.
|
|
@@ -342,7 +334,7 @@ module JsonMend
|
|
|
342
334
|
def try_to_merge_dangling_array(object)
|
|
343
335
|
return false unless peek_char == '['
|
|
344
336
|
|
|
345
|
-
prev_key = object.keys
|
|
337
|
+
prev_key = object.keys[-1]
|
|
346
338
|
return false unless prev_key && object[prev_key].is_a?(Array)
|
|
347
339
|
|
|
348
340
|
@scanner.getch # Consume '['
|
|
@@ -365,7 +357,7 @@ module JsonMend
|
|
|
365
357
|
def parse_array
|
|
366
358
|
with_depth_check do
|
|
367
359
|
arr = []
|
|
368
|
-
|
|
360
|
+
push_context(:array)
|
|
369
361
|
char = peek_char
|
|
370
362
|
# Stop when you find the closing bracket or an invalid character like '}'
|
|
371
363
|
while !@scanner.eos? && !TERMINATORS_ARRAY.include?(char)
|
|
@@ -373,14 +365,14 @@ module JsonMend
|
|
|
373
365
|
char = peek_char
|
|
374
366
|
|
|
375
367
|
# Check for comments explicitly inside array to avoid recursion or garbage consumption issues
|
|
376
|
-
if COMMENT_DELIMETERS.include?(char)
|
|
368
|
+
if char && COMMENT_DELIMETERS.include?(char)
|
|
377
369
|
parse_comment
|
|
378
370
|
char = peek_char
|
|
379
371
|
next
|
|
380
372
|
end
|
|
381
373
|
|
|
382
374
|
value = ''
|
|
383
|
-
if STRING_DELIMITERS.include?(char)
|
|
375
|
+
if char && STRING_DELIMITERS.include?(char)
|
|
384
376
|
# Sometimes it can happen that LLMs forget to start an object and then you think it's a string in an array
|
|
385
377
|
# So we are going to check if this string is followed by a : or not
|
|
386
378
|
# And either parse the string or parse the object
|
|
@@ -409,7 +401,7 @@ module JsonMend
|
|
|
409
401
|
unless @scanner.scan(']')
|
|
410
402
|
@scanner.scan('}') # Consume } if it was the closer
|
|
411
403
|
end
|
|
412
|
-
|
|
404
|
+
pop_context
|
|
413
405
|
|
|
414
406
|
arr
|
|
415
407
|
end
|
|
@@ -422,7 +414,7 @@ module JsonMend
|
|
|
422
414
|
char = peek_char
|
|
423
415
|
|
|
424
416
|
# A valid string can only start with a valid quote or, in our case, with a literal
|
|
425
|
-
while !@scanner.eos? && !STRING_DELIMITERS.include?(char) && !char&.match?(/[\p{L}0-9$_-]/)
|
|
417
|
+
while !@scanner.eos? && char && !STRING_DELIMITERS.include?(char) && !char&.match?(/[\p{L}0-9$_-]/)
|
|
426
418
|
return '' if TERMINATORS_STRING_GUESSED.include?(char)
|
|
427
419
|
|
|
428
420
|
@scanner.getch
|
|
@@ -447,7 +439,7 @@ module JsonMend
|
|
|
447
439
|
|
|
448
440
|
doubled_quotes = rest.first
|
|
449
441
|
|
|
450
|
-
string_parts =
|
|
442
|
+
string_parts = +''
|
|
451
443
|
|
|
452
444
|
# Here things get a bit hairy because a string missing the final quote can also be a key or a value in an object
|
|
453
445
|
# In that case we need to use the ":|,|}" characters as terminators of the string
|
|
@@ -510,13 +502,13 @@ module JsonMend
|
|
|
510
502
|
doubled_quotes = false
|
|
511
503
|
|
|
512
504
|
# There is sometimes a weird case of doubled quotes, we manage this also later in the while loop
|
|
513
|
-
if STRING_DELIMITERS.include?(peek_char) && peek_char == lstring_delimiter
|
|
505
|
+
if peek_char && STRING_DELIMITERS.include?(peek_char) && peek_char == lstring_delimiter
|
|
514
506
|
next_value = peek_char(1)
|
|
515
507
|
|
|
516
508
|
if (
|
|
517
509
|
current_context?(:object_key) && next_value == ':'
|
|
518
510
|
) || (
|
|
519
|
-
current_context?(:object_value) && TERMINATORS_OBJECT_VALUE.include?(next_value)
|
|
511
|
+
current_context?(:object_value) && next_value && TERMINATORS_OBJECT_VALUE.include?(next_value)
|
|
520
512
|
)
|
|
521
513
|
@scanner.getch
|
|
522
514
|
return [true, '']
|
|
@@ -535,11 +527,13 @@ module JsonMend
|
|
|
535
527
|
# Ok this is not a doubled quote, check if this is an empty string or not
|
|
536
528
|
i = skip_whitespaces_at(start_idx: 1)
|
|
537
529
|
next_c = peek_char(i)
|
|
538
|
-
if
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
530
|
+
if next_c
|
|
531
|
+
if STRING_OR_OBJECT_START.include?(next_c)
|
|
532
|
+
@scanner.getch
|
|
533
|
+
return [true, '']
|
|
534
|
+
elsif !TERMINATORS_VALUE.include?(next_c)
|
|
535
|
+
@scanner.getch
|
|
536
|
+
end
|
|
543
537
|
end
|
|
544
538
|
end
|
|
545
539
|
end
|
|
@@ -576,12 +570,12 @@ module JsonMend
|
|
|
576
570
|
end
|
|
577
571
|
|
|
578
572
|
break if context_termination_reached?(
|
|
579
|
-
char
|
|
580
|
-
missing_quotes
|
|
573
|
+
char,
|
|
574
|
+
missing_quotes
|
|
581
575
|
)
|
|
582
576
|
|
|
583
|
-
if current_context?(:object_value) && TERMINATORS_OBJECT_VALUE.include?(char) &&
|
|
584
|
-
(string_parts.empty? || string_parts
|
|
577
|
+
if current_context?(:object_value) && char && TERMINATORS_OBJECT_VALUE.include?(char) &&
|
|
578
|
+
(string_parts.empty? || string_parts[-1] != rstring_delimiter)
|
|
585
579
|
|
|
586
580
|
is_break = check_rstring_delimiter_missing(
|
|
587
581
|
string_parts:,
|
|
@@ -592,7 +586,7 @@ module JsonMend
|
|
|
592
586
|
break if is_break
|
|
593
587
|
end
|
|
594
588
|
|
|
595
|
-
if char == ']' && context_contain?(:array) && string_parts
|
|
589
|
+
if char == ']' && context_contain?(:array) && string_parts[-1] != rstring_delimiter
|
|
596
590
|
i = skip_to_character(rstring_delimiter)
|
|
597
591
|
# No delimiter found
|
|
598
592
|
break unless peek_char(i)
|
|
@@ -610,7 +604,7 @@ module JsonMend
|
|
|
610
604
|
@scanner.getch # Consume the character
|
|
611
605
|
char = peek_char
|
|
612
606
|
|
|
613
|
-
if !@scanner.eos? && string_parts
|
|
607
|
+
if !@scanner.eos? && string_parts[-1] == '\\'
|
|
614
608
|
# This is a special case, if people use real strings this might happen
|
|
615
609
|
is_next, string_parts, char = parse_escape_sequence(
|
|
616
610
|
string_parts:,
|
|
@@ -629,7 +623,7 @@ module JsonMend
|
|
|
629
623
|
break if is_break
|
|
630
624
|
end
|
|
631
625
|
|
|
632
|
-
if char == rstring_delimiter && string_parts
|
|
626
|
+
if char == rstring_delimiter && string_parts[-1] != '\\'
|
|
633
627
|
if check_doubled_quotes(doubled_quotes, rstring_delimiter)
|
|
634
628
|
# Consumed in helper
|
|
635
629
|
elsif check_missing_quotes_in_object_value(missing_quotes, lstring_delimiter, rstring_delimiter)
|
|
@@ -710,7 +704,7 @@ module JsonMend
|
|
|
710
704
|
check_comma_in_object_value = false if check_comma_in_object_value && next_c.match?(/\p{L}/)
|
|
711
705
|
|
|
712
706
|
# If we are in an object context, let's check for the right delimiters
|
|
713
|
-
if (context_contain?(:object) && TERMINATORS_OBJECT_KEY.include?(next_c)) ||
|
|
707
|
+
if (context_contain?(:object) && next_c && TERMINATORS_OBJECT_KEY.include?(next_c)) ||
|
|
714
708
|
(context_contain?(:array) && TERMINATORS_ARRAY_ITEM.include?(next_c)) ||
|
|
715
709
|
(
|
|
716
710
|
check_comma_in_object_value &&
|
|
@@ -735,7 +729,7 @@ module JsonMend
|
|
|
735
729
|
i += 1
|
|
736
730
|
i = skip_whitespaces_at(start_idx: i)
|
|
737
731
|
next_c = peek_char(i)
|
|
738
|
-
return [true, false] if TERMINATORS_OBJECT_VALUE.include?(next_c)
|
|
732
|
+
return [true, false] if next_c && TERMINATORS_OBJECT_VALUE.include?(next_c)
|
|
739
733
|
elsif next_c == rstring_delimiter && peek_char(i - 1) != '\\'
|
|
740
734
|
# Check if self.index:self.index+i is only whitespaces
|
|
741
735
|
return [false, false] if skip_whitespaces_at(start_idx: 1) >= i
|
|
@@ -781,7 +775,7 @@ module JsonMend
|
|
|
781
775
|
prev_byte_idx = @scanner.pos - next_c.bytesize - 1
|
|
782
776
|
is_escaped = prev_byte_idx >= 0 && @scanner.string.getbyte(prev_byte_idx) == 92 # 92 is backslash
|
|
783
777
|
|
|
784
|
-
break if TERMINATORS_VALUE.include?(next_c) || (next_c == rstring_delimiter && !is_escaped)
|
|
778
|
+
break if (next_c && TERMINATORS_VALUE.include?(next_c)) || (next_c == rstring_delimiter && !is_escaped)
|
|
785
779
|
|
|
786
780
|
index += 1
|
|
787
781
|
end
|
|
@@ -805,19 +799,7 @@ module JsonMend
|
|
|
805
799
|
|
|
806
800
|
# Scan forward linearly
|
|
807
801
|
while (c = @scanner.getch)
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
# Check if escaped (count preceding backslashes)
|
|
811
|
-
bk = 1
|
|
812
|
-
slashes = 0
|
|
813
|
-
while (@scanner.pos - 1 - bk >= 0) &&
|
|
814
|
-
(char_code = @scanner.string.getbyte(@scanner.pos - 1 - bk)) &&
|
|
815
|
-
char_code == 92 # 92 is backslash
|
|
816
|
-
slashes += 1
|
|
817
|
-
bk += 1
|
|
818
|
-
end
|
|
819
|
-
|
|
820
|
-
if slashes.even?
|
|
802
|
+
if c == rstring_delimiter
|
|
821
803
|
found_next = true
|
|
822
804
|
break
|
|
823
805
|
end
|
|
@@ -844,7 +826,11 @@ module JsonMend
|
|
|
844
826
|
# Jump directly to the exact byte offset after the second quote!
|
|
845
827
|
@scanner.pos = pos_after_second_quote
|
|
846
828
|
@scanner.skip(/\s+/)
|
|
847
|
-
|
|
829
|
+
|
|
830
|
+
# Safely check the next character using the nil guard
|
|
831
|
+
next_char = @scanner.check(/./)
|
|
832
|
+
is_next_closer = next_char && TERMINATORS_VALUE.include?(next_char)
|
|
833
|
+
|
|
848
834
|
@scanner.pos = saved_pos
|
|
849
835
|
end
|
|
850
836
|
|
|
@@ -934,18 +920,18 @@ module JsonMend
|
|
|
934
920
|
char:,
|
|
935
921
|
rstring_delimiter:
|
|
936
922
|
)
|
|
937
|
-
if !@scanner.eos? && string_parts
|
|
923
|
+
if !@scanner.eos? && string_parts[-1] == '\\'
|
|
938
924
|
# This is a special case, if people use real strings this might happen
|
|
939
925
|
if char == rstring_delimiter || ESCAPE_START_CHARS.include?(char)
|
|
940
|
-
string_parts.
|
|
926
|
+
string_parts.chop!
|
|
941
927
|
string_parts << ESCAPE_MAPPING.fetch(char, char)
|
|
942
928
|
|
|
943
929
|
@scanner.getch # Consume the character
|
|
944
930
|
char = peek_char
|
|
945
|
-
while !@scanner.eos? && string_parts
|
|
931
|
+
while !@scanner.eos? && string_parts[-1] == '\\' && (char == rstring_delimiter || char == '\\')
|
|
946
932
|
# this is a bit of a special case, if I don't do this it will close the loop or create a train of \\
|
|
947
933
|
# I don't love it though
|
|
948
|
-
string_parts.
|
|
934
|
+
string_parts.chop!
|
|
949
935
|
string_parts << char
|
|
950
936
|
@scanner.getch # Consume the character
|
|
951
937
|
char = peek_char
|
|
@@ -959,7 +945,7 @@ module JsonMend
|
|
|
959
945
|
|
|
960
946
|
# Validate valid hex digits
|
|
961
947
|
if (hex_str = @scanner.scan(hex_regex))
|
|
962
|
-
string_parts.
|
|
948
|
+
string_parts.chop!
|
|
963
949
|
hex_val = hex_str.to_i(16)
|
|
964
950
|
|
|
965
951
|
if char == 'u' && hex_val.between?(0xD800, 0xDBFF)
|
|
@@ -985,11 +971,7 @@ module JsonMend
|
|
|
985
971
|
string_parts << "\uFFFD"
|
|
986
972
|
else
|
|
987
973
|
# Regular code point or hex escape
|
|
988
|
-
|
|
989
|
-
string_parts << hex_val.chr('UTF-8')
|
|
990
|
-
rescue RangeError
|
|
991
|
-
string_parts << "\uFFFD"
|
|
992
|
-
end
|
|
974
|
+
string_parts << hex_val.chr('UTF-8')
|
|
993
975
|
end
|
|
994
976
|
|
|
995
977
|
# Scanner is already advanced past digits
|
|
@@ -1000,7 +982,7 @@ module JsonMend
|
|
|
1000
982
|
@scanner.pos = entry_pos
|
|
1001
983
|
end
|
|
1002
984
|
elsif STRING_DELIMITERS.include?(char) && char != rstring_delimiter
|
|
1003
|
-
string_parts.
|
|
985
|
+
string_parts.chop!
|
|
1004
986
|
string_parts << char
|
|
1005
987
|
@scanner.getch # Consume the character
|
|
1006
988
|
char = peek_char
|
|
@@ -1045,19 +1027,19 @@ module JsonMend
|
|
|
1045
1027
|
# we need to update the index only if we had a closing quote
|
|
1046
1028
|
if char == rstring_delimiter
|
|
1047
1029
|
@scanner.getch
|
|
1048
|
-
elsif missing_quotes && current_context?(:object_key) && string_parts
|
|
1049
|
-
string_parts.
|
|
1030
|
+
elsif missing_quotes && current_context?(:object_key) && string_parts[-1] == ','
|
|
1031
|
+
string_parts.chop!
|
|
1050
1032
|
end
|
|
1051
1033
|
|
|
1052
|
-
final_str = string_parts
|
|
1034
|
+
final_str = string_parts
|
|
1053
1035
|
final_str = final_str.rstrip if missing_quotes || final_str.end_with?("\n")
|
|
1054
1036
|
|
|
1055
1037
|
final_str
|
|
1056
1038
|
end
|
|
1057
1039
|
|
|
1058
1040
|
def context_termination_reached?(
|
|
1059
|
-
char
|
|
1060
|
-
missing_quotes
|
|
1041
|
+
char,
|
|
1042
|
+
missing_quotes
|
|
1061
1043
|
)
|
|
1062
1044
|
return false unless missing_quotes
|
|
1063
1045
|
|
|
@@ -1105,7 +1087,7 @@ module JsonMend
|
|
|
1105
1087
|
end
|
|
1106
1088
|
|
|
1107
1089
|
# Handle cases where the number ends with one or more invalid characters.
|
|
1108
|
-
if !scanned_str.empty? && scanned_str.
|
|
1090
|
+
if !scanned_str.empty? && scanned_str.end_with?(*INVALID_NUMBER_TRAILERS)
|
|
1109
1091
|
# Do not rewind scanner, simply discard the invalid trailing chars (garbage)
|
|
1110
1092
|
scanned_str.sub!(INVALID_NUMBER_TRAILERS_REGEX, '')
|
|
1111
1093
|
end
|
|
@@ -1219,13 +1201,7 @@ module JsonMend
|
|
|
1219
1201
|
# It quickly iterates to find a character, handling escaped characters, and
|
|
1220
1202
|
# returns the index (offset) from the scanner
|
|
1221
1203
|
def skip_to_character(characters, start_idx: 0)
|
|
1222
|
-
pattern = SKIP_CHARS_REGEX_CACHE.fetch(characters
|
|
1223
|
-
# :nocov:
|
|
1224
|
-
if pattern.nil?
|
|
1225
|
-
chars = Array(characters).map { |c| Regexp.escape(c.to_s) }
|
|
1226
|
-
pattern = Regexp.new(chars.join('|'))
|
|
1227
|
-
end
|
|
1228
|
-
# :nocov:
|
|
1204
|
+
pattern = SKIP_CHARS_REGEX_CACHE.fetch(characters)
|
|
1229
1205
|
|
|
1230
1206
|
saved_pos = @scanner.pos
|
|
1231
1207
|
# Skip start_idx
|
|
@@ -1302,12 +1278,13 @@ module JsonMend
|
|
|
1302
1278
|
def peek_char(offset = 0)
|
|
1303
1279
|
# Handle the common 0-offset case
|
|
1304
1280
|
if offset.zero?
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
return nil if
|
|
1281
|
+
pos = @scanner.pos
|
|
1282
|
+
str = @scanner.string
|
|
1283
|
+
return nil if pos >= str.bytesize
|
|
1308
1284
|
|
|
1285
|
+
byte = str.getbyte(pos)
|
|
1309
1286
|
# Fast path: If it's a standard ASCII char (0-127), return it directly.
|
|
1310
|
-
return
|
|
1287
|
+
return byte.chr if byte < 128
|
|
1311
1288
|
|
|
1312
1289
|
# Slow path: If it's a multibyte char (e.g. ā), use regex to match the full character.
|
|
1313
1290
|
return @scanner.check(/./m)
|
|
@@ -1324,22 +1301,34 @@ module JsonMend
|
|
|
1324
1301
|
res
|
|
1325
1302
|
end
|
|
1326
1303
|
|
|
1304
|
+
def push_context(value)
|
|
1305
|
+
@context.push(value)
|
|
1306
|
+
@context_counts[value] += 1
|
|
1307
|
+
@current_context = value
|
|
1308
|
+
end
|
|
1309
|
+
|
|
1310
|
+
def pop_context
|
|
1311
|
+
popped_value = @context.pop
|
|
1312
|
+
@context_counts[popped_value] -= 1
|
|
1313
|
+
@current_context = @context.last
|
|
1314
|
+
end
|
|
1315
|
+
|
|
1327
1316
|
def current_context?(value)
|
|
1328
|
-
@
|
|
1317
|
+
@current_context == value
|
|
1329
1318
|
end
|
|
1330
1319
|
|
|
1331
1320
|
def context_contain?(value)
|
|
1332
|
-
@
|
|
1321
|
+
@context_counts[value].positive?
|
|
1333
1322
|
end
|
|
1334
1323
|
|
|
1335
1324
|
# Checks if the character signifies the start of a string or literal
|
|
1336
1325
|
def string_start?(char)
|
|
1337
|
-
STRING_DELIMITERS.include?(char) || char
|
|
1326
|
+
(char && STRING_DELIMITERS.include?(char)) || (char && STRING_START_REGEX.match?(char))
|
|
1338
1327
|
end
|
|
1339
1328
|
|
|
1340
1329
|
# Checks if the character signifies the start of a number
|
|
1341
1330
|
def number_start?(char)
|
|
1342
|
-
char
|
|
1331
|
+
(char && char >= '0' && char <= '9') || char == '-' || char == '.'
|
|
1343
1332
|
end
|
|
1344
1333
|
end
|
|
1345
1334
|
end
|
data/lib/json_mend/version.rb
CHANGED
data/lib/json_mend.rb
CHANGED
|
@@ -15,21 +15,22 @@ module JsonMend
|
|
|
15
15
|
# @return [Object, String] The repaired JSON object or string.
|
|
16
16
|
def repair(json_string, return_objects: false)
|
|
17
17
|
# First, attempt to parse the string with the standard library.
|
|
18
|
-
|
|
18
|
+
begin
|
|
19
19
|
parsed = JSON.parse(
|
|
20
20
|
json_string,
|
|
21
21
|
allow_trailing_comma: true,
|
|
22
22
|
allow_control_characters: true
|
|
23
23
|
)
|
|
24
24
|
|
|
25
|
-
#
|
|
26
|
-
|
|
27
|
-
JSON.generate(parsed)
|
|
25
|
+
# If the user wants objects, return them immediately
|
|
26
|
+
return parsed if return_objects
|
|
28
27
|
|
|
29
|
-
|
|
28
|
+
# Otherwise, generate the string once. This acts as both the
|
|
29
|
+
# UTF-8 verification step AND the final string output.
|
|
30
|
+
return JSON.generate(parsed)
|
|
30
31
|
rescue JSON::ParserError, JSON::GeneratorError
|
|
31
32
|
parser = Parser.new(json_string)
|
|
32
|
-
parser.parse
|
|
33
|
+
repaired_json = parser.parse
|
|
33
34
|
end
|
|
34
35
|
|
|
35
36
|
# Avoids returning `null` for empty results, returns the object directly
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: json_mend
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Oleksii Vasyliev
|
|
@@ -77,14 +77,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
77
77
|
requirements:
|
|
78
78
|
- - ">="
|
|
79
79
|
- !ruby/object:Gem::Version
|
|
80
|
-
version: 3.
|
|
80
|
+
version: 3.3.0
|
|
81
81
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
82
82
|
requirements:
|
|
83
83
|
- - ">="
|
|
84
84
|
- !ruby/object:Gem::Version
|
|
85
85
|
version: '0'
|
|
86
86
|
requirements: []
|
|
87
|
-
rubygems_version: 4.0.
|
|
87
|
+
rubygems_version: 4.0.20
|
|
88
88
|
specification_version: 4
|
|
89
89
|
summary: Repair broken JSON
|
|
90
90
|
test_files: []
|