oj 3.13.11 → 3.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +74 -0
- data/README.md +4 -2
- data/ext/oj/buf.h +11 -6
- data/ext/oj/cache.c +25 -24
- data/ext/oj/cache8.c +10 -9
- data/ext/oj/circarray.c +8 -6
- data/ext/oj/circarray.h +2 -2
- data/ext/oj/code.c +17 -24
- data/ext/oj/code.h +2 -2
- data/ext/oj/compat.c +17 -44
- data/ext/oj/custom.c +70 -141
- data/ext/oj/debug.c +3 -9
- data/ext/oj/dump.c +128 -118
- data/ext/oj/dump.h +12 -8
- data/ext/oj/dump_compat.c +564 -641
- data/ext/oj/dump_leaf.c +17 -63
- data/ext/oj/dump_object.c +70 -199
- data/ext/oj/dump_strict.c +22 -46
- data/ext/oj/encoder.c +1 -1
- data/ext/oj/err.c +2 -13
- data/ext/oj/err.h +9 -12
- data/ext/oj/extconf.rb +14 -5
- data/ext/oj/fast.c +75 -103
- data/ext/oj/intern.c +52 -50
- data/ext/oj/intern.h +4 -8
- data/ext/oj/mem.c +318 -0
- data/ext/oj/mem.h +53 -0
- data/ext/oj/mimic_json.c +75 -47
- data/ext/oj/object.c +49 -66
- data/ext/oj/odd.c +89 -67
- data/ext/oj/odd.h +15 -15
- data/ext/oj/oj.c +140 -99
- data/ext/oj/oj.h +80 -51
- data/ext/oj/parse.c +162 -184
- data/ext/oj/parse.h +7 -10
- data/ext/oj/parser.c +89 -34
- data/ext/oj/parser.h +18 -7
- data/ext/oj/rails.c +82 -146
- data/ext/oj/rails.h +1 -1
- data/ext/oj/reader.c +11 -12
- data/ext/oj/reader.h +4 -2
- data/ext/oj/resolve.c +3 -4
- data/ext/oj/rxclass.c +6 -5
- data/ext/oj/rxclass.h +1 -1
- data/ext/oj/saj.c +20 -31
- data/ext/oj/saj2.c +329 -93
- data/ext/oj/saj2.h +23 -0
- data/ext/oj/scp.c +3 -14
- data/ext/oj/sparse.c +26 -70
- data/ext/oj/stream_writer.c +12 -22
- data/ext/oj/strict.c +20 -52
- data/ext/oj/string_writer.c +21 -21
- data/ext/oj/trace.h +31 -4
- data/ext/oj/usual.c +105 -150
- data/ext/oj/usual.h +68 -0
- data/ext/oj/util.h +1 -1
- data/ext/oj/val_stack.c +1 -1
- data/ext/oj/val_stack.h +8 -7
- data/ext/oj/validate.c +21 -26
- data/ext/oj/wab.c +31 -68
- data/lib/oj/active_support_helper.rb +0 -1
- data/lib/oj/bag.rb +7 -1
- data/lib/oj/easy_hash.rb +4 -5
- data/lib/oj/error.rb +0 -1
- data/lib/oj/json.rb +4 -2
- data/lib/oj/mimic.rb +4 -2
- data/lib/oj/saj.rb +20 -6
- data/lib/oj/state.rb +9 -6
- data/lib/oj/version.rb +1 -2
- data/lib/oj.rb +2 -0
- data/pages/Compatibility.md +1 -1
- data/pages/InstallOptions.md +20 -0
- data/pages/Options.md +10 -0
- data/test/_test_active.rb +8 -9
- data/test/_test_active_mimic.rb +7 -8
- data/test/_test_mimic_rails.rb +17 -20
- data/test/activerecord/result_test.rb +5 -6
- data/test/{activesupport5 → activesupport7}/abstract_unit.rb +16 -12
- data/test/{activesupport5 → activesupport7}/decoding_test.rb +2 -10
- data/test/{activesupport5 → activesupport7}/encoding_test.rb +20 -34
- data/test/{activesupport5 → activesupport7}/encoding_test_cases.rb +6 -0
- data/test/{activesupport5 → activesupport7}/time_zone_test_helpers.rb +8 -0
- data/test/files.rb +15 -15
- data/test/foo.rb +9 -71
- data/test/helper.rb +11 -8
- data/test/isolated/shared.rb +3 -2
- data/test/json_gem/json_addition_test.rb +2 -2
- data/test/json_gem/json_common_interface_test.rb +4 -4
- data/test/json_gem/json_encoding_test.rb +0 -0
- data/test/json_gem/json_ext_parser_test.rb +1 -0
- data/test/json_gem/json_fixtures_test.rb +3 -2
- data/test/json_gem/json_generator_test.rb +48 -36
- data/test/json_gem/json_generic_object_test.rb +11 -11
- data/test/json_gem/json_parser_test.rb +54 -47
- data/test/json_gem/json_string_matching_test.rb +9 -9
- data/test/json_gem/test_helper.rb +7 -3
- data/test/mem.rb +13 -12
- data/test/perf.rb +21 -26
- data/test/perf_compat.rb +31 -33
- data/test/perf_dump.rb +50 -0
- data/test/perf_fast.rb +80 -82
- data/test/perf_file.rb +27 -29
- data/test/perf_object.rb +65 -69
- data/test/perf_once.rb +12 -11
- data/test/perf_parser.rb +42 -48
- data/test/perf_saj.rb +46 -54
- data/test/perf_scp.rb +57 -69
- data/test/perf_simple.rb +41 -39
- data/test/perf_strict.rb +68 -70
- data/test/perf_wab.rb +67 -69
- data/test/prec.rb +3 -3
- data/test/sample/change.rb +0 -1
- data/test/sample/dir.rb +0 -1
- data/test/sample/doc.rb +0 -1
- data/test/sample/file.rb +0 -1
- data/test/sample/group.rb +0 -1
- data/test/sample/hasprops.rb +0 -1
- data/test/sample/layer.rb +0 -1
- data/test/sample/rect.rb +0 -1
- data/test/sample/shape.rb +0 -1
- data/test/sample/text.rb +0 -1
- data/test/sample.rb +16 -16
- data/test/sample_json.rb +8 -8
- data/test/test_compat.rb +76 -42
- data/test/test_custom.rb +72 -51
- data/test/test_debian.rb +7 -10
- data/test/test_fast.rb +86 -90
- data/test/test_file.rb +41 -30
- data/test/test_gc.rb +16 -5
- data/test/test_generate.rb +5 -5
- data/test/test_hash.rb +4 -4
- data/test/test_integer_range.rb +9 -9
- data/test/test_null.rb +20 -20
- data/test/test_object.rb +85 -96
- data/test/test_parser.rb +6 -22
- data/test/test_parser_debug.rb +27 -0
- data/test/test_parser_saj.rb +115 -23
- data/test/test_parser_usual.rb +6 -6
- data/test/test_rails.rb +2 -2
- data/test/test_saj.rb +10 -8
- data/test/test_scp.rb +37 -39
- data/test/test_strict.rb +30 -32
- data/test/test_various.rb +147 -99
- data/test/test_wab.rb +48 -44
- data/test/test_writer.rb +47 -47
- data/test/tests.rb +13 -4
- data/test/tests_mimic.rb +12 -3
- data/test/tests_mimic_addition.rb +12 -3
- metadata +33 -144
- data/test/activesupport4/decoding_test.rb +0 -108
- data/test/activesupport4/encoding_test.rb +0 -531
- data/test/activesupport4/test_helper.rb +0 -41
- data/test/activesupport5/test_helper.rb +0 -72
- data/test/bar.rb +0 -16
- data/test/baz.rb +0 -16
- data/test/bug.rb +0 -16
- data/test/zoo.rb +0 -13
data/ext/oj/parse.c
CHANGED
@@ -12,14 +12,19 @@
|
|
12
12
|
|
13
13
|
#include "buf.h"
|
14
14
|
#include "encode.h"
|
15
|
+
#include "mem.h"
|
15
16
|
#include "oj.h"
|
16
17
|
#include "rxclass.h"
|
17
18
|
#include "val_stack.h"
|
18
19
|
|
20
|
+
#ifdef OJ_USE_SSE4_2
|
21
|
+
#include <nmmintrin.h>
|
22
|
+
#endif
|
23
|
+
|
19
24
|
// Workaround in case INFINITY is not defined in math.h or if the OS is CentOS
|
20
25
|
#define OJ_INFINITY (1.0 / 0.0)
|
21
26
|
|
22
|
-
|
27
|
+
// #define EXP_MAX 1023
|
23
28
|
#define EXP_MAX 100000
|
24
29
|
#define DEC_MAX 15
|
25
30
|
|
@@ -44,11 +49,7 @@ static void skip_comment(ParseInfo pi) {
|
|
44
49
|
pi->cur += 2;
|
45
50
|
return;
|
46
51
|
} else if (pi->end <= pi->cur) {
|
47
|
-
oj_set_error_at(pi,
|
48
|
-
oj_parse_error_class,
|
49
|
-
__FILE__,
|
50
|
-
__LINE__,
|
51
|
-
"comment not terminated");
|
52
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "comment not terminated");
|
52
53
|
return;
|
53
54
|
}
|
54
55
|
}
|
@@ -81,9 +82,8 @@ static void add_value(ParseInfo pi, VALUE rval) {
|
|
81
82
|
break;
|
82
83
|
case NEXT_HASH_VALUE:
|
83
84
|
pi->hash_set_value(pi, parent, rval);
|
84
|
-
if (0 != parent->key && 0 < parent->klen &&
|
85
|
-
(
|
86
|
-
xfree((char *)parent->key);
|
85
|
+
if (0 != parent->key && 0 < parent->klen && (parent->key < pi->json || pi->cur < parent->key)) {
|
86
|
+
OJ_R_FREE((char *)parent->key);
|
87
87
|
parent->key = 0;
|
88
88
|
}
|
89
89
|
parent->next = NEXT_HASH_COMMA;
|
@@ -183,6 +183,46 @@ static void unicode_to_chars(ParseInfo pi, Buf buf, uint32_t code) {
|
|
183
183
|
}
|
184
184
|
}
|
185
185
|
|
186
|
+
static inline const char *scan_string_noSIMD(const char *str, const char *end) {
|
187
|
+
for (; '"' != *str; str++) {
|
188
|
+
if (end <= str || '\0' == *str || '\\' == *str) {
|
189
|
+
break;
|
190
|
+
}
|
191
|
+
}
|
192
|
+
return str;
|
193
|
+
}
|
194
|
+
|
195
|
+
#ifdef OJ_USE_SSE4_2
|
196
|
+
static inline const char *scan_string_SIMD(const char *str, const char *end) {
|
197
|
+
static const char chars[16] = "\x00\\\"";
|
198
|
+
const __m128i terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
|
199
|
+
const char *_end = (const char *)(end - 16);
|
200
|
+
|
201
|
+
for (; str <= _end; str += 16) {
|
202
|
+
const __m128i string = _mm_loadu_si128((const __m128i *)str);
|
203
|
+
const int r = _mm_cmpestri(terminate,
|
204
|
+
3,
|
205
|
+
string,
|
206
|
+
16,
|
207
|
+
_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
|
208
|
+
if (r != 16) {
|
209
|
+
str = (char *)(str + r);
|
210
|
+
return str;
|
211
|
+
}
|
212
|
+
}
|
213
|
+
|
214
|
+
return scan_string_noSIMD(str, end);
|
215
|
+
}
|
216
|
+
#endif
|
217
|
+
|
218
|
+
static const char *(*scan_func)(const char *str, const char *end) = scan_string_noSIMD;
|
219
|
+
|
220
|
+
void oj_scanner_init(void) {
|
221
|
+
#ifdef OJ_USE_SSE4_2
|
222
|
+
scan_func = scan_string_SIMD;
|
223
|
+
#endif
|
224
|
+
}
|
225
|
+
|
186
226
|
// entered at /
|
187
227
|
static void read_escaped_str(ParseInfo pi, const char *start) {
|
188
228
|
struct _buf buf;
|
@@ -192,19 +232,20 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
|
|
192
232
|
Val parent = stack_peek(&pi->stack);
|
193
233
|
|
194
234
|
buf_init(&buf);
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
if (
|
200
|
-
|
201
|
-
|
202
|
-
__FILE__,
|
203
|
-
__LINE__,
|
204
|
-
"quoted string not terminated");
|
235
|
+
buf_append_string(&buf, start, cnt);
|
236
|
+
|
237
|
+
for (s = pi->cur; '"' != *s;) {
|
238
|
+
const char *scanned = scan_func(s, pi->end);
|
239
|
+
if (scanned >= pi->end || '\0' == *scanned) {
|
240
|
+
// if (scanned >= pi->end) {
|
241
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
|
205
242
|
buf_cleanup(&buf);
|
206
243
|
return;
|
207
|
-
}
|
244
|
+
}
|
245
|
+
buf_append_string(&buf, s, (size_t)(scanned - s));
|
246
|
+
s = scanned;
|
247
|
+
|
248
|
+
if ('\\' == *s) {
|
208
249
|
s++;
|
209
250
|
switch (*s) {
|
210
251
|
case 'n': buf_append(&buf, '\n'); break;
|
@@ -234,11 +275,7 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
|
|
234
275
|
break;
|
235
276
|
}
|
236
277
|
pi->cur = s;
|
237
|
-
oj_set_error_at(pi,
|
238
|
-
oj_parse_error_class,
|
239
|
-
__FILE__,
|
240
|
-
__LINE__,
|
241
|
-
"invalid escaped character");
|
278
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid escaped character");
|
242
279
|
buf_cleanup(&buf);
|
243
280
|
return;
|
244
281
|
}
|
@@ -265,16 +302,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
|
|
265
302
|
break;
|
266
303
|
}
|
267
304
|
pi->cur = s;
|
268
|
-
oj_set_error_at(pi,
|
269
|
-
oj_parse_error_class,
|
270
|
-
__FILE__,
|
271
|
-
__LINE__,
|
272
|
-
"invalid escaped character");
|
305
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid escaped character");
|
273
306
|
buf_cleanup(&buf);
|
274
307
|
return;
|
275
308
|
}
|
276
|
-
|
277
|
-
buf_append(&buf, *s);
|
309
|
+
s++;
|
278
310
|
}
|
279
311
|
}
|
280
312
|
if (0 == parent) {
|
@@ -290,7 +322,7 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
|
|
290
322
|
case NEXT_HASH_KEY:
|
291
323
|
if (Qundef == (parent->key_val = pi->hash_key(pi, buf.head, buf_len(&buf)))) {
|
292
324
|
parent->klen = buf_len(&buf);
|
293
|
-
parent->key =
|
325
|
+
parent->key = OJ_MALLOC(parent->klen + 1);
|
294
326
|
memcpy((char *)parent->key, buf.head, parent->klen);
|
295
327
|
*(char *)(parent->key + parent->klen) = '\0';
|
296
328
|
} else {
|
@@ -302,9 +334,8 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
|
|
302
334
|
break;
|
303
335
|
case NEXT_HASH_VALUE:
|
304
336
|
pi->hash_set_cstr(pi, parent, buf.head, buf_len(&buf), start);
|
305
|
-
if (0 != parent->key && 0 < parent->klen &&
|
306
|
-
(
|
307
|
-
xfree((char *)parent->key);
|
337
|
+
if (0 != parent->key && 0 < parent->klen && (parent->key < pi->json || pi->cur < parent->key)) {
|
338
|
+
OJ_R_FREE((char *)parent->key);
|
308
339
|
parent->key = 0;
|
309
340
|
}
|
310
341
|
parent->next = NEXT_HASH_COMMA;
|
@@ -331,22 +362,20 @@ static void read_str(ParseInfo pi) {
|
|
331
362
|
const char *str = pi->cur;
|
332
363
|
Val parent = stack_peek(&pi->stack);
|
333
364
|
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
__FILE__,
|
339
|
-
__LINE__,
|
340
|
-
"quoted string not terminated");
|
341
|
-
return;
|
342
|
-
} else if ('\0' == *pi->cur) {
|
343
|
-
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "NULL byte in string");
|
344
|
-
return;
|
345
|
-
} else if ('\\' == *pi->cur) {
|
346
|
-
read_escaped_str(pi, str);
|
347
|
-
return;
|
348
|
-
}
|
365
|
+
pi->cur = scan_func(pi->cur, pi->end);
|
366
|
+
if (RB_UNLIKELY(pi->end <= pi->cur)) {
|
367
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
|
368
|
+
return;
|
349
369
|
}
|
370
|
+
if (RB_UNLIKELY('\0' == *pi->cur)) {
|
371
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "NULL byte in string");
|
372
|
+
return;
|
373
|
+
}
|
374
|
+
if ('\\' == *pi->cur) {
|
375
|
+
read_escaped_str(pi, str);
|
376
|
+
return;
|
377
|
+
}
|
378
|
+
|
350
379
|
if (0 == parent) { // simple add
|
351
380
|
pi->add_cstr(pi, str, pi->cur - str, str);
|
352
381
|
} else {
|
@@ -370,9 +399,8 @@ static void read_str(ParseInfo pi) {
|
|
370
399
|
break;
|
371
400
|
case NEXT_HASH_VALUE:
|
372
401
|
pi->hash_set_cstr(pi, parent, str, pi->cur - str, str);
|
373
|
-
if (0 != parent->key && 0 < parent->klen &&
|
374
|
-
(
|
375
|
-
xfree((char *)parent->key);
|
402
|
+
if (0 != parent->key && 0 < parent->klen && (parent->key < pi->json || pi->cur < parent->key)) {
|
403
|
+
OJ_R_FREE((char *)parent->key);
|
376
404
|
parent->key = 0;
|
377
405
|
}
|
378
406
|
parent->next = NEXT_HASH_COMMA;
|
@@ -414,7 +442,7 @@ static void read_num(ParseInfo pi) {
|
|
414
442
|
ni.no_big = !pi->options.compat_bigdec;
|
415
443
|
ni.bigdec_load = pi->options.compat_bigdec;
|
416
444
|
} else {
|
417
|
-
ni.no_big
|
445
|
+
ni.no_big = (FloatDec == pi->options.bigdec_load || FastDec == pi->options.bigdec_load ||
|
418
446
|
RubyDec == pi->options.bigdec_load);
|
419
447
|
ni.bigdec_load = pi->options.bigdec_load;
|
420
448
|
}
|
@@ -424,33 +452,21 @@ static void read_num(ParseInfo pi) {
|
|
424
452
|
ni.neg = 1;
|
425
453
|
} else if ('+' == *pi->cur) {
|
426
454
|
if (StrictMode == pi->options.mode) {
|
427
|
-
oj_set_error_at(pi,
|
428
|
-
oj_parse_error_class,
|
429
|
-
__FILE__,
|
430
|
-
__LINE__,
|
431
|
-
"not a number or other value");
|
455
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
|
432
456
|
return;
|
433
457
|
}
|
434
458
|
pi->cur++;
|
435
459
|
}
|
436
460
|
if ('I' == *pi->cur) {
|
437
461
|
if (No == pi->options.allow_nan || 0 != strncmp("Infinity", pi->cur, 8)) {
|
438
|
-
oj_set_error_at(pi,
|
439
|
-
oj_parse_error_class,
|
440
|
-
__FILE__,
|
441
|
-
__LINE__,
|
442
|
-
"not a number or other value");
|
462
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
|
443
463
|
return;
|
444
464
|
}
|
445
465
|
pi->cur += 8;
|
446
466
|
ni.infinity = 1;
|
447
467
|
} else if ('N' == *pi->cur || 'n' == *pi->cur) {
|
448
468
|
if ('a' != pi->cur[1] || ('N' != pi->cur[2] && 'n' != pi->cur[2])) {
|
449
|
-
oj_set_error_at(pi,
|
450
|
-
oj_parse_error_class,
|
451
|
-
__FILE__,
|
452
|
-
__LINE__,
|
453
|
-
"not a number or other value");
|
469
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
|
454
470
|
return;
|
455
471
|
}
|
456
472
|
pi->cur += 3;
|
@@ -459,33 +475,27 @@ static void read_num(ParseInfo pi) {
|
|
459
475
|
int dec_cnt = 0;
|
460
476
|
bool zero1 = false;
|
461
477
|
|
478
|
+
// Skip leading zeros.
|
479
|
+
for (; '0' == *pi->cur; pi->cur++) {
|
480
|
+
zero1 = true;
|
481
|
+
}
|
482
|
+
|
462
483
|
for (; '0' <= *pi->cur && *pi->cur <= '9'; pi->cur++) {
|
463
|
-
|
464
|
-
zero1 = true;
|
465
|
-
}
|
466
|
-
if (0 < ni.i) {
|
467
|
-
dec_cnt++;
|
468
|
-
}
|
469
|
-
if (!ni.big) {
|
470
|
-
int d = (*pi->cur - '0');
|
484
|
+
int d = (*pi->cur - '0');
|
471
485
|
|
472
|
-
|
473
|
-
|
474
|
-
oj_set_error_at(pi,
|
475
|
-
oj_parse_error_class,
|
476
|
-
__FILE__,
|
477
|
-
__LINE__,
|
478
|
-
"not a number");
|
479
|
-
return;
|
480
|
-
}
|
481
|
-
zero1 = false;
|
482
|
-
}
|
483
|
-
ni.i = ni.i * 10 + d;
|
484
|
-
if (INT64_MAX <= ni.i || DEC_MAX < dec_cnt) {
|
485
|
-
ni.big = 1;
|
486
|
-
}
|
486
|
+
if (RB_LIKELY(0 != ni.i)) {
|
487
|
+
dec_cnt++;
|
487
488
|
}
|
489
|
+
ni.i = ni.i * 10 + d;
|
490
|
+
}
|
491
|
+
if (RB_UNLIKELY(0 != ni.i && zero1 && CompatMode == pi->options.mode)) {
|
492
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number");
|
493
|
+
return;
|
488
494
|
}
|
495
|
+
if (INT64_MAX <= ni.i || DEC_MAX < dec_cnt) {
|
496
|
+
ni.big = true;
|
497
|
+
}
|
498
|
+
|
489
499
|
if ('.' == *pi->cur) {
|
490
500
|
pi->cur++;
|
491
501
|
// A trailing . is not a valid decimal but if encountered allow it
|
@@ -505,25 +515,20 @@ static void read_num(ParseInfo pi) {
|
|
505
515
|
for (; '0' <= *pi->cur && *pi->cur <= '9'; pi->cur++) {
|
506
516
|
int d = (*pi->cur - '0');
|
507
517
|
|
508
|
-
if (0
|
518
|
+
if (RB_LIKELY(0 != ni.num || 0 != ni.i)) {
|
509
519
|
dec_cnt++;
|
510
520
|
}
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
}
|
515
|
-
} else {
|
516
|
-
ni.num = ni.num * 10 + d;
|
517
|
-
ni.div *= 10;
|
518
|
-
ni.di++;
|
519
|
-
if (INT64_MAX <= ni.div || DEC_MAX < dec_cnt) {
|
520
|
-
if (!ni.no_big) {
|
521
|
-
ni.big = true;
|
522
|
-
}
|
523
|
-
}
|
524
|
-
}
|
521
|
+
ni.num = ni.num * 10 + d;
|
522
|
+
ni.div *= 10;
|
523
|
+
ni.di++;
|
525
524
|
}
|
526
525
|
}
|
526
|
+
if (INT64_MAX <= ni.div || DEC_MAX < dec_cnt) {
|
527
|
+
if (!ni.no_big) {
|
528
|
+
ni.big = true;
|
529
|
+
}
|
530
|
+
}
|
531
|
+
|
527
532
|
if ('e' == *pi->cur || 'E' == *pi->cur) {
|
528
533
|
int eneg = 0;
|
529
534
|
|
@@ -576,9 +581,8 @@ static void read_num(ParseInfo pi) {
|
|
576
581
|
break;
|
577
582
|
case NEXT_HASH_VALUE:
|
578
583
|
pi->hash_set_num(pi, parent, &ni);
|
579
|
-
if (0 != parent->key && 0 < parent->klen &&
|
580
|
-
(
|
581
|
-
xfree((char *)parent->key);
|
584
|
+
if (0 != parent->key && 0 < parent->klen && (parent->key < pi->json || pi->cur < parent->key)) {
|
585
|
+
OJ_R_FREE((char *)parent->key);
|
582
586
|
parent->key = 0;
|
583
587
|
}
|
584
588
|
parent->next = NEXT_HASH_COMMA;
|
@@ -596,7 +600,7 @@ static void read_num(ParseInfo pi) {
|
|
596
600
|
}
|
597
601
|
|
598
602
|
static void array_start(ParseInfo pi) {
|
599
|
-
|
603
|
+
VALUE v = pi->start_array(pi);
|
600
604
|
|
601
605
|
stack_push(&pi->stack, v, NEXT_ARRAY_NEW);
|
602
606
|
}
|
@@ -620,13 +624,13 @@ static void array_end(ParseInfo pi) {
|
|
620
624
|
}
|
621
625
|
|
622
626
|
static void hash_start(ParseInfo pi) {
|
623
|
-
|
627
|
+
VALUE v = pi->start_hash(pi);
|
624
628
|
|
625
629
|
stack_push(&pi->stack, v, NEXT_HASH_NEW);
|
626
630
|
}
|
627
631
|
|
628
632
|
static void hash_end(ParseInfo pi) {
|
629
|
-
|
633
|
+
Val hash = stack_peek(&pi->stack);
|
630
634
|
|
631
635
|
// leave hash on stack until just before
|
632
636
|
if (0 == hash) {
|
@@ -711,11 +715,7 @@ void oj_parse2(ParseInfo pi) {
|
|
711
715
|
// case '+':
|
712
716
|
case '+':
|
713
717
|
if (CompatMode == pi->options.mode) {
|
714
|
-
oj_set_error_at(pi,
|
715
|
-
oj_parse_error_class,
|
716
|
-
__FILE__,
|
717
|
-
__LINE__,
|
718
|
-
"unexpected character");
|
718
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected character");
|
719
719
|
return;
|
720
720
|
}
|
721
721
|
pi->cur--;
|
@@ -741,11 +741,7 @@ void oj_parse2(ParseInfo pi) {
|
|
741
741
|
pi->cur--;
|
742
742
|
read_num(pi);
|
743
743
|
} else {
|
744
|
-
oj_set_error_at(pi,
|
745
|
-
oj_parse_error_class,
|
746
|
-
__FILE__,
|
747
|
-
__LINE__,
|
748
|
-
"unexpected character");
|
744
|
+
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected character");
|
749
745
|
}
|
750
746
|
break;
|
751
747
|
case 't': read_true(pi); break;
|
@@ -765,9 +761,7 @@ void oj_parse2(ParseInfo pi) {
|
|
765
761
|
}
|
766
762
|
break;
|
767
763
|
case '\0': pi->cur--; return;
|
768
|
-
default:
|
769
|
-
oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected character");
|
770
|
-
return;
|
764
|
+
default: oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected character"); return;
|
771
765
|
}
|
772
766
|
if (err_has(&pi->err)) {
|
773
767
|
return;
|
@@ -804,16 +798,15 @@ static VALUE parse_big_decimal(VALUE str) {
|
|
804
798
|
}
|
805
799
|
|
806
800
|
static long double exp_plus[] = {
|
807
|
-
1.0, 1.0e1, 1.0e2, 1.0e3, 1.0e4, 1.0e5, 1.0e6, 1.0e7, 1.0e8, 1.0e9,
|
808
|
-
1.
|
809
|
-
1.
|
810
|
-
1.
|
811
|
-
1.0e40, 1.0e41, 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, 1.0e49,
|
801
|
+
1.0, 1.0e1, 1.0e2, 1.0e3, 1.0e4, 1.0e5, 1.0e6, 1.0e7, 1.0e8, 1.0e9, 1.0e10, 1.0e11, 1.0e12,
|
802
|
+
1.0e13, 1.0e14, 1.0e15, 1.0e16, 1.0e17, 1.0e18, 1.0e19, 1.0e20, 1.0e21, 1.0e22, 1.0e23, 1.0e24, 1.0e25,
|
803
|
+
1.0e26, 1.0e27, 1.0e28, 1.0e29, 1.0e30, 1.0e31, 1.0e32, 1.0e33, 1.0e34, 1.0e35, 1.0e36, 1.0e37, 1.0e38,
|
804
|
+
1.0e39, 1.0e40, 1.0e41, 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, 1.0e49,
|
812
805
|
};
|
813
806
|
|
814
807
|
VALUE
|
815
808
|
oj_num_as_value(NumInfo ni) {
|
816
|
-
|
809
|
+
VALUE rnum = Qnil;
|
817
810
|
|
818
811
|
if (ni->infinity) {
|
819
812
|
if (ni->neg) {
|
@@ -832,12 +825,12 @@ oj_num_as_value(NumInfo ni) {
|
|
832
825
|
buf[ni->len] = '\0';
|
833
826
|
rnum = rb_cstr_to_inum(buf, 10, 0);
|
834
827
|
} else {
|
835
|
-
char *buf =
|
828
|
+
char *buf = OJ_R_ALLOC_N(char, ni->len + 1);
|
836
829
|
|
837
830
|
memcpy(buf, ni->str, ni->len);
|
838
831
|
buf[ni->len] = '\0';
|
839
832
|
rnum = rb_cstr_to_inum(buf, 10, 0);
|
840
|
-
|
833
|
+
OJ_R_FREE(buf);
|
841
834
|
}
|
842
835
|
} else {
|
843
836
|
if (ni->neg) {
|
@@ -848,7 +841,7 @@ oj_num_as_value(NumInfo ni) {
|
|
848
841
|
}
|
849
842
|
} else { // decimal
|
850
843
|
if (ni->big) {
|
851
|
-
|
844
|
+
VALUE bd = rb_str_new(ni->str, ni->len);
|
852
845
|
|
853
846
|
rnum = rb_rescue2(parse_big_decimal, bd, rescue_big_decimal, bd, rb_eException, 0);
|
854
847
|
if (ni->no_big) {
|
@@ -876,11 +869,11 @@ oj_num_as_value(NumInfo ni) {
|
|
876
869
|
}
|
877
870
|
rnum = rb_float_new((double)ld);
|
878
871
|
} else if (RubyDec == ni->bigdec_load) {
|
879
|
-
|
872
|
+
VALUE sv = rb_str_new(ni->str, ni->len);
|
880
873
|
|
881
874
|
rnum = rb_funcall(sv, rb_intern("to_f"), 0);
|
882
875
|
} else {
|
883
|
-
char
|
876
|
+
char *end;
|
884
877
|
double d = strtod(ni->str, &end);
|
885
878
|
|
886
879
|
if ((long)ni->len != (long)(end - ni->str)) {
|
@@ -892,28 +885,23 @@ oj_num_as_value(NumInfo ni) {
|
|
892
885
|
return rnum;
|
893
886
|
}
|
894
887
|
|
895
|
-
void oj_set_error_at(ParseInfo
|
896
|
-
VALUE err_clas,
|
897
|
-
const char *file,
|
898
|
-
int line,
|
899
|
-
const char *format,
|
900
|
-
...) {
|
888
|
+
void oj_set_error_at(ParseInfo pi, VALUE err_clas, const char *file, int line, const char *format, ...) {
|
901
889
|
va_list ap;
|
902
890
|
char msg[256];
|
903
|
-
char
|
904
|
-
char
|
905
|
-
char
|
891
|
+
char *p = msg;
|
892
|
+
char *end = p + sizeof(msg) - 2;
|
893
|
+
char *start;
|
906
894
|
Val vp;
|
907
|
-
int
|
895
|
+
int mlen;
|
908
896
|
|
909
897
|
va_start(ap, format);
|
910
898
|
mlen = vsnprintf(msg, sizeof(msg) - 1, format, ap);
|
911
899
|
if (0 < mlen) {
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
916
|
-
|
900
|
+
if (sizeof(msg) - 2 < (size_t)mlen) {
|
901
|
+
p = end - 2;
|
902
|
+
} else {
|
903
|
+
p += mlen;
|
904
|
+
}
|
917
905
|
}
|
918
906
|
va_end(ap);
|
919
907
|
pi->err.clas = err_clas;
|
@@ -950,14 +938,7 @@ void oj_set_error_at(ParseInfo pi,
|
|
950
938
|
}
|
951
939
|
*p = '\0';
|
952
940
|
if (0 == pi->json) {
|
953
|
-
oj_err_set(&pi->err,
|
954
|
-
err_clas,
|
955
|
-
"%s at line %d, column %d [%s:%d]",
|
956
|
-
msg,
|
957
|
-
pi->rd.line,
|
958
|
-
pi->rd.col,
|
959
|
-
file,
|
960
|
-
line);
|
941
|
+
oj_err_set(&pi->err, err_clas, "%s at line %d, column %d [%s:%d]", msg, pi->rd.line, pi->rd.col, file, line);
|
961
942
|
} else {
|
962
943
|
_oj_err_set_with_location(&pi->err, err_clas, msg, pi->json, pi->cur - 1, file, line);
|
963
944
|
}
|
@@ -971,11 +952,12 @@ static VALUE protect_parse(VALUE pip) {
|
|
971
952
|
|
972
953
|
extern int oj_utf8_index;
|
973
954
|
|
974
|
-
static void oj_pi_set_input_str(ParseInfo pi,
|
975
|
-
|
955
|
+
static void oj_pi_set_input_str(ParseInfo pi, VALUE *inputp) {
|
956
|
+
int idx = RB_ENCODING_GET(*inputp);
|
976
957
|
|
977
|
-
if (
|
978
|
-
*
|
958
|
+
if (oj_utf8_encoding_index != idx) {
|
959
|
+
rb_encoding *enc = rb_enc_from_index(idx);
|
960
|
+
*inputp = rb_str_conv_enc(*inputp, enc, oj_utf8_encoding);
|
979
961
|
}
|
980
962
|
pi->json = RSTRING_PTR(*inputp);
|
981
963
|
pi->end = pi->json + RSTRING_LEN(*inputp);
|
@@ -983,12 +965,12 @@ static void oj_pi_set_input_str(ParseInfo pi, volatile VALUE *inputp) {
|
|
983
965
|
|
984
966
|
VALUE
|
985
967
|
oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk) {
|
986
|
-
char *
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
int
|
991
|
-
int
|
968
|
+
char *buf = 0;
|
969
|
+
VALUE input;
|
970
|
+
VALUE wrapped_stack;
|
971
|
+
VALUE result = Qnil;
|
972
|
+
int line = 0;
|
973
|
+
int free_json = 0;
|
992
974
|
|
993
975
|
if (argc < 1) {
|
994
976
|
rb_raise(rb_eArgError, "Wrong number of arguments to parse.");
|
@@ -1024,8 +1006,8 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
|
|
1024
1006
|
rb_raise(rb_eTypeError, "Nil is not a valid JSON source.");
|
1025
1007
|
}
|
1026
1008
|
} else {
|
1027
|
-
VALUE
|
1028
|
-
|
1009
|
+
VALUE clas = rb_obj_class(input);
|
1010
|
+
VALUE s;
|
1029
1011
|
|
1030
1012
|
if (oj_stringio_class == clas) {
|
1031
1013
|
s = rb_funcall2(input, oj_string_id, 0, 0);
|
@@ -1037,19 +1019,18 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
|
|
1037
1019
|
size_t len = lseek(fd, 0, SEEK_END);
|
1038
1020
|
|
1039
1021
|
lseek(fd, 0, SEEK_SET);
|
1040
|
-
buf =
|
1022
|
+
buf = OJ_R_ALLOC_N(char, len + 1);
|
1041
1023
|
pi->json = buf;
|
1042
1024
|
pi->end = buf + len;
|
1043
1025
|
if (0 >= (cnt = read(fd, (char *)pi->json, len)) || cnt != (ssize_t)len) {
|
1044
1026
|
if (0 != buf) {
|
1045
|
-
|
1027
|
+
OJ_R_FREE(buf);
|
1046
1028
|
}
|
1047
1029
|
rb_raise(rb_eIOError, "failed to read from IO Object.");
|
1048
1030
|
}
|
1049
1031
|
((char *)pi->json)[len] = '\0';
|
1050
1032
|
/* skip UTF-8 BOM if present */
|
1051
|
-
if (0xEF == (uint8_t)*pi->json && 0xBB == (uint8_t)pi->json[1] &&
|
1052
|
-
0xBF == (uint8_t)pi->json[2]) {
|
1033
|
+
if (0xEF == (uint8_t)*pi->json && 0xBB == (uint8_t)pi->json[1] && 0xBF == (uint8_t)pi->json[2]) {
|
1053
1034
|
pi->cur += 3;
|
1054
1035
|
}
|
1055
1036
|
#endif
|
@@ -1075,8 +1056,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
|
|
1075
1056
|
wrapped_stack = oj_stack_init(&pi->stack);
|
1076
1057
|
rb_protect(protect_parse, (VALUE)pi, &line);
|
1077
1058
|
if (Qundef == pi->stack.head->val && !empty_ok(&pi->options)) {
|
1078
|
-
if (No == pi->options.nilnil ||
|
1079
|
-
(CompatMode == pi->options.mode && 0 < pi->cur - pi->json)) {
|
1059
|
+
if (No == pi->options.nilnil || (CompatMode == pi->options.mode && 0 < pi->cur - pi->json)) {
|
1080
1060
|
oj_set_error_at(pi, oj_json_parser_error_class, __FILE__, __LINE__, "Empty input");
|
1081
1061
|
}
|
1082
1062
|
}
|
@@ -1104,9 +1084,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
|
|
1104
1084
|
switch (v->next) {
|
1105
1085
|
case NEXT_ARRAY_NEW:
|
1106
1086
|
case NEXT_ARRAY_ELEMENT:
|
1107
|
-
case NEXT_ARRAY_COMMA:
|
1108
|
-
oj_set_error_at(pi, err_class, __FILE__, __LINE__, "Array not terminated");
|
1109
|
-
break;
|
1087
|
+
case NEXT_ARRAY_COMMA: oj_set_error_at(pi, err_class, __FILE__, __LINE__, "Array not terminated"); break;
|
1110
1088
|
case NEXT_HASH_NEW:
|
1111
1089
|
case NEXT_HASH_KEY:
|
1112
1090
|
case NEXT_HASH_COLON:
|
@@ -1124,9 +1102,9 @@ CLEANUP:
|
|
1124
1102
|
oj_circ_array_free(pi->circ_array);
|
1125
1103
|
}
|
1126
1104
|
if (0 != buf) {
|
1127
|
-
|
1105
|
+
OJ_R_FREE(buf);
|
1128
1106
|
} else if (free_json) {
|
1129
|
-
|
1107
|
+
OJ_R_FREE(json);
|
1130
1108
|
}
|
1131
1109
|
stack_cleanup(&pi->stack);
|
1132
1110
|
if (pi->str_rx.head != oj_default_options.str_rx.head) {
|
data/ext/oj/parse.h
CHANGED
@@ -32,7 +32,7 @@ typedef struct _numInfo {
|
|
32
32
|
int has_exp;
|
33
33
|
int no_big;
|
34
34
|
int bigdec_load;
|
35
|
-
} *
|
35
|
+
} *NumInfo;
|
36
36
|
|
37
37
|
typedef struct _parseInfo {
|
38
38
|
// used for the string parser
|
@@ -54,11 +54,7 @@ typedef struct _parseInfo {
|
|
54
54
|
VALUE (*start_hash)(struct _parseInfo *pi);
|
55
55
|
void (*end_hash)(struct _parseInfo *pi);
|
56
56
|
VALUE (*hash_key)(struct _parseInfo *pi, const char *key, size_t klen);
|
57
|
-
void (*hash_set_cstr)(struct _parseInfo *pi,
|
58
|
-
Val kval,
|
59
|
-
const char * str,
|
60
|
-
size_t len,
|
61
|
-
const char * orig);
|
57
|
+
void (*hash_set_cstr)(struct _parseInfo *pi, Val kval, const char *str, size_t len, const char *orig);
|
62
58
|
void (*hash_set_num)(struct _parseInfo *pi, Val kval, NumInfo ni);
|
63
59
|
void (*hash_set_value)(struct _parseInfo *pi, Val kval, VALUE value);
|
64
60
|
|
@@ -73,11 +69,10 @@ typedef struct _parseInfo {
|
|
73
69
|
void (*add_value)(struct _parseInfo *pi, VALUE val);
|
74
70
|
VALUE err_class;
|
75
71
|
bool has_callbacks;
|
76
|
-
} *
|
72
|
+
} *ParseInfo;
|
77
73
|
|
78
|
-
extern void
|
79
|
-
extern void
|
80
|
-
oj_set_error_at(ParseInfo pi, VALUE err_clas, const char *file, int line, const char *format, ...);
|
74
|
+
extern void oj_parse2(ParseInfo pi);
|
75
|
+
extern void oj_set_error_at(ParseInfo pi, VALUE err_clas, const char *file, int line, const char *format, ...);
|
81
76
|
extern VALUE oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk);
|
82
77
|
extern VALUE oj_num_as_value(NumInfo ni);
|
83
78
|
|
@@ -97,6 +92,8 @@ static inline void parse_info_init(ParseInfo pi) {
|
|
97
92
|
memset(pi, 0, sizeof(struct _parseInfo));
|
98
93
|
}
|
99
94
|
|
95
|
+
extern void oj_scanner_init(void);
|
96
|
+
|
100
97
|
static inline bool empty_ok(Options options) {
|
101
98
|
switch (options->mode) {
|
102
99
|
case ObjectMode:
|