json 2.19.5 → 2.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +48 -0
- data/LEGAL +3 -3
- data/README.md +16 -7
- data/ext/json/ext/fbuffer/fbuffer.h +32 -24
- data/ext/json/ext/generator/extconf.rb +1 -0
- data/ext/json/ext/generator/generator.c +92 -18
- data/ext/json/ext/json.h +67 -0
- data/ext/json/ext/parser/extconf.rb +23 -0
- data/ext/json/ext/parser/parser.c +1563 -354
- data/ext/json/ext/vendor/fast_float_parser.h +814 -0
- data/lib/json/common.rb +9 -0
- data/lib/json/ext/generator/state.rb +1 -0
- data/lib/json/ext.rb +26 -0
- data/lib/json/truffle_ruby/generator.rb +39 -1
- data/lib/json/version.rb +1 -1
- data/lib/json.rb +34 -5
- metadata +3 -3
- data/ext/json/ext/vendor/ryu.h +0 -819
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
#include "../json.h"
|
|
2
|
-
#include "../vendor/
|
|
2
|
+
#include "../vendor/fast_float_parser.h"
|
|
3
3
|
#include "../simd/simd.h"
|
|
4
4
|
|
|
5
|
-
static VALUE mJSON, eNestingError, Encoding_UTF_8;
|
|
6
|
-
static VALUE CNaN, CInfinity, CMinusInfinity;
|
|
5
|
+
static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
|
|
6
|
+
static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
|
|
7
7
|
|
|
8
|
-
static ID i_new, i_try_convert,
|
|
8
|
+
static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column;
|
|
9
|
+
#ifndef HAVE_RB_STR_TO_INTERNED_STR
|
|
10
|
+
static ID i_uminus;
|
|
11
|
+
#endif
|
|
9
12
|
|
|
10
|
-
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma,
|
|
11
|
-
sym_allow_invalid_escape, sym_symbolize_names,
|
|
12
|
-
sym_allow_duplicate_key;
|
|
13
|
+
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
|
|
14
|
+
sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
|
|
15
|
+
sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
|
|
13
16
|
|
|
14
17
|
static int binary_encindex;
|
|
15
18
|
static int utf8_encindex;
|
|
@@ -58,6 +61,20 @@ typedef struct rvalue_cache_struct {
|
|
|
58
61
|
VALUE entries[JSON_RVALUE_CACHE_CAPA];
|
|
59
62
|
} rvalue_cache;
|
|
60
63
|
|
|
64
|
+
static void rvalue_cache_mark(rvalue_cache *cache)
|
|
65
|
+
{
|
|
66
|
+
for (int index = 0; index < cache->length; index++) {
|
|
67
|
+
rb_gc_mark_movable(cache->entries[index]);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
static void rvalue_cache_compact(rvalue_cache *cache)
|
|
72
|
+
{
|
|
73
|
+
for (int index = 0; index < cache->length; index++) {
|
|
74
|
+
cache->entries[index] = rb_gc_location(cache->entries[index]);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
61
78
|
static rb_encoding *enc_utf8;
|
|
62
79
|
|
|
63
80
|
#define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55
|
|
@@ -206,12 +223,12 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle,
|
|
|
206
223
|
|
|
207
224
|
static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
208
225
|
{
|
|
209
|
-
long required = stack->capa * 2;
|
|
226
|
+
long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA;
|
|
210
227
|
|
|
211
228
|
if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
|
|
212
229
|
stack = rvalue_stack_spill(stack, handle, stack_ref);
|
|
213
230
|
} else {
|
|
214
|
-
|
|
231
|
+
JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa);
|
|
215
232
|
stack->capa = required;
|
|
216
233
|
}
|
|
217
234
|
return stack;
|
|
@@ -219,11 +236,15 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu
|
|
|
219
236
|
|
|
220
237
|
static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref)
|
|
221
238
|
{
|
|
239
|
+
JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle);
|
|
240
|
+
|
|
222
241
|
if (RB_UNLIKELY(stack->head >= stack->capa)) {
|
|
223
242
|
stack = rvalue_stack_grow(stack, handle, stack_ref);
|
|
224
243
|
}
|
|
244
|
+
|
|
225
245
|
stack->ptr[stack->head] = value;
|
|
226
246
|
stack->head++;
|
|
247
|
+
|
|
227
248
|
return value;
|
|
228
249
|
}
|
|
229
250
|
|
|
@@ -243,14 +264,14 @@ static void rvalue_stack_mark(void *ptr)
|
|
|
243
264
|
long index;
|
|
244
265
|
if (stack && stack->ptr) {
|
|
245
266
|
for (index = 0; index < stack->head; index++) {
|
|
246
|
-
|
|
267
|
+
rb_gc_mark_movable(stack->ptr[index]);
|
|
247
268
|
}
|
|
248
269
|
}
|
|
249
270
|
}
|
|
250
271
|
|
|
251
272
|
static void rvalue_stack_free_buffer(rvalue_stack *stack)
|
|
252
273
|
{
|
|
253
|
-
|
|
274
|
+
JSON_SIZED_FREE_N(stack->ptr, stack->capa);
|
|
254
275
|
stack->ptr = NULL;
|
|
255
276
|
}
|
|
256
277
|
|
|
@@ -260,7 +281,7 @@ static void rvalue_stack_free(void *ptr)
|
|
|
260
281
|
if (stack) {
|
|
261
282
|
rvalue_stack_free_buffer(stack);
|
|
262
283
|
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
263
|
-
|
|
284
|
+
JSON_SIZED_FREE(stack);
|
|
264
285
|
#endif
|
|
265
286
|
}
|
|
266
287
|
}
|
|
@@ -268,7 +289,22 @@ static void rvalue_stack_free(void *ptr)
|
|
|
268
289
|
static size_t rvalue_stack_memsize(const void *ptr)
|
|
269
290
|
{
|
|
270
291
|
const rvalue_stack *stack = (const rvalue_stack *)ptr;
|
|
271
|
-
|
|
292
|
+
size_t memsize = sizeof(VALUE) * stack->capa;
|
|
293
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
294
|
+
memsize += sizeof(rvalue_stack);
|
|
295
|
+
#endif
|
|
296
|
+
return memsize;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
static void rvalue_stack_compact(void *ptr)
|
|
300
|
+
{
|
|
301
|
+
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
302
|
+
long index;
|
|
303
|
+
if (stack && stack->ptr) {
|
|
304
|
+
for (index = 0; index < stack->head; index++) {
|
|
305
|
+
stack->ptr[index] = rb_gc_location(stack->ptr[index]);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
272
308
|
}
|
|
273
309
|
|
|
274
310
|
static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
|
|
@@ -277,8 +313,11 @@ static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
|
|
|
277
313
|
.dmark = rvalue_stack_mark,
|
|
278
314
|
.dfree = rvalue_stack_free,
|
|
279
315
|
.dsize = rvalue_stack_memsize,
|
|
316
|
+
.dcompact = rvalue_stack_compact,
|
|
280
317
|
},
|
|
281
|
-
|
|
318
|
+
// We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
|
|
319
|
+
// because it churns a lot of values so trigering write barriers every time is very costly.
|
|
320
|
+
.flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
|
|
282
321
|
};
|
|
283
322
|
|
|
284
323
|
static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
@@ -309,33 +348,62 @@ static void rvalue_stack_eagerly_release(VALUE handle)
|
|
|
309
348
|
}
|
|
310
349
|
}
|
|
311
350
|
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
351
|
+
/* frame stack */
|
|
352
|
+
|
|
353
|
+
// Iterative (non-recursive) parsing keeps an explicit stack of the containers
|
|
354
|
+
// currently being built, instead of relying on the C call stack. Each frame
|
|
355
|
+
// only needs enough bookkeeping to close its container: which kind it is, the
|
|
356
|
+
// rvalue_stack position where its children start (so we know how many to pop),
|
|
357
|
+
// and the cursor at its opening brace (used to rewind for duplicate key
|
|
358
|
+
// errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses
|
|
359
|
+
// the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that
|
|
360
|
+
// it's freed even if parsing raises.
|
|
361
|
+
//
|
|
362
|
+
// The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release
|
|
363
|
+
// and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts
|
|
364
|
+
// -- the element type and the absence of a mark function are the only real
|
|
365
|
+
// differences. Keep the two in sync: a fix to the spill/release or
|
|
366
|
+
// HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the
|
|
367
|
+
// other.
|
|
368
|
+
#define JSON_FRAME_STACK_INITIAL_CAPA 32
|
|
369
|
+
|
|
370
|
+
enum json_frame_type {
|
|
371
|
+
JSON_FRAME_ROOT, // == JSON_PHASE_DONE
|
|
372
|
+
JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA
|
|
373
|
+
JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
// Where a frame is within its container's grammar. This is the entirety of the
|
|
377
|
+
// parser's "what to do next" state: json_parse_any dispatches on the top
|
|
378
|
+
// frame's phase and holds no resume state in C locals, so a parse can stop at
|
|
379
|
+
// any value boundary and be resumed purely from the (persistable) frame stack.
|
|
380
|
+
//
|
|
381
|
+
// The first three phases are deliberately equal to the corresponding json_frame_type
|
|
382
|
+
// to simplify the transition of phase in json_value_completed.
|
|
383
|
+
enum json_frame_phase {
|
|
384
|
+
JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed
|
|
385
|
+
JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']'
|
|
386
|
+
JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}'
|
|
387
|
+
JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':')
|
|
388
|
+
JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',')
|
|
389
|
+
JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':'
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
typedef struct json_frame_struct {
|
|
393
|
+
enum json_frame_type type;
|
|
394
|
+
enum json_frame_phase phase;
|
|
395
|
+
long value_stack_head; // rvalue_stack->head when this container opened
|
|
396
|
+
size_t start_offset; // object frames only (the '{'); NULL otherwise
|
|
397
|
+
} json_frame;
|
|
398
|
+
|
|
399
|
+
typedef struct json_frame_stack_struct {
|
|
400
|
+
enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated
|
|
401
|
+
long capa;
|
|
402
|
+
long head;
|
|
403
|
+
json_frame *ptr;
|
|
404
|
+
} json_frame_stack;
|
|
337
405
|
|
|
338
|
-
enum
|
|
406
|
+
enum deprecatable_action {
|
|
339
407
|
JSON_DEPRECATED = 0,
|
|
340
408
|
JSON_IGNORE,
|
|
341
409
|
JSON_RAISE,
|
|
@@ -345,7 +413,8 @@ typedef struct JSON_ParserStruct {
|
|
|
345
413
|
VALUE on_load_proc;
|
|
346
414
|
VALUE decimal_class;
|
|
347
415
|
ID decimal_method_id;
|
|
348
|
-
enum
|
|
416
|
+
enum deprecatable_action on_duplicate_key;
|
|
417
|
+
enum deprecatable_action on_comment;
|
|
349
418
|
int max_nesting;
|
|
350
419
|
bool allow_nan;
|
|
351
420
|
bool allow_trailing_comma;
|
|
@@ -356,17 +425,152 @@ typedef struct JSON_ParserStruct {
|
|
|
356
425
|
} JSON_ParserConfig;
|
|
357
426
|
|
|
358
427
|
typedef struct JSON_ParserStateStruct {
|
|
359
|
-
VALUE *
|
|
428
|
+
VALUE *value_stack_handle;
|
|
429
|
+
VALUE *frame_stack_handle;
|
|
360
430
|
const char *start;
|
|
361
431
|
const char *cursor;
|
|
362
432
|
const char *end;
|
|
363
|
-
rvalue_stack *
|
|
433
|
+
rvalue_stack *value_stack;
|
|
434
|
+
json_frame_stack *frames;
|
|
364
435
|
rvalue_cache name_cache;
|
|
365
436
|
int in_array;
|
|
366
437
|
int current_nesting;
|
|
367
438
|
unsigned int emitted_deprecations;
|
|
439
|
+
VALUE parser;
|
|
368
440
|
} JSON_ParserState;
|
|
369
441
|
|
|
442
|
+
static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref);
|
|
443
|
+
|
|
444
|
+
static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref)
|
|
445
|
+
{
|
|
446
|
+
long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA;
|
|
447
|
+
|
|
448
|
+
if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
|
|
449
|
+
stack = json_frame_stack_spill(stack, handle, stack_ref);
|
|
450
|
+
} else {
|
|
451
|
+
JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa);
|
|
452
|
+
stack->capa = required;
|
|
453
|
+
}
|
|
454
|
+
return stack;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame)
|
|
458
|
+
{
|
|
459
|
+
json_frame_stack *stack = state->frames;
|
|
460
|
+
|
|
461
|
+
JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle);
|
|
462
|
+
|
|
463
|
+
if (RB_UNLIKELY(stack->head >= stack->capa)) {
|
|
464
|
+
stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames);
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
json_frame *frame_ptr = &stack->ptr[stack->head++];
|
|
468
|
+
*frame_ptr = frame;
|
|
469
|
+
return frame_ptr;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
static inline json_frame *json_frame_stack_peek(json_frame_stack *stack)
|
|
473
|
+
{
|
|
474
|
+
return &stack->ptr[stack->head - 1];
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
static inline void json_frame_stack_pop(json_frame_stack *stack)
|
|
478
|
+
{
|
|
479
|
+
stack->head--;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
static void json_frame_stack_free_buffer(json_frame_stack *stack)
|
|
483
|
+
{
|
|
484
|
+
JSON_SIZED_FREE_N(stack->ptr, stack->capa);
|
|
485
|
+
stack->ptr = NULL;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
static void json_frame_stack_free(void *ptr)
|
|
489
|
+
{
|
|
490
|
+
json_frame_stack *stack = (json_frame_stack *)ptr;
|
|
491
|
+
if (stack) {
|
|
492
|
+
json_frame_stack_free_buffer(stack);
|
|
493
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
494
|
+
JSON_SIZED_FREE(stack);
|
|
495
|
+
#endif
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
static size_t json_frame_stack_memsize(const void *ptr)
|
|
500
|
+
{
|
|
501
|
+
const json_frame_stack *stack = (const json_frame_stack *)ptr;
|
|
502
|
+
|
|
503
|
+
size_t memsize = sizeof(json_frame) * stack->capa;
|
|
504
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
505
|
+
memsize += sizeof(json_frame_stack);
|
|
506
|
+
#endif
|
|
507
|
+
return memsize;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
static const rb_data_type_t JSON_Parser_frame_stack_type = {
|
|
511
|
+
.wrap_struct_name = "JSON::Ext::Parser/frame_stack",
|
|
512
|
+
.function = {
|
|
513
|
+
.dmark = NULL,
|
|
514
|
+
.dfree = json_frame_stack_free,
|
|
515
|
+
.dsize = json_frame_stack_memsize,
|
|
516
|
+
},
|
|
517
|
+
.flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
|
|
518
|
+
};
|
|
519
|
+
|
|
520
|
+
static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
|
|
521
|
+
{
|
|
522
|
+
json_frame_stack *stack;
|
|
523
|
+
*handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
|
|
524
|
+
*stack_ref = stack;
|
|
525
|
+
MEMCPY(stack, old_stack, json_frame_stack, 1);
|
|
526
|
+
|
|
527
|
+
stack->capa = old_stack->capa << 1;
|
|
528
|
+
stack->ptr = ALLOC_N(json_frame, stack->capa);
|
|
529
|
+
stack->type = RVALUE_STACK_HEAP_ALLOCATED;
|
|
530
|
+
MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head);
|
|
531
|
+
return stack;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
static void json_frame_stack_eagerly_release(VALUE handle)
|
|
535
|
+
{
|
|
536
|
+
if (handle) {
|
|
537
|
+
json_frame_stack *stack;
|
|
538
|
+
TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
|
|
539
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
540
|
+
json_frame_stack_free_buffer(stack);
|
|
541
|
+
#else
|
|
542
|
+
json_frame_stack_free(stack);
|
|
543
|
+
RTYPEDDATA_DATA(handle) = NULL;
|
|
544
|
+
#endif
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
|
|
549
|
+
{
|
|
550
|
+
int len = 1;
|
|
551
|
+
if (ch <= 0x7F) {
|
|
552
|
+
buf[0] = (char) ch;
|
|
553
|
+
} else if (ch <= 0x07FF) {
|
|
554
|
+
buf[0] = (char) ((ch >> 6) | 0xC0);
|
|
555
|
+
buf[1] = (char) ((ch & 0x3F) | 0x80);
|
|
556
|
+
len++;
|
|
557
|
+
} else if (ch <= 0xFFFF) {
|
|
558
|
+
buf[0] = (char) ((ch >> 12) | 0xE0);
|
|
559
|
+
buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
|
|
560
|
+
buf[2] = (char) ((ch & 0x3F) | 0x80);
|
|
561
|
+
len += 2;
|
|
562
|
+
} else if (ch <= 0x1fffff) {
|
|
563
|
+
buf[0] =(char) ((ch >> 18) | 0xF0);
|
|
564
|
+
buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
|
|
565
|
+
buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
|
|
566
|
+
buf[3] =(char) ((ch & 0x3F) | 0x80);
|
|
567
|
+
len += 3;
|
|
568
|
+
} else {
|
|
569
|
+
buf[0] = '?';
|
|
570
|
+
}
|
|
571
|
+
return len;
|
|
572
|
+
}
|
|
573
|
+
|
|
370
574
|
static inline size_t rest(JSON_ParserState *state) {
|
|
371
575
|
return state->end - state->cursor;
|
|
372
576
|
}
|
|
@@ -385,12 +589,22 @@ static inline char peek(JSON_ParserState *state)
|
|
|
385
589
|
|
|
386
590
|
static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
|
|
387
591
|
{
|
|
592
|
+
JSON_ASSERT(!state->parser);
|
|
593
|
+
JSON_ASSERT(state->cursor);
|
|
594
|
+
JSON_ASSERT(state->cursor <= state->end);
|
|
595
|
+
|
|
596
|
+
// Redundant but helpful for hardening
|
|
597
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
598
|
+
state->cursor = state->end;
|
|
599
|
+
}
|
|
600
|
+
|
|
388
601
|
const char *cursor = state->cursor;
|
|
389
602
|
long column = 0;
|
|
390
603
|
long line = 1;
|
|
391
604
|
|
|
392
605
|
while (cursor >= state->start) {
|
|
393
606
|
if (*cursor-- == '\n') {
|
|
607
|
+
line++;
|
|
394
608
|
break;
|
|
395
609
|
}
|
|
396
610
|
column++;
|
|
@@ -405,18 +619,24 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum
|
|
|
405
619
|
*column_out = column;
|
|
406
620
|
}
|
|
407
621
|
|
|
622
|
+
static const unsigned int MAX_DEPRECATIONS = 5;
|
|
623
|
+
|
|
408
624
|
static void emit_parse_warning(const char *message, JSON_ParserState *state)
|
|
409
625
|
{
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
626
|
+
VALUE warning;
|
|
627
|
+
if (state->parser) { // line and columns can't be accurate in resumable
|
|
628
|
+
warning = rb_utf8_str_new_cstr(message);
|
|
629
|
+
} else {
|
|
630
|
+
long line, column;
|
|
631
|
+
cursor_position(state, &line, &column);
|
|
632
|
+
warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
|
|
633
|
+
}
|
|
414
634
|
rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning);
|
|
415
635
|
}
|
|
416
636
|
|
|
417
637
|
#define PARSE_ERROR_FRAGMENT_LEN 32
|
|
418
638
|
|
|
419
|
-
static VALUE build_parse_error_message(const char *format, JSON_ParserState *state
|
|
639
|
+
static VALUE build_parse_error_message(const char *format, JSON_ParserState *state)
|
|
420
640
|
{
|
|
421
641
|
unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
|
|
422
642
|
|
|
@@ -450,31 +670,61 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta
|
|
|
450
670
|
}
|
|
451
671
|
}
|
|
452
672
|
|
|
453
|
-
|
|
454
|
-
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
455
|
-
return message;
|
|
673
|
+
return rb_enc_sprintf(enc_utf8, format, ptr);
|
|
456
674
|
}
|
|
457
675
|
|
|
458
|
-
static VALUE parse_error_new(VALUE message, long line, long column)
|
|
676
|
+
static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
|
|
459
677
|
{
|
|
460
|
-
VALUE exc = rb_exc_new_str(
|
|
461
|
-
rb_ivar_set(exc,
|
|
462
|
-
rb_ivar_set(exc,
|
|
678
|
+
VALUE exc = rb_exc_new_str(eParserError, message);
|
|
679
|
+
rb_ivar_set(exc, i_at_line, LONG2NUM(line));
|
|
680
|
+
rb_ivar_set(exc, i_at_column, LONG2NUM(column));
|
|
463
681
|
return exc;
|
|
464
682
|
}
|
|
465
683
|
|
|
466
|
-
NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state)
|
|
684
|
+
NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos)
|
|
685
|
+
{
|
|
686
|
+
if (state->parser) {
|
|
687
|
+
if (eos) {
|
|
688
|
+
// the error will be swallowed by ResumableParser#parse, so no
|
|
689
|
+
// point building a message or backtrace.
|
|
690
|
+
rb_throw_obj(state->parser, state->parser);
|
|
691
|
+
} else {
|
|
692
|
+
// line and columns can't be accurate in resumable
|
|
693
|
+
rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos));
|
|
694
|
+
}
|
|
695
|
+
} else {
|
|
696
|
+
VALUE message = build_parse_error_message(format, state);
|
|
697
|
+
long line, column;
|
|
698
|
+
cursor_position(state, &line, &column);
|
|
699
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
700
|
+
rb_exc_raise(parse_error_new(state, message, line, column, eos));
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state)
|
|
467
705
|
{
|
|
468
|
-
|
|
469
|
-
cursor_position(state, &line, &column);
|
|
470
|
-
VALUE message = build_parse_error_message(format, state, line, column);
|
|
471
|
-
rb_exc_raise(parse_error_new(message, line, column));
|
|
706
|
+
raise_parse_error(format, state, true);
|
|
472
707
|
}
|
|
473
708
|
|
|
474
|
-
NORETURN(static) void
|
|
709
|
+
NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state)
|
|
710
|
+
{
|
|
711
|
+
raise_parse_error(format, state, false);
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos)
|
|
475
715
|
{
|
|
476
716
|
state->cursor = at;
|
|
477
|
-
raise_parse_error(format, state);
|
|
717
|
+
raise_parse_error(format, state, eos);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
721
|
+
{
|
|
722
|
+
raise_parse_error_at(format, state, at, true);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
726
|
+
{
|
|
727
|
+
raise_parse_error_at(format, state, at, false);
|
|
478
728
|
}
|
|
479
729
|
|
|
480
730
|
/* unicode */
|
|
@@ -499,7 +749,7 @@ static const signed char digit_values[256] = {
|
|
|
499
749
|
static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
|
|
500
750
|
{
|
|
501
751
|
if (RB_UNLIKELY(sp > spe - 4)) {
|
|
502
|
-
|
|
752
|
+
raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
503
753
|
}
|
|
504
754
|
|
|
505
755
|
const unsigned char *p = (const unsigned char *)sp;
|
|
@@ -510,7 +760,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
|
|
|
510
760
|
const signed char b3 = digit_values[p[3]];
|
|
511
761
|
|
|
512
762
|
if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
|
|
513
|
-
|
|
763
|
+
raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
514
764
|
}
|
|
515
765
|
|
|
516
766
|
return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
|
|
@@ -522,19 +772,36 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
|
|
|
522
772
|
|
|
523
773
|
static const rb_data_type_t JSON_ParserConfig_type;
|
|
524
774
|
|
|
525
|
-
|
|
526
|
-
|
|
775
|
+
const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
|
|
776
|
+
NOINLINE(static) void
|
|
777
|
+
json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos)
|
|
527
778
|
{
|
|
779
|
+
if (config->on_comment == JSON_RAISE) {
|
|
780
|
+
raise_syntax_error("unexpected token %s", state);
|
|
781
|
+
}
|
|
782
|
+
|
|
528
783
|
const char *start = state->cursor;
|
|
784
|
+
// An incomplete comment suspends a resumable parse by rewinding the cursor
|
|
785
|
+
// and throwing. Callers that already consumed a token not yet committed to
|
|
786
|
+
// the frame stack pass resume_pos so the rewind re-reads that token too.
|
|
787
|
+
// Non-resumable error positions keep pointing at the comment either way.
|
|
788
|
+
const char *rewind_pos = (state->parser && resume_pos) ? resume_pos : start;
|
|
529
789
|
state->cursor++;
|
|
530
790
|
|
|
531
791
|
switch (peek(state)) {
|
|
532
792
|
case '/': {
|
|
533
|
-
|
|
534
|
-
if (!
|
|
793
|
+
const char *newline = memchr(state->cursor, '\n', state->end - state->cursor);
|
|
794
|
+
if (!newline) {
|
|
795
|
+
// state->parser marks resumable mode, where the buffer end is only a
|
|
796
|
+
// chunk boundary: the terminating newline may still arrive, so leave
|
|
797
|
+
// the comment unterminated instead of consuming to end as a one-shot
|
|
798
|
+
// parse would.
|
|
799
|
+
if (state->parser) {
|
|
800
|
+
raise_eos_error_at("unterminated comment, expected end of line", state, rewind_pos);
|
|
801
|
+
}
|
|
535
802
|
state->cursor = state->end;
|
|
536
803
|
} else {
|
|
537
|
-
state->cursor
|
|
804
|
+
state->cursor = newline + 1;
|
|
538
805
|
}
|
|
539
806
|
break;
|
|
540
807
|
}
|
|
@@ -544,7 +811,7 @@ json_eat_comments(JSON_ParserState *state)
|
|
|
544
811
|
while (true) {
|
|
545
812
|
const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
|
|
546
813
|
if (!next_match) {
|
|
547
|
-
|
|
814
|
+
raise_eos_error_at("unterminated comment, expected closing '*/'", state, rewind_pos);
|
|
548
815
|
}
|
|
549
816
|
|
|
550
817
|
state->cursor = next_match + 1;
|
|
@@ -556,13 +823,18 @@ json_eat_comments(JSON_ParserState *state)
|
|
|
556
823
|
break;
|
|
557
824
|
}
|
|
558
825
|
default:
|
|
559
|
-
raise_parse_error_at("unexpected token %s", state, start);
|
|
826
|
+
raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state));
|
|
560
827
|
break;
|
|
561
828
|
}
|
|
829
|
+
|
|
830
|
+
if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) {
|
|
831
|
+
state->emitted_deprecations++;
|
|
832
|
+
emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state);
|
|
833
|
+
}
|
|
562
834
|
}
|
|
563
835
|
|
|
564
836
|
ALWAYS_INLINE(static) void
|
|
565
|
-
|
|
837
|
+
json_eat_whitespace_resume_at(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments, const char *resume_pos)
|
|
566
838
|
{
|
|
567
839
|
while (true) {
|
|
568
840
|
switch (peek(state)) {
|
|
@@ -593,7 +865,11 @@ json_eat_whitespace(JSON_ParserState *state)
|
|
|
593
865
|
state->cursor++;
|
|
594
866
|
break;
|
|
595
867
|
case '/':
|
|
596
|
-
|
|
868
|
+
if (!include_comments) {
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
json_eat_comments(state, config, resume_pos);
|
|
597
873
|
break;
|
|
598
874
|
|
|
599
875
|
default:
|
|
@@ -602,6 +878,12 @@ json_eat_whitespace(JSON_ParserState *state)
|
|
|
602
878
|
}
|
|
603
879
|
}
|
|
604
880
|
|
|
881
|
+
ALWAYS_INLINE(static) void
|
|
882
|
+
json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
|
|
883
|
+
{
|
|
884
|
+
json_eat_whitespace_resume_at(state, config, include_comments, NULL);
|
|
885
|
+
}
|
|
886
|
+
|
|
605
887
|
static inline VALUE build_string(const char *start, const char *end, bool intern, bool symbolize)
|
|
606
888
|
{
|
|
607
889
|
if (symbolize) {
|
|
@@ -747,13 +1029,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
747
1029
|
uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
|
|
748
1030
|
|
|
749
1031
|
if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
|
|
750
|
-
|
|
1032
|
+
raise_syntax_error_at("invalid surrogate pair at %s", state, p);
|
|
751
1033
|
}
|
|
752
1034
|
|
|
753
1035
|
ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
|
|
754
1036
|
pe += 5;
|
|
755
1037
|
} else {
|
|
756
|
-
|
|
1038
|
+
raise_syntax_error_at("incomplete surrogate pair at %s", state, p);
|
|
757
1039
|
break;
|
|
758
1040
|
}
|
|
759
1041
|
}
|
|
@@ -763,20 +1045,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
763
1045
|
p = ++pe;
|
|
764
1046
|
break;
|
|
765
1047
|
}
|
|
1048
|
+
case 0:
|
|
1049
|
+
return Qundef;
|
|
766
1050
|
default:
|
|
767
1051
|
if ((unsigned char)*pe < 0x20) {
|
|
768
1052
|
if (!config->allow_control_characters) {
|
|
769
1053
|
if (*pe == '\n') {
|
|
770
|
-
|
|
1054
|
+
raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
|
|
771
1055
|
}
|
|
772
|
-
|
|
1056
|
+
raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1);
|
|
773
1057
|
}
|
|
774
1058
|
}
|
|
775
1059
|
|
|
776
1060
|
if (config->allow_invalid_escape) {
|
|
777
1061
|
APPEND_CHAR(*pe);
|
|
778
1062
|
} else {
|
|
779
|
-
|
|
1063
|
+
raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1);
|
|
780
1064
|
}
|
|
781
1065
|
break;
|
|
782
1066
|
}
|
|
@@ -872,19 +1156,24 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
|
|
|
872
1156
|
return rb_float_new(negative ? -0.0 : 0.0);
|
|
873
1157
|
}
|
|
874
1158
|
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
1159
|
+
if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
|
|
1160
|
+
// If the value is so small that it definitely underflows to 0.0, return early
|
|
1161
|
+
// to avoid triggering a "Float out of range" warning from rb_cstr_to_dbl.
|
|
1162
|
+
// When mantissa_digits + exponent < -324, value < 10^(-324) < DBL_TRUE_MIN/2,
|
|
1163
|
+
// so it rounds to 0 in IEEE 754 round-to-nearest.
|
|
1164
|
+
if (RB_UNLIKELY(mantissa_digits + exponent < -324)) {
|
|
1165
|
+
return rb_float_new(negative ? -0.0 : 0.0);
|
|
1166
|
+
}
|
|
878
1167
|
return json_decode_large_float(start, end - start);
|
|
879
1168
|
}
|
|
880
1169
|
|
|
881
|
-
return DBL2NUM(
|
|
1170
|
+
return DBL2NUM(ffp_s2d(exponent, mantissa, negative));
|
|
882
1171
|
}
|
|
883
1172
|
|
|
884
1173
|
static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
|
|
885
1174
|
{
|
|
886
|
-
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->
|
|
887
|
-
rvalue_stack_pop(state->
|
|
1175
|
+
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count));
|
|
1176
|
+
rvalue_stack_pop(state->value_stack, count);
|
|
888
1177
|
|
|
889
1178
|
if (config->freeze) {
|
|
890
1179
|
RB_OBJ_FREEZE(array);
|
|
@@ -928,38 +1217,50 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d
|
|
|
928
1217
|
rb_inspect(duplicate_key)
|
|
929
1218
|
);
|
|
930
1219
|
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
1220
|
+
rb_str_concat(message, build_parse_error_message("", state));
|
|
1221
|
+
if (state->parser) { // line and columns can't be accurate in resumable
|
|
1222
|
+
rb_exc_raise(parse_error_new(state, message, 0, 0, false));
|
|
1223
|
+
} else {
|
|
1224
|
+
long line, column;
|
|
1225
|
+
cursor_position(state, &line, &column);
|
|
1226
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
1227
|
+
rb_exc_raise(parse_error_new(state, message, line, column, false));
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
|
|
1232
|
+
{
|
|
1233
|
+
switch (config->on_duplicate_key) {
|
|
1234
|
+
case JSON_IGNORE:
|
|
1235
|
+
return;
|
|
1236
|
+
|
|
1237
|
+
case JSON_DEPRECATED:
|
|
1238
|
+
// Only emit the first few deprecations to avoid spamming.
|
|
1239
|
+
if (state->emitted_deprecations < MAX_DEPRECATIONS) {
|
|
1240
|
+
state->emitted_deprecations++;
|
|
1241
|
+
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
1242
|
+
}
|
|
1243
|
+
return;
|
|
1244
|
+
|
|
1245
|
+
case JSON_RAISE:
|
|
1246
|
+
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
UNREACHABLE;
|
|
935
1250
|
}
|
|
936
1251
|
|
|
937
1252
|
static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
|
|
938
1253
|
{
|
|
939
1254
|
size_t entries_count = count / 2;
|
|
940
1255
|
VALUE object = rb_hash_new_capa(entries_count);
|
|
941
|
-
const VALUE *pairs = rvalue_stack_peek(state->
|
|
1256
|
+
const VALUE *pairs = rvalue_stack_peek(state->value_stack, count);
|
|
942
1257
|
rb_hash_bulk_insert(count, pairs, object);
|
|
943
1258
|
|
|
944
1259
|
if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
|
|
945
|
-
|
|
946
|
-
case JSON_IGNORE:
|
|
947
|
-
break;
|
|
948
|
-
case JSON_DEPRECATED:
|
|
949
|
-
// Only emit the first few deprecations to avoid spamming.
|
|
950
|
-
if (state->emitted_deprecations < 5) {
|
|
951
|
-
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
952
|
-
state->emitted_deprecations++;
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
break;
|
|
956
|
-
case JSON_RAISE:
|
|
957
|
-
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
958
|
-
break;
|
|
959
|
-
}
|
|
1260
|
+
json_on_duplicate_key(state, config, count, pairs);
|
|
960
1261
|
}
|
|
961
1262
|
|
|
962
|
-
rvalue_stack_pop(state->
|
|
1263
|
+
rvalue_stack_pop(state->value_stack, count);
|
|
963
1264
|
|
|
964
1265
|
if (config->freeze) {
|
|
965
1266
|
RB_OBJ_FREEZE(object);
|
|
@@ -973,7 +1274,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
|
|
|
973
1274
|
if (RB_UNLIKELY(config->on_load_proc)) {
|
|
974
1275
|
value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
|
|
975
1276
|
}
|
|
976
|
-
rvalue_stack_push(state->
|
|
1277
|
+
rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack);
|
|
977
1278
|
return value;
|
|
978
1279
|
}
|
|
979
1280
|
|
|
@@ -1022,6 +1323,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
|
|
|
1022
1323
|
}
|
|
1023
1324
|
state->cursor++;
|
|
1024
1325
|
}
|
|
1326
|
+
|
|
1327
|
+
// If the string ended with an unterminated escape sequence, we might
|
|
1328
|
+
// have gone past the end.
|
|
1329
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
1330
|
+
state->cursor = state->end;
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1025
1333
|
return false;
|
|
1026
1334
|
}
|
|
1027
1335
|
|
|
@@ -1039,7 +1347,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1039
1347
|
case '"': {
|
|
1040
1348
|
VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
|
|
1041
1349
|
state->cursor++;
|
|
1042
|
-
return
|
|
1350
|
+
return string;
|
|
1043
1351
|
}
|
|
1044
1352
|
case '\\': {
|
|
1045
1353
|
if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
|
|
@@ -1053,7 +1361,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1053
1361
|
}
|
|
1054
1362
|
default:
|
|
1055
1363
|
if (!config->allow_control_characters) {
|
|
1056
|
-
|
|
1364
|
+
raise_syntax_error("invalid ASCII control character in string: %s", state);
|
|
1057
1365
|
}
|
|
1058
1366
|
break;
|
|
1059
1367
|
}
|
|
@@ -1061,8 +1369,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1061
1369
|
state->cursor++;
|
|
1062
1370
|
} while (string_scan(state));
|
|
1063
1371
|
|
|
1064
|
-
|
|
1065
|
-
return Qfalse;
|
|
1372
|
+
return Qundef;
|
|
1066
1373
|
}
|
|
1067
1374
|
|
|
1068
1375
|
ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
|
|
@@ -1071,15 +1378,19 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars
|
|
|
1071
1378
|
const char *start = state->cursor;
|
|
1072
1379
|
|
|
1073
1380
|
if (RB_UNLIKELY(!string_scan(state))) {
|
|
1074
|
-
|
|
1381
|
+
return Qundef;
|
|
1075
1382
|
}
|
|
1076
1383
|
|
|
1384
|
+
VALUE string;
|
|
1077
1385
|
if (RB_LIKELY(*state->cursor == '"')) {
|
|
1078
|
-
|
|
1386
|
+
string = json_string_fastpath(state, config, start, state->cursor, is_name);
|
|
1079
1387
|
state->cursor++;
|
|
1080
|
-
return json_push_value(state, config, string);
|
|
1081
1388
|
}
|
|
1082
|
-
|
|
1389
|
+
else {
|
|
1390
|
+
string = json_parse_escaped_string(state, config, is_name, start);
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
return string;
|
|
1083
1394
|
}
|
|
1084
1395
|
|
|
1085
1396
|
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
@@ -1152,7 +1463,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat
|
|
|
1152
1463
|
return (int)(state->cursor - start);
|
|
1153
1464
|
}
|
|
1154
1465
|
|
|
1155
|
-
static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
|
|
1466
|
+
static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable)
|
|
1156
1467
|
{
|
|
1157
1468
|
bool integer = true;
|
|
1158
1469
|
const char first_digit = *state->cursor;
|
|
@@ -1166,7 +1477,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1166
1477
|
int mantissa_digits = json_parse_digits(state, &mantissa);
|
|
1167
1478
|
|
|
1168
1479
|
if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
|
|
1169
|
-
|
|
1480
|
+
return Qundef;
|
|
1170
1481
|
}
|
|
1171
1482
|
|
|
1172
1483
|
// Parse fractional part
|
|
@@ -1179,7 +1490,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1179
1490
|
mantissa_digits += fractional_digits;
|
|
1180
1491
|
|
|
1181
1492
|
if (RB_UNLIKELY(!fractional_digits)) {
|
|
1182
|
-
|
|
1493
|
+
return Qundef;
|
|
1183
1494
|
}
|
|
1184
1495
|
}
|
|
1185
1496
|
|
|
@@ -1199,10 +1510,24 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1199
1510
|
int exponent_digits = json_parse_digits(state, &abs_exponent);
|
|
1200
1511
|
|
|
1201
1512
|
if (RB_UNLIKELY(!exponent_digits)) {
|
|
1202
|
-
|
|
1513
|
+
return Qundef;
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
|
|
1517
|
+
exponent = negative_exponent ? INT64_MIN : INT64_MAX;
|
|
1518
|
+
} else {
|
|
1519
|
+
exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
|
|
1203
1520
|
}
|
|
1521
|
+
}
|
|
1204
1522
|
|
|
1205
|
-
|
|
1523
|
+
// A number touching the end of the buffer may still grow in a later chunk,
|
|
1524
|
+
// so the caller will rewind and wait. Decoding it now would build a value
|
|
1525
|
+
// -- for a long run of digits, an expensive bignum -- only to discard it,
|
|
1526
|
+
// and repeating that on every resumed chunk is quadratic in the number's
|
|
1527
|
+
// length. The digit scan above already advanced the cursor, which is all
|
|
1528
|
+
// the caller needs to detect the incomplete number.
|
|
1529
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1530
|
+
return Qundef;
|
|
1206
1531
|
}
|
|
1207
1532
|
|
|
1208
1533
|
if (integer) {
|
|
@@ -1217,229 +1542,420 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1217
1542
|
return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
|
|
1218
1543
|
}
|
|
1219
1544
|
|
|
1220
|
-
|
|
1545
|
+
// How many values (array elements, or interleaved object keys+values) have been
|
|
1546
|
+
// pushed onto the rvalue stack since this container opened. Used to size the
|
|
1547
|
+
// bulk decode on close, and to tell the first key/colon from later ones.
|
|
1548
|
+
static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack)
|
|
1221
1549
|
{
|
|
1222
|
-
return
|
|
1550
|
+
return value_stack->head - frame->value_stack_head;
|
|
1223
1551
|
}
|
|
1224
1552
|
|
|
1225
|
-
|
|
1553
|
+
// A complete value now sits on top of the rvalue stack. Advance the frame that
|
|
1554
|
+
// was waiting for it: the root document is done, or the enclosing container
|
|
1555
|
+
// moves on to expecting a ',' or its closing bracket. The caller passes the
|
|
1556
|
+
// frame it already has in hand -- the one that was expecting the value -- which
|
|
1557
|
+
// after a container close is the freshly re-exposed parent.
|
|
1558
|
+
static inline enum json_frame_phase json_value_completed(json_frame *frame)
|
|
1226
1559
|
{
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1560
|
+
JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT);
|
|
1561
|
+
JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY);
|
|
1562
|
+
JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT);
|
|
1563
|
+
|
|
1564
|
+
return frame->phase = (enum json_frame_phase) frame->type;
|
|
1230
1565
|
}
|
|
1231
1566
|
|
|
1232
|
-
static
|
|
1567
|
+
ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset)
|
|
1233
1568
|
{
|
|
1234
|
-
|
|
1569
|
+
// It is assumed that since `keyword` is always a literal, the compiler is able to constantize this
|
|
1570
|
+
// `strlen` and several other computations in that routine.
|
|
1235
1571
|
|
|
1236
|
-
|
|
1237
|
-
case 'n':
|
|
1238
|
-
if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
|
|
1239
|
-
state->cursor += 4;
|
|
1240
|
-
return json_push_value(state, config, Qnil);
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
raise_parse_error("unexpected token %s", state);
|
|
1244
|
-
break;
|
|
1245
|
-
case 't':
|
|
1246
|
-
if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) {
|
|
1247
|
-
state->cursor += 4;
|
|
1248
|
-
return json_push_value(state, config, Qtrue);
|
|
1249
|
-
}
|
|
1572
|
+
size_t len = strlen(keyword);
|
|
1250
1573
|
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
return json_push_value(state, config, Qfalse);
|
|
1258
|
-
}
|
|
1574
|
+
// Note: memcmp with a small power of two and a literal string compile to an integer comparison /
|
|
1575
|
+
// That's why we sometime compare starting from the first byte and sometimes from the second.
|
|
1576
|
+
if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) {
|
|
1577
|
+
state->cursor += len;
|
|
1578
|
+
return;
|
|
1579
|
+
}
|
|
1259
1580
|
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1581
|
+
bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0;
|
|
1582
|
+
raise_parse_error("unexpected token %s", state, eos);
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
// Parse an arbitrary JSON value iteratively. This is a state machine driven
|
|
1586
|
+
// entirely by the top frame's phase so it can stop at any value boundary and
|
|
1587
|
+
// resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the
|
|
1588
|
+
// bottom of the stack, so the stack is never empty mid-parse and the document
|
|
1589
|
+
// itself is just another frame whose value, once parsed, leaves its phase DONE.
|
|
1590
|
+
// When invoked in resumable mode, it returns true after parsing a complete document.
|
|
1591
|
+
// If reaching EOS without having parsed a complete document, either returns false
|
|
1592
|
+
// of raise a JSON::ParserError tagged with `@eos=true`.
|
|
1593
|
+
ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable)
|
|
1594
|
+
{
|
|
1595
|
+
json_frame *frame = json_frame_stack_peek(state->frames);
|
|
1596
|
+
|
|
1597
|
+
switch (frame->phase) {
|
|
1598
|
+
case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false);
|
|
1599
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1600
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1601
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1602
|
+
case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY;
|
|
1603
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1604
|
+
}
|
|
1605
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1268
1606
|
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
case 'I':
|
|
1272
|
-
if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
|
|
1273
|
-
state->cursor += 8;
|
|
1274
|
-
return json_push_value(state, config, CInfinity);
|
|
1275
|
-
}
|
|
1607
|
+
JSON_PHASE_VALUE: {
|
|
1608
|
+
json_eat_whitespace(state, config, true);
|
|
1276
1609
|
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
if (config->allow_nan) {
|
|
1283
|
-
state->cursor += 9;
|
|
1284
|
-
return json_push_value(state, config, CMinusInfinity);
|
|
1285
|
-
} else {
|
|
1286
|
-
raise_parse_error("unexpected token %s", state);
|
|
1287
|
-
}
|
|
1288
|
-
}
|
|
1289
|
-
return json_push_value(state, config, json_parse_negative_number(state, config));
|
|
1290
|
-
break;
|
|
1291
|
-
}
|
|
1292
|
-
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
|
|
1293
|
-
return json_push_value(state, config, json_parse_positive_number(state, config));
|
|
1294
|
-
break;
|
|
1295
|
-
case '"': {
|
|
1296
|
-
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1297
|
-
return json_parse_string(state, config, false);
|
|
1298
|
-
break;
|
|
1610
|
+
// A trailing comma lands us here expecting an element but finding the
|
|
1611
|
+
// closing bracket; hand off to ARRAY_COMMA to close. An empty array
|
|
1612
|
+
// closes inline at '[', so this position is only reached after a ','.
|
|
1613
|
+
if (config->allow_trailing_comma && frame->type == JSON_FRAME_ARRAY && peek(state) == ']') {
|
|
1614
|
+
goto JSON_PHASE_ARRAY_COMMA;
|
|
1299
1615
|
}
|
|
1300
|
-
case '[': {
|
|
1301
|
-
state->cursor++;
|
|
1302
|
-
json_eat_whitespace(state);
|
|
1303
|
-
long stack_head = state->stack->head;
|
|
1304
1616
|
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
return json_push_value(state, config, json_decode_array(state, config, 0));
|
|
1308
|
-
} else {
|
|
1309
|
-
state->current_nesting++;
|
|
1310
|
-
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1311
|
-
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1312
|
-
}
|
|
1313
|
-
state->in_array++;
|
|
1314
|
-
json_parse_any(state, config);
|
|
1315
|
-
}
|
|
1617
|
+
VALUE value;
|
|
1618
|
+
const char *value_start = state->cursor;
|
|
1316
1619
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1620
|
+
switch (peek(state)) {
|
|
1621
|
+
case 'n':
|
|
1622
|
+
json_match_keyword(state, "null", 0);
|
|
1623
|
+
value = Qnil;
|
|
1624
|
+
break;
|
|
1319
1625
|
|
|
1320
|
-
|
|
1626
|
+
case 't':
|
|
1627
|
+
json_match_keyword(state, "true", 0);
|
|
1628
|
+
value = Qtrue;
|
|
1629
|
+
break;
|
|
1321
1630
|
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
if (peek(state) == ']') {
|
|
1327
|
-
continue;
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
json_parse_any(state, config);
|
|
1331
|
-
continue;
|
|
1332
|
-
}
|
|
1631
|
+
case 'f':
|
|
1632
|
+
json_match_keyword(state, "false", 1);
|
|
1633
|
+
value = Qfalse;
|
|
1634
|
+
break;
|
|
1333
1635
|
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
state->current_nesting--;
|
|
1338
|
-
state->in_array--;
|
|
1339
|
-
return json_push_value(state, config, json_decode_array(state, config, count));
|
|
1636
|
+
case 'N':
|
|
1637
|
+
if (!config->allow_nan) {
|
|
1638
|
+
raise_syntax_error("unexpected token %s", state);
|
|
1340
1639
|
}
|
|
1341
1640
|
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
}
|
|
1346
|
-
case '{': {
|
|
1347
|
-
const char *object_start_cursor = state->cursor;
|
|
1641
|
+
json_match_keyword(state, "NaN", 1);
|
|
1642
|
+
value = CNaN;
|
|
1643
|
+
break;
|
|
1348
1644
|
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1645
|
+
case 'I':
|
|
1646
|
+
if (!config->allow_nan) {
|
|
1647
|
+
raise_syntax_error("unexpected token %s", state);
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
json_match_keyword(state, "Infinity", 0);
|
|
1651
|
+
value = CInfinity;
|
|
1652
|
+
break;
|
|
1352
1653
|
|
|
1353
|
-
|
|
1654
|
+
case '-': {
|
|
1354
1655
|
state->cursor++;
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
if (RB_UNLIKELY(
|
|
1359
|
-
|
|
1656
|
+
|
|
1657
|
+
value = json_parse_number(state, config, true, value_start, resumable);
|
|
1658
|
+
|
|
1659
|
+
if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
|
|
1660
|
+
state->cursor = value_start;
|
|
1661
|
+
json_match_keyword(state, "-Infinity", 1);
|
|
1662
|
+
value = CMinusInfinity;
|
|
1663
|
+
break;
|
|
1360
1664
|
}
|
|
1361
1665
|
|
|
1362
|
-
|
|
1363
|
-
|
|
1666
|
+
// Top level numbers are ambiguous when parsing streams, we can't
|
|
1667
|
+
// know if we parsed all the digits if we hit EOS.
|
|
1668
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1669
|
+
state->cursor = value_start;
|
|
1670
|
+
return false;
|
|
1364
1671
|
}
|
|
1365
|
-
json_parse_string(state, config, true);
|
|
1366
1672
|
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
raise_parse_error("expected ':' after object key", state);
|
|
1673
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1674
|
+
raise_syntax_error_at("invalid number: %s", state, value_start);
|
|
1370
1675
|
}
|
|
1676
|
+
break;
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
|
|
1680
|
+
value = json_parse_number(state, config, false, value_start, resumable);
|
|
1681
|
+
|
|
1682
|
+
// Top level numbers are ambiguous when parsing streams, we can't
|
|
1683
|
+
// know if we parsed all the digits if we hit EOS.
|
|
1684
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1685
|
+
state->cursor = value_start;
|
|
1686
|
+
return false;
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1690
|
+
raise_syntax_error_at("invalid number: %s", state, value_start);
|
|
1691
|
+
}
|
|
1692
|
+
break;
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
case '"': {
|
|
1696
|
+
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1697
|
+
value = json_parse_string(state, config, false);
|
|
1698
|
+
|
|
1699
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1700
|
+
bool is_eos = eos(state);
|
|
1701
|
+
if (resumable && is_eos) {
|
|
1702
|
+
state->cursor = value_start;
|
|
1703
|
+
return false;
|
|
1704
|
+
}
|
|
1705
|
+
raise_parse_error("unexpected end of input, expected closing \"", state, is_eos);
|
|
1706
|
+
}
|
|
1707
|
+
break;
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
case '[': {
|
|
1371
1711
|
state->cursor++;
|
|
1712
|
+
// The '[' is consumed but its frame is only pushed below, so a
|
|
1713
|
+
// comment suspending here must resume from the bracket.
|
|
1714
|
+
json_eat_whitespace_resume_at(state, config, true, value_start);
|
|
1715
|
+
|
|
1716
|
+
const char next = peek(state);
|
|
1717
|
+
if (next == ']') {
|
|
1718
|
+
state->cursor++;
|
|
1719
|
+
value = json_decode_array(state, config, 0);
|
|
1720
|
+
break;
|
|
1721
|
+
} else if (resumable && eos(state)) {
|
|
1722
|
+
state->cursor = value_start;
|
|
1723
|
+
return false;
|
|
1724
|
+
}
|
|
1372
1725
|
|
|
1373
|
-
|
|
1726
|
+
state->current_nesting++;
|
|
1727
|
+
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1728
|
+
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1729
|
+
}
|
|
1730
|
+
state->in_array++;
|
|
1731
|
+
|
|
1732
|
+
// Phase stays VALUE: the next iteration reads the first element.
|
|
1733
|
+
frame = json_frame_stack_push(state, (json_frame){
|
|
1734
|
+
.type = JSON_FRAME_ARRAY,
|
|
1735
|
+
.phase = JSON_PHASE_VALUE,
|
|
1736
|
+
.value_stack_head = state->value_stack->head,
|
|
1737
|
+
});
|
|
1738
|
+
goto JSON_PHASE_VALUE;
|
|
1374
1739
|
}
|
|
1375
1740
|
|
|
1376
|
-
|
|
1377
|
-
|
|
1741
|
+
case '{': {
|
|
1742
|
+
state->cursor++;
|
|
1743
|
+
// Same as '[': the frame is only pushed below.
|
|
1744
|
+
json_eat_whitespace_resume_at(state, config, true, value_start);
|
|
1378
1745
|
|
|
1379
|
-
|
|
1380
|
-
if (next_char == '}') {
|
|
1746
|
+
if (peek(state) == '}') {
|
|
1381
1747
|
state->cursor++;
|
|
1382
|
-
state
|
|
1383
|
-
|
|
1748
|
+
value = json_decode_object(state, config, 0);
|
|
1749
|
+
break;
|
|
1750
|
+
} else if (resumable && eos(state)) {
|
|
1751
|
+
state->cursor = value_start;
|
|
1752
|
+
return false;
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
state->current_nesting++;
|
|
1756
|
+
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1757
|
+
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1758
|
+
}
|
|
1384
1759
|
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1760
|
+
// Phase KEY: the next iteration reads the first key.
|
|
1761
|
+
frame = json_frame_stack_push(state, (json_frame){
|
|
1762
|
+
.type = JSON_FRAME_OBJECT,
|
|
1763
|
+
.phase = JSON_PHASE_OBJECT_KEY,
|
|
1764
|
+
.value_stack_head = state->value_stack->head,
|
|
1765
|
+
.start_offset = value_start - state->start,
|
|
1766
|
+
});
|
|
1767
|
+
goto JSON_PHASE_OBJECT_KEY;
|
|
1768
|
+
}
|
|
1390
1769
|
|
|
1391
|
-
|
|
1770
|
+
case 0:
|
|
1771
|
+
// peek() returns 0 both at end-of-stream and for a literal NUL byte in the
|
|
1772
|
+
// buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at
|
|
1773
|
+
// EOS is just an invalid character.
|
|
1774
|
+
if (eos(state)) {
|
|
1775
|
+
return false;
|
|
1776
|
+
} else {
|
|
1777
|
+
raise_syntax_error("unexpected NULL byte: %s", state);
|
|
1392
1778
|
}
|
|
1779
|
+
default:
|
|
1780
|
+
raise_syntax_error("unexpected character: %s", state);
|
|
1781
|
+
}
|
|
1393
1782
|
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
json_eat_whitespace(state);
|
|
1783
|
+
json_push_value(state, config, value);
|
|
1784
|
+
json_value_completed(frame);
|
|
1397
1785
|
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1786
|
+
switch (frame->phase) {
|
|
1787
|
+
case JSON_PHASE_DONE: return true;
|
|
1788
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1789
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1790
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1791
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1792
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1793
|
+
}
|
|
1794
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1795
|
+
}
|
|
1403
1796
|
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
}
|
|
1407
|
-
json_parse_string(state, config, true);
|
|
1797
|
+
JSON_PHASE_OBJECT_KEY: {
|
|
1798
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1408
1799
|
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1800
|
+
json_eat_whitespace(state, config, true);
|
|
1801
|
+
|
|
1802
|
+
// A trailing comma lands us here expecting a key but finding the closing
|
|
1803
|
+
// brace; hand off to OBJECT_COMMA to close. An empty object closes inline
|
|
1804
|
+
// at '{', so this position is only reached after a ','.
|
|
1805
|
+
if (config->allow_trailing_comma && peek(state) == '}') {
|
|
1806
|
+
goto JSON_PHASE_OBJECT_COMMA;
|
|
1807
|
+
}
|
|
1414
1808
|
|
|
1415
|
-
|
|
1809
|
+
const char *start = state->cursor;
|
|
1416
1810
|
|
|
1417
|
-
|
|
1811
|
+
if (RB_LIKELY(peek(state) == '"')) {
|
|
1812
|
+
VALUE string = json_parse_string(state, config, true);
|
|
1813
|
+
if (UNDEF_P(string)) {
|
|
1814
|
+
if (resumable) {
|
|
1815
|
+
state->cursor = start;
|
|
1816
|
+
return false;
|
|
1817
|
+
} else {
|
|
1818
|
+
raise_syntax_error("unexpected end of input, expected closing \"", state);
|
|
1418
1819
|
}
|
|
1820
|
+
}
|
|
1821
|
+
json_push_value(state, config, string);
|
|
1822
|
+
frame->phase = JSON_PHASE_OBJECT_COLON;
|
|
1823
|
+
goto JSON_PHASE_OBJECT_COLON;
|
|
1824
|
+
} else if (resumable && eos(state)) {
|
|
1825
|
+
return false;
|
|
1826
|
+
} else {
|
|
1827
|
+
// The message differs for the first key vs. a key after a
|
|
1828
|
+
// ',': the first is the only one reached with nothing pushed
|
|
1829
|
+
// for this object yet.
|
|
1830
|
+
if (json_frame_entry_count(frame, state->value_stack) == 0) {
|
|
1831
|
+
raise_syntax_error("expected object key, got %s", state);
|
|
1832
|
+
} else {
|
|
1833
|
+
raise_syntax_error("expected object key, got: %s", state);
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
JSON_PHASE_OBJECT_COLON: {
|
|
1840
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1419
1841
|
|
|
1420
|
-
|
|
1842
|
+
json_eat_whitespace(state, config, true);
|
|
1843
|
+
|
|
1844
|
+
if (RB_LIKELY(peek(state) == ':')) {
|
|
1845
|
+
state->cursor++;
|
|
1846
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
1847
|
+
goto JSON_PHASE_VALUE;
|
|
1848
|
+
} else if (resumable && eos(state)) {
|
|
1849
|
+
return false;
|
|
1850
|
+
} else {
|
|
1851
|
+
// First colon (only the first pair's key is pushed, nothing
|
|
1852
|
+
// else) vs. a later one.
|
|
1853
|
+
if (json_frame_entry_count(frame, state->value_stack) == 1) {
|
|
1854
|
+
raise_syntax_error("expected ':' after object key", state);
|
|
1855
|
+
} else {
|
|
1856
|
+
raise_syntax_error("expected ':' after object key, got: %s", state);
|
|
1421
1857
|
}
|
|
1422
|
-
break;
|
|
1423
1858
|
}
|
|
1859
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1860
|
+
}
|
|
1424
1861
|
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
break;
|
|
1862
|
+
JSON_PHASE_ARRAY_COMMA: {
|
|
1863
|
+
JSON_ASSERT(frame->type == JSON_FRAME_ARRAY);
|
|
1428
1864
|
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1865
|
+
json_eat_whitespace(state, config, true);
|
|
1866
|
+
|
|
1867
|
+
const char next_char = peek(state);
|
|
1868
|
+
|
|
1869
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1870
|
+
state->cursor++;
|
|
1871
|
+
// Commit the phase before eating the whitespace that follows: an
|
|
1872
|
+
// incomplete comment there would suspend the parse, and a phase not
|
|
1873
|
+
// yet advanced past the ',' would drop it on resume. A trailing comma
|
|
1874
|
+
// is recognized in JSON_PHASE_VALUE once the ']' is in the buffer.
|
|
1875
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
1876
|
+
goto JSON_PHASE_VALUE;
|
|
1877
|
+
} else if (next_char == ']') {
|
|
1878
|
+
state->cursor++;
|
|
1879
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
1880
|
+
state->current_nesting--;
|
|
1881
|
+
state->in_array--;
|
|
1882
|
+
|
|
1883
|
+
json_push_value(state, config, json_decode_array(state, config, count));
|
|
1884
|
+
json_frame_stack_pop(state->frames);
|
|
1885
|
+
frame = json_frame_stack_peek(state->frames);
|
|
1886
|
+
|
|
1887
|
+
json_value_completed(frame);
|
|
1888
|
+
|
|
1889
|
+
switch (frame->phase) {
|
|
1890
|
+
case JSON_PHASE_DONE: return true;
|
|
1891
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1892
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1893
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1894
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1895
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1896
|
+
}
|
|
1897
|
+
} else if (resumable && eos(state)) {
|
|
1898
|
+
return false;
|
|
1899
|
+
} else {
|
|
1900
|
+
raise_syntax_error("expected ',' or ']' after array value", state);
|
|
1901
|
+
}
|
|
1902
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1432
1903
|
}
|
|
1433
1904
|
|
|
1434
|
-
|
|
1435
|
-
|
|
1905
|
+
JSON_PHASE_OBJECT_COMMA: {
|
|
1906
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1907
|
+
|
|
1908
|
+
json_eat_whitespace(state, config, true);
|
|
1909
|
+
const char next_char = peek(state);
|
|
1910
|
+
|
|
1911
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1912
|
+
state->cursor++;
|
|
1913
|
+
// Commit the phase before eating the whitespace that follows: an
|
|
1914
|
+
// incomplete comment there would suspend the parse, and a phase not
|
|
1915
|
+
// yet advanced past the ',' would drop it on resume. A trailing comma
|
|
1916
|
+
// is recognized in JSON_PHASE_OBJECT_KEY once the '}' is in the buffer.
|
|
1917
|
+
frame->phase = JSON_PHASE_OBJECT_KEY;
|
|
1918
|
+
goto JSON_PHASE_OBJECT_KEY;
|
|
1919
|
+
} else if (next_char == '}') {
|
|
1920
|
+
state->cursor++;
|
|
1921
|
+
state->current_nesting--;
|
|
1922
|
+
size_t count = json_frame_entry_count(frame, state->value_stack);
|
|
1923
|
+
|
|
1924
|
+
// Temporary rewind cursor in case an error is raised
|
|
1925
|
+
const char *final_cursor = state->cursor;
|
|
1926
|
+
state->cursor = state->start + frame->start_offset;
|
|
1927
|
+
VALUE object = json_decode_object(state, config, count);
|
|
1928
|
+
state->cursor = final_cursor;
|
|
1929
|
+
|
|
1930
|
+
json_push_value(state, config, object);
|
|
1931
|
+
json_frame_stack_pop(state->frames);
|
|
1932
|
+
frame = json_frame_stack_peek(state->frames);
|
|
1933
|
+
json_value_completed(frame);
|
|
1934
|
+
|
|
1935
|
+
switch (frame->phase) {
|
|
1936
|
+
case JSON_PHASE_DONE: return true;
|
|
1937
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1938
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1939
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1940
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1941
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1942
|
+
}
|
|
1943
|
+
} else if (resumable && eos(state)) {
|
|
1944
|
+
return false;
|
|
1945
|
+
} else {
|
|
1946
|
+
raise_syntax_error("expected ',' or '}' after object value, got: %s", state);
|
|
1947
|
+
}
|
|
1948
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1436
1952
|
}
|
|
1437
1953
|
|
|
1438
|
-
static void json_ensure_eof(JSON_ParserState *state)
|
|
1954
|
+
static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
1439
1955
|
{
|
|
1440
|
-
json_eat_whitespace(state);
|
|
1956
|
+
json_eat_whitespace(state, config, true);
|
|
1441
1957
|
if (!eos(state)) {
|
|
1442
|
-
|
|
1958
|
+
raise_syntax_error("unexpected token at end of stream %s", state);
|
|
1443
1959
|
}
|
|
1444
1960
|
}
|
|
1445
1961
|
|
|
@@ -1457,40 +1973,59 @@ static void json_ensure_eof(JSON_ParserState *state)
|
|
|
1457
1973
|
|
|
1458
1974
|
static VALUE convert_encoding(VALUE source)
|
|
1459
1975
|
{
|
|
1460
|
-
|
|
1976
|
+
StringValue(source);
|
|
1977
|
+
int encindex = RB_ENCODING_GET(source);
|
|
1461
1978
|
|
|
1462
|
-
|
|
1979
|
+
if (RB_LIKELY(encindex == utf8_encindex)) {
|
|
1980
|
+
return source;
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
if (encindex == binary_encindex) {
|
|
1984
|
+
// For historical reason, we silently reinterpret binary strings as UTF-8
|
|
1985
|
+
return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
|
|
1989
|
+
StringValue(source);
|
|
1463
1990
|
return source;
|
|
1464
|
-
|
|
1991
|
+
}
|
|
1465
1992
|
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1993
|
+
struct parser_config_init_args {
|
|
1994
|
+
JSON_ParserConfig *config;
|
|
1995
|
+
VALUE self;
|
|
1996
|
+
VALUE unknown_keywords;
|
|
1997
|
+
bool strict;
|
|
1998
|
+
};
|
|
1470
1999
|
|
|
1471
|
-
|
|
2000
|
+
static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
|
|
2001
|
+
{
|
|
2002
|
+
*dest = val;
|
|
2003
|
+
if (self) RB_OBJ_WRITTEN(self, Qundef, val);
|
|
1472
2004
|
}
|
|
1473
2005
|
|
|
1474
2006
|
static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
1475
2007
|
{
|
|
1476
|
-
|
|
2008
|
+
struct parser_config_init_args *args = (struct parser_config_init_args *)data;
|
|
2009
|
+
JSON_ParserConfig *config = args->config;
|
|
2010
|
+
VALUE self = args->self;
|
|
1477
2011
|
|
|
1478
2012
|
if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
|
|
1479
2013
|
else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
|
|
1480
2014
|
else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
|
|
2015
|
+
else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1481
2016
|
else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
|
|
1482
2017
|
else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
|
|
1483
2018
|
else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
|
|
1484
2019
|
else if (key == sym_freeze) { config->freeze = RTEST(val); }
|
|
1485
|
-
else if (key == sym_on_load) { config->on_load_proc
|
|
2020
|
+
else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
|
|
1486
2021
|
else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1487
2022
|
else if (key == sym_decimal_class) {
|
|
1488
2023
|
if (RTEST(val)) {
|
|
1489
2024
|
if (rb_respond_to(val, i_try_convert)) {
|
|
1490
|
-
config->decimal_class
|
|
2025
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1491
2026
|
config->decimal_method_id = i_try_convert;
|
|
1492
2027
|
} else if (rb_respond_to(val, i_new)) {
|
|
1493
|
-
config->decimal_class
|
|
2028
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1494
2029
|
config->decimal_method_id = i_new;
|
|
1495
2030
|
} else if (RB_TYPE_P(val, T_CLASS)) {
|
|
1496
2031
|
VALUE name = rb_class_name(val);
|
|
@@ -1499,7 +2034,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1499
2034
|
if (last_colon) {
|
|
1500
2035
|
const char *mod_path_end = last_colon - 1;
|
|
1501
2036
|
VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
|
|
1502
|
-
config->decimal_class
|
|
2037
|
+
parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
|
|
1503
2038
|
|
|
1504
2039
|
const char *method_name_beg = last_colon + 1;
|
|
1505
2040
|
long before_len = method_name_beg - name_cstr;
|
|
@@ -1507,28 +2042,48 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1507
2042
|
VALUE method_name = rb_str_substr(name, before_len, len);
|
|
1508
2043
|
config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
|
|
1509
2044
|
} else {
|
|
1510
|
-
config->decimal_class
|
|
2045
|
+
parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
|
|
1511
2046
|
config->decimal_method_id = SYM2ID(rb_str_intern(name));
|
|
1512
2047
|
}
|
|
1513
2048
|
}
|
|
1514
2049
|
}
|
|
1515
2050
|
}
|
|
2051
|
+
else if (args->strict) {
|
|
2052
|
+
if (!args->unknown_keywords) {
|
|
2053
|
+
args->unknown_keywords = rb_obj_hide(rb_ary_new());
|
|
2054
|
+
}
|
|
2055
|
+
rb_ary_push(args->unknown_keywords, key);
|
|
2056
|
+
}
|
|
1516
2057
|
|
|
1517
2058
|
return ST_CONTINUE;
|
|
1518
2059
|
}
|
|
1519
2060
|
|
|
1520
|
-
static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
2061
|
+
static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict)
|
|
1521
2062
|
{
|
|
1522
2063
|
config->max_nesting = 100;
|
|
1523
2064
|
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
2065
|
+
struct parser_config_init_args args = {
|
|
2066
|
+
.config = config,
|
|
2067
|
+
.self = self,
|
|
2068
|
+
.strict = strict,
|
|
2069
|
+
};
|
|
2070
|
+
|
|
2071
|
+
if (NIL_P(opts)) return;
|
|
2072
|
+
Check_Type(opts, T_HASH);
|
|
2073
|
+
if (RHASH_SIZE(opts) == 0) return;
|
|
2074
|
+
|
|
2075
|
+
// We assume in most cases few keys are set so it's faster to go over
|
|
2076
|
+
// the provided keys than to check all possible keys.
|
|
2077
|
+
rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
|
|
1531
2078
|
|
|
2079
|
+
if (RB_UNLIKELY(args.unknown_keywords)) {
|
|
2080
|
+
if (RARRAY_LEN(args.unknown_keywords) == 1) {
|
|
2081
|
+
rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0));
|
|
2082
|
+
}
|
|
2083
|
+
else {
|
|
2084
|
+
VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", "));
|
|
2085
|
+
rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords);
|
|
2086
|
+
}
|
|
1532
2087
|
}
|
|
1533
2088
|
}
|
|
1534
2089
|
|
|
@@ -1537,69 +2092,90 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
|
1537
2092
|
*
|
|
1538
2093
|
* Creates a new JSON::Ext::ParserConfig instance.
|
|
1539
2094
|
*
|
|
1540
|
-
*
|
|
1541
|
-
*
|
|
2095
|
+
* Argument +opts+, if given, contains a \Hash of options for the parsing.
|
|
2096
|
+
* See {Parsing Options}[#module-JSON-label-Parsing+Options].
|
|
1542
2097
|
*
|
|
1543
|
-
* _opts_ can have the following keys:
|
|
1544
|
-
* * *max_nesting*: The maximum depth of nesting allowed in the parsed data
|
|
1545
|
-
* structures. Disable depth checking with :max_nesting => false|nil|0, it
|
|
1546
|
-
* defaults to 100.
|
|
1547
|
-
* * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
|
|
1548
|
-
* defiance of RFC 4627 to be parsed by the Parser. This option defaults to
|
|
1549
|
-
* false.
|
|
1550
|
-
* * *symbolize_names*: If set to true, returns symbols for the names
|
|
1551
|
-
* (keys) in a JSON object. Otherwise strings are returned, which is
|
|
1552
|
-
* also the default. It's not possible to use this option in
|
|
1553
|
-
* conjunction with the *create_additions* option.
|
|
1554
|
-
* * *decimal_class*: Specifies which class to use instead of the default
|
|
1555
|
-
* (Float) when parsing decimal numbers. This class must accept a single
|
|
1556
|
-
* string argument in its constructor.
|
|
1557
2098
|
*/
|
|
1558
2099
|
static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
|
|
1559
2100
|
{
|
|
1560
2101
|
rb_check_frozen(self);
|
|
1561
2102
|
GET_PARSER_CONFIG;
|
|
1562
2103
|
|
|
1563
|
-
parser_config_init(config, opts);
|
|
1564
|
-
|
|
1565
|
-
RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
|
|
2104
|
+
parser_config_init(config, opts, self, false);
|
|
1566
2105
|
|
|
1567
2106
|
return self;
|
|
1568
2107
|
}
|
|
1569
2108
|
|
|
1570
|
-
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE
|
|
2109
|
+
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
|
|
1571
2110
|
{
|
|
1572
|
-
Vsource = convert_encoding(
|
|
1573
|
-
|
|
2111
|
+
VALUE Vsource = convert_encoding(src);
|
|
2112
|
+
|
|
2113
|
+
// Ensure the string isn't mutated under us.
|
|
2114
|
+
// The classic API to use is `rb_str_locktmp`, but then we'd
|
|
2115
|
+
// need to use `rb_protect` to make sure we always unlock.
|
|
2116
|
+
if (Vsource == src) {
|
|
2117
|
+
Vsource = rb_str_new_frozen(Vsource);
|
|
2118
|
+
}
|
|
1574
2119
|
|
|
1575
2120
|
VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
|
|
1576
|
-
rvalue_stack
|
|
2121
|
+
rvalue_stack value_stack = {
|
|
1577
2122
|
.type = RVALUE_STACK_STACK_ALLOCATED,
|
|
1578
2123
|
.ptr = rvalue_stack_buffer,
|
|
1579
2124
|
.capa = RVALUE_STACK_INITIAL_CAPA,
|
|
1580
2125
|
};
|
|
1581
2126
|
|
|
2127
|
+
// Seed the frame stack with the root frame, establishing the invariant that
|
|
2128
|
+
// json_parse_any always has a top frame to dispatch on (so the stack is never
|
|
2129
|
+
// empty mid-parse).
|
|
2130
|
+
json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA];
|
|
2131
|
+
frame_stack_buffer[0] = (json_frame){
|
|
2132
|
+
.type = JSON_FRAME_ROOT,
|
|
2133
|
+
.phase = JSON_PHASE_VALUE,
|
|
2134
|
+
};
|
|
2135
|
+
json_frame_stack frames = {
|
|
2136
|
+
.type = RVALUE_STACK_STACK_ALLOCATED,
|
|
2137
|
+
.ptr = frame_stack_buffer,
|
|
2138
|
+
.capa = JSON_FRAME_STACK_INITIAL_CAPA,
|
|
2139
|
+
.head = 1,
|
|
2140
|
+
};
|
|
2141
|
+
|
|
1582
2142
|
long len;
|
|
1583
2143
|
const char *start;
|
|
2144
|
+
|
|
1584
2145
|
RSTRING_GETMEM(Vsource, start, len);
|
|
1585
2146
|
|
|
1586
|
-
VALUE
|
|
2147
|
+
VALUE value_stack_handle = 0;
|
|
2148
|
+
VALUE frame_stack_handle = 0;
|
|
1587
2149
|
JSON_ParserState _state = {
|
|
1588
2150
|
.start = start,
|
|
1589
2151
|
.cursor = start,
|
|
1590
2152
|
.end = start + len,
|
|
1591
|
-
.
|
|
1592
|
-
.
|
|
2153
|
+
.value_stack = &value_stack,
|
|
2154
|
+
.value_stack_handle = &value_stack_handle,
|
|
2155
|
+
.frames = &frames,
|
|
2156
|
+
.frame_stack_handle = &frame_stack_handle,
|
|
1593
2157
|
};
|
|
1594
2158
|
JSON_ParserState *state = &_state;
|
|
1595
2159
|
|
|
1596
|
-
|
|
2160
|
+
bool complete = json_parse_any(state, config, false);
|
|
2161
|
+
|
|
2162
|
+
// The root document value is parsed; it is the lone survivor on
|
|
2163
|
+
// the rvalue stack.
|
|
2164
|
+
VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
|
|
1597
2165
|
|
|
1598
2166
|
// This may be skipped in case of exception, but
|
|
1599
2167
|
// it won't cause a leak.
|
|
1600
|
-
rvalue_stack_eagerly_release(
|
|
1601
|
-
|
|
1602
|
-
|
|
2168
|
+
rvalue_stack_eagerly_release(value_stack_handle);
|
|
2169
|
+
json_frame_stack_eagerly_release(frame_stack_handle);
|
|
2170
|
+
RB_GC_GUARD(value_stack_handle);
|
|
2171
|
+
RB_GC_GUARD(frame_stack_handle);
|
|
2172
|
+
RB_GC_GUARD(Vsource);
|
|
2173
|
+
|
|
2174
|
+
if (complete) {
|
|
2175
|
+
json_ensure_eof(state, config);
|
|
2176
|
+
} else {
|
|
2177
|
+
raise_eos_error("unexpected end of input", state);
|
|
2178
|
+
}
|
|
1603
2179
|
|
|
1604
2180
|
return result;
|
|
1605
2181
|
}
|
|
@@ -1619,12 +2195,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
|
|
|
1619
2195
|
|
|
1620
2196
|
static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
1621
2197
|
{
|
|
1622
|
-
Vsource = convert_encoding(StringValue(Vsource));
|
|
1623
|
-
StringValue(Vsource);
|
|
1624
|
-
|
|
1625
2198
|
JSON_ParserConfig _config = {0};
|
|
1626
2199
|
JSON_ParserConfig *config = &_config;
|
|
1627
|
-
parser_config_init(config, opts);
|
|
2200
|
+
parser_config_init(config, opts, Qfalse, false);
|
|
1628
2201
|
|
|
1629
2202
|
return cParser_parse(config, Vsource);
|
|
1630
2203
|
}
|
|
@@ -1632,23 +2205,35 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
|
1632
2205
|
static void JSON_ParserConfig_mark(void *ptr)
|
|
1633
2206
|
{
|
|
1634
2207
|
JSON_ParserConfig *config = ptr;
|
|
1635
|
-
|
|
1636
|
-
|
|
2208
|
+
rb_gc_mark_movable(config->on_load_proc);
|
|
2209
|
+
rb_gc_mark_movable(config->decimal_class);
|
|
1637
2210
|
}
|
|
1638
2211
|
|
|
1639
2212
|
static size_t JSON_ParserConfig_memsize(const void *ptr)
|
|
1640
2213
|
{
|
|
2214
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
2215
|
+
return 0;
|
|
2216
|
+
#else
|
|
1641
2217
|
return sizeof(JSON_ParserConfig);
|
|
2218
|
+
#endif
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
static void JSON_ParserConfig_compact(void *ptr)
|
|
2222
|
+
{
|
|
2223
|
+
JSON_ParserConfig *config = ptr;
|
|
2224
|
+
config->on_load_proc = rb_gc_location(config->on_load_proc);
|
|
2225
|
+
config->decimal_class = rb_gc_location(config->decimal_class);
|
|
1642
2226
|
}
|
|
1643
2227
|
|
|
1644
2228
|
static const rb_data_type_t JSON_ParserConfig_type = {
|
|
1645
2229
|
.wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
|
|
1646
2230
|
.function = {
|
|
1647
|
-
JSON_ParserConfig_mark,
|
|
1648
|
-
RUBY_DEFAULT_FREE,
|
|
1649
|
-
JSON_ParserConfig_memsize,
|
|
2231
|
+
.dmark = JSON_ParserConfig_mark,
|
|
2232
|
+
.dfree = RUBY_DEFAULT_FREE,
|
|
2233
|
+
.dsize = JSON_ParserConfig_memsize,
|
|
2234
|
+
.dcompact = JSON_ParserConfig_compact,
|
|
1650
2235
|
},
|
|
1651
|
-
.flags =
|
|
2236
|
+
.flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
|
|
1652
2237
|
};
|
|
1653
2238
|
|
|
1654
2239
|
static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
@@ -1657,6 +2242,603 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
|
1657
2242
|
return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config);
|
|
1658
2243
|
}
|
|
1659
2244
|
|
|
2245
|
+
static void json_str_clear(VALUE str)
|
|
2246
|
+
{
|
|
2247
|
+
if (RB_OBJ_FROZEN_RAW(str)) {
|
|
2248
|
+
return;
|
|
2249
|
+
}
|
|
2250
|
+
rb_str_replace(str, JSON_empty_string);
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
typedef struct JSON_ResumableParserStruct {
|
|
2254
|
+
JSON_ParserConfig config;
|
|
2255
|
+
JSON_ParserState state;
|
|
2256
|
+
rvalue_stack value_stack;
|
|
2257
|
+
json_frame_stack frames;
|
|
2258
|
+
VALUE buffer;
|
|
2259
|
+
size_t parsed_bytes;
|
|
2260
|
+
size_t incomplete_bytes;
|
|
2261
|
+
bool complete;
|
|
2262
|
+
bool in_use;
|
|
2263
|
+
} JSON_ResumableParser;
|
|
2264
|
+
|
|
2265
|
+
static void JSON_ResumableParser_mark(void *ptr)
|
|
2266
|
+
{
|
|
2267
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2268
|
+
JSON_ParserConfig_mark(&parser->config);
|
|
2269
|
+
rvalue_stack_mark(&parser->value_stack);
|
|
2270
|
+
rvalue_cache_mark(&parser->state.name_cache);
|
|
2271
|
+
rb_gc_mark(parser->buffer); // pin the buffer
|
|
2272
|
+
rb_gc_mark_movable(parser->state.parser);
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
static void JSON_ResumableParser_free(void *ptr)
|
|
2276
|
+
{
|
|
2277
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2278
|
+
rvalue_stack_free_buffer(&parser->value_stack);
|
|
2279
|
+
json_frame_stack_free_buffer(&parser->frames);
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
static size_t JSON_ResumableParser_memsize(const void *ptr)
|
|
2283
|
+
{
|
|
2284
|
+
const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr;
|
|
2285
|
+
size_t memsize = JSON_ParserConfig_memsize(&parser->config);
|
|
2286
|
+
memsize += rvalue_stack_memsize(&parser->value_stack);
|
|
2287
|
+
memsize += json_frame_stack_memsize(&parser->frames);
|
|
2288
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
2289
|
+
memsize += (
|
|
2290
|
+
sizeof(JSON_ResumableParser)
|
|
2291
|
+
- sizeof(JSON_ParserState)
|
|
2292
|
+
- sizeof(JSON_ParserConfig)
|
|
2293
|
+
- sizeof(rvalue_stack)
|
|
2294
|
+
- sizeof(json_frame_stack)
|
|
2295
|
+
);
|
|
2296
|
+
#endif
|
|
2297
|
+
return memsize;
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2300
|
+
static void JSON_ResumableParser_compact(void *ptr)
|
|
2301
|
+
{
|
|
2302
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2303
|
+
JSON_ParserConfig_compact(&parser->config);
|
|
2304
|
+
rvalue_stack_compact(&parser->value_stack);
|
|
2305
|
+
rvalue_cache_compact(&parser->state.name_cache);
|
|
2306
|
+
parser->buffer = rb_gc_location(parser->buffer);
|
|
2307
|
+
parser->state.parser = rb_gc_location(parser->state.parser);
|
|
2308
|
+
}
|
|
2309
|
+
|
|
2310
|
+
static const rb_data_type_t JSON_ResumableParser_type = {
|
|
2311
|
+
.wrap_struct_name = "JSON::Ext::ResumableParser",
|
|
2312
|
+
.function = {
|
|
2313
|
+
JSON_ResumableParser_mark,
|
|
2314
|
+
JSON_ResumableParser_free,
|
|
2315
|
+
JSON_ResumableParser_memsize,
|
|
2316
|
+
JSON_ResumableParser_compact,
|
|
2317
|
+
},
|
|
2318
|
+
// RUBY_TYPED_WB_PROTECTED is deliberately not declared because
|
|
2319
|
+
// this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
|
|
2320
|
+
// to trigger a lot of write barriers.
|
|
2321
|
+
.flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
|
|
2322
|
+
};
|
|
2323
|
+
|
|
2324
|
+
static VALUE cResumableParser_allocate(VALUE klass)
|
|
2325
|
+
{
|
|
2326
|
+
JSON_ResumableParser *parser;
|
|
2327
|
+
VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
|
|
2328
|
+
parser->state.in_array++;
|
|
2329
|
+
parser->state.parser = obj;
|
|
2330
|
+
return obj;
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
static inline JSON_ResumableParser *cResumableParser_get(VALUE self)
|
|
2334
|
+
{
|
|
2335
|
+
JSON_ResumableParser *parser;
|
|
2336
|
+
TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
|
|
2337
|
+
return parser;
|
|
2338
|
+
}
|
|
2339
|
+
|
|
2340
|
+
/*
|
|
2341
|
+
* call-seq: new(opts => {})
|
|
2342
|
+
*
|
|
2343
|
+
* Creates a new JSON::ResumableParser instance.
|
|
2344
|
+
*
|
|
2345
|
+
* Argument +opts+, if given, contains a \Hash of options for the parsing.
|
|
2346
|
+
* See {Parsing Options}[#module-JSON-label-Parsing+Options].
|
|
2347
|
+
*
|
|
2348
|
+
* A ResumableParser is able to parse partial documents and resume parsing later
|
|
2349
|
+
* when more of the document is provided:
|
|
2350
|
+
*
|
|
2351
|
+
* parser = JSON::ResumableParser.new
|
|
2352
|
+
* parser << '{"user": "george", "role": "ad'
|
|
2353
|
+
* parser.parse # => false
|
|
2354
|
+
* parser.eos? # => true
|
|
2355
|
+
* parser.partial_value # => { "user" => "george", "role" => nil }
|
|
2356
|
+
* parser.rest # => '"ad'
|
|
2357
|
+
*
|
|
2358
|
+
* parser << 'min" }[1, 2, 3]'
|
|
2359
|
+
* parser.parse # => true
|
|
2360
|
+
* parser.value # => { "user" => "george", "role" => "admin" }
|
|
2361
|
+
*
|
|
2362
|
+
* parser.parse # => true
|
|
2363
|
+
* parser.value # => [1, 2, 3]
|
|
2364
|
+
*
|
|
2365
|
+
* === Limitations
|
|
2366
|
+
*
|
|
2367
|
+
* While ResumableParser is able to parse streams of documents without any
|
|
2368
|
+
* explicit separators between them, it is highly recommended to separate documents
|
|
2369
|
+
* by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous.
|
|
2370
|
+
* When parsing a number, ResumableParser will not consider the number complete until something follows:
|
|
2371
|
+
*
|
|
2372
|
+
* parser << '123'
|
|
2373
|
+
* parser.parse # => false
|
|
2374
|
+
* parser << ' '
|
|
2375
|
+
* parser.parse # => true
|
|
2376
|
+
* parser.value # => 123
|
|
2377
|
+
*
|
|
2378
|
+
* === Security
|
|
2379
|
+
*
|
|
2380
|
+
* An incomplete document is buffered in full and there is no size limit, so when reading
|
|
2381
|
+
* from an untrusted source the caller is responsible for bounding how much data is fed.
|
|
2382
|
+
* For example:
|
|
2383
|
+
*
|
|
2384
|
+
* loop do
|
|
2385
|
+
* if parser.parsed_bytes > DOCUMENT_MAX_SIZE
|
|
2386
|
+
* raise "document too large"
|
|
2387
|
+
* end
|
|
2388
|
+
*
|
|
2389
|
+
* parser << read_chunk
|
|
2390
|
+
* while parser.parse
|
|
2391
|
+
* process(parser.value)
|
|
2392
|
+
* end
|
|
2393
|
+
* end
|
|
2394
|
+
*/
|
|
2395
|
+
static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
|
|
2396
|
+
{
|
|
2397
|
+
rb_check_frozen(self);
|
|
2398
|
+
|
|
2399
|
+
VALUE opts = Qfalse;
|
|
2400
|
+
rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts);
|
|
2401
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2402
|
+
|
|
2403
|
+
opts = argc > 0 ? argv[0] : Qnil;
|
|
2404
|
+
parser_config_init(&parser->config, opts, self, true);
|
|
2405
|
+
|
|
2406
|
+
return self;
|
|
2407
|
+
}
|
|
2408
|
+
|
|
2409
|
+
static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock);
|
|
2410
|
+
|
|
2411
|
+
/*
|
|
2412
|
+
* call-seq: self << string -> self
|
|
2413
|
+
*
|
|
2414
|
+
* Appends the given string to the parser's buffer.
|
|
2415
|
+
*/
|
|
2416
|
+
static VALUE cResumableParser_feed(VALUE self, VALUE str)
|
|
2417
|
+
{
|
|
2418
|
+
rb_check_frozen(self);
|
|
2419
|
+
|
|
2420
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2421
|
+
|
|
2422
|
+
str = convert_encoding(str);
|
|
2423
|
+
if (!RSTRING_LEN(str)) {
|
|
2424
|
+
return self;
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
size_t offset = parser->state.cursor - parser->state.start;
|
|
2428
|
+
const size_t remaining = parser->state.end - parser->state.cursor;
|
|
2429
|
+
|
|
2430
|
+
if (!remaining) {
|
|
2431
|
+
if (parser->buffer) {
|
|
2432
|
+
json_str_clear(parser->buffer);
|
|
2433
|
+
}
|
|
2434
|
+
parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
|
|
2435
|
+
offset = 0;
|
|
2436
|
+
} else {
|
|
2437
|
+
JSON_ASSERT(parser->buffer);
|
|
2438
|
+
|
|
2439
|
+
const size_t size = parser->state.end - parser->state.start;
|
|
2440
|
+
const size_t consumed = size - remaining;
|
|
2441
|
+
|
|
2442
|
+
if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
|
|
2443
|
+
VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
|
|
2444
|
+
rb_enc_associate_index(new_buffer, utf8_encindex);
|
|
2445
|
+
|
|
2446
|
+
char *old_ptr = RSTRING_PTR(parser->buffer);
|
|
2447
|
+
memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
|
|
2448
|
+
rb_str_set_len(new_buffer, remaining);
|
|
2449
|
+
offset = 0;
|
|
2450
|
+
parser->buffer = new_buffer;
|
|
2451
|
+
} else if (consumed > (size / 2) && size >= 512) {
|
|
2452
|
+
rb_str_modify(parser->buffer);
|
|
2453
|
+
char *old_ptr = RSTRING_PTR(parser->buffer);
|
|
2454
|
+
memmove(old_ptr, old_ptr + consumed, remaining);
|
|
2455
|
+
rb_str_set_len(parser->buffer, remaining);
|
|
2456
|
+
offset = 0;
|
|
2457
|
+
}
|
|
2458
|
+
rb_str_append(parser->buffer, str);
|
|
2459
|
+
}
|
|
2460
|
+
|
|
2461
|
+
long len;
|
|
2462
|
+
const char *start;
|
|
2463
|
+
RSTRING_GETMEM(parser->buffer, start, len);
|
|
2464
|
+
parser->state.start = start;
|
|
2465
|
+
parser->state.end = start + len;
|
|
2466
|
+
parser->state.cursor = parser->state.start + offset;
|
|
2467
|
+
|
|
2468
|
+
return self;
|
|
2469
|
+
}
|
|
2470
|
+
|
|
2471
|
+
struct json_parse_any_args {
|
|
2472
|
+
JSON_ParserState *state;
|
|
2473
|
+
JSON_ParserConfig *config;
|
|
2474
|
+
VALUE parser;
|
|
2475
|
+
};
|
|
2476
|
+
|
|
2477
|
+
static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args))
|
|
2478
|
+
{
|
|
2479
|
+
struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
|
|
2480
|
+
return (VALUE)json_parse_any(args->state, args->config, true);
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
static VALUE json_parse_any_resumable_safe(VALUE _args)
|
|
2484
|
+
{
|
|
2485
|
+
struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
|
|
2486
|
+
VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args);
|
|
2487
|
+
return result == args->parser ? Qfalse : result;
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2490
|
+
static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock)
|
|
2491
|
+
{
|
|
2492
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2493
|
+
|
|
2494
|
+
if (parser->in_use) {
|
|
2495
|
+
rb_raise(rb_eArgError, "ResumableParser can't be used recursively");
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
if (lock) {
|
|
2499
|
+
parser->in_use = true;
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
// self may have moved, so we need to update all pointers
|
|
2503
|
+
// Investigate: We might be better off keeping JSON_ParserState on the stack
|
|
2504
|
+
// and only persist what we need.
|
|
2505
|
+
parser->state.value_stack = &parser->value_stack;
|
|
2506
|
+
parser->state.frames = &parser->frames;
|
|
2507
|
+
|
|
2508
|
+
return parser;
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
/*
|
|
2512
|
+
* call-seq: parse -> true or false
|
|
2513
|
+
*
|
|
2514
|
+
* Attemps to parse a JSON document from the internal buffer.
|
|
2515
|
+
* Returns whether a complete document could be parsed.
|
|
2516
|
+
*
|
|
2517
|
+
* It does raise +JSON::ParserError+ when encountering invalid \JSON syntax.
|
|
2518
|
+
*
|
|
2519
|
+
* The parsed object can be retrieved by calling #value
|
|
2520
|
+
*/
|
|
2521
|
+
static VALUE cResumableParser_parse(VALUE self)
|
|
2522
|
+
{
|
|
2523
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
|
|
2524
|
+
|
|
2525
|
+
if (parser->complete) {
|
|
2526
|
+
parser->parsed_bytes = 0;
|
|
2527
|
+
parser->incomplete_bytes = 0;
|
|
2528
|
+
parser->complete = false;
|
|
2529
|
+
}
|
|
2530
|
+
|
|
2531
|
+
if (!parser->buffer) {
|
|
2532
|
+
parser->in_use = false;
|
|
2533
|
+
return Qfalse;
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
if (parser->frames.head == 0) {
|
|
2537
|
+
json_frame_stack_push(&parser->state, (json_frame){
|
|
2538
|
+
.type = JSON_FRAME_ROOT,
|
|
2539
|
+
.phase = JSON_PHASE_VALUE,
|
|
2540
|
+
});
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
VALUE Vsource = parser->buffer; // Prevent compaction
|
|
2544
|
+
|
|
2545
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2546
|
+
|
|
2547
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2548
|
+
JSON_ASSERT(parser->value_stack.head == 1);
|
|
2549
|
+
JSON_ASSERT(parser->frames.head == 1);
|
|
2550
|
+
|
|
2551
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
2552
|
+
rvalue_stack_pop(parser->state.value_stack, 1);
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2555
|
+
struct json_parse_any_args args = {
|
|
2556
|
+
.state = &parser->state,
|
|
2557
|
+
.config = &parser->config,
|
|
2558
|
+
.parser = self,
|
|
2559
|
+
};
|
|
2560
|
+
int status;
|
|
2561
|
+
const char *initial_cursor = parser->state.cursor;
|
|
2562
|
+
parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
|
|
2563
|
+
|
|
2564
|
+
if (status) {
|
|
2565
|
+
parser->complete = true; // a parse error is considered complete
|
|
2566
|
+
}
|
|
2567
|
+
|
|
2568
|
+
parser->parsed_bytes += parser->state.cursor - initial_cursor;
|
|
2569
|
+
parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;
|
|
2570
|
+
|
|
2571
|
+
json_eat_whitespace(&parser->state, &parser->config, false);
|
|
2572
|
+
if (eos(&parser->state)) {
|
|
2573
|
+
json_str_clear(parser->buffer);
|
|
2574
|
+
parser->buffer = Qfalse;
|
|
2575
|
+
parser->state.start = parser->state.cursor = parser->state.end = 0;
|
|
2576
|
+
}
|
|
2577
|
+
parser->in_use = false;
|
|
2578
|
+
|
|
2579
|
+
if (status) {
|
|
2580
|
+
rb_jump_tag(status); // reraise
|
|
2581
|
+
}
|
|
2582
|
+
RB_GC_GUARD(Vsource);
|
|
2583
|
+
return parser->complete ? Qtrue : Qfalse;
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
/*
|
|
2587
|
+
* call-seq: value? -> true or false
|
|
2588
|
+
*
|
|
2589
|
+
* Returns whether a parsed value is available.
|
|
2590
|
+
*/
|
|
2591
|
+
static VALUE cResumableParser_value_p(VALUE self)
|
|
2592
|
+
{
|
|
2593
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2594
|
+
|
|
2595
|
+
if (parser->value_stack.head > 0) {
|
|
2596
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2597
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2598
|
+
return Qtrue;
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
return Qfalse;
|
|
2602
|
+
}
|
|
2603
|
+
|
|
2604
|
+
/*
|
|
2605
|
+
* call-seq: value -> object
|
|
2606
|
+
*
|
|
2607
|
+
* Returns and consume the last parsed value.
|
|
2608
|
+
* Raises ArgumentError if there is no parsed value or if it was already retrieved:
|
|
2609
|
+
* parser << '[1][2]'
|
|
2610
|
+
* parser.value # ArgumentError no ready value
|
|
2611
|
+
* parser.parse # => true
|
|
2612
|
+
* parser.value # => [1]
|
|
2613
|
+
* parser.value # ArgumentError no ready value
|
|
2614
|
+
*/
|
|
2615
|
+
static VALUE cResumableParser_value(VALUE self)
|
|
2616
|
+
{
|
|
2617
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2618
|
+
|
|
2619
|
+
if (parser->frames.head > 0) {
|
|
2620
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2621
|
+
|
|
2622
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2623
|
+
VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
|
|
2624
|
+
rvalue_stack_pop(parser->state.value_stack, 1);
|
|
2625
|
+
json_frame_stack_pop(parser->state.frames);
|
|
2626
|
+
return result;
|
|
2627
|
+
}
|
|
2628
|
+
}
|
|
2629
|
+
rb_raise(rb_eArgError, "no ready value");
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
/*
|
|
2633
|
+
* call-seq: clear -> self
|
|
2634
|
+
*
|
|
2635
|
+
* Entirely reset the parser state and buffer.
|
|
2636
|
+
*/
|
|
2637
|
+
static VALUE cResumableParser_clear(VALUE self)
|
|
2638
|
+
{
|
|
2639
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2640
|
+
parser->buffer = 0;
|
|
2641
|
+
parser->complete = true;
|
|
2642
|
+
parser->parsed_bytes = 0;
|
|
2643
|
+
parser->incomplete_bytes = 0;
|
|
2644
|
+
parser->frames.head = 0;
|
|
2645
|
+
parser->value_stack.head = 0;
|
|
2646
|
+
parser->state.name_cache.length = 0;
|
|
2647
|
+
parser->state.current_nesting = 0;
|
|
2648
|
+
parser->state.in_array = 1;
|
|
2649
|
+
parser->state.emitted_deprecations = 0;
|
|
2650
|
+
parser->state.start = parser->state.cursor = parser->state.end = NULL;
|
|
2651
|
+
return self;
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
static VALUE cResumableParser_partial_value_body(VALUE self)
|
|
2655
|
+
{
|
|
2656
|
+
JSON_ResumableParser *original_parser = cResumableParser_get(self);
|
|
2657
|
+
JSON_ResumableParser parser = *original_parser;
|
|
2658
|
+
|
|
2659
|
+
parser.state.frames = &parser.frames;
|
|
2660
|
+
parser.state.value_stack = &parser.value_stack;
|
|
2661
|
+
|
|
2662
|
+
if (parser.value_stack.head == 0) {
|
|
2663
|
+
return Qnil;
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
json_frame *frame = json_frame_stack_peek(parser.state.frames);
|
|
2667
|
+
long missing_object_value = 0;
|
|
2668
|
+
if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
|
|
2669
|
+
missing_object_value = 1;
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
// Copy the value stack as we need to mutate it. The collapse loop folds each
|
|
2673
|
+
// open container by popping its entries and pushing the single result, so a
|
|
2674
|
+
// parent always reclaims its child's slot; head exceeds its live size by at
|
|
2675
|
+
// most one, either for the missing-value placeholder pushed below or for the
|
|
2676
|
+
// result of folding an empty innermost container. That one spare slot keeps
|
|
2677
|
+
// rvalue_stack_push from growing (reallocating) this ALLOCV buffer.
|
|
2678
|
+
long capa = parser.value_stack.head;
|
|
2679
|
+
parser.value_stack.capa = capa + 1;
|
|
2680
|
+
VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, parser.value_stack.capa);
|
|
2681
|
+
MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, capa);
|
|
2682
|
+
parser.value_stack.ptr = value_stack_buffer;
|
|
2683
|
+
|
|
2684
|
+
JSON_ParserState *state = &parser.state;
|
|
2685
|
+
JSON_ParserConfig *config = &parser.config;
|
|
2686
|
+
|
|
2687
|
+
if (missing_object_value) {
|
|
2688
|
+
rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
VALUE partial_result = Qundef;
|
|
2692
|
+
|
|
2693
|
+
while (UNDEF_P(partial_result)) {
|
|
2694
|
+
frame = json_frame_stack_peek(state->frames);
|
|
2695
|
+
|
|
2696
|
+
switch (frame->type) {
|
|
2697
|
+
case JSON_FRAME_ROOT: {
|
|
2698
|
+
partial_result = *rvalue_stack_peek(state->value_stack, 1);
|
|
2699
|
+
break;
|
|
2700
|
+
}
|
|
2701
|
+
|
|
2702
|
+
case JSON_FRAME_ARRAY: {
|
|
2703
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
2704
|
+
json_push_value(state, config, json_decode_array(state, config, count));
|
|
2705
|
+
json_frame_stack_pop(state->frames);
|
|
2706
|
+
|
|
2707
|
+
break;
|
|
2708
|
+
}
|
|
2709
|
+
|
|
2710
|
+
case JSON_FRAME_OBJECT: {
|
|
2711
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
2712
|
+
json_push_value(state, config, json_decode_object(state, config, count));
|
|
2713
|
+
json_frame_stack_pop(state->frames);
|
|
2714
|
+
break;
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2717
|
+
default: {
|
|
2718
|
+
JSON_UNREACHABLE_RETURN(Qundef);
|
|
2719
|
+
break;
|
|
2720
|
+
}
|
|
2721
|
+
}
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
ALLOCV_END(tmpbuf);
|
|
2725
|
+
return partial_result;
|
|
2726
|
+
}
|
|
2727
|
+
|
|
2728
|
+
/*
|
|
2729
|
+
* call-seq: partial_value -> object
|
|
2730
|
+
*
|
|
2731
|
+
* Returns the Ruby objects parsed up to this point:
|
|
2732
|
+
* parser << '[1, [2, 3,'
|
|
2733
|
+
* parser.parse # => false
|
|
2734
|
+
* parser.value # ArgumentError no ready value
|
|
2735
|
+
* parser.partial_value # => [1, [2, 3]]
|
|
2736
|
+
*/
|
|
2737
|
+
static VALUE cResumableParser_partial_value(VALUE self)
|
|
2738
|
+
{
|
|
2739
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
|
|
2740
|
+
|
|
2741
|
+
int status;
|
|
2742
|
+
VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
|
|
2743
|
+
parser->in_use = false;
|
|
2744
|
+
if (status) {
|
|
2745
|
+
rb_jump_tag(status);
|
|
2746
|
+
}
|
|
2747
|
+
return result;
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
/*
|
|
2751
|
+
* call-seq: rest -> string
|
|
2752
|
+
*
|
|
2753
|
+
* Returns a string containing what remains to be parsed in the buffer
|
|
2754
|
+
* parser << '{ "message": "unterminated message'
|
|
2755
|
+
* parser.parse # => false
|
|
2756
|
+
* parser.rest # => '"unterminated message"'
|
|
2757
|
+
*/
|
|
2758
|
+
static VALUE cResumableParser_rest(VALUE self)
|
|
2759
|
+
{
|
|
2760
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2761
|
+
|
|
2762
|
+
if (!parser->buffer) {
|
|
2763
|
+
return rb_utf8_str_new("", 0);
|
|
2764
|
+
}
|
|
2765
|
+
|
|
2766
|
+
size_t offset = parser->state.cursor - parser->state.start;
|
|
2767
|
+
const char *ptr;
|
|
2768
|
+
long len;
|
|
2769
|
+
RSTRING_GETMEM(parser->buffer, ptr, len);
|
|
2770
|
+
return rb_utf8_str_new(ptr + offset, len - offset);
|
|
2771
|
+
}
|
|
2772
|
+
|
|
2773
|
+
/*
|
|
2774
|
+
* call-seq: eos? -> true or false
|
|
2775
|
+
*
|
|
2776
|
+
* Returns whether the internal buffer has been entirely consumed.
|
|
2777
|
+
*/
|
|
2778
|
+
static VALUE cResumableParser_eos_p(VALUE self)
|
|
2779
|
+
{
|
|
2780
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2781
|
+
return eos(&parser->state) ? Qtrue : Qfalse;
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
/*
|
|
2785
|
+
* call-seq: partial_value? -> true or false
|
|
2786
|
+
*
|
|
2787
|
+
* Returns whether a document is currently under construction: an unclosed
|
|
2788
|
+
* container, a key awaiting its value, etc.
|
|
2789
|
+
*
|
|
2790
|
+
* It answers the same question as <tt>!partial_value.nil?</tt>, but as a
|
|
2791
|
+
* cheap predicate on the parser's internal state, without materializing the
|
|
2792
|
+
* partially parsed Ruby objects:
|
|
2793
|
+
* parser << '{"a":1,'
|
|
2794
|
+
* parser.parse # => false
|
|
2795
|
+
* parser.partial_value? # => true
|
|
2796
|
+
*
|
|
2797
|
+
* A fully parsed document whose value hasn't been retrieved yet is not under
|
|
2798
|
+
* construction: #value? returns true and #partial_value? returns false.
|
|
2799
|
+
*/
|
|
2800
|
+
static VALUE cResumableParser_partial_value_p(VALUE self)
|
|
2801
|
+
{
|
|
2802
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2803
|
+
|
|
2804
|
+
// Mirror of #value?: values on the stack while the document isn't DONE
|
|
2805
|
+
// belong to a partially built document. A container whose first key or
|
|
2806
|
+
// element hasn't been parsed yet has no frame nor value registered (the
|
|
2807
|
+
// tokenizer rewinds to the container start on EOS), so that state is
|
|
2808
|
+
// observable through the buffer (#eos?/#rest) instead, keeping this
|
|
2809
|
+
// predicate consistent with #partial_value returning nil.
|
|
2810
|
+
if (parser->value_stack.head > 0) {
|
|
2811
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2812
|
+
if (frame->phase != JSON_PHASE_DONE) {
|
|
2813
|
+
return Qtrue;
|
|
2814
|
+
}
|
|
2815
|
+
}
|
|
2816
|
+
return Qfalse;
|
|
2817
|
+
}
|
|
2818
|
+
|
|
2819
|
+
/*
|
|
2820
|
+
* call-seq: parsed_bytes -> integer
|
|
2821
|
+
*
|
|
2822
|
+
* Returns the number of bytes parsed since the start of the current partial value.
|
|
2823
|
+
* This is intended to be used for securing against untrusted input:
|
|
2824
|
+
*
|
|
2825
|
+
* loop do
|
|
2826
|
+
* if parser.parsed_bytes > DOCUMENT_MAX_SIZE
|
|
2827
|
+
* raise "document too large"
|
|
2828
|
+
* end
|
|
2829
|
+
*
|
|
2830
|
+
* parser << read_chunk
|
|
2831
|
+
* while parser.parse
|
|
2832
|
+
* process(parser.value)
|
|
2833
|
+
* end
|
|
2834
|
+
* end
|
|
2835
|
+
*/
|
|
2836
|
+
static VALUE cResumableParser_parsed_bytes(VALUE self)
|
|
2837
|
+
{
|
|
2838
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2839
|
+
return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
|
|
2840
|
+
}
|
|
2841
|
+
|
|
1660
2842
|
void Init_parser(void)
|
|
1661
2843
|
{
|
|
1662
2844
|
#ifdef HAVE_RB_EXT_RACTOR_SAFE
|
|
@@ -1668,30 +2850,53 @@ void Init_parser(void)
|
|
|
1668
2850
|
mJSON = rb_define_module("JSON");
|
|
1669
2851
|
VALUE mExt = rb_define_module_under(mJSON, "Ext");
|
|
1670
2852
|
VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject);
|
|
2853
|
+
|
|
2854
|
+
rb_global_variable(&eParserError);
|
|
2855
|
+
eParserError = rb_path2class("JSON::ParserError");
|
|
2856
|
+
|
|
2857
|
+
rb_global_variable(&eNestingError);
|
|
1671
2858
|
eNestingError = rb_path2class("JSON::NestingError");
|
|
1672
|
-
|
|
2859
|
+
|
|
1673
2860
|
rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate);
|
|
1674
|
-
|
|
2861
|
+
rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
|
|
1675
2862
|
rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1);
|
|
1676
2863
|
|
|
1677
2864
|
VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject);
|
|
1678
2865
|
rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2);
|
|
1679
2866
|
|
|
2867
|
+
VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject);
|
|
2868
|
+
rb_define_alloc_func(cResumableParser, cResumableParser_allocate);
|
|
2869
|
+
rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1);
|
|
2870
|
+
rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1);
|
|
2871
|
+
rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0);
|
|
2872
|
+
rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
|
|
2873
|
+
rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
|
|
2874
|
+
rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
|
|
2875
|
+
rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0);
|
|
2876
|
+
rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
|
|
2877
|
+
rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
|
|
2878
|
+
rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
|
|
2879
|
+
rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0);
|
|
2880
|
+
|
|
2881
|
+
rb_global_variable(&CNaN);
|
|
1680
2882
|
CNaN = rb_const_get(mJSON, rb_intern("NaN"));
|
|
1681
|
-
rb_gc_register_mark_object(CNaN);
|
|
1682
2883
|
|
|
2884
|
+
rb_global_variable(&CInfinity);
|
|
1683
2885
|
CInfinity = rb_const_get(mJSON, rb_intern("Infinity"));
|
|
1684
|
-
rb_gc_register_mark_object(CInfinity);
|
|
1685
2886
|
|
|
2887
|
+
rb_global_variable(&CMinusInfinity);
|
|
1686
2888
|
CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity"));
|
|
1687
|
-
rb_gc_register_mark_object(CMinusInfinity);
|
|
1688
2889
|
|
|
1689
2890
|
rb_global_variable(&Encoding_UTF_8);
|
|
1690
2891
|
Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
|
|
1691
2892
|
|
|
2893
|
+
rb_global_variable(&JSON_empty_string);
|
|
2894
|
+
JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0));
|
|
2895
|
+
|
|
1692
2896
|
sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
|
|
1693
2897
|
sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
|
|
1694
2898
|
sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
|
|
2899
|
+
sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
|
|
1695
2900
|
sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
|
|
1696
2901
|
sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
|
|
1697
2902
|
sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
|
|
@@ -1702,8 +2907,12 @@ void Init_parser(void)
|
|
|
1702
2907
|
|
|
1703
2908
|
i_new = rb_intern("new");
|
|
1704
2909
|
i_try_convert = rb_intern("try_convert");
|
|
2910
|
+
#ifndef HAVE_RB_STR_TO_INTERNED_STR
|
|
1705
2911
|
i_uminus = rb_intern("-@");
|
|
2912
|
+
#endif
|
|
1706
2913
|
i_encode = rb_intern("encode");
|
|
2914
|
+
i_at_line = rb_intern("@line");
|
|
2915
|
+
i_at_column = rb_intern("@column");
|
|
1707
2916
|
|
|
1708
2917
|
binary_encindex = rb_ascii8bit_encindex();
|
|
1709
2918
|
utf8_encindex = rb_utf8_encindex();
|