json 2.19.1 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +56 -0
- data/LEGAL +3 -3
- data/README.md +11 -2
- data/ext/json/ext/fbuffer/fbuffer.h +32 -24
- data/ext/json/ext/generator/extconf.rb +3 -0
- data/ext/json/ext/generator/generator.c +45 -33
- data/ext/json/ext/json.h +74 -0
- data/ext/json/ext/parser/extconf.rb +37 -0
- data/ext/json/ext/parser/parser.c +1496 -349
- data/ext/json/ext/vendor/fast_float_parser.h +814 -0
- data/lib/json/truffle_ruby/generator.rb +11 -8
- data/lib/json/version.rb +1 -1
- data/lib/json.rb +18 -4
- metadata +3 -3
- data/ext/json/ext/vendor/ryu.h +0 -819
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
#include "../json.h"
|
|
2
|
-
#include "../vendor/
|
|
2
|
+
#include "../vendor/fast_float_parser.h"
|
|
3
3
|
#include "../simd/simd.h"
|
|
4
4
|
|
|
5
|
-
static VALUE mJSON, eNestingError, Encoding_UTF_8;
|
|
6
|
-
static VALUE CNaN, CInfinity, CMinusInfinity;
|
|
5
|
+
static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
|
|
6
|
+
static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
|
|
7
7
|
|
|
8
|
-
static ID i_new, i_try_convert, i_uminus, i_encode;
|
|
8
|
+
static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column;
|
|
9
9
|
|
|
10
|
-
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma,
|
|
11
|
-
sym_allow_invalid_escape, sym_symbolize_names,
|
|
12
|
-
sym_allow_duplicate_key;
|
|
10
|
+
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
|
|
11
|
+
sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
|
|
12
|
+
sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
|
|
13
13
|
|
|
14
14
|
static int binary_encindex;
|
|
15
15
|
static int utf8_encindex;
|
|
@@ -58,6 +58,20 @@ typedef struct rvalue_cache_struct {
|
|
|
58
58
|
VALUE entries[JSON_RVALUE_CACHE_CAPA];
|
|
59
59
|
} rvalue_cache;
|
|
60
60
|
|
|
61
|
+
static void rvalue_cache_mark(rvalue_cache *cache)
|
|
62
|
+
{
|
|
63
|
+
for (int index = 0; index < cache->length; index++) {
|
|
64
|
+
rb_gc_mark_movable(cache->entries[index]);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
static void rvalue_cache_compact(rvalue_cache *cache)
|
|
69
|
+
{
|
|
70
|
+
for (int index = 0; index < cache->length; index++) {
|
|
71
|
+
cache->entries[index] = rb_gc_location(cache->entries[index]);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
61
75
|
static rb_encoding *enc_utf8;
|
|
62
76
|
|
|
63
77
|
#define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55
|
|
@@ -206,12 +220,12 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle,
|
|
|
206
220
|
|
|
207
221
|
static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
208
222
|
{
|
|
209
|
-
long required = stack->capa * 2;
|
|
223
|
+
long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA;
|
|
210
224
|
|
|
211
225
|
if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
|
|
212
226
|
stack = rvalue_stack_spill(stack, handle, stack_ref);
|
|
213
227
|
} else {
|
|
214
|
-
|
|
228
|
+
JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa);
|
|
215
229
|
stack->capa = required;
|
|
216
230
|
}
|
|
217
231
|
return stack;
|
|
@@ -219,11 +233,15 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu
|
|
|
219
233
|
|
|
220
234
|
static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref)
|
|
221
235
|
{
|
|
236
|
+
JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle);
|
|
237
|
+
|
|
222
238
|
if (RB_UNLIKELY(stack->head >= stack->capa)) {
|
|
223
239
|
stack = rvalue_stack_grow(stack, handle, stack_ref);
|
|
224
240
|
}
|
|
241
|
+
|
|
225
242
|
stack->ptr[stack->head] = value;
|
|
226
243
|
stack->head++;
|
|
244
|
+
|
|
227
245
|
return value;
|
|
228
246
|
}
|
|
229
247
|
|
|
@@ -241,35 +259,62 @@ static void rvalue_stack_mark(void *ptr)
|
|
|
241
259
|
{
|
|
242
260
|
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
243
261
|
long index;
|
|
244
|
-
|
|
245
|
-
|
|
262
|
+
if (stack && stack->ptr) {
|
|
263
|
+
for (index = 0; index < stack->head; index++) {
|
|
264
|
+
rb_gc_mark_movable(stack->ptr[index]);
|
|
265
|
+
}
|
|
246
266
|
}
|
|
247
267
|
}
|
|
248
268
|
|
|
269
|
+
static void rvalue_stack_free_buffer(rvalue_stack *stack)
|
|
270
|
+
{
|
|
271
|
+
JSON_SIZED_FREE_N(stack->ptr, stack->capa);
|
|
272
|
+
stack->ptr = NULL;
|
|
273
|
+
}
|
|
274
|
+
|
|
249
275
|
static void rvalue_stack_free(void *ptr)
|
|
250
276
|
{
|
|
251
277
|
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
252
278
|
if (stack) {
|
|
253
|
-
|
|
254
|
-
|
|
279
|
+
rvalue_stack_free_buffer(stack);
|
|
280
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
281
|
+
JSON_SIZED_FREE(stack);
|
|
282
|
+
#endif
|
|
255
283
|
}
|
|
256
284
|
}
|
|
257
285
|
|
|
258
286
|
static size_t rvalue_stack_memsize(const void *ptr)
|
|
259
287
|
{
|
|
260
288
|
const rvalue_stack *stack = (const rvalue_stack *)ptr;
|
|
261
|
-
|
|
289
|
+
size_t memsize = sizeof(VALUE) * stack->capa;
|
|
290
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
291
|
+
memsize += sizeof(rvalue_stack);
|
|
292
|
+
#endif
|
|
293
|
+
return memsize;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
static void rvalue_stack_compact(void *ptr)
|
|
297
|
+
{
|
|
298
|
+
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
299
|
+
long index;
|
|
300
|
+
if (stack && stack->ptr) {
|
|
301
|
+
for (index = 0; index < stack->head; index++) {
|
|
302
|
+
stack->ptr[index] = rb_gc_location(stack->ptr[index]);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
262
305
|
}
|
|
263
306
|
|
|
264
307
|
static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
|
|
265
|
-
"JSON::Ext::Parser/rvalue_stack",
|
|
266
|
-
{
|
|
308
|
+
.wrap_struct_name = "JSON::Ext::Parser/rvalue_stack",
|
|
309
|
+
.function = {
|
|
267
310
|
.dmark = rvalue_stack_mark,
|
|
268
311
|
.dfree = rvalue_stack_free,
|
|
269
312
|
.dsize = rvalue_stack_memsize,
|
|
313
|
+
.dcompact = rvalue_stack_compact,
|
|
270
314
|
},
|
|
271
|
-
|
|
272
|
-
|
|
315
|
+
// We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
|
|
316
|
+
// because it churns a lot of values so trigering write barriers every time is very costly.
|
|
317
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
|
|
273
318
|
};
|
|
274
319
|
|
|
275
320
|
static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
@@ -291,38 +336,71 @@ static void rvalue_stack_eagerly_release(VALUE handle)
|
|
|
291
336
|
if (handle) {
|
|
292
337
|
rvalue_stack *stack;
|
|
293
338
|
TypedData_Get_Struct(handle, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack);
|
|
294
|
-
|
|
339
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
340
|
+
rvalue_stack_free_buffer(stack);
|
|
341
|
+
#else
|
|
295
342
|
rvalue_stack_free(stack);
|
|
343
|
+
RTYPEDDATA_DATA(handle) = NULL;
|
|
344
|
+
#endif
|
|
296
345
|
}
|
|
297
346
|
}
|
|
298
347
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
348
|
+
/* frame stack */
|
|
349
|
+
|
|
350
|
+
// Iterative (non-recursive) parsing keeps an explicit stack of the containers
|
|
351
|
+
// currently being built, instead of relying on the C call stack. Each frame
|
|
352
|
+
// only needs enough bookkeeping to close its container: which kind it is, the
|
|
353
|
+
// rvalue_stack position where its children start (so we know how many to pop),
|
|
354
|
+
// and the cursor at its opening brace (used to rewind for duplicate key
|
|
355
|
+
// errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses
|
|
356
|
+
// the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that
|
|
357
|
+
// it's freed even if parsing raises.
|
|
358
|
+
//
|
|
359
|
+
// The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release
|
|
360
|
+
// and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts
|
|
361
|
+
// -- the element type and the absence of a mark function are the only real
|
|
362
|
+
// differences. Keep the two in sync: a fix to the spill/release or
|
|
363
|
+
// HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the
|
|
364
|
+
// other.
|
|
365
|
+
#define JSON_FRAME_STACK_INITIAL_CAPA 32
|
|
366
|
+
|
|
367
|
+
enum json_frame_type {
|
|
368
|
+
JSON_FRAME_ROOT, // == JSON_PHASE_DONE
|
|
369
|
+
JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA
|
|
370
|
+
JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
// Where a frame is within its container's grammar. This is the entirety of the
|
|
374
|
+
// parser's "what to do next" state: json_parse_any dispatches on the top
|
|
375
|
+
// frame's phase and holds no resume state in C locals, so a parse can stop at
|
|
376
|
+
// any value boundary and be resumed purely from the (persistable) frame stack.
|
|
377
|
+
//
|
|
378
|
+
// The first three phases are deliberately equal to the corresponding json_frame_type
|
|
379
|
+
// to simplify the transition of phase in json_value_completed.
|
|
380
|
+
enum json_frame_phase {
|
|
381
|
+
JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed
|
|
382
|
+
JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']'
|
|
383
|
+
JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}'
|
|
384
|
+
JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':')
|
|
385
|
+
JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',')
|
|
386
|
+
JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':'
|
|
387
|
+
};
|
|
388
|
+
|
|
389
|
+
typedef struct json_frame_struct {
|
|
390
|
+
enum json_frame_type type;
|
|
391
|
+
enum json_frame_phase phase;
|
|
392
|
+
long value_stack_head; // rvalue_stack->head when this container opened
|
|
393
|
+
size_t start_offset; // object frames only (the '{'); NULL otherwise
|
|
394
|
+
} json_frame;
|
|
324
395
|
|
|
325
|
-
|
|
396
|
+
typedef struct json_frame_stack_struct {
|
|
397
|
+
enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated
|
|
398
|
+
long capa;
|
|
399
|
+
long head;
|
|
400
|
+
json_frame *ptr;
|
|
401
|
+
} json_frame_stack;
|
|
402
|
+
|
|
403
|
+
enum deprecatable_action {
|
|
326
404
|
JSON_DEPRECATED = 0,
|
|
327
405
|
JSON_IGNORE,
|
|
328
406
|
JSON_RAISE,
|
|
@@ -332,7 +410,8 @@ typedef struct JSON_ParserStruct {
|
|
|
332
410
|
VALUE on_load_proc;
|
|
333
411
|
VALUE decimal_class;
|
|
334
412
|
ID decimal_method_id;
|
|
335
|
-
enum
|
|
413
|
+
enum deprecatable_action on_duplicate_key;
|
|
414
|
+
enum deprecatable_action on_comment;
|
|
336
415
|
int max_nesting;
|
|
337
416
|
bool allow_nan;
|
|
338
417
|
bool allow_trailing_comma;
|
|
@@ -343,16 +422,152 @@ typedef struct JSON_ParserStruct {
|
|
|
343
422
|
} JSON_ParserConfig;
|
|
344
423
|
|
|
345
424
|
typedef struct JSON_ParserStateStruct {
|
|
346
|
-
VALUE
|
|
425
|
+
VALUE *value_stack_handle;
|
|
426
|
+
VALUE *frame_stack_handle;
|
|
347
427
|
const char *start;
|
|
348
428
|
const char *cursor;
|
|
349
429
|
const char *end;
|
|
350
|
-
rvalue_stack *
|
|
430
|
+
rvalue_stack *value_stack;
|
|
431
|
+
json_frame_stack *frames;
|
|
351
432
|
rvalue_cache name_cache;
|
|
352
433
|
int in_array;
|
|
353
434
|
int current_nesting;
|
|
435
|
+
unsigned int emitted_deprecations;
|
|
436
|
+
VALUE parser;
|
|
354
437
|
} JSON_ParserState;
|
|
355
438
|
|
|
439
|
+
static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref);
|
|
440
|
+
|
|
441
|
+
static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref)
|
|
442
|
+
{
|
|
443
|
+
long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA;
|
|
444
|
+
|
|
445
|
+
if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
|
|
446
|
+
stack = json_frame_stack_spill(stack, handle, stack_ref);
|
|
447
|
+
} else {
|
|
448
|
+
JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa);
|
|
449
|
+
stack->capa = required;
|
|
450
|
+
}
|
|
451
|
+
return stack;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame)
|
|
455
|
+
{
|
|
456
|
+
json_frame_stack *stack = state->frames;
|
|
457
|
+
|
|
458
|
+
JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle);
|
|
459
|
+
|
|
460
|
+
if (RB_UNLIKELY(stack->head >= stack->capa)) {
|
|
461
|
+
stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
json_frame *frame_ptr = &stack->ptr[stack->head++];
|
|
465
|
+
*frame_ptr = frame;
|
|
466
|
+
return frame_ptr;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
static inline json_frame *json_frame_stack_peek(json_frame_stack *stack)
|
|
470
|
+
{
|
|
471
|
+
return &stack->ptr[stack->head - 1];
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
static inline void json_frame_stack_pop(json_frame_stack *stack)
|
|
475
|
+
{
|
|
476
|
+
stack->head--;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
static void json_frame_stack_free_buffer(json_frame_stack *stack)
|
|
480
|
+
{
|
|
481
|
+
JSON_SIZED_FREE_N(stack->ptr, stack->capa);
|
|
482
|
+
stack->ptr = NULL;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
static void json_frame_stack_free(void *ptr)
|
|
486
|
+
{
|
|
487
|
+
json_frame_stack *stack = (json_frame_stack *)ptr;
|
|
488
|
+
if (stack) {
|
|
489
|
+
json_frame_stack_free_buffer(stack);
|
|
490
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
491
|
+
JSON_SIZED_FREE(stack);
|
|
492
|
+
#endif
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
static size_t json_frame_stack_memsize(const void *ptr)
|
|
497
|
+
{
|
|
498
|
+
const json_frame_stack *stack = (const json_frame_stack *)ptr;
|
|
499
|
+
|
|
500
|
+
size_t memsize = sizeof(json_frame) * stack->capa;
|
|
501
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
502
|
+
memsize += sizeof(json_frame_stack);
|
|
503
|
+
#endif
|
|
504
|
+
return memsize;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
static const rb_data_type_t JSON_Parser_frame_stack_type = {
|
|
508
|
+
.wrap_struct_name = "JSON::Ext::Parser/frame_stack",
|
|
509
|
+
.function = {
|
|
510
|
+
.dmark = NULL,
|
|
511
|
+
.dfree = json_frame_stack_free,
|
|
512
|
+
.dsize = json_frame_stack_memsize,
|
|
513
|
+
},
|
|
514
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
|
|
518
|
+
{
|
|
519
|
+
json_frame_stack *stack;
|
|
520
|
+
*handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
|
|
521
|
+
*stack_ref = stack;
|
|
522
|
+
MEMCPY(stack, old_stack, json_frame_stack, 1);
|
|
523
|
+
|
|
524
|
+
stack->capa = old_stack->capa << 1;
|
|
525
|
+
stack->ptr = ALLOC_N(json_frame, stack->capa);
|
|
526
|
+
stack->type = RVALUE_STACK_HEAP_ALLOCATED;
|
|
527
|
+
MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head);
|
|
528
|
+
return stack;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
static void json_frame_stack_eagerly_release(VALUE handle)
|
|
532
|
+
{
|
|
533
|
+
if (handle) {
|
|
534
|
+
json_frame_stack *stack;
|
|
535
|
+
TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
|
|
536
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
537
|
+
json_frame_stack_free_buffer(stack);
|
|
538
|
+
#else
|
|
539
|
+
json_frame_stack_free(stack);
|
|
540
|
+
RTYPEDDATA_DATA(handle) = NULL;
|
|
541
|
+
#endif
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
|
|
546
|
+
{
|
|
547
|
+
int len = 1;
|
|
548
|
+
if (ch <= 0x7F) {
|
|
549
|
+
buf[0] = (char) ch;
|
|
550
|
+
} else if (ch <= 0x07FF) {
|
|
551
|
+
buf[0] = (char) ((ch >> 6) | 0xC0);
|
|
552
|
+
buf[1] = (char) ((ch & 0x3F) | 0x80);
|
|
553
|
+
len++;
|
|
554
|
+
} else if (ch <= 0xFFFF) {
|
|
555
|
+
buf[0] = (char) ((ch >> 12) | 0xE0);
|
|
556
|
+
buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
|
|
557
|
+
buf[2] = (char) ((ch & 0x3F) | 0x80);
|
|
558
|
+
len += 2;
|
|
559
|
+
} else if (ch <= 0x1fffff) {
|
|
560
|
+
buf[0] =(char) ((ch >> 18) | 0xF0);
|
|
561
|
+
buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
|
|
562
|
+
buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
|
|
563
|
+
buf[3] =(char) ((ch & 0x3F) | 0x80);
|
|
564
|
+
len += 3;
|
|
565
|
+
} else {
|
|
566
|
+
buf[0] = '?';
|
|
567
|
+
}
|
|
568
|
+
return len;
|
|
569
|
+
}
|
|
570
|
+
|
|
356
571
|
static inline size_t rest(JSON_ParserState *state) {
|
|
357
572
|
return state->end - state->cursor;
|
|
358
573
|
}
|
|
@@ -371,12 +586,20 @@ static inline char peek(JSON_ParserState *state)
|
|
|
371
586
|
|
|
372
587
|
static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
|
|
373
588
|
{
|
|
589
|
+
JSON_ASSERT(state->cursor <= state->end);
|
|
590
|
+
|
|
591
|
+
// Redundant but helpful for hardening
|
|
592
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
593
|
+
state->cursor = state->end;
|
|
594
|
+
}
|
|
595
|
+
|
|
374
596
|
const char *cursor = state->cursor;
|
|
375
597
|
long column = 0;
|
|
376
598
|
long line = 1;
|
|
377
599
|
|
|
378
600
|
while (cursor >= state->start) {
|
|
379
601
|
if (*cursor-- == '\n') {
|
|
602
|
+
line++;
|
|
380
603
|
break;
|
|
381
604
|
}
|
|
382
605
|
column++;
|
|
@@ -391,6 +614,8 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum
|
|
|
391
614
|
*column_out = column;
|
|
392
615
|
}
|
|
393
616
|
|
|
617
|
+
static const unsigned int MAX_DEPRECATIONS = 5;
|
|
618
|
+
|
|
394
619
|
static void emit_parse_warning(const char *message, JSON_ParserState *state)
|
|
395
620
|
{
|
|
396
621
|
long line, column;
|
|
@@ -402,11 +627,9 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state)
|
|
|
402
627
|
|
|
403
628
|
#define PARSE_ERROR_FRAGMENT_LEN 32
|
|
404
629
|
|
|
405
|
-
|
|
630
|
+
static VALUE build_parse_error_message(const char *format, JSON_ParserState *state)
|
|
406
631
|
{
|
|
407
632
|
unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
|
|
408
|
-
long line, column;
|
|
409
|
-
cursor_position(state, &line, &column);
|
|
410
633
|
|
|
411
634
|
const char *ptr = "EOF";
|
|
412
635
|
if (state->cursor && state->cursor < state->end) {
|
|
@@ -438,20 +661,61 @@ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *st
|
|
|
438
661
|
}
|
|
439
662
|
}
|
|
440
663
|
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
664
|
+
return rb_enc_sprintf(enc_utf8, format, ptr);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
|
|
668
|
+
{
|
|
669
|
+
VALUE exc = rb_exc_new_str(eParserError, message);
|
|
670
|
+
rb_ivar_set(exc, i_at_line, LONG2NUM(line));
|
|
671
|
+
rb_ivar_set(exc, i_at_column, LONG2NUM(column));
|
|
672
|
+
return exc;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos)
|
|
676
|
+
{
|
|
677
|
+
if (state->parser) {
|
|
678
|
+
if (eos) {
|
|
679
|
+
// the error will be swallowed by ResumableParser#parse, so no
|
|
680
|
+
// point building a message or backtrace.
|
|
681
|
+
rb_throw_obj(state->parser, state->parser);
|
|
682
|
+
} else {
|
|
683
|
+
// line and columns can't be accurate in resumable
|
|
684
|
+
rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos));
|
|
685
|
+
}
|
|
686
|
+
} else {
|
|
687
|
+
VALUE message = build_parse_error_message(format, state);
|
|
688
|
+
long line, column;
|
|
689
|
+
cursor_position(state, &line, &column);
|
|
690
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
691
|
+
rb_exc_raise(parse_error_new(state, message, line, column, eos));
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state)
|
|
696
|
+
{
|
|
697
|
+
raise_parse_error(format, state, true);
|
|
698
|
+
}
|
|
444
699
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
rb_exc_raise(exc);
|
|
700
|
+
NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state)
|
|
701
|
+
{
|
|
702
|
+
raise_parse_error(format, state, false);
|
|
449
703
|
}
|
|
450
704
|
|
|
451
|
-
NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
705
|
+
NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos)
|
|
452
706
|
{
|
|
453
707
|
state->cursor = at;
|
|
454
|
-
raise_parse_error(format, state);
|
|
708
|
+
raise_parse_error(format, state, eos);
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
712
|
+
{
|
|
713
|
+
raise_parse_error_at(format, state, at, true);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
717
|
+
{
|
|
718
|
+
raise_parse_error_at(format, state, at, false);
|
|
455
719
|
}
|
|
456
720
|
|
|
457
721
|
/* unicode */
|
|
@@ -476,7 +740,7 @@ static const signed char digit_values[256] = {
|
|
|
476
740
|
static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
|
|
477
741
|
{
|
|
478
742
|
if (RB_UNLIKELY(sp > spe - 4)) {
|
|
479
|
-
|
|
743
|
+
raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
480
744
|
}
|
|
481
745
|
|
|
482
746
|
const unsigned char *p = (const unsigned char *)sp;
|
|
@@ -487,7 +751,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
|
|
|
487
751
|
const signed char b3 = digit_values[p[3]];
|
|
488
752
|
|
|
489
753
|
if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
|
|
490
|
-
|
|
754
|
+
raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
491
755
|
}
|
|
492
756
|
|
|
493
757
|
return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
|
|
@@ -499,9 +763,14 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
|
|
|
499
763
|
|
|
500
764
|
static const rb_data_type_t JSON_ParserConfig_type;
|
|
501
765
|
|
|
502
|
-
|
|
503
|
-
|
|
766
|
+
const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
|
|
767
|
+
NOINLINE(static) void
|
|
768
|
+
json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
504
769
|
{
|
|
770
|
+
if (config->on_comment == JSON_RAISE) {
|
|
771
|
+
raise_syntax_error("unexpected token %s", state);
|
|
772
|
+
}
|
|
773
|
+
|
|
505
774
|
const char *start = state->cursor;
|
|
506
775
|
state->cursor++;
|
|
507
776
|
|
|
@@ -521,7 +790,7 @@ json_eat_comments(JSON_ParserState *state)
|
|
|
521
790
|
while (true) {
|
|
522
791
|
const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
|
|
523
792
|
if (!next_match) {
|
|
524
|
-
|
|
793
|
+
raise_eos_error_at("unterminated comment, expected closing '*/'", state, start);
|
|
525
794
|
}
|
|
526
795
|
|
|
527
796
|
state->cursor = next_match + 1;
|
|
@@ -533,13 +802,18 @@ json_eat_comments(JSON_ParserState *state)
|
|
|
533
802
|
break;
|
|
534
803
|
}
|
|
535
804
|
default:
|
|
536
|
-
raise_parse_error_at("unexpected token %s", state, start);
|
|
805
|
+
raise_parse_error_at("unexpected token %s", state, start, eos(state));
|
|
537
806
|
break;
|
|
538
807
|
}
|
|
808
|
+
|
|
809
|
+
if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) {
|
|
810
|
+
state->emitted_deprecations++;
|
|
811
|
+
emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state);
|
|
812
|
+
}
|
|
539
813
|
}
|
|
540
814
|
|
|
541
815
|
ALWAYS_INLINE(static) void
|
|
542
|
-
json_eat_whitespace(JSON_ParserState *state)
|
|
816
|
+
json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
|
|
543
817
|
{
|
|
544
818
|
while (true) {
|
|
545
819
|
switch (peek(state)) {
|
|
@@ -570,7 +844,11 @@ json_eat_whitespace(JSON_ParserState *state)
|
|
|
570
844
|
state->cursor++;
|
|
571
845
|
break;
|
|
572
846
|
case '/':
|
|
573
|
-
|
|
847
|
+
if (!include_comments) {
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
json_eat_comments(state, config);
|
|
574
852
|
break;
|
|
575
853
|
|
|
576
854
|
default:
|
|
@@ -724,13 +1002,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
724
1002
|
uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
|
|
725
1003
|
|
|
726
1004
|
if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
|
|
727
|
-
|
|
1005
|
+
raise_syntax_error_at("invalid surrogate pair at %s", state, p);
|
|
728
1006
|
}
|
|
729
1007
|
|
|
730
1008
|
ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
|
|
731
1009
|
pe += 5;
|
|
732
1010
|
} else {
|
|
733
|
-
|
|
1011
|
+
raise_syntax_error_at("incomplete surrogate pair at %s", state, p);
|
|
734
1012
|
break;
|
|
735
1013
|
}
|
|
736
1014
|
}
|
|
@@ -740,18 +1018,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
740
1018
|
p = ++pe;
|
|
741
1019
|
break;
|
|
742
1020
|
}
|
|
1021
|
+
case 0:
|
|
1022
|
+
return Qundef;
|
|
743
1023
|
default:
|
|
744
1024
|
if ((unsigned char)*pe < 0x20) {
|
|
745
1025
|
if (!config->allow_control_characters) {
|
|
746
1026
|
if (*pe == '\n') {
|
|
747
|
-
|
|
1027
|
+
raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
|
|
748
1028
|
}
|
|
749
|
-
|
|
1029
|
+
raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1);
|
|
750
1030
|
}
|
|
751
|
-
}
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
if (config->allow_invalid_escape) {
|
|
752
1034
|
APPEND_CHAR(*pe);
|
|
753
1035
|
} else {
|
|
754
|
-
|
|
1036
|
+
raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1);
|
|
755
1037
|
}
|
|
756
1038
|
break;
|
|
757
1039
|
}
|
|
@@ -831,7 +1113,7 @@ NOINLINE(static) VALUE json_decode_large_float(const char *start, long len)
|
|
|
831
1113
|
/* Ruby JSON optimized float decoder using vendored Ryu algorithm
|
|
832
1114
|
* Accepts pre-extracted mantissa and exponent from first-pass validation
|
|
833
1115
|
*/
|
|
834
|
-
static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits,
|
|
1116
|
+
static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int64_t exponent, bool negative,
|
|
835
1117
|
const char *start, const char *end)
|
|
836
1118
|
{
|
|
837
1119
|
if (RB_UNLIKELY(config->decimal_class)) {
|
|
@@ -839,19 +1121,25 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
|
|
|
839
1121
|
return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text);
|
|
840
1122
|
}
|
|
841
1123
|
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1124
|
+
if (RB_UNLIKELY(exponent > INT32_MAX)) {
|
|
1125
|
+
return negative ? CMinusInfinity : CInfinity;
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
if (RB_UNLIKELY(exponent < INT32_MIN)) {
|
|
1129
|
+
return rb_float_new(negative ? -0.0 : 0.0);
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
|
|
845
1133
|
return json_decode_large_float(start, end - start);
|
|
846
1134
|
}
|
|
847
1135
|
|
|
848
|
-
return DBL2NUM(
|
|
1136
|
+
return DBL2NUM(ffp_s2d(exponent, mantissa, negative));
|
|
849
1137
|
}
|
|
850
1138
|
|
|
851
1139
|
static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
|
|
852
1140
|
{
|
|
853
|
-
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->
|
|
854
|
-
rvalue_stack_pop(state->
|
|
1141
|
+
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count));
|
|
1142
|
+
rvalue_stack_pop(state->value_stack, count);
|
|
855
1143
|
|
|
856
1144
|
if (config->freeze) {
|
|
857
1145
|
RB_OBJ_FREEZE(array);
|
|
@@ -895,31 +1183,50 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d
|
|
|
895
1183
|
rb_inspect(duplicate_key)
|
|
896
1184
|
);
|
|
897
1185
|
|
|
898
|
-
|
|
899
|
-
|
|
1186
|
+
rb_str_concat(message, build_parse_error_message("", state));
|
|
1187
|
+
if (state->parser) { // line and columns can't be accurate in resumable
|
|
1188
|
+
rb_exc_raise(parse_error_new(state, message, 0, 0, false));
|
|
1189
|
+
} else {
|
|
1190
|
+
long line, column;
|
|
1191
|
+
cursor_position(state, &line, &column);
|
|
1192
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
1193
|
+
rb_exc_raise(parse_error_new(state, message, line, column, false));
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
|
|
1198
|
+
{
|
|
1199
|
+
switch (config->on_duplicate_key) {
|
|
1200
|
+
case JSON_IGNORE:
|
|
1201
|
+
return;
|
|
1202
|
+
|
|
1203
|
+
case JSON_DEPRECATED:
|
|
1204
|
+
// Only emit the first few deprecations to avoid spamming.
|
|
1205
|
+
if (state->emitted_deprecations < MAX_DEPRECATIONS) {
|
|
1206
|
+
state->emitted_deprecations++;
|
|
1207
|
+
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
1208
|
+
}
|
|
1209
|
+
return;
|
|
1210
|
+
|
|
1211
|
+
case JSON_RAISE:
|
|
1212
|
+
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
UNREACHABLE;
|
|
900
1216
|
}
|
|
901
1217
|
|
|
902
1218
|
static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
|
|
903
1219
|
{
|
|
904
1220
|
size_t entries_count = count / 2;
|
|
905
1221
|
VALUE object = rb_hash_new_capa(entries_count);
|
|
906
|
-
const VALUE *pairs = rvalue_stack_peek(state->
|
|
1222
|
+
const VALUE *pairs = rvalue_stack_peek(state->value_stack, count);
|
|
907
1223
|
rb_hash_bulk_insert(count, pairs, object);
|
|
908
1224
|
|
|
909
1225
|
if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
|
|
910
|
-
|
|
911
|
-
case JSON_IGNORE:
|
|
912
|
-
break;
|
|
913
|
-
case JSON_DEPRECATED:
|
|
914
|
-
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
915
|
-
break;
|
|
916
|
-
case JSON_RAISE:
|
|
917
|
-
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
918
|
-
break;
|
|
919
|
-
}
|
|
1226
|
+
json_on_duplicate_key(state, config, count, pairs);
|
|
920
1227
|
}
|
|
921
1228
|
|
|
922
|
-
rvalue_stack_pop(state->
|
|
1229
|
+
rvalue_stack_pop(state->value_stack, count);
|
|
923
1230
|
|
|
924
1231
|
if (config->freeze) {
|
|
925
1232
|
RB_OBJ_FREEZE(object);
|
|
@@ -933,7 +1240,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
|
|
|
933
1240
|
if (RB_UNLIKELY(config->on_load_proc)) {
|
|
934
1241
|
value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
|
|
935
1242
|
}
|
|
936
|
-
rvalue_stack_push(state->
|
|
1243
|
+
rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack);
|
|
937
1244
|
return value;
|
|
938
1245
|
}
|
|
939
1246
|
|
|
@@ -982,6 +1289,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
|
|
|
982
1289
|
}
|
|
983
1290
|
state->cursor++;
|
|
984
1291
|
}
|
|
1292
|
+
|
|
1293
|
+
// If the string ended with an unterminated escape sequence, we might
|
|
1294
|
+
// have gone past the end.
|
|
1295
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
1296
|
+
state->cursor = state->end;
|
|
1297
|
+
}
|
|
1298
|
+
|
|
985
1299
|
return false;
|
|
986
1300
|
}
|
|
987
1301
|
|
|
@@ -999,7 +1313,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
999
1313
|
case '"': {
|
|
1000
1314
|
VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
|
|
1001
1315
|
state->cursor++;
|
|
1002
|
-
return
|
|
1316
|
+
return string;
|
|
1003
1317
|
}
|
|
1004
1318
|
case '\\': {
|
|
1005
1319
|
if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
|
|
@@ -1013,7 +1327,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1013
1327
|
}
|
|
1014
1328
|
default:
|
|
1015
1329
|
if (!config->allow_control_characters) {
|
|
1016
|
-
|
|
1330
|
+
raise_syntax_error("invalid ASCII control character in string: %s", state);
|
|
1017
1331
|
}
|
|
1018
1332
|
break;
|
|
1019
1333
|
}
|
|
@@ -1021,8 +1335,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1021
1335
|
state->cursor++;
|
|
1022
1336
|
} while (string_scan(state));
|
|
1023
1337
|
|
|
1024
|
-
|
|
1025
|
-
return Qfalse;
|
|
1338
|
+
return Qundef;
|
|
1026
1339
|
}
|
|
1027
1340
|
|
|
1028
1341
|
ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
|
|
@@ -1031,15 +1344,19 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars
|
|
|
1031
1344
|
const char *start = state->cursor;
|
|
1032
1345
|
|
|
1033
1346
|
if (RB_UNLIKELY(!string_scan(state))) {
|
|
1034
|
-
|
|
1347
|
+
return Qundef;
|
|
1035
1348
|
}
|
|
1036
1349
|
|
|
1350
|
+
VALUE string;
|
|
1037
1351
|
if (RB_LIKELY(*state->cursor == '"')) {
|
|
1038
|
-
|
|
1352
|
+
string = json_string_fastpath(state, config, start, state->cursor, is_name);
|
|
1039
1353
|
state->cursor++;
|
|
1040
|
-
return json_push_value(state, config, string);
|
|
1041
1354
|
}
|
|
1042
|
-
|
|
1355
|
+
else {
|
|
1356
|
+
string = json_parse_escaped_string(state, config, is_name, start);
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
return string;
|
|
1043
1360
|
}
|
|
1044
1361
|
|
|
1045
1362
|
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
@@ -1118,7 +1435,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1118
1435
|
const char first_digit = *state->cursor;
|
|
1119
1436
|
|
|
1120
1437
|
// Variables for Ryu optimization - extract digits during parsing
|
|
1121
|
-
|
|
1438
|
+
int64_t exponent = 0;
|
|
1122
1439
|
int decimal_point_pos = -1;
|
|
1123
1440
|
uint64_t mantissa = 0;
|
|
1124
1441
|
|
|
@@ -1126,7 +1443,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1126
1443
|
int mantissa_digits = json_parse_digits(state, &mantissa);
|
|
1127
1444
|
|
|
1128
1445
|
if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
|
|
1129
|
-
|
|
1446
|
+
return Qundef;
|
|
1130
1447
|
}
|
|
1131
1448
|
|
|
1132
1449
|
// Parse fractional part
|
|
@@ -1139,7 +1456,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1139
1456
|
mantissa_digits += fractional_digits;
|
|
1140
1457
|
|
|
1141
1458
|
if (RB_UNLIKELY(!fractional_digits)) {
|
|
1142
|
-
|
|
1459
|
+
return Qundef;
|
|
1143
1460
|
}
|
|
1144
1461
|
}
|
|
1145
1462
|
|
|
@@ -1159,10 +1476,14 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1159
1476
|
int exponent_digits = json_parse_digits(state, &abs_exponent);
|
|
1160
1477
|
|
|
1161
1478
|
if (RB_UNLIKELY(!exponent_digits)) {
|
|
1162
|
-
|
|
1479
|
+
return Qundef;
|
|
1163
1480
|
}
|
|
1164
1481
|
|
|
1165
|
-
|
|
1482
|
+
if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
|
|
1483
|
+
exponent = negative_exponent ? INT64_MIN : INT64_MAX;
|
|
1484
|
+
} else {
|
|
1485
|
+
exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
|
|
1486
|
+
}
|
|
1166
1487
|
}
|
|
1167
1488
|
|
|
1168
1489
|
if (integer) {
|
|
@@ -1177,229 +1498,411 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1177
1498
|
return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
|
|
1178
1499
|
}
|
|
1179
1500
|
|
|
1180
|
-
|
|
1501
|
+
// How many values (array elements, or interleaved object keys+values) have been
|
|
1502
|
+
// pushed onto the rvalue stack since this container opened. Used to size the
|
|
1503
|
+
// bulk decode on close, and to tell the first key/colon from later ones.
|
|
1504
|
+
static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack)
|
|
1181
1505
|
{
|
|
1182
|
-
return
|
|
1506
|
+
return value_stack->head - frame->value_stack_head;
|
|
1183
1507
|
}
|
|
1184
1508
|
|
|
1185
|
-
|
|
1509
|
+
// A complete value now sits on top of the rvalue stack. Advance the frame that
|
|
1510
|
+
// was waiting for it: the root document is done, or the enclosing container
|
|
1511
|
+
// moves on to expecting a ',' or its closing bracket. The caller passes the
|
|
1512
|
+
// frame it already has in hand -- the one that was expecting the value -- which
|
|
1513
|
+
// after a container close is the freshly re-exposed parent.
|
|
1514
|
+
static inline enum json_frame_phase json_value_completed(json_frame *frame)
|
|
1186
1515
|
{
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1516
|
+
JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT);
|
|
1517
|
+
JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY);
|
|
1518
|
+
JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT);
|
|
1519
|
+
|
|
1520
|
+
return frame->phase = (enum json_frame_phase) frame->type;
|
|
1190
1521
|
}
|
|
1191
1522
|
|
|
1192
|
-
static
|
|
1523
|
+
ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset)
|
|
1193
1524
|
{
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
switch (peek(state)) {
|
|
1197
|
-
case 'n':
|
|
1198
|
-
if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
|
|
1199
|
-
state->cursor += 4;
|
|
1200
|
-
return json_push_value(state, config, Qnil);
|
|
1201
|
-
}
|
|
1525
|
+
// It is assumed that since `keyword` is always a literal, the compiler is able to constantize this
|
|
1526
|
+
// `strlen` and several other computations in that routine.
|
|
1202
1527
|
|
|
1203
|
-
|
|
1204
|
-
break;
|
|
1205
|
-
case 't':
|
|
1206
|
-
if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) {
|
|
1207
|
-
state->cursor += 4;
|
|
1208
|
-
return json_push_value(state, config, Qtrue);
|
|
1209
|
-
}
|
|
1528
|
+
size_t len = strlen(keyword);
|
|
1210
1529
|
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
return json_push_value(state, config, Qfalse);
|
|
1218
|
-
}
|
|
1530
|
+
// Note: memcmp with a small power of two and a literal string compile to an integer comparison /
|
|
1531
|
+
// That's why we sometime compare starting from the first byte and sometimes from the second.
|
|
1532
|
+
if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) {
|
|
1533
|
+
state->cursor += len;
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1219
1536
|
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1537
|
+
bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0;
|
|
1538
|
+
raise_parse_error("unexpected token %s", state, eos);
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
// Parse an arbitrary JSON value iteratively. This is a state machine driven
|
|
1542
|
+
// entirely by the top frame's phase so it can stop at any value boundary and
|
|
1543
|
+
// resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the
|
|
1544
|
+
// bottom of the stack, so the stack is never empty mid-parse and the document
|
|
1545
|
+
// itself is just another frame whose value, once parsed, leaves its phase DONE.
|
|
1546
|
+
// When invoked in resumable mode, it returns true after parsing a complete document.
|
|
1547
|
+
// If reaching EOS without having parsed a complete document, either returns false
|
|
1548
|
+
// of raise a JSON::ParserError tagged with `@eos=true`.
|
|
1549
|
+
ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable)
|
|
1550
|
+
{
|
|
1551
|
+
json_frame *frame = json_frame_stack_peek(state->frames);
|
|
1552
|
+
|
|
1553
|
+
switch (frame->phase) {
|
|
1554
|
+
case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false);
|
|
1555
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1556
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1557
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1558
|
+
case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY;
|
|
1559
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1560
|
+
}
|
|
1561
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1228
1562
|
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
case 'I':
|
|
1232
|
-
if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
|
|
1233
|
-
state->cursor += 8;
|
|
1234
|
-
return json_push_value(state, config, CInfinity);
|
|
1235
|
-
}
|
|
1563
|
+
JSON_PHASE_VALUE: {
|
|
1564
|
+
json_eat_whitespace(state, config, true);
|
|
1236
1565
|
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
case '-': {
|
|
1240
|
-
// Note: memcmp with a small power of two compile to an integer comparison
|
|
1241
|
-
if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) {
|
|
1242
|
-
if (config->allow_nan) {
|
|
1243
|
-
state->cursor += 9;
|
|
1244
|
-
return json_push_value(state, config, CMinusInfinity);
|
|
1245
|
-
} else {
|
|
1246
|
-
raise_parse_error("unexpected token %s", state);
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
return json_push_value(state, config, json_parse_negative_number(state, config));
|
|
1250
|
-
break;
|
|
1251
|
-
}
|
|
1252
|
-
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
|
|
1253
|
-
return json_push_value(state, config, json_parse_positive_number(state, config));
|
|
1254
|
-
break;
|
|
1255
|
-
case '"': {
|
|
1256
|
-
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1257
|
-
return json_parse_string(state, config, false);
|
|
1258
|
-
break;
|
|
1259
|
-
}
|
|
1260
|
-
case '[': {
|
|
1261
|
-
state->cursor++;
|
|
1262
|
-
json_eat_whitespace(state);
|
|
1263
|
-
long stack_head = state->stack->head;
|
|
1566
|
+
VALUE value;
|
|
1567
|
+
const char *value_start = state->cursor;
|
|
1264
1568
|
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1271
|
-
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1272
|
-
}
|
|
1273
|
-
state->in_array++;
|
|
1274
|
-
json_parse_any(state, config);
|
|
1275
|
-
}
|
|
1569
|
+
switch (peek(state)) {
|
|
1570
|
+
case 'n':
|
|
1571
|
+
json_match_keyword(state, "null", 0);
|
|
1572
|
+
value = Qnil;
|
|
1573
|
+
break;
|
|
1276
1574
|
|
|
1277
|
-
|
|
1278
|
-
|
|
1575
|
+
case 't':
|
|
1576
|
+
json_match_keyword(state, "true", 0);
|
|
1577
|
+
value = Qtrue;
|
|
1578
|
+
break;
|
|
1279
1579
|
|
|
1280
|
-
|
|
1580
|
+
case 'f':
|
|
1581
|
+
json_match_keyword(state, "false", 1);
|
|
1582
|
+
value = Qfalse;
|
|
1583
|
+
break;
|
|
1281
1584
|
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
json_eat_whitespace(state);
|
|
1286
|
-
if (peek(state) == ']') {
|
|
1287
|
-
continue;
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
json_parse_any(state, config);
|
|
1291
|
-
continue;
|
|
1585
|
+
case 'N':
|
|
1586
|
+
if (!config->allow_nan) {
|
|
1587
|
+
raise_syntax_error("unexpected token %s", state);
|
|
1292
1588
|
}
|
|
1293
1589
|
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1590
|
+
json_match_keyword(state, "NaN", 1);
|
|
1591
|
+
value = CNaN;
|
|
1592
|
+
break;
|
|
1593
|
+
|
|
1594
|
+
case 'I':
|
|
1595
|
+
if (!config->allow_nan) {
|
|
1596
|
+
raise_syntax_error("unexpected token %s", state);
|
|
1300
1597
|
}
|
|
1301
1598
|
|
|
1302
|
-
|
|
1599
|
+
json_match_keyword(state, "Infinity", 0);
|
|
1600
|
+
value = CInfinity;
|
|
1601
|
+
break;
|
|
1602
|
+
|
|
1603
|
+
case '-': {
|
|
1604
|
+
state->cursor++;
|
|
1605
|
+
|
|
1606
|
+
value = json_parse_number(state, config, true, value_start);
|
|
1607
|
+
|
|
1608
|
+
if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
|
|
1609
|
+
state->cursor = value_start;
|
|
1610
|
+
json_match_keyword(state, "-Infinity", 1);
|
|
1611
|
+
value = CMinusInfinity;
|
|
1612
|
+
break;
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
// Top level numbers are ambiguous when parsing streams, we can't
|
|
1616
|
+
// know if we parsed all the digits if we hit EOS.
|
|
1617
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1618
|
+
state->cursor = value_start;
|
|
1619
|
+
return false;
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1623
|
+
raise_syntax_error_at("invalid number: %s", state, value_start);
|
|
1624
|
+
}
|
|
1625
|
+
break;
|
|
1303
1626
|
}
|
|
1304
|
-
break;
|
|
1305
|
-
}
|
|
1306
|
-
case '{': {
|
|
1307
|
-
const char *object_start_cursor = state->cursor;
|
|
1308
1627
|
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1628
|
+
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
|
|
1629
|
+
value = json_parse_number(state, config, false, value_start);
|
|
1630
|
+
|
|
1631
|
+
// Top level numbers are ambiguous when parsing streams, we can't
|
|
1632
|
+
// know if we parsed all the digits if we hit EOS.
|
|
1633
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1634
|
+
state->cursor = value_start;
|
|
1635
|
+
return false;
|
|
1636
|
+
}
|
|
1312
1637
|
|
|
1313
|
-
|
|
1638
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1639
|
+
raise_syntax_error_at("invalid number: %s", state, value_start);
|
|
1640
|
+
}
|
|
1641
|
+
break;
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
case '"': {
|
|
1645
|
+
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1646
|
+
value = json_parse_string(state, config, false);
|
|
1647
|
+
|
|
1648
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1649
|
+
bool is_eos = eos(state);
|
|
1650
|
+
if (resumable && is_eos) {
|
|
1651
|
+
state->cursor = value_start;
|
|
1652
|
+
return false;
|
|
1653
|
+
}
|
|
1654
|
+
raise_parse_error("unexpected end of input, expected closing \"", state, is_eos);
|
|
1655
|
+
}
|
|
1656
|
+
break;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
case '[': {
|
|
1314
1660
|
state->cursor++;
|
|
1315
|
-
|
|
1316
|
-
|
|
1661
|
+
json_eat_whitespace(state, config, true);
|
|
1662
|
+
|
|
1663
|
+
const char next = peek(state);
|
|
1664
|
+
if (next == ']') {
|
|
1665
|
+
state->cursor++;
|
|
1666
|
+
value = json_decode_array(state, config, 0);
|
|
1667
|
+
break;
|
|
1668
|
+
} else if (resumable && eos(state)) {
|
|
1669
|
+
state->cursor = value_start;
|
|
1670
|
+
return false;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1317
1673
|
state->current_nesting++;
|
|
1318
1674
|
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1319
1675
|
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1320
1676
|
}
|
|
1677
|
+
state->in_array++;
|
|
1678
|
+
|
|
1679
|
+
// Phase stays VALUE: the next iteration reads the first element.
|
|
1680
|
+
frame = json_frame_stack_push(state, (json_frame){
|
|
1681
|
+
.type = JSON_FRAME_ARRAY,
|
|
1682
|
+
.phase = JSON_PHASE_VALUE,
|
|
1683
|
+
.value_stack_head = state->value_stack->head,
|
|
1684
|
+
});
|
|
1685
|
+
goto JSON_PHASE_VALUE;
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
case '{': {
|
|
1689
|
+
state->cursor++;
|
|
1690
|
+
json_eat_whitespace(state, config, true);
|
|
1321
1691
|
|
|
1322
|
-
if (peek(state)
|
|
1323
|
-
|
|
1692
|
+
if (peek(state) == '}') {
|
|
1693
|
+
state->cursor++;
|
|
1694
|
+
value = json_decode_object(state, config, 0);
|
|
1695
|
+
break;
|
|
1696
|
+
} else if (resumable && eos(state)) {
|
|
1697
|
+
state->cursor = value_start;
|
|
1698
|
+
return false;
|
|
1324
1699
|
}
|
|
1325
|
-
json_parse_string(state, config, true);
|
|
1326
1700
|
|
|
1327
|
-
|
|
1328
|
-
if (
|
|
1329
|
-
|
|
1701
|
+
state->current_nesting++;
|
|
1702
|
+
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1703
|
+
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1330
1704
|
}
|
|
1331
|
-
state->cursor++;
|
|
1332
1705
|
|
|
1333
|
-
|
|
1706
|
+
// Phase KEY: the next iteration reads the first key.
|
|
1707
|
+
frame = json_frame_stack_push(state, (json_frame){
|
|
1708
|
+
.type = JSON_FRAME_OBJECT,
|
|
1709
|
+
.phase = JSON_PHASE_OBJECT_KEY,
|
|
1710
|
+
.value_stack_head = state->value_stack->head,
|
|
1711
|
+
.start_offset = value_start - state->start,
|
|
1712
|
+
});
|
|
1713
|
+
goto JSON_PHASE_OBJECT_KEY;
|
|
1334
1714
|
}
|
|
1335
1715
|
|
|
1336
|
-
|
|
1337
|
-
|
|
1716
|
+
case 0:
|
|
1717
|
+
// peek() returns 0 both at end-of-stream and for a literal NUL byte in the
|
|
1718
|
+
// buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at
|
|
1719
|
+
// EOS is just an invalid character.
|
|
1720
|
+
if (eos(state)) {
|
|
1721
|
+
return false;
|
|
1722
|
+
} else {
|
|
1723
|
+
raise_syntax_error("unexpected NULL byte: %s", state);
|
|
1724
|
+
}
|
|
1725
|
+
default:
|
|
1726
|
+
raise_syntax_error("unexpected character: %s", state);
|
|
1727
|
+
}
|
|
1338
1728
|
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1729
|
+
json_push_value(state, config, value);
|
|
1730
|
+
json_value_completed(frame);
|
|
1731
|
+
|
|
1732
|
+
switch (frame->phase) {
|
|
1733
|
+
case JSON_PHASE_DONE: return true;
|
|
1734
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1735
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1736
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1737
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1738
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1739
|
+
}
|
|
1740
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
JSON_PHASE_OBJECT_KEY: {
|
|
1744
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1344
1745
|
|
|
1345
|
-
|
|
1346
|
-
const char *final_cursor = state->cursor;
|
|
1347
|
-
state->cursor = object_start_cursor;
|
|
1348
|
-
VALUE object = json_decode_object(state, config, count);
|
|
1349
|
-
state->cursor = final_cursor;
|
|
1746
|
+
json_eat_whitespace(state, config, true);
|
|
1350
1747
|
|
|
1351
|
-
|
|
1748
|
+
const char *start = state->cursor;
|
|
1749
|
+
|
|
1750
|
+
if (RB_LIKELY(peek(state) == '"')) {
|
|
1751
|
+
VALUE string = json_parse_string(state, config, true);
|
|
1752
|
+
if (UNDEF_P(string)) {
|
|
1753
|
+
if (resumable) {
|
|
1754
|
+
state->cursor = start;
|
|
1755
|
+
return false;
|
|
1756
|
+
} else {
|
|
1757
|
+
raise_syntax_error("unexpected end of input, expected closing \"", state);
|
|
1352
1758
|
}
|
|
1759
|
+
}
|
|
1760
|
+
json_push_value(state, config, string);
|
|
1761
|
+
frame->phase = JSON_PHASE_OBJECT_COLON;
|
|
1762
|
+
goto JSON_PHASE_OBJECT_COLON;
|
|
1763
|
+
} else if (resumable && eos(state)) {
|
|
1764
|
+
return false;
|
|
1765
|
+
} else {
|
|
1766
|
+
// The message differs for the first key vs. a key after a
|
|
1767
|
+
// ',': the first is the only one reached with nothing pushed
|
|
1768
|
+
// for this object yet.
|
|
1769
|
+
if (json_frame_entry_count(frame, state->value_stack) == 0) {
|
|
1770
|
+
raise_syntax_error("expected object key, got %s", state);
|
|
1771
|
+
} else {
|
|
1772
|
+
raise_syntax_error("expected object key, got: %s", state);
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1776
|
+
}
|
|
1353
1777
|
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
json_eat_whitespace(state);
|
|
1778
|
+
JSON_PHASE_OBJECT_COLON: {
|
|
1779
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1357
1780
|
|
|
1358
|
-
|
|
1359
|
-
if (peek(state) == '}') {
|
|
1360
|
-
continue;
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1781
|
+
json_eat_whitespace(state, config, true);
|
|
1363
1782
|
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1783
|
+
if (RB_LIKELY(peek(state) == ':')) {
|
|
1784
|
+
state->cursor++;
|
|
1785
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
1786
|
+
goto JSON_PHASE_VALUE;
|
|
1787
|
+
} else if (resumable && eos(state)) {
|
|
1788
|
+
return false;
|
|
1789
|
+
} else {
|
|
1790
|
+
// First colon (only the first pair's key is pushed, nothing
|
|
1791
|
+
// else) vs. a later one.
|
|
1792
|
+
if (json_frame_entry_count(frame, state->value_stack) == 1) {
|
|
1793
|
+
raise_syntax_error("expected ':' after object key", state);
|
|
1794
|
+
} else {
|
|
1795
|
+
raise_syntax_error("expected ':' after object key, got: %s", state);
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1799
|
+
}
|
|
1368
1800
|
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
raise_parse_error("expected ':' after object key, got: %s", state);
|
|
1372
|
-
}
|
|
1373
|
-
state->cursor++;
|
|
1801
|
+
JSON_PHASE_ARRAY_COMMA: {
|
|
1802
|
+
JSON_ASSERT(frame->type == JSON_FRAME_ARRAY);
|
|
1374
1803
|
|
|
1375
|
-
|
|
1804
|
+
json_eat_whitespace(state, config, true);
|
|
1376
1805
|
|
|
1377
|
-
|
|
1378
|
-
}
|
|
1806
|
+
const char next_char = peek(state);
|
|
1379
1807
|
|
|
1380
|
-
|
|
1808
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1809
|
+
state->cursor++;
|
|
1810
|
+
if (config->allow_trailing_comma) {
|
|
1811
|
+
json_eat_whitespace(state, config, true);
|
|
1812
|
+
if (peek(state) == ']') {
|
|
1813
|
+
// Trailing comma: stay in COMMA to close on the next iteration.
|
|
1814
|
+
goto JSON_PHASE_ARRAY_COMMA;
|
|
1815
|
+
}
|
|
1381
1816
|
}
|
|
1382
|
-
|
|
1817
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
1818
|
+
goto JSON_PHASE_VALUE;
|
|
1819
|
+
} else if (next_char == ']') {
|
|
1820
|
+
state->cursor++;
|
|
1821
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
1822
|
+
state->current_nesting--;
|
|
1823
|
+
state->in_array--;
|
|
1824
|
+
|
|
1825
|
+
json_push_value(state, config, json_decode_array(state, config, count));
|
|
1826
|
+
json_frame_stack_pop(state->frames);
|
|
1827
|
+
frame = json_frame_stack_peek(state->frames);
|
|
1828
|
+
|
|
1829
|
+
json_value_completed(frame);
|
|
1830
|
+
|
|
1831
|
+
switch (frame->phase) {
|
|
1832
|
+
case JSON_PHASE_DONE: return true;
|
|
1833
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1834
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1835
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1836
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1837
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1838
|
+
}
|
|
1839
|
+
} else if (resumable && eos(state)) {
|
|
1840
|
+
return false;
|
|
1841
|
+
} else {
|
|
1842
|
+
raise_syntax_error("expected ',' or ']' after array value", state);
|
|
1383
1843
|
}
|
|
1844
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1845
|
+
}
|
|
1384
1846
|
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
break;
|
|
1847
|
+
JSON_PHASE_OBJECT_COMMA: {
|
|
1848
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1388
1849
|
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1850
|
+
json_eat_whitespace(state, config, true);
|
|
1851
|
+
const char next_char = peek(state);
|
|
1852
|
+
|
|
1853
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1854
|
+
state->cursor++;
|
|
1855
|
+
json_eat_whitespace(state, config, true);
|
|
1856
|
+
|
|
1857
|
+
if (config->allow_trailing_comma) {
|
|
1858
|
+
if (peek(state) == '}') {
|
|
1859
|
+
// Trailing comma: stay in COMMA to close on the next iteration.
|
|
1860
|
+
goto JSON_PHASE_OBJECT_COMMA;
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
frame->phase = JSON_PHASE_OBJECT_KEY;
|
|
1865
|
+
goto JSON_PHASE_OBJECT_KEY;
|
|
1866
|
+
} else if (next_char == '}') {
|
|
1867
|
+
state->cursor++;
|
|
1868
|
+
state->current_nesting--;
|
|
1869
|
+
size_t count = json_frame_entry_count(frame, state->value_stack);
|
|
1870
|
+
|
|
1871
|
+
// Temporary rewind cursor in case an error is raised
|
|
1872
|
+
const char *final_cursor = state->cursor;
|
|
1873
|
+
state->cursor = state->start + frame->start_offset;
|
|
1874
|
+
VALUE object = json_decode_object(state, config, count);
|
|
1875
|
+
state->cursor = final_cursor;
|
|
1876
|
+
|
|
1877
|
+
json_push_value(state, config, object);
|
|
1878
|
+
json_frame_stack_pop(state->frames);
|
|
1879
|
+
frame = json_frame_stack_peek(state->frames);
|
|
1880
|
+
json_value_completed(frame);
|
|
1881
|
+
|
|
1882
|
+
switch (frame->phase) {
|
|
1883
|
+
case JSON_PHASE_DONE: return true;
|
|
1884
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1885
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1886
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1887
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1888
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1889
|
+
}
|
|
1890
|
+
} else if (resumable && eos(state)) {
|
|
1891
|
+
return false;
|
|
1892
|
+
} else {
|
|
1893
|
+
raise_syntax_error("expected ',' or '}' after object value, got: %s", state);
|
|
1894
|
+
}
|
|
1895
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1392
1896
|
}
|
|
1393
1897
|
|
|
1394
|
-
|
|
1395
|
-
return Qundef;
|
|
1898
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1396
1899
|
}
|
|
1397
1900
|
|
|
1398
|
-
static void json_ensure_eof(JSON_ParserState *state)
|
|
1901
|
+
static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
1399
1902
|
{
|
|
1400
|
-
json_eat_whitespace(state);
|
|
1903
|
+
json_eat_whitespace(state, config, true);
|
|
1401
1904
|
if (!eos(state)) {
|
|
1402
|
-
|
|
1905
|
+
raise_syntax_error("unexpected token at end of stream %s", state);
|
|
1403
1906
|
}
|
|
1404
1907
|
}
|
|
1405
1908
|
|
|
@@ -1417,40 +1920,59 @@ static void json_ensure_eof(JSON_ParserState *state)
|
|
|
1417
1920
|
|
|
1418
1921
|
static VALUE convert_encoding(VALUE source)
|
|
1419
1922
|
{
|
|
1420
|
-
|
|
1923
|
+
StringValue(source);
|
|
1924
|
+
int encindex = RB_ENCODING_GET(source);
|
|
1925
|
+
|
|
1926
|
+
if (RB_LIKELY(encindex == utf8_encindex)) {
|
|
1927
|
+
return source;
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
if (encindex == binary_encindex) {
|
|
1931
|
+
// For historical reason, we silently reinterpret binary strings as UTF-8
|
|
1932
|
+
return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
|
|
1933
|
+
}
|
|
1421
1934
|
|
|
1422
|
-
|
|
1935
|
+
source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
|
|
1936
|
+
StringValue(source);
|
|
1423
1937
|
return source;
|
|
1424
|
-
|
|
1938
|
+
}
|
|
1425
1939
|
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1940
|
+
struct parser_config_init_args {
|
|
1941
|
+
JSON_ParserConfig *config;
|
|
1942
|
+
VALUE self;
|
|
1943
|
+
VALUE unknown_keywords;
|
|
1944
|
+
bool strict;
|
|
1945
|
+
};
|
|
1430
1946
|
|
|
1431
|
-
|
|
1947
|
+
static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
|
|
1948
|
+
{
|
|
1949
|
+
*dest = val;
|
|
1950
|
+
if (self) RB_OBJ_WRITTEN(self, Qundef, val);
|
|
1432
1951
|
}
|
|
1433
1952
|
|
|
1434
1953
|
static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
1435
1954
|
{
|
|
1436
|
-
|
|
1955
|
+
struct parser_config_init_args *args = (struct parser_config_init_args *)data;
|
|
1956
|
+
JSON_ParserConfig *config = args->config;
|
|
1957
|
+
VALUE self = args->self;
|
|
1437
1958
|
|
|
1438
1959
|
if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
|
|
1439
1960
|
else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
|
|
1440
1961
|
else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
|
|
1962
|
+
else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1441
1963
|
else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
|
|
1442
1964
|
else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
|
|
1443
1965
|
else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
|
|
1444
1966
|
else if (key == sym_freeze) { config->freeze = RTEST(val); }
|
|
1445
|
-
else if (key == sym_on_load) { config->on_load_proc
|
|
1967
|
+
else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
|
|
1446
1968
|
else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1447
1969
|
else if (key == sym_decimal_class) {
|
|
1448
1970
|
if (RTEST(val)) {
|
|
1449
1971
|
if (rb_respond_to(val, i_try_convert)) {
|
|
1450
|
-
config->decimal_class
|
|
1972
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1451
1973
|
config->decimal_method_id = i_try_convert;
|
|
1452
1974
|
} else if (rb_respond_to(val, i_new)) {
|
|
1453
|
-
config->decimal_class
|
|
1975
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1454
1976
|
config->decimal_method_id = i_new;
|
|
1455
1977
|
} else if (RB_TYPE_P(val, T_CLASS)) {
|
|
1456
1978
|
VALUE name = rb_class_name(val);
|
|
@@ -1459,7 +1981,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1459
1981
|
if (last_colon) {
|
|
1460
1982
|
const char *mod_path_end = last_colon - 1;
|
|
1461
1983
|
VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
|
|
1462
|
-
config->decimal_class
|
|
1984
|
+
parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
|
|
1463
1985
|
|
|
1464
1986
|
const char *method_name_beg = last_colon + 1;
|
|
1465
1987
|
long before_len = method_name_beg - name_cstr;
|
|
@@ -1467,28 +1989,48 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1467
1989
|
VALUE method_name = rb_str_substr(name, before_len, len);
|
|
1468
1990
|
config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
|
|
1469
1991
|
} else {
|
|
1470
|
-
config->decimal_class
|
|
1992
|
+
parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
|
|
1471
1993
|
config->decimal_method_id = SYM2ID(rb_str_intern(name));
|
|
1472
1994
|
}
|
|
1473
1995
|
}
|
|
1474
1996
|
}
|
|
1475
1997
|
}
|
|
1998
|
+
else if (args->strict) {
|
|
1999
|
+
if (!args->unknown_keywords) {
|
|
2000
|
+
args->unknown_keywords = rb_obj_hide(rb_ary_new());
|
|
2001
|
+
}
|
|
2002
|
+
rb_ary_push(args->unknown_keywords, key);
|
|
2003
|
+
}
|
|
1476
2004
|
|
|
1477
2005
|
return ST_CONTINUE;
|
|
1478
2006
|
}
|
|
1479
2007
|
|
|
1480
|
-
static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
2008
|
+
static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict)
|
|
1481
2009
|
{
|
|
1482
2010
|
config->max_nesting = 100;
|
|
1483
2011
|
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
2012
|
+
struct parser_config_init_args args = {
|
|
2013
|
+
.config = config,
|
|
2014
|
+
.self = self,
|
|
2015
|
+
.strict = strict,
|
|
2016
|
+
};
|
|
2017
|
+
|
|
2018
|
+
if (NIL_P(opts)) return;
|
|
2019
|
+
Check_Type(opts, T_HASH);
|
|
2020
|
+
if (RHASH_SIZE(opts) == 0) return;
|
|
2021
|
+
|
|
2022
|
+
// We assume in most cases few keys are set so it's faster to go over
|
|
2023
|
+
// the provided keys than to check all possible keys.
|
|
2024
|
+
rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
|
|
1491
2025
|
|
|
2026
|
+
if (RB_UNLIKELY(args.unknown_keywords)) {
|
|
2027
|
+
if (RARRAY_LEN(args.unknown_keywords) == 1) {
|
|
2028
|
+
rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0));
|
|
2029
|
+
}
|
|
2030
|
+
else {
|
|
2031
|
+
VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", "));
|
|
2032
|
+
rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords);
|
|
2033
|
+
}
|
|
1492
2034
|
}
|
|
1493
2035
|
}
|
|
1494
2036
|
|
|
@@ -1497,67 +2039,90 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
|
1497
2039
|
*
|
|
1498
2040
|
* Creates a new JSON::Ext::ParserConfig instance.
|
|
1499
2041
|
*
|
|
1500
|
-
*
|
|
1501
|
-
*
|
|
2042
|
+
* Argument +opts+, if given, contains a \Hash of options for the parsing.
|
|
2043
|
+
* See {Parsing Options}[#module-JSON-label-Parsing+Options].
|
|
1502
2044
|
*
|
|
1503
|
-
* _opts_ can have the following keys:
|
|
1504
|
-
* * *max_nesting*: The maximum depth of nesting allowed in the parsed data
|
|
1505
|
-
* structures. Disable depth checking with :max_nesting => false|nil|0, it
|
|
1506
|
-
* defaults to 100.
|
|
1507
|
-
* * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
|
|
1508
|
-
* defiance of RFC 4627 to be parsed by the Parser. This option defaults to
|
|
1509
|
-
* false.
|
|
1510
|
-
* * *symbolize_names*: If set to true, returns symbols for the names
|
|
1511
|
-
* (keys) in a JSON object. Otherwise strings are returned, which is
|
|
1512
|
-
* also the default. It's not possible to use this option in
|
|
1513
|
-
* conjunction with the *create_additions* option.
|
|
1514
|
-
* * *decimal_class*: Specifies which class to use instead of the default
|
|
1515
|
-
* (Float) when parsing decimal numbers. This class must accept a single
|
|
1516
|
-
* string argument in its constructor.
|
|
1517
2045
|
*/
|
|
1518
2046
|
static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
|
|
1519
2047
|
{
|
|
1520
2048
|
rb_check_frozen(self);
|
|
1521
2049
|
GET_PARSER_CONFIG;
|
|
1522
2050
|
|
|
1523
|
-
parser_config_init(config, opts);
|
|
1524
|
-
|
|
1525
|
-
RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
|
|
2051
|
+
parser_config_init(config, opts, self, false);
|
|
1526
2052
|
|
|
1527
2053
|
return self;
|
|
1528
2054
|
}
|
|
1529
2055
|
|
|
1530
|
-
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE
|
|
2056
|
+
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
|
|
1531
2057
|
{
|
|
1532
|
-
Vsource = convert_encoding(
|
|
1533
|
-
|
|
2058
|
+
VALUE Vsource = convert_encoding(src);
|
|
2059
|
+
|
|
2060
|
+
// Ensure the string isn't mutated under us.
|
|
2061
|
+
// The classic API to use is `rb_str_locktmp`, but then we'd
|
|
2062
|
+
// need to use `rb_protect` to make sure we always unlock.
|
|
2063
|
+
if (Vsource == src) {
|
|
2064
|
+
Vsource = rb_str_new_frozen(Vsource);
|
|
2065
|
+
}
|
|
1534
2066
|
|
|
1535
2067
|
VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
|
|
1536
|
-
rvalue_stack
|
|
2068
|
+
rvalue_stack value_stack = {
|
|
1537
2069
|
.type = RVALUE_STACK_STACK_ALLOCATED,
|
|
1538
2070
|
.ptr = rvalue_stack_buffer,
|
|
1539
2071
|
.capa = RVALUE_STACK_INITIAL_CAPA,
|
|
1540
2072
|
};
|
|
1541
2073
|
|
|
2074
|
+
// Seed the frame stack with the root frame, establishing the invariant that
|
|
2075
|
+
// json_parse_any always has a top frame to dispatch on (so the stack is never
|
|
2076
|
+
// empty mid-parse).
|
|
2077
|
+
json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA];
|
|
2078
|
+
frame_stack_buffer[0] = (json_frame){
|
|
2079
|
+
.type = JSON_FRAME_ROOT,
|
|
2080
|
+
.phase = JSON_PHASE_VALUE,
|
|
2081
|
+
};
|
|
2082
|
+
json_frame_stack frames = {
|
|
2083
|
+
.type = RVALUE_STACK_STACK_ALLOCATED,
|
|
2084
|
+
.ptr = frame_stack_buffer,
|
|
2085
|
+
.capa = JSON_FRAME_STACK_INITIAL_CAPA,
|
|
2086
|
+
.head = 1,
|
|
2087
|
+
};
|
|
2088
|
+
|
|
1542
2089
|
long len;
|
|
1543
2090
|
const char *start;
|
|
2091
|
+
|
|
1544
2092
|
RSTRING_GETMEM(Vsource, start, len);
|
|
1545
2093
|
|
|
2094
|
+
VALUE value_stack_handle = 0;
|
|
2095
|
+
VALUE frame_stack_handle = 0;
|
|
1546
2096
|
JSON_ParserState _state = {
|
|
1547
2097
|
.start = start,
|
|
1548
2098
|
.cursor = start,
|
|
1549
2099
|
.end = start + len,
|
|
1550
|
-
.
|
|
2100
|
+
.value_stack = &value_stack,
|
|
2101
|
+
.value_stack_handle = &value_stack_handle,
|
|
2102
|
+
.frames = &frames,
|
|
2103
|
+
.frame_stack_handle = &frame_stack_handle,
|
|
1551
2104
|
};
|
|
1552
2105
|
JSON_ParserState *state = &_state;
|
|
1553
2106
|
|
|
1554
|
-
|
|
2107
|
+
bool complete = json_parse_any(state, config, false);
|
|
2108
|
+
|
|
2109
|
+
// The root document value is parsed; it is the lone survivor on
|
|
2110
|
+
// the rvalue stack.
|
|
2111
|
+
VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
|
|
1555
2112
|
|
|
1556
2113
|
// This may be skipped in case of exception, but
|
|
1557
2114
|
// it won't cause a leak.
|
|
1558
|
-
rvalue_stack_eagerly_release(
|
|
1559
|
-
|
|
1560
|
-
|
|
2115
|
+
rvalue_stack_eagerly_release(value_stack_handle);
|
|
2116
|
+
json_frame_stack_eagerly_release(frame_stack_handle);
|
|
2117
|
+
RB_GC_GUARD(value_stack_handle);
|
|
2118
|
+
RB_GC_GUARD(frame_stack_handle);
|
|
2119
|
+
RB_GC_GUARD(Vsource);
|
|
2120
|
+
|
|
2121
|
+
if (complete) {
|
|
2122
|
+
json_ensure_eof(state, config);
|
|
2123
|
+
} else {
|
|
2124
|
+
raise_eos_error("unexpected end of input", state);
|
|
2125
|
+
}
|
|
1561
2126
|
|
|
1562
2127
|
return result;
|
|
1563
2128
|
}
|
|
@@ -1577,12 +2142,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
|
|
|
1577
2142
|
|
|
1578
2143
|
static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
1579
2144
|
{
|
|
1580
|
-
Vsource = convert_encoding(StringValue(Vsource));
|
|
1581
|
-
StringValue(Vsource);
|
|
1582
|
-
|
|
1583
2145
|
JSON_ParserConfig _config = {0};
|
|
1584
2146
|
JSON_ParserConfig *config = &_config;
|
|
1585
|
-
parser_config_init(config, opts);
|
|
2147
|
+
parser_config_init(config, opts, Qfalse, false);
|
|
1586
2148
|
|
|
1587
2149
|
return cParser_parse(config, Vsource);
|
|
1588
2150
|
}
|
|
@@ -1590,30 +2152,35 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
|
1590
2152
|
static void JSON_ParserConfig_mark(void *ptr)
|
|
1591
2153
|
{
|
|
1592
2154
|
JSON_ParserConfig *config = ptr;
|
|
1593
|
-
|
|
1594
|
-
|
|
2155
|
+
rb_gc_mark_movable(config->on_load_proc);
|
|
2156
|
+
rb_gc_mark_movable(config->decimal_class);
|
|
1595
2157
|
}
|
|
1596
2158
|
|
|
1597
|
-
static
|
|
2159
|
+
static size_t JSON_ParserConfig_memsize(const void *ptr)
|
|
1598
2160
|
{
|
|
1599
|
-
|
|
1600
|
-
|
|
2161
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
2162
|
+
return 0;
|
|
2163
|
+
#else
|
|
2164
|
+
return sizeof(JSON_ParserConfig);
|
|
2165
|
+
#endif
|
|
1601
2166
|
}
|
|
1602
2167
|
|
|
1603
|
-
static
|
|
2168
|
+
static void JSON_ParserConfig_compact(void *ptr)
|
|
1604
2169
|
{
|
|
1605
|
-
|
|
2170
|
+
JSON_ParserConfig *config = ptr;
|
|
2171
|
+
config->on_load_proc = rb_gc_location(config->on_load_proc);
|
|
2172
|
+
config->decimal_class = rb_gc_location(config->decimal_class);
|
|
1606
2173
|
}
|
|
1607
2174
|
|
|
1608
2175
|
static const rb_data_type_t JSON_ParserConfig_type = {
|
|
1609
|
-
"JSON::Ext::Parser/ParserConfig",
|
|
1610
|
-
{
|
|
1611
|
-
JSON_ParserConfig_mark,
|
|
1612
|
-
|
|
1613
|
-
JSON_ParserConfig_memsize,
|
|
2176
|
+
.wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
|
|
2177
|
+
.function = {
|
|
2178
|
+
.dmark = JSON_ParserConfig_mark,
|
|
2179
|
+
.dfree = RUBY_DEFAULT_FREE,
|
|
2180
|
+
.dsize = JSON_ParserConfig_memsize,
|
|
2181
|
+
.dcompact = JSON_ParserConfig_compact,
|
|
1614
2182
|
},
|
|
1615
|
-
|
|
1616
|
-
RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE,
|
|
2183
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
|
|
1617
2184
|
};
|
|
1618
2185
|
|
|
1619
2186
|
static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
@@ -1622,6 +2189,562 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
|
1622
2189
|
return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config);
|
|
1623
2190
|
}
|
|
1624
2191
|
|
|
2192
|
+
static void json_str_clear(VALUE str)
|
|
2193
|
+
{
|
|
2194
|
+
if (RB_OBJ_FROZEN_RAW(str)) {
|
|
2195
|
+
return;
|
|
2196
|
+
}
|
|
2197
|
+
rb_str_replace(str, JSON_empty_string);
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
typedef struct JSON_ResumableParserStruct {
|
|
2201
|
+
JSON_ParserConfig config;
|
|
2202
|
+
JSON_ParserState state;
|
|
2203
|
+
rvalue_stack value_stack;
|
|
2204
|
+
json_frame_stack frames;
|
|
2205
|
+
VALUE buffer;
|
|
2206
|
+
size_t parsed_bytes;
|
|
2207
|
+
size_t incomplete_bytes;
|
|
2208
|
+
bool complete;
|
|
2209
|
+
bool in_use;
|
|
2210
|
+
} JSON_ResumableParser;
|
|
2211
|
+
|
|
2212
|
+
static void JSON_ResumableParser_mark(void *ptr)
|
|
2213
|
+
{
|
|
2214
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2215
|
+
JSON_ParserConfig_mark(&parser->config);
|
|
2216
|
+
rvalue_stack_mark(&parser->value_stack);
|
|
2217
|
+
rvalue_cache_mark(&parser->state.name_cache);
|
|
2218
|
+
rb_gc_mark(parser->buffer); // pin the buffer
|
|
2219
|
+
rb_gc_mark_movable(parser->state.parser);
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
static void JSON_ResumableParser_free(void *ptr)
|
|
2223
|
+
{
|
|
2224
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2225
|
+
rvalue_stack_free_buffer(&parser->value_stack);
|
|
2226
|
+
json_frame_stack_free_buffer(&parser->frames);
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
static size_t JSON_ResumableParser_memsize(const void *ptr)
|
|
2230
|
+
{
|
|
2231
|
+
const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr;
|
|
2232
|
+
size_t memsize = JSON_ParserConfig_memsize(&parser->config);
|
|
2233
|
+
memsize += rvalue_stack_memsize(&parser->value_stack);
|
|
2234
|
+
memsize += json_frame_stack_memsize(&parser->frames);
|
|
2235
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
2236
|
+
memsize += (
|
|
2237
|
+
sizeof(JSON_ResumableParser)
|
|
2238
|
+
- sizeof(JSON_ParserState)
|
|
2239
|
+
- sizeof(JSON_ParserConfig)
|
|
2240
|
+
- sizeof(rvalue_stack)
|
|
2241
|
+
- sizeof(json_frame_stack)
|
|
2242
|
+
);
|
|
2243
|
+
#endif
|
|
2244
|
+
return memsize;
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
static void JSON_ResumableParser_compact(void *ptr)
|
|
2248
|
+
{
|
|
2249
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2250
|
+
JSON_ParserConfig_compact(&parser->config);
|
|
2251
|
+
rvalue_stack_compact(&parser->value_stack);
|
|
2252
|
+
rvalue_cache_compact(&parser->state.name_cache);
|
|
2253
|
+
parser->buffer = rb_gc_location(parser->buffer);
|
|
2254
|
+
parser->state.parser = rb_gc_location(parser->state.parser);
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
static const rb_data_type_t JSON_ResumableParser_type = {
|
|
2258
|
+
.wrap_struct_name = "JSON::Ext::ResumableParser",
|
|
2259
|
+
.function = {
|
|
2260
|
+
JSON_ResumableParser_mark,
|
|
2261
|
+
JSON_ResumableParser_free,
|
|
2262
|
+
JSON_ResumableParser_memsize,
|
|
2263
|
+
JSON_ResumableParser_compact,
|
|
2264
|
+
},
|
|
2265
|
+
// RUBY_TYPED_WB_PROTECTED is deliberately not declared because
|
|
2266
|
+
// this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
|
|
2267
|
+
// to trigger a lot of write barriers.
|
|
2268
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
|
|
2269
|
+
};
|
|
2270
|
+
|
|
2271
|
+
static VALUE cResumableParser_allocate(VALUE klass)
|
|
2272
|
+
{
|
|
2273
|
+
JSON_ResumableParser *parser;
|
|
2274
|
+
VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
|
|
2275
|
+
parser->state.in_array++;
|
|
2276
|
+
parser->state.parser = obj;
|
|
2277
|
+
return obj;
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
static inline JSON_ResumableParser *cResumableParser_get(VALUE self)
|
|
2281
|
+
{
|
|
2282
|
+
JSON_ResumableParser *parser;
|
|
2283
|
+
TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
|
|
2284
|
+
return parser;
|
|
2285
|
+
}
|
|
2286
|
+
|
|
2287
|
+
/*
|
|
2288
|
+
* call-seq: new(opts => {})
|
|
2289
|
+
*
|
|
2290
|
+
* Creates a new JSON::ResumableParser instance.
|
|
2291
|
+
*
|
|
2292
|
+
* Argument +opts+, if given, contains a \Hash of options for the parsing.
|
|
2293
|
+
* See {Parsing Options}[#module-JSON-label-Parsing+Options].
|
|
2294
|
+
*
|
|
2295
|
+
* A ResumableParser is able to parse partial documents and resume parsing later
|
|
2296
|
+
* when more of the document is provided:
|
|
2297
|
+
*
|
|
2298
|
+
* parser = JSON::ResumableParser.new
|
|
2299
|
+
* parser << '{"user": "george", "role": "ad'
|
|
2300
|
+
* parser.parse # => false
|
|
2301
|
+
* parser.eos? # => true
|
|
2302
|
+
* parser.partial_value # => { "user" => "george", "role" => nil }
|
|
2303
|
+
* parser.rest # => '"ad'
|
|
2304
|
+
*
|
|
2305
|
+
* parser << 'min" }[1, 2, 3]'
|
|
2306
|
+
* parser.parse # => true
|
|
2307
|
+
* parser.value # => { "user" => "george", "role" => "admin" }
|
|
2308
|
+
*
|
|
2309
|
+
* parser.parse # => true
|
|
2310
|
+
* parser.value # => [1, 2, 3]
|
|
2311
|
+
*
|
|
2312
|
+
* === Limitations
|
|
2313
|
+
*
|
|
2314
|
+
* While ResumableParser is able to parse streams of documents without any
|
|
2315
|
+
* explicit separators between them, it is highly recommended to separate documents
|
|
2316
|
+
* by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous.
|
|
2317
|
+
* When parsing a number, ResumableParser will not consider the number complete until something follows:
|
|
2318
|
+
*
|
|
2319
|
+
* parser << '123'
|
|
2320
|
+
* parser.parse # => false
|
|
2321
|
+
* parser << ' '
|
|
2322
|
+
* parser.parse # => true
|
|
2323
|
+
* parser.value # => 123
|
|
2324
|
+
*
|
|
2325
|
+
* === Security
|
|
2326
|
+
*
|
|
2327
|
+
* An incomplete document is buffered in full and there is no size limit, so when reading
|
|
2328
|
+
* from an untrusted source the caller is responsible for bounding how much data is fed.
|
|
2329
|
+
* For example:
|
|
2330
|
+
*
|
|
2331
|
+
* loop do
|
|
2332
|
+
* if parser.parsed_bytes > DOCUMENT_MAX_SIZE
|
|
2333
|
+
* raise "document too large"
|
|
2334
|
+
* end
|
|
2335
|
+
*
|
|
2336
|
+
* parser << read_chunk
|
|
2337
|
+
* while parser.parse
|
|
2338
|
+
* process(parser.value)
|
|
2339
|
+
* end
|
|
2340
|
+
* end
|
|
2341
|
+
*/
|
|
2342
|
+
static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
|
|
2343
|
+
{
|
|
2344
|
+
rb_check_frozen(self);
|
|
2345
|
+
|
|
2346
|
+
VALUE opts = Qfalse;
|
|
2347
|
+
rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts);
|
|
2348
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2349
|
+
|
|
2350
|
+
opts = argc > 0 ? argv[0] : Qnil;
|
|
2351
|
+
parser_config_init(&parser->config, opts, self, true);
|
|
2352
|
+
|
|
2353
|
+
return self;
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock);
|
|
2357
|
+
|
|
2358
|
+
/*
|
|
2359
|
+
* call-seq: self << string -> self
|
|
2360
|
+
*
|
|
2361
|
+
* Appends the given string to the parser's buffer.
|
|
2362
|
+
*/
|
|
2363
|
+
static VALUE cResumableParser_feed(VALUE self, VALUE str)
|
|
2364
|
+
{
|
|
2365
|
+
rb_check_frozen(self);
|
|
2366
|
+
|
|
2367
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2368
|
+
|
|
2369
|
+
str = convert_encoding(str);
|
|
2370
|
+
if (!RSTRING_LEN(str)) {
|
|
2371
|
+
return self;
|
|
2372
|
+
}
|
|
2373
|
+
|
|
2374
|
+
size_t offset = parser->state.cursor - parser->state.start;
|
|
2375
|
+
const size_t remaining = parser->state.end - parser->state.cursor;
|
|
2376
|
+
|
|
2377
|
+
if (!remaining) {
|
|
2378
|
+
if (parser->buffer) {
|
|
2379
|
+
json_str_clear(parser->buffer);
|
|
2380
|
+
}
|
|
2381
|
+
parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
|
|
2382
|
+
offset = 0;
|
|
2383
|
+
} else {
|
|
2384
|
+
JSON_ASSERT(parser->buffer);
|
|
2385
|
+
|
|
2386
|
+
const size_t size = parser->state.end - parser->state.start;
|
|
2387
|
+
const size_t consumed = size - remaining;
|
|
2388
|
+
|
|
2389
|
+
if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
|
|
2390
|
+
VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
|
|
2391
|
+
rb_enc_associate_index(new_buffer, utf8_encindex);
|
|
2392
|
+
|
|
2393
|
+
char *old_ptr = RSTRING_PTR(parser->buffer);
|
|
2394
|
+
memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
|
|
2395
|
+
rb_str_set_len(new_buffer, remaining);
|
|
2396
|
+
offset = 0;
|
|
2397
|
+
parser->buffer = new_buffer;
|
|
2398
|
+
} else if (consumed > (size / 2) && size >= 512) {
|
|
2399
|
+
rb_str_modify(parser->buffer);
|
|
2400
|
+
char *old_ptr = RSTRING_PTR(parser->buffer);
|
|
2401
|
+
memmove(old_ptr, old_ptr + consumed, remaining);
|
|
2402
|
+
rb_str_set_len(parser->buffer, remaining);
|
|
2403
|
+
offset = 0;
|
|
2404
|
+
}
|
|
2405
|
+
rb_str_append(parser->buffer, str);
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
long len;
|
|
2409
|
+
const char *start;
|
|
2410
|
+
RSTRING_GETMEM(parser->buffer, start, len);
|
|
2411
|
+
parser->state.start = start;
|
|
2412
|
+
parser->state.end = start + len;
|
|
2413
|
+
parser->state.cursor = parser->state.start + offset;
|
|
2414
|
+
|
|
2415
|
+
return self;
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
struct json_parse_any_args {
|
|
2419
|
+
JSON_ParserState *state;
|
|
2420
|
+
JSON_ParserConfig *config;
|
|
2421
|
+
VALUE parser;
|
|
2422
|
+
};
|
|
2423
|
+
|
|
2424
|
+
static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args))
|
|
2425
|
+
{
|
|
2426
|
+
struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
|
|
2427
|
+
return (VALUE)json_parse_any(args->state, args->config, true);
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
static VALUE json_parse_any_resumable_safe(VALUE _args)
|
|
2431
|
+
{
|
|
2432
|
+
struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
|
|
2433
|
+
VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args);
|
|
2434
|
+
return result == args->parser ? Qfalse : result;
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock)
|
|
2438
|
+
{
|
|
2439
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2440
|
+
|
|
2441
|
+
if (parser->in_use) {
|
|
2442
|
+
rb_raise(rb_eArgError, "ResumableParser can't be used recursively");
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
if (lock) {
|
|
2446
|
+
parser->in_use = true;
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
// self may have moved, so we need to update all pointers
|
|
2450
|
+
// Investigate: We might be better off keeping JSON_ParserState on the stack
|
|
2451
|
+
// and only persist what we need.
|
|
2452
|
+
parser->state.value_stack = &parser->value_stack;
|
|
2453
|
+
parser->state.frames = &parser->frames;
|
|
2454
|
+
|
|
2455
|
+
return parser;
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
/*
|
|
2459
|
+
* call-seq: parse -> true or false
|
|
2460
|
+
*
|
|
2461
|
+
* Attemps to parse a JSON document from the internal buffer.
|
|
2462
|
+
* Returns whether a complete document could be parsed.
|
|
2463
|
+
*
|
|
2464
|
+
* It does raise +JSON::ParserError+ when encountering invalid \JSON syntax.
|
|
2465
|
+
*
|
|
2466
|
+
* The parsed object can be retrieved by calling #value
|
|
2467
|
+
*/
|
|
2468
|
+
static VALUE cResumableParser_parse(VALUE self)
|
|
2469
|
+
{
|
|
2470
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
|
|
2471
|
+
|
|
2472
|
+
if (parser->complete) {
|
|
2473
|
+
parser->parsed_bytes = 0;
|
|
2474
|
+
parser->incomplete_bytes = 0;
|
|
2475
|
+
parser->complete = false;
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
if (!parser->buffer) {
|
|
2479
|
+
parser->in_use = false;
|
|
2480
|
+
return Qfalse;
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
if (parser->frames.head == 0) {
|
|
2484
|
+
json_frame_stack_push(&parser->state, (json_frame){
|
|
2485
|
+
.type = JSON_FRAME_ROOT,
|
|
2486
|
+
.phase = JSON_PHASE_VALUE,
|
|
2487
|
+
});
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2490
|
+
VALUE Vsource = parser->buffer; // Prevent compaction
|
|
2491
|
+
|
|
2492
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2493
|
+
|
|
2494
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2495
|
+
JSON_ASSERT(parser->value_stack.head == 1);
|
|
2496
|
+
JSON_ASSERT(parser->frames.head == 1);
|
|
2497
|
+
|
|
2498
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
2499
|
+
rvalue_stack_pop(parser->state.value_stack, 1);
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
struct json_parse_any_args args = {
|
|
2503
|
+
.state = &parser->state,
|
|
2504
|
+
.config = &parser->config,
|
|
2505
|
+
.parser = self,
|
|
2506
|
+
};
|
|
2507
|
+
int status;
|
|
2508
|
+
const char *initial_cursor = parser->state.cursor;
|
|
2509
|
+
parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
|
|
2510
|
+
|
|
2511
|
+
if (status) {
|
|
2512
|
+
parser->complete = true; // a parse error is considered complete
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
parser->parsed_bytes += parser->state.cursor - initial_cursor;
|
|
2516
|
+
parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;
|
|
2517
|
+
|
|
2518
|
+
json_eat_whitespace(&parser->state, &parser->config, false);
|
|
2519
|
+
if (eos(&parser->state)) {
|
|
2520
|
+
json_str_clear(parser->buffer);
|
|
2521
|
+
parser->buffer = Qfalse;
|
|
2522
|
+
}
|
|
2523
|
+
parser->in_use = false;
|
|
2524
|
+
|
|
2525
|
+
if (status) {
|
|
2526
|
+
rb_jump_tag(status); // reraise
|
|
2527
|
+
}
|
|
2528
|
+
RB_GC_GUARD(Vsource);
|
|
2529
|
+
return parser->complete ? Qtrue : Qfalse;
|
|
2530
|
+
}
|
|
2531
|
+
|
|
2532
|
+
/*
|
|
2533
|
+
* call-seq: value? -> true or false
|
|
2534
|
+
*
|
|
2535
|
+
* Returns whether a parsed value is available.
|
|
2536
|
+
*/
|
|
2537
|
+
static VALUE cResumableParser_value_p(VALUE self)
|
|
2538
|
+
{
|
|
2539
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2540
|
+
|
|
2541
|
+
if (parser->value_stack.head > 0) {
|
|
2542
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2543
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2544
|
+
return Qtrue;
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
return Qfalse;
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
/*
|
|
2551
|
+
* call-seq: value -> object
|
|
2552
|
+
*
|
|
2553
|
+
* Returns and consume the last parsed value.
|
|
2554
|
+
* Raises ArgumentError if there is no parsed value or if it was already retrieved:
|
|
2555
|
+
* parser << '[1][2]'
|
|
2556
|
+
* parser.value # ArgumentError no ready value
|
|
2557
|
+
* parser.parse # => true
|
|
2558
|
+
* parser.value # => [1]
|
|
2559
|
+
* parser.value # ArgumentError no ready value
|
|
2560
|
+
*/
|
|
2561
|
+
static VALUE cResumableParser_value(VALUE self)
|
|
2562
|
+
{
|
|
2563
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2564
|
+
|
|
2565
|
+
if (parser->frames.head > 0) {
|
|
2566
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2567
|
+
|
|
2568
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2569
|
+
VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
|
|
2570
|
+
rvalue_stack_pop(parser->state.value_stack, 1);
|
|
2571
|
+
json_frame_stack_pop(parser->state.frames);
|
|
2572
|
+
return result;
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
rb_raise(rb_eArgError, "no ready value");
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
/*
|
|
2579
|
+
* call-seq: clear -> self
|
|
2580
|
+
*
|
|
2581
|
+
* Entirely reset the parser state and buffer.
|
|
2582
|
+
*/
|
|
2583
|
+
static VALUE cResumableParser_clear(VALUE self)
|
|
2584
|
+
{
|
|
2585
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2586
|
+
parser->buffer = 0;
|
|
2587
|
+
parser->complete = true;
|
|
2588
|
+
parser->parsed_bytes = 0;
|
|
2589
|
+
parser->incomplete_bytes = 0;
|
|
2590
|
+
parser->frames.head = 0;
|
|
2591
|
+
parser->value_stack.head = 0;
|
|
2592
|
+
parser->state.name_cache.length = 0;
|
|
2593
|
+
parser->state.current_nesting = 0;
|
|
2594
|
+
parser->state.in_array = 1;
|
|
2595
|
+
parser->state.emitted_deprecations = 0;
|
|
2596
|
+
parser->state.start = parser->state.cursor = parser->state.end = NULL;
|
|
2597
|
+
return self;
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
static VALUE cResumableParser_partial_value_body(VALUE self)
|
|
2601
|
+
{
|
|
2602
|
+
JSON_ResumableParser *original_parser = cResumableParser_get(self);
|
|
2603
|
+
JSON_ResumableParser parser = *original_parser;
|
|
2604
|
+
|
|
2605
|
+
parser.state.frames = &parser.frames;
|
|
2606
|
+
parser.state.value_stack = &parser.value_stack;
|
|
2607
|
+
|
|
2608
|
+
if (parser.value_stack.head == 0) {
|
|
2609
|
+
return Qnil;
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
json_frame *frame = json_frame_stack_peek(parser.state.frames);
|
|
2613
|
+
long missing_object_value = 0;
|
|
2614
|
+
if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
|
|
2615
|
+
missing_object_value = 1;
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
// Copy the value stack as we need to mutate it.
|
|
2619
|
+
long capa = parser.value_stack.head;
|
|
2620
|
+
parser.value_stack.capa = (capa + missing_object_value);
|
|
2621
|
+
VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, capa + missing_object_value);
|
|
2622
|
+
MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, parser.value_stack.capa);
|
|
2623
|
+
parser.value_stack.ptr = value_stack_buffer;
|
|
2624
|
+
|
|
2625
|
+
JSON_ParserState *state = &parser.state;
|
|
2626
|
+
JSON_ParserConfig *config = &parser.config;
|
|
2627
|
+
|
|
2628
|
+
if (missing_object_value) {
|
|
2629
|
+
rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
VALUE partial_result = Qundef;
|
|
2633
|
+
|
|
2634
|
+
while (UNDEF_P(partial_result)) {
|
|
2635
|
+
frame = json_frame_stack_peek(state->frames);
|
|
2636
|
+
|
|
2637
|
+
switch (frame->type) {
|
|
2638
|
+
case JSON_FRAME_ROOT: {
|
|
2639
|
+
partial_result = *rvalue_stack_peek(state->value_stack, 1);
|
|
2640
|
+
break;
|
|
2641
|
+
}
|
|
2642
|
+
|
|
2643
|
+
case JSON_FRAME_ARRAY: {
|
|
2644
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
2645
|
+
json_push_value(state, config, json_decode_array(state, config, count));
|
|
2646
|
+
json_frame_stack_pop(state->frames);
|
|
2647
|
+
|
|
2648
|
+
break;
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
case JSON_FRAME_OBJECT: {
|
|
2652
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
2653
|
+
json_push_value(state, config, json_decode_object(state, config, count));
|
|
2654
|
+
json_frame_stack_pop(state->frames);
|
|
2655
|
+
break;
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2658
|
+
default: {
|
|
2659
|
+
JSON_UNREACHABLE_RETURN(Qundef);
|
|
2660
|
+
break;
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
ALLOCV_END(tmpbuf);
|
|
2666
|
+
return partial_result;
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
/*
|
|
2670
|
+
* call-seq: partial_value -> object
|
|
2671
|
+
*
|
|
2672
|
+
* Returns the Ruby objects parsed up to this point:
|
|
2673
|
+
* parser << '[1, [2, 3,'
|
|
2674
|
+
* parser.parse # => false
|
|
2675
|
+
* parser.value # ArgumentError no ready value
|
|
2676
|
+
* parser.partial_value # => [1, [2, 3]]
|
|
2677
|
+
*/
|
|
2678
|
+
static VALUE cResumableParser_partial_value(VALUE self)
|
|
2679
|
+
{
|
|
2680
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
|
|
2681
|
+
|
|
2682
|
+
int status;
|
|
2683
|
+
VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
|
|
2684
|
+
parser->in_use = false;
|
|
2685
|
+
if (status) {
|
|
2686
|
+
rb_jump_tag(status);
|
|
2687
|
+
}
|
|
2688
|
+
return result;
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
/*
|
|
2692
|
+
* call-seq: rest -> string
|
|
2693
|
+
*
|
|
2694
|
+
* Returns a string containing what remains to be parsed in the buffer
|
|
2695
|
+
* parser << '{ "message": "unterminated message'
|
|
2696
|
+
* parser.parse # => false
|
|
2697
|
+
* parser.rest # => '"unterminated message"'
|
|
2698
|
+
*/
|
|
2699
|
+
static VALUE cResumableParser_rest(VALUE self)
|
|
2700
|
+
{
|
|
2701
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2702
|
+
|
|
2703
|
+
if (!parser->buffer) {
|
|
2704
|
+
return rb_utf8_str_new("", 0);
|
|
2705
|
+
}
|
|
2706
|
+
|
|
2707
|
+
size_t offset = parser->state.cursor - parser->state.start;
|
|
2708
|
+
const char *ptr;
|
|
2709
|
+
long len;
|
|
2710
|
+
RSTRING_GETMEM(parser->buffer, ptr, len);
|
|
2711
|
+
return rb_utf8_str_new(ptr + offset, len - offset);
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
/*
|
|
2715
|
+
* call-seq: value? -> true or false
|
|
2716
|
+
*
|
|
2717
|
+
* Returns whether the internal buffer has been entirely consumed.
|
|
2718
|
+
*/
|
|
2719
|
+
static VALUE cResumableParser_eos_p(VALUE self)
|
|
2720
|
+
{
|
|
2721
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2722
|
+
return eos(&parser->state) ? Qtrue : Qfalse;
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
/*
|
|
2726
|
+
* call-seq: parsed_bytes -> integer
|
|
2727
|
+
*
|
|
2728
|
+
* Returns the number of bytes parsed since the start of the current partial value.
|
|
2729
|
+
* This is intended to be used for securing against untrusted input:
|
|
2730
|
+
*
|
|
2731
|
+
* loop do
|
|
2732
|
+
* if parser.parsed_bytes > DOCUMENT_MAX_SIZE
|
|
2733
|
+
* raise "document too large"
|
|
2734
|
+
* end
|
|
2735
|
+
*
|
|
2736
|
+
* parser << read_chunk
|
|
2737
|
+
* while parser.parse
|
|
2738
|
+
* process(parser.value)
|
|
2739
|
+
* end
|
|
2740
|
+
* end
|
|
2741
|
+
*/
|
|
2742
|
+
static VALUE cResumableParser_parsed_bytes(VALUE self)
|
|
2743
|
+
{
|
|
2744
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2745
|
+
return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
|
|
2746
|
+
}
|
|
2747
|
+
|
|
1625
2748
|
void Init_parser(void)
|
|
1626
2749
|
{
|
|
1627
2750
|
#ifdef HAVE_RB_EXT_RACTOR_SAFE
|
|
@@ -1633,30 +2756,52 @@ void Init_parser(void)
|
|
|
1633
2756
|
mJSON = rb_define_module("JSON");
|
|
1634
2757
|
VALUE mExt = rb_define_module_under(mJSON, "Ext");
|
|
1635
2758
|
VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject);
|
|
2759
|
+
|
|
2760
|
+
rb_global_variable(&eParserError);
|
|
2761
|
+
eParserError = rb_path2class("JSON::ParserError");
|
|
2762
|
+
|
|
2763
|
+
rb_global_variable(&eNestingError);
|
|
1636
2764
|
eNestingError = rb_path2class("JSON::NestingError");
|
|
1637
|
-
|
|
2765
|
+
|
|
1638
2766
|
rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate);
|
|
1639
|
-
|
|
2767
|
+
rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
|
|
1640
2768
|
rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1);
|
|
1641
2769
|
|
|
1642
2770
|
VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject);
|
|
1643
2771
|
rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2);
|
|
1644
2772
|
|
|
2773
|
+
VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject);
|
|
2774
|
+
rb_define_alloc_func(cResumableParser, cResumableParser_allocate);
|
|
2775
|
+
rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1);
|
|
2776
|
+
rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1);
|
|
2777
|
+
rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0);
|
|
2778
|
+
rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
|
|
2779
|
+
rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
|
|
2780
|
+
rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
|
|
2781
|
+
rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
|
|
2782
|
+
rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
|
|
2783
|
+
rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
|
|
2784
|
+
rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0);
|
|
2785
|
+
|
|
2786
|
+
rb_global_variable(&CNaN);
|
|
1645
2787
|
CNaN = rb_const_get(mJSON, rb_intern("NaN"));
|
|
1646
|
-
rb_gc_register_mark_object(CNaN);
|
|
1647
2788
|
|
|
2789
|
+
rb_global_variable(&CInfinity);
|
|
1648
2790
|
CInfinity = rb_const_get(mJSON, rb_intern("Infinity"));
|
|
1649
|
-
rb_gc_register_mark_object(CInfinity);
|
|
1650
2791
|
|
|
2792
|
+
rb_global_variable(&CMinusInfinity);
|
|
1651
2793
|
CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity"));
|
|
1652
|
-
rb_gc_register_mark_object(CMinusInfinity);
|
|
1653
2794
|
|
|
1654
2795
|
rb_global_variable(&Encoding_UTF_8);
|
|
1655
2796
|
Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
|
|
1656
2797
|
|
|
2798
|
+
rb_global_variable(&JSON_empty_string);
|
|
2799
|
+
JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0));
|
|
2800
|
+
|
|
1657
2801
|
sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
|
|
1658
2802
|
sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
|
|
1659
2803
|
sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
|
|
2804
|
+
sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
|
|
1660
2805
|
sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
|
|
1661
2806
|
sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
|
|
1662
2807
|
sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
|
|
@@ -1669,6 +2814,8 @@ void Init_parser(void)
|
|
|
1669
2814
|
i_try_convert = rb_intern("try_convert");
|
|
1670
2815
|
i_uminus = rb_intern("-@");
|
|
1671
2816
|
i_encode = rb_intern("encode");
|
|
2817
|
+
i_at_line = rb_intern("@line");
|
|
2818
|
+
i_at_column = rb_intern("@column");
|
|
1672
2819
|
|
|
1673
2820
|
binary_encindex = rb_ascii8bit_encindex();
|
|
1674
2821
|
utf8_encindex = rb_utf8_encindex();
|