json 2.18.1 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +65 -0
- data/LEGAL +3 -3
- data/README.md +11 -2
- data/ext/json/ext/fbuffer/fbuffer.h +46 -36
- data/ext/json/ext/generator/extconf.rb +3 -0
- data/ext/json/ext/generator/generator.c +98 -325
- data/ext/json/ext/json.h +78 -0
- data/ext/json/ext/parser/extconf.rb +37 -0
- data/ext/json/ext/parser/parser.c +1535 -380
- data/ext/json/ext/simd/simd.h +0 -10
- data/ext/json/ext/vendor/fast_float_parser.h +814 -0
- data/lib/json/common.rb +41 -10
- data/lib/json/ext/generator/state.rb +1 -1
- data/lib/json/truffle_ruby/generator.rb +24 -9
- data/lib/json/version.rb +1 -1
- data/lib/json.rb +30 -4
- metadata +3 -3
- data/ext/json/ext/vendor/ryu.h +0 -819
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
#include "../json.h"
|
|
2
|
-
#include "../vendor/
|
|
2
|
+
#include "../vendor/fast_float_parser.h"
|
|
3
3
|
#include "../simd/simd.h"
|
|
4
4
|
|
|
5
|
-
static VALUE mJSON, eNestingError, Encoding_UTF_8;
|
|
6
|
-
static VALUE CNaN, CInfinity, CMinusInfinity;
|
|
5
|
+
static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
|
|
6
|
+
static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
|
|
7
7
|
|
|
8
|
-
static ID i_new, i_try_convert, i_uminus, i_encode;
|
|
8
|
+
static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column;
|
|
9
9
|
|
|
10
|
-
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma,
|
|
11
|
-
|
|
10
|
+
static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
|
|
11
|
+
sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
|
|
12
|
+
sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
|
|
12
13
|
|
|
13
14
|
static int binary_encindex;
|
|
14
15
|
static int utf8_encindex;
|
|
@@ -57,6 +58,20 @@ typedef struct rvalue_cache_struct {
|
|
|
57
58
|
VALUE entries[JSON_RVALUE_CACHE_CAPA];
|
|
58
59
|
} rvalue_cache;
|
|
59
60
|
|
|
61
|
+
static void rvalue_cache_mark(rvalue_cache *cache)
|
|
62
|
+
{
|
|
63
|
+
for (int index = 0; index < cache->length; index++) {
|
|
64
|
+
rb_gc_mark_movable(cache->entries[index]);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
static void rvalue_cache_compact(rvalue_cache *cache)
|
|
69
|
+
{
|
|
70
|
+
for (int index = 0; index < cache->length; index++) {
|
|
71
|
+
cache->entries[index] = rb_gc_location(cache->entries[index]);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
60
75
|
static rb_encoding *enc_utf8;
|
|
61
76
|
|
|
62
77
|
#define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55
|
|
@@ -205,12 +220,12 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle,
|
|
|
205
220
|
|
|
206
221
|
static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
207
222
|
{
|
|
208
|
-
long required = stack->capa * 2;
|
|
223
|
+
long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA;
|
|
209
224
|
|
|
210
225
|
if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
|
|
211
226
|
stack = rvalue_stack_spill(stack, handle, stack_ref);
|
|
212
227
|
} else {
|
|
213
|
-
|
|
228
|
+
JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa);
|
|
214
229
|
stack->capa = required;
|
|
215
230
|
}
|
|
216
231
|
return stack;
|
|
@@ -218,11 +233,15 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu
|
|
|
218
233
|
|
|
219
234
|
static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref)
|
|
220
235
|
{
|
|
236
|
+
JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle);
|
|
237
|
+
|
|
221
238
|
if (RB_UNLIKELY(stack->head >= stack->capa)) {
|
|
222
239
|
stack = rvalue_stack_grow(stack, handle, stack_ref);
|
|
223
240
|
}
|
|
241
|
+
|
|
224
242
|
stack->ptr[stack->head] = value;
|
|
225
243
|
stack->head++;
|
|
244
|
+
|
|
226
245
|
return value;
|
|
227
246
|
}
|
|
228
247
|
|
|
@@ -240,35 +259,62 @@ static void rvalue_stack_mark(void *ptr)
|
|
|
240
259
|
{
|
|
241
260
|
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
242
261
|
long index;
|
|
243
|
-
|
|
244
|
-
|
|
262
|
+
if (stack && stack->ptr) {
|
|
263
|
+
for (index = 0; index < stack->head; index++) {
|
|
264
|
+
rb_gc_mark_movable(stack->ptr[index]);
|
|
265
|
+
}
|
|
245
266
|
}
|
|
246
267
|
}
|
|
247
268
|
|
|
269
|
+
static void rvalue_stack_free_buffer(rvalue_stack *stack)
|
|
270
|
+
{
|
|
271
|
+
JSON_SIZED_FREE_N(stack->ptr, stack->capa);
|
|
272
|
+
stack->ptr = NULL;
|
|
273
|
+
}
|
|
274
|
+
|
|
248
275
|
static void rvalue_stack_free(void *ptr)
|
|
249
276
|
{
|
|
250
277
|
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
251
278
|
if (stack) {
|
|
252
|
-
|
|
253
|
-
|
|
279
|
+
rvalue_stack_free_buffer(stack);
|
|
280
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
281
|
+
JSON_SIZED_FREE(stack);
|
|
282
|
+
#endif
|
|
254
283
|
}
|
|
255
284
|
}
|
|
256
285
|
|
|
257
286
|
static size_t rvalue_stack_memsize(const void *ptr)
|
|
258
287
|
{
|
|
259
288
|
const rvalue_stack *stack = (const rvalue_stack *)ptr;
|
|
260
|
-
|
|
289
|
+
size_t memsize = sizeof(VALUE) * stack->capa;
|
|
290
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
291
|
+
memsize += sizeof(rvalue_stack);
|
|
292
|
+
#endif
|
|
293
|
+
return memsize;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
static void rvalue_stack_compact(void *ptr)
|
|
297
|
+
{
|
|
298
|
+
rvalue_stack *stack = (rvalue_stack *)ptr;
|
|
299
|
+
long index;
|
|
300
|
+
if (stack && stack->ptr) {
|
|
301
|
+
for (index = 0; index < stack->head; index++) {
|
|
302
|
+
stack->ptr[index] = rb_gc_location(stack->ptr[index]);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
261
305
|
}
|
|
262
306
|
|
|
263
307
|
static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
|
|
264
|
-
"JSON::Ext::Parser/rvalue_stack",
|
|
265
|
-
{
|
|
308
|
+
.wrap_struct_name = "JSON::Ext::Parser/rvalue_stack",
|
|
309
|
+
.function = {
|
|
266
310
|
.dmark = rvalue_stack_mark,
|
|
267
311
|
.dfree = rvalue_stack_free,
|
|
268
312
|
.dsize = rvalue_stack_memsize,
|
|
313
|
+
.dcompact = rvalue_stack_compact,
|
|
269
314
|
},
|
|
270
|
-
|
|
271
|
-
|
|
315
|
+
// We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
|
|
316
|
+
// because it churns a lot of values so trigering write barriers every time is very costly.
|
|
317
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
|
|
272
318
|
};
|
|
273
319
|
|
|
274
320
|
static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
|
|
@@ -290,38 +336,71 @@ static void rvalue_stack_eagerly_release(VALUE handle)
|
|
|
290
336
|
if (handle) {
|
|
291
337
|
rvalue_stack *stack;
|
|
292
338
|
TypedData_Get_Struct(handle, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack);
|
|
293
|
-
|
|
339
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
340
|
+
rvalue_stack_free_buffer(stack);
|
|
341
|
+
#else
|
|
294
342
|
rvalue_stack_free(stack);
|
|
343
|
+
RTYPEDDATA_DATA(handle) = NULL;
|
|
344
|
+
#endif
|
|
295
345
|
}
|
|
296
346
|
}
|
|
297
347
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
348
|
+
/* frame stack */
|
|
349
|
+
|
|
350
|
+
// Iterative (non-recursive) parsing keeps an explicit stack of the containers
|
|
351
|
+
// currently being built, instead of relying on the C call stack. Each frame
|
|
352
|
+
// only needs enough bookkeeping to close its container: which kind it is, the
|
|
353
|
+
// rvalue_stack position where its children start (so we know how many to pop),
|
|
354
|
+
// and the cursor at its opening brace (used to rewind for duplicate key
|
|
355
|
+
// errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses
|
|
356
|
+
// the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that
|
|
357
|
+
// it's freed even if parsing raises.
|
|
358
|
+
//
|
|
359
|
+
// The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release
|
|
360
|
+
// and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts
|
|
361
|
+
// -- the element type and the absence of a mark function are the only real
|
|
362
|
+
// differences. Keep the two in sync: a fix to the spill/release or
|
|
363
|
+
// HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the
|
|
364
|
+
// other.
|
|
365
|
+
#define JSON_FRAME_STACK_INITIAL_CAPA 32
|
|
366
|
+
|
|
367
|
+
enum json_frame_type {
|
|
368
|
+
JSON_FRAME_ROOT, // == JSON_PHASE_DONE
|
|
369
|
+
JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA
|
|
370
|
+
JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
// Where a frame is within its container's grammar. This is the entirety of the
|
|
374
|
+
// parser's "what to do next" state: json_parse_any dispatches on the top
|
|
375
|
+
// frame's phase and holds no resume state in C locals, so a parse can stop at
|
|
376
|
+
// any value boundary and be resumed purely from the (persistable) frame stack.
|
|
377
|
+
//
|
|
378
|
+
// The first three phases are deliberately equal to the corresponding json_frame_type
|
|
379
|
+
// to simplify the transition of phase in json_value_completed.
|
|
380
|
+
enum json_frame_phase {
|
|
381
|
+
JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed
|
|
382
|
+
JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']'
|
|
383
|
+
JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}'
|
|
384
|
+
JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':')
|
|
385
|
+
JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',')
|
|
386
|
+
JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':'
|
|
387
|
+
};
|
|
388
|
+
|
|
389
|
+
typedef struct json_frame_struct {
|
|
390
|
+
enum json_frame_type type;
|
|
391
|
+
enum json_frame_phase phase;
|
|
392
|
+
long value_stack_head; // rvalue_stack->head when this container opened
|
|
393
|
+
size_t start_offset; // object frames only (the '{'); NULL otherwise
|
|
394
|
+
} json_frame;
|
|
323
395
|
|
|
324
|
-
|
|
396
|
+
typedef struct json_frame_stack_struct {
|
|
397
|
+
enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated
|
|
398
|
+
long capa;
|
|
399
|
+
long head;
|
|
400
|
+
json_frame *ptr;
|
|
401
|
+
} json_frame_stack;
|
|
402
|
+
|
|
403
|
+
enum deprecatable_action {
|
|
325
404
|
JSON_DEPRECATED = 0,
|
|
326
405
|
JSON_IGNORE,
|
|
327
406
|
JSON_RAISE,
|
|
@@ -331,26 +410,164 @@ typedef struct JSON_ParserStruct {
|
|
|
331
410
|
VALUE on_load_proc;
|
|
332
411
|
VALUE decimal_class;
|
|
333
412
|
ID decimal_method_id;
|
|
334
|
-
enum
|
|
413
|
+
enum deprecatable_action on_duplicate_key;
|
|
414
|
+
enum deprecatable_action on_comment;
|
|
335
415
|
int max_nesting;
|
|
336
416
|
bool allow_nan;
|
|
337
417
|
bool allow_trailing_comma;
|
|
338
418
|
bool allow_control_characters;
|
|
419
|
+
bool allow_invalid_escape;
|
|
339
420
|
bool symbolize_names;
|
|
340
421
|
bool freeze;
|
|
341
422
|
} JSON_ParserConfig;
|
|
342
423
|
|
|
343
424
|
typedef struct JSON_ParserStateStruct {
|
|
344
|
-
VALUE
|
|
425
|
+
VALUE *value_stack_handle;
|
|
426
|
+
VALUE *frame_stack_handle;
|
|
345
427
|
const char *start;
|
|
346
428
|
const char *cursor;
|
|
347
429
|
const char *end;
|
|
348
|
-
rvalue_stack *
|
|
430
|
+
rvalue_stack *value_stack;
|
|
431
|
+
json_frame_stack *frames;
|
|
349
432
|
rvalue_cache name_cache;
|
|
350
433
|
int in_array;
|
|
351
434
|
int current_nesting;
|
|
435
|
+
unsigned int emitted_deprecations;
|
|
436
|
+
VALUE parser;
|
|
352
437
|
} JSON_ParserState;
|
|
353
438
|
|
|
439
|
+
static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref);
|
|
440
|
+
|
|
441
|
+
static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref)
|
|
442
|
+
{
|
|
443
|
+
long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA;
|
|
444
|
+
|
|
445
|
+
if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
|
|
446
|
+
stack = json_frame_stack_spill(stack, handle, stack_ref);
|
|
447
|
+
} else {
|
|
448
|
+
JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa);
|
|
449
|
+
stack->capa = required;
|
|
450
|
+
}
|
|
451
|
+
return stack;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame)
|
|
455
|
+
{
|
|
456
|
+
json_frame_stack *stack = state->frames;
|
|
457
|
+
|
|
458
|
+
JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle);
|
|
459
|
+
|
|
460
|
+
if (RB_UNLIKELY(stack->head >= stack->capa)) {
|
|
461
|
+
stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
json_frame *frame_ptr = &stack->ptr[stack->head++];
|
|
465
|
+
*frame_ptr = frame;
|
|
466
|
+
return frame_ptr;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
static inline json_frame *json_frame_stack_peek(json_frame_stack *stack)
|
|
470
|
+
{
|
|
471
|
+
return &stack->ptr[stack->head - 1];
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
static inline void json_frame_stack_pop(json_frame_stack *stack)
|
|
475
|
+
{
|
|
476
|
+
stack->head--;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
static void json_frame_stack_free_buffer(json_frame_stack *stack)
|
|
480
|
+
{
|
|
481
|
+
JSON_SIZED_FREE_N(stack->ptr, stack->capa);
|
|
482
|
+
stack->ptr = NULL;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
static void json_frame_stack_free(void *ptr)
|
|
486
|
+
{
|
|
487
|
+
json_frame_stack *stack = (json_frame_stack *)ptr;
|
|
488
|
+
if (stack) {
|
|
489
|
+
json_frame_stack_free_buffer(stack);
|
|
490
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
491
|
+
JSON_SIZED_FREE(stack);
|
|
492
|
+
#endif
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
static size_t json_frame_stack_memsize(const void *ptr)
|
|
497
|
+
{
|
|
498
|
+
const json_frame_stack *stack = (const json_frame_stack *)ptr;
|
|
499
|
+
|
|
500
|
+
size_t memsize = sizeof(json_frame) * stack->capa;
|
|
501
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
502
|
+
memsize += sizeof(json_frame_stack);
|
|
503
|
+
#endif
|
|
504
|
+
return memsize;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
static const rb_data_type_t JSON_Parser_frame_stack_type = {
|
|
508
|
+
.wrap_struct_name = "JSON::Ext::Parser/frame_stack",
|
|
509
|
+
.function = {
|
|
510
|
+
.dmark = NULL,
|
|
511
|
+
.dfree = json_frame_stack_free,
|
|
512
|
+
.dsize = json_frame_stack_memsize,
|
|
513
|
+
},
|
|
514
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
|
|
518
|
+
{
|
|
519
|
+
json_frame_stack *stack;
|
|
520
|
+
*handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
|
|
521
|
+
*stack_ref = stack;
|
|
522
|
+
MEMCPY(stack, old_stack, json_frame_stack, 1);
|
|
523
|
+
|
|
524
|
+
stack->capa = old_stack->capa << 1;
|
|
525
|
+
stack->ptr = ALLOC_N(json_frame, stack->capa);
|
|
526
|
+
stack->type = RVALUE_STACK_HEAP_ALLOCATED;
|
|
527
|
+
MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head);
|
|
528
|
+
return stack;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
static void json_frame_stack_eagerly_release(VALUE handle)
|
|
532
|
+
{
|
|
533
|
+
if (handle) {
|
|
534
|
+
json_frame_stack *stack;
|
|
535
|
+
TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
|
|
536
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
537
|
+
json_frame_stack_free_buffer(stack);
|
|
538
|
+
#else
|
|
539
|
+
json_frame_stack_free(stack);
|
|
540
|
+
RTYPEDDATA_DATA(handle) = NULL;
|
|
541
|
+
#endif
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
|
|
546
|
+
{
|
|
547
|
+
int len = 1;
|
|
548
|
+
if (ch <= 0x7F) {
|
|
549
|
+
buf[0] = (char) ch;
|
|
550
|
+
} else if (ch <= 0x07FF) {
|
|
551
|
+
buf[0] = (char) ((ch >> 6) | 0xC0);
|
|
552
|
+
buf[1] = (char) ((ch & 0x3F) | 0x80);
|
|
553
|
+
len++;
|
|
554
|
+
} else if (ch <= 0xFFFF) {
|
|
555
|
+
buf[0] = (char) ((ch >> 12) | 0xE0);
|
|
556
|
+
buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
|
|
557
|
+
buf[2] = (char) ((ch & 0x3F) | 0x80);
|
|
558
|
+
len += 2;
|
|
559
|
+
} else if (ch <= 0x1fffff) {
|
|
560
|
+
buf[0] =(char) ((ch >> 18) | 0xF0);
|
|
561
|
+
buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
|
|
562
|
+
buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
|
|
563
|
+
buf[3] =(char) ((ch & 0x3F) | 0x80);
|
|
564
|
+
len += 3;
|
|
565
|
+
} else {
|
|
566
|
+
buf[0] = '?';
|
|
567
|
+
}
|
|
568
|
+
return len;
|
|
569
|
+
}
|
|
570
|
+
|
|
354
571
|
static inline size_t rest(JSON_ParserState *state) {
|
|
355
572
|
return state->end - state->cursor;
|
|
356
573
|
}
|
|
@@ -369,12 +586,20 @@ static inline char peek(JSON_ParserState *state)
|
|
|
369
586
|
|
|
370
587
|
static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
|
|
371
588
|
{
|
|
589
|
+
JSON_ASSERT(state->cursor <= state->end);
|
|
590
|
+
|
|
591
|
+
// Redundant but helpful for hardening
|
|
592
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
593
|
+
state->cursor = state->end;
|
|
594
|
+
}
|
|
595
|
+
|
|
372
596
|
const char *cursor = state->cursor;
|
|
373
597
|
long column = 0;
|
|
374
598
|
long line = 1;
|
|
375
599
|
|
|
376
600
|
while (cursor >= state->start) {
|
|
377
601
|
if (*cursor-- == '\n') {
|
|
602
|
+
line++;
|
|
378
603
|
break;
|
|
379
604
|
}
|
|
380
605
|
column++;
|
|
@@ -389,6 +614,8 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum
|
|
|
389
614
|
*column_out = column;
|
|
390
615
|
}
|
|
391
616
|
|
|
617
|
+
static const unsigned int MAX_DEPRECATIONS = 5;
|
|
618
|
+
|
|
392
619
|
static void emit_parse_warning(const char *message, JSON_ParserState *state)
|
|
393
620
|
{
|
|
394
621
|
long line, column;
|
|
@@ -400,14 +627,9 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state)
|
|
|
400
627
|
|
|
401
628
|
#define PARSE_ERROR_FRAGMENT_LEN 32
|
|
402
629
|
|
|
403
|
-
|
|
404
|
-
RBIMPL_ATTR_NORETURN()
|
|
405
|
-
#endif
|
|
406
|
-
static void raise_parse_error(const char *format, JSON_ParserState *state)
|
|
630
|
+
static VALUE build_parse_error_message(const char *format, JSON_ParserState *state)
|
|
407
631
|
{
|
|
408
632
|
unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
|
|
409
|
-
long line, column;
|
|
410
|
-
cursor_position(state, &line, &column);
|
|
411
633
|
|
|
412
634
|
const char *ptr = "EOF";
|
|
413
635
|
if (state->cursor && state->cursor < state->end) {
|
|
@@ -439,23 +661,61 @@ static void raise_parse_error(const char *format, JSON_ParserState *state)
|
|
|
439
661
|
}
|
|
440
662
|
}
|
|
441
663
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
664
|
+
return rb_enc_sprintf(enc_utf8, format, ptr);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
|
|
668
|
+
{
|
|
669
|
+
VALUE exc = rb_exc_new_str(eParserError, message);
|
|
670
|
+
rb_ivar_set(exc, i_at_line, LONG2NUM(line));
|
|
671
|
+
rb_ivar_set(exc, i_at_column, LONG2NUM(column));
|
|
672
|
+
return exc;
|
|
673
|
+
}
|
|
445
674
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
675
|
+
NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos)
|
|
676
|
+
{
|
|
677
|
+
if (state->parser) {
|
|
678
|
+
if (eos) {
|
|
679
|
+
// the error will be swallowed by ResumableParser#parse, so no
|
|
680
|
+
// point building a message or backtrace.
|
|
681
|
+
rb_throw_obj(state->parser, state->parser);
|
|
682
|
+
} else {
|
|
683
|
+
// line and columns can't be accurate in resumable
|
|
684
|
+
rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos));
|
|
685
|
+
}
|
|
686
|
+
} else {
|
|
687
|
+
VALUE message = build_parse_error_message(format, state);
|
|
688
|
+
long line, column;
|
|
689
|
+
cursor_position(state, &line, &column);
|
|
690
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
691
|
+
rb_exc_raise(parse_error_new(state, message, line, column, eos));
|
|
692
|
+
}
|
|
450
693
|
}
|
|
451
694
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
695
|
+
NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state)
|
|
696
|
+
{
|
|
697
|
+
raise_parse_error(format, state, true);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state)
|
|
701
|
+
{
|
|
702
|
+
raise_parse_error(format, state, false);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos)
|
|
456
706
|
{
|
|
457
707
|
state->cursor = at;
|
|
458
|
-
raise_parse_error(format, state);
|
|
708
|
+
raise_parse_error(format, state, eos);
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
712
|
+
{
|
|
713
|
+
raise_parse_error_at(format, state, at, true);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at)
|
|
717
|
+
{
|
|
718
|
+
raise_parse_error_at(format, state, at, false);
|
|
459
719
|
}
|
|
460
720
|
|
|
461
721
|
/* unicode */
|
|
@@ -480,7 +740,7 @@ static const signed char digit_values[256] = {
|
|
|
480
740
|
static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
|
|
481
741
|
{
|
|
482
742
|
if (RB_UNLIKELY(sp > spe - 4)) {
|
|
483
|
-
|
|
743
|
+
raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
484
744
|
}
|
|
485
745
|
|
|
486
746
|
const unsigned char *p = (const unsigned char *)sp;
|
|
@@ -491,7 +751,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
|
|
|
491
751
|
const signed char b3 = digit_values[p[3]];
|
|
492
752
|
|
|
493
753
|
if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
|
|
494
|
-
|
|
754
|
+
raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
|
|
495
755
|
}
|
|
496
756
|
|
|
497
757
|
return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
|
|
@@ -503,9 +763,14 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
|
|
|
503
763
|
|
|
504
764
|
static const rb_data_type_t JSON_ParserConfig_type;
|
|
505
765
|
|
|
506
|
-
|
|
507
|
-
|
|
766
|
+
const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
|
|
767
|
+
NOINLINE(static) void
|
|
768
|
+
json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
508
769
|
{
|
|
770
|
+
if (config->on_comment == JSON_RAISE) {
|
|
771
|
+
raise_syntax_error("unexpected token %s", state);
|
|
772
|
+
}
|
|
773
|
+
|
|
509
774
|
const char *start = state->cursor;
|
|
510
775
|
state->cursor++;
|
|
511
776
|
|
|
@@ -525,7 +790,7 @@ json_eat_comments(JSON_ParserState *state)
|
|
|
525
790
|
while (true) {
|
|
526
791
|
const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
|
|
527
792
|
if (!next_match) {
|
|
528
|
-
|
|
793
|
+
raise_eos_error_at("unterminated comment, expected closing '*/'", state, start);
|
|
529
794
|
}
|
|
530
795
|
|
|
531
796
|
state->cursor = next_match + 1;
|
|
@@ -537,13 +802,18 @@ json_eat_comments(JSON_ParserState *state)
|
|
|
537
802
|
break;
|
|
538
803
|
}
|
|
539
804
|
default:
|
|
540
|
-
raise_parse_error_at("unexpected token %s", state, start);
|
|
805
|
+
raise_parse_error_at("unexpected token %s", state, start, eos(state));
|
|
541
806
|
break;
|
|
542
807
|
}
|
|
808
|
+
|
|
809
|
+
if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) {
|
|
810
|
+
state->emitted_deprecations++;
|
|
811
|
+
emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state);
|
|
812
|
+
}
|
|
543
813
|
}
|
|
544
814
|
|
|
545
815
|
ALWAYS_INLINE(static) void
|
|
546
|
-
json_eat_whitespace(JSON_ParserState *state)
|
|
816
|
+
json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
|
|
547
817
|
{
|
|
548
818
|
while (true) {
|
|
549
819
|
switch (peek(state)) {
|
|
@@ -574,7 +844,11 @@ json_eat_whitespace(JSON_ParserState *state)
|
|
|
574
844
|
state->cursor++;
|
|
575
845
|
break;
|
|
576
846
|
case '/':
|
|
577
|
-
|
|
847
|
+
if (!include_comments) {
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
json_eat_comments(state, config);
|
|
578
852
|
break;
|
|
579
853
|
|
|
580
854
|
default:
|
|
@@ -728,13 +1002,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
728
1002
|
uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
|
|
729
1003
|
|
|
730
1004
|
if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
|
|
731
|
-
|
|
1005
|
+
raise_syntax_error_at("invalid surrogate pair at %s", state, p);
|
|
732
1006
|
}
|
|
733
1007
|
|
|
734
1008
|
ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
|
|
735
1009
|
pe += 5;
|
|
736
1010
|
} else {
|
|
737
|
-
|
|
1011
|
+
raise_syntax_error_at("incomplete surrogate pair at %s", state, p);
|
|
738
1012
|
break;
|
|
739
1013
|
}
|
|
740
1014
|
}
|
|
@@ -744,16 +1018,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
744
1018
|
p = ++pe;
|
|
745
1019
|
break;
|
|
746
1020
|
}
|
|
1021
|
+
case 0:
|
|
1022
|
+
return Qundef;
|
|
747
1023
|
default:
|
|
748
1024
|
if ((unsigned char)*pe < 0x20) {
|
|
749
1025
|
if (!config->allow_control_characters) {
|
|
750
1026
|
if (*pe == '\n') {
|
|
751
|
-
|
|
1027
|
+
raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
|
|
752
1028
|
}
|
|
753
|
-
|
|
1029
|
+
raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1);
|
|
754
1030
|
}
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
if (config->allow_invalid_escape) {
|
|
1034
|
+
APPEND_CHAR(*pe);
|
|
755
1035
|
} else {
|
|
756
|
-
|
|
1036
|
+
raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1);
|
|
757
1037
|
}
|
|
758
1038
|
break;
|
|
759
1039
|
}
|
|
@@ -776,20 +1056,39 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
|
|
|
776
1056
|
}
|
|
777
1057
|
|
|
778
1058
|
#define MAX_FAST_INTEGER_SIZE 18
|
|
1059
|
+
#define MAX_NUMBER_STACK_BUFFER 128
|
|
1060
|
+
|
|
1061
|
+
typedef VALUE (*json_number_decode_func_t)(const char *ptr);
|
|
1062
|
+
|
|
1063
|
+
static inline VALUE json_decode_large_number(const char *start, long len, json_number_decode_func_t func)
|
|
1064
|
+
{
|
|
1065
|
+
if (RB_LIKELY(len < MAX_NUMBER_STACK_BUFFER)) {
|
|
1066
|
+
char buffer[MAX_NUMBER_STACK_BUFFER];
|
|
1067
|
+
MEMCPY(buffer, start, char, len);
|
|
1068
|
+
buffer[len] = '\0';
|
|
1069
|
+
return func(buffer);
|
|
1070
|
+
} else {
|
|
1071
|
+
VALUE buffer_v = rb_str_tmp_new(len);
|
|
1072
|
+
char *buffer = RSTRING_PTR(buffer_v);
|
|
1073
|
+
MEMCPY(buffer, start, char, len);
|
|
1074
|
+
buffer[len] = '\0';
|
|
1075
|
+
VALUE number = func(buffer);
|
|
1076
|
+
RB_GC_GUARD(buffer_v);
|
|
1077
|
+
return number;
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
static VALUE json_decode_inum(const char *buffer)
|
|
1082
|
+
{
|
|
1083
|
+
return rb_cstr2inum(buffer, 10);
|
|
1084
|
+
}
|
|
779
1085
|
|
|
780
|
-
static VALUE json_decode_large_integer(const char *start, long len)
|
|
1086
|
+
NOINLINE(static) VALUE json_decode_large_integer(const char *start, long len)
|
|
781
1087
|
{
|
|
782
|
-
|
|
783
|
-
char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1);
|
|
784
|
-
MEMCPY(buffer, start, char, len);
|
|
785
|
-
buffer[len] = '\0';
|
|
786
|
-
VALUE number = rb_cstr2inum(buffer, 10);
|
|
787
|
-
RB_ALLOCV_END(buffer_v);
|
|
788
|
-
return number;
|
|
1088
|
+
return json_decode_large_number(start, len, json_decode_inum);
|
|
789
1089
|
}
|
|
790
1090
|
|
|
791
|
-
static inline VALUE
|
|
792
|
-
json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end)
|
|
1091
|
+
static inline VALUE json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end)
|
|
793
1092
|
{
|
|
794
1093
|
if (RB_LIKELY(mantissa_digits < MAX_FAST_INTEGER_SIZE)) {
|
|
795
1094
|
if (negative) {
|
|
@@ -801,28 +1100,20 @@ json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const
|
|
|
801
1100
|
return json_decode_large_integer(start, end - start);
|
|
802
1101
|
}
|
|
803
1102
|
|
|
804
|
-
static VALUE
|
|
1103
|
+
static VALUE json_decode_dnum(const char *buffer)
|
|
805
1104
|
{
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
MEMCPY(buffer, start, char, len);
|
|
809
|
-
buffer[len] = '\0';
|
|
810
|
-
return DBL2NUM(rb_cstr_to_dbl(buffer, 1));
|
|
811
|
-
}
|
|
1105
|
+
return DBL2NUM(rb_cstr_to_dbl(buffer, 1));
|
|
1106
|
+
}
|
|
812
1107
|
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
buffer[len] = '\0';
|
|
817
|
-
VALUE number = DBL2NUM(rb_cstr_to_dbl(buffer, 1));
|
|
818
|
-
RB_ALLOCV_END(buffer_v);
|
|
819
|
-
return number;
|
|
1108
|
+
NOINLINE(static) VALUE json_decode_large_float(const char *start, long len)
|
|
1109
|
+
{
|
|
1110
|
+
return json_decode_large_number(start, len, json_decode_dnum);
|
|
820
1111
|
}
|
|
821
1112
|
|
|
822
1113
|
/* Ruby JSON optimized float decoder using vendored Ryu algorithm
|
|
823
1114
|
* Accepts pre-extracted mantissa and exponent from first-pass validation
|
|
824
1115
|
*/
|
|
825
|
-
static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits,
|
|
1116
|
+
static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int64_t exponent, bool negative,
|
|
826
1117
|
const char *start, const char *end)
|
|
827
1118
|
{
|
|
828
1119
|
if (RB_UNLIKELY(config->decimal_class)) {
|
|
@@ -830,19 +1121,25 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
|
|
|
830
1121
|
return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text);
|
|
831
1122
|
}
|
|
832
1123
|
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
1124
|
+
if (RB_UNLIKELY(exponent > INT32_MAX)) {
|
|
1125
|
+
return negative ? CMinusInfinity : CInfinity;
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
if (RB_UNLIKELY(exponent < INT32_MIN)) {
|
|
1129
|
+
return rb_float_new(negative ? -0.0 : 0.0);
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
|
|
836
1133
|
return json_decode_large_float(start, end - start);
|
|
837
1134
|
}
|
|
838
1135
|
|
|
839
|
-
return DBL2NUM(
|
|
1136
|
+
return DBL2NUM(ffp_s2d(exponent, mantissa, negative));
|
|
840
1137
|
}
|
|
841
1138
|
|
|
842
1139
|
static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
|
|
843
1140
|
{
|
|
844
|
-
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->
|
|
845
|
-
rvalue_stack_pop(state->
|
|
1141
|
+
VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count));
|
|
1142
|
+
rvalue_stack_pop(state->value_stack, count);
|
|
846
1143
|
|
|
847
1144
|
if (config->freeze) {
|
|
848
1145
|
RB_OBJ_FREEZE(array);
|
|
@@ -868,7 +1165,7 @@ static VALUE json_find_duplicated_key(size_t count, const VALUE *pairs)
|
|
|
868
1165
|
return Qfalse;
|
|
869
1166
|
}
|
|
870
1167
|
|
|
871
|
-
static void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key)
|
|
1168
|
+
NOINLINE(static) void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key)
|
|
872
1169
|
{
|
|
873
1170
|
VALUE message = rb_sprintf(
|
|
874
1171
|
"detected duplicate key %"PRIsVALUE" in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`",
|
|
@@ -879,41 +1176,57 @@ static void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_
|
|
|
879
1176
|
RB_GC_GUARD(message);
|
|
880
1177
|
}
|
|
881
1178
|
|
|
882
|
-
|
|
883
|
-
RBIMPL_ATTR_NORETURN()
|
|
884
|
-
#endif
|
|
885
|
-
static void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key)
|
|
1179
|
+
NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key)
|
|
886
1180
|
{
|
|
887
1181
|
VALUE message = rb_sprintf(
|
|
888
1182
|
"duplicate key %"PRIsVALUE,
|
|
889
1183
|
rb_inspect(duplicate_key)
|
|
890
1184
|
);
|
|
891
1185
|
|
|
892
|
-
|
|
893
|
-
|
|
1186
|
+
rb_str_concat(message, build_parse_error_message("", state));
|
|
1187
|
+
if (state->parser) { // line and columns can't be accurate in resumable
|
|
1188
|
+
rb_exc_raise(parse_error_new(state, message, 0, 0, false));
|
|
1189
|
+
} else {
|
|
1190
|
+
long line, column;
|
|
1191
|
+
cursor_position(state, &line, &column);
|
|
1192
|
+
rb_str_catf(message, " at line %ld column %ld", line, column);
|
|
1193
|
+
rb_exc_raise(parse_error_new(state, message, line, column, false));
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
|
|
1198
|
+
{
|
|
1199
|
+
switch (config->on_duplicate_key) {
|
|
1200
|
+
case JSON_IGNORE:
|
|
1201
|
+
return;
|
|
1202
|
+
|
|
1203
|
+
case JSON_DEPRECATED:
|
|
1204
|
+
// Only emit the first few deprecations to avoid spamming.
|
|
1205
|
+
if (state->emitted_deprecations < MAX_DEPRECATIONS) {
|
|
1206
|
+
state->emitted_deprecations++;
|
|
1207
|
+
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
1208
|
+
}
|
|
1209
|
+
return;
|
|
1210
|
+
|
|
1211
|
+
case JSON_RAISE:
|
|
1212
|
+
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
UNREACHABLE;
|
|
894
1216
|
}
|
|
895
1217
|
|
|
896
1218
|
static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
|
|
897
1219
|
{
|
|
898
1220
|
size_t entries_count = count / 2;
|
|
899
1221
|
VALUE object = rb_hash_new_capa(entries_count);
|
|
900
|
-
const VALUE *pairs = rvalue_stack_peek(state->
|
|
1222
|
+
const VALUE *pairs = rvalue_stack_peek(state->value_stack, count);
|
|
901
1223
|
rb_hash_bulk_insert(count, pairs, object);
|
|
902
1224
|
|
|
903
1225
|
if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
|
|
904
|
-
|
|
905
|
-
case JSON_IGNORE:
|
|
906
|
-
break;
|
|
907
|
-
case JSON_DEPRECATED:
|
|
908
|
-
emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
|
|
909
|
-
break;
|
|
910
|
-
case JSON_RAISE:
|
|
911
|
-
raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
|
|
912
|
-
break;
|
|
913
|
-
}
|
|
1226
|
+
json_on_duplicate_key(state, config, count, pairs);
|
|
914
1227
|
}
|
|
915
1228
|
|
|
916
|
-
rvalue_stack_pop(state->
|
|
1229
|
+
rvalue_stack_pop(state->value_stack, count);
|
|
917
1230
|
|
|
918
1231
|
if (config->freeze) {
|
|
919
1232
|
RB_OBJ_FREEZE(object);
|
|
@@ -927,7 +1240,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
|
|
|
927
1240
|
if (RB_UNLIKELY(config->on_load_proc)) {
|
|
928
1241
|
value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
|
|
929
1242
|
}
|
|
930
|
-
rvalue_stack_push(state->
|
|
1243
|
+
rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack);
|
|
931
1244
|
return value;
|
|
932
1245
|
}
|
|
933
1246
|
|
|
@@ -976,6 +1289,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
|
|
|
976
1289
|
}
|
|
977
1290
|
state->cursor++;
|
|
978
1291
|
}
|
|
1292
|
+
|
|
1293
|
+
// If the string ended with an unterminated escape sequence, we might
|
|
1294
|
+
// have gone past the end.
|
|
1295
|
+
if (RB_UNLIKELY(state->cursor > state->end)) {
|
|
1296
|
+
state->cursor = state->end;
|
|
1297
|
+
}
|
|
1298
|
+
|
|
979
1299
|
return false;
|
|
980
1300
|
}
|
|
981
1301
|
|
|
@@ -993,7 +1313,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
993
1313
|
case '"': {
|
|
994
1314
|
VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
|
|
995
1315
|
state->cursor++;
|
|
996
|
-
return
|
|
1316
|
+
return string;
|
|
997
1317
|
}
|
|
998
1318
|
case '\\': {
|
|
999
1319
|
if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
|
|
@@ -1007,7 +1327,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1007
1327
|
}
|
|
1008
1328
|
default:
|
|
1009
1329
|
if (!config->allow_control_characters) {
|
|
1010
|
-
|
|
1330
|
+
raise_syntax_error("invalid ASCII control character in string: %s", state);
|
|
1011
1331
|
}
|
|
1012
1332
|
break;
|
|
1013
1333
|
}
|
|
@@ -1015,8 +1335,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
|
|
|
1015
1335
|
state->cursor++;
|
|
1016
1336
|
} while (string_scan(state));
|
|
1017
1337
|
|
|
1018
|
-
|
|
1019
|
-
return Qfalse;
|
|
1338
|
+
return Qundef;
|
|
1020
1339
|
}
|
|
1021
1340
|
|
|
1022
1341
|
ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
|
|
@@ -1025,15 +1344,19 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars
|
|
|
1025
1344
|
const char *start = state->cursor;
|
|
1026
1345
|
|
|
1027
1346
|
if (RB_UNLIKELY(!string_scan(state))) {
|
|
1028
|
-
|
|
1347
|
+
return Qundef;
|
|
1029
1348
|
}
|
|
1030
1349
|
|
|
1350
|
+
VALUE string;
|
|
1031
1351
|
if (RB_LIKELY(*state->cursor == '"')) {
|
|
1032
|
-
|
|
1352
|
+
string = json_string_fastpath(state, config, start, state->cursor, is_name);
|
|
1033
1353
|
state->cursor++;
|
|
1034
|
-
return json_push_value(state, config, string);
|
|
1035
1354
|
}
|
|
1036
|
-
|
|
1355
|
+
else {
|
|
1356
|
+
string = json_parse_escaped_string(state, config, is_name, start);
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
return string;
|
|
1037
1360
|
}
|
|
1038
1361
|
|
|
1039
1362
|
#if JSON_CPU_LITTLE_ENDIAN_64BITS
|
|
@@ -1112,7 +1435,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1112
1435
|
const char first_digit = *state->cursor;
|
|
1113
1436
|
|
|
1114
1437
|
// Variables for Ryu optimization - extract digits during parsing
|
|
1115
|
-
|
|
1438
|
+
int64_t exponent = 0;
|
|
1116
1439
|
int decimal_point_pos = -1;
|
|
1117
1440
|
uint64_t mantissa = 0;
|
|
1118
1441
|
|
|
@@ -1120,7 +1443,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1120
1443
|
int mantissa_digits = json_parse_digits(state, &mantissa);
|
|
1121
1444
|
|
|
1122
1445
|
if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
|
|
1123
|
-
|
|
1446
|
+
return Qundef;
|
|
1124
1447
|
}
|
|
1125
1448
|
|
|
1126
1449
|
// Parse fractional part
|
|
@@ -1133,7 +1456,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1133
1456
|
mantissa_digits += fractional_digits;
|
|
1134
1457
|
|
|
1135
1458
|
if (RB_UNLIKELY(!fractional_digits)) {
|
|
1136
|
-
|
|
1459
|
+
return Qundef;
|
|
1137
1460
|
}
|
|
1138
1461
|
}
|
|
1139
1462
|
|
|
@@ -1153,10 +1476,14 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1153
1476
|
int exponent_digits = json_parse_digits(state, &abs_exponent);
|
|
1154
1477
|
|
|
1155
1478
|
if (RB_UNLIKELY(!exponent_digits)) {
|
|
1156
|
-
|
|
1479
|
+
return Qundef;
|
|
1157
1480
|
}
|
|
1158
1481
|
|
|
1159
|
-
|
|
1482
|
+
if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
|
|
1483
|
+
exponent = negative_exponent ? INT64_MIN : INT64_MAX;
|
|
1484
|
+
} else {
|
|
1485
|
+
exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
|
|
1486
|
+
}
|
|
1160
1487
|
}
|
|
1161
1488
|
|
|
1162
1489
|
if (integer) {
|
|
@@ -1171,229 +1498,411 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
|
|
|
1171
1498
|
return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
|
|
1172
1499
|
}
|
|
1173
1500
|
|
|
1174
|
-
|
|
1501
|
+
// How many values (array elements, or interleaved object keys+values) have been
|
|
1502
|
+
// pushed onto the rvalue stack since this container opened. Used to size the
|
|
1503
|
+
// bulk decode on close, and to tell the first key/colon from later ones.
|
|
1504
|
+
static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack)
|
|
1175
1505
|
{
|
|
1176
|
-
return
|
|
1506
|
+
return value_stack->head - frame->value_stack_head;
|
|
1177
1507
|
}
|
|
1178
1508
|
|
|
1179
|
-
|
|
1509
|
+
// A complete value now sits on top of the rvalue stack. Advance the frame that
|
|
1510
|
+
// was waiting for it: the root document is done, or the enclosing container
|
|
1511
|
+
// moves on to expecting a ',' or its closing bracket. The caller passes the
|
|
1512
|
+
// frame it already has in hand -- the one that was expecting the value -- which
|
|
1513
|
+
// after a container close is the freshly re-exposed parent.
|
|
1514
|
+
static inline enum json_frame_phase json_value_completed(json_frame *frame)
|
|
1180
1515
|
{
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1516
|
+
JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT);
|
|
1517
|
+
JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY);
|
|
1518
|
+
JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT);
|
|
1519
|
+
|
|
1520
|
+
return frame->phase = (enum json_frame_phase) frame->type;
|
|
1184
1521
|
}
|
|
1185
1522
|
|
|
1186
|
-
static
|
|
1523
|
+
ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset)
|
|
1187
1524
|
{
|
|
1188
|
-
|
|
1525
|
+
// It is assumed that since `keyword` is always a literal, the compiler is able to constantize this
|
|
1526
|
+
// `strlen` and several other computations in that routine.
|
|
1189
1527
|
|
|
1190
|
-
|
|
1191
|
-
case 'n':
|
|
1192
|
-
if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
|
|
1193
|
-
state->cursor += 4;
|
|
1194
|
-
return json_push_value(state, config, Qnil);
|
|
1195
|
-
}
|
|
1528
|
+
size_t len = strlen(keyword);
|
|
1196
1529
|
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
}
|
|
1530
|
+
// Note: memcmp with a small power of two and a literal string compile to an integer comparison /
|
|
1531
|
+
// That's why we sometime compare starting from the first byte and sometimes from the second.
|
|
1532
|
+
if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) {
|
|
1533
|
+
state->cursor += len;
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1204
1536
|
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
// Note: memcmp with a small power of two compile to an integer comparison
|
|
1209
|
-
if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) {
|
|
1210
|
-
state->cursor += 5;
|
|
1211
|
-
return json_push_value(state, config, Qfalse);
|
|
1212
|
-
}
|
|
1537
|
+
bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0;
|
|
1538
|
+
raise_parse_error("unexpected token %s", state, eos);
|
|
1539
|
+
}
|
|
1213
1540
|
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1541
|
+
// Parse an arbitrary JSON value iteratively. This is a state machine driven
|
|
1542
|
+
// entirely by the top frame's phase so it can stop at any value boundary and
|
|
1543
|
+
// resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the
|
|
1544
|
+
// bottom of the stack, so the stack is never empty mid-parse and the document
|
|
1545
|
+
// itself is just another frame whose value, once parsed, leaves its phase DONE.
|
|
1546
|
+
// When invoked in resumable mode, it returns true after parsing a complete document.
|
|
1547
|
+
// If reaching EOS without having parsed a complete document, either returns false
|
|
1548
|
+
// of raise a JSON::ParserError tagged with `@eos=true`.
|
|
1549
|
+
ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable)
|
|
1550
|
+
{
|
|
1551
|
+
json_frame *frame = json_frame_stack_peek(state->frames);
|
|
1552
|
+
|
|
1553
|
+
switch (frame->phase) {
|
|
1554
|
+
case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false);
|
|
1555
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1556
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1557
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1558
|
+
case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY;
|
|
1559
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1560
|
+
}
|
|
1561
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1222
1562
|
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
case 'I':
|
|
1226
|
-
if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
|
|
1227
|
-
state->cursor += 8;
|
|
1228
|
-
return json_push_value(state, config, CInfinity);
|
|
1229
|
-
}
|
|
1563
|
+
JSON_PHASE_VALUE: {
|
|
1564
|
+
json_eat_whitespace(state, config, true);
|
|
1230
1565
|
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
case '-': {
|
|
1234
|
-
// Note: memcmp with a small power of two compile to an integer comparison
|
|
1235
|
-
if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) {
|
|
1236
|
-
if (config->allow_nan) {
|
|
1237
|
-
state->cursor += 9;
|
|
1238
|
-
return json_push_value(state, config, CMinusInfinity);
|
|
1239
|
-
} else {
|
|
1240
|
-
raise_parse_error("unexpected token %s", state);
|
|
1241
|
-
}
|
|
1242
|
-
}
|
|
1243
|
-
return json_push_value(state, config, json_parse_negative_number(state, config));
|
|
1244
|
-
break;
|
|
1245
|
-
}
|
|
1246
|
-
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
|
|
1247
|
-
return json_push_value(state, config, json_parse_positive_number(state, config));
|
|
1248
|
-
break;
|
|
1249
|
-
case '"': {
|
|
1250
|
-
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1251
|
-
return json_parse_string(state, config, false);
|
|
1252
|
-
break;
|
|
1253
|
-
}
|
|
1254
|
-
case '[': {
|
|
1255
|
-
state->cursor++;
|
|
1256
|
-
json_eat_whitespace(state);
|
|
1257
|
-
long stack_head = state->stack->head;
|
|
1566
|
+
VALUE value;
|
|
1567
|
+
const char *value_start = state->cursor;
|
|
1258
1568
|
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1569
|
+
switch (peek(state)) {
|
|
1570
|
+
case 'n':
|
|
1571
|
+
json_match_keyword(state, "null", 0);
|
|
1572
|
+
value = Qnil;
|
|
1573
|
+
break;
|
|
1574
|
+
|
|
1575
|
+
case 't':
|
|
1576
|
+
json_match_keyword(state, "true", 0);
|
|
1577
|
+
value = Qtrue;
|
|
1578
|
+
break;
|
|
1579
|
+
|
|
1580
|
+
case 'f':
|
|
1581
|
+
json_match_keyword(state, "false", 1);
|
|
1582
|
+
value = Qfalse;
|
|
1583
|
+
break;
|
|
1584
|
+
|
|
1585
|
+
case 'N':
|
|
1586
|
+
if (!config->allow_nan) {
|
|
1587
|
+
raise_syntax_error("unexpected token %s", state);
|
|
1266
1588
|
}
|
|
1267
|
-
state->in_array++;
|
|
1268
|
-
json_parse_any(state, config);
|
|
1269
|
-
}
|
|
1270
1589
|
|
|
1271
|
-
|
|
1272
|
-
|
|
1590
|
+
json_match_keyword(state, "NaN", 1);
|
|
1591
|
+
value = CNaN;
|
|
1592
|
+
break;
|
|
1273
1593
|
|
|
1274
|
-
|
|
1594
|
+
case 'I':
|
|
1595
|
+
if (!config->allow_nan) {
|
|
1596
|
+
raise_syntax_error("unexpected token %s", state);
|
|
1597
|
+
}
|
|
1275
1598
|
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1599
|
+
json_match_keyword(state, "Infinity", 0);
|
|
1600
|
+
value = CInfinity;
|
|
1601
|
+
break;
|
|
1602
|
+
|
|
1603
|
+
case '-': {
|
|
1604
|
+
state->cursor++;
|
|
1605
|
+
|
|
1606
|
+
value = json_parse_number(state, config, true, value_start);
|
|
1607
|
+
|
|
1608
|
+
if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
|
|
1609
|
+
state->cursor = value_start;
|
|
1610
|
+
json_match_keyword(state, "-Infinity", 1);
|
|
1611
|
+
value = CMinusInfinity;
|
|
1612
|
+
break;
|
|
1286
1613
|
}
|
|
1287
1614
|
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
state->
|
|
1292
|
-
|
|
1293
|
-
return json_push_value(state, config, json_decode_array(state, config, count));
|
|
1615
|
+
// Top level numbers are ambiguous when parsing streams, we can't
|
|
1616
|
+
// know if we parsed all the digits if we hit EOS.
|
|
1617
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1618
|
+
state->cursor = value_start;
|
|
1619
|
+
return false;
|
|
1294
1620
|
}
|
|
1295
1621
|
|
|
1296
|
-
|
|
1622
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1623
|
+
raise_syntax_error_at("invalid number: %s", state, value_start);
|
|
1624
|
+
}
|
|
1625
|
+
break;
|
|
1297
1626
|
}
|
|
1298
|
-
break;
|
|
1299
|
-
}
|
|
1300
|
-
case '{': {
|
|
1301
|
-
const char *object_start_cursor = state->cursor;
|
|
1302
1627
|
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1628
|
+
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
|
|
1629
|
+
value = json_parse_number(state, config, false, value_start);
|
|
1630
|
+
|
|
1631
|
+
// Top level numbers are ambiguous when parsing streams, we can't
|
|
1632
|
+
// know if we parsed all the digits if we hit EOS.
|
|
1633
|
+
if (RB_UNLIKELY(resumable && eos(state))) {
|
|
1634
|
+
state->cursor = value_start;
|
|
1635
|
+
return false;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1639
|
+
raise_syntax_error_at("invalid number: %s", state, value_start);
|
|
1640
|
+
}
|
|
1641
|
+
break;
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
case '"': {
|
|
1645
|
+
// %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
|
|
1646
|
+
value = json_parse_string(state, config, false);
|
|
1647
|
+
|
|
1648
|
+
if (RB_UNLIKELY(UNDEF_P(value))) {
|
|
1649
|
+
bool is_eos = eos(state);
|
|
1650
|
+
if (resumable && is_eos) {
|
|
1651
|
+
state->cursor = value_start;
|
|
1652
|
+
return false;
|
|
1653
|
+
}
|
|
1654
|
+
raise_parse_error("unexpected end of input, expected closing \"", state, is_eos);
|
|
1655
|
+
}
|
|
1656
|
+
break;
|
|
1657
|
+
}
|
|
1306
1658
|
|
|
1307
|
-
|
|
1659
|
+
case '[': {
|
|
1308
1660
|
state->cursor++;
|
|
1309
|
-
|
|
1310
|
-
|
|
1661
|
+
json_eat_whitespace(state, config, true);
|
|
1662
|
+
|
|
1663
|
+
const char next = peek(state);
|
|
1664
|
+
if (next == ']') {
|
|
1665
|
+
state->cursor++;
|
|
1666
|
+
value = json_decode_array(state, config, 0);
|
|
1667
|
+
break;
|
|
1668
|
+
} else if (resumable && eos(state)) {
|
|
1669
|
+
state->cursor = value_start;
|
|
1670
|
+
return false;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1311
1673
|
state->current_nesting++;
|
|
1312
1674
|
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1313
1675
|
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1314
1676
|
}
|
|
1677
|
+
state->in_array++;
|
|
1678
|
+
|
|
1679
|
+
// Phase stays VALUE: the next iteration reads the first element.
|
|
1680
|
+
frame = json_frame_stack_push(state, (json_frame){
|
|
1681
|
+
.type = JSON_FRAME_ARRAY,
|
|
1682
|
+
.phase = JSON_PHASE_VALUE,
|
|
1683
|
+
.value_stack_head = state->value_stack->head,
|
|
1684
|
+
});
|
|
1685
|
+
goto JSON_PHASE_VALUE;
|
|
1686
|
+
}
|
|
1315
1687
|
|
|
1316
|
-
|
|
1317
|
-
|
|
1688
|
+
case '{': {
|
|
1689
|
+
state->cursor++;
|
|
1690
|
+
json_eat_whitespace(state, config, true);
|
|
1691
|
+
|
|
1692
|
+
if (peek(state) == '}') {
|
|
1693
|
+
state->cursor++;
|
|
1694
|
+
value = json_decode_object(state, config, 0);
|
|
1695
|
+
break;
|
|
1696
|
+
} else if (resumable && eos(state)) {
|
|
1697
|
+
state->cursor = value_start;
|
|
1698
|
+
return false;
|
|
1318
1699
|
}
|
|
1319
|
-
json_parse_string(state, config, true);
|
|
1320
1700
|
|
|
1321
|
-
|
|
1322
|
-
if (
|
|
1323
|
-
|
|
1701
|
+
state->current_nesting++;
|
|
1702
|
+
if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
|
|
1703
|
+
rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
|
|
1324
1704
|
}
|
|
1325
|
-
state->cursor++;
|
|
1326
1705
|
|
|
1327
|
-
|
|
1706
|
+
// Phase KEY: the next iteration reads the first key.
|
|
1707
|
+
frame = json_frame_stack_push(state, (json_frame){
|
|
1708
|
+
.type = JSON_FRAME_OBJECT,
|
|
1709
|
+
.phase = JSON_PHASE_OBJECT_KEY,
|
|
1710
|
+
.value_stack_head = state->value_stack->head,
|
|
1711
|
+
.start_offset = value_start - state->start,
|
|
1712
|
+
});
|
|
1713
|
+
goto JSON_PHASE_OBJECT_KEY;
|
|
1328
1714
|
}
|
|
1329
1715
|
|
|
1330
|
-
|
|
1331
|
-
|
|
1716
|
+
case 0:
|
|
1717
|
+
// peek() returns 0 both at end-of-stream and for a literal NUL byte in the
|
|
1718
|
+
// buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at
|
|
1719
|
+
// EOS is just an invalid character.
|
|
1720
|
+
if (eos(state)) {
|
|
1721
|
+
return false;
|
|
1722
|
+
} else {
|
|
1723
|
+
raise_syntax_error("unexpected NULL byte: %s", state);
|
|
1724
|
+
}
|
|
1725
|
+
default:
|
|
1726
|
+
raise_syntax_error("unexpected character: %s", state);
|
|
1727
|
+
}
|
|
1332
1728
|
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
state->cursor++;
|
|
1336
|
-
state->current_nesting--;
|
|
1337
|
-
size_t count = state->stack->head - stack_head;
|
|
1729
|
+
json_push_value(state, config, value);
|
|
1730
|
+
json_value_completed(frame);
|
|
1338
1731
|
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1732
|
+
switch (frame->phase) {
|
|
1733
|
+
case JSON_PHASE_DONE: return true;
|
|
1734
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1735
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1736
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1737
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1738
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1739
|
+
}
|
|
1740
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
JSON_PHASE_OBJECT_KEY: {
|
|
1744
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1344
1745
|
|
|
1345
|
-
|
|
1746
|
+
json_eat_whitespace(state, config, true);
|
|
1747
|
+
|
|
1748
|
+
const char *start = state->cursor;
|
|
1749
|
+
|
|
1750
|
+
if (RB_LIKELY(peek(state) == '"')) {
|
|
1751
|
+
VALUE string = json_parse_string(state, config, true);
|
|
1752
|
+
if (UNDEF_P(string)) {
|
|
1753
|
+
if (resumable) {
|
|
1754
|
+
state->cursor = start;
|
|
1755
|
+
return false;
|
|
1756
|
+
} else {
|
|
1757
|
+
raise_syntax_error("unexpected end of input, expected closing \"", state);
|
|
1346
1758
|
}
|
|
1759
|
+
}
|
|
1760
|
+
json_push_value(state, config, string);
|
|
1761
|
+
frame->phase = JSON_PHASE_OBJECT_COLON;
|
|
1762
|
+
goto JSON_PHASE_OBJECT_COLON;
|
|
1763
|
+
} else if (resumable && eos(state)) {
|
|
1764
|
+
return false;
|
|
1765
|
+
} else {
|
|
1766
|
+
// The message differs for the first key vs. a key after a
|
|
1767
|
+
// ',': the first is the only one reached with nothing pushed
|
|
1768
|
+
// for this object yet.
|
|
1769
|
+
if (json_frame_entry_count(frame, state->value_stack) == 0) {
|
|
1770
|
+
raise_syntax_error("expected object key, got %s", state);
|
|
1771
|
+
} else {
|
|
1772
|
+
raise_syntax_error("expected object key, got: %s", state);
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1776
|
+
}
|
|
1347
1777
|
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
json_eat_whitespace(state);
|
|
1778
|
+
JSON_PHASE_OBJECT_COLON: {
|
|
1779
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1351
1780
|
|
|
1352
|
-
|
|
1353
|
-
if (peek(state) == '}') {
|
|
1354
|
-
continue;
|
|
1355
|
-
}
|
|
1356
|
-
}
|
|
1781
|
+
json_eat_whitespace(state, config, true);
|
|
1357
1782
|
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1783
|
+
if (RB_LIKELY(peek(state) == ':')) {
|
|
1784
|
+
state->cursor++;
|
|
1785
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
1786
|
+
goto JSON_PHASE_VALUE;
|
|
1787
|
+
} else if (resumable && eos(state)) {
|
|
1788
|
+
return false;
|
|
1789
|
+
} else {
|
|
1790
|
+
// First colon (only the first pair's key is pushed, nothing
|
|
1791
|
+
// else) vs. a later one.
|
|
1792
|
+
if (json_frame_entry_count(frame, state->value_stack) == 1) {
|
|
1793
|
+
raise_syntax_error("expected ':' after object key", state);
|
|
1794
|
+
} else {
|
|
1795
|
+
raise_syntax_error("expected ':' after object key, got: %s", state);
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1799
|
+
}
|
|
1362
1800
|
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
raise_parse_error("expected ':' after object key, got: %s", state);
|
|
1366
|
-
}
|
|
1367
|
-
state->cursor++;
|
|
1801
|
+
JSON_PHASE_ARRAY_COMMA: {
|
|
1802
|
+
JSON_ASSERT(frame->type == JSON_FRAME_ARRAY);
|
|
1368
1803
|
|
|
1369
|
-
|
|
1804
|
+
json_eat_whitespace(state, config, true);
|
|
1370
1805
|
|
|
1371
|
-
|
|
1372
|
-
}
|
|
1806
|
+
const char next_char = peek(state);
|
|
1373
1807
|
|
|
1374
|
-
|
|
1808
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1809
|
+
state->cursor++;
|
|
1810
|
+
if (config->allow_trailing_comma) {
|
|
1811
|
+
json_eat_whitespace(state, config, true);
|
|
1812
|
+
if (peek(state) == ']') {
|
|
1813
|
+
// Trailing comma: stay in COMMA to close on the next iteration.
|
|
1814
|
+
goto JSON_PHASE_ARRAY_COMMA;
|
|
1815
|
+
}
|
|
1375
1816
|
}
|
|
1376
|
-
|
|
1817
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
1818
|
+
goto JSON_PHASE_VALUE;
|
|
1819
|
+
} else if (next_char == ']') {
|
|
1820
|
+
state->cursor++;
|
|
1821
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
1822
|
+
state->current_nesting--;
|
|
1823
|
+
state->in_array--;
|
|
1824
|
+
|
|
1825
|
+
json_push_value(state, config, json_decode_array(state, config, count));
|
|
1826
|
+
json_frame_stack_pop(state->frames);
|
|
1827
|
+
frame = json_frame_stack_peek(state->frames);
|
|
1828
|
+
|
|
1829
|
+
json_value_completed(frame);
|
|
1830
|
+
|
|
1831
|
+
switch (frame->phase) {
|
|
1832
|
+
case JSON_PHASE_DONE: return true;
|
|
1833
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1834
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1835
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1836
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1837
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1838
|
+
}
|
|
1839
|
+
} else if (resumable && eos(state)) {
|
|
1840
|
+
return false;
|
|
1841
|
+
} else {
|
|
1842
|
+
raise_syntax_error("expected ',' or ']' after array value", state);
|
|
1377
1843
|
}
|
|
1844
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1845
|
+
}
|
|
1378
1846
|
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
break;
|
|
1847
|
+
JSON_PHASE_OBJECT_COMMA: {
|
|
1848
|
+
JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
|
|
1382
1849
|
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1850
|
+
json_eat_whitespace(state, config, true);
|
|
1851
|
+
const char next_char = peek(state);
|
|
1852
|
+
|
|
1853
|
+
if (RB_LIKELY(next_char == ',')) {
|
|
1854
|
+
state->cursor++;
|
|
1855
|
+
json_eat_whitespace(state, config, true);
|
|
1856
|
+
|
|
1857
|
+
if (config->allow_trailing_comma) {
|
|
1858
|
+
if (peek(state) == '}') {
|
|
1859
|
+
// Trailing comma: stay in COMMA to close on the next iteration.
|
|
1860
|
+
goto JSON_PHASE_OBJECT_COMMA;
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
frame->phase = JSON_PHASE_OBJECT_KEY;
|
|
1865
|
+
goto JSON_PHASE_OBJECT_KEY;
|
|
1866
|
+
} else if (next_char == '}') {
|
|
1867
|
+
state->cursor++;
|
|
1868
|
+
state->current_nesting--;
|
|
1869
|
+
size_t count = json_frame_entry_count(frame, state->value_stack);
|
|
1870
|
+
|
|
1871
|
+
// Temporary rewind cursor in case an error is raised
|
|
1872
|
+
const char *final_cursor = state->cursor;
|
|
1873
|
+
state->cursor = state->start + frame->start_offset;
|
|
1874
|
+
VALUE object = json_decode_object(state, config, count);
|
|
1875
|
+
state->cursor = final_cursor;
|
|
1876
|
+
|
|
1877
|
+
json_push_value(state, config, object);
|
|
1878
|
+
json_frame_stack_pop(state->frames);
|
|
1879
|
+
frame = json_frame_stack_peek(state->frames);
|
|
1880
|
+
json_value_completed(frame);
|
|
1881
|
+
|
|
1882
|
+
switch (frame->phase) {
|
|
1883
|
+
case JSON_PHASE_DONE: return true;
|
|
1884
|
+
case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
|
|
1885
|
+
case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
|
|
1886
|
+
case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
|
|
1887
|
+
case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
|
|
1888
|
+
case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
|
|
1889
|
+
}
|
|
1890
|
+
} else if (resumable && eos(state)) {
|
|
1891
|
+
return false;
|
|
1892
|
+
} else {
|
|
1893
|
+
raise_syntax_error("expected ',' or '}' after object value, got: %s", state);
|
|
1894
|
+
}
|
|
1895
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1386
1896
|
}
|
|
1387
1897
|
|
|
1388
|
-
|
|
1389
|
-
return Qundef;
|
|
1898
|
+
JSON_UNREACHABLE_RETURN(false);
|
|
1390
1899
|
}
|
|
1391
1900
|
|
|
1392
|
-
static void json_ensure_eof(JSON_ParserState *state)
|
|
1901
|
+
static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config)
|
|
1393
1902
|
{
|
|
1394
|
-
json_eat_whitespace(state);
|
|
1903
|
+
json_eat_whitespace(state, config, true);
|
|
1395
1904
|
if (!eos(state)) {
|
|
1396
|
-
|
|
1905
|
+
raise_syntax_error("unexpected token at end of stream %s", state);
|
|
1397
1906
|
}
|
|
1398
1907
|
}
|
|
1399
1908
|
|
|
@@ -1411,39 +1920,59 @@ static void json_ensure_eof(JSON_ParserState *state)
|
|
|
1411
1920
|
|
|
1412
1921
|
static VALUE convert_encoding(VALUE source)
|
|
1413
1922
|
{
|
|
1414
|
-
|
|
1923
|
+
StringValue(source);
|
|
1924
|
+
int encindex = RB_ENCODING_GET(source);
|
|
1925
|
+
|
|
1926
|
+
if (RB_LIKELY(encindex == utf8_encindex)) {
|
|
1927
|
+
return source;
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
if (encindex == binary_encindex) {
|
|
1931
|
+
// For historical reason, we silently reinterpret binary strings as UTF-8
|
|
1932
|
+
return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
|
|
1933
|
+
}
|
|
1415
1934
|
|
|
1416
|
-
|
|
1935
|
+
source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
|
|
1936
|
+
StringValue(source);
|
|
1417
1937
|
return source;
|
|
1418
|
-
|
|
1938
|
+
}
|
|
1419
1939
|
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1940
|
+
struct parser_config_init_args {
|
|
1941
|
+
JSON_ParserConfig *config;
|
|
1942
|
+
VALUE self;
|
|
1943
|
+
VALUE unknown_keywords;
|
|
1944
|
+
bool strict;
|
|
1945
|
+
};
|
|
1424
1946
|
|
|
1425
|
-
|
|
1947
|
+
static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
|
|
1948
|
+
{
|
|
1949
|
+
*dest = val;
|
|
1950
|
+
if (self) RB_OBJ_WRITTEN(self, Qundef, val);
|
|
1426
1951
|
}
|
|
1427
1952
|
|
|
1428
1953
|
static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
1429
1954
|
{
|
|
1430
|
-
|
|
1955
|
+
struct parser_config_init_args *args = (struct parser_config_init_args *)data;
|
|
1956
|
+
JSON_ParserConfig *config = args->config;
|
|
1957
|
+
VALUE self = args->self;
|
|
1431
1958
|
|
|
1432
1959
|
if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
|
|
1433
1960
|
else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
|
|
1434
1961
|
else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
|
|
1962
|
+
else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1435
1963
|
else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
|
|
1964
|
+
else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
|
|
1436
1965
|
else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
|
|
1437
1966
|
else if (key == sym_freeze) { config->freeze = RTEST(val); }
|
|
1438
|
-
else if (key == sym_on_load) { config->on_load_proc
|
|
1967
|
+
else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
|
|
1439
1968
|
else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
|
|
1440
1969
|
else if (key == sym_decimal_class) {
|
|
1441
1970
|
if (RTEST(val)) {
|
|
1442
1971
|
if (rb_respond_to(val, i_try_convert)) {
|
|
1443
|
-
config->decimal_class
|
|
1972
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1444
1973
|
config->decimal_method_id = i_try_convert;
|
|
1445
1974
|
} else if (rb_respond_to(val, i_new)) {
|
|
1446
|
-
config->decimal_class
|
|
1975
|
+
parser_config_wb_write(self, &config->decimal_class, val);
|
|
1447
1976
|
config->decimal_method_id = i_new;
|
|
1448
1977
|
} else if (RB_TYPE_P(val, T_CLASS)) {
|
|
1449
1978
|
VALUE name = rb_class_name(val);
|
|
@@ -1452,7 +1981,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1452
1981
|
if (last_colon) {
|
|
1453
1982
|
const char *mod_path_end = last_colon - 1;
|
|
1454
1983
|
VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
|
|
1455
|
-
config->decimal_class
|
|
1984
|
+
parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
|
|
1456
1985
|
|
|
1457
1986
|
const char *method_name_beg = last_colon + 1;
|
|
1458
1987
|
long before_len = method_name_beg - name_cstr;
|
|
@@ -1460,28 +1989,48 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
|
|
|
1460
1989
|
VALUE method_name = rb_str_substr(name, before_len, len);
|
|
1461
1990
|
config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
|
|
1462
1991
|
} else {
|
|
1463
|
-
config->decimal_class
|
|
1992
|
+
parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
|
|
1464
1993
|
config->decimal_method_id = SYM2ID(rb_str_intern(name));
|
|
1465
1994
|
}
|
|
1466
1995
|
}
|
|
1467
1996
|
}
|
|
1468
1997
|
}
|
|
1998
|
+
else if (args->strict) {
|
|
1999
|
+
if (!args->unknown_keywords) {
|
|
2000
|
+
args->unknown_keywords = rb_obj_hide(rb_ary_new());
|
|
2001
|
+
}
|
|
2002
|
+
rb_ary_push(args->unknown_keywords, key);
|
|
2003
|
+
}
|
|
1469
2004
|
|
|
1470
2005
|
return ST_CONTINUE;
|
|
1471
2006
|
}
|
|
1472
2007
|
|
|
1473
|
-
static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
2008
|
+
static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict)
|
|
1474
2009
|
{
|
|
1475
2010
|
config->max_nesting = 100;
|
|
1476
2011
|
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
rb_hash_foreach(opts, parser_config_init_i, (VALUE)config);
|
|
1483
|
-
}
|
|
2012
|
+
struct parser_config_init_args args = {
|
|
2013
|
+
.config = config,
|
|
2014
|
+
.self = self,
|
|
2015
|
+
.strict = strict,
|
|
2016
|
+
};
|
|
1484
2017
|
|
|
2018
|
+
if (NIL_P(opts)) return;
|
|
2019
|
+
Check_Type(opts, T_HASH);
|
|
2020
|
+
if (RHASH_SIZE(opts) == 0) return;
|
|
2021
|
+
|
|
2022
|
+
// We assume in most cases few keys are set so it's faster to go over
|
|
2023
|
+
// the provided keys than to check all possible keys.
|
|
2024
|
+
rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
|
|
2025
|
+
|
|
2026
|
+
if (RB_UNLIKELY(args.unknown_keywords)) {
|
|
2027
|
+
if (RARRAY_LEN(args.unknown_keywords) == 1) {
|
|
2028
|
+
rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0));
|
|
2029
|
+
}
|
|
2030
|
+
else {
|
|
2031
|
+
VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", "));
|
|
2032
|
+
rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords);
|
|
2033
|
+
}
|
|
1485
2034
|
}
|
|
1486
2035
|
}
|
|
1487
2036
|
|
|
@@ -1490,67 +2039,90 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
|
|
|
1490
2039
|
*
|
|
1491
2040
|
* Creates a new JSON::Ext::ParserConfig instance.
|
|
1492
2041
|
*
|
|
1493
|
-
*
|
|
1494
|
-
*
|
|
2042
|
+
* Argument +opts+, if given, contains a \Hash of options for the parsing.
|
|
2043
|
+
* See {Parsing Options}[#module-JSON-label-Parsing+Options].
|
|
1495
2044
|
*
|
|
1496
|
-
* _opts_ can have the following keys:
|
|
1497
|
-
* * *max_nesting*: The maximum depth of nesting allowed in the parsed data
|
|
1498
|
-
* structures. Disable depth checking with :max_nesting => false|nil|0, it
|
|
1499
|
-
* defaults to 100.
|
|
1500
|
-
* * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
|
|
1501
|
-
* defiance of RFC 4627 to be parsed by the Parser. This option defaults to
|
|
1502
|
-
* false.
|
|
1503
|
-
* * *symbolize_names*: If set to true, returns symbols for the names
|
|
1504
|
-
* (keys) in a JSON object. Otherwise strings are returned, which is
|
|
1505
|
-
* also the default. It's not possible to use this option in
|
|
1506
|
-
* conjunction with the *create_additions* option.
|
|
1507
|
-
* * *decimal_class*: Specifies which class to use instead of the default
|
|
1508
|
-
* (Float) when parsing decimal numbers. This class must accept a single
|
|
1509
|
-
* string argument in its constructor.
|
|
1510
2045
|
*/
|
|
1511
2046
|
static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
|
|
1512
2047
|
{
|
|
1513
2048
|
rb_check_frozen(self);
|
|
1514
2049
|
GET_PARSER_CONFIG;
|
|
1515
2050
|
|
|
1516
|
-
parser_config_init(config, opts);
|
|
1517
|
-
|
|
1518
|
-
RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
|
|
2051
|
+
parser_config_init(config, opts, self, false);
|
|
1519
2052
|
|
|
1520
2053
|
return self;
|
|
1521
2054
|
}
|
|
1522
2055
|
|
|
1523
|
-
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE
|
|
2056
|
+
static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
|
|
1524
2057
|
{
|
|
1525
|
-
Vsource = convert_encoding(
|
|
1526
|
-
|
|
2058
|
+
VALUE Vsource = convert_encoding(src);
|
|
2059
|
+
|
|
2060
|
+
// Ensure the string isn't mutated under us.
|
|
2061
|
+
// The classic API to use is `rb_str_locktmp`, but then we'd
|
|
2062
|
+
// need to use `rb_protect` to make sure we always unlock.
|
|
2063
|
+
if (Vsource == src) {
|
|
2064
|
+
Vsource = rb_str_new_frozen(Vsource);
|
|
2065
|
+
}
|
|
1527
2066
|
|
|
1528
2067
|
VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
|
|
1529
|
-
rvalue_stack
|
|
2068
|
+
rvalue_stack value_stack = {
|
|
1530
2069
|
.type = RVALUE_STACK_STACK_ALLOCATED,
|
|
1531
2070
|
.ptr = rvalue_stack_buffer,
|
|
1532
2071
|
.capa = RVALUE_STACK_INITIAL_CAPA,
|
|
1533
2072
|
};
|
|
1534
2073
|
|
|
2074
|
+
// Seed the frame stack with the root frame, establishing the invariant that
|
|
2075
|
+
// json_parse_any always has a top frame to dispatch on (so the stack is never
|
|
2076
|
+
// empty mid-parse).
|
|
2077
|
+
json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA];
|
|
2078
|
+
frame_stack_buffer[0] = (json_frame){
|
|
2079
|
+
.type = JSON_FRAME_ROOT,
|
|
2080
|
+
.phase = JSON_PHASE_VALUE,
|
|
2081
|
+
};
|
|
2082
|
+
json_frame_stack frames = {
|
|
2083
|
+
.type = RVALUE_STACK_STACK_ALLOCATED,
|
|
2084
|
+
.ptr = frame_stack_buffer,
|
|
2085
|
+
.capa = JSON_FRAME_STACK_INITIAL_CAPA,
|
|
2086
|
+
.head = 1,
|
|
2087
|
+
};
|
|
2088
|
+
|
|
1535
2089
|
long len;
|
|
1536
2090
|
const char *start;
|
|
2091
|
+
|
|
1537
2092
|
RSTRING_GETMEM(Vsource, start, len);
|
|
1538
2093
|
|
|
2094
|
+
VALUE value_stack_handle = 0;
|
|
2095
|
+
VALUE frame_stack_handle = 0;
|
|
1539
2096
|
JSON_ParserState _state = {
|
|
1540
2097
|
.start = start,
|
|
1541
2098
|
.cursor = start,
|
|
1542
2099
|
.end = start + len,
|
|
1543
|
-
.
|
|
2100
|
+
.value_stack = &value_stack,
|
|
2101
|
+
.value_stack_handle = &value_stack_handle,
|
|
2102
|
+
.frames = &frames,
|
|
2103
|
+
.frame_stack_handle = &frame_stack_handle,
|
|
1544
2104
|
};
|
|
1545
2105
|
JSON_ParserState *state = &_state;
|
|
1546
2106
|
|
|
1547
|
-
|
|
2107
|
+
bool complete = json_parse_any(state, config, false);
|
|
2108
|
+
|
|
2109
|
+
// The root document value is parsed; it is the lone survivor on
|
|
2110
|
+
// the rvalue stack.
|
|
2111
|
+
VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
|
|
1548
2112
|
|
|
1549
2113
|
// This may be skipped in case of exception, but
|
|
1550
2114
|
// it won't cause a leak.
|
|
1551
|
-
rvalue_stack_eagerly_release(
|
|
1552
|
-
|
|
1553
|
-
|
|
2115
|
+
rvalue_stack_eagerly_release(value_stack_handle);
|
|
2116
|
+
json_frame_stack_eagerly_release(frame_stack_handle);
|
|
2117
|
+
RB_GC_GUARD(value_stack_handle);
|
|
2118
|
+
RB_GC_GUARD(frame_stack_handle);
|
|
2119
|
+
RB_GC_GUARD(Vsource);
|
|
2120
|
+
|
|
2121
|
+
if (complete) {
|
|
2122
|
+
json_ensure_eof(state, config);
|
|
2123
|
+
} else {
|
|
2124
|
+
raise_eos_error("unexpected end of input", state);
|
|
2125
|
+
}
|
|
1554
2126
|
|
|
1555
2127
|
return result;
|
|
1556
2128
|
}
|
|
@@ -1570,12 +2142,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
|
|
|
1570
2142
|
|
|
1571
2143
|
static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
1572
2144
|
{
|
|
1573
|
-
Vsource = convert_encoding(StringValue(Vsource));
|
|
1574
|
-
StringValue(Vsource);
|
|
1575
|
-
|
|
1576
2145
|
JSON_ParserConfig _config = {0};
|
|
1577
2146
|
JSON_ParserConfig *config = &_config;
|
|
1578
|
-
parser_config_init(config, opts);
|
|
2147
|
+
parser_config_init(config, opts, Qfalse, false);
|
|
1579
2148
|
|
|
1580
2149
|
return cParser_parse(config, Vsource);
|
|
1581
2150
|
}
|
|
@@ -1583,30 +2152,35 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
|
|
|
1583
2152
|
static void JSON_ParserConfig_mark(void *ptr)
|
|
1584
2153
|
{
|
|
1585
2154
|
JSON_ParserConfig *config = ptr;
|
|
1586
|
-
|
|
1587
|
-
|
|
2155
|
+
rb_gc_mark_movable(config->on_load_proc);
|
|
2156
|
+
rb_gc_mark_movable(config->decimal_class);
|
|
1588
2157
|
}
|
|
1589
2158
|
|
|
1590
|
-
static
|
|
2159
|
+
static size_t JSON_ParserConfig_memsize(const void *ptr)
|
|
1591
2160
|
{
|
|
1592
|
-
|
|
1593
|
-
|
|
2161
|
+
#ifdef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
2162
|
+
return 0;
|
|
2163
|
+
#else
|
|
2164
|
+
return sizeof(JSON_ParserConfig);
|
|
2165
|
+
#endif
|
|
1594
2166
|
}
|
|
1595
2167
|
|
|
1596
|
-
static
|
|
2168
|
+
static void JSON_ParserConfig_compact(void *ptr)
|
|
1597
2169
|
{
|
|
1598
|
-
|
|
2170
|
+
JSON_ParserConfig *config = ptr;
|
|
2171
|
+
config->on_load_proc = rb_gc_location(config->on_load_proc);
|
|
2172
|
+
config->decimal_class = rb_gc_location(config->decimal_class);
|
|
1599
2173
|
}
|
|
1600
2174
|
|
|
1601
2175
|
static const rb_data_type_t JSON_ParserConfig_type = {
|
|
1602
|
-
"JSON::Ext::Parser/ParserConfig",
|
|
1603
|
-
{
|
|
1604
|
-
JSON_ParserConfig_mark,
|
|
1605
|
-
|
|
1606
|
-
JSON_ParserConfig_memsize,
|
|
2176
|
+
.wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
|
|
2177
|
+
.function = {
|
|
2178
|
+
.dmark = JSON_ParserConfig_mark,
|
|
2179
|
+
.dfree = RUBY_DEFAULT_FREE,
|
|
2180
|
+
.dsize = JSON_ParserConfig_memsize,
|
|
2181
|
+
.dcompact = JSON_ParserConfig_compact,
|
|
1607
2182
|
},
|
|
1608
|
-
|
|
1609
|
-
RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE,
|
|
2183
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
|
|
1610
2184
|
};
|
|
1611
2185
|
|
|
1612
2186
|
static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
@@ -1615,6 +2189,562 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
|
|
|
1615
2189
|
return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config);
|
|
1616
2190
|
}
|
|
1617
2191
|
|
|
2192
|
+
static void json_str_clear(VALUE str)
|
|
2193
|
+
{
|
|
2194
|
+
if (RB_OBJ_FROZEN_RAW(str)) {
|
|
2195
|
+
return;
|
|
2196
|
+
}
|
|
2197
|
+
rb_str_replace(str, JSON_empty_string);
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
typedef struct JSON_ResumableParserStruct {
|
|
2201
|
+
JSON_ParserConfig config;
|
|
2202
|
+
JSON_ParserState state;
|
|
2203
|
+
rvalue_stack value_stack;
|
|
2204
|
+
json_frame_stack frames;
|
|
2205
|
+
VALUE buffer;
|
|
2206
|
+
size_t parsed_bytes;
|
|
2207
|
+
size_t incomplete_bytes;
|
|
2208
|
+
bool complete;
|
|
2209
|
+
bool in_use;
|
|
2210
|
+
} JSON_ResumableParser;
|
|
2211
|
+
|
|
2212
|
+
static void JSON_ResumableParser_mark(void *ptr)
|
|
2213
|
+
{
|
|
2214
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2215
|
+
JSON_ParserConfig_mark(&parser->config);
|
|
2216
|
+
rvalue_stack_mark(&parser->value_stack);
|
|
2217
|
+
rvalue_cache_mark(&parser->state.name_cache);
|
|
2218
|
+
rb_gc_mark(parser->buffer); // pin the buffer
|
|
2219
|
+
rb_gc_mark_movable(parser->state.parser);
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
static void JSON_ResumableParser_free(void *ptr)
|
|
2223
|
+
{
|
|
2224
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2225
|
+
rvalue_stack_free_buffer(&parser->value_stack);
|
|
2226
|
+
json_frame_stack_free_buffer(&parser->frames);
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
static size_t JSON_ResumableParser_memsize(const void *ptr)
|
|
2230
|
+
{
|
|
2231
|
+
const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr;
|
|
2232
|
+
size_t memsize = JSON_ParserConfig_memsize(&parser->config);
|
|
2233
|
+
memsize += rvalue_stack_memsize(&parser->value_stack);
|
|
2234
|
+
memsize += json_frame_stack_memsize(&parser->frames);
|
|
2235
|
+
#ifndef HAVE_RUBY_TYPED_EMBEDDABLE
|
|
2236
|
+
memsize += (
|
|
2237
|
+
sizeof(JSON_ResumableParser)
|
|
2238
|
+
- sizeof(JSON_ParserState)
|
|
2239
|
+
- sizeof(JSON_ParserConfig)
|
|
2240
|
+
- sizeof(rvalue_stack)
|
|
2241
|
+
- sizeof(json_frame_stack)
|
|
2242
|
+
);
|
|
2243
|
+
#endif
|
|
2244
|
+
return memsize;
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
static void JSON_ResumableParser_compact(void *ptr)
|
|
2248
|
+
{
|
|
2249
|
+
JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
|
|
2250
|
+
JSON_ParserConfig_compact(&parser->config);
|
|
2251
|
+
rvalue_stack_compact(&parser->value_stack);
|
|
2252
|
+
rvalue_cache_compact(&parser->state.name_cache);
|
|
2253
|
+
parser->buffer = rb_gc_location(parser->buffer);
|
|
2254
|
+
parser->state.parser = rb_gc_location(parser->state.parser);
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
static const rb_data_type_t JSON_ResumableParser_type = {
|
|
2258
|
+
.wrap_struct_name = "JSON::Ext::ResumableParser",
|
|
2259
|
+
.function = {
|
|
2260
|
+
JSON_ResumableParser_mark,
|
|
2261
|
+
JSON_ResumableParser_free,
|
|
2262
|
+
JSON_ResumableParser_memsize,
|
|
2263
|
+
JSON_ResumableParser_compact,
|
|
2264
|
+
},
|
|
2265
|
+
// RUBY_TYPED_WB_PROTECTED is deliberately not declared because
|
|
2266
|
+
// this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
|
|
2267
|
+
// to trigger a lot of write barriers.
|
|
2268
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
|
|
2269
|
+
};
|
|
2270
|
+
|
|
2271
|
+
static VALUE cResumableParser_allocate(VALUE klass)
|
|
2272
|
+
{
|
|
2273
|
+
JSON_ResumableParser *parser;
|
|
2274
|
+
VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
|
|
2275
|
+
parser->state.in_array++;
|
|
2276
|
+
parser->state.parser = obj;
|
|
2277
|
+
return obj;
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
static inline JSON_ResumableParser *cResumableParser_get(VALUE self)
|
|
2281
|
+
{
|
|
2282
|
+
JSON_ResumableParser *parser;
|
|
2283
|
+
TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
|
|
2284
|
+
return parser;
|
|
2285
|
+
}
|
|
2286
|
+
|
|
2287
|
+
/*
|
|
2288
|
+
* call-seq: new(opts => {})
|
|
2289
|
+
*
|
|
2290
|
+
* Creates a new JSON::ResumableParser instance.
|
|
2291
|
+
*
|
|
2292
|
+
* Argument +opts+, if given, contains a \Hash of options for the parsing.
|
|
2293
|
+
* See {Parsing Options}[#module-JSON-label-Parsing+Options].
|
|
2294
|
+
*
|
|
2295
|
+
* A ResumableParser is able to parse partial documents and resume parsing later
|
|
2296
|
+
* when more of the document is provided:
|
|
2297
|
+
*
|
|
2298
|
+
* parser = JSON::ResumableParser.new
|
|
2299
|
+
* parser << '{"user": "george", "role": "ad'
|
|
2300
|
+
* parser.parse # => false
|
|
2301
|
+
* parser.eos? # => true
|
|
2302
|
+
* parser.partial_value # => { "user" => "george", "role" => nil }
|
|
2303
|
+
* parser.rest # => '"ad'
|
|
2304
|
+
*
|
|
2305
|
+
* parser << 'min" }[1, 2, 3]'
|
|
2306
|
+
* parser.parse # => true
|
|
2307
|
+
* parser.value # => { "user" => "george", "role" => "admin" }
|
|
2308
|
+
*
|
|
2309
|
+
* parser.parse # => true
|
|
2310
|
+
* parser.value # => [1, 2, 3]
|
|
2311
|
+
*
|
|
2312
|
+
* === Limitations
|
|
2313
|
+
*
|
|
2314
|
+
* While ResumableParser is able to parse streams of documents without any
|
|
2315
|
+
* explicit separators between them, it is highly recommended to separate documents
|
|
2316
|
+
* by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous.
|
|
2317
|
+
* When parsing a number, ResumableParser will not consider the number complete until something follows:
|
|
2318
|
+
*
|
|
2319
|
+
* parser << '123'
|
|
2320
|
+
* parser.parse # => false
|
|
2321
|
+
* parser << ' '
|
|
2322
|
+
* parser.parse # => true
|
|
2323
|
+
* parser.value # => 123
|
|
2324
|
+
*
|
|
2325
|
+
* === Security
|
|
2326
|
+
*
|
|
2327
|
+
* An incomplete document is buffered in full and there is no size limit, so when reading
|
|
2328
|
+
* from an untrusted source the caller is responsible for bounding how much data is fed.
|
|
2329
|
+
* For example:
|
|
2330
|
+
*
|
|
2331
|
+
* loop do
|
|
2332
|
+
* if parser.parsed_bytes > DOCUMENT_MAX_SIZE
|
|
2333
|
+
* raise "document too large"
|
|
2334
|
+
* end
|
|
2335
|
+
*
|
|
2336
|
+
* parser << read_chunk
|
|
2337
|
+
* while parser.parse
|
|
2338
|
+
* process(parser.value)
|
|
2339
|
+
* end
|
|
2340
|
+
* end
|
|
2341
|
+
*/
|
|
2342
|
+
static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
|
|
2343
|
+
{
|
|
2344
|
+
rb_check_frozen(self);
|
|
2345
|
+
|
|
2346
|
+
VALUE opts = Qfalse;
|
|
2347
|
+
rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts);
|
|
2348
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2349
|
+
|
|
2350
|
+
opts = argc > 0 ? argv[0] : Qnil;
|
|
2351
|
+
parser_config_init(&parser->config, opts, self, true);
|
|
2352
|
+
|
|
2353
|
+
return self;
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock);
|
|
2357
|
+
|
|
2358
|
+
/*
|
|
2359
|
+
* call-seq: self << string -> self
|
|
2360
|
+
*
|
|
2361
|
+
* Appends the given string to the parser's buffer.
|
|
2362
|
+
*/
|
|
2363
|
+
static VALUE cResumableParser_feed(VALUE self, VALUE str)
|
|
2364
|
+
{
|
|
2365
|
+
rb_check_frozen(self);
|
|
2366
|
+
|
|
2367
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2368
|
+
|
|
2369
|
+
str = convert_encoding(str);
|
|
2370
|
+
if (!RSTRING_LEN(str)) {
|
|
2371
|
+
return self;
|
|
2372
|
+
}
|
|
2373
|
+
|
|
2374
|
+
size_t offset = parser->state.cursor - parser->state.start;
|
|
2375
|
+
const size_t remaining = parser->state.end - parser->state.cursor;
|
|
2376
|
+
|
|
2377
|
+
if (!remaining) {
|
|
2378
|
+
if (parser->buffer) {
|
|
2379
|
+
json_str_clear(parser->buffer);
|
|
2380
|
+
}
|
|
2381
|
+
parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
|
|
2382
|
+
offset = 0;
|
|
2383
|
+
} else {
|
|
2384
|
+
JSON_ASSERT(parser->buffer);
|
|
2385
|
+
|
|
2386
|
+
const size_t size = parser->state.end - parser->state.start;
|
|
2387
|
+
const size_t consumed = size - remaining;
|
|
2388
|
+
|
|
2389
|
+
if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
|
|
2390
|
+
VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
|
|
2391
|
+
rb_enc_associate_index(new_buffer, utf8_encindex);
|
|
2392
|
+
|
|
2393
|
+
char *old_ptr = RSTRING_PTR(parser->buffer);
|
|
2394
|
+
memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
|
|
2395
|
+
rb_str_set_len(new_buffer, remaining);
|
|
2396
|
+
offset = 0;
|
|
2397
|
+
parser->buffer = new_buffer;
|
|
2398
|
+
} else if (consumed > (size / 2) && size >= 512) {
|
|
2399
|
+
rb_str_modify(parser->buffer);
|
|
2400
|
+
char *old_ptr = RSTRING_PTR(parser->buffer);
|
|
2401
|
+
memmove(old_ptr, old_ptr + consumed, remaining);
|
|
2402
|
+
rb_str_set_len(parser->buffer, remaining);
|
|
2403
|
+
offset = 0;
|
|
2404
|
+
}
|
|
2405
|
+
rb_str_append(parser->buffer, str);
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
long len;
|
|
2409
|
+
const char *start;
|
|
2410
|
+
RSTRING_GETMEM(parser->buffer, start, len);
|
|
2411
|
+
parser->state.start = start;
|
|
2412
|
+
parser->state.end = start + len;
|
|
2413
|
+
parser->state.cursor = parser->state.start + offset;
|
|
2414
|
+
|
|
2415
|
+
return self;
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
struct json_parse_any_args {
|
|
2419
|
+
JSON_ParserState *state;
|
|
2420
|
+
JSON_ParserConfig *config;
|
|
2421
|
+
VALUE parser;
|
|
2422
|
+
};
|
|
2423
|
+
|
|
2424
|
+
static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args))
|
|
2425
|
+
{
|
|
2426
|
+
struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
|
|
2427
|
+
return (VALUE)json_parse_any(args->state, args->config, true);
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
static VALUE json_parse_any_resumable_safe(VALUE _args)
|
|
2431
|
+
{
|
|
2432
|
+
struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
|
|
2433
|
+
VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args);
|
|
2434
|
+
return result == args->parser ? Qfalse : result;
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock)
|
|
2438
|
+
{
|
|
2439
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2440
|
+
|
|
2441
|
+
if (parser->in_use) {
|
|
2442
|
+
rb_raise(rb_eArgError, "ResumableParser can't be used recursively");
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
if (lock) {
|
|
2446
|
+
parser->in_use = true;
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
// self may have moved, so we need to update all pointers
|
|
2450
|
+
// Investigate: We might be better off keeping JSON_ParserState on the stack
|
|
2451
|
+
// and only persist what we need.
|
|
2452
|
+
parser->state.value_stack = &parser->value_stack;
|
|
2453
|
+
parser->state.frames = &parser->frames;
|
|
2454
|
+
|
|
2455
|
+
return parser;
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
/*
|
|
2459
|
+
* call-seq: parse -> true or false
|
|
2460
|
+
*
|
|
2461
|
+
* Attemps to parse a JSON document from the internal buffer.
|
|
2462
|
+
* Returns whether a complete document could be parsed.
|
|
2463
|
+
*
|
|
2464
|
+
* It does raise +JSON::ParserError+ when encountering invalid \JSON syntax.
|
|
2465
|
+
*
|
|
2466
|
+
* The parsed object can be retrieved by calling #value
|
|
2467
|
+
*/
|
|
2468
|
+
static VALUE cResumableParser_parse(VALUE self)
|
|
2469
|
+
{
|
|
2470
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
|
|
2471
|
+
|
|
2472
|
+
if (parser->complete) {
|
|
2473
|
+
parser->parsed_bytes = 0;
|
|
2474
|
+
parser->incomplete_bytes = 0;
|
|
2475
|
+
parser->complete = false;
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
if (!parser->buffer) {
|
|
2479
|
+
parser->in_use = false;
|
|
2480
|
+
return Qfalse;
|
|
2481
|
+
}
|
|
2482
|
+
|
|
2483
|
+
if (parser->frames.head == 0) {
|
|
2484
|
+
json_frame_stack_push(&parser->state, (json_frame){
|
|
2485
|
+
.type = JSON_FRAME_ROOT,
|
|
2486
|
+
.phase = JSON_PHASE_VALUE,
|
|
2487
|
+
});
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2490
|
+
VALUE Vsource = parser->buffer; // Prevent compaction
|
|
2491
|
+
|
|
2492
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2493
|
+
|
|
2494
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2495
|
+
JSON_ASSERT(parser->value_stack.head == 1);
|
|
2496
|
+
JSON_ASSERT(parser->frames.head == 1);
|
|
2497
|
+
|
|
2498
|
+
frame->phase = JSON_PHASE_VALUE;
|
|
2499
|
+
rvalue_stack_pop(parser->state.value_stack, 1);
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
struct json_parse_any_args args = {
|
|
2503
|
+
.state = &parser->state,
|
|
2504
|
+
.config = &parser->config,
|
|
2505
|
+
.parser = self,
|
|
2506
|
+
};
|
|
2507
|
+
int status;
|
|
2508
|
+
const char *initial_cursor = parser->state.cursor;
|
|
2509
|
+
parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
|
|
2510
|
+
|
|
2511
|
+
if (status) {
|
|
2512
|
+
parser->complete = true; // a parse error is considered complete
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
parser->parsed_bytes += parser->state.cursor - initial_cursor;
|
|
2516
|
+
parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;
|
|
2517
|
+
|
|
2518
|
+
json_eat_whitespace(&parser->state, &parser->config, false);
|
|
2519
|
+
if (eos(&parser->state)) {
|
|
2520
|
+
json_str_clear(parser->buffer);
|
|
2521
|
+
parser->buffer = Qfalse;
|
|
2522
|
+
}
|
|
2523
|
+
parser->in_use = false;
|
|
2524
|
+
|
|
2525
|
+
if (status) {
|
|
2526
|
+
rb_jump_tag(status); // reraise
|
|
2527
|
+
}
|
|
2528
|
+
RB_GC_GUARD(Vsource);
|
|
2529
|
+
return parser->complete ? Qtrue : Qfalse;
|
|
2530
|
+
}
|
|
2531
|
+
|
|
2532
|
+
/*
|
|
2533
|
+
* call-seq: value? -> true or false
|
|
2534
|
+
*
|
|
2535
|
+
* Returns whether a parsed value is available.
|
|
2536
|
+
*/
|
|
2537
|
+
static VALUE cResumableParser_value_p(VALUE self)
|
|
2538
|
+
{
|
|
2539
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2540
|
+
|
|
2541
|
+
if (parser->value_stack.head > 0) {
|
|
2542
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2543
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2544
|
+
return Qtrue;
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
return Qfalse;
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
/*
|
|
2551
|
+
* call-seq: value -> object
|
|
2552
|
+
*
|
|
2553
|
+
* Returns and consume the last parsed value.
|
|
2554
|
+
* Raises ArgumentError if there is no parsed value or if it was already retrieved:
|
|
2555
|
+
* parser << '[1][2]'
|
|
2556
|
+
* parser.value # ArgumentError no ready value
|
|
2557
|
+
* parser.parse # => true
|
|
2558
|
+
* parser.value # => [1]
|
|
2559
|
+
* parser.value # ArgumentError no ready value
|
|
2560
|
+
*/
|
|
2561
|
+
static VALUE cResumableParser_value(VALUE self)
|
|
2562
|
+
{
|
|
2563
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2564
|
+
|
|
2565
|
+
if (parser->frames.head > 0) {
|
|
2566
|
+
json_frame *frame = json_frame_stack_peek(&parser->frames);
|
|
2567
|
+
|
|
2568
|
+
if (frame->phase == JSON_PHASE_DONE) {
|
|
2569
|
+
VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
|
|
2570
|
+
rvalue_stack_pop(parser->state.value_stack, 1);
|
|
2571
|
+
json_frame_stack_pop(parser->state.frames);
|
|
2572
|
+
return result;
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
rb_raise(rb_eArgError, "no ready value");
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
/*
|
|
2579
|
+
* call-seq: clear -> self
|
|
2580
|
+
*
|
|
2581
|
+
* Entirely reset the parser state and buffer.
|
|
2582
|
+
*/
|
|
2583
|
+
static VALUE cResumableParser_clear(VALUE self)
|
|
2584
|
+
{
|
|
2585
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
|
|
2586
|
+
parser->buffer = 0;
|
|
2587
|
+
parser->complete = true;
|
|
2588
|
+
parser->parsed_bytes = 0;
|
|
2589
|
+
parser->incomplete_bytes = 0;
|
|
2590
|
+
parser->frames.head = 0;
|
|
2591
|
+
parser->value_stack.head = 0;
|
|
2592
|
+
parser->state.name_cache.length = 0;
|
|
2593
|
+
parser->state.current_nesting = 0;
|
|
2594
|
+
parser->state.in_array = 1;
|
|
2595
|
+
parser->state.emitted_deprecations = 0;
|
|
2596
|
+
parser->state.start = parser->state.cursor = parser->state.end = NULL;
|
|
2597
|
+
return self;
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
static VALUE cResumableParser_partial_value_body(VALUE self)
|
|
2601
|
+
{
|
|
2602
|
+
JSON_ResumableParser *original_parser = cResumableParser_get(self);
|
|
2603
|
+
JSON_ResumableParser parser = *original_parser;
|
|
2604
|
+
|
|
2605
|
+
parser.state.frames = &parser.frames;
|
|
2606
|
+
parser.state.value_stack = &parser.value_stack;
|
|
2607
|
+
|
|
2608
|
+
if (parser.value_stack.head == 0) {
|
|
2609
|
+
return Qnil;
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
json_frame *frame = json_frame_stack_peek(parser.state.frames);
|
|
2613
|
+
long missing_object_value = 0;
|
|
2614
|
+
if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
|
|
2615
|
+
missing_object_value = 1;
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
// Copy the value stack as we need to mutate it.
|
|
2619
|
+
long capa = parser.value_stack.head;
|
|
2620
|
+
parser.value_stack.capa = (capa + missing_object_value);
|
|
2621
|
+
VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, capa + missing_object_value);
|
|
2622
|
+
MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, parser.value_stack.capa);
|
|
2623
|
+
parser.value_stack.ptr = value_stack_buffer;
|
|
2624
|
+
|
|
2625
|
+
JSON_ParserState *state = &parser.state;
|
|
2626
|
+
JSON_ParserConfig *config = &parser.config;
|
|
2627
|
+
|
|
2628
|
+
if (missing_object_value) {
|
|
2629
|
+
rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
VALUE partial_result = Qundef;
|
|
2633
|
+
|
|
2634
|
+
while (UNDEF_P(partial_result)) {
|
|
2635
|
+
frame = json_frame_stack_peek(state->frames);
|
|
2636
|
+
|
|
2637
|
+
switch (frame->type) {
|
|
2638
|
+
case JSON_FRAME_ROOT: {
|
|
2639
|
+
partial_result = *rvalue_stack_peek(state->value_stack, 1);
|
|
2640
|
+
break;
|
|
2641
|
+
}
|
|
2642
|
+
|
|
2643
|
+
case JSON_FRAME_ARRAY: {
|
|
2644
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
2645
|
+
json_push_value(state, config, json_decode_array(state, config, count));
|
|
2646
|
+
json_frame_stack_pop(state->frames);
|
|
2647
|
+
|
|
2648
|
+
break;
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
case JSON_FRAME_OBJECT: {
|
|
2652
|
+
long count = json_frame_entry_count(frame, state->value_stack);
|
|
2653
|
+
json_push_value(state, config, json_decode_object(state, config, count));
|
|
2654
|
+
json_frame_stack_pop(state->frames);
|
|
2655
|
+
break;
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2658
|
+
default: {
|
|
2659
|
+
JSON_UNREACHABLE_RETURN(Qundef);
|
|
2660
|
+
break;
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
ALLOCV_END(tmpbuf);
|
|
2666
|
+
return partial_result;
|
|
2667
|
+
}
|
|
2668
|
+
|
|
2669
|
+
/*
|
|
2670
|
+
* call-seq: partial_value -> object
|
|
2671
|
+
*
|
|
2672
|
+
* Returns the Ruby objects parsed up to this point:
|
|
2673
|
+
* parser << '[1, [2, 3,'
|
|
2674
|
+
* parser.parse # => false
|
|
2675
|
+
* parser.value # ArgumentError no ready value
|
|
2676
|
+
* parser.partial_value # => [1, [2, 3]]
|
|
2677
|
+
*/
|
|
2678
|
+
static VALUE cResumableParser_partial_value(VALUE self)
|
|
2679
|
+
{
|
|
2680
|
+
JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
|
|
2681
|
+
|
|
2682
|
+
int status;
|
|
2683
|
+
VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
|
|
2684
|
+
parser->in_use = false;
|
|
2685
|
+
if (status) {
|
|
2686
|
+
rb_jump_tag(status);
|
|
2687
|
+
}
|
|
2688
|
+
return result;
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
/*
|
|
2692
|
+
* call-seq: rest -> string
|
|
2693
|
+
*
|
|
2694
|
+
* Returns a string containing what remains to be parsed in the buffer
|
|
2695
|
+
* parser << '{ "message": "unterminated message'
|
|
2696
|
+
* parser.parse # => false
|
|
2697
|
+
* parser.rest # => '"unterminated message"'
|
|
2698
|
+
*/
|
|
2699
|
+
static VALUE cResumableParser_rest(VALUE self)
|
|
2700
|
+
{
|
|
2701
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2702
|
+
|
|
2703
|
+
if (!parser->buffer) {
|
|
2704
|
+
return rb_utf8_str_new("", 0);
|
|
2705
|
+
}
|
|
2706
|
+
|
|
2707
|
+
size_t offset = parser->state.cursor - parser->state.start;
|
|
2708
|
+
const char *ptr;
|
|
2709
|
+
long len;
|
|
2710
|
+
RSTRING_GETMEM(parser->buffer, ptr, len);
|
|
2711
|
+
return rb_utf8_str_new(ptr + offset, len - offset);
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
/*
|
|
2715
|
+
* call-seq: value? -> true or false
|
|
2716
|
+
*
|
|
2717
|
+
* Returns whether the internal buffer has been entirely consumed.
|
|
2718
|
+
*/
|
|
2719
|
+
static VALUE cResumableParser_eos_p(VALUE self)
|
|
2720
|
+
{
|
|
2721
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2722
|
+
return eos(&parser->state) ? Qtrue : Qfalse;
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
/*
|
|
2726
|
+
* call-seq: parsed_bytes -> integer
|
|
2727
|
+
*
|
|
2728
|
+
* Returns the number of bytes parsed since the start of the current partial value.
|
|
2729
|
+
* This is intended to be used for securing against untrusted input:
|
|
2730
|
+
*
|
|
2731
|
+
* loop do
|
|
2732
|
+
* if parser.parsed_bytes > DOCUMENT_MAX_SIZE
|
|
2733
|
+
* raise "document too large"
|
|
2734
|
+
* end
|
|
2735
|
+
*
|
|
2736
|
+
* parser << read_chunk
|
|
2737
|
+
* while parser.parse
|
|
2738
|
+
* process(parser.value)
|
|
2739
|
+
* end
|
|
2740
|
+
* end
|
|
2741
|
+
*/
|
|
2742
|
+
static VALUE cResumableParser_parsed_bytes(VALUE self)
|
|
2743
|
+
{
|
|
2744
|
+
JSON_ResumableParser *parser = cResumableParser_get(self);
|
|
2745
|
+
return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
|
|
2746
|
+
}
|
|
2747
|
+
|
|
1618
2748
|
void Init_parser(void)
|
|
1619
2749
|
{
|
|
1620
2750
|
#ifdef HAVE_RB_EXT_RACTOR_SAFE
|
|
@@ -1626,31 +2756,54 @@ void Init_parser(void)
|
|
|
1626
2756
|
mJSON = rb_define_module("JSON");
|
|
1627
2757
|
VALUE mExt = rb_define_module_under(mJSON, "Ext");
|
|
1628
2758
|
VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject);
|
|
2759
|
+
|
|
2760
|
+
rb_global_variable(&eParserError);
|
|
2761
|
+
eParserError = rb_path2class("JSON::ParserError");
|
|
2762
|
+
|
|
2763
|
+
rb_global_variable(&eNestingError);
|
|
1629
2764
|
eNestingError = rb_path2class("JSON::NestingError");
|
|
1630
|
-
|
|
2765
|
+
|
|
1631
2766
|
rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate);
|
|
1632
|
-
|
|
2767
|
+
rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
|
|
1633
2768
|
rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1);
|
|
1634
2769
|
|
|
1635
2770
|
VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject);
|
|
1636
2771
|
rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2);
|
|
1637
2772
|
|
|
2773
|
+
VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject);
|
|
2774
|
+
rb_define_alloc_func(cResumableParser, cResumableParser_allocate);
|
|
2775
|
+
rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1);
|
|
2776
|
+
rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1);
|
|
2777
|
+
rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0);
|
|
2778
|
+
rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
|
|
2779
|
+
rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
|
|
2780
|
+
rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
|
|
2781
|
+
rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
|
|
2782
|
+
rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
|
|
2783
|
+
rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
|
|
2784
|
+
rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0);
|
|
2785
|
+
|
|
2786
|
+
rb_global_variable(&CNaN);
|
|
1638
2787
|
CNaN = rb_const_get(mJSON, rb_intern("NaN"));
|
|
1639
|
-
rb_gc_register_mark_object(CNaN);
|
|
1640
2788
|
|
|
2789
|
+
rb_global_variable(&CInfinity);
|
|
1641
2790
|
CInfinity = rb_const_get(mJSON, rb_intern("Infinity"));
|
|
1642
|
-
rb_gc_register_mark_object(CInfinity);
|
|
1643
2791
|
|
|
2792
|
+
rb_global_variable(&CMinusInfinity);
|
|
1644
2793
|
CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity"));
|
|
1645
|
-
rb_gc_register_mark_object(CMinusInfinity);
|
|
1646
2794
|
|
|
1647
2795
|
rb_global_variable(&Encoding_UTF_8);
|
|
1648
2796
|
Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
|
|
1649
2797
|
|
|
2798
|
+
rb_global_variable(&JSON_empty_string);
|
|
2799
|
+
JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0));
|
|
2800
|
+
|
|
1650
2801
|
sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
|
|
1651
2802
|
sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
|
|
1652
2803
|
sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
|
|
2804
|
+
sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
|
|
1653
2805
|
sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
|
|
2806
|
+
sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
|
|
1654
2807
|
sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
|
|
1655
2808
|
sym_freeze = ID2SYM(rb_intern("freeze"));
|
|
1656
2809
|
sym_on_load = ID2SYM(rb_intern("on_load"));
|
|
@@ -1661,6 +2814,8 @@ void Init_parser(void)
|
|
|
1661
2814
|
i_try_convert = rb_intern("try_convert");
|
|
1662
2815
|
i_uminus = rb_intern("-@");
|
|
1663
2816
|
i_encode = rb_intern("encode");
|
|
2817
|
+
i_at_line = rb_intern("@line");
|
|
2818
|
+
i_at_column = rb_intern("@column");
|
|
1664
2819
|
|
|
1665
2820
|
binary_encindex = rb_ascii8bit_encindex();
|
|
1666
2821
|
utf8_encindex = rb_utf8_encindex();
|