json 2.19.5 → 2.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,18 @@
1
1
  #include "../json.h"
2
- #include "../vendor/ryu.h"
2
+ #include "../vendor/fast_float_parser.h"
3
3
  #include "../simd/simd.h"
4
4
 
5
- static VALUE mJSON, eNestingError, Encoding_UTF_8;
6
- static VALUE CNaN, CInfinity, CMinusInfinity;
5
+ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
6
+ static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
7
7
 
8
- static ID i_new, i_try_convert, i_uminus, i_encode;
8
+ static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column;
9
+ #ifndef HAVE_RB_STR_TO_INTERNED_STR
10
+ static ID i_uminus;
11
+ #endif
9
12
 
10
- static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters,
11
- sym_allow_invalid_escape, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load,
12
- sym_allow_duplicate_key;
13
+ static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
14
+ sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
15
+ sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
13
16
 
14
17
  static int binary_encindex;
15
18
  static int utf8_encindex;
@@ -58,6 +61,20 @@ typedef struct rvalue_cache_struct {
58
61
  VALUE entries[JSON_RVALUE_CACHE_CAPA];
59
62
  } rvalue_cache;
60
63
 
64
+ static void rvalue_cache_mark(rvalue_cache *cache)
65
+ {
66
+ for (int index = 0; index < cache->length; index++) {
67
+ rb_gc_mark_movable(cache->entries[index]);
68
+ }
69
+ }
70
+
71
+ static void rvalue_cache_compact(rvalue_cache *cache)
72
+ {
73
+ for (int index = 0; index < cache->length; index++) {
74
+ cache->entries[index] = rb_gc_location(cache->entries[index]);
75
+ }
76
+ }
77
+
61
78
  static rb_encoding *enc_utf8;
62
79
 
63
80
  #define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55
@@ -206,12 +223,12 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle,
206
223
 
207
224
  static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref)
208
225
  {
209
- long required = stack->capa * 2;
226
+ long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA;
210
227
 
211
228
  if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
212
229
  stack = rvalue_stack_spill(stack, handle, stack_ref);
213
230
  } else {
214
- REALLOC_N(stack->ptr, VALUE, required);
231
+ JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa);
215
232
  stack->capa = required;
216
233
  }
217
234
  return stack;
@@ -219,11 +236,15 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu
219
236
 
220
237
  static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref)
221
238
  {
239
+ JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle);
240
+
222
241
  if (RB_UNLIKELY(stack->head >= stack->capa)) {
223
242
  stack = rvalue_stack_grow(stack, handle, stack_ref);
224
243
  }
244
+
225
245
  stack->ptr[stack->head] = value;
226
246
  stack->head++;
247
+
227
248
  return value;
228
249
  }
229
250
 
@@ -243,14 +264,14 @@ static void rvalue_stack_mark(void *ptr)
243
264
  long index;
244
265
  if (stack && stack->ptr) {
245
266
  for (index = 0; index < stack->head; index++) {
246
- rb_gc_mark(stack->ptr[index]);
267
+ rb_gc_mark_movable(stack->ptr[index]);
247
268
  }
248
269
  }
249
270
  }
250
271
 
251
272
  static void rvalue_stack_free_buffer(rvalue_stack *stack)
252
273
  {
253
- ruby_xfree(stack->ptr);
274
+ JSON_SIZED_FREE_N(stack->ptr, stack->capa);
254
275
  stack->ptr = NULL;
255
276
  }
256
277
 
@@ -260,7 +281,7 @@ static void rvalue_stack_free(void *ptr)
260
281
  if (stack) {
261
282
  rvalue_stack_free_buffer(stack);
262
283
  #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
263
- ruby_xfree(stack);
284
+ JSON_SIZED_FREE(stack);
264
285
  #endif
265
286
  }
266
287
  }
@@ -268,7 +289,22 @@ static void rvalue_stack_free(void *ptr)
268
289
  static size_t rvalue_stack_memsize(const void *ptr)
269
290
  {
270
291
  const rvalue_stack *stack = (const rvalue_stack *)ptr;
271
- return sizeof(rvalue_stack) + sizeof(VALUE) * stack->capa;
292
+ size_t memsize = sizeof(VALUE) * stack->capa;
293
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
294
+ memsize += sizeof(rvalue_stack);
295
+ #endif
296
+ return memsize;
297
+ }
298
+
299
+ static void rvalue_stack_compact(void *ptr)
300
+ {
301
+ rvalue_stack *stack = (rvalue_stack *)ptr;
302
+ long index;
303
+ if (stack && stack->ptr) {
304
+ for (index = 0; index < stack->head; index++) {
305
+ stack->ptr[index] = rb_gc_location(stack->ptr[index]);
306
+ }
307
+ }
272
308
  }
273
309
 
274
310
  static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
@@ -277,8 +313,11 @@ static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
277
313
  .dmark = rvalue_stack_mark,
278
314
  .dfree = rvalue_stack_free,
279
315
  .dsize = rvalue_stack_memsize,
316
+ .dcompact = rvalue_stack_compact,
280
317
  },
281
- .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE,
318
+ // We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
319
+ // because it churns a lot of values so trigering write barriers every time is very costly.
320
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
282
321
  };
283
322
 
284
323
  static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
@@ -309,33 +348,62 @@ static void rvalue_stack_eagerly_release(VALUE handle)
309
348
  }
310
349
  }
311
350
 
312
- static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
313
- {
314
- int len = 1;
315
- if (ch <= 0x7F) {
316
- buf[0] = (char) ch;
317
- } else if (ch <= 0x07FF) {
318
- buf[0] = (char) ((ch >> 6) | 0xC0);
319
- buf[1] = (char) ((ch & 0x3F) | 0x80);
320
- len++;
321
- } else if (ch <= 0xFFFF) {
322
- buf[0] = (char) ((ch >> 12) | 0xE0);
323
- buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
324
- buf[2] = (char) ((ch & 0x3F) | 0x80);
325
- len += 2;
326
- } else if (ch <= 0x1fffff) {
327
- buf[0] =(char) ((ch >> 18) | 0xF0);
328
- buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
329
- buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
330
- buf[3] =(char) ((ch & 0x3F) | 0x80);
331
- len += 3;
332
- } else {
333
- buf[0] = '?';
334
- }
335
- return len;
336
- }
351
+ /* frame stack */
352
+
353
+ // Iterative (non-recursive) parsing keeps an explicit stack of the containers
354
+ // currently being built, instead of relying on the C call stack. Each frame
355
+ // only needs enough bookkeeping to close its container: which kind it is, the
356
+ // rvalue_stack position where its children start (so we know how many to pop),
357
+ // and the cursor at its opening brace (used to rewind for duplicate key
358
+ // errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses
359
+ // the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that
360
+ // it's freed even if parsing raises.
361
+ //
362
+ // The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release
363
+ // and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts
364
+ // -- the element type and the absence of a mark function are the only real
365
+ // differences. Keep the two in sync: a fix to the spill/release or
366
+ // HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the
367
+ // other.
368
+ #define JSON_FRAME_STACK_INITIAL_CAPA 32
369
+
370
+ enum json_frame_type {
371
+ JSON_FRAME_ROOT, // == JSON_PHASE_DONE
372
+ JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA
373
+ JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA
374
+ };
375
+
376
+ // Where a frame is within its container's grammar. This is the entirety of the
377
+ // parser's "what to do next" state: json_parse_any dispatches on the top
378
+ // frame's phase and holds no resume state in C locals, so a parse can stop at
379
+ // any value boundary and be resumed purely from the (persistable) frame stack.
380
+ //
381
+ // The first three phases are deliberately equal to the corresponding json_frame_type
382
+ // to simplify the transition of phase in json_value_completed.
383
+ enum json_frame_phase {
384
+ JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed
385
+ JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']'
386
+ JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}'
387
+ JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':')
388
+ JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',')
389
+ JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':'
390
+ };
391
+
392
+ typedef struct json_frame_struct {
393
+ enum json_frame_type type;
394
+ enum json_frame_phase phase;
395
+ long value_stack_head; // rvalue_stack->head when this container opened
396
+ size_t start_offset; // object frames only (the '{'); NULL otherwise
397
+ } json_frame;
398
+
399
+ typedef struct json_frame_stack_struct {
400
+ enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated
401
+ long capa;
402
+ long head;
403
+ json_frame *ptr;
404
+ } json_frame_stack;
337
405
 
338
- enum duplicate_key_action {
406
+ enum deprecatable_action {
339
407
  JSON_DEPRECATED = 0,
340
408
  JSON_IGNORE,
341
409
  JSON_RAISE,
@@ -345,7 +413,8 @@ typedef struct JSON_ParserStruct {
345
413
  VALUE on_load_proc;
346
414
  VALUE decimal_class;
347
415
  ID decimal_method_id;
348
- enum duplicate_key_action on_duplicate_key;
416
+ enum deprecatable_action on_duplicate_key;
417
+ enum deprecatable_action on_comment;
349
418
  int max_nesting;
350
419
  bool allow_nan;
351
420
  bool allow_trailing_comma;
@@ -356,17 +425,152 @@ typedef struct JSON_ParserStruct {
356
425
  } JSON_ParserConfig;
357
426
 
358
427
  typedef struct JSON_ParserStateStruct {
359
- VALUE *stack_handle;
428
+ VALUE *value_stack_handle;
429
+ VALUE *frame_stack_handle;
360
430
  const char *start;
361
431
  const char *cursor;
362
432
  const char *end;
363
- rvalue_stack *stack;
433
+ rvalue_stack *value_stack;
434
+ json_frame_stack *frames;
364
435
  rvalue_cache name_cache;
365
436
  int in_array;
366
437
  int current_nesting;
367
438
  unsigned int emitted_deprecations;
439
+ VALUE parser;
368
440
  } JSON_ParserState;
369
441
 
442
+ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref);
443
+
444
+ static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref)
445
+ {
446
+ long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA;
447
+
448
+ if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
449
+ stack = json_frame_stack_spill(stack, handle, stack_ref);
450
+ } else {
451
+ JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa);
452
+ stack->capa = required;
453
+ }
454
+ return stack;
455
+ }
456
+
457
+ static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame)
458
+ {
459
+ json_frame_stack *stack = state->frames;
460
+
461
+ JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle);
462
+
463
+ if (RB_UNLIKELY(stack->head >= stack->capa)) {
464
+ stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames);
465
+ }
466
+
467
+ json_frame *frame_ptr = &stack->ptr[stack->head++];
468
+ *frame_ptr = frame;
469
+ return frame_ptr;
470
+ }
471
+
472
+ static inline json_frame *json_frame_stack_peek(json_frame_stack *stack)
473
+ {
474
+ return &stack->ptr[stack->head - 1];
475
+ }
476
+
477
+ static inline void json_frame_stack_pop(json_frame_stack *stack)
478
+ {
479
+ stack->head--;
480
+ }
481
+
482
+ static void json_frame_stack_free_buffer(json_frame_stack *stack)
483
+ {
484
+ JSON_SIZED_FREE_N(stack->ptr, stack->capa);
485
+ stack->ptr = NULL;
486
+ }
487
+
488
+ static void json_frame_stack_free(void *ptr)
489
+ {
490
+ json_frame_stack *stack = (json_frame_stack *)ptr;
491
+ if (stack) {
492
+ json_frame_stack_free_buffer(stack);
493
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
494
+ JSON_SIZED_FREE(stack);
495
+ #endif
496
+ }
497
+ }
498
+
499
+ static size_t json_frame_stack_memsize(const void *ptr)
500
+ {
501
+ const json_frame_stack *stack = (const json_frame_stack *)ptr;
502
+
503
+ size_t memsize = sizeof(json_frame) * stack->capa;
504
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
505
+ memsize += sizeof(json_frame_stack);
506
+ #endif
507
+ return memsize;
508
+ }
509
+
510
+ static const rb_data_type_t JSON_Parser_frame_stack_type = {
511
+ .wrap_struct_name = "JSON::Ext::Parser/frame_stack",
512
+ .function = {
513
+ .dmark = NULL,
514
+ .dfree = json_frame_stack_free,
515
+ .dsize = json_frame_stack_memsize,
516
+ },
517
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
518
+ };
519
+
520
+ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
521
+ {
522
+ json_frame_stack *stack;
523
+ *handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
524
+ *stack_ref = stack;
525
+ MEMCPY(stack, old_stack, json_frame_stack, 1);
526
+
527
+ stack->capa = old_stack->capa << 1;
528
+ stack->ptr = ALLOC_N(json_frame, stack->capa);
529
+ stack->type = RVALUE_STACK_HEAP_ALLOCATED;
530
+ MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head);
531
+ return stack;
532
+ }
533
+
534
+ static void json_frame_stack_eagerly_release(VALUE handle)
535
+ {
536
+ if (handle) {
537
+ json_frame_stack *stack;
538
+ TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
539
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
540
+ json_frame_stack_free_buffer(stack);
541
+ #else
542
+ json_frame_stack_free(stack);
543
+ RTYPEDDATA_DATA(handle) = NULL;
544
+ #endif
545
+ }
546
+ }
547
+
548
+ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
549
+ {
550
+ int len = 1;
551
+ if (ch <= 0x7F) {
552
+ buf[0] = (char) ch;
553
+ } else if (ch <= 0x07FF) {
554
+ buf[0] = (char) ((ch >> 6) | 0xC0);
555
+ buf[1] = (char) ((ch & 0x3F) | 0x80);
556
+ len++;
557
+ } else if (ch <= 0xFFFF) {
558
+ buf[0] = (char) ((ch >> 12) | 0xE0);
559
+ buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
560
+ buf[2] = (char) ((ch & 0x3F) | 0x80);
561
+ len += 2;
562
+ } else if (ch <= 0x1fffff) {
563
+ buf[0] =(char) ((ch >> 18) | 0xF0);
564
+ buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
565
+ buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
566
+ buf[3] =(char) ((ch & 0x3F) | 0x80);
567
+ len += 3;
568
+ } else {
569
+ buf[0] = '?';
570
+ }
571
+ return len;
572
+ }
573
+
370
574
  static inline size_t rest(JSON_ParserState *state) {
371
575
  return state->end - state->cursor;
372
576
  }
@@ -385,12 +589,22 @@ static inline char peek(JSON_ParserState *state)
385
589
 
386
590
  static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
387
591
  {
592
+ JSON_ASSERT(!state->parser);
593
+ JSON_ASSERT(state->cursor);
594
+ JSON_ASSERT(state->cursor <= state->end);
595
+
596
+ // Redundant but helpful for hardening
597
+ if (RB_UNLIKELY(state->cursor > state->end)) {
598
+ state->cursor = state->end;
599
+ }
600
+
388
601
  const char *cursor = state->cursor;
389
602
  long column = 0;
390
603
  long line = 1;
391
604
 
392
605
  while (cursor >= state->start) {
393
606
  if (*cursor-- == '\n') {
607
+ line++;
394
608
  break;
395
609
  }
396
610
  column++;
@@ -405,18 +619,24 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum
405
619
  *column_out = column;
406
620
  }
407
621
 
622
+ static const unsigned int MAX_DEPRECATIONS = 5;
623
+
408
624
  static void emit_parse_warning(const char *message, JSON_ParserState *state)
409
625
  {
410
- long line, column;
411
- cursor_position(state, &line, &column);
412
-
413
- VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
626
+ VALUE warning;
627
+ if (state->parser) { // line and columns can't be accurate in resumable
628
+ warning = rb_utf8_str_new_cstr(message);
629
+ } else {
630
+ long line, column;
631
+ cursor_position(state, &line, &column);
632
+ warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
633
+ }
414
634
  rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning);
415
635
  }
416
636
 
417
637
  #define PARSE_ERROR_FRAGMENT_LEN 32
418
638
 
419
- static VALUE build_parse_error_message(const char *format, JSON_ParserState *state, long line, long column)
639
+ static VALUE build_parse_error_message(const char *format, JSON_ParserState *state)
420
640
  {
421
641
  unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
422
642
 
@@ -450,31 +670,61 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta
450
670
  }
451
671
  }
452
672
 
453
- VALUE message = rb_enc_sprintf(enc_utf8, format, ptr);
454
- rb_str_catf(message, " at line %ld column %ld", line, column);
455
- return message;
673
+ return rb_enc_sprintf(enc_utf8, format, ptr);
456
674
  }
457
675
 
458
- static VALUE parse_error_new(VALUE message, long line, long column)
676
+ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
459
677
  {
460
- VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message);
461
- rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line));
462
- rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column));
678
+ VALUE exc = rb_exc_new_str(eParserError, message);
679
+ rb_ivar_set(exc, i_at_line, LONG2NUM(line));
680
+ rb_ivar_set(exc, i_at_column, LONG2NUM(column));
463
681
  return exc;
464
682
  }
465
683
 
466
- NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state)
684
+ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos)
685
+ {
686
+ if (state->parser) {
687
+ if (eos) {
688
+ // the error will be swallowed by ResumableParser#parse, so no
689
+ // point building a message or backtrace.
690
+ rb_throw_obj(state->parser, state->parser);
691
+ } else {
692
+ // line and columns can't be accurate in resumable
693
+ rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos));
694
+ }
695
+ } else {
696
+ VALUE message = build_parse_error_message(format, state);
697
+ long line, column;
698
+ cursor_position(state, &line, &column);
699
+ rb_str_catf(message, " at line %ld column %ld", line, column);
700
+ rb_exc_raise(parse_error_new(state, message, line, column, eos));
701
+ }
702
+ }
703
+
704
+ NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state)
467
705
  {
468
- long line, column;
469
- cursor_position(state, &line, &column);
470
- VALUE message = build_parse_error_message(format, state, line, column);
471
- rb_exc_raise(parse_error_new(message, line, column));
706
+ raise_parse_error(format, state, true);
472
707
  }
473
708
 
474
- NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at)
709
+ NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state)
710
+ {
711
+ raise_parse_error(format, state, false);
712
+ }
713
+
714
+ NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos)
475
715
  {
476
716
  state->cursor = at;
477
- raise_parse_error(format, state);
717
+ raise_parse_error(format, state, eos);
718
+ }
719
+
720
+ NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at)
721
+ {
722
+ raise_parse_error_at(format, state, at, true);
723
+ }
724
+
725
+ NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at)
726
+ {
727
+ raise_parse_error_at(format, state, at, false);
478
728
  }
479
729
 
480
730
  /* unicode */
@@ -499,7 +749,7 @@ static const signed char digit_values[256] = {
499
749
  static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
500
750
  {
501
751
  if (RB_UNLIKELY(sp > spe - 4)) {
502
- raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
752
+ raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
503
753
  }
504
754
 
505
755
  const unsigned char *p = (const unsigned char *)sp;
@@ -510,7 +760,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
510
760
  const signed char b3 = digit_values[p[3]];
511
761
 
512
762
  if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
513
- raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
763
+ raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
514
764
  }
515
765
 
516
766
  return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
@@ -522,19 +772,36 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
522
772
 
523
773
  static const rb_data_type_t JSON_ParserConfig_type;
524
774
 
525
- static void
526
- json_eat_comments(JSON_ParserState *state)
775
+ const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
776
+ NOINLINE(static) void
777
+ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos)
527
778
  {
779
+ if (config->on_comment == JSON_RAISE) {
780
+ raise_syntax_error("unexpected token %s", state);
781
+ }
782
+
528
783
  const char *start = state->cursor;
784
+ // An incomplete comment suspends a resumable parse by rewinding the cursor
785
+ // and throwing. Callers that already consumed a token not yet committed to
786
+ // the frame stack pass resume_pos so the rewind re-reads that token too.
787
+ // Non-resumable error positions keep pointing at the comment either way.
788
+ const char *rewind_pos = (state->parser && resume_pos) ? resume_pos : start;
529
789
  state->cursor++;
530
790
 
531
791
  switch (peek(state)) {
532
792
  case '/': {
533
- state->cursor = memchr(state->cursor, '\n', state->end - state->cursor);
534
- if (!state->cursor) {
793
+ const char *newline = memchr(state->cursor, '\n', state->end - state->cursor);
794
+ if (!newline) {
795
+ // state->parser marks resumable mode, where the buffer end is only a
796
+ // chunk boundary: the terminating newline may still arrive, so leave
797
+ // the comment unterminated instead of consuming to end as a one-shot
798
+ // parse would.
799
+ if (state->parser) {
800
+ raise_eos_error_at("unterminated comment, expected end of line", state, rewind_pos);
801
+ }
535
802
  state->cursor = state->end;
536
803
  } else {
537
- state->cursor++;
804
+ state->cursor = newline + 1;
538
805
  }
539
806
  break;
540
807
  }
@@ -544,7 +811,7 @@ json_eat_comments(JSON_ParserState *state)
544
811
  while (true) {
545
812
  const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
546
813
  if (!next_match) {
547
- raise_parse_error_at("unterminated comment, expected closing '*/'", state, start);
814
+ raise_eos_error_at("unterminated comment, expected closing '*/'", state, rewind_pos);
548
815
  }
549
816
 
550
817
  state->cursor = next_match + 1;
@@ -556,13 +823,18 @@ json_eat_comments(JSON_ParserState *state)
556
823
  break;
557
824
  }
558
825
  default:
559
- raise_parse_error_at("unexpected token %s", state, start);
826
+ raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state));
560
827
  break;
561
828
  }
829
+
830
+ if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) {
831
+ state->emitted_deprecations++;
832
+ emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state);
833
+ }
562
834
  }
563
835
 
564
836
  ALWAYS_INLINE(static) void
565
- json_eat_whitespace(JSON_ParserState *state)
837
+ json_eat_whitespace_resume_at(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments, const char *resume_pos)
566
838
  {
567
839
  while (true) {
568
840
  switch (peek(state)) {
@@ -593,7 +865,11 @@ json_eat_whitespace(JSON_ParserState *state)
593
865
  state->cursor++;
594
866
  break;
595
867
  case '/':
596
- json_eat_comments(state);
868
+ if (!include_comments) {
869
+ return;
870
+ }
871
+
872
+ json_eat_comments(state, config, resume_pos);
597
873
  break;
598
874
 
599
875
  default:
@@ -602,6 +878,12 @@ json_eat_whitespace(JSON_ParserState *state)
602
878
  }
603
879
  }
604
880
 
881
+ ALWAYS_INLINE(static) void
882
+ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
883
+ {
884
+ json_eat_whitespace_resume_at(state, config, include_comments, NULL);
885
+ }
886
+
605
887
  static inline VALUE build_string(const char *start, const char *end, bool intern, bool symbolize)
606
888
  {
607
889
  if (symbolize) {
@@ -747,13 +1029,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
747
1029
  uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
748
1030
 
749
1031
  if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
750
- raise_parse_error_at("invalid surrogate pair at %s", state, p);
1032
+ raise_syntax_error_at("invalid surrogate pair at %s", state, p);
751
1033
  }
752
1034
 
753
1035
  ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
754
1036
  pe += 5;
755
1037
  } else {
756
- raise_parse_error_at("incomplete surrogate pair at %s", state, p);
1038
+ raise_syntax_error_at("incomplete surrogate pair at %s", state, p);
757
1039
  break;
758
1040
  }
759
1041
  }
@@ -763,20 +1045,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
763
1045
  p = ++pe;
764
1046
  break;
765
1047
  }
1048
+ case 0:
1049
+ return Qundef;
766
1050
  default:
767
1051
  if ((unsigned char)*pe < 0x20) {
768
1052
  if (!config->allow_control_characters) {
769
1053
  if (*pe == '\n') {
770
- raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
1054
+ raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
771
1055
  }
772
- raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1);
1056
+ raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1);
773
1057
  }
774
1058
  }
775
1059
 
776
1060
  if (config->allow_invalid_escape) {
777
1061
  APPEND_CHAR(*pe);
778
1062
  } else {
779
- raise_parse_error_at("invalid escape character in string: %s", state, pe - 1);
1063
+ raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1);
780
1064
  }
781
1065
  break;
782
1066
  }
@@ -872,19 +1156,24 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
872
1156
  return rb_float_new(negative ? -0.0 : 0.0);
873
1157
  }
874
1158
 
875
- // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case)
876
- // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308)
877
- if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) {
1159
+ if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
1160
+ // If the value is so small that it definitely underflows to 0.0, return early
1161
+ // to avoid triggering a "Float out of range" warning from rb_cstr_to_dbl.
1162
+ // When mantissa_digits + exponent < -324, value < 10^(-324) < DBL_TRUE_MIN/2,
1163
+ // so it rounds to 0 in IEEE 754 round-to-nearest.
1164
+ if (RB_UNLIKELY(mantissa_digits + exponent < -324)) {
1165
+ return rb_float_new(negative ? -0.0 : 0.0);
1166
+ }
878
1167
  return json_decode_large_float(start, end - start);
879
1168
  }
880
1169
 
881
- return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, (int32_t)exponent, negative));
1170
+ return DBL2NUM(ffp_s2d(exponent, mantissa, negative));
882
1171
  }
883
1172
 
884
1173
  static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
885
1174
  {
886
- VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count));
887
- rvalue_stack_pop(state->stack, count);
1175
+ VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count));
1176
+ rvalue_stack_pop(state->value_stack, count);
888
1177
 
889
1178
  if (config->freeze) {
890
1179
  RB_OBJ_FREEZE(array);
@@ -928,38 +1217,50 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d
928
1217
  rb_inspect(duplicate_key)
929
1218
  );
930
1219
 
931
- long line, column;
932
- cursor_position(state, &line, &column);
933
- rb_str_concat(message, build_parse_error_message("", state, line, column)) ;
934
- rb_exc_raise(parse_error_new(message, line, column));
1220
+ rb_str_concat(message, build_parse_error_message("", state));
1221
+ if (state->parser) { // line and columns can't be accurate in resumable
1222
+ rb_exc_raise(parse_error_new(state, message, 0, 0, false));
1223
+ } else {
1224
+ long line, column;
1225
+ cursor_position(state, &line, &column);
1226
+ rb_str_catf(message, " at line %ld column %ld", line, column);
1227
+ rb_exc_raise(parse_error_new(state, message, line, column, false));
1228
+ }
1229
+ }
1230
+
1231
+ NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
1232
+ {
1233
+ switch (config->on_duplicate_key) {
1234
+ case JSON_IGNORE:
1235
+ return;
1236
+
1237
+ case JSON_DEPRECATED:
1238
+ // Only emit the first few deprecations to avoid spamming.
1239
+ if (state->emitted_deprecations < MAX_DEPRECATIONS) {
1240
+ state->emitted_deprecations++;
1241
+ emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
1242
+ }
1243
+ return;
1244
+
1245
+ case JSON_RAISE:
1246
+ raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
1247
+ return;
1248
+ }
1249
+ UNREACHABLE;
935
1250
  }
936
1251
 
937
1252
  static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
938
1253
  {
939
1254
  size_t entries_count = count / 2;
940
1255
  VALUE object = rb_hash_new_capa(entries_count);
941
- const VALUE *pairs = rvalue_stack_peek(state->stack, count);
1256
+ const VALUE *pairs = rvalue_stack_peek(state->value_stack, count);
942
1257
  rb_hash_bulk_insert(count, pairs, object);
943
1258
 
944
1259
  if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
945
- switch (config->on_duplicate_key) {
946
- case JSON_IGNORE:
947
- break;
948
- case JSON_DEPRECATED:
949
- // Only emit the first few deprecations to avoid spamming.
950
- if (state->emitted_deprecations < 5) {
951
- emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
952
- state->emitted_deprecations++;
953
- }
954
-
955
- break;
956
- case JSON_RAISE:
957
- raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
958
- break;
959
- }
1260
+ json_on_duplicate_key(state, config, count, pairs);
960
1261
  }
961
1262
 
962
- rvalue_stack_pop(state->stack, count);
1263
+ rvalue_stack_pop(state->value_stack, count);
963
1264
 
964
1265
  if (config->freeze) {
965
1266
  RB_OBJ_FREEZE(object);
@@ -973,7 +1274,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
973
1274
  if (RB_UNLIKELY(config->on_load_proc)) {
974
1275
  value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
975
1276
  }
976
- rvalue_stack_push(state->stack, value, state->stack_handle, &state->stack);
1277
+ rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack);
977
1278
  return value;
978
1279
  }
979
1280
 
@@ -1022,6 +1323,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
1022
1323
  }
1023
1324
  state->cursor++;
1024
1325
  }
1326
+
1327
+ // If the string ended with an unterminated escape sequence, we might
1328
+ // have gone past the end.
1329
+ if (RB_UNLIKELY(state->cursor > state->end)) {
1330
+ state->cursor = state->end;
1331
+ }
1332
+
1025
1333
  return false;
1026
1334
  }
1027
1335
 
@@ -1039,7 +1347,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1039
1347
  case '"': {
1040
1348
  VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
1041
1349
  state->cursor++;
1042
- return json_push_value(state, config, string);
1350
+ return string;
1043
1351
  }
1044
1352
  case '\\': {
1045
1353
  if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
@@ -1053,7 +1361,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1053
1361
  }
1054
1362
  default:
1055
1363
  if (!config->allow_control_characters) {
1056
- raise_parse_error("invalid ASCII control character in string: %s", state);
1364
+ raise_syntax_error("invalid ASCII control character in string: %s", state);
1057
1365
  }
1058
1366
  break;
1059
1367
  }
@@ -1061,8 +1369,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1061
1369
  state->cursor++;
1062
1370
  } while (string_scan(state));
1063
1371
 
1064
- raise_parse_error("unexpected end of input, expected closing \"", state);
1065
- return Qfalse;
1372
+ return Qundef;
1066
1373
  }
1067
1374
 
1068
1375
  ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
@@ -1071,15 +1378,19 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars
1071
1378
  const char *start = state->cursor;
1072
1379
 
1073
1380
  if (RB_UNLIKELY(!string_scan(state))) {
1074
- raise_parse_error("unexpected end of input, expected closing \"", state);
1381
+ return Qundef;
1075
1382
  }
1076
1383
 
1384
+ VALUE string;
1077
1385
  if (RB_LIKELY(*state->cursor == '"')) {
1078
- VALUE string = json_string_fastpath(state, config, start, state->cursor, is_name);
1386
+ string = json_string_fastpath(state, config, start, state->cursor, is_name);
1079
1387
  state->cursor++;
1080
- return json_push_value(state, config, string);
1081
1388
  }
1082
- return json_parse_escaped_string(state, config, is_name, start);
1389
+ else {
1390
+ string = json_parse_escaped_string(state, config, is_name, start);
1391
+ }
1392
+
1393
+ return string;
1083
1394
  }
1084
1395
 
1085
1396
  #if JSON_CPU_LITTLE_ENDIAN_64BITS
@@ -1152,7 +1463,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat
1152
1463
  return (int)(state->cursor - start);
1153
1464
  }
1154
1465
 
1155
- static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
1466
+ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable)
1156
1467
  {
1157
1468
  bool integer = true;
1158
1469
  const char first_digit = *state->cursor;
@@ -1166,7 +1477,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1166
1477
  int mantissa_digits = json_parse_digits(state, &mantissa);
1167
1478
 
1168
1479
  if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
1169
- raise_parse_error_at("invalid number: %s", state, start);
1480
+ return Qundef;
1170
1481
  }
1171
1482
 
1172
1483
  // Parse fractional part
@@ -1179,7 +1490,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1179
1490
  mantissa_digits += fractional_digits;
1180
1491
 
1181
1492
  if (RB_UNLIKELY(!fractional_digits)) {
1182
- raise_parse_error_at("invalid number: %s", state, start);
1493
+ return Qundef;
1183
1494
  }
1184
1495
  }
1185
1496
 
@@ -1199,10 +1510,24 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1199
1510
  int exponent_digits = json_parse_digits(state, &abs_exponent);
1200
1511
 
1201
1512
  if (RB_UNLIKELY(!exponent_digits)) {
1202
- raise_parse_error_at("invalid number: %s", state, start);
1513
+ return Qundef;
1514
+ }
1515
+
1516
+ if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
1517
+ exponent = negative_exponent ? INT64_MIN : INT64_MAX;
1518
+ } else {
1519
+ exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
1203
1520
  }
1521
+ }
1204
1522
 
1205
- exponent = negative_exponent ? -abs_exponent : abs_exponent;
1523
+ // A number touching the end of the buffer may still grow in a later chunk,
1524
+ // so the caller will rewind and wait. Decoding it now would build a value
1525
+ // -- for a long run of digits, an expensive bignum -- only to discard it,
1526
+ // and repeating that on every resumed chunk is quadratic in the number's
1527
+ // length. The digit scan above already advanced the cursor, which is all
1528
+ // the caller needs to detect the incomplete number.
1529
+ if (RB_UNLIKELY(resumable && eos(state))) {
1530
+ return Qundef;
1206
1531
  }
1207
1532
 
1208
1533
  if (integer) {
@@ -1217,229 +1542,420 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1217
1542
  return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
1218
1543
  }
1219
1544
 
1220
- static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_ParserConfig *config)
1545
+ // How many values (array elements, or interleaved object keys+values) have been
1546
+ // pushed onto the rvalue stack since this container opened. Used to size the
1547
+ // bulk decode on close, and to tell the first key/colon from later ones.
1548
+ static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack)
1221
1549
  {
1222
- return json_parse_number(state, config, false, state->cursor);
1550
+ return value_stack->head - frame->value_stack_head;
1223
1551
  }
1224
1552
 
1225
- static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config)
1553
+ // A complete value now sits on top of the rvalue stack. Advance the frame that
1554
+ // was waiting for it: the root document is done, or the enclosing container
1555
+ // moves on to expecting a ',' or its closing bracket. The caller passes the
1556
+ // frame it already has in hand -- the one that was expecting the value -- which
1557
+ // after a container close is the freshly re-exposed parent.
1558
+ static inline enum json_frame_phase json_value_completed(json_frame *frame)
1226
1559
  {
1227
- const char *start = state->cursor;
1228
- state->cursor++;
1229
- return json_parse_number(state, config, true, start);
1560
+ JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT);
1561
+ JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY);
1562
+ JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT);
1563
+
1564
+ return frame->phase = (enum json_frame_phase) frame->type;
1230
1565
  }
1231
1566
 
1232
- static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
1567
+ ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset)
1233
1568
  {
1234
- json_eat_whitespace(state);
1569
+ // It is assumed that since `keyword` is always a literal, the compiler is able to constantize this
1570
+ // `strlen` and several other computations in that routine.
1235
1571
 
1236
- switch (peek(state)) {
1237
- case 'n':
1238
- if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
1239
- state->cursor += 4;
1240
- return json_push_value(state, config, Qnil);
1241
- }
1242
-
1243
- raise_parse_error("unexpected token %s", state);
1244
- break;
1245
- case 't':
1246
- if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) {
1247
- state->cursor += 4;
1248
- return json_push_value(state, config, Qtrue);
1249
- }
1572
+ size_t len = strlen(keyword);
1250
1573
 
1251
- raise_parse_error("unexpected token %s", state);
1252
- break;
1253
- case 'f':
1254
- // Note: memcmp with a small power of two compile to an integer comparison
1255
- if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) {
1256
- state->cursor += 5;
1257
- return json_push_value(state, config, Qfalse);
1258
- }
1574
+ // Note: memcmp with a small power of two and a literal string compile to an integer comparison /
1575
+ // That's why we sometime compare starting from the first byte and sometimes from the second.
1576
+ if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) {
1577
+ state->cursor += len;
1578
+ return;
1579
+ }
1259
1580
 
1260
- raise_parse_error("unexpected token %s", state);
1261
- break;
1262
- case 'N':
1263
- // Note: memcmp with a small power of two compile to an integer comparison
1264
- if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) {
1265
- state->cursor += 3;
1266
- return json_push_value(state, config, CNaN);
1267
- }
1581
+ bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0;
1582
+ raise_parse_error("unexpected token %s", state, eos);
1583
+ }
1584
+
1585
+ // Parse an arbitrary JSON value iteratively. This is a state machine driven
1586
+ // entirely by the top frame's phase so it can stop at any value boundary and
1587
+ // resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the
1588
+ // bottom of the stack, so the stack is never empty mid-parse and the document
1589
+ // itself is just another frame whose value, once parsed, leaves its phase DONE.
1590
+ // When invoked in resumable mode, it returns true after parsing a complete document.
1591
+ // If reaching EOS without having parsed a complete document, either returns false
1592
+ // of raise a JSON::ParserError tagged with `@eos=true`.
1593
+ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable)
1594
+ {
1595
+ json_frame *frame = json_frame_stack_peek(state->frames);
1596
+
1597
+ switch (frame->phase) {
1598
+ case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false);
1599
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1600
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1601
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1602
+ case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY;
1603
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1604
+ }
1605
+ JSON_UNREACHABLE_RETURN(false);
1268
1606
 
1269
- raise_parse_error("unexpected token %s", state);
1270
- break;
1271
- case 'I':
1272
- if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
1273
- state->cursor += 8;
1274
- return json_push_value(state, config, CInfinity);
1275
- }
1607
+ JSON_PHASE_VALUE: {
1608
+ json_eat_whitespace(state, config, true);
1276
1609
 
1277
- raise_parse_error("unexpected token %s", state);
1278
- break;
1279
- case '-': {
1280
- // Note: memcmp with a small power of two compile to an integer comparison
1281
- if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) {
1282
- if (config->allow_nan) {
1283
- state->cursor += 9;
1284
- return json_push_value(state, config, CMinusInfinity);
1285
- } else {
1286
- raise_parse_error("unexpected token %s", state);
1287
- }
1288
- }
1289
- return json_push_value(state, config, json_parse_negative_number(state, config));
1290
- break;
1291
- }
1292
- case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
1293
- return json_push_value(state, config, json_parse_positive_number(state, config));
1294
- break;
1295
- case '"': {
1296
- // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
1297
- return json_parse_string(state, config, false);
1298
- break;
1610
+ // A trailing comma lands us here expecting an element but finding the
1611
+ // closing bracket; hand off to ARRAY_COMMA to close. An empty array
1612
+ // closes inline at '[', so this position is only reached after a ','.
1613
+ if (config->allow_trailing_comma && frame->type == JSON_FRAME_ARRAY && peek(state) == ']') {
1614
+ goto JSON_PHASE_ARRAY_COMMA;
1299
1615
  }
1300
- case '[': {
1301
- state->cursor++;
1302
- json_eat_whitespace(state);
1303
- long stack_head = state->stack->head;
1304
1616
 
1305
- if (peek(state) == ']') {
1306
- state->cursor++;
1307
- return json_push_value(state, config, json_decode_array(state, config, 0));
1308
- } else {
1309
- state->current_nesting++;
1310
- if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1311
- rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1312
- }
1313
- state->in_array++;
1314
- json_parse_any(state, config);
1315
- }
1617
+ VALUE value;
1618
+ const char *value_start = state->cursor;
1316
1619
 
1317
- while (true) {
1318
- json_eat_whitespace(state);
1620
+ switch (peek(state)) {
1621
+ case 'n':
1622
+ json_match_keyword(state, "null", 0);
1623
+ value = Qnil;
1624
+ break;
1319
1625
 
1320
- const char next_char = peek(state);
1626
+ case 't':
1627
+ json_match_keyword(state, "true", 0);
1628
+ value = Qtrue;
1629
+ break;
1321
1630
 
1322
- if (RB_LIKELY(next_char == ',')) {
1323
- state->cursor++;
1324
- if (config->allow_trailing_comma) {
1325
- json_eat_whitespace(state);
1326
- if (peek(state) == ']') {
1327
- continue;
1328
- }
1329
- }
1330
- json_parse_any(state, config);
1331
- continue;
1332
- }
1631
+ case 'f':
1632
+ json_match_keyword(state, "false", 1);
1633
+ value = Qfalse;
1634
+ break;
1333
1635
 
1334
- if (next_char == ']') {
1335
- state->cursor++;
1336
- long count = state->stack->head - stack_head;
1337
- state->current_nesting--;
1338
- state->in_array--;
1339
- return json_push_value(state, config, json_decode_array(state, config, count));
1636
+ case 'N':
1637
+ if (!config->allow_nan) {
1638
+ raise_syntax_error("unexpected token %s", state);
1340
1639
  }
1341
1640
 
1342
- raise_parse_error("expected ',' or ']' after array value", state);
1343
- }
1344
- break;
1345
- }
1346
- case '{': {
1347
- const char *object_start_cursor = state->cursor;
1641
+ json_match_keyword(state, "NaN", 1);
1642
+ value = CNaN;
1643
+ break;
1348
1644
 
1349
- state->cursor++;
1350
- json_eat_whitespace(state);
1351
- long stack_head = state->stack->head;
1645
+ case 'I':
1646
+ if (!config->allow_nan) {
1647
+ raise_syntax_error("unexpected token %s", state);
1648
+ }
1649
+
1650
+ json_match_keyword(state, "Infinity", 0);
1651
+ value = CInfinity;
1652
+ break;
1352
1653
 
1353
- if (peek(state) == '}') {
1654
+ case '-': {
1354
1655
  state->cursor++;
1355
- return json_push_value(state, config, json_decode_object(state, config, 0));
1356
- } else {
1357
- state->current_nesting++;
1358
- if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1359
- rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1656
+
1657
+ value = json_parse_number(state, config, true, value_start, resumable);
1658
+
1659
+ if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
1660
+ state->cursor = value_start;
1661
+ json_match_keyword(state, "-Infinity", 1);
1662
+ value = CMinusInfinity;
1663
+ break;
1360
1664
  }
1361
1665
 
1362
- if (peek(state) != '"') {
1363
- raise_parse_error("expected object key, got %s", state);
1666
+ // Top level numbers are ambiguous when parsing streams, we can't
1667
+ // know if we parsed all the digits if we hit EOS.
1668
+ if (RB_UNLIKELY(resumable && eos(state))) {
1669
+ state->cursor = value_start;
1670
+ return false;
1364
1671
  }
1365
- json_parse_string(state, config, true);
1366
1672
 
1367
- json_eat_whitespace(state);
1368
- if (peek(state) != ':') {
1369
- raise_parse_error("expected ':' after object key", state);
1673
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1674
+ raise_syntax_error_at("invalid number: %s", state, value_start);
1370
1675
  }
1676
+ break;
1677
+ }
1678
+
1679
+ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
1680
+ value = json_parse_number(state, config, false, value_start, resumable);
1681
+
1682
+ // Top level numbers are ambiguous when parsing streams, we can't
1683
+ // know if we parsed all the digits if we hit EOS.
1684
+ if (RB_UNLIKELY(resumable && eos(state))) {
1685
+ state->cursor = value_start;
1686
+ return false;
1687
+ }
1688
+
1689
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1690
+ raise_syntax_error_at("invalid number: %s", state, value_start);
1691
+ }
1692
+ break;
1693
+ }
1694
+
1695
+ case '"': {
1696
+ // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
1697
+ value = json_parse_string(state, config, false);
1698
+
1699
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1700
+ bool is_eos = eos(state);
1701
+ if (resumable && is_eos) {
1702
+ state->cursor = value_start;
1703
+ return false;
1704
+ }
1705
+ raise_parse_error("unexpected end of input, expected closing \"", state, is_eos);
1706
+ }
1707
+ break;
1708
+ }
1709
+
1710
+ case '[': {
1371
1711
  state->cursor++;
1712
+ // The '[' is consumed but its frame is only pushed below, so a
1713
+ // comment suspending here must resume from the bracket.
1714
+ json_eat_whitespace_resume_at(state, config, true, value_start);
1715
+
1716
+ const char next = peek(state);
1717
+ if (next == ']') {
1718
+ state->cursor++;
1719
+ value = json_decode_array(state, config, 0);
1720
+ break;
1721
+ } else if (resumable && eos(state)) {
1722
+ state->cursor = value_start;
1723
+ return false;
1724
+ }
1372
1725
 
1373
- json_parse_any(state, config);
1726
+ state->current_nesting++;
1727
+ if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1728
+ rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1729
+ }
1730
+ state->in_array++;
1731
+
1732
+ // Phase stays VALUE: the next iteration reads the first element.
1733
+ frame = json_frame_stack_push(state, (json_frame){
1734
+ .type = JSON_FRAME_ARRAY,
1735
+ .phase = JSON_PHASE_VALUE,
1736
+ .value_stack_head = state->value_stack->head,
1737
+ });
1738
+ goto JSON_PHASE_VALUE;
1374
1739
  }
1375
1740
 
1376
- while (true) {
1377
- json_eat_whitespace(state);
1741
+ case '{': {
1742
+ state->cursor++;
1743
+ // Same as '[': the frame is only pushed below.
1744
+ json_eat_whitespace_resume_at(state, config, true, value_start);
1378
1745
 
1379
- const char next_char = peek(state);
1380
- if (next_char == '}') {
1746
+ if (peek(state) == '}') {
1381
1747
  state->cursor++;
1382
- state->current_nesting--;
1383
- size_t count = state->stack->head - stack_head;
1748
+ value = json_decode_object(state, config, 0);
1749
+ break;
1750
+ } else if (resumable && eos(state)) {
1751
+ state->cursor = value_start;
1752
+ return false;
1753
+ }
1754
+
1755
+ state->current_nesting++;
1756
+ if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1757
+ rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1758
+ }
1384
1759
 
1385
- // Temporary rewind cursor in case an error is raised
1386
- const char *final_cursor = state->cursor;
1387
- state->cursor = object_start_cursor;
1388
- VALUE object = json_decode_object(state, config, count);
1389
- state->cursor = final_cursor;
1760
+ // Phase KEY: the next iteration reads the first key.
1761
+ frame = json_frame_stack_push(state, (json_frame){
1762
+ .type = JSON_FRAME_OBJECT,
1763
+ .phase = JSON_PHASE_OBJECT_KEY,
1764
+ .value_stack_head = state->value_stack->head,
1765
+ .start_offset = value_start - state->start,
1766
+ });
1767
+ goto JSON_PHASE_OBJECT_KEY;
1768
+ }
1390
1769
 
1391
- return json_push_value(state, config, object);
1770
+ case 0:
1771
+ // peek() returns 0 both at end-of-stream and for a literal NUL byte in the
1772
+ // buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at
1773
+ // EOS is just an invalid character.
1774
+ if (eos(state)) {
1775
+ return false;
1776
+ } else {
1777
+ raise_syntax_error("unexpected NULL byte: %s", state);
1392
1778
  }
1779
+ default:
1780
+ raise_syntax_error("unexpected character: %s", state);
1781
+ }
1393
1782
 
1394
- if (next_char == ',') {
1395
- state->cursor++;
1396
- json_eat_whitespace(state);
1783
+ json_push_value(state, config, value);
1784
+ json_value_completed(frame);
1397
1785
 
1398
- if (config->allow_trailing_comma) {
1399
- if (peek(state) == '}') {
1400
- continue;
1401
- }
1402
- }
1786
+ switch (frame->phase) {
1787
+ case JSON_PHASE_DONE: return true;
1788
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1789
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1790
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1791
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1792
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1793
+ }
1794
+ JSON_UNREACHABLE_RETURN(false);
1795
+ }
1403
1796
 
1404
- if (RB_UNLIKELY(peek(state) != '"')) {
1405
- raise_parse_error("expected object key, got: %s", state);
1406
- }
1407
- json_parse_string(state, config, true);
1797
+ JSON_PHASE_OBJECT_KEY: {
1798
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1408
1799
 
1409
- json_eat_whitespace(state);
1410
- if (RB_UNLIKELY(peek(state) != ':')) {
1411
- raise_parse_error("expected ':' after object key, got: %s", state);
1412
- }
1413
- state->cursor++;
1800
+ json_eat_whitespace(state, config, true);
1801
+
1802
+ // A trailing comma lands us here expecting a key but finding the closing
1803
+ // brace; hand off to OBJECT_COMMA to close. An empty object closes inline
1804
+ // at '{', so this position is only reached after a ','.
1805
+ if (config->allow_trailing_comma && peek(state) == '}') {
1806
+ goto JSON_PHASE_OBJECT_COMMA;
1807
+ }
1414
1808
 
1415
- json_parse_any(state, config);
1809
+ const char *start = state->cursor;
1416
1810
 
1417
- continue;
1811
+ if (RB_LIKELY(peek(state) == '"')) {
1812
+ VALUE string = json_parse_string(state, config, true);
1813
+ if (UNDEF_P(string)) {
1814
+ if (resumable) {
1815
+ state->cursor = start;
1816
+ return false;
1817
+ } else {
1818
+ raise_syntax_error("unexpected end of input, expected closing \"", state);
1418
1819
  }
1820
+ }
1821
+ json_push_value(state, config, string);
1822
+ frame->phase = JSON_PHASE_OBJECT_COLON;
1823
+ goto JSON_PHASE_OBJECT_COLON;
1824
+ } else if (resumable && eos(state)) {
1825
+ return false;
1826
+ } else {
1827
+ // The message differs for the first key vs. a key after a
1828
+ // ',': the first is the only one reached with nothing pushed
1829
+ // for this object yet.
1830
+ if (json_frame_entry_count(frame, state->value_stack) == 0) {
1831
+ raise_syntax_error("expected object key, got %s", state);
1832
+ } else {
1833
+ raise_syntax_error("expected object key, got: %s", state);
1834
+ }
1835
+ }
1836
+ JSON_UNREACHABLE_RETURN(false);
1837
+ }
1838
+
1839
+ JSON_PHASE_OBJECT_COLON: {
1840
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1419
1841
 
1420
- raise_parse_error("expected ',' or '}' after object value, got: %s", state);
1842
+ json_eat_whitespace(state, config, true);
1843
+
1844
+ if (RB_LIKELY(peek(state) == ':')) {
1845
+ state->cursor++;
1846
+ frame->phase = JSON_PHASE_VALUE;
1847
+ goto JSON_PHASE_VALUE;
1848
+ } else if (resumable && eos(state)) {
1849
+ return false;
1850
+ } else {
1851
+ // First colon (only the first pair's key is pushed, nothing
1852
+ // else) vs. a later one.
1853
+ if (json_frame_entry_count(frame, state->value_stack) == 1) {
1854
+ raise_syntax_error("expected ':' after object key", state);
1855
+ } else {
1856
+ raise_syntax_error("expected ':' after object key, got: %s", state);
1421
1857
  }
1422
- break;
1423
1858
  }
1859
+ JSON_UNREACHABLE_RETURN(false);
1860
+ }
1424
1861
 
1425
- case 0:
1426
- raise_parse_error("unexpected end of input", state);
1427
- break;
1862
+ JSON_PHASE_ARRAY_COMMA: {
1863
+ JSON_ASSERT(frame->type == JSON_FRAME_ARRAY);
1428
1864
 
1429
- default:
1430
- raise_parse_error("unexpected character: %s", state);
1431
- break;
1865
+ json_eat_whitespace(state, config, true);
1866
+
1867
+ const char next_char = peek(state);
1868
+
1869
+ if (RB_LIKELY(next_char == ',')) {
1870
+ state->cursor++;
1871
+ // Commit the phase before eating the whitespace that follows: an
1872
+ // incomplete comment there would suspend the parse, and a phase not
1873
+ // yet advanced past the ',' would drop it on resume. A trailing comma
1874
+ // is recognized in JSON_PHASE_VALUE once the ']' is in the buffer.
1875
+ frame->phase = JSON_PHASE_VALUE;
1876
+ goto JSON_PHASE_VALUE;
1877
+ } else if (next_char == ']') {
1878
+ state->cursor++;
1879
+ long count = json_frame_entry_count(frame, state->value_stack);
1880
+ state->current_nesting--;
1881
+ state->in_array--;
1882
+
1883
+ json_push_value(state, config, json_decode_array(state, config, count));
1884
+ json_frame_stack_pop(state->frames);
1885
+ frame = json_frame_stack_peek(state->frames);
1886
+
1887
+ json_value_completed(frame);
1888
+
1889
+ switch (frame->phase) {
1890
+ case JSON_PHASE_DONE: return true;
1891
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1892
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1893
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1894
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1895
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1896
+ }
1897
+ } else if (resumable && eos(state)) {
1898
+ return false;
1899
+ } else {
1900
+ raise_syntax_error("expected ',' or ']' after array value", state);
1901
+ }
1902
+ JSON_UNREACHABLE_RETURN(false);
1432
1903
  }
1433
1904
 
1434
- raise_parse_error("unreachable: %s", state);
1435
- return Qundef;
1905
+ JSON_PHASE_OBJECT_COMMA: {
1906
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1907
+
1908
+ json_eat_whitespace(state, config, true);
1909
+ const char next_char = peek(state);
1910
+
1911
+ if (RB_LIKELY(next_char == ',')) {
1912
+ state->cursor++;
1913
+ // Commit the phase before eating the whitespace that follows: an
1914
+ // incomplete comment there would suspend the parse, and a phase not
1915
+ // yet advanced past the ',' would drop it on resume. A trailing comma
1916
+ // is recognized in JSON_PHASE_OBJECT_KEY once the '}' is in the buffer.
1917
+ frame->phase = JSON_PHASE_OBJECT_KEY;
1918
+ goto JSON_PHASE_OBJECT_KEY;
1919
+ } else if (next_char == '}') {
1920
+ state->cursor++;
1921
+ state->current_nesting--;
1922
+ size_t count = json_frame_entry_count(frame, state->value_stack);
1923
+
1924
+ // Temporary rewind cursor in case an error is raised
1925
+ const char *final_cursor = state->cursor;
1926
+ state->cursor = state->start + frame->start_offset;
1927
+ VALUE object = json_decode_object(state, config, count);
1928
+ state->cursor = final_cursor;
1929
+
1930
+ json_push_value(state, config, object);
1931
+ json_frame_stack_pop(state->frames);
1932
+ frame = json_frame_stack_peek(state->frames);
1933
+ json_value_completed(frame);
1934
+
1935
+ switch (frame->phase) {
1936
+ case JSON_PHASE_DONE: return true;
1937
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1938
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1939
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1940
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1941
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1942
+ }
1943
+ } else if (resumable && eos(state)) {
1944
+ return false;
1945
+ } else {
1946
+ raise_syntax_error("expected ',' or '}' after object value, got: %s", state);
1947
+ }
1948
+ JSON_UNREACHABLE_RETURN(false);
1949
+ }
1950
+
1951
+ JSON_UNREACHABLE_RETURN(false);
1436
1952
  }
1437
1953
 
1438
- static void json_ensure_eof(JSON_ParserState *state)
1954
+ static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config)
1439
1955
  {
1440
- json_eat_whitespace(state);
1956
+ json_eat_whitespace(state, config, true);
1441
1957
  if (!eos(state)) {
1442
- raise_parse_error("unexpected token at end of stream %s", state);
1958
+ raise_syntax_error("unexpected token at end of stream %s", state);
1443
1959
  }
1444
1960
  }
1445
1961
 
@@ -1457,40 +1973,59 @@ static void json_ensure_eof(JSON_ParserState *state)
1457
1973
 
1458
1974
  static VALUE convert_encoding(VALUE source)
1459
1975
  {
1460
- int encindex = RB_ENCODING_GET(source);
1976
+ StringValue(source);
1977
+ int encindex = RB_ENCODING_GET(source);
1461
1978
 
1462
- if (RB_LIKELY(encindex == utf8_encindex)) {
1979
+ if (RB_LIKELY(encindex == utf8_encindex)) {
1980
+ return source;
1981
+ }
1982
+
1983
+ if (encindex == binary_encindex) {
1984
+ // For historical reason, we silently reinterpret binary strings as UTF-8
1985
+ return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
1986
+ }
1987
+
1988
+ source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
1989
+ StringValue(source);
1463
1990
  return source;
1464
- }
1991
+ }
1465
1992
 
1466
- if (encindex == binary_encindex) {
1467
- // For historical reason, we silently reinterpret binary strings as UTF-8
1468
- return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
1469
- }
1993
+ struct parser_config_init_args {
1994
+ JSON_ParserConfig *config;
1995
+ VALUE self;
1996
+ VALUE unknown_keywords;
1997
+ bool strict;
1998
+ };
1470
1999
 
1471
- return rb_funcall(source, i_encode, 1, Encoding_UTF_8);
2000
+ static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
2001
+ {
2002
+ *dest = val;
2003
+ if (self) RB_OBJ_WRITTEN(self, Qundef, val);
1472
2004
  }
1473
2005
 
1474
2006
  static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1475
2007
  {
1476
- JSON_ParserConfig *config = (JSON_ParserConfig *)data;
2008
+ struct parser_config_init_args *args = (struct parser_config_init_args *)data;
2009
+ JSON_ParserConfig *config = args->config;
2010
+ VALUE self = args->self;
1477
2011
 
1478
2012
  if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
1479
2013
  else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
1480
2014
  else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
2015
+ else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
1481
2016
  else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
1482
2017
  else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
1483
2018
  else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
1484
2019
  else if (key == sym_freeze) { config->freeze = RTEST(val); }
1485
- else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; }
2020
+ else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
1486
2021
  else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
1487
2022
  else if (key == sym_decimal_class) {
1488
2023
  if (RTEST(val)) {
1489
2024
  if (rb_respond_to(val, i_try_convert)) {
1490
- config->decimal_class = val;
2025
+ parser_config_wb_write(self, &config->decimal_class, val);
1491
2026
  config->decimal_method_id = i_try_convert;
1492
2027
  } else if (rb_respond_to(val, i_new)) {
1493
- config->decimal_class = val;
2028
+ parser_config_wb_write(self, &config->decimal_class, val);
1494
2029
  config->decimal_method_id = i_new;
1495
2030
  } else if (RB_TYPE_P(val, T_CLASS)) {
1496
2031
  VALUE name = rb_class_name(val);
@@ -1499,7 +2034,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1499
2034
  if (last_colon) {
1500
2035
  const char *mod_path_end = last_colon - 1;
1501
2036
  VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
1502
- config->decimal_class = rb_path_to_class(mod_path);
2037
+ parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
1503
2038
 
1504
2039
  const char *method_name_beg = last_colon + 1;
1505
2040
  long before_len = method_name_beg - name_cstr;
@@ -1507,28 +2042,48 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1507
2042
  VALUE method_name = rb_str_substr(name, before_len, len);
1508
2043
  config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
1509
2044
  } else {
1510
- config->decimal_class = rb_mKernel;
2045
+ parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
1511
2046
  config->decimal_method_id = SYM2ID(rb_str_intern(name));
1512
2047
  }
1513
2048
  }
1514
2049
  }
1515
2050
  }
2051
+ else if (args->strict) {
2052
+ if (!args->unknown_keywords) {
2053
+ args->unknown_keywords = rb_obj_hide(rb_ary_new());
2054
+ }
2055
+ rb_ary_push(args->unknown_keywords, key);
2056
+ }
1516
2057
 
1517
2058
  return ST_CONTINUE;
1518
2059
  }
1519
2060
 
1520
- static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
2061
+ static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict)
1521
2062
  {
1522
2063
  config->max_nesting = 100;
1523
2064
 
1524
- if (!NIL_P(opts)) {
1525
- Check_Type(opts, T_HASH);
1526
- if (RHASH_SIZE(opts) > 0) {
1527
- // We assume in most cases few keys are set so it's faster to go over
1528
- // the provided keys than to check all possible keys.
1529
- rb_hash_foreach(opts, parser_config_init_i, (VALUE)config);
1530
- }
2065
+ struct parser_config_init_args args = {
2066
+ .config = config,
2067
+ .self = self,
2068
+ .strict = strict,
2069
+ };
2070
+
2071
+ if (NIL_P(opts)) return;
2072
+ Check_Type(opts, T_HASH);
2073
+ if (RHASH_SIZE(opts) == 0) return;
2074
+
2075
+ // We assume in most cases few keys are set so it's faster to go over
2076
+ // the provided keys than to check all possible keys.
2077
+ rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
1531
2078
 
2079
+ if (RB_UNLIKELY(args.unknown_keywords)) {
2080
+ if (RARRAY_LEN(args.unknown_keywords) == 1) {
2081
+ rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0));
2082
+ }
2083
+ else {
2084
+ VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", "));
2085
+ rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords);
2086
+ }
1532
2087
  }
1533
2088
  }
1534
2089
 
@@ -1537,69 +2092,90 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
1537
2092
  *
1538
2093
  * Creates a new JSON::Ext::ParserConfig instance.
1539
2094
  *
1540
- * It will be configured by the _opts_ hash. _opts_ can have the following
1541
- * keys:
2095
+ * Argument +opts+, if given, contains a \Hash of options for the parsing.
2096
+ * See {Parsing Options}[#module-JSON-label-Parsing+Options].
1542
2097
  *
1543
- * _opts_ can have the following keys:
1544
- * * *max_nesting*: The maximum depth of nesting allowed in the parsed data
1545
- * structures. Disable depth checking with :max_nesting => false|nil|0, it
1546
- * defaults to 100.
1547
- * * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
1548
- * defiance of RFC 4627 to be parsed by the Parser. This option defaults to
1549
- * false.
1550
- * * *symbolize_names*: If set to true, returns symbols for the names
1551
- * (keys) in a JSON object. Otherwise strings are returned, which is
1552
- * also the default. It's not possible to use this option in
1553
- * conjunction with the *create_additions* option.
1554
- * * *decimal_class*: Specifies which class to use instead of the default
1555
- * (Float) when parsing decimal numbers. This class must accept a single
1556
- * string argument in its constructor.
1557
2098
  */
1558
2099
  static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
1559
2100
  {
1560
2101
  rb_check_frozen(self);
1561
2102
  GET_PARSER_CONFIG;
1562
2103
 
1563
- parser_config_init(config, opts);
1564
-
1565
- RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
2104
+ parser_config_init(config, opts, self, false);
1566
2105
 
1567
2106
  return self;
1568
2107
  }
1569
2108
 
1570
- static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource)
2109
+ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
1571
2110
  {
1572
- Vsource = convert_encoding(StringValue(Vsource));
1573
- StringValue(Vsource);
2111
+ VALUE Vsource = convert_encoding(src);
2112
+
2113
+ // Ensure the string isn't mutated under us.
2114
+ // The classic API to use is `rb_str_locktmp`, but then we'd
2115
+ // need to use `rb_protect` to make sure we always unlock.
2116
+ if (Vsource == src) {
2117
+ Vsource = rb_str_new_frozen(Vsource);
2118
+ }
1574
2119
 
1575
2120
  VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
1576
- rvalue_stack stack = {
2121
+ rvalue_stack value_stack = {
1577
2122
  .type = RVALUE_STACK_STACK_ALLOCATED,
1578
2123
  .ptr = rvalue_stack_buffer,
1579
2124
  .capa = RVALUE_STACK_INITIAL_CAPA,
1580
2125
  };
1581
2126
 
2127
+ // Seed the frame stack with the root frame, establishing the invariant that
2128
+ // json_parse_any always has a top frame to dispatch on (so the stack is never
2129
+ // empty mid-parse).
2130
+ json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA];
2131
+ frame_stack_buffer[0] = (json_frame){
2132
+ .type = JSON_FRAME_ROOT,
2133
+ .phase = JSON_PHASE_VALUE,
2134
+ };
2135
+ json_frame_stack frames = {
2136
+ .type = RVALUE_STACK_STACK_ALLOCATED,
2137
+ .ptr = frame_stack_buffer,
2138
+ .capa = JSON_FRAME_STACK_INITIAL_CAPA,
2139
+ .head = 1,
2140
+ };
2141
+
1582
2142
  long len;
1583
2143
  const char *start;
2144
+
1584
2145
  RSTRING_GETMEM(Vsource, start, len);
1585
2146
 
1586
- VALUE stack_handle = 0;
2147
+ VALUE value_stack_handle = 0;
2148
+ VALUE frame_stack_handle = 0;
1587
2149
  JSON_ParserState _state = {
1588
2150
  .start = start,
1589
2151
  .cursor = start,
1590
2152
  .end = start + len,
1591
- .stack = &stack,
1592
- .stack_handle = &stack_handle,
2153
+ .value_stack = &value_stack,
2154
+ .value_stack_handle = &value_stack_handle,
2155
+ .frames = &frames,
2156
+ .frame_stack_handle = &frame_stack_handle,
1593
2157
  };
1594
2158
  JSON_ParserState *state = &_state;
1595
2159
 
1596
- VALUE result = json_parse_any(state, config);
2160
+ bool complete = json_parse_any(state, config, false);
2161
+
2162
+ // The root document value is parsed; it is the lone survivor on
2163
+ // the rvalue stack.
2164
+ VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
1597
2165
 
1598
2166
  // This may be skipped in case of exception, but
1599
2167
  // it won't cause a leak.
1600
- rvalue_stack_eagerly_release(stack_handle);
1601
- RB_GC_GUARD(stack_handle);
1602
- json_ensure_eof(state);
2168
+ rvalue_stack_eagerly_release(value_stack_handle);
2169
+ json_frame_stack_eagerly_release(frame_stack_handle);
2170
+ RB_GC_GUARD(value_stack_handle);
2171
+ RB_GC_GUARD(frame_stack_handle);
2172
+ RB_GC_GUARD(Vsource);
2173
+
2174
+ if (complete) {
2175
+ json_ensure_eof(state, config);
2176
+ } else {
2177
+ raise_eos_error("unexpected end of input", state);
2178
+ }
1603
2179
 
1604
2180
  return result;
1605
2181
  }
@@ -1619,12 +2195,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
1619
2195
 
1620
2196
  static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
1621
2197
  {
1622
- Vsource = convert_encoding(StringValue(Vsource));
1623
- StringValue(Vsource);
1624
-
1625
2198
  JSON_ParserConfig _config = {0};
1626
2199
  JSON_ParserConfig *config = &_config;
1627
- parser_config_init(config, opts);
2200
+ parser_config_init(config, opts, Qfalse, false);
1628
2201
 
1629
2202
  return cParser_parse(config, Vsource);
1630
2203
  }
@@ -1632,23 +2205,35 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
1632
2205
  static void JSON_ParserConfig_mark(void *ptr)
1633
2206
  {
1634
2207
  JSON_ParserConfig *config = ptr;
1635
- rb_gc_mark(config->on_load_proc);
1636
- rb_gc_mark(config->decimal_class);
2208
+ rb_gc_mark_movable(config->on_load_proc);
2209
+ rb_gc_mark_movable(config->decimal_class);
1637
2210
  }
1638
2211
 
1639
2212
  static size_t JSON_ParserConfig_memsize(const void *ptr)
1640
2213
  {
2214
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
2215
+ return 0;
2216
+ #else
1641
2217
  return sizeof(JSON_ParserConfig);
2218
+ #endif
2219
+ }
2220
+
2221
+ static void JSON_ParserConfig_compact(void *ptr)
2222
+ {
2223
+ JSON_ParserConfig *config = ptr;
2224
+ config->on_load_proc = rb_gc_location(config->on_load_proc);
2225
+ config->decimal_class = rb_gc_location(config->decimal_class);
1642
2226
  }
1643
2227
 
1644
2228
  static const rb_data_type_t JSON_ParserConfig_type = {
1645
2229
  .wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
1646
2230
  .function = {
1647
- JSON_ParserConfig_mark,
1648
- RUBY_DEFAULT_FREE,
1649
- JSON_ParserConfig_memsize,
2231
+ .dmark = JSON_ParserConfig_mark,
2232
+ .dfree = RUBY_DEFAULT_FREE,
2233
+ .dsize = JSON_ParserConfig_memsize,
2234
+ .dcompact = JSON_ParserConfig_compact,
1650
2235
  },
1651
- .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
2236
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
1652
2237
  };
1653
2238
 
1654
2239
  static VALUE cJSON_parser_s_allocate(VALUE klass)
@@ -1657,6 +2242,603 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
1657
2242
  return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config);
1658
2243
  }
1659
2244
 
2245
+ static void json_str_clear(VALUE str)
2246
+ {
2247
+ if (RB_OBJ_FROZEN_RAW(str)) {
2248
+ return;
2249
+ }
2250
+ rb_str_replace(str, JSON_empty_string);
2251
+ }
2252
+
2253
+ typedef struct JSON_ResumableParserStruct {
2254
+ JSON_ParserConfig config;
2255
+ JSON_ParserState state;
2256
+ rvalue_stack value_stack;
2257
+ json_frame_stack frames;
2258
+ VALUE buffer;
2259
+ size_t parsed_bytes;
2260
+ size_t incomplete_bytes;
2261
+ bool complete;
2262
+ bool in_use;
2263
+ } JSON_ResumableParser;
2264
+
2265
+ static void JSON_ResumableParser_mark(void *ptr)
2266
+ {
2267
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2268
+ JSON_ParserConfig_mark(&parser->config);
2269
+ rvalue_stack_mark(&parser->value_stack);
2270
+ rvalue_cache_mark(&parser->state.name_cache);
2271
+ rb_gc_mark(parser->buffer); // pin the buffer
2272
+ rb_gc_mark_movable(parser->state.parser);
2273
+ }
2274
+
2275
+ static void JSON_ResumableParser_free(void *ptr)
2276
+ {
2277
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2278
+ rvalue_stack_free_buffer(&parser->value_stack);
2279
+ json_frame_stack_free_buffer(&parser->frames);
2280
+ }
2281
+
2282
+ static size_t JSON_ResumableParser_memsize(const void *ptr)
2283
+ {
2284
+ const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr;
2285
+ size_t memsize = JSON_ParserConfig_memsize(&parser->config);
2286
+ memsize += rvalue_stack_memsize(&parser->value_stack);
2287
+ memsize += json_frame_stack_memsize(&parser->frames);
2288
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
2289
+ memsize += (
2290
+ sizeof(JSON_ResumableParser)
2291
+ - sizeof(JSON_ParserState)
2292
+ - sizeof(JSON_ParserConfig)
2293
+ - sizeof(rvalue_stack)
2294
+ - sizeof(json_frame_stack)
2295
+ );
2296
+ #endif
2297
+ return memsize;
2298
+ }
2299
+
2300
+ static void JSON_ResumableParser_compact(void *ptr)
2301
+ {
2302
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2303
+ JSON_ParserConfig_compact(&parser->config);
2304
+ rvalue_stack_compact(&parser->value_stack);
2305
+ rvalue_cache_compact(&parser->state.name_cache);
2306
+ parser->buffer = rb_gc_location(parser->buffer);
2307
+ parser->state.parser = rb_gc_location(parser->state.parser);
2308
+ }
2309
+
2310
+ static const rb_data_type_t JSON_ResumableParser_type = {
2311
+ .wrap_struct_name = "JSON::Ext::ResumableParser",
2312
+ .function = {
2313
+ JSON_ResumableParser_mark,
2314
+ JSON_ResumableParser_free,
2315
+ JSON_ResumableParser_memsize,
2316
+ JSON_ResumableParser_compact,
2317
+ },
2318
+ // RUBY_TYPED_WB_PROTECTED is deliberately not declared because
2319
+ // this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
2320
+ // to trigger a lot of write barriers.
2321
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
2322
+ };
2323
+
2324
+ static VALUE cResumableParser_allocate(VALUE klass)
2325
+ {
2326
+ JSON_ResumableParser *parser;
2327
+ VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
2328
+ parser->state.in_array++;
2329
+ parser->state.parser = obj;
2330
+ return obj;
2331
+ }
2332
+
2333
+ static inline JSON_ResumableParser *cResumableParser_get(VALUE self)
2334
+ {
2335
+ JSON_ResumableParser *parser;
2336
+ TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
2337
+ return parser;
2338
+ }
2339
+
2340
+ /*
2341
+ * call-seq: new(opts => {})
2342
+ *
2343
+ * Creates a new JSON::ResumableParser instance.
2344
+ *
2345
+ * Argument +opts+, if given, contains a \Hash of options for the parsing.
2346
+ * See {Parsing Options}[#module-JSON-label-Parsing+Options].
2347
+ *
2348
+ * A ResumableParser is able to parse partial documents and resume parsing later
2349
+ * when more of the document is provided:
2350
+ *
2351
+ * parser = JSON::ResumableParser.new
2352
+ * parser << '{"user": "george", "role": "ad'
2353
+ * parser.parse # => false
2354
+ * parser.eos? # => true
2355
+ * parser.partial_value # => { "user" => "george", "role" => nil }
2356
+ * parser.rest # => '"ad'
2357
+ *
2358
+ * parser << 'min" }[1, 2, 3]'
2359
+ * parser.parse # => true
2360
+ * parser.value # => { "user" => "george", "role" => "admin" }
2361
+ *
2362
+ * parser.parse # => true
2363
+ * parser.value # => [1, 2, 3]
2364
+ *
2365
+ * === Limitations
2366
+ *
2367
+ * While ResumableParser is able to parse streams of documents without any
2368
+ * explicit separators between them, it is highly recommended to separate documents
2369
+ * by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous.
2370
+ * When parsing a number, ResumableParser will not consider the number complete until something follows:
2371
+ *
2372
+ * parser << '123'
2373
+ * parser.parse # => false
2374
+ * parser << ' '
2375
+ * parser.parse # => true
2376
+ * parser.value # => 123
2377
+ *
2378
+ * === Security
2379
+ *
2380
+ * An incomplete document is buffered in full and there is no size limit, so when reading
2381
+ * from an untrusted source the caller is responsible for bounding how much data is fed.
2382
+ * For example:
2383
+ *
2384
+ * loop do
2385
+ * if parser.parsed_bytes > DOCUMENT_MAX_SIZE
2386
+ * raise "document too large"
2387
+ * end
2388
+ *
2389
+ * parser << read_chunk
2390
+ * while parser.parse
2391
+ * process(parser.value)
2392
+ * end
2393
+ * end
2394
+ */
2395
+ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
2396
+ {
2397
+ rb_check_frozen(self);
2398
+
2399
+ VALUE opts = Qfalse;
2400
+ rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts);
2401
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2402
+
2403
+ opts = argc > 0 ? argv[0] : Qnil;
2404
+ parser_config_init(&parser->config, opts, self, true);
2405
+
2406
+ return self;
2407
+ }
2408
+
2409
+ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock);
2410
+
2411
+ /*
2412
+ * call-seq: self << string -> self
2413
+ *
2414
+ * Appends the given string to the parser's buffer.
2415
+ */
2416
+ static VALUE cResumableParser_feed(VALUE self, VALUE str)
2417
+ {
2418
+ rb_check_frozen(self);
2419
+
2420
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2421
+
2422
+ str = convert_encoding(str);
2423
+ if (!RSTRING_LEN(str)) {
2424
+ return self;
2425
+ }
2426
+
2427
+ size_t offset = parser->state.cursor - parser->state.start;
2428
+ const size_t remaining = parser->state.end - parser->state.cursor;
2429
+
2430
+ if (!remaining) {
2431
+ if (parser->buffer) {
2432
+ json_str_clear(parser->buffer);
2433
+ }
2434
+ parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
2435
+ offset = 0;
2436
+ } else {
2437
+ JSON_ASSERT(parser->buffer);
2438
+
2439
+ const size_t size = parser->state.end - parser->state.start;
2440
+ const size_t consumed = size - remaining;
2441
+
2442
+ if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
2443
+ VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
2444
+ rb_enc_associate_index(new_buffer, utf8_encindex);
2445
+
2446
+ char *old_ptr = RSTRING_PTR(parser->buffer);
2447
+ memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
2448
+ rb_str_set_len(new_buffer, remaining);
2449
+ offset = 0;
2450
+ parser->buffer = new_buffer;
2451
+ } else if (consumed > (size / 2) && size >= 512) {
2452
+ rb_str_modify(parser->buffer);
2453
+ char *old_ptr = RSTRING_PTR(parser->buffer);
2454
+ memmove(old_ptr, old_ptr + consumed, remaining);
2455
+ rb_str_set_len(parser->buffer, remaining);
2456
+ offset = 0;
2457
+ }
2458
+ rb_str_append(parser->buffer, str);
2459
+ }
2460
+
2461
+ long len;
2462
+ const char *start;
2463
+ RSTRING_GETMEM(parser->buffer, start, len);
2464
+ parser->state.start = start;
2465
+ parser->state.end = start + len;
2466
+ parser->state.cursor = parser->state.start + offset;
2467
+
2468
+ return self;
2469
+ }
2470
+
2471
+ struct json_parse_any_args {
2472
+ JSON_ParserState *state;
2473
+ JSON_ParserConfig *config;
2474
+ VALUE parser;
2475
+ };
2476
+
2477
+ static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args))
2478
+ {
2479
+ struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
2480
+ return (VALUE)json_parse_any(args->state, args->config, true);
2481
+ }
2482
+
2483
+ static VALUE json_parse_any_resumable_safe(VALUE _args)
2484
+ {
2485
+ struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
2486
+ VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args);
2487
+ return result == args->parser ? Qfalse : result;
2488
+ }
2489
+
2490
+ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock)
2491
+ {
2492
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2493
+
2494
+ if (parser->in_use) {
2495
+ rb_raise(rb_eArgError, "ResumableParser can't be used recursively");
2496
+ }
2497
+
2498
+ if (lock) {
2499
+ parser->in_use = true;
2500
+ }
2501
+
2502
+ // self may have moved, so we need to update all pointers
2503
+ // Investigate: We might be better off keeping JSON_ParserState on the stack
2504
+ // and only persist what we need.
2505
+ parser->state.value_stack = &parser->value_stack;
2506
+ parser->state.frames = &parser->frames;
2507
+
2508
+ return parser;
2509
+ }
2510
+
2511
+ /*
2512
+ * call-seq: parse -> true or false
2513
+ *
2514
+ * Attemps to parse a JSON document from the internal buffer.
2515
+ * Returns whether a complete document could be parsed.
2516
+ *
2517
+ * It does raise +JSON::ParserError+ when encountering invalid \JSON syntax.
2518
+ *
2519
+ * The parsed object can be retrieved by calling #value
2520
+ */
2521
+ static VALUE cResumableParser_parse(VALUE self)
2522
+ {
2523
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
2524
+
2525
+ if (parser->complete) {
2526
+ parser->parsed_bytes = 0;
2527
+ parser->incomplete_bytes = 0;
2528
+ parser->complete = false;
2529
+ }
2530
+
2531
+ if (!parser->buffer) {
2532
+ parser->in_use = false;
2533
+ return Qfalse;
2534
+ }
2535
+
2536
+ if (parser->frames.head == 0) {
2537
+ json_frame_stack_push(&parser->state, (json_frame){
2538
+ .type = JSON_FRAME_ROOT,
2539
+ .phase = JSON_PHASE_VALUE,
2540
+ });
2541
+ }
2542
+
2543
+ VALUE Vsource = parser->buffer; // Prevent compaction
2544
+
2545
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2546
+
2547
+ if (frame->phase == JSON_PHASE_DONE) {
2548
+ JSON_ASSERT(parser->value_stack.head == 1);
2549
+ JSON_ASSERT(parser->frames.head == 1);
2550
+
2551
+ frame->phase = JSON_PHASE_VALUE;
2552
+ rvalue_stack_pop(parser->state.value_stack, 1);
2553
+ }
2554
+
2555
+ struct json_parse_any_args args = {
2556
+ .state = &parser->state,
2557
+ .config = &parser->config,
2558
+ .parser = self,
2559
+ };
2560
+ int status;
2561
+ const char *initial_cursor = parser->state.cursor;
2562
+ parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
2563
+
2564
+ if (status) {
2565
+ parser->complete = true; // a parse error is considered complete
2566
+ }
2567
+
2568
+ parser->parsed_bytes += parser->state.cursor - initial_cursor;
2569
+ parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;
2570
+
2571
+ json_eat_whitespace(&parser->state, &parser->config, false);
2572
+ if (eos(&parser->state)) {
2573
+ json_str_clear(parser->buffer);
2574
+ parser->buffer = Qfalse;
2575
+ parser->state.start = parser->state.cursor = parser->state.end = 0;
2576
+ }
2577
+ parser->in_use = false;
2578
+
2579
+ if (status) {
2580
+ rb_jump_tag(status); // reraise
2581
+ }
2582
+ RB_GC_GUARD(Vsource);
2583
+ return parser->complete ? Qtrue : Qfalse;
2584
+ }
2585
+
2586
+ /*
2587
+ * call-seq: value? -> true or false
2588
+ *
2589
+ * Returns whether a parsed value is available.
2590
+ */
2591
+ static VALUE cResumableParser_value_p(VALUE self)
2592
+ {
2593
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2594
+
2595
+ if (parser->value_stack.head > 0) {
2596
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2597
+ if (frame->phase == JSON_PHASE_DONE) {
2598
+ return Qtrue;
2599
+ }
2600
+ }
2601
+ return Qfalse;
2602
+ }
2603
+
2604
+ /*
2605
+ * call-seq: value -> object
2606
+ *
2607
+ * Returns and consume the last parsed value.
2608
+ * Raises ArgumentError if there is no parsed value or if it was already retrieved:
2609
+ * parser << '[1][2]'
2610
+ * parser.value # ArgumentError no ready value
2611
+ * parser.parse # => true
2612
+ * parser.value # => [1]
2613
+ * parser.value # ArgumentError no ready value
2614
+ */
2615
+ static VALUE cResumableParser_value(VALUE self)
2616
+ {
2617
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2618
+
2619
+ if (parser->frames.head > 0) {
2620
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2621
+
2622
+ if (frame->phase == JSON_PHASE_DONE) {
2623
+ VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
2624
+ rvalue_stack_pop(parser->state.value_stack, 1);
2625
+ json_frame_stack_pop(parser->state.frames);
2626
+ return result;
2627
+ }
2628
+ }
2629
+ rb_raise(rb_eArgError, "no ready value");
2630
+ }
2631
+
2632
+ /*
2633
+ * call-seq: clear -> self
2634
+ *
2635
+ * Entirely reset the parser state and buffer.
2636
+ */
2637
+ static VALUE cResumableParser_clear(VALUE self)
2638
+ {
2639
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2640
+ parser->buffer = 0;
2641
+ parser->complete = true;
2642
+ parser->parsed_bytes = 0;
2643
+ parser->incomplete_bytes = 0;
2644
+ parser->frames.head = 0;
2645
+ parser->value_stack.head = 0;
2646
+ parser->state.name_cache.length = 0;
2647
+ parser->state.current_nesting = 0;
2648
+ parser->state.in_array = 1;
2649
+ parser->state.emitted_deprecations = 0;
2650
+ parser->state.start = parser->state.cursor = parser->state.end = NULL;
2651
+ return self;
2652
+ }
2653
+
2654
+ static VALUE cResumableParser_partial_value_body(VALUE self)
2655
+ {
2656
+ JSON_ResumableParser *original_parser = cResumableParser_get(self);
2657
+ JSON_ResumableParser parser = *original_parser;
2658
+
2659
+ parser.state.frames = &parser.frames;
2660
+ parser.state.value_stack = &parser.value_stack;
2661
+
2662
+ if (parser.value_stack.head == 0) {
2663
+ return Qnil;
2664
+ }
2665
+
2666
+ json_frame *frame = json_frame_stack_peek(parser.state.frames);
2667
+ long missing_object_value = 0;
2668
+ if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
2669
+ missing_object_value = 1;
2670
+ }
2671
+
2672
+ // Copy the value stack as we need to mutate it. The collapse loop folds each
2673
+ // open container by popping its entries and pushing the single result, so a
2674
+ // parent always reclaims its child's slot; head exceeds its live size by at
2675
+ // most one, either for the missing-value placeholder pushed below or for the
2676
+ // result of folding an empty innermost container. That one spare slot keeps
2677
+ // rvalue_stack_push from growing (reallocating) this ALLOCV buffer.
2678
+ long capa = parser.value_stack.head;
2679
+ parser.value_stack.capa = capa + 1;
2680
+ VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, parser.value_stack.capa);
2681
+ MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, capa);
2682
+ parser.value_stack.ptr = value_stack_buffer;
2683
+
2684
+ JSON_ParserState *state = &parser.state;
2685
+ JSON_ParserConfig *config = &parser.config;
2686
+
2687
+ if (missing_object_value) {
2688
+ rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
2689
+ }
2690
+
2691
+ VALUE partial_result = Qundef;
2692
+
2693
+ while (UNDEF_P(partial_result)) {
2694
+ frame = json_frame_stack_peek(state->frames);
2695
+
2696
+ switch (frame->type) {
2697
+ case JSON_FRAME_ROOT: {
2698
+ partial_result = *rvalue_stack_peek(state->value_stack, 1);
2699
+ break;
2700
+ }
2701
+
2702
+ case JSON_FRAME_ARRAY: {
2703
+ long count = json_frame_entry_count(frame, state->value_stack);
2704
+ json_push_value(state, config, json_decode_array(state, config, count));
2705
+ json_frame_stack_pop(state->frames);
2706
+
2707
+ break;
2708
+ }
2709
+
2710
+ case JSON_FRAME_OBJECT: {
2711
+ long count = json_frame_entry_count(frame, state->value_stack);
2712
+ json_push_value(state, config, json_decode_object(state, config, count));
2713
+ json_frame_stack_pop(state->frames);
2714
+ break;
2715
+ }
2716
+
2717
+ default: {
2718
+ JSON_UNREACHABLE_RETURN(Qundef);
2719
+ break;
2720
+ }
2721
+ }
2722
+ }
2723
+
2724
+ ALLOCV_END(tmpbuf);
2725
+ return partial_result;
2726
+ }
2727
+
2728
+ /*
2729
+ * call-seq: partial_value -> object
2730
+ *
2731
+ * Returns the Ruby objects parsed up to this point:
2732
+ * parser << '[1, [2, 3,'
2733
+ * parser.parse # => false
2734
+ * parser.value # ArgumentError no ready value
2735
+ * parser.partial_value # => [1, [2, 3]]
2736
+ */
2737
+ static VALUE cResumableParser_partial_value(VALUE self)
2738
+ {
2739
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
2740
+
2741
+ int status;
2742
+ VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
2743
+ parser->in_use = false;
2744
+ if (status) {
2745
+ rb_jump_tag(status);
2746
+ }
2747
+ return result;
2748
+ }
2749
+
2750
+ /*
2751
+ * call-seq: rest -> string
2752
+ *
2753
+ * Returns a string containing what remains to be parsed in the buffer
2754
+ * parser << '{ "message": "unterminated message'
2755
+ * parser.parse # => false
2756
+ * parser.rest # => '"unterminated message"'
2757
+ */
2758
+ static VALUE cResumableParser_rest(VALUE self)
2759
+ {
2760
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2761
+
2762
+ if (!parser->buffer) {
2763
+ return rb_utf8_str_new("", 0);
2764
+ }
2765
+
2766
+ size_t offset = parser->state.cursor - parser->state.start;
2767
+ const char *ptr;
2768
+ long len;
2769
+ RSTRING_GETMEM(parser->buffer, ptr, len);
2770
+ return rb_utf8_str_new(ptr + offset, len - offset);
2771
+ }
2772
+
2773
+ /*
2774
+ * call-seq: eos? -> true or false
2775
+ *
2776
+ * Returns whether the internal buffer has been entirely consumed.
2777
+ */
2778
+ static VALUE cResumableParser_eos_p(VALUE self)
2779
+ {
2780
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2781
+ return eos(&parser->state) ? Qtrue : Qfalse;
2782
+ }
2783
+
2784
+ /*
2785
+ * call-seq: partial_value? -> true or false
2786
+ *
2787
+ * Returns whether a document is currently under construction: an unclosed
2788
+ * container, a key awaiting its value, etc.
2789
+ *
2790
+ * It answers the same question as <tt>!partial_value.nil?</tt>, but as a
2791
+ * cheap predicate on the parser's internal state, without materializing the
2792
+ * partially parsed Ruby objects:
2793
+ * parser << '{"a":1,'
2794
+ * parser.parse # => false
2795
+ * parser.partial_value? # => true
2796
+ *
2797
+ * A fully parsed document whose value hasn't been retrieved yet is not under
2798
+ * construction: #value? returns true and #partial_value? returns false.
2799
+ */
2800
+ static VALUE cResumableParser_partial_value_p(VALUE self)
2801
+ {
2802
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2803
+
2804
+ // Mirror of #value?: values on the stack while the document isn't DONE
2805
+ // belong to a partially built document. A container whose first key or
2806
+ // element hasn't been parsed yet has no frame nor value registered (the
2807
+ // tokenizer rewinds to the container start on EOS), so that state is
2808
+ // observable through the buffer (#eos?/#rest) instead, keeping this
2809
+ // predicate consistent with #partial_value returning nil.
2810
+ if (parser->value_stack.head > 0) {
2811
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2812
+ if (frame->phase != JSON_PHASE_DONE) {
2813
+ return Qtrue;
2814
+ }
2815
+ }
2816
+ return Qfalse;
2817
+ }
2818
+
2819
+ /*
2820
+ * call-seq: parsed_bytes -> integer
2821
+ *
2822
+ * Returns the number of bytes parsed since the start of the current partial value.
2823
+ * This is intended to be used for securing against untrusted input:
2824
+ *
2825
+ * loop do
2826
+ * if parser.parsed_bytes > DOCUMENT_MAX_SIZE
2827
+ * raise "document too large"
2828
+ * end
2829
+ *
2830
+ * parser << read_chunk
2831
+ * while parser.parse
2832
+ * process(parser.value)
2833
+ * end
2834
+ * end
2835
+ */
2836
+ static VALUE cResumableParser_parsed_bytes(VALUE self)
2837
+ {
2838
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2839
+ return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
2840
+ }
2841
+
1660
2842
  void Init_parser(void)
1661
2843
  {
1662
2844
  #ifdef HAVE_RB_EXT_RACTOR_SAFE
@@ -1668,30 +2850,53 @@ void Init_parser(void)
1668
2850
  mJSON = rb_define_module("JSON");
1669
2851
  VALUE mExt = rb_define_module_under(mJSON, "Ext");
1670
2852
  VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject);
2853
+
2854
+ rb_global_variable(&eParserError);
2855
+ eParserError = rb_path2class("JSON::ParserError");
2856
+
2857
+ rb_global_variable(&eNestingError);
1671
2858
  eNestingError = rb_path2class("JSON::NestingError");
1672
- rb_gc_register_mark_object(eNestingError);
2859
+
1673
2860
  rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate);
1674
- rb_define_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
2861
+ rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
1675
2862
  rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1);
1676
2863
 
1677
2864
  VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject);
1678
2865
  rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2);
1679
2866
 
2867
+ VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject);
2868
+ rb_define_alloc_func(cResumableParser, cResumableParser_allocate);
2869
+ rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1);
2870
+ rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1);
2871
+ rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0);
2872
+ rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
2873
+ rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
2874
+ rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
2875
+ rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0);
2876
+ rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
2877
+ rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
2878
+ rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
2879
+ rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0);
2880
+
2881
+ rb_global_variable(&CNaN);
1680
2882
  CNaN = rb_const_get(mJSON, rb_intern("NaN"));
1681
- rb_gc_register_mark_object(CNaN);
1682
2883
 
2884
+ rb_global_variable(&CInfinity);
1683
2885
  CInfinity = rb_const_get(mJSON, rb_intern("Infinity"));
1684
- rb_gc_register_mark_object(CInfinity);
1685
2886
 
2887
+ rb_global_variable(&CMinusInfinity);
1686
2888
  CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity"));
1687
- rb_gc_register_mark_object(CMinusInfinity);
1688
2889
 
1689
2890
  rb_global_variable(&Encoding_UTF_8);
1690
2891
  Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
1691
2892
 
2893
+ rb_global_variable(&JSON_empty_string);
2894
+ JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0));
2895
+
1692
2896
  sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
1693
2897
  sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
1694
2898
  sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
2899
+ sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
1695
2900
  sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
1696
2901
  sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
1697
2902
  sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
@@ -1702,8 +2907,12 @@ void Init_parser(void)
1702
2907
 
1703
2908
  i_new = rb_intern("new");
1704
2909
  i_try_convert = rb_intern("try_convert");
2910
+ #ifndef HAVE_RB_STR_TO_INTERNED_STR
1705
2911
  i_uminus = rb_intern("-@");
2912
+ #endif
1706
2913
  i_encode = rb_intern("encode");
2914
+ i_at_line = rb_intern("@line");
2915
+ i_at_column = rb_intern("@column");
1707
2916
 
1708
2917
  binary_encindex = rb_ascii8bit_encindex();
1709
2918
  utf8_encindex = rb_utf8_encindex();