json 2.19.2 → 2.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,18 @@
1
1
  #include "../json.h"
2
- #include "../vendor/ryu.h"
2
+ #include "../vendor/fast_float_parser.h"
3
3
  #include "../simd/simd.h"
4
4
 
5
- static VALUE mJSON, eNestingError, Encoding_UTF_8;
6
- static VALUE CNaN, CInfinity, CMinusInfinity;
5
+ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
6
+ static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
7
7
 
8
- static ID i_new, i_try_convert, i_uminus, i_encode;
8
+ static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column;
9
+ #ifndef HAVE_RB_STR_TO_INTERNED_STR
10
+ static ID i_uminus;
11
+ #endif
9
12
 
10
- static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters,
11
- sym_allow_invalid_escape, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load,
12
- sym_allow_duplicate_key;
13
+ static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
14
+ sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
15
+ sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
13
16
 
14
17
  static int binary_encindex;
15
18
  static int utf8_encindex;
@@ -58,6 +61,20 @@ typedef struct rvalue_cache_struct {
58
61
  VALUE entries[JSON_RVALUE_CACHE_CAPA];
59
62
  } rvalue_cache;
60
63
 
64
+ static void rvalue_cache_mark(rvalue_cache *cache)
65
+ {
66
+ for (int index = 0; index < cache->length; index++) {
67
+ rb_gc_mark_movable(cache->entries[index]);
68
+ }
69
+ }
70
+
71
+ static void rvalue_cache_compact(rvalue_cache *cache)
72
+ {
73
+ for (int index = 0; index < cache->length; index++) {
74
+ cache->entries[index] = rb_gc_location(cache->entries[index]);
75
+ }
76
+ }
77
+
61
78
  static rb_encoding *enc_utf8;
62
79
 
63
80
  #define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55
@@ -206,12 +223,12 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle,
206
223
 
207
224
  static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref)
208
225
  {
209
- long required = stack->capa * 2;
226
+ long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA;
210
227
 
211
228
  if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
212
229
  stack = rvalue_stack_spill(stack, handle, stack_ref);
213
230
  } else {
214
- REALLOC_N(stack->ptr, VALUE, required);
231
+ JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa);
215
232
  stack->capa = required;
216
233
  }
217
234
  return stack;
@@ -219,11 +236,15 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu
219
236
 
220
237
  static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref)
221
238
  {
239
+ JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle);
240
+
222
241
  if (RB_UNLIKELY(stack->head >= stack->capa)) {
223
242
  stack = rvalue_stack_grow(stack, handle, stack_ref);
224
243
  }
244
+
225
245
  stack->ptr[stack->head] = value;
226
246
  stack->head++;
247
+
227
248
  return value;
228
249
  }
229
250
 
@@ -241,35 +262,62 @@ static void rvalue_stack_mark(void *ptr)
241
262
  {
242
263
  rvalue_stack *stack = (rvalue_stack *)ptr;
243
264
  long index;
244
- for (index = 0; index < stack->head; index++) {
245
- rb_gc_mark(stack->ptr[index]);
265
+ if (stack && stack->ptr) {
266
+ for (index = 0; index < stack->head; index++) {
267
+ rb_gc_mark_movable(stack->ptr[index]);
268
+ }
246
269
  }
247
270
  }
248
271
 
272
+ static void rvalue_stack_free_buffer(rvalue_stack *stack)
273
+ {
274
+ JSON_SIZED_FREE_N(stack->ptr, stack->capa);
275
+ stack->ptr = NULL;
276
+ }
277
+
249
278
  static void rvalue_stack_free(void *ptr)
250
279
  {
251
280
  rvalue_stack *stack = (rvalue_stack *)ptr;
252
281
  if (stack) {
253
- ruby_xfree(stack->ptr);
254
- ruby_xfree(stack);
282
+ rvalue_stack_free_buffer(stack);
283
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
284
+ JSON_SIZED_FREE(stack);
285
+ #endif
255
286
  }
256
287
  }
257
288
 
258
289
  static size_t rvalue_stack_memsize(const void *ptr)
259
290
  {
260
291
  const rvalue_stack *stack = (const rvalue_stack *)ptr;
261
- return sizeof(rvalue_stack) + sizeof(VALUE) * stack->capa;
292
+ size_t memsize = sizeof(VALUE) * stack->capa;
293
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
294
+ memsize += sizeof(rvalue_stack);
295
+ #endif
296
+ return memsize;
297
+ }
298
+
299
+ static void rvalue_stack_compact(void *ptr)
300
+ {
301
+ rvalue_stack *stack = (rvalue_stack *)ptr;
302
+ long index;
303
+ if (stack && stack->ptr) {
304
+ for (index = 0; index < stack->head; index++) {
305
+ stack->ptr[index] = rb_gc_location(stack->ptr[index]);
306
+ }
307
+ }
262
308
  }
263
309
 
264
310
  static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
265
- "JSON::Ext::Parser/rvalue_stack",
266
- {
311
+ .wrap_struct_name = "JSON::Ext::Parser/rvalue_stack",
312
+ .function = {
267
313
  .dmark = rvalue_stack_mark,
268
314
  .dfree = rvalue_stack_free,
269
315
  .dsize = rvalue_stack_memsize,
316
+ .dcompact = rvalue_stack_compact,
270
317
  },
271
- 0, 0,
272
- RUBY_TYPED_FREE_IMMEDIATELY,
318
+ // We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
319
+ // because it churns a lot of values so trigering write barriers every time is very costly.
320
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
273
321
  };
274
322
 
275
323
  static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
@@ -291,38 +339,71 @@ static void rvalue_stack_eagerly_release(VALUE handle)
291
339
  if (handle) {
292
340
  rvalue_stack *stack;
293
341
  TypedData_Get_Struct(handle, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack);
294
- RTYPEDDATA_DATA(handle) = NULL;
342
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
343
+ rvalue_stack_free_buffer(stack);
344
+ #else
295
345
  rvalue_stack_free(stack);
346
+ RTYPEDDATA_DATA(handle) = NULL;
347
+ #endif
296
348
  }
297
349
  }
298
350
 
299
- static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
300
- {
301
- int len = 1;
302
- if (ch <= 0x7F) {
303
- buf[0] = (char) ch;
304
- } else if (ch <= 0x07FF) {
305
- buf[0] = (char) ((ch >> 6) | 0xC0);
306
- buf[1] = (char) ((ch & 0x3F) | 0x80);
307
- len++;
308
- } else if (ch <= 0xFFFF) {
309
- buf[0] = (char) ((ch >> 12) | 0xE0);
310
- buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
311
- buf[2] = (char) ((ch & 0x3F) | 0x80);
312
- len += 2;
313
- } else if (ch <= 0x1fffff) {
314
- buf[0] =(char) ((ch >> 18) | 0xF0);
315
- buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
316
- buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
317
- buf[3] =(char) ((ch & 0x3F) | 0x80);
318
- len += 3;
319
- } else {
320
- buf[0] = '?';
321
- }
322
- return len;
323
- }
351
+ /* frame stack */
352
+
353
+ // Iterative (non-recursive) parsing keeps an explicit stack of the containers
354
+ // currently being built, instead of relying on the C call stack. Each frame
355
+ // only needs enough bookkeeping to close its container: which kind it is, the
356
+ // rvalue_stack position where its children start (so we know how many to pop),
357
+ // and the cursor at its opening brace (used to rewind for duplicate key
358
+ // errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses
359
+ // the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that
360
+ // it's freed even if parsing raises.
361
+ //
362
+ // The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release
363
+ // and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts
364
+ // -- the element type and the absence of a mark function are the only real
365
+ // differences. Keep the two in sync: a fix to the spill/release or
366
+ // HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the
367
+ // other.
368
+ #define JSON_FRAME_STACK_INITIAL_CAPA 32
369
+
370
+ enum json_frame_type {
371
+ JSON_FRAME_ROOT, // == JSON_PHASE_DONE
372
+ JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA
373
+ JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA
374
+ };
375
+
376
+ // Where a frame is within its container's grammar. This is the entirety of the
377
+ // parser's "what to do next" state: json_parse_any dispatches on the top
378
+ // frame's phase and holds no resume state in C locals, so a parse can stop at
379
+ // any value boundary and be resumed purely from the (persistable) frame stack.
380
+ //
381
+ // The first three phases are deliberately equal to the corresponding json_frame_type
382
+ // to simplify the transition of phase in json_value_completed.
383
+ enum json_frame_phase {
384
+ JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed
385
+ JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']'
386
+ JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}'
387
+ JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':')
388
+ JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',')
389
+ JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':'
390
+ };
391
+
392
+ typedef struct json_frame_struct {
393
+ enum json_frame_type type;
394
+ enum json_frame_phase phase;
395
+ long value_stack_head; // rvalue_stack->head when this container opened
396
+ size_t start_offset; // object frames only (the '{'); NULL otherwise
397
+ } json_frame;
398
+
399
+ typedef struct json_frame_stack_struct {
400
+ enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated
401
+ long capa;
402
+ long head;
403
+ json_frame *ptr;
404
+ } json_frame_stack;
324
405
 
325
- enum duplicate_key_action {
406
+ enum deprecatable_action {
326
407
  JSON_DEPRECATED = 0,
327
408
  JSON_IGNORE,
328
409
  JSON_RAISE,
@@ -332,7 +413,8 @@ typedef struct JSON_ParserStruct {
332
413
  VALUE on_load_proc;
333
414
  VALUE decimal_class;
334
415
  ID decimal_method_id;
335
- enum duplicate_key_action on_duplicate_key;
416
+ enum deprecatable_action on_duplicate_key;
417
+ enum deprecatable_action on_comment;
336
418
  int max_nesting;
337
419
  bool allow_nan;
338
420
  bool allow_trailing_comma;
@@ -343,16 +425,152 @@ typedef struct JSON_ParserStruct {
343
425
  } JSON_ParserConfig;
344
426
 
345
427
  typedef struct JSON_ParserStateStruct {
346
- VALUE stack_handle;
428
+ VALUE *value_stack_handle;
429
+ VALUE *frame_stack_handle;
347
430
  const char *start;
348
431
  const char *cursor;
349
432
  const char *end;
350
- rvalue_stack *stack;
433
+ rvalue_stack *value_stack;
434
+ json_frame_stack *frames;
351
435
  rvalue_cache name_cache;
352
436
  int in_array;
353
437
  int current_nesting;
438
+ unsigned int emitted_deprecations;
439
+ VALUE parser;
354
440
  } JSON_ParserState;
355
441
 
442
+ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref);
443
+
444
+ static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref)
445
+ {
446
+ long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA;
447
+
448
+ if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
449
+ stack = json_frame_stack_spill(stack, handle, stack_ref);
450
+ } else {
451
+ JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa);
452
+ stack->capa = required;
453
+ }
454
+ return stack;
455
+ }
456
+
457
+ static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame)
458
+ {
459
+ json_frame_stack *stack = state->frames;
460
+
461
+ JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle);
462
+
463
+ if (RB_UNLIKELY(stack->head >= stack->capa)) {
464
+ stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames);
465
+ }
466
+
467
+ json_frame *frame_ptr = &stack->ptr[stack->head++];
468
+ *frame_ptr = frame;
469
+ return frame_ptr;
470
+ }
471
+
472
+ static inline json_frame *json_frame_stack_peek(json_frame_stack *stack)
473
+ {
474
+ return &stack->ptr[stack->head - 1];
475
+ }
476
+
477
+ static inline void json_frame_stack_pop(json_frame_stack *stack)
478
+ {
479
+ stack->head--;
480
+ }
481
+
482
+ static void json_frame_stack_free_buffer(json_frame_stack *stack)
483
+ {
484
+ JSON_SIZED_FREE_N(stack->ptr, stack->capa);
485
+ stack->ptr = NULL;
486
+ }
487
+
488
+ static void json_frame_stack_free(void *ptr)
489
+ {
490
+ json_frame_stack *stack = (json_frame_stack *)ptr;
491
+ if (stack) {
492
+ json_frame_stack_free_buffer(stack);
493
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
494
+ JSON_SIZED_FREE(stack);
495
+ #endif
496
+ }
497
+ }
498
+
499
+ static size_t json_frame_stack_memsize(const void *ptr)
500
+ {
501
+ const json_frame_stack *stack = (const json_frame_stack *)ptr;
502
+
503
+ size_t memsize = sizeof(json_frame) * stack->capa;
504
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
505
+ memsize += sizeof(json_frame_stack);
506
+ #endif
507
+ return memsize;
508
+ }
509
+
510
+ static const rb_data_type_t JSON_Parser_frame_stack_type = {
511
+ .wrap_struct_name = "JSON::Ext::Parser/frame_stack",
512
+ .function = {
513
+ .dmark = NULL,
514
+ .dfree = json_frame_stack_free,
515
+ .dsize = json_frame_stack_memsize,
516
+ },
517
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
518
+ };
519
+
520
+ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
521
+ {
522
+ json_frame_stack *stack;
523
+ *handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
524
+ *stack_ref = stack;
525
+ MEMCPY(stack, old_stack, json_frame_stack, 1);
526
+
527
+ stack->capa = old_stack->capa << 1;
528
+ stack->ptr = ALLOC_N(json_frame, stack->capa);
529
+ stack->type = RVALUE_STACK_HEAP_ALLOCATED;
530
+ MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head);
531
+ return stack;
532
+ }
533
+
534
+ static void json_frame_stack_eagerly_release(VALUE handle)
535
+ {
536
+ if (handle) {
537
+ json_frame_stack *stack;
538
+ TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
539
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
540
+ json_frame_stack_free_buffer(stack);
541
+ #else
542
+ json_frame_stack_free(stack);
543
+ RTYPEDDATA_DATA(handle) = NULL;
544
+ #endif
545
+ }
546
+ }
547
+
548
+ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
549
+ {
550
+ int len = 1;
551
+ if (ch <= 0x7F) {
552
+ buf[0] = (char) ch;
553
+ } else if (ch <= 0x07FF) {
554
+ buf[0] = (char) ((ch >> 6) | 0xC0);
555
+ buf[1] = (char) ((ch & 0x3F) | 0x80);
556
+ len++;
557
+ } else if (ch <= 0xFFFF) {
558
+ buf[0] = (char) ((ch >> 12) | 0xE0);
559
+ buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
560
+ buf[2] = (char) ((ch & 0x3F) | 0x80);
561
+ len += 2;
562
+ } else if (ch <= 0x1fffff) {
563
+ buf[0] =(char) ((ch >> 18) | 0xF0);
564
+ buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
565
+ buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
566
+ buf[3] =(char) ((ch & 0x3F) | 0x80);
567
+ len += 3;
568
+ } else {
569
+ buf[0] = '?';
570
+ }
571
+ return len;
572
+ }
573
+
356
574
  static inline size_t rest(JSON_ParserState *state) {
357
575
  return state->end - state->cursor;
358
576
  }
@@ -371,12 +589,22 @@ static inline char peek(JSON_ParserState *state)
371
589
 
372
590
  static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
373
591
  {
592
+ JSON_ASSERT(!state->parser);
593
+ JSON_ASSERT(state->cursor);
594
+ JSON_ASSERT(state->cursor <= state->end);
595
+
596
+ // Redundant but helpful for hardening
597
+ if (RB_UNLIKELY(state->cursor > state->end)) {
598
+ state->cursor = state->end;
599
+ }
600
+
374
601
  const char *cursor = state->cursor;
375
602
  long column = 0;
376
603
  long line = 1;
377
604
 
378
605
  while (cursor >= state->start) {
379
606
  if (*cursor-- == '\n') {
607
+ line++;
380
608
  break;
381
609
  }
382
610
  column++;
@@ -391,18 +619,24 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum
391
619
  *column_out = column;
392
620
  }
393
621
 
622
+ static const unsigned int MAX_DEPRECATIONS = 5;
623
+
394
624
  static void emit_parse_warning(const char *message, JSON_ParserState *state)
395
625
  {
396
- long line, column;
397
- cursor_position(state, &line, &column);
398
-
399
- VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
626
+ VALUE warning;
627
+ if (state->parser) { // line and columns can't be accurate in resumable
628
+ warning = rb_utf8_str_new_cstr(message);
629
+ } else {
630
+ long line, column;
631
+ cursor_position(state, &line, &column);
632
+ warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
633
+ }
400
634
  rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning);
401
635
  }
402
636
 
403
637
  #define PARSE_ERROR_FRAGMENT_LEN 32
404
638
 
405
- static VALUE build_parse_error_message(const char *format, JSON_ParserState *state, long line, long column)
639
+ static VALUE build_parse_error_message(const char *format, JSON_ParserState *state)
406
640
  {
407
641
  unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
408
642
 
@@ -436,32 +670,61 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta
436
670
  }
437
671
  }
438
672
 
439
- VALUE msg = rb_sprintf(format, ptr);
440
- VALUE message = rb_enc_sprintf(enc_utf8, "%s at line %ld column %ld", RSTRING_PTR(msg), line, column);
441
- RB_GC_GUARD(msg);
442
- return message;
673
+ return rb_enc_sprintf(enc_utf8, format, ptr);
443
674
  }
444
675
 
445
- static VALUE parse_error_new(VALUE message, long line, long column)
676
+ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
446
677
  {
447
- VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message);
448
- rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line));
449
- rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column));
678
+ VALUE exc = rb_exc_new_str(eParserError, message);
679
+ rb_ivar_set(exc, i_at_line, LONG2NUM(line));
680
+ rb_ivar_set(exc, i_at_column, LONG2NUM(column));
450
681
  return exc;
451
682
  }
452
683
 
453
- NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state)
684
+ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos)
685
+ {
686
+ if (state->parser) {
687
+ if (eos) {
688
+ // the error will be swallowed by ResumableParser#parse, so no
689
+ // point building a message or backtrace.
690
+ rb_throw_obj(state->parser, state->parser);
691
+ } else {
692
+ // line and columns can't be accurate in resumable
693
+ rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos));
694
+ }
695
+ } else {
696
+ VALUE message = build_parse_error_message(format, state);
697
+ long line, column;
698
+ cursor_position(state, &line, &column);
699
+ rb_str_catf(message, " at line %ld column %ld", line, column);
700
+ rb_exc_raise(parse_error_new(state, message, line, column, eos));
701
+ }
702
+ }
703
+
704
+ NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state)
705
+ {
706
+ raise_parse_error(format, state, true);
707
+ }
708
+
709
+ NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state)
454
710
  {
455
- long line, column;
456
- cursor_position(state, &line, &column);
457
- VALUE message = build_parse_error_message(format, state, line, column);
458
- rb_exc_raise(parse_error_new(message, line, column));
711
+ raise_parse_error(format, state, false);
459
712
  }
460
713
 
461
- NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at)
714
+ NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos)
462
715
  {
463
716
  state->cursor = at;
464
- raise_parse_error(format, state);
717
+ raise_parse_error(format, state, eos);
718
+ }
719
+
720
+ NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at)
721
+ {
722
+ raise_parse_error_at(format, state, at, true);
723
+ }
724
+
725
+ NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at)
726
+ {
727
+ raise_parse_error_at(format, state, at, false);
465
728
  }
466
729
 
467
730
  /* unicode */
@@ -486,7 +749,7 @@ static const signed char digit_values[256] = {
486
749
  static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
487
750
  {
488
751
  if (RB_UNLIKELY(sp > spe - 4)) {
489
- raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
752
+ raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
490
753
  }
491
754
 
492
755
  const unsigned char *p = (const unsigned char *)sp;
@@ -497,7 +760,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
497
760
  const signed char b3 = digit_values[p[3]];
498
761
 
499
762
  if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
500
- raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
763
+ raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
501
764
  }
502
765
 
503
766
  return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
@@ -509,19 +772,36 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
509
772
 
510
773
  static const rb_data_type_t JSON_ParserConfig_type;
511
774
 
512
- static void
513
- json_eat_comments(JSON_ParserState *state)
775
+ const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
776
+ NOINLINE(static) void
777
+ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos)
514
778
  {
779
+ if (config->on_comment == JSON_RAISE) {
780
+ raise_syntax_error("unexpected token %s", state);
781
+ }
782
+
515
783
  const char *start = state->cursor;
784
+ // An incomplete comment suspends a resumable parse by rewinding the cursor
785
+ // and throwing. Callers that already consumed a token not yet committed to
786
+ // the frame stack pass resume_pos so the rewind re-reads that token too.
787
+ // Non-resumable error positions keep pointing at the comment either way.
788
+ const char *rewind_pos = (state->parser && resume_pos) ? resume_pos : start;
516
789
  state->cursor++;
517
790
 
518
791
  switch (peek(state)) {
519
792
  case '/': {
520
- state->cursor = memchr(state->cursor, '\n', state->end - state->cursor);
521
- if (!state->cursor) {
793
+ const char *newline = memchr(state->cursor, '\n', state->end - state->cursor);
794
+ if (!newline) {
795
+ // state->parser marks resumable mode, where the buffer end is only a
796
+ // chunk boundary: the terminating newline may still arrive, so leave
797
+ // the comment unterminated instead of consuming to end as a one-shot
798
+ // parse would.
799
+ if (state->parser) {
800
+ raise_eos_error_at("unterminated comment, expected end of line", state, rewind_pos);
801
+ }
522
802
  state->cursor = state->end;
523
803
  } else {
524
- state->cursor++;
804
+ state->cursor = newline + 1;
525
805
  }
526
806
  break;
527
807
  }
@@ -531,7 +811,7 @@ json_eat_comments(JSON_ParserState *state)
531
811
  while (true) {
532
812
  const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
533
813
  if (!next_match) {
534
- raise_parse_error_at("unterminated comment, expected closing '*/'", state, start);
814
+ raise_eos_error_at("unterminated comment, expected closing '*/'", state, rewind_pos);
535
815
  }
536
816
 
537
817
  state->cursor = next_match + 1;
@@ -543,13 +823,18 @@ json_eat_comments(JSON_ParserState *state)
543
823
  break;
544
824
  }
545
825
  default:
546
- raise_parse_error_at("unexpected token %s", state, start);
826
+ raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state));
547
827
  break;
548
828
  }
829
+
830
+ if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) {
831
+ state->emitted_deprecations++;
832
+ emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state);
833
+ }
549
834
  }
550
835
 
551
836
  ALWAYS_INLINE(static) void
552
- json_eat_whitespace(JSON_ParserState *state)
837
+ json_eat_whitespace_resume_at(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments, const char *resume_pos)
553
838
  {
554
839
  while (true) {
555
840
  switch (peek(state)) {
@@ -580,7 +865,11 @@ json_eat_whitespace(JSON_ParserState *state)
580
865
  state->cursor++;
581
866
  break;
582
867
  case '/':
583
- json_eat_comments(state);
868
+ if (!include_comments) {
869
+ return;
870
+ }
871
+
872
+ json_eat_comments(state, config, resume_pos);
584
873
  break;
585
874
 
586
875
  default:
@@ -589,6 +878,12 @@ json_eat_whitespace(JSON_ParserState *state)
589
878
  }
590
879
  }
591
880
 
881
+ ALWAYS_INLINE(static) void
882
+ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
883
+ {
884
+ json_eat_whitespace_resume_at(state, config, include_comments, NULL);
885
+ }
886
+
592
887
  static inline VALUE build_string(const char *start, const char *end, bool intern, bool symbolize)
593
888
  {
594
889
  if (symbolize) {
@@ -734,13 +1029,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
734
1029
  uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
735
1030
 
736
1031
  if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
737
- raise_parse_error_at("invalid surrogate pair at %s", state, p);
1032
+ raise_syntax_error_at("invalid surrogate pair at %s", state, p);
738
1033
  }
739
1034
 
740
1035
  ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
741
1036
  pe += 5;
742
1037
  } else {
743
- raise_parse_error_at("incomplete surrogate pair at %s", state, p);
1038
+ raise_syntax_error_at("incomplete surrogate pair at %s", state, p);
744
1039
  break;
745
1040
  }
746
1041
  }
@@ -750,18 +1045,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
750
1045
  p = ++pe;
751
1046
  break;
752
1047
  }
1048
+ case 0:
1049
+ return Qundef;
753
1050
  default:
754
1051
  if ((unsigned char)*pe < 0x20) {
755
1052
  if (!config->allow_control_characters) {
756
1053
  if (*pe == '\n') {
757
- raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
1054
+ raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
758
1055
  }
759
- raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1);
1056
+ raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1);
760
1057
  }
761
- } else if (config->allow_invalid_escape) {
1058
+ }
1059
+
1060
+ if (config->allow_invalid_escape) {
762
1061
  APPEND_CHAR(*pe);
763
1062
  } else {
764
- raise_parse_error_at("invalid escape character in string: %s", state, pe - 1);
1063
+ raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1);
765
1064
  }
766
1065
  break;
767
1066
  }
@@ -841,7 +1140,7 @@ NOINLINE(static) VALUE json_decode_large_float(const char *start, long len)
841
1140
  /* Ruby JSON optimized float decoder using vendored Ryu algorithm
842
1141
  * Accepts pre-extracted mantissa and exponent from first-pass validation
843
1142
  */
844
- static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int32_t exponent, bool negative,
1143
+ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int64_t exponent, bool negative,
845
1144
  const char *start, const char *end)
846
1145
  {
847
1146
  if (RB_UNLIKELY(config->decimal_class)) {
@@ -849,19 +1148,32 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
849
1148
  return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text);
850
1149
  }
851
1150
 
852
- // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case)
853
- // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308)
854
- if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) {
1151
+ if (RB_UNLIKELY(exponent > INT32_MAX)) {
1152
+ return negative ? CMinusInfinity : CInfinity;
1153
+ }
1154
+
1155
+ if (RB_UNLIKELY(exponent < INT32_MIN)) {
1156
+ return rb_float_new(negative ? -0.0 : 0.0);
1157
+ }
1158
+
1159
+ if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
1160
+ // If the value is so small that it definitely underflows to 0.0, return early
1161
+ // to avoid triggering a "Float out of range" warning from rb_cstr_to_dbl.
1162
+ // When mantissa_digits + exponent < -324, value < 10^(-324) < DBL_TRUE_MIN/2,
1163
+ // so it rounds to 0 in IEEE 754 round-to-nearest.
1164
+ if (RB_UNLIKELY(mantissa_digits + exponent < -324)) {
1165
+ return rb_float_new(negative ? -0.0 : 0.0);
1166
+ }
855
1167
  return json_decode_large_float(start, end - start);
856
1168
  }
857
1169
 
858
- return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, exponent, negative));
1170
+ return DBL2NUM(ffp_s2d(exponent, mantissa, negative));
859
1171
  }
860
1172
 
861
1173
  static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
862
1174
  {
863
- VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count));
864
- rvalue_stack_pop(state->stack, count);
1175
+ VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count));
1176
+ rvalue_stack_pop(state->value_stack, count);
865
1177
 
866
1178
  if (config->freeze) {
867
1179
  RB_OBJ_FREEZE(array);
@@ -905,36 +1217,50 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d
905
1217
  rb_inspect(duplicate_key)
906
1218
  );
907
1219
 
908
- long line, column;
909
- cursor_position(state, &line, &column);
910
- rb_str_concat(message, build_parse_error_message("", state, line, column)) ;
911
- rb_exc_raise(parse_error_new(message, line, column));
1220
+ rb_str_concat(message, build_parse_error_message("", state));
1221
+ if (state->parser) { // line and columns can't be accurate in resumable
1222
+ rb_exc_raise(parse_error_new(state, message, 0, 0, false));
1223
+ } else {
1224
+ long line, column;
1225
+ cursor_position(state, &line, &column);
1226
+ rb_str_catf(message, " at line %ld column %ld", line, column);
1227
+ rb_exc_raise(parse_error_new(state, message, line, column, false));
1228
+ }
1229
+ }
912
1230
 
913
- raise_parse_error(RSTRING_PTR(message), state);
914
- RB_GC_GUARD(message);
1231
+ NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
1232
+ {
1233
+ switch (config->on_duplicate_key) {
1234
+ case JSON_IGNORE:
1235
+ return;
1236
+
1237
+ case JSON_DEPRECATED:
1238
+ // Only emit the first few deprecations to avoid spamming.
1239
+ if (state->emitted_deprecations < MAX_DEPRECATIONS) {
1240
+ state->emitted_deprecations++;
1241
+ emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
1242
+ }
1243
+ return;
1244
+
1245
+ case JSON_RAISE:
1246
+ raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
1247
+ return;
1248
+ }
1249
+ UNREACHABLE;
915
1250
  }
916
1251
 
917
1252
  static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
918
1253
  {
919
1254
  size_t entries_count = count / 2;
920
1255
  VALUE object = rb_hash_new_capa(entries_count);
921
- const VALUE *pairs = rvalue_stack_peek(state->stack, count);
1256
+ const VALUE *pairs = rvalue_stack_peek(state->value_stack, count);
922
1257
  rb_hash_bulk_insert(count, pairs, object);
923
1258
 
924
1259
  if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
925
- switch (config->on_duplicate_key) {
926
- case JSON_IGNORE:
927
- break;
928
- case JSON_DEPRECATED:
929
- emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
930
- break;
931
- case JSON_RAISE:
932
- raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
933
- break;
934
- }
1260
+ json_on_duplicate_key(state, config, count, pairs);
935
1261
  }
936
1262
 
937
- rvalue_stack_pop(state->stack, count);
1263
+ rvalue_stack_pop(state->value_stack, count);
938
1264
 
939
1265
  if (config->freeze) {
940
1266
  RB_OBJ_FREEZE(object);
@@ -948,7 +1274,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
948
1274
  if (RB_UNLIKELY(config->on_load_proc)) {
949
1275
  value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
950
1276
  }
951
- rvalue_stack_push(state->stack, value, &state->stack_handle, &state->stack);
1277
+ rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack);
952
1278
  return value;
953
1279
  }
954
1280
 
@@ -997,6 +1323,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
997
1323
  }
998
1324
  state->cursor++;
999
1325
  }
1326
+
1327
+ // If the string ended with an unterminated escape sequence, we might
1328
+ // have gone past the end.
1329
+ if (RB_UNLIKELY(state->cursor > state->end)) {
1330
+ state->cursor = state->end;
1331
+ }
1332
+
1000
1333
  return false;
1001
1334
  }
1002
1335
 
@@ -1014,7 +1347,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1014
1347
  case '"': {
1015
1348
  VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
1016
1349
  state->cursor++;
1017
- return json_push_value(state, config, string);
1350
+ return string;
1018
1351
  }
1019
1352
  case '\\': {
1020
1353
  if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
@@ -1028,7 +1361,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1028
1361
  }
1029
1362
  default:
1030
1363
  if (!config->allow_control_characters) {
1031
- raise_parse_error("invalid ASCII control character in string: %s", state);
1364
+ raise_syntax_error("invalid ASCII control character in string: %s", state);
1032
1365
  }
1033
1366
  break;
1034
1367
  }
@@ -1036,8 +1369,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1036
1369
  state->cursor++;
1037
1370
  } while (string_scan(state));
1038
1371
 
1039
- raise_parse_error("unexpected end of input, expected closing \"", state);
1040
- return Qfalse;
1372
+ return Qundef;
1041
1373
  }
1042
1374
 
1043
1375
  ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
@@ -1046,15 +1378,19 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars
1046
1378
  const char *start = state->cursor;
1047
1379
 
1048
1380
  if (RB_UNLIKELY(!string_scan(state))) {
1049
- raise_parse_error("unexpected end of input, expected closing \"", state);
1381
+ return Qundef;
1050
1382
  }
1051
1383
 
1384
+ VALUE string;
1052
1385
  if (RB_LIKELY(*state->cursor == '"')) {
1053
- VALUE string = json_string_fastpath(state, config, start, state->cursor, is_name);
1386
+ string = json_string_fastpath(state, config, start, state->cursor, is_name);
1054
1387
  state->cursor++;
1055
- return json_push_value(state, config, string);
1056
1388
  }
1057
- return json_parse_escaped_string(state, config, is_name, start);
1389
+ else {
1390
+ string = json_parse_escaped_string(state, config, is_name, start);
1391
+ }
1392
+
1393
+ return string;
1058
1394
  }
1059
1395
 
1060
1396
  #if JSON_CPU_LITTLE_ENDIAN_64BITS
@@ -1127,13 +1463,13 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat
1127
1463
  return (int)(state->cursor - start);
1128
1464
  }
1129
1465
 
1130
- static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
1466
+ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable)
1131
1467
  {
1132
1468
  bool integer = true;
1133
1469
  const char first_digit = *state->cursor;
1134
1470
 
1135
1471
  // Variables for Ryu optimization - extract digits during parsing
1136
- int32_t exponent = 0;
1472
+ int64_t exponent = 0;
1137
1473
  int decimal_point_pos = -1;
1138
1474
  uint64_t mantissa = 0;
1139
1475
 
@@ -1141,7 +1477,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1141
1477
  int mantissa_digits = json_parse_digits(state, &mantissa);
1142
1478
 
1143
1479
  if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
1144
- raise_parse_error_at("invalid number: %s", state, start);
1480
+ return Qundef;
1145
1481
  }
1146
1482
 
1147
1483
  // Parse fractional part
@@ -1154,7 +1490,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1154
1490
  mantissa_digits += fractional_digits;
1155
1491
 
1156
1492
  if (RB_UNLIKELY(!fractional_digits)) {
1157
- raise_parse_error_at("invalid number: %s", state, start);
1493
+ return Qundef;
1158
1494
  }
1159
1495
  }
1160
1496
 
@@ -1174,10 +1510,24 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1174
1510
  int exponent_digits = json_parse_digits(state, &abs_exponent);
1175
1511
 
1176
1512
  if (RB_UNLIKELY(!exponent_digits)) {
1177
- raise_parse_error_at("invalid number: %s", state, start);
1513
+ return Qundef;
1514
+ }
1515
+
1516
+ if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
1517
+ exponent = negative_exponent ? INT64_MIN : INT64_MAX;
1518
+ } else {
1519
+ exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
1178
1520
  }
1521
+ }
1179
1522
 
1180
- exponent = negative_exponent ? -((int32_t)abs_exponent) : ((int32_t)abs_exponent);
1523
+ // A number touching the end of the buffer may still grow in a later chunk,
1524
+ // so the caller will rewind and wait. Decoding it now would build a value
1525
+ // -- for a long run of digits, an expensive bignum -- only to discard it,
1526
+ // and repeating that on every resumed chunk is quadratic in the number's
1527
+ // length. The digit scan above already advanced the cursor, which is all
1528
+ // the caller needs to detect the incomplete number.
1529
+ if (RB_UNLIKELY(resumable && eos(state))) {
1530
+ return Qundef;
1181
1531
  }
1182
1532
 
1183
1533
  if (integer) {
@@ -1192,229 +1542,420 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1192
1542
  return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
1193
1543
  }
1194
1544
 
1195
- static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_ParserConfig *config)
1545
+ // How many values (array elements, or interleaved object keys+values) have been
1546
+ // pushed onto the rvalue stack since this container opened. Used to size the
1547
+ // bulk decode on close, and to tell the first key/colon from later ones.
1548
+ static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack)
1196
1549
  {
1197
- return json_parse_number(state, config, false, state->cursor);
1550
+ return value_stack->head - frame->value_stack_head;
1198
1551
  }
1199
1552
 
1200
- static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config)
1553
+ // A complete value now sits on top of the rvalue stack. Advance the frame that
1554
+ // was waiting for it: the root document is done, or the enclosing container
1555
+ // moves on to expecting a ',' or its closing bracket. The caller passes the
1556
+ // frame it already has in hand -- the one that was expecting the value -- which
1557
+ // after a container close is the freshly re-exposed parent.
1558
+ static inline enum json_frame_phase json_value_completed(json_frame *frame)
1201
1559
  {
1202
- const char *start = state->cursor;
1203
- state->cursor++;
1204
- return json_parse_number(state, config, true, start);
1560
+ JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT);
1561
+ JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY);
1562
+ JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT);
1563
+
1564
+ return frame->phase = (enum json_frame_phase) frame->type;
1205
1565
  }
1206
1566
 
1207
- static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
1567
+ ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset)
1208
1568
  {
1209
- json_eat_whitespace(state);
1210
-
1211
- switch (peek(state)) {
1212
- case 'n':
1213
- if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
1214
- state->cursor += 4;
1215
- return json_push_value(state, config, Qnil);
1216
- }
1569
+ // It is assumed that since `keyword` is always a literal, the compiler is able to constantize this
1570
+ // `strlen` and several other computations in that routine.
1217
1571
 
1218
- raise_parse_error("unexpected token %s", state);
1219
- break;
1220
- case 't':
1221
- if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) {
1222
- state->cursor += 4;
1223
- return json_push_value(state, config, Qtrue);
1224
- }
1572
+ size_t len = strlen(keyword);
1225
1573
 
1226
- raise_parse_error("unexpected token %s", state);
1227
- break;
1228
- case 'f':
1229
- // Note: memcmp with a small power of two compile to an integer comparison
1230
- if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) {
1231
- state->cursor += 5;
1232
- return json_push_value(state, config, Qfalse);
1233
- }
1574
+ // Note: memcmp with a small power of two and a literal string compile to an integer comparison /
1575
+ // That's why we sometime compare starting from the first byte and sometimes from the second.
1576
+ if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) {
1577
+ state->cursor += len;
1578
+ return;
1579
+ }
1234
1580
 
1235
- raise_parse_error("unexpected token %s", state);
1236
- break;
1237
- case 'N':
1238
- // Note: memcmp with a small power of two compile to an integer comparison
1239
- if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) {
1240
- state->cursor += 3;
1241
- return json_push_value(state, config, CNaN);
1242
- }
1581
+ bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0;
1582
+ raise_parse_error("unexpected token %s", state, eos);
1583
+ }
1584
+
1585
+ // Parse an arbitrary JSON value iteratively. This is a state machine driven
1586
+ // entirely by the top frame's phase so it can stop at any value boundary and
1587
+ // resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the
1588
+ // bottom of the stack, so the stack is never empty mid-parse and the document
1589
+ // itself is just another frame whose value, once parsed, leaves its phase DONE.
1590
+ // When invoked in resumable mode, it returns true after parsing a complete document.
1591
+ // If reaching EOS without having parsed a complete document, either returns false
1592
+ // of raise a JSON::ParserError tagged with `@eos=true`.
1593
+ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable)
1594
+ {
1595
+ json_frame *frame = json_frame_stack_peek(state->frames);
1596
+
1597
+ switch (frame->phase) {
1598
+ case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false);
1599
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1600
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1601
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1602
+ case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY;
1603
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1604
+ }
1605
+ JSON_UNREACHABLE_RETURN(false);
1243
1606
 
1244
- raise_parse_error("unexpected token %s", state);
1245
- break;
1246
- case 'I':
1247
- if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
1248
- state->cursor += 8;
1249
- return json_push_value(state, config, CInfinity);
1250
- }
1607
+ JSON_PHASE_VALUE: {
1608
+ json_eat_whitespace(state, config, true);
1251
1609
 
1252
- raise_parse_error("unexpected token %s", state);
1253
- break;
1254
- case '-': {
1255
- // Note: memcmp with a small power of two compile to an integer comparison
1256
- if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) {
1257
- if (config->allow_nan) {
1258
- state->cursor += 9;
1259
- return json_push_value(state, config, CMinusInfinity);
1260
- } else {
1261
- raise_parse_error("unexpected token %s", state);
1262
- }
1263
- }
1264
- return json_push_value(state, config, json_parse_negative_number(state, config));
1265
- break;
1266
- }
1267
- case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
1268
- return json_push_value(state, config, json_parse_positive_number(state, config));
1269
- break;
1270
- case '"': {
1271
- // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
1272
- return json_parse_string(state, config, false);
1273
- break;
1610
+ // A trailing comma lands us here expecting an element but finding the
1611
+ // closing bracket; hand off to ARRAY_COMMA to close. An empty array
1612
+ // closes inline at '[', so this position is only reached after a ','.
1613
+ if (config->allow_trailing_comma && frame->type == JSON_FRAME_ARRAY && peek(state) == ']') {
1614
+ goto JSON_PHASE_ARRAY_COMMA;
1274
1615
  }
1275
- case '[': {
1276
- state->cursor++;
1277
- json_eat_whitespace(state);
1278
- long stack_head = state->stack->head;
1279
1616
 
1280
- if (peek(state) == ']') {
1281
- state->cursor++;
1282
- return json_push_value(state, config, json_decode_array(state, config, 0));
1283
- } else {
1284
- state->current_nesting++;
1285
- if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1286
- rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1287
- }
1288
- state->in_array++;
1289
- json_parse_any(state, config);
1290
- }
1617
+ VALUE value;
1618
+ const char *value_start = state->cursor;
1291
1619
 
1292
- while (true) {
1293
- json_eat_whitespace(state);
1620
+ switch (peek(state)) {
1621
+ case 'n':
1622
+ json_match_keyword(state, "null", 0);
1623
+ value = Qnil;
1624
+ break;
1294
1625
 
1295
- const char next_char = peek(state);
1626
+ case 't':
1627
+ json_match_keyword(state, "true", 0);
1628
+ value = Qtrue;
1629
+ break;
1296
1630
 
1297
- if (RB_LIKELY(next_char == ',')) {
1298
- state->cursor++;
1299
- if (config->allow_trailing_comma) {
1300
- json_eat_whitespace(state);
1301
- if (peek(state) == ']') {
1302
- continue;
1303
- }
1304
- }
1305
- json_parse_any(state, config);
1306
- continue;
1631
+ case 'f':
1632
+ json_match_keyword(state, "false", 1);
1633
+ value = Qfalse;
1634
+ break;
1635
+
1636
+ case 'N':
1637
+ if (!config->allow_nan) {
1638
+ raise_syntax_error("unexpected token %s", state);
1307
1639
  }
1308
1640
 
1309
- if (next_char == ']') {
1310
- state->cursor++;
1311
- long count = state->stack->head - stack_head;
1312
- state->current_nesting--;
1313
- state->in_array--;
1314
- return json_push_value(state, config, json_decode_array(state, config, count));
1641
+ json_match_keyword(state, "NaN", 1);
1642
+ value = CNaN;
1643
+ break;
1644
+
1645
+ case 'I':
1646
+ if (!config->allow_nan) {
1647
+ raise_syntax_error("unexpected token %s", state);
1648
+ }
1649
+
1650
+ json_match_keyword(state, "Infinity", 0);
1651
+ value = CInfinity;
1652
+ break;
1653
+
1654
+ case '-': {
1655
+ state->cursor++;
1656
+
1657
+ value = json_parse_number(state, config, true, value_start, resumable);
1658
+
1659
+ if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
1660
+ state->cursor = value_start;
1661
+ json_match_keyword(state, "-Infinity", 1);
1662
+ value = CMinusInfinity;
1663
+ break;
1315
1664
  }
1316
1665
 
1317
- raise_parse_error("expected ',' or ']' after array value", state);
1666
+ // Top level numbers are ambiguous when parsing streams, we can't
1667
+ // know if we parsed all the digits if we hit EOS.
1668
+ if (RB_UNLIKELY(resumable && eos(state))) {
1669
+ state->cursor = value_start;
1670
+ return false;
1671
+ }
1672
+
1673
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1674
+ raise_syntax_error_at("invalid number: %s", state, value_start);
1675
+ }
1676
+ break;
1318
1677
  }
1319
- break;
1320
- }
1321
- case '{': {
1322
- const char *object_start_cursor = state->cursor;
1323
1678
 
1324
- state->cursor++;
1325
- json_eat_whitespace(state);
1326
- long stack_head = state->stack->head;
1679
+ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
1680
+ value = json_parse_number(state, config, false, value_start, resumable);
1681
+
1682
+ // Top level numbers are ambiguous when parsing streams, we can't
1683
+ // know if we parsed all the digits if we hit EOS.
1684
+ if (RB_UNLIKELY(resumable && eos(state))) {
1685
+ state->cursor = value_start;
1686
+ return false;
1687
+ }
1688
+
1689
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1690
+ raise_syntax_error_at("invalid number: %s", state, value_start);
1691
+ }
1692
+ break;
1693
+ }
1694
+
1695
+ case '"': {
1696
+ // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
1697
+ value = json_parse_string(state, config, false);
1698
+
1699
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1700
+ bool is_eos = eos(state);
1701
+ if (resumable && is_eos) {
1702
+ state->cursor = value_start;
1703
+ return false;
1704
+ }
1705
+ raise_parse_error("unexpected end of input, expected closing \"", state, is_eos);
1706
+ }
1707
+ break;
1708
+ }
1327
1709
 
1328
- if (peek(state) == '}') {
1710
+ case '[': {
1329
1711
  state->cursor++;
1330
- return json_push_value(state, config, json_decode_object(state, config, 0));
1331
- } else {
1712
+ // The '[' is consumed but its frame is only pushed below, so a
1713
+ // comment suspending here must resume from the bracket.
1714
+ json_eat_whitespace_resume_at(state, config, true, value_start);
1715
+
1716
+ const char next = peek(state);
1717
+ if (next == ']') {
1718
+ state->cursor++;
1719
+ value = json_decode_array(state, config, 0);
1720
+ break;
1721
+ } else if (resumable && eos(state)) {
1722
+ state->cursor = value_start;
1723
+ return false;
1724
+ }
1725
+
1332
1726
  state->current_nesting++;
1333
1727
  if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1334
1728
  rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1335
1729
  }
1730
+ state->in_array++;
1731
+
1732
+ // Phase stays VALUE: the next iteration reads the first element.
1733
+ frame = json_frame_stack_push(state, (json_frame){
1734
+ .type = JSON_FRAME_ARRAY,
1735
+ .phase = JSON_PHASE_VALUE,
1736
+ .value_stack_head = state->value_stack->head,
1737
+ });
1738
+ goto JSON_PHASE_VALUE;
1739
+ }
1740
+
1741
+ case '{': {
1742
+ state->cursor++;
1743
+ // Same as '[': the frame is only pushed below.
1744
+ json_eat_whitespace_resume_at(state, config, true, value_start);
1336
1745
 
1337
- if (peek(state) != '"') {
1338
- raise_parse_error("expected object key, got %s", state);
1746
+ if (peek(state) == '}') {
1747
+ state->cursor++;
1748
+ value = json_decode_object(state, config, 0);
1749
+ break;
1750
+ } else if (resumable && eos(state)) {
1751
+ state->cursor = value_start;
1752
+ return false;
1339
1753
  }
1340
- json_parse_string(state, config, true);
1341
1754
 
1342
- json_eat_whitespace(state);
1343
- if (peek(state) != ':') {
1344
- raise_parse_error("expected ':' after object key", state);
1755
+ state->current_nesting++;
1756
+ if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1757
+ rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1345
1758
  }
1346
- state->cursor++;
1347
1759
 
1348
- json_parse_any(state, config);
1760
+ // Phase KEY: the next iteration reads the first key.
1761
+ frame = json_frame_stack_push(state, (json_frame){
1762
+ .type = JSON_FRAME_OBJECT,
1763
+ .phase = JSON_PHASE_OBJECT_KEY,
1764
+ .value_stack_head = state->value_stack->head,
1765
+ .start_offset = value_start - state->start,
1766
+ });
1767
+ goto JSON_PHASE_OBJECT_KEY;
1349
1768
  }
1350
1769
 
1351
- while (true) {
1352
- json_eat_whitespace(state);
1770
+ case 0:
1771
+ // peek() returns 0 both at end-of-stream and for a literal NUL byte in the
1772
+ // buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at
1773
+ // EOS is just an invalid character.
1774
+ if (eos(state)) {
1775
+ return false;
1776
+ } else {
1777
+ raise_syntax_error("unexpected NULL byte: %s", state);
1778
+ }
1779
+ default:
1780
+ raise_syntax_error("unexpected character: %s", state);
1781
+ }
1353
1782
 
1354
- const char next_char = peek(state);
1355
- if (next_char == '}') {
1356
- state->cursor++;
1357
- state->current_nesting--;
1358
- size_t count = state->stack->head - stack_head;
1783
+ json_push_value(state, config, value);
1784
+ json_value_completed(frame);
1785
+
1786
+ switch (frame->phase) {
1787
+ case JSON_PHASE_DONE: return true;
1788
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1789
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1790
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1791
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1792
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1793
+ }
1794
+ JSON_UNREACHABLE_RETURN(false);
1795
+ }
1796
+
1797
+ JSON_PHASE_OBJECT_KEY: {
1798
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1359
1799
 
1360
- // Temporary rewind cursor in case an error is raised
1361
- const char *final_cursor = state->cursor;
1362
- state->cursor = object_start_cursor;
1363
- VALUE object = json_decode_object(state, config, count);
1364
- state->cursor = final_cursor;
1800
+ json_eat_whitespace(state, config, true);
1365
1801
 
1366
- return json_push_value(state, config, object);
1802
+ // A trailing comma lands us here expecting a key but finding the closing
1803
+ // brace; hand off to OBJECT_COMMA to close. An empty object closes inline
1804
+ // at '{', so this position is only reached after a ','.
1805
+ if (config->allow_trailing_comma && peek(state) == '}') {
1806
+ goto JSON_PHASE_OBJECT_COMMA;
1807
+ }
1808
+
1809
+ const char *start = state->cursor;
1810
+
1811
+ if (RB_LIKELY(peek(state) == '"')) {
1812
+ VALUE string = json_parse_string(state, config, true);
1813
+ if (UNDEF_P(string)) {
1814
+ if (resumable) {
1815
+ state->cursor = start;
1816
+ return false;
1817
+ } else {
1818
+ raise_syntax_error("unexpected end of input, expected closing \"", state);
1367
1819
  }
1820
+ }
1821
+ json_push_value(state, config, string);
1822
+ frame->phase = JSON_PHASE_OBJECT_COLON;
1823
+ goto JSON_PHASE_OBJECT_COLON;
1824
+ } else if (resumable && eos(state)) {
1825
+ return false;
1826
+ } else {
1827
+ // The message differs for the first key vs. a key after a
1828
+ // ',': the first is the only one reached with nothing pushed
1829
+ // for this object yet.
1830
+ if (json_frame_entry_count(frame, state->value_stack) == 0) {
1831
+ raise_syntax_error("expected object key, got %s", state);
1832
+ } else {
1833
+ raise_syntax_error("expected object key, got: %s", state);
1834
+ }
1835
+ }
1836
+ JSON_UNREACHABLE_RETURN(false);
1837
+ }
1368
1838
 
1369
- if (next_char == ',') {
1370
- state->cursor++;
1371
- json_eat_whitespace(state);
1839
+ JSON_PHASE_OBJECT_COLON: {
1840
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1372
1841
 
1373
- if (config->allow_trailing_comma) {
1374
- if (peek(state) == '}') {
1375
- continue;
1376
- }
1377
- }
1842
+ json_eat_whitespace(state, config, true);
1378
1843
 
1379
- if (RB_UNLIKELY(peek(state) != '"')) {
1380
- raise_parse_error("expected object key, got: %s", state);
1381
- }
1382
- json_parse_string(state, config, true);
1844
+ if (RB_LIKELY(peek(state) == ':')) {
1845
+ state->cursor++;
1846
+ frame->phase = JSON_PHASE_VALUE;
1847
+ goto JSON_PHASE_VALUE;
1848
+ } else if (resumable && eos(state)) {
1849
+ return false;
1850
+ } else {
1851
+ // First colon (only the first pair's key is pushed, nothing
1852
+ // else) vs. a later one.
1853
+ if (json_frame_entry_count(frame, state->value_stack) == 1) {
1854
+ raise_syntax_error("expected ':' after object key", state);
1855
+ } else {
1856
+ raise_syntax_error("expected ':' after object key, got: %s", state);
1857
+ }
1858
+ }
1859
+ JSON_UNREACHABLE_RETURN(false);
1860
+ }
1383
1861
 
1384
- json_eat_whitespace(state);
1385
- if (RB_UNLIKELY(peek(state) != ':')) {
1386
- raise_parse_error("expected ':' after object key, got: %s", state);
1387
- }
1388
- state->cursor++;
1862
+ JSON_PHASE_ARRAY_COMMA: {
1863
+ JSON_ASSERT(frame->type == JSON_FRAME_ARRAY);
1389
1864
 
1390
- json_parse_any(state, config);
1865
+ json_eat_whitespace(state, config, true);
1391
1866
 
1392
- continue;
1393
- }
1867
+ const char next_char = peek(state);
1394
1868
 
1395
- raise_parse_error("expected ',' or '}' after object value, got: %s", state);
1869
+ if (RB_LIKELY(next_char == ',')) {
1870
+ state->cursor++;
1871
+ // Commit the phase before eating the whitespace that follows: an
1872
+ // incomplete comment there would suspend the parse, and a phase not
1873
+ // yet advanced past the ',' would drop it on resume. A trailing comma
1874
+ // is recognized in JSON_PHASE_VALUE once the ']' is in the buffer.
1875
+ frame->phase = JSON_PHASE_VALUE;
1876
+ goto JSON_PHASE_VALUE;
1877
+ } else if (next_char == ']') {
1878
+ state->cursor++;
1879
+ long count = json_frame_entry_count(frame, state->value_stack);
1880
+ state->current_nesting--;
1881
+ state->in_array--;
1882
+
1883
+ json_push_value(state, config, json_decode_array(state, config, count));
1884
+ json_frame_stack_pop(state->frames);
1885
+ frame = json_frame_stack_peek(state->frames);
1886
+
1887
+ json_value_completed(frame);
1888
+
1889
+ switch (frame->phase) {
1890
+ case JSON_PHASE_DONE: return true;
1891
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1892
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1893
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1894
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1895
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1396
1896
  }
1397
- break;
1897
+ } else if (resumable && eos(state)) {
1898
+ return false;
1899
+ } else {
1900
+ raise_syntax_error("expected ',' or ']' after array value", state);
1398
1901
  }
1902
+ JSON_UNREACHABLE_RETURN(false);
1903
+ }
1399
1904
 
1400
- case 0:
1401
- raise_parse_error("unexpected end of input", state);
1402
- break;
1905
+ JSON_PHASE_OBJECT_COMMA: {
1906
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1403
1907
 
1404
- default:
1405
- raise_parse_error("unexpected character: %s", state);
1406
- break;
1908
+ json_eat_whitespace(state, config, true);
1909
+ const char next_char = peek(state);
1910
+
1911
+ if (RB_LIKELY(next_char == ',')) {
1912
+ state->cursor++;
1913
+ // Commit the phase before eating the whitespace that follows: an
1914
+ // incomplete comment there would suspend the parse, and a phase not
1915
+ // yet advanced past the ',' would drop it on resume. A trailing comma
1916
+ // is recognized in JSON_PHASE_OBJECT_KEY once the '}' is in the buffer.
1917
+ frame->phase = JSON_PHASE_OBJECT_KEY;
1918
+ goto JSON_PHASE_OBJECT_KEY;
1919
+ } else if (next_char == '}') {
1920
+ state->cursor++;
1921
+ state->current_nesting--;
1922
+ size_t count = json_frame_entry_count(frame, state->value_stack);
1923
+
1924
+ // Temporary rewind cursor in case an error is raised
1925
+ const char *final_cursor = state->cursor;
1926
+ state->cursor = state->start + frame->start_offset;
1927
+ VALUE object = json_decode_object(state, config, count);
1928
+ state->cursor = final_cursor;
1929
+
1930
+ json_push_value(state, config, object);
1931
+ json_frame_stack_pop(state->frames);
1932
+ frame = json_frame_stack_peek(state->frames);
1933
+ json_value_completed(frame);
1934
+
1935
+ switch (frame->phase) {
1936
+ case JSON_PHASE_DONE: return true;
1937
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1938
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1939
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1940
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1941
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1942
+ }
1943
+ } else if (resumable && eos(state)) {
1944
+ return false;
1945
+ } else {
1946
+ raise_syntax_error("expected ',' or '}' after object value, got: %s", state);
1947
+ }
1948
+ JSON_UNREACHABLE_RETURN(false);
1407
1949
  }
1408
1950
 
1409
- raise_parse_error("unreachable: %s", state);
1410
- return Qundef;
1951
+ JSON_UNREACHABLE_RETURN(false);
1411
1952
  }
1412
1953
 
1413
- static void json_ensure_eof(JSON_ParserState *state)
1954
+ static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config)
1414
1955
  {
1415
- json_eat_whitespace(state);
1956
+ json_eat_whitespace(state, config, true);
1416
1957
  if (!eos(state)) {
1417
- raise_parse_error("unexpected token at end of stream %s", state);
1958
+ raise_syntax_error("unexpected token at end of stream %s", state);
1418
1959
  }
1419
1960
  }
1420
1961
 
@@ -1432,40 +1973,59 @@ static void json_ensure_eof(JSON_ParserState *state)
1432
1973
 
1433
1974
  static VALUE convert_encoding(VALUE source)
1434
1975
  {
1435
- int encindex = RB_ENCODING_GET(source);
1976
+ StringValue(source);
1977
+ int encindex = RB_ENCODING_GET(source);
1978
+
1979
+ if (RB_LIKELY(encindex == utf8_encindex)) {
1980
+ return source;
1981
+ }
1436
1982
 
1437
- if (RB_LIKELY(encindex == utf8_encindex)) {
1983
+ if (encindex == binary_encindex) {
1984
+ // For historical reason, we silently reinterpret binary strings as UTF-8
1985
+ return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
1986
+ }
1987
+
1988
+ source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
1989
+ StringValue(source);
1438
1990
  return source;
1439
- }
1991
+ }
1440
1992
 
1441
- if (encindex == binary_encindex) {
1442
- // For historical reason, we silently reinterpret binary strings as UTF-8
1443
- return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
1444
- }
1993
+ struct parser_config_init_args {
1994
+ JSON_ParserConfig *config;
1995
+ VALUE self;
1996
+ VALUE unknown_keywords;
1997
+ bool strict;
1998
+ };
1445
1999
 
1446
- return rb_funcall(source, i_encode, 1, Encoding_UTF_8);
2000
+ static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
2001
+ {
2002
+ *dest = val;
2003
+ if (self) RB_OBJ_WRITTEN(self, Qundef, val);
1447
2004
  }
1448
2005
 
1449
2006
  static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1450
2007
  {
1451
- JSON_ParserConfig *config = (JSON_ParserConfig *)data;
2008
+ struct parser_config_init_args *args = (struct parser_config_init_args *)data;
2009
+ JSON_ParserConfig *config = args->config;
2010
+ VALUE self = args->self;
1452
2011
 
1453
2012
  if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
1454
2013
  else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
1455
2014
  else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
2015
+ else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
1456
2016
  else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
1457
2017
  else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
1458
2018
  else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
1459
2019
  else if (key == sym_freeze) { config->freeze = RTEST(val); }
1460
- else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; }
2020
+ else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
1461
2021
  else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
1462
2022
  else if (key == sym_decimal_class) {
1463
2023
  if (RTEST(val)) {
1464
2024
  if (rb_respond_to(val, i_try_convert)) {
1465
- config->decimal_class = val;
2025
+ parser_config_wb_write(self, &config->decimal_class, val);
1466
2026
  config->decimal_method_id = i_try_convert;
1467
2027
  } else if (rb_respond_to(val, i_new)) {
1468
- config->decimal_class = val;
2028
+ parser_config_wb_write(self, &config->decimal_class, val);
1469
2029
  config->decimal_method_id = i_new;
1470
2030
  } else if (RB_TYPE_P(val, T_CLASS)) {
1471
2031
  VALUE name = rb_class_name(val);
@@ -1474,7 +2034,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1474
2034
  if (last_colon) {
1475
2035
  const char *mod_path_end = last_colon - 1;
1476
2036
  VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
1477
- config->decimal_class = rb_path_to_class(mod_path);
2037
+ parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
1478
2038
 
1479
2039
  const char *method_name_beg = last_colon + 1;
1480
2040
  long before_len = method_name_beg - name_cstr;
@@ -1482,28 +2042,48 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1482
2042
  VALUE method_name = rb_str_substr(name, before_len, len);
1483
2043
  config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
1484
2044
  } else {
1485
- config->decimal_class = rb_mKernel;
2045
+ parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
1486
2046
  config->decimal_method_id = SYM2ID(rb_str_intern(name));
1487
2047
  }
1488
2048
  }
1489
2049
  }
1490
2050
  }
2051
+ else if (args->strict) {
2052
+ if (!args->unknown_keywords) {
2053
+ args->unknown_keywords = rb_obj_hide(rb_ary_new());
2054
+ }
2055
+ rb_ary_push(args->unknown_keywords, key);
2056
+ }
1491
2057
 
1492
2058
  return ST_CONTINUE;
1493
2059
  }
1494
2060
 
1495
- static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
2061
+ static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict)
1496
2062
  {
1497
2063
  config->max_nesting = 100;
1498
2064
 
1499
- if (!NIL_P(opts)) {
1500
- Check_Type(opts, T_HASH);
1501
- if (RHASH_SIZE(opts) > 0) {
1502
- // We assume in most cases few keys are set so it's faster to go over
1503
- // the provided keys than to check all possible keys.
1504
- rb_hash_foreach(opts, parser_config_init_i, (VALUE)config);
1505
- }
2065
+ struct parser_config_init_args args = {
2066
+ .config = config,
2067
+ .self = self,
2068
+ .strict = strict,
2069
+ };
2070
+
2071
+ if (NIL_P(opts)) return;
2072
+ Check_Type(opts, T_HASH);
2073
+ if (RHASH_SIZE(opts) == 0) return;
1506
2074
 
2075
+ // We assume in most cases few keys are set so it's faster to go over
2076
+ // the provided keys than to check all possible keys.
2077
+ rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
2078
+
2079
+ if (RB_UNLIKELY(args.unknown_keywords)) {
2080
+ if (RARRAY_LEN(args.unknown_keywords) == 1) {
2081
+ rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0));
2082
+ }
2083
+ else {
2084
+ VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", "));
2085
+ rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords);
2086
+ }
1507
2087
  }
1508
2088
  }
1509
2089
 
@@ -1512,67 +2092,90 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
1512
2092
  *
1513
2093
  * Creates a new JSON::Ext::ParserConfig instance.
1514
2094
  *
1515
- * It will be configured by the _opts_ hash. _opts_ can have the following
1516
- * keys:
2095
+ * Argument +opts+, if given, contains a \Hash of options for the parsing.
2096
+ * See {Parsing Options}[#module-JSON-label-Parsing+Options].
1517
2097
  *
1518
- * _opts_ can have the following keys:
1519
- * * *max_nesting*: The maximum depth of nesting allowed in the parsed data
1520
- * structures. Disable depth checking with :max_nesting => false|nil|0, it
1521
- * defaults to 100.
1522
- * * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
1523
- * defiance of RFC 4627 to be parsed by the Parser. This option defaults to
1524
- * false.
1525
- * * *symbolize_names*: If set to true, returns symbols for the names
1526
- * (keys) in a JSON object. Otherwise strings are returned, which is
1527
- * also the default. It's not possible to use this option in
1528
- * conjunction with the *create_additions* option.
1529
- * * *decimal_class*: Specifies which class to use instead of the default
1530
- * (Float) when parsing decimal numbers. This class must accept a single
1531
- * string argument in its constructor.
1532
2098
  */
1533
2099
  static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
1534
2100
  {
1535
2101
  rb_check_frozen(self);
1536
2102
  GET_PARSER_CONFIG;
1537
2103
 
1538
- parser_config_init(config, opts);
1539
-
1540
- RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
2104
+ parser_config_init(config, opts, self, false);
1541
2105
 
1542
2106
  return self;
1543
2107
  }
1544
2108
 
1545
- static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource)
2109
+ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
1546
2110
  {
1547
- Vsource = convert_encoding(StringValue(Vsource));
1548
- StringValue(Vsource);
2111
+ VALUE Vsource = convert_encoding(src);
2112
+
2113
+ // Ensure the string isn't mutated under us.
2114
+ // The classic API to use is `rb_str_locktmp`, but then we'd
2115
+ // need to use `rb_protect` to make sure we always unlock.
2116
+ if (Vsource == src) {
2117
+ Vsource = rb_str_new_frozen(Vsource);
2118
+ }
1549
2119
 
1550
2120
  VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
1551
- rvalue_stack stack = {
2121
+ rvalue_stack value_stack = {
1552
2122
  .type = RVALUE_STACK_STACK_ALLOCATED,
1553
2123
  .ptr = rvalue_stack_buffer,
1554
2124
  .capa = RVALUE_STACK_INITIAL_CAPA,
1555
2125
  };
1556
2126
 
2127
+ // Seed the frame stack with the root frame, establishing the invariant that
2128
+ // json_parse_any always has a top frame to dispatch on (so the stack is never
2129
+ // empty mid-parse).
2130
+ json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA];
2131
+ frame_stack_buffer[0] = (json_frame){
2132
+ .type = JSON_FRAME_ROOT,
2133
+ .phase = JSON_PHASE_VALUE,
2134
+ };
2135
+ json_frame_stack frames = {
2136
+ .type = RVALUE_STACK_STACK_ALLOCATED,
2137
+ .ptr = frame_stack_buffer,
2138
+ .capa = JSON_FRAME_STACK_INITIAL_CAPA,
2139
+ .head = 1,
2140
+ };
2141
+
1557
2142
  long len;
1558
2143
  const char *start;
2144
+
1559
2145
  RSTRING_GETMEM(Vsource, start, len);
1560
2146
 
2147
+ VALUE value_stack_handle = 0;
2148
+ VALUE frame_stack_handle = 0;
1561
2149
  JSON_ParserState _state = {
1562
2150
  .start = start,
1563
2151
  .cursor = start,
1564
2152
  .end = start + len,
1565
- .stack = &stack,
2153
+ .value_stack = &value_stack,
2154
+ .value_stack_handle = &value_stack_handle,
2155
+ .frames = &frames,
2156
+ .frame_stack_handle = &frame_stack_handle,
1566
2157
  };
1567
2158
  JSON_ParserState *state = &_state;
1568
2159
 
1569
- VALUE result = json_parse_any(state, config);
2160
+ bool complete = json_parse_any(state, config, false);
2161
+
2162
+ // The root document value is parsed; it is the lone survivor on
2163
+ // the rvalue stack.
2164
+ VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
1570
2165
 
1571
2166
  // This may be skipped in case of exception, but
1572
2167
  // it won't cause a leak.
1573
- rvalue_stack_eagerly_release(state->stack_handle);
1574
-
1575
- json_ensure_eof(state);
2168
+ rvalue_stack_eagerly_release(value_stack_handle);
2169
+ json_frame_stack_eagerly_release(frame_stack_handle);
2170
+ RB_GC_GUARD(value_stack_handle);
2171
+ RB_GC_GUARD(frame_stack_handle);
2172
+ RB_GC_GUARD(Vsource);
2173
+
2174
+ if (complete) {
2175
+ json_ensure_eof(state, config);
2176
+ } else {
2177
+ raise_eos_error("unexpected end of input", state);
2178
+ }
1576
2179
 
1577
2180
  return result;
1578
2181
  }
@@ -1592,12 +2195,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
1592
2195
 
1593
2196
  static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
1594
2197
  {
1595
- Vsource = convert_encoding(StringValue(Vsource));
1596
- StringValue(Vsource);
1597
-
1598
2198
  JSON_ParserConfig _config = {0};
1599
2199
  JSON_ParserConfig *config = &_config;
1600
- parser_config_init(config, opts);
2200
+ parser_config_init(config, opts, Qfalse, false);
1601
2201
 
1602
2202
  return cParser_parse(config, Vsource);
1603
2203
  }
@@ -1605,30 +2205,35 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
1605
2205
  static void JSON_ParserConfig_mark(void *ptr)
1606
2206
  {
1607
2207
  JSON_ParserConfig *config = ptr;
1608
- rb_gc_mark(config->on_load_proc);
1609
- rb_gc_mark(config->decimal_class);
2208
+ rb_gc_mark_movable(config->on_load_proc);
2209
+ rb_gc_mark_movable(config->decimal_class);
1610
2210
  }
1611
2211
 
1612
- static void JSON_ParserConfig_free(void *ptr)
2212
+ static size_t JSON_ParserConfig_memsize(const void *ptr)
1613
2213
  {
1614
- JSON_ParserConfig *config = ptr;
1615
- ruby_xfree(config);
2214
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
2215
+ return 0;
2216
+ #else
2217
+ return sizeof(JSON_ParserConfig);
2218
+ #endif
1616
2219
  }
1617
2220
 
1618
- static size_t JSON_ParserConfig_memsize(const void *ptr)
2221
+ static void JSON_ParserConfig_compact(void *ptr)
1619
2222
  {
1620
- return sizeof(JSON_ParserConfig);
2223
+ JSON_ParserConfig *config = ptr;
2224
+ config->on_load_proc = rb_gc_location(config->on_load_proc);
2225
+ config->decimal_class = rb_gc_location(config->decimal_class);
1621
2226
  }
1622
2227
 
1623
2228
  static const rb_data_type_t JSON_ParserConfig_type = {
1624
- "JSON::Ext::Parser/ParserConfig",
1625
- {
1626
- JSON_ParserConfig_mark,
1627
- JSON_ParserConfig_free,
1628
- JSON_ParserConfig_memsize,
2229
+ .wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
2230
+ .function = {
2231
+ .dmark = JSON_ParserConfig_mark,
2232
+ .dfree = RUBY_DEFAULT_FREE,
2233
+ .dsize = JSON_ParserConfig_memsize,
2234
+ .dcompact = JSON_ParserConfig_compact,
1629
2235
  },
1630
- 0, 0,
1631
- RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE,
2236
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
1632
2237
  };
1633
2238
 
1634
2239
  static VALUE cJSON_parser_s_allocate(VALUE klass)
@@ -1637,6 +2242,603 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
1637
2242
  return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config);
1638
2243
  }
1639
2244
 
2245
+ static void json_str_clear(VALUE str)
2246
+ {
2247
+ if (RB_OBJ_FROZEN_RAW(str)) {
2248
+ return;
2249
+ }
2250
+ rb_str_replace(str, JSON_empty_string);
2251
+ }
2252
+
2253
+ typedef struct JSON_ResumableParserStruct {
2254
+ JSON_ParserConfig config;
2255
+ JSON_ParserState state;
2256
+ rvalue_stack value_stack;
2257
+ json_frame_stack frames;
2258
+ VALUE buffer;
2259
+ size_t parsed_bytes;
2260
+ size_t incomplete_bytes;
2261
+ bool complete;
2262
+ bool in_use;
2263
+ } JSON_ResumableParser;
2264
+
2265
+ static void JSON_ResumableParser_mark(void *ptr)
2266
+ {
2267
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2268
+ JSON_ParserConfig_mark(&parser->config);
2269
+ rvalue_stack_mark(&parser->value_stack);
2270
+ rvalue_cache_mark(&parser->state.name_cache);
2271
+ rb_gc_mark(parser->buffer); // pin the buffer
2272
+ rb_gc_mark_movable(parser->state.parser);
2273
+ }
2274
+
2275
+ static void JSON_ResumableParser_free(void *ptr)
2276
+ {
2277
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2278
+ rvalue_stack_free_buffer(&parser->value_stack);
2279
+ json_frame_stack_free_buffer(&parser->frames);
2280
+ }
2281
+
2282
+ static size_t JSON_ResumableParser_memsize(const void *ptr)
2283
+ {
2284
+ const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr;
2285
+ size_t memsize = JSON_ParserConfig_memsize(&parser->config);
2286
+ memsize += rvalue_stack_memsize(&parser->value_stack);
2287
+ memsize += json_frame_stack_memsize(&parser->frames);
2288
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
2289
+ memsize += (
2290
+ sizeof(JSON_ResumableParser)
2291
+ - sizeof(JSON_ParserState)
2292
+ - sizeof(JSON_ParserConfig)
2293
+ - sizeof(rvalue_stack)
2294
+ - sizeof(json_frame_stack)
2295
+ );
2296
+ #endif
2297
+ return memsize;
2298
+ }
2299
+
2300
+ static void JSON_ResumableParser_compact(void *ptr)
2301
+ {
2302
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2303
+ JSON_ParserConfig_compact(&parser->config);
2304
+ rvalue_stack_compact(&parser->value_stack);
2305
+ rvalue_cache_compact(&parser->state.name_cache);
2306
+ parser->buffer = rb_gc_location(parser->buffer);
2307
+ parser->state.parser = rb_gc_location(parser->state.parser);
2308
+ }
2309
+
2310
+ static const rb_data_type_t JSON_ResumableParser_type = {
2311
+ .wrap_struct_name = "JSON::Ext::ResumableParser",
2312
+ .function = {
2313
+ JSON_ResumableParser_mark,
2314
+ JSON_ResumableParser_free,
2315
+ JSON_ResumableParser_memsize,
2316
+ JSON_ResumableParser_compact,
2317
+ },
2318
+ // RUBY_TYPED_WB_PROTECTED is deliberately not declared because
2319
+ // this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
2320
+ // to trigger a lot of write barriers.
2321
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
2322
+ };
2323
+
2324
+ static VALUE cResumableParser_allocate(VALUE klass)
2325
+ {
2326
+ JSON_ResumableParser *parser;
2327
+ VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
2328
+ parser->state.in_array++;
2329
+ parser->state.parser = obj;
2330
+ return obj;
2331
+ }
2332
+
2333
+ static inline JSON_ResumableParser *cResumableParser_get(VALUE self)
2334
+ {
2335
+ JSON_ResumableParser *parser;
2336
+ TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
2337
+ return parser;
2338
+ }
2339
+
2340
+ /*
2341
+ * call-seq: new(opts => {})
2342
+ *
2343
+ * Creates a new JSON::ResumableParser instance.
2344
+ *
2345
+ * Argument +opts+, if given, contains a \Hash of options for the parsing.
2346
+ * See {Parsing Options}[#module-JSON-label-Parsing+Options].
2347
+ *
2348
+ * A ResumableParser is able to parse partial documents and resume parsing later
2349
+ * when more of the document is provided:
2350
+ *
2351
+ * parser = JSON::ResumableParser.new
2352
+ * parser << '{"user": "george", "role": "ad'
2353
+ * parser.parse # => false
2354
+ * parser.eos? # => true
2355
+ * parser.partial_value # => { "user" => "george", "role" => nil }
2356
+ * parser.rest # => '"ad'
2357
+ *
2358
+ * parser << 'min" }[1, 2, 3]'
2359
+ * parser.parse # => true
2360
+ * parser.value # => { "user" => "george", "role" => "admin" }
2361
+ *
2362
+ * parser.parse # => true
2363
+ * parser.value # => [1, 2, 3]
2364
+ *
2365
+ * === Limitations
2366
+ *
2367
+ * While ResumableParser is able to parse streams of documents without any
2368
+ * explicit separators between them, it is highly recommended to separate documents
2369
+ * by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous.
2370
+ * When parsing a number, ResumableParser will not consider the number complete until something follows:
2371
+ *
2372
+ * parser << '123'
2373
+ * parser.parse # => false
2374
+ * parser << ' '
2375
+ * parser.parse # => true
2376
+ * parser.value # => 123
2377
+ *
2378
+ * === Security
2379
+ *
2380
+ * An incomplete document is buffered in full and there is no size limit, so when reading
2381
+ * from an untrusted source the caller is responsible for bounding how much data is fed.
2382
+ * For example:
2383
+ *
2384
+ * loop do
2385
+ * if parser.parsed_bytes > DOCUMENT_MAX_SIZE
2386
+ * raise "document too large"
2387
+ * end
2388
+ *
2389
+ * parser << read_chunk
2390
+ * while parser.parse
2391
+ * process(parser.value)
2392
+ * end
2393
+ * end
2394
+ */
2395
+ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
2396
+ {
2397
+ rb_check_frozen(self);
2398
+
2399
+ VALUE opts = Qfalse;
2400
+ rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts);
2401
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2402
+
2403
+ opts = argc > 0 ? argv[0] : Qnil;
2404
+ parser_config_init(&parser->config, opts, self, true);
2405
+
2406
+ return self;
2407
+ }
2408
+
2409
+ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock);
2410
+
2411
+ /*
2412
+ * call-seq: self << string -> self
2413
+ *
2414
+ * Appends the given string to the parser's buffer.
2415
+ */
2416
+ static VALUE cResumableParser_feed(VALUE self, VALUE str)
2417
+ {
2418
+ rb_check_frozen(self);
2419
+
2420
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2421
+
2422
+ str = convert_encoding(str);
2423
+ if (!RSTRING_LEN(str)) {
2424
+ return self;
2425
+ }
2426
+
2427
+ size_t offset = parser->state.cursor - parser->state.start;
2428
+ const size_t remaining = parser->state.end - parser->state.cursor;
2429
+
2430
+ if (!remaining) {
2431
+ if (parser->buffer) {
2432
+ json_str_clear(parser->buffer);
2433
+ }
2434
+ parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
2435
+ offset = 0;
2436
+ } else {
2437
+ JSON_ASSERT(parser->buffer);
2438
+
2439
+ const size_t size = parser->state.end - parser->state.start;
2440
+ const size_t consumed = size - remaining;
2441
+
2442
+ if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
2443
+ VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
2444
+ rb_enc_associate_index(new_buffer, utf8_encindex);
2445
+
2446
+ char *old_ptr = RSTRING_PTR(parser->buffer);
2447
+ memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
2448
+ rb_str_set_len(new_buffer, remaining);
2449
+ offset = 0;
2450
+ parser->buffer = new_buffer;
2451
+ } else if (consumed > (size / 2) && size >= 512) {
2452
+ rb_str_modify(parser->buffer);
2453
+ char *old_ptr = RSTRING_PTR(parser->buffer);
2454
+ memmove(old_ptr, old_ptr + consumed, remaining);
2455
+ rb_str_set_len(parser->buffer, remaining);
2456
+ offset = 0;
2457
+ }
2458
+ rb_str_append(parser->buffer, str);
2459
+ }
2460
+
2461
+ long len;
2462
+ const char *start;
2463
+ RSTRING_GETMEM(parser->buffer, start, len);
2464
+ parser->state.start = start;
2465
+ parser->state.end = start + len;
2466
+ parser->state.cursor = parser->state.start + offset;
2467
+
2468
+ return self;
2469
+ }
2470
+
2471
+ struct json_parse_any_args {
2472
+ JSON_ParserState *state;
2473
+ JSON_ParserConfig *config;
2474
+ VALUE parser;
2475
+ };
2476
+
2477
+ static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args))
2478
+ {
2479
+ struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
2480
+ return (VALUE)json_parse_any(args->state, args->config, true);
2481
+ }
2482
+
2483
+ static VALUE json_parse_any_resumable_safe(VALUE _args)
2484
+ {
2485
+ struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
2486
+ VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args);
2487
+ return result == args->parser ? Qfalse : result;
2488
+ }
2489
+
2490
+ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock)
2491
+ {
2492
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2493
+
2494
+ if (parser->in_use) {
2495
+ rb_raise(rb_eArgError, "ResumableParser can't be used recursively");
2496
+ }
2497
+
2498
+ if (lock) {
2499
+ parser->in_use = true;
2500
+ }
2501
+
2502
+ // self may have moved, so we need to update all pointers
2503
+ // Investigate: We might be better off keeping JSON_ParserState on the stack
2504
+ // and only persist what we need.
2505
+ parser->state.value_stack = &parser->value_stack;
2506
+ parser->state.frames = &parser->frames;
2507
+
2508
+ return parser;
2509
+ }
2510
+
2511
+ /*
2512
+ * call-seq: parse -> true or false
2513
+ *
2514
+ * Attemps to parse a JSON document from the internal buffer.
2515
+ * Returns whether a complete document could be parsed.
2516
+ *
2517
+ * It does raise +JSON::ParserError+ when encountering invalid \JSON syntax.
2518
+ *
2519
+ * The parsed object can be retrieved by calling #value
2520
+ */
2521
+ static VALUE cResumableParser_parse(VALUE self)
2522
+ {
2523
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
2524
+
2525
+ if (parser->complete) {
2526
+ parser->parsed_bytes = 0;
2527
+ parser->incomplete_bytes = 0;
2528
+ parser->complete = false;
2529
+ }
2530
+
2531
+ if (!parser->buffer) {
2532
+ parser->in_use = false;
2533
+ return Qfalse;
2534
+ }
2535
+
2536
+ if (parser->frames.head == 0) {
2537
+ json_frame_stack_push(&parser->state, (json_frame){
2538
+ .type = JSON_FRAME_ROOT,
2539
+ .phase = JSON_PHASE_VALUE,
2540
+ });
2541
+ }
2542
+
2543
+ VALUE Vsource = parser->buffer; // Prevent compaction
2544
+
2545
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2546
+
2547
+ if (frame->phase == JSON_PHASE_DONE) {
2548
+ JSON_ASSERT(parser->value_stack.head == 1);
2549
+ JSON_ASSERT(parser->frames.head == 1);
2550
+
2551
+ frame->phase = JSON_PHASE_VALUE;
2552
+ rvalue_stack_pop(parser->state.value_stack, 1);
2553
+ }
2554
+
2555
+ struct json_parse_any_args args = {
2556
+ .state = &parser->state,
2557
+ .config = &parser->config,
2558
+ .parser = self,
2559
+ };
2560
+ int status;
2561
+ const char *initial_cursor = parser->state.cursor;
2562
+ parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
2563
+
2564
+ if (status) {
2565
+ parser->complete = true; // a parse error is considered complete
2566
+ }
2567
+
2568
+ parser->parsed_bytes += parser->state.cursor - initial_cursor;
2569
+ parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;
2570
+
2571
+ json_eat_whitespace(&parser->state, &parser->config, false);
2572
+ if (eos(&parser->state)) {
2573
+ json_str_clear(parser->buffer);
2574
+ parser->buffer = Qfalse;
2575
+ parser->state.start = parser->state.cursor = parser->state.end = 0;
2576
+ }
2577
+ parser->in_use = false;
2578
+
2579
+ if (status) {
2580
+ rb_jump_tag(status); // reraise
2581
+ }
2582
+ RB_GC_GUARD(Vsource);
2583
+ return parser->complete ? Qtrue : Qfalse;
2584
+ }
2585
+
2586
+ /*
2587
+ * call-seq: value? -> true or false
2588
+ *
2589
+ * Returns whether a parsed value is available.
2590
+ */
2591
+ static VALUE cResumableParser_value_p(VALUE self)
2592
+ {
2593
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2594
+
2595
+ if (parser->value_stack.head > 0) {
2596
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2597
+ if (frame->phase == JSON_PHASE_DONE) {
2598
+ return Qtrue;
2599
+ }
2600
+ }
2601
+ return Qfalse;
2602
+ }
2603
+
2604
+ /*
2605
+ * call-seq: value -> object
2606
+ *
2607
+ * Returns and consume the last parsed value.
2608
+ * Raises ArgumentError if there is no parsed value or if it was already retrieved:
2609
+ * parser << '[1][2]'
2610
+ * parser.value # ArgumentError no ready value
2611
+ * parser.parse # => true
2612
+ * parser.value # => [1]
2613
+ * parser.value # ArgumentError no ready value
2614
+ */
2615
+ static VALUE cResumableParser_value(VALUE self)
2616
+ {
2617
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2618
+
2619
+ if (parser->frames.head > 0) {
2620
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2621
+
2622
+ if (frame->phase == JSON_PHASE_DONE) {
2623
+ VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
2624
+ rvalue_stack_pop(parser->state.value_stack, 1);
2625
+ json_frame_stack_pop(parser->state.frames);
2626
+ return result;
2627
+ }
2628
+ }
2629
+ rb_raise(rb_eArgError, "no ready value");
2630
+ }
2631
+
2632
+ /*
2633
+ * call-seq: clear -> self
2634
+ *
2635
+ * Entirely reset the parser state and buffer.
2636
+ */
2637
+ static VALUE cResumableParser_clear(VALUE self)
2638
+ {
2639
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2640
+ parser->buffer = 0;
2641
+ parser->complete = true;
2642
+ parser->parsed_bytes = 0;
2643
+ parser->incomplete_bytes = 0;
2644
+ parser->frames.head = 0;
2645
+ parser->value_stack.head = 0;
2646
+ parser->state.name_cache.length = 0;
2647
+ parser->state.current_nesting = 0;
2648
+ parser->state.in_array = 1;
2649
+ parser->state.emitted_deprecations = 0;
2650
+ parser->state.start = parser->state.cursor = parser->state.end = NULL;
2651
+ return self;
2652
+ }
2653
+
2654
+ static VALUE cResumableParser_partial_value_body(VALUE self)
2655
+ {
2656
+ JSON_ResumableParser *original_parser = cResumableParser_get(self);
2657
+ JSON_ResumableParser parser = *original_parser;
2658
+
2659
+ parser.state.frames = &parser.frames;
2660
+ parser.state.value_stack = &parser.value_stack;
2661
+
2662
+ if (parser.value_stack.head == 0) {
2663
+ return Qnil;
2664
+ }
2665
+
2666
+ json_frame *frame = json_frame_stack_peek(parser.state.frames);
2667
+ long missing_object_value = 0;
2668
+ if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
2669
+ missing_object_value = 1;
2670
+ }
2671
+
2672
+ // Copy the value stack as we need to mutate it. The collapse loop folds each
2673
+ // open container by popping its entries and pushing the single result, so a
2674
+ // parent always reclaims its child's slot; head exceeds its live size by at
2675
+ // most one, either for the missing-value placeholder pushed below or for the
2676
+ // result of folding an empty innermost container. That one spare slot keeps
2677
+ // rvalue_stack_push from growing (reallocating) this ALLOCV buffer.
2678
+ long capa = parser.value_stack.head;
2679
+ parser.value_stack.capa = capa + 1;
2680
+ VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, parser.value_stack.capa);
2681
+ MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, capa);
2682
+ parser.value_stack.ptr = value_stack_buffer;
2683
+
2684
+ JSON_ParserState *state = &parser.state;
2685
+ JSON_ParserConfig *config = &parser.config;
2686
+
2687
+ if (missing_object_value) {
2688
+ rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
2689
+ }
2690
+
2691
+ VALUE partial_result = Qundef;
2692
+
2693
+ while (UNDEF_P(partial_result)) {
2694
+ frame = json_frame_stack_peek(state->frames);
2695
+
2696
+ switch (frame->type) {
2697
+ case JSON_FRAME_ROOT: {
2698
+ partial_result = *rvalue_stack_peek(state->value_stack, 1);
2699
+ break;
2700
+ }
2701
+
2702
+ case JSON_FRAME_ARRAY: {
2703
+ long count = json_frame_entry_count(frame, state->value_stack);
2704
+ json_push_value(state, config, json_decode_array(state, config, count));
2705
+ json_frame_stack_pop(state->frames);
2706
+
2707
+ break;
2708
+ }
2709
+
2710
+ case JSON_FRAME_OBJECT: {
2711
+ long count = json_frame_entry_count(frame, state->value_stack);
2712
+ json_push_value(state, config, json_decode_object(state, config, count));
2713
+ json_frame_stack_pop(state->frames);
2714
+ break;
2715
+ }
2716
+
2717
+ default: {
2718
+ JSON_UNREACHABLE_RETURN(Qundef);
2719
+ break;
2720
+ }
2721
+ }
2722
+ }
2723
+
2724
+ ALLOCV_END(tmpbuf);
2725
+ return partial_result;
2726
+ }
2727
+
2728
+ /*
2729
+ * call-seq: partial_value -> object
2730
+ *
2731
+ * Returns the Ruby objects parsed up to this point:
2732
+ * parser << '[1, [2, 3,'
2733
+ * parser.parse # => false
2734
+ * parser.value # ArgumentError no ready value
2735
+ * parser.partial_value # => [1, [2, 3]]
2736
+ */
2737
+ static VALUE cResumableParser_partial_value(VALUE self)
2738
+ {
2739
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
2740
+
2741
+ int status;
2742
+ VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
2743
+ parser->in_use = false;
2744
+ if (status) {
2745
+ rb_jump_tag(status);
2746
+ }
2747
+ return result;
2748
+ }
2749
+
2750
+ /*
2751
+ * call-seq: rest -> string
2752
+ *
2753
+ * Returns a string containing what remains to be parsed in the buffer
2754
+ * parser << '{ "message": "unterminated message'
2755
+ * parser.parse # => false
2756
+ * parser.rest # => '"unterminated message"'
2757
+ */
2758
+ static VALUE cResumableParser_rest(VALUE self)
2759
+ {
2760
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2761
+
2762
+ if (!parser->buffer) {
2763
+ return rb_utf8_str_new("", 0);
2764
+ }
2765
+
2766
+ size_t offset = parser->state.cursor - parser->state.start;
2767
+ const char *ptr;
2768
+ long len;
2769
+ RSTRING_GETMEM(parser->buffer, ptr, len);
2770
+ return rb_utf8_str_new(ptr + offset, len - offset);
2771
+ }
2772
+
2773
+ /*
2774
+ * call-seq: eos? -> true or false
2775
+ *
2776
+ * Returns whether the internal buffer has been entirely consumed.
2777
+ */
2778
+ static VALUE cResumableParser_eos_p(VALUE self)
2779
+ {
2780
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2781
+ return eos(&parser->state) ? Qtrue : Qfalse;
2782
+ }
2783
+
2784
+ /*
2785
+ * call-seq: partial_value? -> true or false
2786
+ *
2787
+ * Returns whether a document is currently under construction: an unclosed
2788
+ * container, a key awaiting its value, etc.
2789
+ *
2790
+ * It answers the same question as <tt>!partial_value.nil?</tt>, but as a
2791
+ * cheap predicate on the parser's internal state, without materializing the
2792
+ * partially parsed Ruby objects:
2793
+ * parser << '{"a":1,'
2794
+ * parser.parse # => false
2795
+ * parser.partial_value? # => true
2796
+ *
2797
+ * A fully parsed document whose value hasn't been retrieved yet is not under
2798
+ * construction: #value? returns true and #partial_value? returns false.
2799
+ */
2800
+ static VALUE cResumableParser_partial_value_p(VALUE self)
2801
+ {
2802
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2803
+
2804
+ // Mirror of #value?: values on the stack while the document isn't DONE
2805
+ // belong to a partially built document. A container whose first key or
2806
+ // element hasn't been parsed yet has no frame nor value registered (the
2807
+ // tokenizer rewinds to the container start on EOS), so that state is
2808
+ // observable through the buffer (#eos?/#rest) instead, keeping this
2809
+ // predicate consistent with #partial_value returning nil.
2810
+ if (parser->value_stack.head > 0) {
2811
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2812
+ if (frame->phase != JSON_PHASE_DONE) {
2813
+ return Qtrue;
2814
+ }
2815
+ }
2816
+ return Qfalse;
2817
+ }
2818
+
2819
+ /*
2820
+ * call-seq: parsed_bytes -> integer
2821
+ *
2822
+ * Returns the number of bytes parsed since the start of the current partial value.
2823
+ * This is intended to be used for securing against untrusted input:
2824
+ *
2825
+ * loop do
2826
+ * if parser.parsed_bytes > DOCUMENT_MAX_SIZE
2827
+ * raise "document too large"
2828
+ * end
2829
+ *
2830
+ * parser << read_chunk
2831
+ * while parser.parse
2832
+ * process(parser.value)
2833
+ * end
2834
+ * end
2835
+ */
2836
+ static VALUE cResumableParser_parsed_bytes(VALUE self)
2837
+ {
2838
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2839
+ return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
2840
+ }
2841
+
1640
2842
  void Init_parser(void)
1641
2843
  {
1642
2844
  #ifdef HAVE_RB_EXT_RACTOR_SAFE
@@ -1648,30 +2850,53 @@ void Init_parser(void)
1648
2850
  mJSON = rb_define_module("JSON");
1649
2851
  VALUE mExt = rb_define_module_under(mJSON, "Ext");
1650
2852
  VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject);
2853
+
2854
+ rb_global_variable(&eParserError);
2855
+ eParserError = rb_path2class("JSON::ParserError");
2856
+
2857
+ rb_global_variable(&eNestingError);
1651
2858
  eNestingError = rb_path2class("JSON::NestingError");
1652
- rb_gc_register_mark_object(eNestingError);
2859
+
1653
2860
  rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate);
1654
- rb_define_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
2861
+ rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
1655
2862
  rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1);
1656
2863
 
1657
2864
  VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject);
1658
2865
  rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2);
1659
2866
 
2867
+ VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject);
2868
+ rb_define_alloc_func(cResumableParser, cResumableParser_allocate);
2869
+ rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1);
2870
+ rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1);
2871
+ rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0);
2872
+ rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
2873
+ rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
2874
+ rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
2875
+ rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0);
2876
+ rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
2877
+ rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
2878
+ rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
2879
+ rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0);
2880
+
2881
+ rb_global_variable(&CNaN);
1660
2882
  CNaN = rb_const_get(mJSON, rb_intern("NaN"));
1661
- rb_gc_register_mark_object(CNaN);
1662
2883
 
2884
+ rb_global_variable(&CInfinity);
1663
2885
  CInfinity = rb_const_get(mJSON, rb_intern("Infinity"));
1664
- rb_gc_register_mark_object(CInfinity);
1665
2886
 
2887
+ rb_global_variable(&CMinusInfinity);
1666
2888
  CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity"));
1667
- rb_gc_register_mark_object(CMinusInfinity);
1668
2889
 
1669
2890
  rb_global_variable(&Encoding_UTF_8);
1670
2891
  Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
1671
2892
 
2893
+ rb_global_variable(&JSON_empty_string);
2894
+ JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0));
2895
+
1672
2896
  sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
1673
2897
  sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
1674
2898
  sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
2899
+ sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
1675
2900
  sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
1676
2901
  sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
1677
2902
  sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
@@ -1682,8 +2907,12 @@ void Init_parser(void)
1682
2907
 
1683
2908
  i_new = rb_intern("new");
1684
2909
  i_try_convert = rb_intern("try_convert");
2910
+ #ifndef HAVE_RB_STR_TO_INTERNED_STR
1685
2911
  i_uminus = rb_intern("-@");
2912
+ #endif
1686
2913
  i_encode = rb_intern("encode");
2914
+ i_at_line = rb_intern("@line");
2915
+ i_at_column = rb_intern("@column");
1687
2916
 
1688
2917
  binary_encindex = rb_ascii8bit_encindex();
1689
2918
  utf8_encindex = rb_utf8_encindex();