json 2.19.3 → 2.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,18 @@
1
1
  #include "../json.h"
2
- #include "../vendor/ryu.h"
2
+ #include "../vendor/fast_float_parser.h"
3
3
  #include "../simd/simd.h"
4
4
 
5
- static VALUE mJSON, eNestingError, Encoding_UTF_8;
6
- static VALUE CNaN, CInfinity, CMinusInfinity;
5
+ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8;
6
+ static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string;
7
7
 
8
- static ID i_new, i_try_convert, i_uminus, i_encode;
8
+ static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column;
9
+ #ifndef HAVE_RB_STR_TO_INTERNED_STR
10
+ static ID i_uminus;
11
+ #endif
9
12
 
10
- static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters,
11
- sym_allow_invalid_escape, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load,
12
- sym_allow_duplicate_key;
13
+ static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments,
14
+ sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names,
15
+ sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
13
16
 
14
17
  static int binary_encindex;
15
18
  static int utf8_encindex;
@@ -58,6 +61,20 @@ typedef struct rvalue_cache_struct {
58
61
  VALUE entries[JSON_RVALUE_CACHE_CAPA];
59
62
  } rvalue_cache;
60
63
 
64
+ static void rvalue_cache_mark(rvalue_cache *cache)
65
+ {
66
+ for (int index = 0; index < cache->length; index++) {
67
+ rb_gc_mark_movable(cache->entries[index]);
68
+ }
69
+ }
70
+
71
+ static void rvalue_cache_compact(rvalue_cache *cache)
72
+ {
73
+ for (int index = 0; index < cache->length; index++) {
74
+ cache->entries[index] = rb_gc_location(cache->entries[index]);
75
+ }
76
+ }
77
+
61
78
  static rb_encoding *enc_utf8;
62
79
 
63
80
  #define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55
@@ -206,12 +223,12 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle,
206
223
 
207
224
  static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref)
208
225
  {
209
- long required = stack->capa * 2;
226
+ long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA;
210
227
 
211
228
  if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
212
229
  stack = rvalue_stack_spill(stack, handle, stack_ref);
213
230
  } else {
214
- REALLOC_N(stack->ptr, VALUE, required);
231
+ JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa);
215
232
  stack->capa = required;
216
233
  }
217
234
  return stack;
@@ -219,11 +236,15 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu
219
236
 
220
237
  static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref)
221
238
  {
239
+ JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle);
240
+
222
241
  if (RB_UNLIKELY(stack->head >= stack->capa)) {
223
242
  stack = rvalue_stack_grow(stack, handle, stack_ref);
224
243
  }
244
+
225
245
  stack->ptr[stack->head] = value;
226
246
  stack->head++;
247
+
227
248
  return value;
228
249
  }
229
250
 
@@ -241,35 +262,62 @@ static void rvalue_stack_mark(void *ptr)
241
262
  {
242
263
  rvalue_stack *stack = (rvalue_stack *)ptr;
243
264
  long index;
244
- for (index = 0; index < stack->head; index++) {
245
- rb_gc_mark(stack->ptr[index]);
265
+ if (stack && stack->ptr) {
266
+ for (index = 0; index < stack->head; index++) {
267
+ rb_gc_mark_movable(stack->ptr[index]);
268
+ }
246
269
  }
247
270
  }
248
271
 
272
+ static void rvalue_stack_free_buffer(rvalue_stack *stack)
273
+ {
274
+ JSON_SIZED_FREE_N(stack->ptr, stack->capa);
275
+ stack->ptr = NULL;
276
+ }
277
+
249
278
  static void rvalue_stack_free(void *ptr)
250
279
  {
251
280
  rvalue_stack *stack = (rvalue_stack *)ptr;
252
281
  if (stack) {
253
- ruby_xfree(stack->ptr);
254
- ruby_xfree(stack);
282
+ rvalue_stack_free_buffer(stack);
283
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
284
+ JSON_SIZED_FREE(stack);
285
+ #endif
255
286
  }
256
287
  }
257
288
 
258
289
  static size_t rvalue_stack_memsize(const void *ptr)
259
290
  {
260
291
  const rvalue_stack *stack = (const rvalue_stack *)ptr;
261
- return sizeof(rvalue_stack) + sizeof(VALUE) * stack->capa;
292
+ size_t memsize = sizeof(VALUE) * stack->capa;
293
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
294
+ memsize += sizeof(rvalue_stack);
295
+ #endif
296
+ return memsize;
297
+ }
298
+
299
+ static void rvalue_stack_compact(void *ptr)
300
+ {
301
+ rvalue_stack *stack = (rvalue_stack *)ptr;
302
+ long index;
303
+ if (stack && stack->ptr) {
304
+ for (index = 0; index < stack->head; index++) {
305
+ stack->ptr[index] = rb_gc_location(stack->ptr[index]);
306
+ }
307
+ }
262
308
  }
263
309
 
264
310
  static const rb_data_type_t JSON_Parser_rvalue_stack_type = {
265
- "JSON::Ext::Parser/rvalue_stack",
266
- {
311
+ .wrap_struct_name = "JSON::Ext::Parser/rvalue_stack",
312
+ .function = {
267
313
  .dmark = rvalue_stack_mark,
268
314
  .dfree = rvalue_stack_free,
269
315
  .dsize = rvalue_stack_memsize,
316
+ .dcompact = rvalue_stack_compact,
270
317
  },
271
- 0, 0,
272
- RUBY_TYPED_FREE_IMMEDIATELY,
318
+ // We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED
319
+ // because it churns a lot of values so trigering write barriers every time is very costly.
320
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
273
321
  };
274
322
 
275
323
  static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref)
@@ -291,38 +339,71 @@ static void rvalue_stack_eagerly_release(VALUE handle)
291
339
  if (handle) {
292
340
  rvalue_stack *stack;
293
341
  TypedData_Get_Struct(handle, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack);
294
- RTYPEDDATA_DATA(handle) = NULL;
342
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
343
+ rvalue_stack_free_buffer(stack);
344
+ #else
295
345
  rvalue_stack_free(stack);
346
+ RTYPEDDATA_DATA(handle) = NULL;
347
+ #endif
296
348
  }
297
349
  }
298
350
 
299
- static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
300
- {
301
- int len = 1;
302
- if (ch <= 0x7F) {
303
- buf[0] = (char) ch;
304
- } else if (ch <= 0x07FF) {
305
- buf[0] = (char) ((ch >> 6) | 0xC0);
306
- buf[1] = (char) ((ch & 0x3F) | 0x80);
307
- len++;
308
- } else if (ch <= 0xFFFF) {
309
- buf[0] = (char) ((ch >> 12) | 0xE0);
310
- buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
311
- buf[2] = (char) ((ch & 0x3F) | 0x80);
312
- len += 2;
313
- } else if (ch <= 0x1fffff) {
314
- buf[0] =(char) ((ch >> 18) | 0xF0);
315
- buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
316
- buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
317
- buf[3] =(char) ((ch & 0x3F) | 0x80);
318
- len += 3;
319
- } else {
320
- buf[0] = '?';
321
- }
322
- return len;
323
- }
351
+ /* frame stack */
352
+
353
+ // Iterative (non-recursive) parsing keeps an explicit stack of the containers
354
+ // currently being built, instead of relying on the C call stack. Each frame
355
+ // only needs enough bookkeeping to close its container: which kind it is, the
356
+ // rvalue_stack position where its children start (so we know how many to pop),
357
+ // and the cursor at its opening brace (used to rewind for duplicate key
358
+ // errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses
359
+ // the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that
360
+ // it's freed even if parsing raises.
361
+ //
362
+ // The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release
363
+ // and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts
364
+ // -- the element type and the absence of a mark function are the only real
365
+ // differences. Keep the two in sync: a fix to the spill/release or
366
+ // HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the
367
+ // other.
368
+ #define JSON_FRAME_STACK_INITIAL_CAPA 32
369
+
370
+ enum json_frame_type {
371
+ JSON_FRAME_ROOT, // == JSON_PHASE_DONE
372
+ JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA
373
+ JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA
374
+ };
375
+
376
+ // Where a frame is within its container's grammar. This is the entirety of the
377
+ // parser's "what to do next" state: json_parse_any dispatches on the top
378
+ // frame's phase and holds no resume state in C locals, so a parse can stop at
379
+ // any value boundary and be resumed purely from the (persistable) frame stack.
380
+ //
381
+ // The first three phases are deliberately equal to the corresponding json_frame_type
382
+ // to simplify the transition of phase in json_value_completed.
383
+ enum json_frame_phase {
384
+ JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed
385
+ JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']'
386
+ JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}'
387
+ JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':')
388
+ JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',')
389
+ JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':'
390
+ };
391
+
392
+ typedef struct json_frame_struct {
393
+ enum json_frame_type type;
394
+ enum json_frame_phase phase;
395
+ long value_stack_head; // rvalue_stack->head when this container opened
396
+ size_t start_offset; // object frames only (the '{'); NULL otherwise
397
+ } json_frame;
398
+
399
+ typedef struct json_frame_stack_struct {
400
+ enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated
401
+ long capa;
402
+ long head;
403
+ json_frame *ptr;
404
+ } json_frame_stack;
324
405
 
325
- enum duplicate_key_action {
406
+ enum deprecatable_action {
326
407
  JSON_DEPRECATED = 0,
327
408
  JSON_IGNORE,
328
409
  JSON_RAISE,
@@ -332,7 +413,8 @@ typedef struct JSON_ParserStruct {
332
413
  VALUE on_load_proc;
333
414
  VALUE decimal_class;
334
415
  ID decimal_method_id;
335
- enum duplicate_key_action on_duplicate_key;
416
+ enum deprecatable_action on_duplicate_key;
417
+ enum deprecatable_action on_comment;
336
418
  int max_nesting;
337
419
  bool allow_nan;
338
420
  bool allow_trailing_comma;
@@ -343,16 +425,152 @@ typedef struct JSON_ParserStruct {
343
425
  } JSON_ParserConfig;
344
426
 
345
427
  typedef struct JSON_ParserStateStruct {
346
- VALUE stack_handle;
428
+ VALUE *value_stack_handle;
429
+ VALUE *frame_stack_handle;
347
430
  const char *start;
348
431
  const char *cursor;
349
432
  const char *end;
350
- rvalue_stack *stack;
433
+ rvalue_stack *value_stack;
434
+ json_frame_stack *frames;
351
435
  rvalue_cache name_cache;
352
436
  int in_array;
353
437
  int current_nesting;
438
+ unsigned int emitted_deprecations;
439
+ VALUE parser;
354
440
  } JSON_ParserState;
355
441
 
442
+ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref);
443
+
444
+ static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref)
445
+ {
446
+ long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA;
447
+
448
+ if (stack->type == RVALUE_STACK_STACK_ALLOCATED) {
449
+ stack = json_frame_stack_spill(stack, handle, stack_ref);
450
+ } else {
451
+ JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa);
452
+ stack->capa = required;
453
+ }
454
+ return stack;
455
+ }
456
+
457
+ static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame)
458
+ {
459
+ json_frame_stack *stack = state->frames;
460
+
461
+ JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle);
462
+
463
+ if (RB_UNLIKELY(stack->head >= stack->capa)) {
464
+ stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames);
465
+ }
466
+
467
+ json_frame *frame_ptr = &stack->ptr[stack->head++];
468
+ *frame_ptr = frame;
469
+ return frame_ptr;
470
+ }
471
+
472
+ static inline json_frame *json_frame_stack_peek(json_frame_stack *stack)
473
+ {
474
+ return &stack->ptr[stack->head - 1];
475
+ }
476
+
477
+ static inline void json_frame_stack_pop(json_frame_stack *stack)
478
+ {
479
+ stack->head--;
480
+ }
481
+
482
+ static void json_frame_stack_free_buffer(json_frame_stack *stack)
483
+ {
484
+ JSON_SIZED_FREE_N(stack->ptr, stack->capa);
485
+ stack->ptr = NULL;
486
+ }
487
+
488
+ static void json_frame_stack_free(void *ptr)
489
+ {
490
+ json_frame_stack *stack = (json_frame_stack *)ptr;
491
+ if (stack) {
492
+ json_frame_stack_free_buffer(stack);
493
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
494
+ JSON_SIZED_FREE(stack);
495
+ #endif
496
+ }
497
+ }
498
+
499
+ static size_t json_frame_stack_memsize(const void *ptr)
500
+ {
501
+ const json_frame_stack *stack = (const json_frame_stack *)ptr;
502
+
503
+ size_t memsize = sizeof(json_frame) * stack->capa;
504
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
505
+ memsize += sizeof(json_frame_stack);
506
+ #endif
507
+ return memsize;
508
+ }
509
+
510
+ static const rb_data_type_t JSON_Parser_frame_stack_type = {
511
+ .wrap_struct_name = "JSON::Ext::Parser/frame_stack",
512
+ .function = {
513
+ .dmark = NULL,
514
+ .dfree = json_frame_stack_free,
515
+ .dsize = json_frame_stack_memsize,
516
+ },
517
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE,
518
+ };
519
+
520
+ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref)
521
+ {
522
+ json_frame_stack *stack;
523
+ *handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
524
+ *stack_ref = stack;
525
+ MEMCPY(stack, old_stack, json_frame_stack, 1);
526
+
527
+ stack->capa = old_stack->capa << 1;
528
+ stack->ptr = ALLOC_N(json_frame, stack->capa);
529
+ stack->type = RVALUE_STACK_HEAP_ALLOCATED;
530
+ MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head);
531
+ return stack;
532
+ }
533
+
534
+ static void json_frame_stack_eagerly_release(VALUE handle)
535
+ {
536
+ if (handle) {
537
+ json_frame_stack *stack;
538
+ TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack);
539
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
540
+ json_frame_stack_free_buffer(stack);
541
+ #else
542
+ json_frame_stack_free(stack);
543
+ RTYPEDDATA_DATA(handle) = NULL;
544
+ #endif
545
+ }
546
+ }
547
+
548
+ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
549
+ {
550
+ int len = 1;
551
+ if (ch <= 0x7F) {
552
+ buf[0] = (char) ch;
553
+ } else if (ch <= 0x07FF) {
554
+ buf[0] = (char) ((ch >> 6) | 0xC0);
555
+ buf[1] = (char) ((ch & 0x3F) | 0x80);
556
+ len++;
557
+ } else if (ch <= 0xFFFF) {
558
+ buf[0] = (char) ((ch >> 12) | 0xE0);
559
+ buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80);
560
+ buf[2] = (char) ((ch & 0x3F) | 0x80);
561
+ len += 2;
562
+ } else if (ch <= 0x1fffff) {
563
+ buf[0] =(char) ((ch >> 18) | 0xF0);
564
+ buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80);
565
+ buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80);
566
+ buf[3] =(char) ((ch & 0x3F) | 0x80);
567
+ len += 3;
568
+ } else {
569
+ buf[0] = '?';
570
+ }
571
+ return len;
572
+ }
573
+
356
574
  static inline size_t rest(JSON_ParserState *state) {
357
575
  return state->end - state->cursor;
358
576
  }
@@ -371,12 +589,20 @@ static inline char peek(JSON_ParserState *state)
371
589
 
372
590
  static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
373
591
  {
592
+ JSON_ASSERT(state->cursor <= state->end);
593
+
594
+ // Redundant but helpful for hardening
595
+ if (RB_UNLIKELY(state->cursor > state->end)) {
596
+ state->cursor = state->end;
597
+ }
598
+
374
599
  const char *cursor = state->cursor;
375
600
  long column = 0;
376
601
  long line = 1;
377
602
 
378
603
  while (cursor >= state->start) {
379
604
  if (*cursor-- == '\n') {
605
+ line++;
380
606
  break;
381
607
  }
382
608
  column++;
@@ -391,6 +617,8 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum
391
617
  *column_out = column;
392
618
  }
393
619
 
620
+ static const unsigned int MAX_DEPRECATIONS = 5;
621
+
394
622
  static void emit_parse_warning(const char *message, JSON_ParserState *state)
395
623
  {
396
624
  long line, column;
@@ -402,7 +630,7 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state)
402
630
 
403
631
  #define PARSE_ERROR_FRAGMENT_LEN 32
404
632
 
405
- static VALUE build_parse_error_message(const char *format, JSON_ParserState *state, long line, long column)
633
+ static VALUE build_parse_error_message(const char *format, JSON_ParserState *state)
406
634
  {
407
635
  unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
408
636
 
@@ -436,32 +664,61 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta
436
664
  }
437
665
  }
438
666
 
439
- VALUE msg = rb_sprintf(format, ptr);
440
- VALUE message = rb_enc_sprintf(enc_utf8, "%s at line %ld column %ld", RSTRING_PTR(msg), line, column);
441
- RB_GC_GUARD(msg);
442
- return message;
667
+ return rb_enc_sprintf(enc_utf8, format, ptr);
443
668
  }
444
669
 
445
- static VALUE parse_error_new(VALUE message, long line, long column)
670
+ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos)
446
671
  {
447
- VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message);
448
- rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line));
449
- rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column));
672
+ VALUE exc = rb_exc_new_str(eParserError, message);
673
+ rb_ivar_set(exc, i_at_line, LONG2NUM(line));
674
+ rb_ivar_set(exc, i_at_column, LONG2NUM(column));
450
675
  return exc;
451
676
  }
452
677
 
453
- NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state)
678
+ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos)
454
679
  {
455
- long line, column;
456
- cursor_position(state, &line, &column);
457
- VALUE message = build_parse_error_message(format, state, line, column);
458
- rb_exc_raise(parse_error_new(message, line, column));
680
+ if (state->parser) {
681
+ if (eos) {
682
+ // the error will be swallowed by ResumableParser#parse, so no
683
+ // point building a message or backtrace.
684
+ rb_throw_obj(state->parser, state->parser);
685
+ } else {
686
+ // line and columns can't be accurate in resumable
687
+ rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos));
688
+ }
689
+ } else {
690
+ VALUE message = build_parse_error_message(format, state);
691
+ long line, column;
692
+ cursor_position(state, &line, &column);
693
+ rb_str_catf(message, " at line %ld column %ld", line, column);
694
+ rb_exc_raise(parse_error_new(state, message, line, column, eos));
695
+ }
696
+ }
697
+
698
+ NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state)
699
+ {
700
+ raise_parse_error(format, state, true);
701
+ }
702
+
703
+ NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state)
704
+ {
705
+ raise_parse_error(format, state, false);
459
706
  }
460
707
 
461
- NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at)
708
+ NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos)
462
709
  {
463
710
  state->cursor = at;
464
- raise_parse_error(format, state);
711
+ raise_parse_error(format, state, eos);
712
+ }
713
+
714
+ NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at)
715
+ {
716
+ raise_parse_error_at(format, state, at, true);
717
+ }
718
+
719
+ NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at)
720
+ {
721
+ raise_parse_error_at(format, state, at, false);
465
722
  }
466
723
 
467
724
  /* unicode */
@@ -486,7 +743,7 @@ static const signed char digit_values[256] = {
486
743
  static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe)
487
744
  {
488
745
  if (RB_UNLIKELY(sp > spe - 4)) {
489
- raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
746
+ raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
490
747
  }
491
748
 
492
749
  const unsigned char *p = (const unsigned char *)sp;
@@ -497,7 +754,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
497
754
  const signed char b3 = digit_values[p[3]];
498
755
 
499
756
  if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) {
500
- raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
757
+ raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2);
501
758
  }
502
759
 
503
760
  return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3;
@@ -509,19 +766,36 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const
509
766
 
510
767
  static const rb_data_type_t JSON_ParserConfig_type;
511
768
 
512
- static void
513
- json_eat_comments(JSON_ParserState *state)
769
+ const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`";
770
+ NOINLINE(static) void
771
+ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos)
514
772
  {
773
+ if (config->on_comment == JSON_RAISE) {
774
+ raise_syntax_error("unexpected token %s", state);
775
+ }
776
+
515
777
  const char *start = state->cursor;
778
+ // An incomplete comment suspends a resumable parse by rewinding the cursor
779
+ // and throwing. Callers that already consumed a token not yet committed to
780
+ // the frame stack pass resume_pos so the rewind re-reads that token too.
781
+ // Non-resumable error positions keep pointing at the comment either way.
782
+ const char *rewind_pos = (state->parser && resume_pos) ? resume_pos : start;
516
783
  state->cursor++;
517
784
 
518
785
  switch (peek(state)) {
519
786
  case '/': {
520
- state->cursor = memchr(state->cursor, '\n', state->end - state->cursor);
521
- if (!state->cursor) {
787
+ const char *newline = memchr(state->cursor, '\n', state->end - state->cursor);
788
+ if (!newline) {
789
+ // state->parser marks resumable mode, where the buffer end is only a
790
+ // chunk boundary: the terminating newline may still arrive, so leave
791
+ // the comment unterminated instead of consuming to end as a one-shot
792
+ // parse would.
793
+ if (state->parser) {
794
+ raise_eos_error_at("unterminated comment, expected end of line", state, rewind_pos);
795
+ }
522
796
  state->cursor = state->end;
523
797
  } else {
524
- state->cursor++;
798
+ state->cursor = newline + 1;
525
799
  }
526
800
  break;
527
801
  }
@@ -531,7 +805,7 @@ json_eat_comments(JSON_ParserState *state)
531
805
  while (true) {
532
806
  const char *next_match = memchr(state->cursor, '*', state->end - state->cursor);
533
807
  if (!next_match) {
534
- raise_parse_error_at("unterminated comment, expected closing '*/'", state, start);
808
+ raise_eos_error_at("unterminated comment, expected closing '*/'", state, rewind_pos);
535
809
  }
536
810
 
537
811
  state->cursor = next_match + 1;
@@ -543,13 +817,18 @@ json_eat_comments(JSON_ParserState *state)
543
817
  break;
544
818
  }
545
819
  default:
546
- raise_parse_error_at("unexpected token %s", state, start);
820
+ raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state));
547
821
  break;
548
822
  }
823
+
824
+ if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) {
825
+ state->emitted_deprecations++;
826
+ emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state);
827
+ }
549
828
  }
550
829
 
551
830
  ALWAYS_INLINE(static) void
552
- json_eat_whitespace(JSON_ParserState *state)
831
+ json_eat_whitespace_resume_at(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments, const char *resume_pos)
553
832
  {
554
833
  while (true) {
555
834
  switch (peek(state)) {
@@ -580,7 +859,11 @@ json_eat_whitespace(JSON_ParserState *state)
580
859
  state->cursor++;
581
860
  break;
582
861
  case '/':
583
- json_eat_comments(state);
862
+ if (!include_comments) {
863
+ return;
864
+ }
865
+
866
+ json_eat_comments(state, config, resume_pos);
584
867
  break;
585
868
 
586
869
  default:
@@ -589,6 +872,12 @@ json_eat_whitespace(JSON_ParserState *state)
589
872
  }
590
873
  }
591
874
 
875
+ ALWAYS_INLINE(static) void
876
+ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments)
877
+ {
878
+ json_eat_whitespace_resume_at(state, config, include_comments, NULL);
879
+ }
880
+
592
881
  static inline VALUE build_string(const char *start, const char *end, bool intern, bool symbolize)
593
882
  {
594
883
  if (symbolize) {
@@ -734,13 +1023,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
734
1023
  uint32_t sur = unescape_unicode(state, pe + 2, stringEnd);
735
1024
 
736
1025
  if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) {
737
- raise_parse_error_at("invalid surrogate pair at %s", state, p);
1026
+ raise_syntax_error_at("invalid surrogate pair at %s", state, p);
738
1027
  }
739
1028
 
740
1029
  ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF));
741
1030
  pe += 5;
742
1031
  } else {
743
- raise_parse_error_at("incomplete surrogate pair at %s", state, p);
1032
+ raise_syntax_error_at("incomplete surrogate pair at %s", state, p);
744
1033
  break;
745
1034
  }
746
1035
  }
@@ -750,20 +1039,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser
750
1039
  p = ++pe;
751
1040
  break;
752
1041
  }
1042
+ case 0:
1043
+ return Qundef;
753
1044
  default:
754
1045
  if ((unsigned char)*pe < 0x20) {
755
1046
  if (!config->allow_control_characters) {
756
1047
  if (*pe == '\n') {
757
- raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
1048
+ raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1);
758
1049
  }
759
- raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1);
1050
+ raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1);
760
1051
  }
761
1052
  }
762
1053
 
763
1054
  if (config->allow_invalid_escape) {
764
1055
  APPEND_CHAR(*pe);
765
1056
  } else {
766
- raise_parse_error_at("invalid escape character in string: %s", state, pe - 1);
1057
+ raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1);
767
1058
  }
768
1059
  break;
769
1060
  }
@@ -843,7 +1134,7 @@ NOINLINE(static) VALUE json_decode_large_float(const char *start, long len)
843
1134
  /* Ruby JSON optimized float decoder using vendored Ryu algorithm
844
1135
  * Accepts pre-extracted mantissa and exponent from first-pass validation
845
1136
  */
846
- static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int32_t exponent, bool negative,
1137
+ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int64_t exponent, bool negative,
847
1138
  const char *start, const char *end)
848
1139
  {
849
1140
  if (RB_UNLIKELY(config->decimal_class)) {
@@ -851,19 +1142,32 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis
851
1142
  return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text);
852
1143
  }
853
1144
 
854
- // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case)
855
- // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308)
856
- if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) {
1145
+ if (RB_UNLIKELY(exponent > INT32_MAX)) {
1146
+ return negative ? CMinusInfinity : CInfinity;
1147
+ }
1148
+
1149
+ if (RB_UNLIKELY(exponent < INT32_MIN)) {
1150
+ return rb_float_new(negative ? -0.0 : 0.0);
1151
+ }
1152
+
1153
+ if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) {
1154
+ // If the value is so small that it definitely underflows to 0.0, return early
1155
+ // to avoid triggering a "Float out of range" warning from rb_cstr_to_dbl.
1156
+ // When mantissa_digits + exponent < -324, value < 10^(-324) < DBL_TRUE_MIN/2,
1157
+ // so it rounds to 0 in IEEE 754 round-to-nearest.
1158
+ if (RB_UNLIKELY(mantissa_digits + exponent < -324)) {
1159
+ return rb_float_new(negative ? -0.0 : 0.0);
1160
+ }
857
1161
  return json_decode_large_float(start, end - start);
858
1162
  }
859
1163
 
860
- return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, exponent, negative));
1164
+ return DBL2NUM(ffp_s2d(exponent, mantissa, negative));
861
1165
  }
862
1166
 
863
1167
  static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count)
864
1168
  {
865
- VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count));
866
- rvalue_stack_pop(state->stack, count);
1169
+ VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count));
1170
+ rvalue_stack_pop(state->value_stack, count);
867
1171
 
868
1172
  if (config->freeze) {
869
1173
  RB_OBJ_FREEZE(array);
@@ -907,36 +1211,50 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d
907
1211
  rb_inspect(duplicate_key)
908
1212
  );
909
1213
 
910
- long line, column;
911
- cursor_position(state, &line, &column);
912
- rb_str_concat(message, build_parse_error_message("", state, line, column)) ;
913
- rb_exc_raise(parse_error_new(message, line, column));
1214
+ rb_str_concat(message, build_parse_error_message("", state));
1215
+ if (state->parser) { // line and columns can't be accurate in resumable
1216
+ rb_exc_raise(parse_error_new(state, message, 0, 0, false));
1217
+ } else {
1218
+ long line, column;
1219
+ cursor_position(state, &line, &column);
1220
+ rb_str_catf(message, " at line %ld column %ld", line, column);
1221
+ rb_exc_raise(parse_error_new(state, message, line, column, false));
1222
+ }
1223
+ }
914
1224
 
915
- raise_parse_error(RSTRING_PTR(message), state);
916
- RB_GC_GUARD(message);
1225
+ NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs)
1226
+ {
1227
+ switch (config->on_duplicate_key) {
1228
+ case JSON_IGNORE:
1229
+ return;
1230
+
1231
+ case JSON_DEPRECATED:
1232
+ // Only emit the first few deprecations to avoid spamming.
1233
+ if (state->emitted_deprecations < MAX_DEPRECATIONS) {
1234
+ state->emitted_deprecations++;
1235
+ emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
1236
+ }
1237
+ return;
1238
+
1239
+ case JSON_RAISE:
1240
+ raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
1241
+ return;
1242
+ }
1243
+ UNREACHABLE;
917
1244
  }
918
1245
 
919
1246
  static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
920
1247
  {
921
1248
  size_t entries_count = count / 2;
922
1249
  VALUE object = rb_hash_new_capa(entries_count);
923
- const VALUE *pairs = rvalue_stack_peek(state->stack, count);
1250
+ const VALUE *pairs = rvalue_stack_peek(state->value_stack, count);
924
1251
  rb_hash_bulk_insert(count, pairs, object);
925
1252
 
926
1253
  if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
927
- switch (config->on_duplicate_key) {
928
- case JSON_IGNORE:
929
- break;
930
- case JSON_DEPRECATED:
931
- emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
932
- break;
933
- case JSON_RAISE:
934
- raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
935
- break;
936
- }
1254
+ json_on_duplicate_key(state, config, count, pairs);
937
1255
  }
938
1256
 
939
- rvalue_stack_pop(state->stack, count);
1257
+ rvalue_stack_pop(state->value_stack, count);
940
1258
 
941
1259
  if (config->freeze) {
942
1260
  RB_OBJ_FREEZE(object);
@@ -950,7 +1268,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
950
1268
  if (RB_UNLIKELY(config->on_load_proc)) {
951
1269
  value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil);
952
1270
  }
953
- rvalue_stack_push(state->stack, value, &state->stack_handle, &state->stack);
1271
+ rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack);
954
1272
  return value;
955
1273
  }
956
1274
 
@@ -999,6 +1317,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state)
999
1317
  }
1000
1318
  state->cursor++;
1001
1319
  }
1320
+
1321
+ // If the string ended with an unterminated escape sequence, we might
1322
+ // have gone past the end.
1323
+ if (RB_UNLIKELY(state->cursor > state->end)) {
1324
+ state->cursor = state->end;
1325
+ }
1326
+
1002
1327
  return false;
1003
1328
  }
1004
1329
 
@@ -1016,7 +1341,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1016
1341
  case '"': {
1017
1342
  VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions);
1018
1343
  state->cursor++;
1019
- return json_push_value(state, config, string);
1344
+ return string;
1020
1345
  }
1021
1346
  case '\\': {
1022
1347
  if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) {
@@ -1030,7 +1355,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1030
1355
  }
1031
1356
  default:
1032
1357
  if (!config->allow_control_characters) {
1033
- raise_parse_error("invalid ASCII control character in string: %s", state);
1358
+ raise_syntax_error("invalid ASCII control character in string: %s", state);
1034
1359
  }
1035
1360
  break;
1036
1361
  }
@@ -1038,8 +1363,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi
1038
1363
  state->cursor++;
1039
1364
  } while (string_scan(state));
1040
1365
 
1041
- raise_parse_error("unexpected end of input, expected closing \"", state);
1042
- return Qfalse;
1366
+ return Qundef;
1043
1367
  }
1044
1368
 
1045
1369
  ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
@@ -1048,15 +1372,19 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars
1048
1372
  const char *start = state->cursor;
1049
1373
 
1050
1374
  if (RB_UNLIKELY(!string_scan(state))) {
1051
- raise_parse_error("unexpected end of input, expected closing \"", state);
1375
+ return Qundef;
1052
1376
  }
1053
1377
 
1378
+ VALUE string;
1054
1379
  if (RB_LIKELY(*state->cursor == '"')) {
1055
- VALUE string = json_string_fastpath(state, config, start, state->cursor, is_name);
1380
+ string = json_string_fastpath(state, config, start, state->cursor, is_name);
1056
1381
  state->cursor++;
1057
- return json_push_value(state, config, string);
1058
1382
  }
1059
- return json_parse_escaped_string(state, config, is_name, start);
1383
+ else {
1384
+ string = json_parse_escaped_string(state, config, is_name, start);
1385
+ }
1386
+
1387
+ return string;
1060
1388
  }
1061
1389
 
1062
1390
  #if JSON_CPU_LITTLE_ENDIAN_64BITS
@@ -1129,13 +1457,13 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat
1129
1457
  return (int)(state->cursor - start);
1130
1458
  }
1131
1459
 
1132
- static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start)
1460
+ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable)
1133
1461
  {
1134
1462
  bool integer = true;
1135
1463
  const char first_digit = *state->cursor;
1136
1464
 
1137
1465
  // Variables for Ryu optimization - extract digits during parsing
1138
- int32_t exponent = 0;
1466
+ int64_t exponent = 0;
1139
1467
  int decimal_point_pos = -1;
1140
1468
  uint64_t mantissa = 0;
1141
1469
 
@@ -1143,7 +1471,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1143
1471
  int mantissa_digits = json_parse_digits(state, &mantissa);
1144
1472
 
1145
1473
  if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) {
1146
- raise_parse_error_at("invalid number: %s", state, start);
1474
+ return Qundef;
1147
1475
  }
1148
1476
 
1149
1477
  // Parse fractional part
@@ -1156,7 +1484,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1156
1484
  mantissa_digits += fractional_digits;
1157
1485
 
1158
1486
  if (RB_UNLIKELY(!fractional_digits)) {
1159
- raise_parse_error_at("invalid number: %s", state, start);
1487
+ return Qundef;
1160
1488
  }
1161
1489
  }
1162
1490
 
@@ -1176,10 +1504,24 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1176
1504
  int exponent_digits = json_parse_digits(state, &abs_exponent);
1177
1505
 
1178
1506
  if (RB_UNLIKELY(!exponent_digits)) {
1179
- raise_parse_error_at("invalid number: %s", state, start);
1507
+ return Qundef;
1508
+ }
1509
+
1510
+ if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) {
1511
+ exponent = negative_exponent ? INT64_MIN : INT64_MAX;
1512
+ } else {
1513
+ exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent;
1180
1514
  }
1515
+ }
1181
1516
 
1182
- exponent = negative_exponent ? -((int32_t)abs_exponent) : ((int32_t)abs_exponent);
1517
+ // A number touching the end of the buffer may still grow in a later chunk,
1518
+ // so the caller will rewind and wait. Decoding it now would build a value
1519
+ // -- for a long run of digits, an expensive bignum -- only to discard it,
1520
+ // and repeating that on every resumed chunk is quadratic in the number's
1521
+ // length. The digit scan above already advanced the cursor, which is all
1522
+ // the caller needs to detect the incomplete number.
1523
+ if (RB_UNLIKELY(resumable && eos(state))) {
1524
+ return Qundef;
1183
1525
  }
1184
1526
 
1185
1527
  if (integer) {
@@ -1194,229 +1536,420 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig
1194
1536
  return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor);
1195
1537
  }
1196
1538
 
1197
- static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_ParserConfig *config)
1539
+ // How many values (array elements, or interleaved object keys+values) have been
1540
+ // pushed onto the rvalue stack since this container opened. Used to size the
1541
+ // bulk decode on close, and to tell the first key/colon from later ones.
1542
+ static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack)
1198
1543
  {
1199
- return json_parse_number(state, config, false, state->cursor);
1544
+ return value_stack->head - frame->value_stack_head;
1200
1545
  }
1201
1546
 
1202
- static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config)
1547
+ // A complete value now sits on top of the rvalue stack. Advance the frame that
1548
+ // was waiting for it: the root document is done, or the enclosing container
1549
+ // moves on to expecting a ',' or its closing bracket. The caller passes the
1550
+ // frame it already has in hand -- the one that was expecting the value -- which
1551
+ // after a container close is the freshly re-exposed parent.
1552
+ static inline enum json_frame_phase json_value_completed(json_frame *frame)
1203
1553
  {
1204
- const char *start = state->cursor;
1205
- state->cursor++;
1206
- return json_parse_number(state, config, true, start);
1554
+ JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT);
1555
+ JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY);
1556
+ JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT);
1557
+
1558
+ return frame->phase = (enum json_frame_phase) frame->type;
1207
1559
  }
1208
1560
 
1209
- static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
1561
+ ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset)
1210
1562
  {
1211
- json_eat_whitespace(state);
1563
+ // It is assumed that since `keyword` is always a literal, the compiler is able to constantize this
1564
+ // `strlen` and several other computations in that routine.
1212
1565
 
1213
- switch (peek(state)) {
1214
- case 'n':
1215
- if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) {
1216
- state->cursor += 4;
1217
- return json_push_value(state, config, Qnil);
1218
- }
1219
-
1220
- raise_parse_error("unexpected token %s", state);
1221
- break;
1222
- case 't':
1223
- if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) {
1224
- state->cursor += 4;
1225
- return json_push_value(state, config, Qtrue);
1226
- }
1566
+ size_t len = strlen(keyword);
1227
1567
 
1228
- raise_parse_error("unexpected token %s", state);
1229
- break;
1230
- case 'f':
1231
- // Note: memcmp with a small power of two compile to an integer comparison
1232
- if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) {
1233
- state->cursor += 5;
1234
- return json_push_value(state, config, Qfalse);
1235
- }
1568
+ // Note: memcmp with a small power of two and a literal string compile to an integer comparison /
1569
+ // That's why we sometime compare starting from the first byte and sometimes from the second.
1570
+ if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) {
1571
+ state->cursor += len;
1572
+ return;
1573
+ }
1236
1574
 
1237
- raise_parse_error("unexpected token %s", state);
1238
- break;
1239
- case 'N':
1240
- // Note: memcmp with a small power of two compile to an integer comparison
1241
- if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) {
1242
- state->cursor += 3;
1243
- return json_push_value(state, config, CNaN);
1244
- }
1575
+ bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0;
1576
+ raise_parse_error("unexpected token %s", state, eos);
1577
+ }
1578
+
1579
+ // Parse an arbitrary JSON value iteratively. This is a state machine driven
1580
+ // entirely by the top frame's phase so it can stop at any value boundary and
1581
+ // resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the
1582
+ // bottom of the stack, so the stack is never empty mid-parse and the document
1583
+ // itself is just another frame whose value, once parsed, leaves its phase DONE.
1584
+ // When invoked in resumable mode, it returns true after parsing a complete document.
1585
+ // If reaching EOS without having parsed a complete document, either returns false
1586
+ // of raise a JSON::ParserError tagged with `@eos=true`.
1587
+ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable)
1588
+ {
1589
+ json_frame *frame = json_frame_stack_peek(state->frames);
1590
+
1591
+ switch (frame->phase) {
1592
+ case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false);
1593
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1594
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1595
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1596
+ case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY;
1597
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1598
+ }
1599
+ JSON_UNREACHABLE_RETURN(false);
1245
1600
 
1246
- raise_parse_error("unexpected token %s", state);
1247
- break;
1248
- case 'I':
1249
- if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) {
1250
- state->cursor += 8;
1251
- return json_push_value(state, config, CInfinity);
1252
- }
1601
+ JSON_PHASE_VALUE: {
1602
+ json_eat_whitespace(state, config, true);
1253
1603
 
1254
- raise_parse_error("unexpected token %s", state);
1255
- break;
1256
- case '-': {
1257
- // Note: memcmp with a small power of two compile to an integer comparison
1258
- if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) {
1259
- if (config->allow_nan) {
1260
- state->cursor += 9;
1261
- return json_push_value(state, config, CMinusInfinity);
1262
- } else {
1263
- raise_parse_error("unexpected token %s", state);
1264
- }
1265
- }
1266
- return json_push_value(state, config, json_parse_negative_number(state, config));
1267
- break;
1268
- }
1269
- case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
1270
- return json_push_value(state, config, json_parse_positive_number(state, config));
1271
- break;
1272
- case '"': {
1273
- // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
1274
- return json_parse_string(state, config, false);
1275
- break;
1604
+ // A trailing comma lands us here expecting an element but finding the
1605
+ // closing bracket; hand off to ARRAY_COMMA to close. An empty array
1606
+ // closes inline at '[', so this position is only reached after a ','.
1607
+ if (config->allow_trailing_comma && frame->type == JSON_FRAME_ARRAY && peek(state) == ']') {
1608
+ goto JSON_PHASE_ARRAY_COMMA;
1276
1609
  }
1277
- case '[': {
1278
- state->cursor++;
1279
- json_eat_whitespace(state);
1280
- long stack_head = state->stack->head;
1281
1610
 
1282
- if (peek(state) == ']') {
1283
- state->cursor++;
1284
- return json_push_value(state, config, json_decode_array(state, config, 0));
1285
- } else {
1286
- state->current_nesting++;
1287
- if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1288
- rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1289
- }
1290
- state->in_array++;
1291
- json_parse_any(state, config);
1292
- }
1611
+ VALUE value;
1612
+ const char *value_start = state->cursor;
1293
1613
 
1294
- while (true) {
1295
- json_eat_whitespace(state);
1614
+ switch (peek(state)) {
1615
+ case 'n':
1616
+ json_match_keyword(state, "null", 0);
1617
+ value = Qnil;
1618
+ break;
1296
1619
 
1297
- const char next_char = peek(state);
1620
+ case 't':
1621
+ json_match_keyword(state, "true", 0);
1622
+ value = Qtrue;
1623
+ break;
1298
1624
 
1299
- if (RB_LIKELY(next_char == ',')) {
1300
- state->cursor++;
1301
- if (config->allow_trailing_comma) {
1302
- json_eat_whitespace(state);
1303
- if (peek(state) == ']') {
1304
- continue;
1305
- }
1306
- }
1307
- json_parse_any(state, config);
1308
- continue;
1309
- }
1625
+ case 'f':
1626
+ json_match_keyword(state, "false", 1);
1627
+ value = Qfalse;
1628
+ break;
1310
1629
 
1311
- if (next_char == ']') {
1312
- state->cursor++;
1313
- long count = state->stack->head - stack_head;
1314
- state->current_nesting--;
1315
- state->in_array--;
1316
- return json_push_value(state, config, json_decode_array(state, config, count));
1630
+ case 'N':
1631
+ if (!config->allow_nan) {
1632
+ raise_syntax_error("unexpected token %s", state);
1633
+ }
1634
+
1635
+ json_match_keyword(state, "NaN", 1);
1636
+ value = CNaN;
1637
+ break;
1638
+
1639
+ case 'I':
1640
+ if (!config->allow_nan) {
1641
+ raise_syntax_error("unexpected token %s", state);
1642
+ }
1643
+
1644
+ json_match_keyword(state, "Infinity", 0);
1645
+ value = CInfinity;
1646
+ break;
1647
+
1648
+ case '-': {
1649
+ state->cursor++;
1650
+
1651
+ value = json_parse_number(state, config, true, value_start, resumable);
1652
+
1653
+ if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) {
1654
+ state->cursor = value_start;
1655
+ json_match_keyword(state, "-Infinity", 1);
1656
+ value = CMinusInfinity;
1657
+ break;
1658
+ }
1659
+
1660
+ // Top level numbers are ambiguous when parsing streams, we can't
1661
+ // know if we parsed all the digits if we hit EOS.
1662
+ if (RB_UNLIKELY(resumable && eos(state))) {
1663
+ state->cursor = value_start;
1664
+ return false;
1317
1665
  }
1318
1666
 
1319
- raise_parse_error("expected ',' or ']' after array value", state);
1667
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1668
+ raise_syntax_error_at("invalid number: %s", state, value_start);
1669
+ }
1670
+ break;
1320
1671
  }
1321
- break;
1322
- }
1323
- case '{': {
1324
- const char *object_start_cursor = state->cursor;
1325
1672
 
1326
- state->cursor++;
1327
- json_eat_whitespace(state);
1328
- long stack_head = state->stack->head;
1673
+ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
1674
+ value = json_parse_number(state, config, false, value_start, resumable);
1329
1675
 
1330
- if (peek(state) == '}') {
1676
+ // Top level numbers are ambiguous when parsing streams, we can't
1677
+ // know if we parsed all the digits if we hit EOS.
1678
+ if (RB_UNLIKELY(resumable && eos(state))) {
1679
+ state->cursor = value_start;
1680
+ return false;
1681
+ }
1682
+
1683
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1684
+ raise_syntax_error_at("invalid number: %s", state, value_start);
1685
+ }
1686
+ break;
1687
+ }
1688
+
1689
+ case '"': {
1690
+ // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"}
1691
+ value = json_parse_string(state, config, false);
1692
+
1693
+ if (RB_UNLIKELY(UNDEF_P(value))) {
1694
+ bool is_eos = eos(state);
1695
+ if (resumable && is_eos) {
1696
+ state->cursor = value_start;
1697
+ return false;
1698
+ }
1699
+ raise_parse_error("unexpected end of input, expected closing \"", state, is_eos);
1700
+ }
1701
+ break;
1702
+ }
1703
+
1704
+ case '[': {
1331
1705
  state->cursor++;
1332
- return json_push_value(state, config, json_decode_object(state, config, 0));
1333
- } else {
1706
+ // The '[' is consumed but its frame is only pushed below, so a
1707
+ // comment suspending here must resume from the bracket.
1708
+ json_eat_whitespace_resume_at(state, config, true, value_start);
1709
+
1710
+ const char next = peek(state);
1711
+ if (next == ']') {
1712
+ state->cursor++;
1713
+ value = json_decode_array(state, config, 0);
1714
+ break;
1715
+ } else if (resumable && eos(state)) {
1716
+ state->cursor = value_start;
1717
+ return false;
1718
+ }
1719
+
1334
1720
  state->current_nesting++;
1335
1721
  if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1336
1722
  rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1337
1723
  }
1724
+ state->in_array++;
1725
+
1726
+ // Phase stays VALUE: the next iteration reads the first element.
1727
+ frame = json_frame_stack_push(state, (json_frame){
1728
+ .type = JSON_FRAME_ARRAY,
1729
+ .phase = JSON_PHASE_VALUE,
1730
+ .value_stack_head = state->value_stack->head,
1731
+ });
1732
+ goto JSON_PHASE_VALUE;
1733
+ }
1734
+
1735
+ case '{': {
1736
+ state->cursor++;
1737
+ // Same as '[': the frame is only pushed below.
1738
+ json_eat_whitespace_resume_at(state, config, true, value_start);
1338
1739
 
1339
- if (peek(state) != '"') {
1340
- raise_parse_error("expected object key, got %s", state);
1740
+ if (peek(state) == '}') {
1741
+ state->cursor++;
1742
+ value = json_decode_object(state, config, 0);
1743
+ break;
1744
+ } else if (resumable && eos(state)) {
1745
+ state->cursor = value_start;
1746
+ return false;
1341
1747
  }
1342
- json_parse_string(state, config, true);
1343
1748
 
1344
- json_eat_whitespace(state);
1345
- if (peek(state) != ':') {
1346
- raise_parse_error("expected ':' after object key", state);
1749
+ state->current_nesting++;
1750
+ if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) {
1751
+ rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting);
1347
1752
  }
1348
- state->cursor++;
1349
1753
 
1350
- json_parse_any(state, config);
1754
+ // Phase KEY: the next iteration reads the first key.
1755
+ frame = json_frame_stack_push(state, (json_frame){
1756
+ .type = JSON_FRAME_OBJECT,
1757
+ .phase = JSON_PHASE_OBJECT_KEY,
1758
+ .value_stack_head = state->value_stack->head,
1759
+ .start_offset = value_start - state->start,
1760
+ });
1761
+ goto JSON_PHASE_OBJECT_KEY;
1351
1762
  }
1352
1763
 
1353
- while (true) {
1354
- json_eat_whitespace(state);
1764
+ case 0:
1765
+ // peek() returns 0 both at end-of-stream and for a literal NUL byte in the
1766
+ // buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at
1767
+ // EOS is just an invalid character.
1768
+ if (eos(state)) {
1769
+ return false;
1770
+ } else {
1771
+ raise_syntax_error("unexpected NULL byte: %s", state);
1772
+ }
1773
+ default:
1774
+ raise_syntax_error("unexpected character: %s", state);
1775
+ }
1355
1776
 
1356
- const char next_char = peek(state);
1357
- if (next_char == '}') {
1358
- state->cursor++;
1359
- state->current_nesting--;
1360
- size_t count = state->stack->head - stack_head;
1777
+ json_push_value(state, config, value);
1778
+ json_value_completed(frame);
1779
+
1780
+ switch (frame->phase) {
1781
+ case JSON_PHASE_DONE: return true;
1782
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1783
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1784
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1785
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1786
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1787
+ }
1788
+ JSON_UNREACHABLE_RETURN(false);
1789
+ }
1790
+
1791
+ JSON_PHASE_OBJECT_KEY: {
1792
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1361
1793
 
1362
- // Temporary rewind cursor in case an error is raised
1363
- const char *final_cursor = state->cursor;
1364
- state->cursor = object_start_cursor;
1365
- VALUE object = json_decode_object(state, config, count);
1366
- state->cursor = final_cursor;
1794
+ json_eat_whitespace(state, config, true);
1367
1795
 
1368
- return json_push_value(state, config, object);
1796
+ // A trailing comma lands us here expecting a key but finding the closing
1797
+ // brace; hand off to OBJECT_COMMA to close. An empty object closes inline
1798
+ // at '{', so this position is only reached after a ','.
1799
+ if (config->allow_trailing_comma && peek(state) == '}') {
1800
+ goto JSON_PHASE_OBJECT_COMMA;
1801
+ }
1802
+
1803
+ const char *start = state->cursor;
1804
+
1805
+ if (RB_LIKELY(peek(state) == '"')) {
1806
+ VALUE string = json_parse_string(state, config, true);
1807
+ if (UNDEF_P(string)) {
1808
+ if (resumable) {
1809
+ state->cursor = start;
1810
+ return false;
1811
+ } else {
1812
+ raise_syntax_error("unexpected end of input, expected closing \"", state);
1369
1813
  }
1814
+ }
1815
+ json_push_value(state, config, string);
1816
+ frame->phase = JSON_PHASE_OBJECT_COLON;
1817
+ goto JSON_PHASE_OBJECT_COLON;
1818
+ } else if (resumable && eos(state)) {
1819
+ return false;
1820
+ } else {
1821
+ // The message differs for the first key vs. a key after a
1822
+ // ',': the first is the only one reached with nothing pushed
1823
+ // for this object yet.
1824
+ if (json_frame_entry_count(frame, state->value_stack) == 0) {
1825
+ raise_syntax_error("expected object key, got %s", state);
1826
+ } else {
1827
+ raise_syntax_error("expected object key, got: %s", state);
1828
+ }
1829
+ }
1830
+ JSON_UNREACHABLE_RETURN(false);
1831
+ }
1370
1832
 
1371
- if (next_char == ',') {
1372
- state->cursor++;
1373
- json_eat_whitespace(state);
1833
+ JSON_PHASE_OBJECT_COLON: {
1834
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1374
1835
 
1375
- if (config->allow_trailing_comma) {
1376
- if (peek(state) == '}') {
1377
- continue;
1378
- }
1379
- }
1836
+ json_eat_whitespace(state, config, true);
1380
1837
 
1381
- if (RB_UNLIKELY(peek(state) != '"')) {
1382
- raise_parse_error("expected object key, got: %s", state);
1383
- }
1384
- json_parse_string(state, config, true);
1838
+ if (RB_LIKELY(peek(state) == ':')) {
1839
+ state->cursor++;
1840
+ frame->phase = JSON_PHASE_VALUE;
1841
+ goto JSON_PHASE_VALUE;
1842
+ } else if (resumable && eos(state)) {
1843
+ return false;
1844
+ } else {
1845
+ // First colon (only the first pair's key is pushed, nothing
1846
+ // else) vs. a later one.
1847
+ if (json_frame_entry_count(frame, state->value_stack) == 1) {
1848
+ raise_syntax_error("expected ':' after object key", state);
1849
+ } else {
1850
+ raise_syntax_error("expected ':' after object key, got: %s", state);
1851
+ }
1852
+ }
1853
+ JSON_UNREACHABLE_RETURN(false);
1854
+ }
1385
1855
 
1386
- json_eat_whitespace(state);
1387
- if (RB_UNLIKELY(peek(state) != ':')) {
1388
- raise_parse_error("expected ':' after object key, got: %s", state);
1389
- }
1390
- state->cursor++;
1856
+ JSON_PHASE_ARRAY_COMMA: {
1857
+ JSON_ASSERT(frame->type == JSON_FRAME_ARRAY);
1391
1858
 
1392
- json_parse_any(state, config);
1859
+ json_eat_whitespace(state, config, true);
1393
1860
 
1394
- continue;
1395
- }
1861
+ const char next_char = peek(state);
1396
1862
 
1397
- raise_parse_error("expected ',' or '}' after object value, got: %s", state);
1863
+ if (RB_LIKELY(next_char == ',')) {
1864
+ state->cursor++;
1865
+ // Commit the phase before eating the whitespace that follows: an
1866
+ // incomplete comment there would suspend the parse, and a phase not
1867
+ // yet advanced past the ',' would drop it on resume. A trailing comma
1868
+ // is recognized in JSON_PHASE_VALUE once the ']' is in the buffer.
1869
+ frame->phase = JSON_PHASE_VALUE;
1870
+ goto JSON_PHASE_VALUE;
1871
+ } else if (next_char == ']') {
1872
+ state->cursor++;
1873
+ long count = json_frame_entry_count(frame, state->value_stack);
1874
+ state->current_nesting--;
1875
+ state->in_array--;
1876
+
1877
+ json_push_value(state, config, json_decode_array(state, config, count));
1878
+ json_frame_stack_pop(state->frames);
1879
+ frame = json_frame_stack_peek(state->frames);
1880
+
1881
+ json_value_completed(frame);
1882
+
1883
+ switch (frame->phase) {
1884
+ case JSON_PHASE_DONE: return true;
1885
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1886
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1887
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1888
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1889
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1398
1890
  }
1399
- break;
1891
+ } else if (resumable && eos(state)) {
1892
+ return false;
1893
+ } else {
1894
+ raise_syntax_error("expected ',' or ']' after array value", state);
1400
1895
  }
1896
+ JSON_UNREACHABLE_RETURN(false);
1897
+ }
1401
1898
 
1402
- case 0:
1403
- raise_parse_error("unexpected end of input", state);
1404
- break;
1899
+ JSON_PHASE_OBJECT_COMMA: {
1900
+ JSON_ASSERT(frame->type == JSON_FRAME_OBJECT);
1405
1901
 
1406
- default:
1407
- raise_parse_error("unexpected character: %s", state);
1408
- break;
1902
+ json_eat_whitespace(state, config, true);
1903
+ const char next_char = peek(state);
1904
+
1905
+ if (RB_LIKELY(next_char == ',')) {
1906
+ state->cursor++;
1907
+ // Commit the phase before eating the whitespace that follows: an
1908
+ // incomplete comment there would suspend the parse, and a phase not
1909
+ // yet advanced past the ',' would drop it on resume. A trailing comma
1910
+ // is recognized in JSON_PHASE_OBJECT_KEY once the '}' is in the buffer.
1911
+ frame->phase = JSON_PHASE_OBJECT_KEY;
1912
+ goto JSON_PHASE_OBJECT_KEY;
1913
+ } else if (next_char == '}') {
1914
+ state->cursor++;
1915
+ state->current_nesting--;
1916
+ size_t count = json_frame_entry_count(frame, state->value_stack);
1917
+
1918
+ // Temporary rewind cursor in case an error is raised
1919
+ const char *final_cursor = state->cursor;
1920
+ state->cursor = state->start + frame->start_offset;
1921
+ VALUE object = json_decode_object(state, config, count);
1922
+ state->cursor = final_cursor;
1923
+
1924
+ json_push_value(state, config, object);
1925
+ json_frame_stack_pop(state->frames);
1926
+ frame = json_frame_stack_peek(state->frames);
1927
+ json_value_completed(frame);
1928
+
1929
+ switch (frame->phase) {
1930
+ case JSON_PHASE_DONE: return true;
1931
+ case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA;
1932
+ case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA;
1933
+ case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE;
1934
+ case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false);
1935
+ case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON;
1936
+ }
1937
+ } else if (resumable && eos(state)) {
1938
+ return false;
1939
+ } else {
1940
+ raise_syntax_error("expected ',' or '}' after object value, got: %s", state);
1941
+ }
1942
+ JSON_UNREACHABLE_RETURN(false);
1409
1943
  }
1410
1944
 
1411
- raise_parse_error("unreachable: %s", state);
1412
- return Qundef;
1945
+ JSON_UNREACHABLE_RETURN(false);
1413
1946
  }
1414
1947
 
1415
- static void json_ensure_eof(JSON_ParserState *state)
1948
+ static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config)
1416
1949
  {
1417
- json_eat_whitespace(state);
1950
+ json_eat_whitespace(state, config, true);
1418
1951
  if (!eos(state)) {
1419
- raise_parse_error("unexpected token at end of stream %s", state);
1952
+ raise_syntax_error("unexpected token at end of stream %s", state);
1420
1953
  }
1421
1954
  }
1422
1955
 
@@ -1434,40 +1967,59 @@ static void json_ensure_eof(JSON_ParserState *state)
1434
1967
 
1435
1968
  static VALUE convert_encoding(VALUE source)
1436
1969
  {
1437
- int encindex = RB_ENCODING_GET(source);
1970
+ StringValue(source);
1971
+ int encindex = RB_ENCODING_GET(source);
1972
+
1973
+ if (RB_LIKELY(encindex == utf8_encindex)) {
1974
+ return source;
1975
+ }
1976
+
1977
+ if (encindex == binary_encindex) {
1978
+ // For historical reason, we silently reinterpret binary strings as UTF-8
1979
+ return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
1980
+ }
1438
1981
 
1439
- if (RB_LIKELY(encindex == utf8_encindex)) {
1982
+ source = rb_funcall(source, i_encode, 1, Encoding_UTF_8);
1983
+ StringValue(source);
1440
1984
  return source;
1441
- }
1985
+ }
1442
1986
 
1443
- if (encindex == binary_encindex) {
1444
- // For historical reason, we silently reinterpret binary strings as UTF-8
1445
- return rb_enc_associate_index(rb_str_dup(source), utf8_encindex);
1446
- }
1987
+ struct parser_config_init_args {
1988
+ JSON_ParserConfig *config;
1989
+ VALUE self;
1990
+ VALUE unknown_keywords;
1991
+ bool strict;
1992
+ };
1447
1993
 
1448
- return rb_funcall(source, i_encode, 1, Encoding_UTF_8);
1994
+ static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val)
1995
+ {
1996
+ *dest = val;
1997
+ if (self) RB_OBJ_WRITTEN(self, Qundef, val);
1449
1998
  }
1450
1999
 
1451
2000
  static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1452
2001
  {
1453
- JSON_ParserConfig *config = (JSON_ParserConfig *)data;
2002
+ struct parser_config_init_args *args = (struct parser_config_init_args *)data;
2003
+ JSON_ParserConfig *config = args->config;
2004
+ VALUE self = args->self;
1454
2005
 
1455
2006
  if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; }
1456
2007
  else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); }
1457
2008
  else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); }
2009
+ else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
1458
2010
  else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); }
1459
2011
  else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); }
1460
2012
  else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); }
1461
2013
  else if (key == sym_freeze) { config->freeze = RTEST(val); }
1462
- else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; }
2014
+ else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); }
1463
2015
  else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
1464
2016
  else if (key == sym_decimal_class) {
1465
2017
  if (RTEST(val)) {
1466
2018
  if (rb_respond_to(val, i_try_convert)) {
1467
- config->decimal_class = val;
2019
+ parser_config_wb_write(self, &config->decimal_class, val);
1468
2020
  config->decimal_method_id = i_try_convert;
1469
2021
  } else if (rb_respond_to(val, i_new)) {
1470
- config->decimal_class = val;
2022
+ parser_config_wb_write(self, &config->decimal_class, val);
1471
2023
  config->decimal_method_id = i_new;
1472
2024
  } else if (RB_TYPE_P(val, T_CLASS)) {
1473
2025
  VALUE name = rb_class_name(val);
@@ -1476,7 +2028,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1476
2028
  if (last_colon) {
1477
2029
  const char *mod_path_end = last_colon - 1;
1478
2030
  VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr);
1479
- config->decimal_class = rb_path_to_class(mod_path);
2031
+ parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path));
1480
2032
 
1481
2033
  const char *method_name_beg = last_colon + 1;
1482
2034
  long before_len = method_name_beg - name_cstr;
@@ -1484,28 +2036,48 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
1484
2036
  VALUE method_name = rb_str_substr(name, before_len, len);
1485
2037
  config->decimal_method_id = SYM2ID(rb_str_intern(method_name));
1486
2038
  } else {
1487
- config->decimal_class = rb_mKernel;
2039
+ parser_config_wb_write(self, &config->decimal_class, rb_mKernel);
1488
2040
  config->decimal_method_id = SYM2ID(rb_str_intern(name));
1489
2041
  }
1490
2042
  }
1491
2043
  }
1492
2044
  }
2045
+ else if (args->strict) {
2046
+ if (!args->unknown_keywords) {
2047
+ args->unknown_keywords = rb_obj_hide(rb_ary_new());
2048
+ }
2049
+ rb_ary_push(args->unknown_keywords, key);
2050
+ }
1493
2051
 
1494
2052
  return ST_CONTINUE;
1495
2053
  }
1496
2054
 
1497
- static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
2055
+ static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict)
1498
2056
  {
1499
2057
  config->max_nesting = 100;
1500
2058
 
1501
- if (!NIL_P(opts)) {
1502
- Check_Type(opts, T_HASH);
1503
- if (RHASH_SIZE(opts) > 0) {
1504
- // We assume in most cases few keys are set so it's faster to go over
1505
- // the provided keys than to check all possible keys.
1506
- rb_hash_foreach(opts, parser_config_init_i, (VALUE)config);
1507
- }
2059
+ struct parser_config_init_args args = {
2060
+ .config = config,
2061
+ .self = self,
2062
+ .strict = strict,
2063
+ };
1508
2064
 
2065
+ if (NIL_P(opts)) return;
2066
+ Check_Type(opts, T_HASH);
2067
+ if (RHASH_SIZE(opts) == 0) return;
2068
+
2069
+ // We assume in most cases few keys are set so it's faster to go over
2070
+ // the provided keys than to check all possible keys.
2071
+ rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args);
2072
+
2073
+ if (RB_UNLIKELY(args.unknown_keywords)) {
2074
+ if (RARRAY_LEN(args.unknown_keywords) == 1) {
2075
+ rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0));
2076
+ }
2077
+ else {
2078
+ VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", "));
2079
+ rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords);
2080
+ }
1509
2081
  }
1510
2082
  }
1511
2083
 
@@ -1514,67 +2086,90 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts)
1514
2086
  *
1515
2087
  * Creates a new JSON::Ext::ParserConfig instance.
1516
2088
  *
1517
- * It will be configured by the _opts_ hash. _opts_ can have the following
1518
- * keys:
2089
+ * Argument +opts+, if given, contains a \Hash of options for the parsing.
2090
+ * See {Parsing Options}[#module-JSON-label-Parsing+Options].
1519
2091
  *
1520
- * _opts_ can have the following keys:
1521
- * * *max_nesting*: The maximum depth of nesting allowed in the parsed data
1522
- * structures. Disable depth checking with :max_nesting => false|nil|0, it
1523
- * defaults to 100.
1524
- * * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in
1525
- * defiance of RFC 4627 to be parsed by the Parser. This option defaults to
1526
- * false.
1527
- * * *symbolize_names*: If set to true, returns symbols for the names
1528
- * (keys) in a JSON object. Otherwise strings are returned, which is
1529
- * also the default. It's not possible to use this option in
1530
- * conjunction with the *create_additions* option.
1531
- * * *decimal_class*: Specifies which class to use instead of the default
1532
- * (Float) when parsing decimal numbers. This class must accept a single
1533
- * string argument in its constructor.
1534
2092
  */
1535
2093
  static VALUE cParserConfig_initialize(VALUE self, VALUE opts)
1536
2094
  {
1537
2095
  rb_check_frozen(self);
1538
2096
  GET_PARSER_CONFIG;
1539
2097
 
1540
- parser_config_init(config, opts);
1541
-
1542
- RB_OBJ_WRITTEN(self, Qundef, config->decimal_class);
2098
+ parser_config_init(config, opts, self, false);
1543
2099
 
1544
2100
  return self;
1545
2101
  }
1546
2102
 
1547
- static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource)
2103
+ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src)
1548
2104
  {
1549
- Vsource = convert_encoding(StringValue(Vsource));
1550
- StringValue(Vsource);
2105
+ VALUE Vsource = convert_encoding(src);
2106
+
2107
+ // Ensure the string isn't mutated under us.
2108
+ // The classic API to use is `rb_str_locktmp`, but then we'd
2109
+ // need to use `rb_protect` to make sure we always unlock.
2110
+ if (Vsource == src) {
2111
+ Vsource = rb_str_new_frozen(Vsource);
2112
+ }
1551
2113
 
1552
2114
  VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA];
1553
- rvalue_stack stack = {
2115
+ rvalue_stack value_stack = {
1554
2116
  .type = RVALUE_STACK_STACK_ALLOCATED,
1555
2117
  .ptr = rvalue_stack_buffer,
1556
2118
  .capa = RVALUE_STACK_INITIAL_CAPA,
1557
2119
  };
1558
2120
 
2121
+ // Seed the frame stack with the root frame, establishing the invariant that
2122
+ // json_parse_any always has a top frame to dispatch on (so the stack is never
2123
+ // empty mid-parse).
2124
+ json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA];
2125
+ frame_stack_buffer[0] = (json_frame){
2126
+ .type = JSON_FRAME_ROOT,
2127
+ .phase = JSON_PHASE_VALUE,
2128
+ };
2129
+ json_frame_stack frames = {
2130
+ .type = RVALUE_STACK_STACK_ALLOCATED,
2131
+ .ptr = frame_stack_buffer,
2132
+ .capa = JSON_FRAME_STACK_INITIAL_CAPA,
2133
+ .head = 1,
2134
+ };
2135
+
1559
2136
  long len;
1560
2137
  const char *start;
2138
+
1561
2139
  RSTRING_GETMEM(Vsource, start, len);
1562
2140
 
2141
+ VALUE value_stack_handle = 0;
2142
+ VALUE frame_stack_handle = 0;
1563
2143
  JSON_ParserState _state = {
1564
2144
  .start = start,
1565
2145
  .cursor = start,
1566
2146
  .end = start + len,
1567
- .stack = &stack,
2147
+ .value_stack = &value_stack,
2148
+ .value_stack_handle = &value_stack_handle,
2149
+ .frames = &frames,
2150
+ .frame_stack_handle = &frame_stack_handle,
1568
2151
  };
1569
2152
  JSON_ParserState *state = &_state;
1570
2153
 
1571
- VALUE result = json_parse_any(state, config);
2154
+ bool complete = json_parse_any(state, config, false);
2155
+
2156
+ // The root document value is parsed; it is the lone survivor on
2157
+ // the rvalue stack.
2158
+ VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef;
1572
2159
 
1573
2160
  // This may be skipped in case of exception, but
1574
2161
  // it won't cause a leak.
1575
- rvalue_stack_eagerly_release(state->stack_handle);
1576
-
1577
- json_ensure_eof(state);
2162
+ rvalue_stack_eagerly_release(value_stack_handle);
2163
+ json_frame_stack_eagerly_release(frame_stack_handle);
2164
+ RB_GC_GUARD(value_stack_handle);
2165
+ RB_GC_GUARD(frame_stack_handle);
2166
+ RB_GC_GUARD(Vsource);
2167
+
2168
+ if (complete) {
2169
+ json_ensure_eof(state, config);
2170
+ } else {
2171
+ raise_eos_error("unexpected end of input", state);
2172
+ }
1578
2173
 
1579
2174
  return result;
1580
2175
  }
@@ -1594,12 +2189,9 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource)
1594
2189
 
1595
2190
  static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
1596
2191
  {
1597
- Vsource = convert_encoding(StringValue(Vsource));
1598
- StringValue(Vsource);
1599
-
1600
2192
  JSON_ParserConfig _config = {0};
1601
2193
  JSON_ParserConfig *config = &_config;
1602
- parser_config_init(config, opts);
2194
+ parser_config_init(config, opts, Qfalse, false);
1603
2195
 
1604
2196
  return cParser_parse(config, Vsource);
1605
2197
  }
@@ -1607,30 +2199,35 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts)
1607
2199
  static void JSON_ParserConfig_mark(void *ptr)
1608
2200
  {
1609
2201
  JSON_ParserConfig *config = ptr;
1610
- rb_gc_mark(config->on_load_proc);
1611
- rb_gc_mark(config->decimal_class);
2202
+ rb_gc_mark_movable(config->on_load_proc);
2203
+ rb_gc_mark_movable(config->decimal_class);
1612
2204
  }
1613
2205
 
1614
- static void JSON_ParserConfig_free(void *ptr)
2206
+ static size_t JSON_ParserConfig_memsize(const void *ptr)
1615
2207
  {
1616
- JSON_ParserConfig *config = ptr;
1617
- ruby_xfree(config);
2208
+ #ifdef HAVE_RUBY_TYPED_EMBEDDABLE
2209
+ return 0;
2210
+ #else
2211
+ return sizeof(JSON_ParserConfig);
2212
+ #endif
1618
2213
  }
1619
2214
 
1620
- static size_t JSON_ParserConfig_memsize(const void *ptr)
2215
+ static void JSON_ParserConfig_compact(void *ptr)
1621
2216
  {
1622
- return sizeof(JSON_ParserConfig);
2217
+ JSON_ParserConfig *config = ptr;
2218
+ config->on_load_proc = rb_gc_location(config->on_load_proc);
2219
+ config->decimal_class = rb_gc_location(config->decimal_class);
1623
2220
  }
1624
2221
 
1625
2222
  static const rb_data_type_t JSON_ParserConfig_type = {
1626
- "JSON::Ext::Parser/ParserConfig",
1627
- {
1628
- JSON_ParserConfig_mark,
1629
- JSON_ParserConfig_free,
1630
- JSON_ParserConfig_memsize,
2223
+ .wrap_struct_name = "JSON::Ext::Parser/ParserConfig",
2224
+ .function = {
2225
+ .dmark = JSON_ParserConfig_mark,
2226
+ .dfree = RUBY_DEFAULT_FREE,
2227
+ .dsize = JSON_ParserConfig_memsize,
2228
+ .dcompact = JSON_ParserConfig_compact,
1631
2229
  },
1632
- 0, 0,
1633
- RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE,
2230
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE,
1634
2231
  };
1635
2232
 
1636
2233
  static VALUE cJSON_parser_s_allocate(VALUE klass)
@@ -1639,6 +2236,602 @@ static VALUE cJSON_parser_s_allocate(VALUE klass)
1639
2236
  return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config);
1640
2237
  }
1641
2238
 
2239
+ static void json_str_clear(VALUE str)
2240
+ {
2241
+ if (RB_OBJ_FROZEN_RAW(str)) {
2242
+ return;
2243
+ }
2244
+ rb_str_replace(str, JSON_empty_string);
2245
+ }
2246
+
2247
+ typedef struct JSON_ResumableParserStruct {
2248
+ JSON_ParserConfig config;
2249
+ JSON_ParserState state;
2250
+ rvalue_stack value_stack;
2251
+ json_frame_stack frames;
2252
+ VALUE buffer;
2253
+ size_t parsed_bytes;
2254
+ size_t incomplete_bytes;
2255
+ bool complete;
2256
+ bool in_use;
2257
+ } JSON_ResumableParser;
2258
+
2259
+ static void JSON_ResumableParser_mark(void *ptr)
2260
+ {
2261
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2262
+ JSON_ParserConfig_mark(&parser->config);
2263
+ rvalue_stack_mark(&parser->value_stack);
2264
+ rvalue_cache_mark(&parser->state.name_cache);
2265
+ rb_gc_mark(parser->buffer); // pin the buffer
2266
+ rb_gc_mark_movable(parser->state.parser);
2267
+ }
2268
+
2269
+ static void JSON_ResumableParser_free(void *ptr)
2270
+ {
2271
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2272
+ rvalue_stack_free_buffer(&parser->value_stack);
2273
+ json_frame_stack_free_buffer(&parser->frames);
2274
+ }
2275
+
2276
+ static size_t JSON_ResumableParser_memsize(const void *ptr)
2277
+ {
2278
+ const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr;
2279
+ size_t memsize = JSON_ParserConfig_memsize(&parser->config);
2280
+ memsize += rvalue_stack_memsize(&parser->value_stack);
2281
+ memsize += json_frame_stack_memsize(&parser->frames);
2282
+ #ifndef HAVE_RUBY_TYPED_EMBEDDABLE
2283
+ memsize += (
2284
+ sizeof(JSON_ResumableParser)
2285
+ - sizeof(JSON_ParserState)
2286
+ - sizeof(JSON_ParserConfig)
2287
+ - sizeof(rvalue_stack)
2288
+ - sizeof(json_frame_stack)
2289
+ );
2290
+ #endif
2291
+ return memsize;
2292
+ }
2293
+
2294
+ static void JSON_ResumableParser_compact(void *ptr)
2295
+ {
2296
+ JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr;
2297
+ JSON_ParserConfig_compact(&parser->config);
2298
+ rvalue_stack_compact(&parser->value_stack);
2299
+ rvalue_cache_compact(&parser->state.name_cache);
2300
+ parser->buffer = rb_gc_location(parser->buffer);
2301
+ parser->state.parser = rb_gc_location(parser->state.parser);
2302
+ }
2303
+
2304
+ static const rb_data_type_t JSON_ResumableParser_type = {
2305
+ .wrap_struct_name = "JSON::Ext::ResumableParser",
2306
+ .function = {
2307
+ JSON_ResumableParser_mark,
2308
+ JSON_ResumableParser_free,
2309
+ JSON_ResumableParser_memsize,
2310
+ JSON_ResumableParser_compact,
2311
+ },
2312
+ // RUBY_TYPED_WB_PROTECTED is deliberately not declared because
2313
+ // this is a superset of JSON_Parser_rvalue_stack_type, so we'd need
2314
+ // to trigger a lot of write barriers.
2315
+ .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE,
2316
+ };
2317
+
2318
+ static VALUE cResumableParser_allocate(VALUE klass)
2319
+ {
2320
+ JSON_ResumableParser *parser;
2321
+ VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
2322
+ parser->state.in_array++;
2323
+ parser->state.parser = obj;
2324
+ return obj;
2325
+ }
2326
+
2327
+ static inline JSON_ResumableParser *cResumableParser_get(VALUE self)
2328
+ {
2329
+ JSON_ResumableParser *parser;
2330
+ TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser);
2331
+ return parser;
2332
+ }
2333
+
2334
+ /*
2335
+ * call-seq: new(opts => {})
2336
+ *
2337
+ * Creates a new JSON::ResumableParser instance.
2338
+ *
2339
+ * Argument +opts+, if given, contains a \Hash of options for the parsing.
2340
+ * See {Parsing Options}[#module-JSON-label-Parsing+Options].
2341
+ *
2342
+ * A ResumableParser is able to parse partial documents and resume parsing later
2343
+ * when more of the document is provided:
2344
+ *
2345
+ * parser = JSON::ResumableParser.new
2346
+ * parser << '{"user": "george", "role": "ad'
2347
+ * parser.parse # => false
2348
+ * parser.eos? # => true
2349
+ * parser.partial_value # => { "user" => "george", "role" => nil }
2350
+ * parser.rest # => '"ad'
2351
+ *
2352
+ * parser << 'min" }[1, 2, 3]'
2353
+ * parser.parse # => true
2354
+ * parser.value # => { "user" => "george", "role" => "admin" }
2355
+ *
2356
+ * parser.parse # => true
2357
+ * parser.value # => [1, 2, 3]
2358
+ *
2359
+ * === Limitations
2360
+ *
2361
+ * While ResumableParser is able to parse streams of documents without any
2362
+ * explicit separators between them, it is highly recommended to separate documents
2363
+ * by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous.
2364
+ * When parsing a number, ResumableParser will not consider the number complete until something follows:
2365
+ *
2366
+ * parser << '123'
2367
+ * parser.parse # => false
2368
+ * parser << ' '
2369
+ * parser.parse # => true
2370
+ * parser.value # => 123
2371
+ *
2372
+ * === Security
2373
+ *
2374
+ * An incomplete document is buffered in full and there is no size limit, so when reading
2375
+ * from an untrusted source the caller is responsible for bounding how much data is fed.
2376
+ * For example:
2377
+ *
2378
+ * loop do
2379
+ * if parser.parsed_bytes > DOCUMENT_MAX_SIZE
2380
+ * raise "document too large"
2381
+ * end
2382
+ *
2383
+ * parser << read_chunk
2384
+ * while parser.parse
2385
+ * process(parser.value)
2386
+ * end
2387
+ * end
2388
+ */
2389
+ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self)
2390
+ {
2391
+ rb_check_frozen(self);
2392
+
2393
+ VALUE opts = Qfalse;
2394
+ rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts);
2395
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2396
+
2397
+ opts = argc > 0 ? argv[0] : Qnil;
2398
+ parser_config_init(&parser->config, opts, self, true);
2399
+
2400
+ return self;
2401
+ }
2402
+
2403
+ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock);
2404
+
2405
+ /*
2406
+ * call-seq: self << string -> self
2407
+ *
2408
+ * Appends the given string to the parser's buffer.
2409
+ */
2410
+ static VALUE cResumableParser_feed(VALUE self, VALUE str)
2411
+ {
2412
+ rb_check_frozen(self);
2413
+
2414
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2415
+
2416
+ str = convert_encoding(str);
2417
+ if (!RSTRING_LEN(str)) {
2418
+ return self;
2419
+ }
2420
+
2421
+ size_t offset = parser->state.cursor - parser->state.start;
2422
+ const size_t remaining = parser->state.end - parser->state.cursor;
2423
+
2424
+ if (!remaining) {
2425
+ if (parser->buffer) {
2426
+ json_str_clear(parser->buffer);
2427
+ }
2428
+ parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
2429
+ offset = 0;
2430
+ } else {
2431
+ JSON_ASSERT(parser->buffer);
2432
+
2433
+ const size_t size = parser->state.end - parser->state.start;
2434
+ const size_t consumed = size - remaining;
2435
+
2436
+ if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
2437
+ VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
2438
+ rb_enc_associate_index(new_buffer, utf8_encindex);
2439
+
2440
+ char *old_ptr = RSTRING_PTR(parser->buffer);
2441
+ memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
2442
+ rb_str_set_len(new_buffer, remaining);
2443
+ offset = 0;
2444
+ parser->buffer = new_buffer;
2445
+ } else if (consumed > (size / 2) && size >= 512) {
2446
+ rb_str_modify(parser->buffer);
2447
+ char *old_ptr = RSTRING_PTR(parser->buffer);
2448
+ memmove(old_ptr, old_ptr + consumed, remaining);
2449
+ rb_str_set_len(parser->buffer, remaining);
2450
+ offset = 0;
2451
+ }
2452
+ rb_str_append(parser->buffer, str);
2453
+ }
2454
+
2455
+ long len;
2456
+ const char *start;
2457
+ RSTRING_GETMEM(parser->buffer, start, len);
2458
+ parser->state.start = start;
2459
+ parser->state.end = start + len;
2460
+ parser->state.cursor = parser->state.start + offset;
2461
+
2462
+ return self;
2463
+ }
2464
+
2465
+ struct json_parse_any_args {
2466
+ JSON_ParserState *state;
2467
+ JSON_ParserConfig *config;
2468
+ VALUE parser;
2469
+ };
2470
+
2471
+ static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args))
2472
+ {
2473
+ struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
2474
+ return (VALUE)json_parse_any(args->state, args->config, true);
2475
+ }
2476
+
2477
+ static VALUE json_parse_any_resumable_safe(VALUE _args)
2478
+ {
2479
+ struct json_parse_any_args *args = (struct json_parse_any_args *)_args;
2480
+ VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args);
2481
+ return result == args->parser ? Qfalse : result;
2482
+ }
2483
+
2484
+ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock)
2485
+ {
2486
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2487
+
2488
+ if (parser->in_use) {
2489
+ rb_raise(rb_eArgError, "ResumableParser can't be used recursively");
2490
+ }
2491
+
2492
+ if (lock) {
2493
+ parser->in_use = true;
2494
+ }
2495
+
2496
+ // self may have moved, so we need to update all pointers
2497
+ // Investigate: We might be better off keeping JSON_ParserState on the stack
2498
+ // and only persist what we need.
2499
+ parser->state.value_stack = &parser->value_stack;
2500
+ parser->state.frames = &parser->frames;
2501
+
2502
+ return parser;
2503
+ }
2504
+
2505
+ /*
2506
+ * call-seq: parse -> true or false
2507
+ *
2508
+ * Attemps to parse a JSON document from the internal buffer.
2509
+ * Returns whether a complete document could be parsed.
2510
+ *
2511
+ * It does raise +JSON::ParserError+ when encountering invalid \JSON syntax.
2512
+ *
2513
+ * The parsed object can be retrieved by calling #value
2514
+ */
2515
+ static VALUE cResumableParser_parse(VALUE self)
2516
+ {
2517
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
2518
+
2519
+ if (parser->complete) {
2520
+ parser->parsed_bytes = 0;
2521
+ parser->incomplete_bytes = 0;
2522
+ parser->complete = false;
2523
+ }
2524
+
2525
+ if (!parser->buffer) {
2526
+ parser->in_use = false;
2527
+ return Qfalse;
2528
+ }
2529
+
2530
+ if (parser->frames.head == 0) {
2531
+ json_frame_stack_push(&parser->state, (json_frame){
2532
+ .type = JSON_FRAME_ROOT,
2533
+ .phase = JSON_PHASE_VALUE,
2534
+ });
2535
+ }
2536
+
2537
+ VALUE Vsource = parser->buffer; // Prevent compaction
2538
+
2539
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2540
+
2541
+ if (frame->phase == JSON_PHASE_DONE) {
2542
+ JSON_ASSERT(parser->value_stack.head == 1);
2543
+ JSON_ASSERT(parser->frames.head == 1);
2544
+
2545
+ frame->phase = JSON_PHASE_VALUE;
2546
+ rvalue_stack_pop(parser->state.value_stack, 1);
2547
+ }
2548
+
2549
+ struct json_parse_any_args args = {
2550
+ .state = &parser->state,
2551
+ .config = &parser->config,
2552
+ .parser = self,
2553
+ };
2554
+ int status;
2555
+ const char *initial_cursor = parser->state.cursor;
2556
+ parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);
2557
+
2558
+ if (status) {
2559
+ parser->complete = true; // a parse error is considered complete
2560
+ }
2561
+
2562
+ parser->parsed_bytes += parser->state.cursor - initial_cursor;
2563
+ parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;
2564
+
2565
+ json_eat_whitespace(&parser->state, &parser->config, false);
2566
+ if (eos(&parser->state)) {
2567
+ json_str_clear(parser->buffer);
2568
+ parser->buffer = Qfalse;
2569
+ }
2570
+ parser->in_use = false;
2571
+
2572
+ if (status) {
2573
+ rb_jump_tag(status); // reraise
2574
+ }
2575
+ RB_GC_GUARD(Vsource);
2576
+ return parser->complete ? Qtrue : Qfalse;
2577
+ }
2578
+
2579
+ /*
2580
+ * call-seq: value? -> true or false
2581
+ *
2582
+ * Returns whether a parsed value is available.
2583
+ */
2584
+ static VALUE cResumableParser_value_p(VALUE self)
2585
+ {
2586
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2587
+
2588
+ if (parser->value_stack.head > 0) {
2589
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2590
+ if (frame->phase == JSON_PHASE_DONE) {
2591
+ return Qtrue;
2592
+ }
2593
+ }
2594
+ return Qfalse;
2595
+ }
2596
+
2597
+ /*
2598
+ * call-seq: value -> object
2599
+ *
2600
+ * Returns and consume the last parsed value.
2601
+ * Raises ArgumentError if there is no parsed value or if it was already retrieved:
2602
+ * parser << '[1][2]'
2603
+ * parser.value # ArgumentError no ready value
2604
+ * parser.parse # => true
2605
+ * parser.value # => [1]
2606
+ * parser.value # ArgumentError no ready value
2607
+ */
2608
+ static VALUE cResumableParser_value(VALUE self)
2609
+ {
2610
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2611
+
2612
+ if (parser->frames.head > 0) {
2613
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2614
+
2615
+ if (frame->phase == JSON_PHASE_DONE) {
2616
+ VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
2617
+ rvalue_stack_pop(parser->state.value_stack, 1);
2618
+ json_frame_stack_pop(parser->state.frames);
2619
+ return result;
2620
+ }
2621
+ }
2622
+ rb_raise(rb_eArgError, "no ready value");
2623
+ }
2624
+
2625
+ /*
2626
+ * call-seq: clear -> self
2627
+ *
2628
+ * Entirely reset the parser state and buffer.
2629
+ */
2630
+ static VALUE cResumableParser_clear(VALUE self)
2631
+ {
2632
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
2633
+ parser->buffer = 0;
2634
+ parser->complete = true;
2635
+ parser->parsed_bytes = 0;
2636
+ parser->incomplete_bytes = 0;
2637
+ parser->frames.head = 0;
2638
+ parser->value_stack.head = 0;
2639
+ parser->state.name_cache.length = 0;
2640
+ parser->state.current_nesting = 0;
2641
+ parser->state.in_array = 1;
2642
+ parser->state.emitted_deprecations = 0;
2643
+ parser->state.start = parser->state.cursor = parser->state.end = NULL;
2644
+ return self;
2645
+ }
2646
+
2647
+ static VALUE cResumableParser_partial_value_body(VALUE self)
2648
+ {
2649
+ JSON_ResumableParser *original_parser = cResumableParser_get(self);
2650
+ JSON_ResumableParser parser = *original_parser;
2651
+
2652
+ parser.state.frames = &parser.frames;
2653
+ parser.state.value_stack = &parser.value_stack;
2654
+
2655
+ if (parser.value_stack.head == 0) {
2656
+ return Qnil;
2657
+ }
2658
+
2659
+ json_frame *frame = json_frame_stack_peek(parser.state.frames);
2660
+ long missing_object_value = 0;
2661
+ if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) {
2662
+ missing_object_value = 1;
2663
+ }
2664
+
2665
+ // Copy the value stack as we need to mutate it. The collapse loop folds each
2666
+ // open container by popping its entries and pushing the single result, so a
2667
+ // parent always reclaims its child's slot; head exceeds its live size by at
2668
+ // most one, either for the missing-value placeholder pushed below or for the
2669
+ // result of folding an empty innermost container. That one spare slot keeps
2670
+ // rvalue_stack_push from growing (reallocating) this ALLOCV buffer.
2671
+ long capa = parser.value_stack.head;
2672
+ parser.value_stack.capa = capa + 1;
2673
+ VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, parser.value_stack.capa);
2674
+ MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, capa);
2675
+ parser.value_stack.ptr = value_stack_buffer;
2676
+
2677
+ JSON_ParserState *state = &parser.state;
2678
+ JSON_ParserConfig *config = &parser.config;
2679
+
2680
+ if (missing_object_value) {
2681
+ rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack);
2682
+ }
2683
+
2684
+ VALUE partial_result = Qundef;
2685
+
2686
+ while (UNDEF_P(partial_result)) {
2687
+ frame = json_frame_stack_peek(state->frames);
2688
+
2689
+ switch (frame->type) {
2690
+ case JSON_FRAME_ROOT: {
2691
+ partial_result = *rvalue_stack_peek(state->value_stack, 1);
2692
+ break;
2693
+ }
2694
+
2695
+ case JSON_FRAME_ARRAY: {
2696
+ long count = json_frame_entry_count(frame, state->value_stack);
2697
+ json_push_value(state, config, json_decode_array(state, config, count));
2698
+ json_frame_stack_pop(state->frames);
2699
+
2700
+ break;
2701
+ }
2702
+
2703
+ case JSON_FRAME_OBJECT: {
2704
+ long count = json_frame_entry_count(frame, state->value_stack);
2705
+ json_push_value(state, config, json_decode_object(state, config, count));
2706
+ json_frame_stack_pop(state->frames);
2707
+ break;
2708
+ }
2709
+
2710
+ default: {
2711
+ JSON_UNREACHABLE_RETURN(Qundef);
2712
+ break;
2713
+ }
2714
+ }
2715
+ }
2716
+
2717
+ ALLOCV_END(tmpbuf);
2718
+ return partial_result;
2719
+ }
2720
+
2721
+ /*
2722
+ * call-seq: partial_value -> object
2723
+ *
2724
+ * Returns the Ruby objects parsed up to this point:
2725
+ * parser << '[1, [2, 3,'
2726
+ * parser.parse # => false
2727
+ * parser.value # ArgumentError no ready value
2728
+ * parser.partial_value # => [1, [2, 3]]
2729
+ */
2730
+ static VALUE cResumableParser_partial_value(VALUE self)
2731
+ {
2732
+ JSON_ResumableParser *parser = ResumableParser_acquire(self, true);
2733
+
2734
+ int status;
2735
+ VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
2736
+ parser->in_use = false;
2737
+ if (status) {
2738
+ rb_jump_tag(status);
2739
+ }
2740
+ return result;
2741
+ }
2742
+
2743
+ /*
2744
+ * call-seq: rest -> string
2745
+ *
2746
+ * Returns a string containing what remains to be parsed in the buffer
2747
+ * parser << '{ "message": "unterminated message'
2748
+ * parser.parse # => false
2749
+ * parser.rest # => '"unterminated message"'
2750
+ */
2751
+ static VALUE cResumableParser_rest(VALUE self)
2752
+ {
2753
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2754
+
2755
+ if (!parser->buffer) {
2756
+ return rb_utf8_str_new("", 0);
2757
+ }
2758
+
2759
+ size_t offset = parser->state.cursor - parser->state.start;
2760
+ const char *ptr;
2761
+ long len;
2762
+ RSTRING_GETMEM(parser->buffer, ptr, len);
2763
+ return rb_utf8_str_new(ptr + offset, len - offset);
2764
+ }
2765
+
2766
+ /*
2767
+ * call-seq: eos? -> true or false
2768
+ *
2769
+ * Returns whether the internal buffer has been entirely consumed.
2770
+ */
2771
+ static VALUE cResumableParser_eos_p(VALUE self)
2772
+ {
2773
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2774
+ return eos(&parser->state) ? Qtrue : Qfalse;
2775
+ }
2776
+
2777
+ /*
2778
+ * call-seq: partial_value? -> true or false
2779
+ *
2780
+ * Returns whether a document is currently under construction: an unclosed
2781
+ * container, a key awaiting its value, etc.
2782
+ *
2783
+ * It answers the same question as <tt>!partial_value.nil?</tt>, but as a
2784
+ * cheap predicate on the parser's internal state, without materializing the
2785
+ * partially parsed Ruby objects:
2786
+ * parser << '{"a":1,'
2787
+ * parser.parse # => false
2788
+ * parser.partial_value? # => true
2789
+ *
2790
+ * A fully parsed document whose value hasn't been retrieved yet is not under
2791
+ * construction: #value? returns true and #partial_value? returns false.
2792
+ */
2793
+ static VALUE cResumableParser_partial_value_p(VALUE self)
2794
+ {
2795
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2796
+
2797
+ // Mirror of #value?: values on the stack while the document isn't DONE
2798
+ // belong to a partially built document. A container whose first key or
2799
+ // element hasn't been parsed yet has no frame nor value registered (the
2800
+ // tokenizer rewinds to the container start on EOS), so that state is
2801
+ // observable through the buffer (#eos?/#rest) instead, keeping this
2802
+ // predicate consistent with #partial_value returning nil.
2803
+ if (parser->value_stack.head > 0) {
2804
+ json_frame *frame = json_frame_stack_peek(&parser->frames);
2805
+ if (frame->phase != JSON_PHASE_DONE) {
2806
+ return Qtrue;
2807
+ }
2808
+ }
2809
+ return Qfalse;
2810
+ }
2811
+
2812
+ /*
2813
+ * call-seq: parsed_bytes -> integer
2814
+ *
2815
+ * Returns the number of bytes parsed since the start of the current partial value.
2816
+ * This is intended to be used for securing against untrusted input:
2817
+ *
2818
+ * loop do
2819
+ * if parser.parsed_bytes > DOCUMENT_MAX_SIZE
2820
+ * raise "document too large"
2821
+ * end
2822
+ *
2823
+ * parser << read_chunk
2824
+ * while parser.parse
2825
+ * process(parser.value)
2826
+ * end
2827
+ * end
2828
+ */
2829
+ static VALUE cResumableParser_parsed_bytes(VALUE self)
2830
+ {
2831
+ JSON_ResumableParser *parser = cResumableParser_get(self);
2832
+ return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
2833
+ }
2834
+
1642
2835
  void Init_parser(void)
1643
2836
  {
1644
2837
  #ifdef HAVE_RB_EXT_RACTOR_SAFE
@@ -1650,30 +2843,53 @@ void Init_parser(void)
1650
2843
  mJSON = rb_define_module("JSON");
1651
2844
  VALUE mExt = rb_define_module_under(mJSON, "Ext");
1652
2845
  VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject);
2846
+
2847
+ rb_global_variable(&eParserError);
2848
+ eParserError = rb_path2class("JSON::ParserError");
2849
+
2850
+ rb_global_variable(&eNestingError);
1653
2851
  eNestingError = rb_path2class("JSON::NestingError");
1654
- rb_gc_register_mark_object(eNestingError);
2852
+
1655
2853
  rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate);
1656
- rb_define_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
2854
+ rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1);
1657
2855
  rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1);
1658
2856
 
1659
2857
  VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject);
1660
2858
  rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2);
1661
2859
 
2860
+ VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject);
2861
+ rb_define_alloc_func(cResumableParser, cResumableParser_allocate);
2862
+ rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1);
2863
+ rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1);
2864
+ rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0);
2865
+ rb_define_method(cResumableParser, "value", cResumableParser_value, 0);
2866
+ rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0);
2867
+ rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0);
2868
+ rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0);
2869
+ rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0);
2870
+ rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0);
2871
+ rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0);
2872
+ rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0);
2873
+
2874
+ rb_global_variable(&CNaN);
1662
2875
  CNaN = rb_const_get(mJSON, rb_intern("NaN"));
1663
- rb_gc_register_mark_object(CNaN);
1664
2876
 
2877
+ rb_global_variable(&CInfinity);
1665
2878
  CInfinity = rb_const_get(mJSON, rb_intern("Infinity"));
1666
- rb_gc_register_mark_object(CInfinity);
1667
2879
 
2880
+ rb_global_variable(&CMinusInfinity);
1668
2881
  CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity"));
1669
- rb_gc_register_mark_object(CMinusInfinity);
1670
2882
 
1671
2883
  rb_global_variable(&Encoding_UTF_8);
1672
2884
  Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
1673
2885
 
2886
+ rb_global_variable(&JSON_empty_string);
2887
+ JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0));
2888
+
1674
2889
  sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
1675
2890
  sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
1676
2891
  sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma"));
2892
+ sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
1677
2893
  sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters"));
1678
2894
  sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape"));
1679
2895
  sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));
@@ -1684,8 +2900,12 @@ void Init_parser(void)
1684
2900
 
1685
2901
  i_new = rb_intern("new");
1686
2902
  i_try_convert = rb_intern("try_convert");
2903
+ #ifndef HAVE_RB_STR_TO_INTERNED_STR
1687
2904
  i_uminus = rb_intern("-@");
2905
+ #endif
1688
2906
  i_encode = rb_intern("encode");
2907
+ i_at_line = rb_intern("@line");
2908
+ i_at_column = rb_intern("@column");
1689
2909
 
1690
2910
  binary_encindex = rb_ascii8bit_encindex();
1691
2911
  utf8_encindex = rb_utf8_encindex();