oj 3.13.14 → 3.13.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +32 -0
  3. data/README.md +2 -0
  4. data/ext/oj/buf.h +4 -0
  5. data/ext/oj/compat.c +10 -10
  6. data/ext/oj/custom.c +34 -53
  7. data/ext/oj/dump.c +24 -13
  8. data/ext/oj/dump_compat.c +5 -10
  9. data/ext/oj/dump_object.c +5 -60
  10. data/ext/oj/dump_strict.c +5 -5
  11. data/ext/oj/extconf.rb +5 -4
  12. data/ext/oj/fast.c +15 -13
  13. data/ext/oj/intern.c +6 -9
  14. data/ext/oj/introspect.c +96 -0
  15. data/ext/oj/mimic_json.c +18 -8
  16. data/ext/oj/object.c +42 -41
  17. data/ext/oj/oj.c +27 -4
  18. data/ext/oj/oj.h +4 -1
  19. data/ext/oj/parse.c +111 -76
  20. data/ext/oj/parse.h +2 -0
  21. data/ext/oj/parser.c +61 -4
  22. data/ext/oj/parser.h +12 -0
  23. data/ext/oj/rails.c +5 -10
  24. data/ext/oj/saj2.c +333 -85
  25. data/ext/oj/saj2.h +23 -0
  26. data/ext/oj/sparse.c +4 -0
  27. data/ext/oj/strict.c +13 -13
  28. data/ext/oj/usual.c +82 -129
  29. data/ext/oj/usual.h +68 -0
  30. data/ext/oj/val_stack.c +1 -1
  31. data/ext/oj/validate.c +21 -26
  32. data/ext/oj/wab.c +15 -20
  33. data/lib/oj/saj.rb +20 -6
  34. data/lib/oj/state.rb +1 -1
  35. data/lib/oj/version.rb +1 -1
  36. data/pages/Compatibility.md +1 -1
  37. data/test/bar.rb +3 -1
  38. data/test/helper.rb +8 -2
  39. data/test/json_gem/json_generator_test.rb +3 -4
  40. data/test/json_gem/json_parser_test.rb +8 -1
  41. data/test/json_gem/test_helper.rb +7 -3
  42. data/test/test_compat.rb +25 -0
  43. data/test/test_custom.rb +13 -2
  44. data/test/test_file.rb +23 -7
  45. data/test/test_gc.rb +11 -0
  46. data/test/test_object.rb +3 -10
  47. data/test/test_parser.rb +3 -19
  48. data/test/test_parser_debug.rb +27 -0
  49. data/test/test_parser_saj.rb +92 -2
  50. data/test/test_scp.rb +2 -4
  51. data/test/test_strict.rb +2 -0
  52. data/test/test_various.rb +8 -3
  53. data/test/test_wab.rb +2 -0
  54. data/test/tests.rb +9 -0
  55. data/test/tests_mimic.rb +9 -0
  56. data/test/tests_mimic_addition.rb +9 -0
  57. metadata +7 -107
data/ext/oj/oj.c CHANGED
@@ -32,6 +32,7 @@ ID oj_array_append_id;
32
32
  ID oj_array_end_id;
33
33
  ID oj_array_start_id;
34
34
  ID oj_as_json_id;
35
+ ID oj_at_id;
35
36
  ID oj_begin_id;
36
37
  ID oj_bigdecimal_id;
37
38
  ID oj_end_id;
@@ -45,7 +46,6 @@ ID oj_hash_key_id;
45
46
  ID oj_hash_set_id;
46
47
  ID oj_hash_start_id;
47
48
  ID oj_iconv_id;
48
- ID oj_instance_variables_id;
49
49
  ID oj_json_create_id;
50
50
  ID oj_length_id;
51
51
  ID oj_new_id;
@@ -90,7 +90,9 @@ VALUE oj_array_class_sym;
90
90
  VALUE oj_create_additions_sym;
91
91
  VALUE oj_decimal_class_sym;
92
92
  VALUE oj_hash_class_sym;
93
+ VALUE oj_in_sym;
93
94
  VALUE oj_indent_sym;
95
+ VALUE oj_nanosecond_sym;
94
96
  VALUE oj_object_class_sym;
95
97
  VALUE oj_quirks_mode_sym;
96
98
  VALUE oj_safe_sym;
@@ -137,6 +139,7 @@ static VALUE rails_sym;
137
139
  static VALUE raise_sym;
138
140
  static VALUE ruby_sym;
139
141
  static VALUE sec_prec_sym;
142
+ static VALUE slash_sym;
140
143
  static VALUE strict_sym;
141
144
  static VALUE symbol_keys_sym;
142
145
  static VALUE time_format_sym;
@@ -405,6 +408,7 @@ static VALUE get_def_opts(VALUE self) {
405
408
  switch (oj_default_options.escape_mode) {
406
409
  case NLEsc: rb_hash_aset(opts, escape_mode_sym, newline_sym); break;
407
410
  case JSONEsc: rb_hash_aset(opts, escape_mode_sym, json_sym); break;
411
+ case SlashEsc: rb_hash_aset(opts, escape_mode_sym, slash_sym); break;
408
412
  case XSSEsc: rb_hash_aset(opts, escape_mode_sym, xss_safe_sym); break;
409
413
  case ASCIIEsc: rb_hash_aset(opts, escape_mode_sym, ascii_sym); break;
410
414
  case JXEsc: rb_hash_aset(opts, escape_mode_sym, unicode_xss_sym); break;
@@ -734,6 +738,8 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
734
738
  copts->escape_mode = NLEsc;
735
739
  } else if (json_sym == v) {
736
740
  copts->escape_mode = JSONEsc;
741
+ } else if (slash_sym == v) {
742
+ copts->escape_mode = SlashEsc;
737
743
  } else if (xss_safe_sym == v) {
738
744
  copts->escape_mode = XSSEsc;
739
745
  } else if (ascii_sym == v) {
@@ -926,7 +932,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
926
932
  if (Qnil == v) {
927
933
  return ST_CONTINUE;
928
934
  }
929
- if (TYPE(v) == T_STRUCT && rb_obj_class(v) == rb_cRange) {
935
+ if (rb_obj_class(v) == rb_cRange) {
930
936
  VALUE min = rb_funcall(v, oj_begin_id, 0);
931
937
  VALUE max = rb_funcall(v, oj_end_id, 0);
932
938
 
@@ -1148,7 +1154,17 @@ static VALUE load_file(int argc, VALUE *argv, VALUE self) {
1148
1154
  }
1149
1155
  }
1150
1156
  path = StringValuePtr(*argv);
1151
- if (0 == (fd = open(path, O_RDONLY))) {
1157
+ #ifdef _WIN32
1158
+ {
1159
+ WCHAR *wide_path;
1160
+ wide_path = rb_w32_mbstr_to_wstr(CP_UTF8, path, -1, NULL);
1161
+ fd = rb_w32_wopen(wide_path, O_RDONLY);
1162
+ free(wide_path);
1163
+ }
1164
+ #else
1165
+ fd = open(path, O_RDONLY);
1166
+ #endif
1167
+ if (0 == fd) {
1152
1168
  rb_raise(rb_eIOError, "%s", strerror(errno));
1153
1169
  }
1154
1170
  switch (mode) {
@@ -1801,6 +1817,7 @@ void Init_oj(void) {
1801
1817
  oj_array_end_id = rb_intern("array_end");
1802
1818
  oj_array_start_id = rb_intern("array_start");
1803
1819
  oj_as_json_id = rb_intern("as_json");
1820
+ oj_at_id = rb_intern("at");
1804
1821
  oj_begin_id = rb_intern("begin");
1805
1822
  oj_bigdecimal_id = rb_intern("BigDecimal");
1806
1823
  oj_end_id = rb_intern("end");
@@ -1814,7 +1831,6 @@ void Init_oj(void) {
1814
1831
  oj_hash_set_id = rb_intern("hash_set");
1815
1832
  oj_hash_start_id = rb_intern("hash_start");
1816
1833
  oj_iconv_id = rb_intern("iconv");
1817
- oj_instance_variables_id = rb_intern("instance_variables");
1818
1834
  oj_json_create_id = rb_intern("json_create");
1819
1835
  oj_length_id = rb_intern("length");
1820
1836
  oj_new_id = rb_intern("new");
@@ -1949,10 +1965,14 @@ void Init_oj(void) {
1949
1965
  rb_gc_register_address(&oj_decimal_class_sym);
1950
1966
  oj_hash_class_sym = ID2SYM(rb_intern("hash_class"));
1951
1967
  rb_gc_register_address(&oj_hash_class_sym);
1968
+ oj_in_sym = ID2SYM(rb_intern("in"));
1969
+ rb_gc_register_address(&oj_in_sym);
1952
1970
  oj_indent_sym = ID2SYM(rb_intern("indent"));
1953
1971
  rb_gc_register_address(&oj_indent_sym);
1954
1972
  oj_max_nesting_sym = ID2SYM(rb_intern("max_nesting"));
1955
1973
  rb_gc_register_address(&oj_max_nesting_sym);
1974
+ oj_nanosecond_sym = ID2SYM(rb_intern("nanosecond"));
1975
+ rb_gc_register_address(&oj_nanosecond_sym);
1956
1976
  oj_object_class_sym = ID2SYM(rb_intern("object_class"));
1957
1977
  rb_gc_register_address(&oj_object_class_sym);
1958
1978
  oj_object_nl_sym = ID2SYM(rb_intern("object_nl"));
@@ -1977,6 +1997,8 @@ void Init_oj(void) {
1977
1997
  rb_gc_register_address(&ruby_sym);
1978
1998
  sec_prec_sym = ID2SYM(rb_intern("second_precision"));
1979
1999
  rb_gc_register_address(&sec_prec_sym);
2000
+ slash_sym = ID2SYM(rb_intern("slash"));
2001
+ rb_gc_register_address(&slash_sym);
1980
2002
  strict_sym = ID2SYM(rb_intern("strict"));
1981
2003
  rb_gc_register_address(&strict_sym);
1982
2004
  symbol_keys_sym = ID2SYM(rb_intern("symbol_keys"));
@@ -2029,4 +2051,5 @@ void Init_oj(void) {
2029
2051
  oj_init_doc();
2030
2052
 
2031
2053
  oj_parser_init();
2054
+ oj_scanner_init();
2032
2055
  }
data/ext/oj/oj.h CHANGED
@@ -66,6 +66,7 @@ typedef enum { UnixTime = 'u', UnixZTime = 'z', XmlTime = 'x', RubyTime = 'r' }
66
66
  typedef enum {
67
67
  NLEsc = 'n',
68
68
  JSONEsc = 'j',
69
+ SlashEsc = 's',
69
70
  XSSEsc = 'x',
70
71
  ASCIIEsc = 'a',
71
72
  JXEsc = 'g', // json gem
@@ -316,7 +317,9 @@ extern VALUE oj_ascii_only_sym;
316
317
  extern VALUE oj_create_additions_sym;
317
318
  extern VALUE oj_decimal_class_sym;
318
319
  extern VALUE oj_hash_class_sym;
320
+ extern VALUE oj_in_sym;
319
321
  extern VALUE oj_indent_sym;
322
+ extern VALUE oj_nanosecond_sym;
320
323
  extern VALUE oj_max_nesting_sym;
321
324
  extern VALUE oj_object_class_sym;
322
325
  extern VALUE oj_object_nl_sym;
@@ -333,6 +336,7 @@ extern ID oj_array_append_id;
333
336
  extern ID oj_array_end_id;
334
337
  extern ID oj_array_start_id;
335
338
  extern ID oj_as_json_id;
339
+ extern ID oj_at_id;
336
340
  extern ID oj_begin_id;
337
341
  extern ID oj_bigdecimal_id;
338
342
  extern ID oj_end_id;
@@ -346,7 +350,6 @@ extern ID oj_hash_key_id;
346
350
  extern ID oj_hash_set_id;
347
351
  extern ID oj_hash_start_id;
348
352
  extern ID oj_iconv_id;
349
- extern ID oj_instance_variables_id;
350
353
  extern ID oj_json_create_id;
351
354
  extern ID oj_length_id;
352
355
  extern ID oj_new_id;
data/ext/oj/parse.c CHANGED
@@ -183,6 +183,42 @@ static void unicode_to_chars(ParseInfo pi, Buf buf, uint32_t code) {
183
183
  }
184
184
  }
185
185
 
186
+ static inline const char *scan_string_noSIMD(const char *str, const char *end) {
187
+ for (; '"' != *str; str++) {
188
+ if (end <= str || '\0' == *str || '\\' == *str) {
189
+ break;
190
+ }
191
+ }
192
+ return str;
193
+ }
194
+
195
+ #ifdef OJ_USE_SSE4_2
196
+ static inline const char *scan_string_SIMD(const char *str, const char *end) {
197
+ static const char chars[16] = "\x00\\\"";
198
+ const __m128i terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
199
+ const char *_end = (const char *)(end - 16);
200
+
201
+ for (; str <= _end; str += 16) {
202
+ const __m128i string = _mm_loadu_si128((const __m128i *)str);
203
+ const int r = _mm_cmpestri(terminate, 3, string, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
204
+ if (r != 16) {
205
+ str = (char*)(str + r);
206
+ return str;
207
+ }
208
+ }
209
+
210
+ return scan_string_noSIMD(str, end);
211
+ }
212
+ #endif
213
+
214
+ static const char *(*scan_func) (const char *str, const char *end) = scan_string_noSIMD;
215
+
216
+ void oj_scanner_init(void) {
217
+ #ifdef OJ_USE_SSE4_2
218
+ scan_func = scan_string_SIMD;
219
+ #endif
220
+ }
221
+
186
222
  // entered at /
187
223
  static void read_escaped_str(ParseInfo pi, const char *start) {
188
224
  struct _buf buf;
@@ -192,11 +228,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
192
228
  Val parent = stack_peek(&pi->stack);
193
229
 
194
230
  buf_init(&buf);
195
- if (0 < cnt) {
196
- buf_append_string(&buf, start, cnt);
197
- }
198
- for (s = pi->cur; '"' != *s; s++) {
199
- if (s >= pi->end) {
231
+ buf_append_string(&buf, start, cnt);
232
+
233
+ for (s = pi->cur; '"' != *s;) {
234
+ const char *scanned = scan_func(s, pi->end);
235
+ if (scanned >= pi->end) {
200
236
  oj_set_error_at(pi,
201
237
  oj_parse_error_class,
202
238
  __FILE__,
@@ -204,7 +240,12 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
204
240
  "quoted string not terminated");
205
241
  buf_cleanup(&buf);
206
242
  return;
207
- } else if ('\\' == *s) {
243
+ }
244
+
245
+ buf_append_string(&buf, s, (size_t)(scanned - s));
246
+ s = scanned;
247
+
248
+ if ('\\' == *s) {
208
249
  s++;
209
250
  switch (*s) {
210
251
  case 'n': buf_append(&buf, '\n'); break;
@@ -273,8 +314,7 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
273
314
  buf_cleanup(&buf);
274
315
  return;
275
316
  }
276
- } else {
277
- buf_append(&buf, *s);
317
+ s++;
278
318
  }
279
319
  }
280
320
  if (0 == parent) {
@@ -331,22 +371,24 @@ static void read_str(ParseInfo pi) {
331
371
  const char *str = pi->cur;
332
372
  Val parent = stack_peek(&pi->stack);
333
373
 
334
- for (; '"' != *pi->cur; pi->cur++) {
335
- if (pi->end <= pi->cur) {
336
- oj_set_error_at(pi,
337
- oj_parse_error_class,
338
- __FILE__,
339
- __LINE__,
340
- "quoted string not terminated");
341
- return;
342
- } else if ('\0' == *pi->cur) {
343
- oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "NULL byte in string");
344
- return;
345
- } else if ('\\' == *pi->cur) {
346
- read_escaped_str(pi, str);
347
- return;
348
- }
374
+ pi->cur = scan_func(pi->cur, pi->end);
375
+ if (RB_UNLIKELY(pi->end <= pi->cur)) {
376
+ oj_set_error_at(pi,
377
+ oj_parse_error_class,
378
+ __FILE__,
379
+ __LINE__,
380
+ "quoted string not terminated");
381
+ return;
349
382
  }
383
+ if (RB_UNLIKELY('\0' == *pi->cur)) {
384
+ oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "NULL byte in string");
385
+ return;
386
+ }
387
+ if ('\\' == *pi->cur) {
388
+ read_escaped_str(pi, str);
389
+ return;
390
+ }
391
+
350
392
  if (0 == parent) { // simple add
351
393
  pi->add_cstr(pi, str, pi->cur - str, str);
352
394
  } else {
@@ -459,33 +501,31 @@ static void read_num(ParseInfo pi) {
459
501
  int dec_cnt = 0;
460
502
  bool zero1 = false;
461
503
 
504
+ // Skip leading zeros.
505
+ for (; '0' == *pi->cur; pi->cur++) {
506
+ zero1 = true;
507
+ }
508
+
462
509
  for (; '0' <= *pi->cur && *pi->cur <= '9'; pi->cur++) {
463
- if (0 == ni.i && '0' == *pi->cur) {
464
- zero1 = true;
465
- }
466
- if (0 < ni.i) {
467
- dec_cnt++;
468
- }
469
- if (!ni.big) {
470
- int d = (*pi->cur - '0');
510
+ int d = (*pi->cur - '0');
471
511
 
472
- if (0 < d) {
473
- if (zero1 && CompatMode == pi->options.mode) {
474
- oj_set_error_at(pi,
475
- oj_parse_error_class,
476
- __FILE__,
477
- __LINE__,
478
- "not a number");
479
- return;
480
- }
481
- zero1 = false;
482
- }
483
- ni.i = ni.i * 10 + d;
484
- if (INT64_MAX <= ni.i || DEC_MAX < dec_cnt) {
485
- ni.big = 1;
486
- }
512
+ if (RB_LIKELY(0 != ni.i)) {
513
+ dec_cnt++;
487
514
  }
515
+ ni.i = ni.i * 10 + d;
488
516
  }
517
+ if (RB_UNLIKELY(0 != ni.i && zero1 && CompatMode == pi->options.mode)) {
518
+ oj_set_error_at(pi,
519
+ oj_parse_error_class,
520
+ __FILE__,
521
+ __LINE__,
522
+ "not a number");
523
+ return;
524
+ }
525
+ if (INT64_MAX <= ni.i || DEC_MAX < dec_cnt) {
526
+ ni.big = true;
527
+ }
528
+
489
529
  if ('.' == *pi->cur) {
490
530
  pi->cur++;
491
531
  // A trailing . is not a valid decimal but if encountered allow it
@@ -505,25 +545,20 @@ static void read_num(ParseInfo pi) {
505
545
  for (; '0' <= *pi->cur && *pi->cur <= '9'; pi->cur++) {
506
546
  int d = (*pi->cur - '0');
507
547
 
508
- if (0 < ni.num || 0 < ni.i) {
548
+ if (RB_LIKELY(0 != ni.num || 0 != ni.i)) {
509
549
  dec_cnt++;
510
550
  }
511
- if (INT64_MAX <= ni.div) {
512
- if (!ni.no_big) {
513
- ni.big = true;
514
- }
515
- } else {
516
- ni.num = ni.num * 10 + d;
517
- ni.div *= 10;
518
- ni.di++;
519
- if (INT64_MAX <= ni.div || DEC_MAX < dec_cnt) {
520
- if (!ni.no_big) {
521
- ni.big = true;
522
- }
523
- }
524
- }
551
+ ni.num = ni.num * 10 + d;
552
+ ni.div *= 10;
553
+ ni.di++;
554
+ }
555
+ }
556
+ if (INT64_MAX <= ni.div || DEC_MAX < dec_cnt) {
557
+ if (!ni.no_big) {
558
+ ni.big = true;
525
559
  }
526
560
  }
561
+
527
562
  if ('e' == *pi->cur || 'E' == *pi->cur) {
528
563
  int eneg = 0;
529
564
 
@@ -596,7 +631,7 @@ static void read_num(ParseInfo pi) {
596
631
  }
597
632
 
598
633
  static void array_start(ParseInfo pi) {
599
- volatile VALUE v = pi->start_array(pi);
634
+ VALUE v = pi->start_array(pi);
600
635
 
601
636
  stack_push(&pi->stack, v, NEXT_ARRAY_NEW);
602
637
  }
@@ -620,13 +655,13 @@ static void array_end(ParseInfo pi) {
620
655
  }
621
656
 
622
657
  static void hash_start(ParseInfo pi) {
623
- volatile VALUE v = pi->start_hash(pi);
658
+ VALUE v = pi->start_hash(pi);
624
659
 
625
660
  stack_push(&pi->stack, v, NEXT_HASH_NEW);
626
661
  }
627
662
 
628
663
  static void hash_end(ParseInfo pi) {
629
- volatile Val hash = stack_peek(&pi->stack);
664
+ Val hash = stack_peek(&pi->stack);
630
665
 
631
666
  // leave hash on stack until just before
632
667
  if (0 == hash) {
@@ -813,7 +848,7 @@ static long double exp_plus[] = {
813
848
 
814
849
  VALUE
815
850
  oj_num_as_value(NumInfo ni) {
816
- volatile VALUE rnum = Qnil;
851
+ VALUE rnum = Qnil;
817
852
 
818
853
  if (ni->infinity) {
819
854
  if (ni->neg) {
@@ -848,7 +883,7 @@ oj_num_as_value(NumInfo ni) {
848
883
  }
849
884
  } else { // decimal
850
885
  if (ni->big) {
851
- volatile VALUE bd = rb_str_new(ni->str, ni->len);
886
+ VALUE bd = rb_str_new(ni->str, ni->len);
852
887
 
853
888
  rnum = rb_rescue2(parse_big_decimal, bd, rescue_big_decimal, bd, rb_eException, 0);
854
889
  if (ni->no_big) {
@@ -876,7 +911,7 @@ oj_num_as_value(NumInfo ni) {
876
911
  }
877
912
  rnum = rb_float_new((double)ld);
878
913
  } else if (RubyDec == ni->bigdec_load) {
879
- volatile VALUE sv = rb_str_new(ni->str, ni->len);
914
+ VALUE sv = rb_str_new(ni->str, ni->len);
880
915
 
881
916
  rnum = rb_funcall(sv, rb_intern("to_f"), 0);
882
917
  } else {
@@ -971,7 +1006,7 @@ static VALUE protect_parse(VALUE pip) {
971
1006
 
972
1007
  extern int oj_utf8_index;
973
1008
 
974
- static void oj_pi_set_input_str(ParseInfo pi, volatile VALUE *inputp) {
1009
+ static void oj_pi_set_input_str(ParseInfo pi, VALUE *inputp) {
975
1010
  int idx = RB_ENCODING_GET(*inputp);
976
1011
 
977
1012
  if (oj_utf8_encoding_index != idx) {
@@ -984,12 +1019,12 @@ static void oj_pi_set_input_str(ParseInfo pi, volatile VALUE *inputp) {
984
1019
 
985
1020
  VALUE
986
1021
  oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk) {
987
- char * buf = 0;
988
- volatile VALUE input;
989
- volatile VALUE wrapped_stack;
990
- volatile VALUE result = Qnil;
991
- int line = 0;
992
- int free_json = 0;
1022
+ char * buf = 0;
1023
+ VALUE input;
1024
+ VALUE wrapped_stack;
1025
+ VALUE result = Qnil;
1026
+ int line = 0;
1027
+ int free_json = 0;
993
1028
 
994
1029
  if (argc < 1) {
995
1030
  rb_raise(rb_eArgError, "Wrong number of arguments to parse.");
@@ -1025,8 +1060,8 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
1025
1060
  rb_raise(rb_eTypeError, "Nil is not a valid JSON source.");
1026
1061
  }
1027
1062
  } else {
1028
- VALUE clas = rb_obj_class(input);
1029
- volatile VALUE s;
1063
+ VALUE clas = rb_obj_class(input);
1064
+ VALUE s;
1030
1065
 
1031
1066
  if (oj_stringio_class == clas) {
1032
1067
  s = rb_funcall2(input, oj_string_id, 0, 0);
data/ext/oj/parse.h CHANGED
@@ -97,6 +97,8 @@ static inline void parse_info_init(ParseInfo pi) {
97
97
  memset(pi, 0, sizeof(struct _parseInfo));
98
98
  }
99
99
 
100
+ extern void oj_scanner_init(void);
101
+
100
102
  static inline bool empty_ok(Options options) {
101
103
  switch (options->mode) {
102
104
  case ObjectMode:
data/ext/oj/parser.c CHANGED
@@ -533,6 +533,7 @@ static void calc_num(ojParser p) {
533
533
  // nothing to do
534
534
  break;
535
535
  }
536
+ p->type = OJ_NONE;
536
537
  }
537
538
 
538
539
  static void big_change(ojParser p) {
@@ -598,6 +599,8 @@ static void parse(ojParser p, const byte *json) {
598
599
  const byte *b = json;
599
600
  int i;
600
601
 
602
+ p->line = 1;
603
+ p->col = -1;
601
604
  #if DEBUG
602
605
  printf("*** parse - mode: %c %s\n", p->map[256], (const char *)json);
603
606
  #endif
@@ -652,6 +655,7 @@ static void parse(ojParser p, const byte *json) {
652
655
  }
653
656
  buf_append_string(&p->buf, (const char *)start, b - start);
654
657
  if ('"' == *b) {
658
+ p->cur = b - json;
655
659
  p->funcs[p->stack[p->depth]].add_str(p);
656
660
  p->map = (0 == p->depth) ? value_map : after_map;
657
661
  break;
@@ -661,12 +665,14 @@ static void parse(ojParser p, const byte *json) {
661
665
  p->next_map = (0 == p->depth) ? value_map : after_map;
662
666
  break;
663
667
  case OPEN_OBJECT:
668
+ p->cur = b - json;
664
669
  p->funcs[p->stack[p->depth]].open_object(p);
665
670
  p->depth++;
666
671
  p->stack[p->depth] = OBJECT_FUN;
667
672
  p->map = key1_map;
668
673
  break;
669
674
  case NUM_CLOSE_OBJECT:
675
+ p->cur = b - json;
670
676
  calc_num(p);
671
677
  // flow through
672
678
  case CLOSE_OBJECT:
@@ -677,15 +683,18 @@ static void parse(ojParser p, const byte *json) {
677
683
  return;
678
684
  }
679
685
  p->depth--;
686
+ p->cur = b - json;
680
687
  p->funcs[p->stack[p->depth]].close_object(p);
681
688
  break;
682
689
  case OPEN_ARRAY:
690
+ p->cur = b - json;
683
691
  p->funcs[p->stack[p->depth]].open_array(p);
684
692
  p->depth++;
685
693
  p->stack[p->depth] = ARRAY_FUN;
686
694
  p->map = value_map;
687
695
  break;
688
696
  case NUM_CLOSE_ARRAY:
697
+ p->cur = b - json;
689
698
  calc_num(p);
690
699
  // flow through
691
700
  case CLOSE_ARRAY:
@@ -696,9 +705,11 @@ static void parse(ojParser p, const byte *json) {
696
705
  return;
697
706
  }
698
707
  p->depth--;
708
+ p->cur = b - json;
699
709
  p->funcs[p->stack[p->depth]].close_array(p);
700
710
  break;
701
711
  case NUM_COMMA:
712
+ p->cur = b - json;
702
713
  calc_num(p);
703
714
  if (0 < p->depth && OBJECT_FUN == p->stack[p->depth]) {
704
715
  p->map = key_map;
@@ -860,8 +871,14 @@ static void parse(ojParser p, const byte *json) {
860
871
  b--;
861
872
  p->map = big_exp_map;
862
873
  break;
863
- case NUM_SPC: calc_num(p); break;
864
- case NUM_NEWLINE: calc_num(p); b++;
874
+ case NUM_SPC:
875
+ p->cur = b - json;
876
+ calc_num(p);
877
+ break;
878
+ case NUM_NEWLINE:
879
+ p->cur = b - json;
880
+ calc_num(p);
881
+ b++;
865
882
  #ifdef SPACE_JUMP
866
883
  // for (uint32_t *sj = (uint32_t*)b; 0x20202020 == *sj; sj++) { b += 4; }
867
884
  for (uint16_t *sj = (uint16_t *)b; 0x2020 == *sj; sj++) {
@@ -882,6 +899,7 @@ static void parse(ojParser p, const byte *json) {
882
899
  buf_append_string(&p->buf, (const char *)start, b - start);
883
900
  }
884
901
  if ('"' == *b) {
902
+ p->cur = b - json;
885
903
  p->funcs[p->stack[p->depth]].add_str(p);
886
904
  p->map = p->next_map;
887
905
  break;
@@ -890,6 +908,7 @@ static void parse(ojParser p, const byte *json) {
890
908
  break;
891
909
  case STR_SLASH: p->map = esc_map; break;
892
910
  case STR_QUOTE:
911
+ p->cur = b - json;
893
912
  p->funcs[p->stack[p->depth]].add_str(p);
894
913
  p->map = p->next_map;
895
914
  break;
@@ -967,6 +986,7 @@ static void parse(ojParser p, const byte *json) {
967
986
  case VAL_NULL:
968
987
  if ('u' == b[1] && 'l' == b[2] && 'l' == b[3]) {
969
988
  b += 3;
989
+ p->cur = b - json;
970
990
  p->funcs[p->stack[p->depth]].add_null(p);
971
991
  p->map = (0 == p->depth) ? value_map : after_map;
972
992
  break;
@@ -992,6 +1012,7 @@ static void parse(ojParser p, const byte *json) {
992
1012
  case VAL_TRUE:
993
1013
  if ('r' == b[1] && 'u' == b[2] && 'e' == b[3]) {
994
1014
  b += 3;
1015
+ p->cur = b - json;
995
1016
  p->funcs[p->stack[p->depth]].add_true(p);
996
1017
  p->map = (0 == p->depth) ? value_map : after_map;
997
1018
  break;
@@ -1017,6 +1038,7 @@ static void parse(ojParser p, const byte *json) {
1017
1038
  case VAL_FALSE:
1018
1039
  if ('a' == b[1] && 'l' == b[2] && 's' == b[3] && 'e' == b[4]) {
1019
1040
  b += 4;
1041
+ p->cur = b - json;
1020
1042
  p->funcs[p->stack[p->depth]].add_false(p);
1021
1043
  p->map = (0 == p->depth) ? value_map : after_map;
1022
1044
  break;
@@ -1050,6 +1072,7 @@ static void parse(ojParser p, const byte *json) {
1050
1072
  parse_error(p, "expected null");
1051
1073
  return;
1052
1074
  }
1075
+ p->cur = b - json;
1053
1076
  p->funcs[p->stack[p->depth]].add_null(p);
1054
1077
  p->map = (0 == p->depth) ? value_map : after_map;
1055
1078
  }
@@ -1061,6 +1084,7 @@ static void parse(ojParser p, const byte *json) {
1061
1084
  parse_error(p, "expected false");
1062
1085
  return;
1063
1086
  }
1087
+ p->cur = b - json;
1064
1088
  p->funcs[p->stack[p->depth]].add_false(p);
1065
1089
  p->map = (0 == p->depth) ? value_map : after_map;
1066
1090
  }
@@ -1072,6 +1096,7 @@ static void parse(ojParser p, const byte *json) {
1072
1096
  parse_error(p, "expected true");
1073
1097
  return;
1074
1098
  }
1099
+ p->cur = b - json;
1075
1100
  p->funcs[p->stack[p->depth]].add_true(p);
1076
1101
  p->map = (0 == p->depth) ? value_map : after_map;
1077
1102
  }
@@ -1089,6 +1114,9 @@ static void parse(ojParser p, const byte *json) {
1089
1114
  p->map = trail_map;
1090
1115
  }
1091
1116
  }
1117
+ if (0 < p->depth) {
1118
+ parse_error(p, "parse error, not closed");
1119
+ }
1092
1120
  if (0 == p->depth) {
1093
1121
  switch (p->map[256]) {
1094
1122
  case '0':
@@ -1099,7 +1127,10 @@ static void parse(ojParser p, const byte *json) {
1099
1127
  case 'D':
1100
1128
  case 'g':
1101
1129
  case 'B':
1102
- case 'Y': calc_num(p); break;
1130
+ case 'Y':
1131
+ p->cur = b - json;
1132
+ calc_num(p);
1133
+ break;
1103
1134
  }
1104
1135
  }
1105
1136
  return;
@@ -1223,6 +1254,32 @@ static VALUE parser_new(int argc, VALUE *argv, VALUE self) {
1223
1254
  return Data_Wrap_Struct(parser_class, parser_mark, parser_free, p);
1224
1255
  }
1225
1256
 
1257
+ // Create a new parser without setting the delegate. The parser is
1258
+ // wrapped. The parser is (ojParser)DATA_PTR(value) where value is the return
1259
+ // from this function. A delegate must be added before the parser can be
1260
+ // used. Optionally oj_parser_set_options can be called if the options are not
1261
+ // set directly.
1262
+ VALUE oj_parser_new() {
1263
+ ojParser p = ALLOC(struct _ojParser);
1264
+
1265
+ #if HAVE_RB_EXT_RACTOR_SAFE
1266
+ // This doesn't seem to do anything.
1267
+ rb_ext_ractor_safe(true);
1268
+ #endif
1269
+ memset(p, 0, sizeof(struct _ojParser));
1270
+ buf_init(&p->key);
1271
+ buf_init(&p->buf);
1272
+ p->map = value_map;
1273
+
1274
+ return Data_Wrap_Struct(parser_class, parser_mark, parser_free, p);
1275
+ }
1276
+
1277
+ // Set set the options from a hash (ropts).
1278
+ void oj_parser_set_option(ojParser p, VALUE ropts) {
1279
+ Check_Type(ropts, T_HASH);
1280
+ rb_hash_foreach(ropts, opt_cb, (VALUE)p);
1281
+ }
1282
+
1226
1283
  /* Document-method: method_missing(value)
1227
1284
  * call-seq: method_missing(value)
1228
1285
  *
@@ -1456,7 +1513,7 @@ static VALUE saj_parser = Qundef;
1456
1513
  /* Document-method: saj
1457
1514
  * call-seq: saj
1458
1515
  *
1459
- * Returns the default saj parser. Note the default SAJ parser can not be used
1516
+ * Returns the default SAJ parser. Note the default SAJ parser can not be used
1460
1517
  * concurrently in more than one thread.
1461
1518
  */
1462
1519
  static VALUE parser_saj(VALUE self) {