oj 2.9.0 → 2.9.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of oj might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
- ---
2
- SHA1:
3
- metadata.gz: b9dd0aba891fc55691f3a56dfeebb3a9b429a3e9
4
- data.tar.gz: df3fc2acd752b893079e08bad70b7bcafee0df15
5
- SHA512:
6
- metadata.gz: 9e0c17e03b86f6e6b495ee3c75cfac64a400e04cd6c3fac52e444d2f48af44d4f69ea327a1ef4db39692fec5e16ca2270c80b4bcc8d66ba80da22a27d492d443
7
- data.tar.gz: 91a9dc1eafd5182a4c1d6d930a9b673b88a3151cb9dff09580943f04eff7a323bb4c4fe25de1df34e427dab08afdc9dd587fd7a6b61d43d2ffe29e6f648ad949
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 89fa4d9b461087f8729149310d3ff78f6c93208f
4
+ data.tar.gz: ae041dd8632861dbb9a456fe28212ee33828b866
5
+ SHA512:
6
+ metadata.gz: a4c73cfee1fad82bb875f6c0e57695ae5ed7c0de7ea573e46296624ae98d8cb4ea05d9b24c8847f0dd99b12ccbb98763a333f1c3a6e0e1b60b00f49d8ccb7c0c
7
+ data.tar.gz: 82419fd8c95688ede5884b41f37a67e04236eb666660407487076dad1f492cfef964ed819d4976eabd6d22b35eb0d3dd25fdc0b96a87d6a4d1b1ba71770c1068
data/README.md CHANGED
@@ -26,12 +26,17 @@ Follow [@peterohler on Twitter](http://twitter.com/#!/peterohler) for announceme
26
26
 
27
27
  [![Build Status](https://secure.travis-ci.org/ohler55/oj.png?branch=master)](http://travis-ci.org/ohler55/oj)
28
28
 
29
- ### Current Release 2.9.0
29
+ ### Current Release 2.9.1
30
30
 
31
- - Added support for detection and handling of String, Array, and Hash subclasses.
31
+ - Fixed mimic load when given a block to evalate. It conflicted with the new
32
+ load option.
32
33
 
33
- - Oj.load() can now take a block which will be yielded to on every object
34
- parsed when used with a file or string with multiple JSON entries.
34
+ - Added a true stream that is used when the input argument to load is an IO
35
+ object such as a stream or file. This is slightly slower for smaller files
36
+ but allows reading of huge files that will not fit in memory and is more
37
+ efficient on even larger files that would fit into memory. The load_file()
38
+ method uses the new stream parser so multiple GB files can be processed
39
+ without used execessive memory.
35
40
 
36
41
  [Older release notes](http://www.ohler.com/dev/oj_misc/release_notes.html).
37
42
 
@@ -97,7 +97,7 @@ buf_append(Buf buf, char c) {
97
97
  }
98
98
  *buf->tail = c;
99
99
  buf->tail++;
100
- *buf->tail = '\0'; // TBD temp for debugging
100
+ //*buf->tail = '\0'; // for debugging
101
101
  }
102
102
 
103
103
  #endif /* __OJ_BUF_H__ */
@@ -45,11 +45,7 @@ hash_set_cstr(ParseInfo pi, const char *key, size_t klen, const char *str, size_
45
45
  *pi->options.create_id == *key &&
46
46
  pi->options.create_id_len == klen &&
47
47
  0 == strncmp(pi->options.create_id, key, klen)) {
48
- if (str < pi->json || pi->cur < str) {
49
- parent->classname = oj_strndup(str, len);
50
- } else {
51
- parent->classname = str;
52
- }
48
+ parent->classname = oj_strndup(str, len);
53
49
  parent->clen = len;
54
50
  } else {
55
51
  volatile VALUE rstr = rb_str_new(str, len);
@@ -75,7 +71,7 @@ end_hash(struct _ParseInfo *pi) {
75
71
  if (Qundef != clas) { // else an error
76
72
  parent->val = rb_funcall(clas, oj_json_create_id, 1, parent->val);
77
73
  }
78
- if (parent->classname < pi->json || pi->end < parent->classname) {
74
+ if (0 != parent->classname) {
79
75
  xfree((char*)parent->classname);
80
76
  parent->classname = 0;
81
77
  }
@@ -96,7 +92,11 @@ oj_compat_parse(int argc, VALUE *argv, VALUE self) {
96
92
  pi.options = oj_default_options;
97
93
  oj_set_compat_callbacks(&pi);
98
94
 
99
- return oj_pi_parse(argc, argv, &pi, 0, 0);
95
+ if (T_STRING == rb_type(*argv)) {
96
+ return oj_pi_parse(argc, argv, &pi, 0, 0, 1);
97
+ } else {
98
+ return oj_pi_sparse(argc, argv, &pi, 0);
99
+ }
100
100
  }
101
101
 
102
102
  VALUE
@@ -108,5 +108,5 @@ oj_compat_parse_cstr(int argc, VALUE *argv, char *json, size_t len) {
108
108
  pi.end_hash = end_hash;
109
109
  pi.hash_set_cstr = hash_set_cstr;
110
110
 
111
- return oj_pi_parse(argc, argv, &pi, json, len);
111
+ return oj_pi_parse(argc, argv, &pi, json, len, 1);
112
112
  }
@@ -1302,6 +1302,7 @@ isRbxHashAttr(const char *attr) {
1302
1302
  "@mask",
1303
1303
  "@size",
1304
1304
  "@entries",
1305
+ "@default_proc",
1305
1306
  0 };
1306
1307
  const char **ap;
1307
1308
 
@@ -1452,6 +1453,7 @@ dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out) {
1452
1453
  ID vid;
1453
1454
  const char *attr;
1454
1455
  int i;
1456
+ int first = 1;
1455
1457
 
1456
1458
  cnt = (int)RARRAY_LEN(vars);
1457
1459
  #endif
@@ -1474,6 +1476,11 @@ dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out) {
1474
1476
  continue;
1475
1477
  }
1476
1478
  #endif
1479
+ if (first) {
1480
+ first = 0;
1481
+ } else {
1482
+ *out->cur++ = ',';
1483
+ }
1477
1484
  if (out->end - out->cur <= (long)size) {
1478
1485
  grow(out, size);
1479
1486
  }
@@ -1494,9 +1501,6 @@ dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out) {
1494
1501
  if (out->end - out->cur <= 2) {
1495
1502
  grow(out, 2);
1496
1503
  }
1497
- if (1 < i) {
1498
- *out->cur++ = ',';
1499
- }
1500
1504
  }
1501
1505
  #endif
1502
1506
  #if HAS_EXCEPTION_MAGIC
@@ -1807,6 +1811,17 @@ oj_dump_obj_to_json(VALUE obj, Options copts, Out out) {
1807
1811
  }
1808
1812
  out->indent = copts->indent;
1809
1813
  dump_val(obj, 0, out);
1814
+ if (0 < out->indent) {
1815
+ switch (*(out->cur - 1)) {
1816
+ case ']':
1817
+ case '}':
1818
+ grow(out, 1);
1819
+ *out->cur++ = '\n';
1820
+ *out->cur = '\0';
1821
+ default:
1822
+ break;
1823
+ }
1824
+ }
1810
1825
  if (Yes == copts->circular) {
1811
1826
  oj_cache8_delete(out->circ_cache);
1812
1827
  }
@@ -45,7 +45,7 @@ extern VALUE oj_parse_error_class;
45
45
  extern void oj_err_set(Err e, VALUE clas, const char *format, ...);
46
46
  extern void _oj_err_set_with_location(Err err, VALUE eclas, const char *msg, const char *json, const char *current, const char* file, int line);
47
47
  extern void oj_err_raise(Err e);
48
- // TBD remove
48
+
49
49
  #define raise_error(msg, json, current) _oj_raise_error(msg, json, current, __FILE__, __LINE__)
50
50
  extern void _oj_raise_error(const char *msg, const char *json, const char *current, const char* file, int line);
51
51
 
@@ -522,21 +522,30 @@ add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
522
522
  pi->stack.head->val = str_to_value(pi, str, len, orig);
523
523
  }
524
524
 
525
+ void
526
+ oj_set_object_callbacks(ParseInfo pi) {
527
+ oj_set_strict_callbacks(pi);
528
+ pi->end_hash = end_hash;
529
+ pi->start_hash = start_hash;
530
+ pi->hash_set_cstr = hash_set_cstr;
531
+ pi->hash_set_num = hash_set_num;
532
+ pi->hash_set_value = hash_set_value;
533
+ pi->add_cstr = add_cstr;
534
+ pi->array_append_cstr = array_append_cstr;
535
+ }
536
+
525
537
  VALUE
526
538
  oj_object_parse(int argc, VALUE *argv, VALUE self) {
527
539
  struct _ParseInfo pi;
528
540
 
529
541
  pi.options = oj_default_options;
530
- oj_set_strict_callbacks(&pi);
531
- pi.end_hash = end_hash;
532
- pi.start_hash = start_hash;
533
- pi.hash_set_cstr = hash_set_cstr;
534
- pi.hash_set_num = hash_set_num;
535
- pi.hash_set_value = hash_set_value;
536
- pi.add_cstr = add_cstr;
537
- pi.array_append_cstr = array_append_cstr;
542
+ oj_set_object_callbacks(&pi);
538
543
 
539
- return oj_pi_parse(argc, argv, &pi, 0, 0);
544
+ if (T_STRING == rb_type(*argv)) {
545
+ return oj_pi_parse(argc, argv, &pi, 0, 0, 1);
546
+ } else {
547
+ return oj_pi_sparse(argc, argv, &pi, 0);
548
+ }
540
549
  }
541
550
 
542
551
  VALUE
@@ -553,5 +562,5 @@ oj_object_parse_cstr(int argc, VALUE *argv, char *json, size_t len) {
553
562
  pi.add_cstr = add_cstr;
554
563
  pi.array_append_cstr = array_append_cstr;
555
564
 
556
- return oj_pi_parse(argc, argv, &pi, json, len);
565
+ return oj_pi_parse(argc, argv, &pi, json, len, 1);
557
566
  }
@@ -35,6 +35,7 @@
35
35
  #include <string.h>
36
36
  #include <sys/types.h>
37
37
  #include <unistd.h>
38
+ #include <fcntl.h>
38
39
 
39
40
  #include "oj.h"
40
41
  #include "parse.h"
@@ -67,6 +68,7 @@ ID oj_json_create_id;
67
68
  ID oj_length_id;
68
69
  ID oj_new_id;
69
70
  ID oj_read_id;
71
+ ID oj_readpartial_id;
70
72
  ID oj_replace_id;
71
73
  ID oj_string_id;
72
74
  ID oj_to_hash_id;
@@ -619,6 +621,10 @@ oj_parse_options(VALUE ropts, Options copts) {
619
621
  * any, will be yielded to. If no block then the last element read will be
620
622
  * returned.
621
623
  *
624
+ * This parser operates on string and will attempt to load files into memory if
625
+ * a file object is passed as the first argument. A stream input will be parsed
626
+ * using a stream parser but others use the slightly faster string parser.
627
+ *
622
628
  * @param [String|IO] json JSON String or an Object that responds to read()
623
629
  * @param [Hash] options load options (same as default_options)
624
630
  */
@@ -676,37 +682,24 @@ load(int argc, VALUE *argv, VALUE self) {
676
682
  * If the input file is not a valid JSON document (an empty file is not a valid
677
683
  * JSON document) an exception is raised.
678
684
  *
685
+ * This is a stream based parser which allows a large or huge file to be loaded
686
+ * without pulling the whole file into memory.
687
+ *
679
688
  * @param [String] path path to a file containing a JSON document
680
689
  * @param [Hash] options load options (same as default_options)
681
690
  */
682
691
  static VALUE
683
692
  load_file(int argc, VALUE *argv, VALUE self) {
684
693
  char *path;
685
- char *json;
686
- FILE *f;
687
- unsigned long len;
694
+ int fd;
688
695
  Mode mode = oj_default_options.mode;
689
-
690
- Check_Type(*argv, T_STRING);
691
- path = StringValuePtr(*argv);
692
- if (0 == (f = fopen(path, "r"))) {
693
- rb_raise(rb_eIOError, "%s", strerror(errno));
694
- }
695
- fseek(f, 0, SEEK_END);
696
- len = ftell(f);
697
- json = ALLOC_N(char, len + 1);
698
- fseek(f, 0, SEEK_SET);
699
- if (len != fread(json, 1, len, f)) {
700
- xfree(json);
701
- fclose(f);
702
- rb_raise(rb_const_get_at(Oj, rb_intern("LoadError")), "Failed to read %ld bytes from %s.", len, path);
703
- }
704
- fclose(f);
705
- json[len] = '\0';
696
+ struct _ParseInfo pi;
706
697
 
707
698
  if (1 > argc) {
708
699
  rb_raise(rb_eArgError, "Wrong number of arguments to load().");
709
700
  }
701
+ Check_Type(*argv, T_STRING);
702
+ pi.options = oj_default_options;
710
703
  if (2 <= argc) {
711
704
  VALUE ropts = argv[1];
712
705
  VALUE v;
@@ -725,18 +718,25 @@ load_file(int argc, VALUE *argv, VALUE self) {
725
718
  }
726
719
  }
727
720
  }
728
- // The json string is freed in the parser when it is finished with it.
721
+ path = StringValuePtr(*argv);
722
+ if (0 == (fd = open(path, O_RDONLY))) {
723
+ rb_raise(rb_eIOError, "%s", strerror(errno));
724
+ }
729
725
  switch (mode) {
730
726
  case StrictMode:
731
- return oj_strict_parse_cstr(argc, argv, json, len);
727
+ oj_set_strict_callbacks(&pi);
728
+ return oj_pi_sparse(argc, argv, &pi, fd);
732
729
  case NullMode:
733
730
  case CompatMode:
734
- return oj_compat_parse_cstr(argc, argv, json, len);
731
+ oj_set_compat_callbacks(&pi);
732
+ return oj_pi_sparse(argc, argv, &pi, fd);
735
733
  case ObjectMode:
736
734
  default:
737
735
  break;
738
736
  }
739
- return oj_object_parse_cstr(argc, argv, json, len);
737
+ oj_set_object_callbacks(&pi);
738
+
739
+ return oj_pi_sparse(argc, argv, &pi, fd);
740
740
  }
741
741
 
742
742
  /* call-seq: safe_load(doc)
@@ -760,7 +760,7 @@ safe_load(VALUE self, VALUE doc) {
760
760
  oj_set_strict_callbacks(&pi);
761
761
  *args = doc;
762
762
 
763
- return oj_pi_parse(1, args, &pi, 0, 0);
763
+ return oj_pi_parse(1, args, &pi, 0, 0, 1);
764
764
  }
765
765
 
766
766
  /* call-seq: saj_parse(handler, io)
@@ -1452,11 +1452,11 @@ mimic_walk(VALUE key, VALUE obj, VALUE proc) {
1452
1452
  break;
1453
1453
  case T_ARRAY:
1454
1454
  {
1455
- VALUE *np = RARRAY_PTR(obj);
1456
1455
  size_t cnt = RARRAY_LEN(obj);
1457
-
1458
- for (; 0 < cnt; cnt--, np++) {
1459
- mimic_walk(Qnil, *np, proc);
1456
+ size_t i;
1457
+
1458
+ for (i = 0; i < cnt; i++) {
1459
+ mimic_walk(Qnil, rb_ary_entry(obj, i), proc);
1460
1460
  }
1461
1461
  break;
1462
1462
  }
@@ -1482,7 +1482,12 @@ mimic_walk(VALUE key, VALUE obj, VALUE proc) {
1482
1482
 
1483
1483
  static VALUE
1484
1484
  mimic_load(int argc, VALUE *argv, VALUE self) {
1485
- VALUE obj = load(1, argv, self);
1485
+ struct _ParseInfo pi;
1486
+
1487
+ pi.options = oj_default_options;
1488
+ oj_set_compat_callbacks(&pi);
1489
+
1490
+ VALUE obj = oj_pi_parse(argc, argv, &pi, 0, 0, 0);
1486
1491
  VALUE p = Qnil;
1487
1492
 
1488
1493
  if (2 <= argc) {
@@ -1638,7 +1643,7 @@ mimic_parse(int argc, VALUE *argv, VALUE self) {
1638
1643
  }
1639
1644
  *args = *argv;
1640
1645
 
1641
- return oj_pi_parse(1, args, &pi, 0, 0);
1646
+ return oj_pi_parse(1, args, &pi, 0, 0, 0);
1642
1647
  }
1643
1648
 
1644
1649
  static VALUE
@@ -1878,6 +1883,7 @@ void Init_oj() {
1878
1883
  oj_length_id = rb_intern("length");
1879
1884
  oj_new_id = rb_intern("new");
1880
1885
  oj_read_id = rb_intern("read");
1886
+ oj_readpartial_id = rb_intern("readpartial");
1881
1887
  oj_replace_id = rb_intern("replace");
1882
1888
  oj_string_id = rb_intern("string");
1883
1889
  oj_to_hash_id = rb_intern("to_hash");
@@ -199,6 +199,7 @@ extern VALUE oj_saj_parse(int argc, VALUE *argv, VALUE self);
199
199
  extern VALUE oj_sc_parse(int argc, VALUE *argv, VALUE self);
200
200
 
201
201
  extern VALUE oj_strict_parse(int argc, VALUE *argv, VALUE self);
202
+ extern VALUE oj_strict_sparse(int argc, VALUE *argv, VALUE self);
202
203
  extern VALUE oj_compat_parse(int argc, VALUE *argv, VALUE self);
203
204
  extern VALUE oj_object_parse(int argc, VALUE *argv, VALUE self);
204
205
 
@@ -262,6 +263,7 @@ extern ID oj_json_create_id;
262
263
  extern ID oj_length_id;
263
264
  extern ID oj_new_id;
264
265
  extern ID oj_read_id;
266
+ extern ID oj_readpartial_id;
265
267
  extern ID oj_replace_id;
266
268
  extern ID oj_string_id;
267
269
  extern ID oj_to_hash_id;
@@ -722,7 +722,11 @@ oj_set_error_at(ParseInfo pi, VALUE err_clas, const char* file, int line, const
722
722
  vsnprintf(msg, sizeof(msg) - 1, format, ap);
723
723
  va_end(ap);
724
724
  pi->err.clas = err_clas;
725
- _oj_err_set_with_location(&pi->err, err_clas, msg, pi->json, pi->cur - 1, file, line);
725
+ if (0 == pi->json) {
726
+ oj_err_set(&pi->err, err_clas, "%s at line %d, column %d [%s:%d]", msg, pi->rd.line, pi->rd.col, file, line);
727
+ } else {
728
+ _oj_err_set_with_location(&pi->err, err_clas, msg, pi->json, pi->cur - 1, file, line);
729
+ }
726
730
  }
727
731
 
728
732
  static VALUE
@@ -733,7 +737,7 @@ protect_parse(VALUE pip) {
733
737
  }
734
738
 
735
739
  VALUE
736
- oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
740
+ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk) {
737
741
  char *buf = 0;
738
742
  volatile VALUE input;
739
743
  volatile VALUE wrapped_stack;
@@ -748,7 +752,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
748
752
  if (2 == argc) {
749
753
  oj_parse_options(argv[1], &pi->options);
750
754
  }
751
- if (rb_block_given_p()) {
755
+ if (yieldOk && rb_block_given_p()) {
752
756
  pi->proc = Qnil;
753
757
  } else {
754
758
  pi->proc = Qundef;
@@ -758,7 +762,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
758
762
  pi->json = json;
759
763
  pi->end = json + len;
760
764
  free_json = 1;
761
- } else if (rb_type(input) == T_STRING) {
765
+ } else if (T_STRING == rb_type(input)) {
762
766
  pi->json = rb_string_value_cstr((VALUE*)&input);
763
767
  pi->end = pi->json + RSTRING_LEN(input);
764
768
  } else if (Qnil == input && Yes == pi->options.nilnil) {
@@ -796,9 +800,8 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
796
800
  }
797
801
  #endif
798
802
  } else if (rb_respond_to(input, oj_read_id)) {
799
- s = rb_funcall2(input, oj_read_id, 0, 0);
800
- pi->json = rb_string_value_cstr((VALUE*)&s);
801
- pi->end = pi->json + RSTRING_LEN(s);
803
+ // use stream parser instead
804
+ return oj_pi_sparse(argc, argv, pi, 0);
802
805
  } else {
803
806
  rb_raise(rb_eArgError, "strict_parse() expected a String or IO Object.");
804
807
  }
@@ -32,11 +32,13 @@
32
32
  #define __OJ_PARSE_H__
33
33
 
34
34
  #include <stdarg.h>
35
+ #include <stdio.h>
35
36
 
36
37
  #include "ruby.h"
37
38
  #include "oj.h"
38
39
  #include "val_stack.h"
39
40
  #include "circarray.h"
41
+ #include "reader.h"
40
42
 
41
43
  typedef struct _NumInfo {
42
44
  int64_t i;
@@ -54,9 +56,13 @@ typedef struct _NumInfo {
54
56
  } *NumInfo;
55
57
 
56
58
  typedef struct _ParseInfo {
59
+ // used for the string parser
57
60
  const char *json;
58
61
  const char *cur;
59
62
  const char *end;
63
+ // used for the stream parser
64
+ struct _Reader rd;
65
+
60
66
  struct _Err err;
61
67
  struct _Options options;
62
68
  void *cbc;
@@ -83,10 +89,14 @@ typedef struct _ParseInfo {
83
89
 
84
90
  extern void oj_parse2(ParseInfo pi);
85
91
  extern void oj_set_error_at(ParseInfo pi, VALUE err_clas, const char* file, int line, const char *format, ...);
86
- extern VALUE oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len);
92
+ extern VALUE oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk);
87
93
  extern VALUE oj_num_as_value(NumInfo ni);
88
94
 
89
95
  extern void oj_set_strict_callbacks(ParseInfo pi);
96
+ extern void oj_set_object_callbacks(ParseInfo pi);
90
97
  extern void oj_set_compat_callbacks(ParseInfo pi);
91
98
 
99
+ extern void oj_sparse2(ParseInfo pi);
100
+ extern VALUE oj_pi_sparse(int argc, VALUE *argv, ParseInfo pi, int fd);
101
+
92
102
  #endif /* __OJ_PARSE_H__ */