oj 2.9.0 → 2.9.1

checksums.yaml

@@ -1,7 +1,7 @@
---- 
-SHA1: 
-  metadata.gz: b9dd0aba891fc55691f3a56dfeebb3a9b429a3e9
-  data.tar.gz: df3fc2acd752b893079e08bad70b7bcafee0df15
-SHA512: 
-  metadata.gz: 9e0c17e03b86f6e6b495ee3c75cfac64a400e04cd6c3fac52e444d2f48af44d4f69ea327a1ef4db39692fec5e16ca2270c80b4bcc8d66ba80da22a27d492d443
-  data.tar.gz: 91a9dc1eafd5182a4c1d6d930a9b673b88a3151cb9dff09580943f04eff7a323bb4c4fe25de1df34e427dab08afdc9dd587fd7a6b61d43d2ffe29e6f648ad949
+---
+SHA1:
+  metadata.gz: 89fa4d9b461087f8729149310d3ff78f6c93208f
+  data.tar.gz: ae041dd8632861dbb9a456fe28212ee33828b866
+SHA512:
+  metadata.gz: a4c73cfee1fad82bb875f6c0e57695ae5ed7c0de7ea573e46296624ae98d8cb4ea05d9b24c8847f0dd99b12ccbb98763a333f1c3a6e0e1b60b00f49d8ccb7c0c
+  data.tar.gz: 82419fd8c95688ede5884b41f37a67e04236eb666660407487076dad1f492cfef964ed819d4976eabd6d22b35eb0d3dd25fdc0b96a87d6a4d1b1ba71770c1068

data/README.md

@@ -26,12 +26,17 @@ Follow [@peterohler on Twitter](http://twitter.com/#!/peterohler) for announceme
 
 [![Build Status](https://secure.travis-ci.org/ohler55/oj.png?branch=master)](http://travis-ci.org/ohler55/oj)
 
-### Current Release 2.9.0
+### Current Release 2.9.1
 
- - Added support for detection and handling of String, Array, and Hash subclasses.
+ - Fixed mimic load when given a block to evalate. It conflicted with the new
+   load option.
 
- - Oj.load() can now take a block which will be yielded to on every object
-   parsed when used with a file or string with multiple JSON entries.
+ - Added a true stream that is used when the input argument to load is an IO
+   object such as a stream or file. This is slightly slower for smaller files
+   but allows reading of huge files that will not fit in memory and is more
+   efficient on even larger files that would fit into memory. The load_file()
+   method uses the new stream parser so multiple GB files can be processed
+   without used execessive memory.
 
 [Older release notes](http://www.ohler.com/dev/oj_misc/release_notes.html).
 

data/ext/oj/buf.h

@@ -97,7 +97,7 @@ buf_append(Buf buf, char c) {
     }
     *buf->tail = c;
     buf->tail++;
-    *buf->tail = '\0'; // TBD temp for debugging
+    //*buf->tail = '\0'; // for debugging
 }
 
 #endif /* __OJ_BUF_H__ */

data/ext/oj/compat.c

@@ -45,11 +45,7 @@ hash_set_cstr(ParseInfo pi, const char *key, size_t klen, const char *str, size_
 	*pi->options.create_id == *key &&
 	pi->options.create_id_len == klen &&
 	0 == strncmp(pi->options.create_id, key, klen)) {
-	if (str < pi->json || pi->cur < str) {
-	    parent->classname = oj_strndup(str, len);
-	} else {
-	    parent->classname = str;
-	}
+	parent->classname = oj_strndup(str, len);
 	parent->clen = len;
     } else {
 	volatile VALUE	rstr = rb_str_new(str, len);
@@ -75,7 +71,7 @@ end_hash(struct _ParseInfo *pi) {
 	if (Qundef != clas) { // else an error
 	    parent->val = rb_funcall(clas, oj_json_create_id, 1, parent->val);
 	}
-	if (parent->classname < pi->json || pi->end < parent->classname) {
+	if (0 != parent->classname) {
 	    xfree((char*)parent->classname);
 	    parent->classname = 0;
 	}
@@ -96,7 +92,11 @@ oj_compat_parse(int argc, VALUE *argv, VALUE self) {
     pi.options = oj_default_options;
     oj_set_compat_callbacks(&pi);
 
-    return oj_pi_parse(argc, argv, &pi, 0, 0);
+    if (T_STRING == rb_type(*argv)) {
+	return oj_pi_parse(argc, argv, &pi, 0, 0, 1);
+    } else {
+	return oj_pi_sparse(argc, argv, &pi, 0);
+    }
 }
 
 VALUE
@@ -108,5 +108,5 @@ oj_compat_parse_cstr(int argc, VALUE *argv, char *json, size_t len) {
     pi.end_hash = end_hash;
     pi.hash_set_cstr = hash_set_cstr;
 
-    return oj_pi_parse(argc, argv, &pi, json, len);
+    return oj_pi_parse(argc, argv, &pi, json, len, 1);
 }

data/ext/oj/dump.c

@@ -1302,6 +1302,7 @@ isRbxHashAttr(const char *attr) {
 	"@mask",
 	"@size",
 	"@entries",
+	"@default_proc",
 	0 };
     const char	**ap;
 
@@ -1452,6 +1453,7 @@ dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out) {
 	ID		vid;
 	const char	*attr;
 	int		i;
+	int		first = 1;
 
 	cnt = (int)RARRAY_LEN(vars);
 #endif
@@ -1474,6 +1476,11 @@ dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out) {
 		continue;
 	    }
 #endif
+	    if (first) {
+		first = 0;
+	    } else {
+		*out->cur++ = ',';
+	    }
 	    if (out->end - out->cur <= (long)size) {
 		grow(out, size);
 	    }
@@ -1494,9 +1501,6 @@ dump_obj_attrs(VALUE obj, VALUE clas, slot_t id, int depth, Out out) {
 	    if (out->end - out->cur <= 2) {
 		grow(out, 2);
 	    }
-	    if (1 < i) {
-	      *out->cur++ = ',';
-	    }
 	}
 #endif
 #if HAS_EXCEPTION_MAGIC
@@ -1807,6 +1811,17 @@ oj_dump_obj_to_json(VALUE obj, Options copts, Out out) {
     }
     out->indent = copts->indent;
     dump_val(obj, 0, out);
+    if (0 < out->indent) {
+	switch (*(out->cur - 1)) {
+	case ']':
+	case '}':
+	    grow(out, 1);
+	    *out->cur++ = '\n';
+	    *out->cur = '\0';
+	default:
+	    break;
+	}
+    }
     if (Yes == copts->circular) {
 	oj_cache8_delete(out->circ_cache);
     }

data/ext/oj/err.h

@@ -45,7 +45,7 @@ extern VALUE	oj_parse_error_class;
 extern void	oj_err_set(Err e, VALUE clas, const char *format, ...);
 extern void	_oj_err_set_with_location(Err err, VALUE eclas, const char *msg, const char *json, const char *current, const char* file, int line);
 extern void	oj_err_raise(Err e);
-// TBD remove
+
 #define raise_error(msg, json, current) _oj_raise_error(msg, json, current, __FILE__, __LINE__)
 extern void	_oj_raise_error(const char *msg, const char *json, const char *current, const char* file, int line);
 

data/ext/oj/object.c

@@ -522,21 +522,30 @@ add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
     pi->stack.head->val = str_to_value(pi, str, len, orig);
 }
 
+void
+oj_set_object_callbacks(ParseInfo pi) {
+    oj_set_strict_callbacks(pi);
+    pi->end_hash = end_hash;
+    pi->start_hash = start_hash;
+    pi->hash_set_cstr = hash_set_cstr;
+    pi->hash_set_num = hash_set_num;
+    pi->hash_set_value = hash_set_value;
+    pi->add_cstr = add_cstr;
+    pi->array_append_cstr = array_append_cstr;
+}
+
 VALUE
 oj_object_parse(int argc, VALUE *argv, VALUE self) {
     struct _ParseInfo	pi;
 
     pi.options = oj_default_options;
-    oj_set_strict_callbacks(&pi);
-    pi.end_hash = end_hash;
-    pi.start_hash = start_hash;
-    pi.hash_set_cstr = hash_set_cstr;
-    pi.hash_set_num = hash_set_num;
-    pi.hash_set_value = hash_set_value;
-    pi.add_cstr = add_cstr;
-    pi.array_append_cstr = array_append_cstr;
+    oj_set_object_callbacks(&pi);
 
-    return oj_pi_parse(argc, argv, &pi, 0, 0);
+    if (T_STRING == rb_type(*argv)) {
+	return oj_pi_parse(argc, argv, &pi, 0, 0, 1);
+    } else {
+	return oj_pi_sparse(argc, argv, &pi, 0);
+    }
 }
 
 VALUE
@@ -553,5 +562,5 @@ oj_object_parse_cstr(int argc, VALUE *argv, char *json, size_t len) {
     pi.add_cstr = add_cstr;
     pi.array_append_cstr = array_append_cstr;
 
-    return oj_pi_parse(argc, argv, &pi, json, len);
+    return oj_pi_parse(argc, argv, &pi, json, len, 1);
 }

data/ext/oj/oj.c

@@ -35,6 +35,7 @@
 #include <string.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <fcntl.h>
 
 #include "oj.h"
 #include "parse.h"
@@ -67,6 +68,7 @@ ID	oj_json_create_id;
 ID	oj_length_id;
 ID	oj_new_id;
 ID	oj_read_id;
+ID	oj_readpartial_id;
 ID	oj_replace_id;
 ID	oj_string_id;
 ID	oj_to_hash_id;
@@ -619,6 +621,10 @@ oj_parse_options(VALUE ropts, Options copts) {
  * any, will be yielded to. If no block then the last element read will be
  * returned.
  *
+ * This parser operates on string and will attempt to load files into memory if
+ * a file object is passed as the first argument. A stream input will be parsed
+ * using a stream parser but others use the slightly faster string parser.
+ *
  * @param [String|IO] json JSON String or an Object that responds to read()
  * @param [Hash] options load options (same as default_options)
  */
@@ -676,37 +682,24 @@ load(int argc, VALUE *argv, VALUE self) {
  * If the input file is not a valid JSON document (an empty file is not a valid
  * JSON document) an exception is raised.
  *
+ * This is a stream based parser which allows a large or huge file to be loaded
+ * without pulling the whole file into memory.
+ *
  * @param [String] path path to a file containing a JSON document
  * @param [Hash] options load options (same as default_options)
  */
 static VALUE
 load_file(int argc, VALUE *argv, VALUE self) {
     char		*path;
-    char		*json;
-    FILE		*f;
-    unsigned long	len;
+    int			fd;
     Mode		mode = oj_default_options.mode;
-
-    Check_Type(*argv, T_STRING);
-    path = StringValuePtr(*argv);
-    if (0 == (f = fopen(path, "r"))) {
-	rb_raise(rb_eIOError, "%s", strerror(errno));
-    }
-    fseek(f, 0, SEEK_END);
-    len = ftell(f);
-    json = ALLOC_N(char, len + 1);
-    fseek(f, 0, SEEK_SET);
-    if (len != fread(json, 1, len, f)) {
-	xfree(json);
-	fclose(f);
-	rb_raise(rb_const_get_at(Oj, rb_intern("LoadError")), "Failed to read %ld bytes from %s.", len, path);
-    }
-    fclose(f);
-    json[len] = '\0';
+    struct _ParseInfo	pi;
 
     if (1 > argc) {
 	rb_raise(rb_eArgError, "Wrong number of arguments to load().");
     }
+    Check_Type(*argv, T_STRING);
+    pi.options = oj_default_options;
     if (2 <= argc) {
 	VALUE	ropts = argv[1];
 	VALUE	v;
@@ -725,18 +718,25 @@ load_file(int argc, VALUE *argv, VALUE self) {
 	    }
 	}
     }
-    // The json string is freed in the parser when it is finished with it.
+    path = StringValuePtr(*argv);
+    if (0 == (fd = open(path, O_RDONLY))) {
+	rb_raise(rb_eIOError, "%s", strerror(errno));
+    }
     switch (mode) {
     case StrictMode:
-	return oj_strict_parse_cstr(argc, argv, json, len);
+	oj_set_strict_callbacks(&pi);
+	return oj_pi_sparse(argc, argv, &pi, fd);
     case NullMode:
     case CompatMode:
-	return oj_compat_parse_cstr(argc, argv, json, len);
+	oj_set_compat_callbacks(&pi);
+	return oj_pi_sparse(argc, argv, &pi, fd);
     case ObjectMode:
     default:
 	break;
     }
-    return oj_object_parse_cstr(argc, argv, json, len);
+    oj_set_object_callbacks(&pi);
+
+    return oj_pi_sparse(argc, argv, &pi, fd);
 }
 
 /* call-seq: safe_load(doc)
@@ -760,7 +760,7 @@ safe_load(VALUE self, VALUE doc) {
     oj_set_strict_callbacks(&pi);
     *args = doc;
 
-    return oj_pi_parse(1, args, &pi, 0, 0);
+    return oj_pi_parse(1, args, &pi, 0, 0, 1);
 }
 
 /* call-seq: saj_parse(handler, io)
@@ -1452,11 +1452,11 @@ mimic_walk(VALUE key, VALUE obj, VALUE proc) {
 	break;
     case T_ARRAY:
 	{
-	    VALUE	*np = RARRAY_PTR(obj);
 	    size_t	cnt = RARRAY_LEN(obj);
-	
-	    for (; 0 < cnt; cnt--, np++) {
-		mimic_walk(Qnil, *np, proc);
+	    size_t	i;
+
+	    for (i = 0; i < cnt; i++) {
+		mimic_walk(Qnil, rb_ary_entry(obj, i), proc);
 	    }
 	    break;
 	}
@@ -1482,7 +1482,12 @@ mimic_walk(VALUE key, VALUE obj, VALUE proc) {
 
 static VALUE
 mimic_load(int argc, VALUE *argv, VALUE self) {
-    VALUE	obj = load(1, argv, self);
+    struct _ParseInfo	pi;
+
+    pi.options = oj_default_options;
+    oj_set_compat_callbacks(&pi);
+
+    VALUE	obj = oj_pi_parse(argc, argv, &pi, 0, 0, 0);
     VALUE	p = Qnil;
 
     if (2 <= argc) {
@@ -1638,7 +1643,7 @@ mimic_parse(int argc, VALUE *argv, VALUE self) {
     }
     *args = *argv;
 
-    return oj_pi_parse(1, args, &pi, 0, 0);
+    return oj_pi_parse(1, args, &pi, 0, 0, 0);
 }
 
 static VALUE
@@ -1878,6 +1883,7 @@ void Init_oj() {
     oj_length_id = rb_intern("length");
     oj_new_id = rb_intern("new");
     oj_read_id = rb_intern("read");
+    oj_readpartial_id = rb_intern("readpartial");
     oj_replace_id = rb_intern("replace");
     oj_string_id = rb_intern("string");
     oj_to_hash_id = rb_intern("to_hash");

data/ext/oj/oj.h

@@ -199,6 +199,7 @@ extern VALUE	oj_saj_parse(int argc, VALUE *argv, VALUE self);
 extern VALUE	oj_sc_parse(int argc, VALUE *argv, VALUE self);
 
 extern VALUE	oj_strict_parse(int argc, VALUE *argv, VALUE self);
+extern VALUE	oj_strict_sparse(int argc, VALUE *argv, VALUE self);
 extern VALUE	oj_compat_parse(int argc, VALUE *argv, VALUE self);
 extern VALUE	oj_object_parse(int argc, VALUE *argv, VALUE self);
 
@@ -262,6 +263,7 @@ extern ID	oj_json_create_id;
 extern ID	oj_length_id;
 extern ID	oj_new_id;
 extern ID	oj_read_id;
+extern ID	oj_readpartial_id;
 extern ID	oj_replace_id;
 extern ID	oj_string_id;
 extern ID	oj_to_hash_id;

data/ext/oj/parse.c

@@ -722,7 +722,11 @@ oj_set_error_at(ParseInfo pi, VALUE err_clas, const char* file, int line, const
     vsnprintf(msg, sizeof(msg) - 1, format, ap);
     va_end(ap);
     pi->err.clas = err_clas;
-    _oj_err_set_with_location(&pi->err, err_clas, msg, pi->json, pi->cur - 1, file, line);
+    if (0 == pi->json) {
+	oj_err_set(&pi->err, err_clas, "%s at line %d, column %d [%s:%d]", msg, pi->rd.line, pi->rd.col, file, line);
+    } else {
+	_oj_err_set_with_location(&pi->err, err_clas, msg, pi->json, pi->cur - 1, file, line);
+    }
 }
 
 static VALUE
@@ -733,7 +737,7 @@ protect_parse(VALUE pip) {
 }
 
 VALUE
-oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
+oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk) {
     char		*buf = 0;
     volatile VALUE	input;
     volatile VALUE	wrapped_stack;
@@ -748,7 +752,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
     if (2 == argc) {
 	oj_parse_options(argv[1], &pi->options);
     }
-    if (rb_block_given_p()) {
+    if (yieldOk && rb_block_given_p()) {
 	pi->proc = Qnil;
     } else {
 	pi->proc = Qundef;
@@ -758,7 +762,7 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
 	pi->json = json;
 	pi->end = json + len;
 	free_json = 1;
-    } else if (rb_type(input) == T_STRING) {
+    } else if (T_STRING == rb_type(input)) {
 	pi->json = rb_string_value_cstr((VALUE*)&input);
 	pi->end = pi->json + RSTRING_LEN(input);
     } else if (Qnil == input && Yes == pi->options.nilnil) {
@@ -796,9 +800,8 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len) {
 	    }
 #endif
 	} else if (rb_respond_to(input, oj_read_id)) {
-	    s = rb_funcall2(input, oj_read_id, 0, 0);
-	    pi->json = rb_string_value_cstr((VALUE*)&s);
-	    pi->end = pi->json + RSTRING_LEN(s);
+	    // use stream parser instead
+	    return oj_pi_sparse(argc, argv, pi, 0);
 	} else {
 	    rb_raise(rb_eArgError, "strict_parse() expected a String or IO Object.");
 	}

data/ext/oj/parse.h

@@ -32,11 +32,13 @@
 #define __OJ_PARSE_H__
 
 #include <stdarg.h>
+#include <stdio.h>
 
 #include "ruby.h"
 #include "oj.h"
 #include "val_stack.h"
 #include "circarray.h"
+#include "reader.h"
 
 typedef struct _NumInfo {
     int64_t	i;
@@ -54,9 +56,13 @@ typedef struct _NumInfo {
 } *NumInfo;
 
 typedef struct _ParseInfo {
+    // used for the string parser
     const char		*json;
     const char		*cur;
     const char		*end;
+    // used for the stream parser
+    struct _Reader	rd;
+
     struct _Err		err;
     struct _Options	options;
     void		*cbc;
@@ -83,10 +89,14 @@ typedef struct _ParseInfo {
 
 extern void	oj_parse2(ParseInfo pi);
 extern void	oj_set_error_at(ParseInfo pi, VALUE err_clas, const char* file, int line, const char *format, ...);
-extern VALUE	oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len);
+extern VALUE	oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yieldOk);
 extern VALUE	oj_num_as_value(NumInfo ni);
 
 extern void	oj_set_strict_callbacks(ParseInfo pi);
+extern void	oj_set_object_callbacks(ParseInfo pi);
 extern void	oj_set_compat_callbacks(ParseInfo pi);
 
+extern void	oj_sparse2(ParseInfo pi);
+extern VALUE	oj_pi_sparse(int argc, VALUE *argv, ParseInfo pi, int fd);
+
 #endif /* __OJ_PARSE_H__ */