oj 1.0.5 → 1.0.6

data/README.md

@@ -24,9 +24,9 @@ A fast JSON parser and Object marshaller as a Ruby gem.
 
 ## <a name="release">Release Notes</a>
 
-### Release 1.0.5
+### Release 1.0.6
 
- - Added :ascii_only options for dumping JSON where all high-bit characters are encoded as escaped sequences.
+ - Gave Oj::Doc a speed increase. It is now 8 times fast than JSON::Ext.
 
 ## <a name="description">Description</a>
 

data/ext/oj/dump.c

@@ -97,6 +97,14 @@ static void     grow(Out out, size_t len);
 static size_t	json_friendly_size(const u_char *str, size_t len);
 static size_t	ascii_friendly_size(const u_char *str, size_t len);
 
+static void	dump_leaf_to_json(Leaf leaf, Options copts, Out out);
+static void	dump_leaf(Leaf leaf, int depth, Out out);
+static void	dump_leaf_str(Leaf leaf, Out out);
+static void	dump_leaf_fixnum(Leaf leaf, Out out);
+static void	dump_leaf_float(Leaf leaf, Out out);
+static void	dump_leaf_array(Leaf leaf, int depth, Out out);
+static void	dump_leaf_hash(Leaf leaf, int depth, Out out);
+
 
 static const char	hex_chars[17] = "0123456789abcdef";
 
@@ -287,7 +295,7 @@ static void
 dump_fixnum(VALUE obj, Out out) {
     char        buf[32];
     char        *b = buf + sizeof(buf) - 1;
-    long        num = NUM2LONG(obj);
+    long	num = NUM2LONG(obj);
     int         neg = 0;
 
     if (0 > num) {
@@ -407,7 +415,6 @@ dump_cstr(const char *str, size_t cnt, int is_sym, int escape1, Out out) {
 		    *out->cur++ = '0';
 		    dump_hex((u_char)*str, out);
 		} else { // continuation?
-		    // TBD lead with \u00 . grab next char?
 		    *out->cur++ = '0';
 		    *out->cur++ = '0';
 		    dump_hex((u_char)*str, out);
@@ -1128,7 +1135,224 @@ oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
         rb_raise(rb_eIOError, "%s\n", strerror(errno));
     }
     if (size != fwrite(out.buf, 1, size, f)) {
-        int err = ferror(f);
+        int	err = ferror(f);
+
+        rb_raise(rb_eIOError, "Write failed. [%d:%s]\n", err, strerror(err));
+    }
+    xfree(out.buf);
+    fclose(f);
+}
+
+// dump leaf functions
+
+inline static void
+dump_chars(const char *s, size_t size, Out out) {
+    if (out->end - out->cur <= (long)size) {
+	grow(out, size);
+    }
+    memcpy(out->cur, s, size);
+    out->cur += size;
+    *out->cur = '\0';
+}
+
+static void
+dump_leaf_str(Leaf leaf, Out out) {
+    switch (leaf->value_type) {
+    case STR_VAL:
+	dump_cstr(leaf->str, strlen(leaf->str), 0, 0, out);
+	break;
+    case RUBY_VAL:
+	dump_cstr(StringValuePtr(leaf->value), RSTRING_LEN(leaf->value), 0, 0, out);
+	break;
+    case COL_VAL:
+    default:
+	rb_raise(rb_eTypeError, "Unexpected value type %02x.", leaf->value_type);
+	break;
+    }
+}
+
+static void
+dump_leaf_fixnum(Leaf leaf, Out out) {
+    switch (leaf->value_type) {
+    case STR_VAL:
+	dump_chars(leaf->str, strlen(leaf->str), out);
+	break;
+    case RUBY_VAL:
+	if (T_BIGNUM == rb_type(leaf->value)) {
+	    dump_bignum(leaf->value, out);
+	} else {
+	    dump_fixnum(leaf->value, out);
+	}
+	break;
+    case COL_VAL:
+    default:
+	rb_raise(rb_eTypeError, "Unexpected value type %02x.", leaf->value_type);
+	break;
+    }
+}
+
+static void
+dump_leaf_float(Leaf leaf, Out out) {
+    switch (leaf->value_type) {
+    case STR_VAL:
+	dump_chars(leaf->str, strlen(leaf->str), out);
+	break;
+    case RUBY_VAL:
+	dump_float(leaf->value, out);
+	break;
+    case COL_VAL:
+    default:
+	rb_raise(rb_eTypeError, "Unexpected value type %02x.", leaf->value_type);
+	break;
+    }
+}
+
+static void
+dump_leaf_array(Leaf leaf, int depth, Out out) {
+    size_t	size;
+    int		d2 = depth + 1;
+
+    size = 2;
+    if (out->end - out->cur <= (long)size) {
+        grow(out, size);
+    }
+    *out->cur++ = '[';
+    if (0 == leaf->elements) {
+	*out->cur++ = ']';
+    } else {
+	Leaf	first = leaf->elements->next;
+	Leaf	e = first;
+
+	size = d2 * out->indent + 2;
+	do {
+	    if (out->end - out->cur <= (long)size) {
+		grow(out, size);
+	    }
+	    fill_indent(out, d2);
+	    dump_leaf(e, d2, out);
+	    if (e->next != first) {
+		*out->cur++ = ',';
+	    }
+	    e = e->next;
+	} while (e != first);
+	size = depth * out->indent + 1;
+	if (out->end - out->cur <= (long)size) {
+	    grow(out, size);
+	}
+	fill_indent(out, depth);
+	*out->cur++ = ']';
+    }
+    *out->cur = '\0';
+}
+
+static void
+dump_leaf_hash(Leaf leaf, int depth, Out out) {
+    size_t	size;
+    int		d2 = depth + 1;
+
+    size = 2;
+    if (out->end - out->cur <= (long)size) {
+        grow(out, size);
+    }
+    *out->cur++ = '{';
+    if (0 == leaf->elements) {
+	*out->cur++ = '}';
+    } else {
+	Leaf	first = leaf->elements->next;
+	Leaf	e = first;
+
+	size = d2 * out->indent + 2;
+	do {
+	    if (out->end - out->cur <= (long)size) {
+		grow(out, size);
+	    }
+	    fill_indent(out, d2);
+	    dump_cstr(e->key, strlen(e->key), 0, 0, out);
+	    *out->cur++ = ':';
+	    dump_leaf(e, d2, out);
+	    if (e->next != first) {
+		*out->cur++ = ',';
+	    }
+	    e = e->next;
+	} while (e != first);
+	size = depth * out->indent + 1;
+	if (out->end - out->cur <= (long)size) {
+	    grow(out, size);
+	}
+	fill_indent(out, depth);
+	*out->cur++ = '}';
+    }
+    *out->cur = '\0';
+}
+
+static void
+dump_leaf(Leaf leaf, int depth, Out out) {
+    switch (leaf->type) {
+    case T_NIL:
+	dump_nil(out);
+	break;
+    case T_TRUE:
+	dump_true(out);
+	break;
+    case T_FALSE:
+	dump_false(out);
+	break;
+    case T_STRING:
+	dump_leaf_str(leaf, out);
+	break;
+    case T_FIXNUM:
+	dump_leaf_fixnum(leaf, out);
+	break;
+    case T_FLOAT:
+	dump_leaf_float(leaf, out);
+	break;
+    case T_ARRAY:
+	dump_leaf_array(leaf, depth, out);
+	break;
+    case T_HASH:
+	dump_leaf_hash(leaf, depth, out);
+	break;
+    default:
+	rb_raise(rb_eTypeError, "Unexpected type %02x.", leaf->type);
+	break;
+    }
+}
+
+static void
+dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
+    out->buf = ALLOC_N(char, 65336);
+    out->end = out->buf + 65325; // 10 less than end plus extra for possible errors
+    out->cur = out->buf;
+    out->circ_cnt = 0;
+    out->opts = copts;
+    out->hash_cnt = 0;
+    out->indent = copts->indent;
+    dump_leaf(leaf, 0, out);
+}
+
+char*
+oj_write_leaf_to_str(Leaf leaf, Options copts) {
+    struct _Out out;
+
+    dump_leaf_to_json(leaf, copts, &out);
+
+    return out.buf;
+}
+
+void
+oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts) {
+    struct _Out out;
+    size_t      size;
+    FILE        *f;    
+
+    dump_leaf_to_json(leaf, copts, &out);
+    size = out.cur - out.buf;
+    if (0 == (f = fopen(path, "w"))) {
+        rb_raise(rb_eIOError, "%s\n", strerror(errno));
+    }
+    if (size != fwrite(out.buf, 1, size, f)) {
+        int	err = ferror(f);
+
         rb_raise(rb_eIOError, "Write failed. [%d:%s]\n", err, strerror(err));
     }
     xfree(out.buf);

data/ext/oj/fast.c

@@ -37,29 +37,6 @@
 #include "oj.h"
 
 #define MAX_STACK	100
-
-enum {
-    STR_VAL  = 0x00,
-    COL_VAL  = 0x01,
-    RUBY_VAL = 0x02
-};
-    
-typedef struct _Leaf {
-    struct _Leaf	*next;
-    union {
-	const char	*key;      // hash key
-	size_t		index;     // array index, 0 is not set
-    };
-    union {
-	char		*str;      // pointer to location in json string
-	struct _Leaf	*elements; // array and hash elements
-	VALUE		value;
-    };
-    uint8_t		type;
-    uint8_t		parent_type;
-    uint8_t		value_type;
-} *Leaf;
-
 //#define BATCH_SIZE	(4096 / sizeof(struct _Leaf) - 1)
 #define BATCH_SIZE	100
 
@@ -81,8 +58,6 @@ typedef struct _Doc {
     unsigned long	size;        // number of leaves/branches in the doc
     VALUE		self;
     Batch		batches;
-    //Leaf		where_array[MAX_STACK];
-    //size_t		where_len;   // length of allocated if longer than where_array
     struct _Batch	batch0;
 } *Doc;
 
@@ -748,8 +723,6 @@ read_quoted_value(ParseInfo pi) {
 // doc support functions
 inline static void
 doc_init(Doc doc) {
-    //doc->where_path = doc->where_array;
-    //doc->where_len = 0;
     doc->where = doc->where_path;
     *doc->where = 0;
     doc->data = 0;
@@ -776,11 +749,6 @@ doc_free(Doc doc) {
 		xfree(b);
 	    }
 	}
-	/*
-	if (doc->where_array != doc->where_path) {
-	    free(doc->where_path);
-	}
-	*/
 	//xfree(f);
     }
 }
@@ -792,6 +760,7 @@ protect_open_proc(VALUE x) {
     pi->doc->data = read_next(pi); // parse
     *pi->doc->where = pi->doc->data;
     pi->doc->where = pi->doc->where_path;
+
     return rb_yield(pi->doc->self); // caller processing
 }
 
@@ -848,6 +817,9 @@ static Leaf
 get_leaf(Leaf *stack, Leaf *lp, const char *path) {
     Leaf	leaf = *lp;
 
+    if (MAX_STACK <= lp - stack) {
+	rb_raise(rb_eIOError, "Path too deep. limit is %d levels.\n", MAX_STACK);
+    }
     if ('\0' != *path) {
 	if ('.' == *path && '.' == *(path + 1)) {
 	    path += 2;
@@ -864,7 +836,6 @@ get_leaf(Leaf *stack, Leaf *lp, const char *path) {
 	    Leaf	e = first;
 	    int		type = leaf->type;
 
-	    // TBD fail if stack too deep
 	    leaf = 0;
 	    if (T_ARRAY == type) {
 		int	cnt = 0;
@@ -933,7 +904,9 @@ each_leaf(Doc doc, VALUE self) {
 
 static int
 move_step(Doc doc, const char *path, int loc) {
-    // TBD raise if too deep
+    if (MAX_STACK <= doc->where - doc->where_path) {
+	rb_raise(rb_eIOError, "Path too deep. limit is %d levels.\n", MAX_STACK);
+    }
     if ('\0' == *path) {
 	loc = 0;
     } else {
@@ -1425,13 +1398,12 @@ doc_each_value(int argc, VALUE *argv, VALUE self) {
     return Qnil;
 }
 
-// TBD improve to be more direct for higher performance
-
 /* call-seq: dump(path=nil) => String
  *
  * Dumps the document or nodes to a new JSON document. It uses the default
  * options for generating the JSON.
  * @param [String] path if provided it identified the top of the branch to dump to JSON
+ * @param [String] filename if provided it is the filename to write the output to
  * @example
  *   Oj::Doc.open('[3,[2,1]]') { |doc|
  *       doc.dump('/2')
@@ -1443,16 +1415,31 @@ doc_dump(int argc, VALUE *argv, VALUE self) {
     Doc		doc = DATA_PTR(self);
     Leaf	leaf;
     const char	*path = 0;
-    const char	*json;
+    const char	*filename = 0;
 
     if (1 <= argc) {
-	Check_Type(*argv, T_STRING);
-	path = StringValuePtr(*argv);
+	if (Qnil != *argv) {
+	    Check_Type(*argv, T_STRING);
+	    path = StringValuePtr(*argv);
+	}
+	if (2 <= argc) {
+	    Check_Type(argv[1], T_STRING);
+	    filename = StringValuePtr(argv[1]);
+	}
     }
     if (0 != (leaf = get_doc_leaf(doc, path))) {
-	json = oj_write_obj_to_str(leaf_value(doc, leaf), &oj_default_options);
+	char	*json;
+	VALUE	rjson;
 
-	return rb_str_new2(json);
+	if (0 == filename) {
+	    json = oj_write_leaf_to_str(leaf, &oj_default_options);
+	    rjson = rb_str_new2(json);
+	    xfree(json);
+	} else {
+	    oj_write_leaf_to_file(leaf, filename, &oj_default_options);
+	    rjson = Qnil;
+	}
+	return rjson;
     }
     return Qnil;
 }

data/ext/oj/oj.h

@@ -89,9 +89,33 @@ typedef struct _Options {
     char        mode;		// Mode
 } *Options;
 
+enum {
+    STR_VAL  = 0x00,
+    COL_VAL  = 0x01,
+    RUBY_VAL = 0x02
+};
+    
+typedef struct _Leaf {
+    struct _Leaf	*next;
+    union {
+	const char	*key;      // hash key
+	size_t		index;     // array index, 0 is not set
+    };
+    union {
+	char		*str;      // pointer to location in json string
+	struct _Leaf	*elements; // array and hash elements
+	VALUE		value;
+    };
+    uint8_t		type;
+    uint8_t		parent_type;
+    uint8_t		value_type;
+} *Leaf;
+
 extern VALUE	oj_parse(char *json, Options options);
 extern char*	oj_write_obj_to_str(VALUE obj, Options copts);
 extern void	oj_write_obj_to_file(VALUE obj, const char *path, Options copts);
+extern char*	oj_write_leaf_to_str(Leaf leaf, Options copts);
+extern void	oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts);
 
 extern void	_oj_raise_error(const char *msg, const char *xml, const char *current, const char* file, int line);
 

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '1.0.5'
+  VERSION = '1.0.6'
 end

data/test/perf_fast.rb

@@ -14,9 +14,10 @@ require 'oj'
 
 $verbose = false
 $indent = 0
-$iter = 100000
+$iter = 10000
 $gets = 0
 $fetch = false
+$write = false
 
 opts = OptionParser.new
 opts.on("-v", "verbose")                                  { $verbose = true }
@@ -24,6 +25,7 @@ opts.on("-c", "--count [Int]", Integer, "iterations")     { |i| $iter = i }
 opts.on("-i", "--indent [Int]", Integer, "indentation")   { |i| $indent = i }
 opts.on("-g", "--gets [Int]", Integer, "number of gets")  { |i| $gets = i }
 opts.on("-f", "fetch")                                    { $fetch = true }
+opts.on("-w", "write")                                    { $write = true }
 opts.on("-h", "--help", "Show this display")              { puts opts; Process.exit!(0) }
 files = opts.parse(ARGV)
 
@@ -82,6 +84,17 @@ perf.add('Yajl', 'parse') { Yajl::Parser.parse($json) } unless $failed.has_key?(
 perf.add('JSON::Ext', 'parse') { JSON::Ext::Parser.new($json).parse } unless $failed.has_key?('JSON::Ext')
 perf.run($iter)
 
+puts '-' * 80
+puts "JSON generation Performance"
+Oj::Doc.open($json) do |doc|
+  perf = Perf.new()
+  perf.add('Oj::Doc', 'dump') { doc.dump() }
+  perf.add('Yajl', 'encode') { Yajl::Encoder.encode($obj) }
+  perf.add('JSON::Ext', 'fast_generate') { JSON.fast_generate($obj) }
+  perf.before('JSON::Ext') { JSON.generator = JSON::Ext::Generator }
+  perf.run($iter)
+end
+
 if 0 < $gets
   puts '-' * 80
   puts "Parse and get all values Performance"
@@ -113,6 +126,19 @@ if $fetch
   end
 end
 
+if $write
+  puts '-' * 80
+  puts "JSON write to file Performance"
+  Oj::Doc.open($json) do |doc|
+    perf = Perf.new()
+    perf.add('Oj::Doc', 'dump') { doc.dump(nil, 'oj.json') }
+    perf.add('Yajl', 'encode') { File.open('yajl.json', 'w') { |f| Yajl::Encoder.encode($obj, f) } }
+    perf.add('JSON::Ext', 'fast_generate') { File.open('json_ext.json', 'w') { |f| f.write(JSON.fast_generate($obj)) } }
+    perf.before('JSON::Ext') { JSON.generator = JSON::Ext::Generator }
+    perf.run($iter)
+  end
+end
+
 unless $failed.empty?
   puts "The following packages were not included for the reason listed"
   $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-17 00:00:00.000000000 Z
+date: 2012-03-20 00:00:00.000000000 Z
 dependencies: []
 description: ! 'The fastest JSON parser and object serializer. '
 email: peter@ohler.com