oj 2.12.1 → 2.12.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2a621ea1cf610e5b78949d860fc54f2c6fe43e0c
+  data.tar.gz: 11cd52b0f0747e95007d678b291560b6271b9235
+SHA512:
+  metadata.gz: 76bdb63802a6f728259dd9cb7351d64f38862cf8a522bdb56c208cdc5803cb52a4625493c4c0d7d103d5e6a7a2522bfd847a1942ccb56ce2a59a3b22ae9d969e
+  data.tar.gz: 5a215c6a81a876ee0f1a777ff90f1d55b1ad5eb809da6b431eb21b79f7bcaab7a552febdcc090f12e17aa0acf0ea885c78d70940b13e298d8524db1c227c0500

data/README.md

@@ -26,9 +26,11 @@ Follow [@peterohler on Twitter](http://twitter.com/#!/peterohler) for announceme
 
 [![Build Status](https://secure.travis-ci.org/ohler55/oj.png?branch=master)](http://travis-ci.org/ohler55/oj)
 
-## Current Release 2.12.1
+## Current Release 2.12.2
 
- - Changed the unix_zone encoded format to be the utc epoch.
+ - Fixed Oj::Doc bug that causes a crash on local name lookups.
+
+ - Fixed Oj::Doc unicode parsing. 
 
 [Older release notes](http://www.ohler.com/dev/oj_misc/release_notes.html).
 
@@ -80,6 +82,11 @@ Rails 3.2 and 4.0. As of the Oj 2.6.0 release the default behavior is to not use
 the `to_json()` method unless the `:use_to_json` option is set. This provides
 another work around to the rails older and newer behavior.
 
+The latest ActiveRecord is able to work with Oj by simply using the line:
+```
+serialize :my_attr, Oj
+```
+
 In version Rails 4.1, multi_json has been removed, and this patch is unnecessary and will no longer work.
 Instead, use the `oj_mimic_json` [gem](https://github.com/ohler55/oj_mimic_json) along with `oj` in your `Gemfile` to have Oj mimic the JSON gem and be used in its place by `ActiveSupport` JSON handling:
 ```

data/ext/oj/dump.c

@@ -2228,7 +2228,7 @@ dump_leaf_hash(Leaf leaf, int depth, Out out) {
 
 static void
 dump_leaf(Leaf leaf, int depth, Out out) {
-    switch (leaf->type) {
+    switch (leaf->rtype) {
     case T_NIL:
 	dump_nil(out);
 	break;
@@ -2254,7 +2254,7 @@ dump_leaf(Leaf leaf, int depth, Out out) {
 	dump_leaf_hash(leaf, depth, out);
 	break;
     default:
-	rb_raise(rb_eTypeError, "Unexpected type %02x.\n", leaf->type);
+	rb_raise(rb_eTypeError, "Unexpected type %02x.\n", leaf->rtype);
 	break;
     }
 }

data/ext/oj/fast.c

@@ -170,7 +170,7 @@ ulong_fill(char *s, size_t num) {
 inline static void
 leaf_init(Leaf leaf, int type) {
     leaf->next = 0;
-    leaf->type = type;
+    leaf->rtype = type;
     leaf->parent_type = T_NONE;
     switch (type) {
     case T_ARRAY:
@@ -206,6 +206,8 @@ leaf_new(Doc doc, int type) {
     if (0 == doc->batches || BATCH_SIZE == doc->batches->next_avail) {
 	Batch	b = ALLOC(struct _Batch);
 
+	// Initializes all leaves with a NO_VAL value_type
+	memset(b, 0, sizeof(struct _Batch));
 	b->next = doc->batches;
 	doc->batches = b;
 	b->next_avail = 0;
@@ -232,7 +234,7 @@ leaf_append_element(Leaf parent, Leaf element) {
 static VALUE
 leaf_value(Doc doc, Leaf leaf) {
     if (RUBY_VAL != leaf->value_type) {
-	switch (leaf->type) {
+	switch (leaf->rtype) {
 	case T_NIL:
 	    leaf->value = Qnil;
 	    break;
@@ -260,7 +262,7 @@ leaf_value(Doc doc, Leaf leaf) {
 	    return leaf_hash_value(doc, leaf);
 	    break;
 	default:
-	    rb_raise(rb_const_get_at(Oj, rb_intern("Error")), "Unexpected type %02x.", leaf->type);
+	    rb_raise(rb_const_get_at(Oj, rb_intern("Error")), "Unexpected type %02x.", leaf->rtype);
 	    break;
 	}
     }
@@ -348,79 +350,11 @@ leaf_fixnum_value(Leaf leaf) {
     leaf->value_type = RUBY_VAL;
 }
 
-#ifdef JRUBY_RUBY
-static void
-leaf_float_value(Leaf leaf) {
-    char	*s = leaf->str;
-    int64_t	n = 0;
-    long	a = 0;
-    long	div = 1;
-    long	e = 0;
-    int		neg = 0;
-    int		eneg = 0;
-    int		big = 0;
-
-    if ('-' == *s) {
-	s++;
-	neg = 1;
-    } else if ('+' == *s) {
-	s++;
-    }
-    for (; '0' <= *s && *s <= '9'; s++) {
-	n = n * 10 + (*s - '0');
-	if (NUM_MAX <= n) {
-	    big = 1;
-	}
-    }
-    if (big) {
-	char	c = *s;
-	
-	*s = '\0';
-	leaf->value = rb_cstr_to_inum(leaf->str, 10, 0);
-	*s = c;
-    } else {
-	double	d;
-
-	if ('.' == *s) {
-	    s++;
-	    for (; '0' <= *s && *s <= '9'; s++) {
-		a = a * 10 + (*s - '0');
-		div *= 10;
-	    }
-	}
-	if ('e' == *s || 'E' == *s) {
-	    s++;
-	    if ('-' == *s) {
-		s++;
-		eneg = 1;
-	    } else if ('+' == *s) {
-		s++;
-	    }
-	    for (; '0' <= *s && *s <= '9'; s++) {
-		e = e * 10 + (*s - '0');
-	    }
-	}
-	d = (double)n + (double)a / (double)div;
-	if (neg) {
-	    d = -d;
-	}
-	if (0 != e) {
-	    if (eneg) {
-		e = -e;
-	    }
-	    d *= pow(10.0, e);
-	}
-	leaf->value = rb_float_new(d);
-    }
-    leaf->value_type = RUBY_VAL;
-}
-#else
 static void
 leaf_float_value(Leaf leaf) {
     leaf->value = rb_float_new(rb_cstr_to_dbl(leaf->str, 1));
     leaf->value_type = RUBY_VAL;
 }
-#endif
 
 static VALUE
 leaf_array_value(Doc doc, Leaf leaf) {
@@ -675,33 +609,59 @@ read_nil(ParseInfo pi) {
     return leaf;
 }
 
-static char
-read_hex(ParseInfo pi, char *h) {
-    uint8_t	b = 0;
+static uint32_t
+read_4hex(ParseInfo pi, const char *h) {
+    uint32_t	b = 0;
+    int		i;
+
+    for (i = 0; i < 4; i++, h++) {
+	b = b << 4;
+	if ('0' <= *h && *h <= '9') {
+	    b += *h - '0';
+	} else if ('A' <= *h && *h <= 'F') {
+	    b += *h - 'A' + 10;
+	} else if ('a' <= *h && *h <= 'f') {
+	    b += *h - 'a' + 10;
+	} else {
+	    raise_error("invalid hex character", pi->str, pi->s);
+	}
+    }
+    return b;
+}
 
-    if ('0' <= *h && *h <= '9') {
-	b = *h - '0';
-    } else if ('A' <= *h && *h <= 'F') {
-	b = *h - 'A' + 10;
-    } else if ('a' <= *h && *h <= 'f') {
-	b = *h - 'a' + 10;
-    } else {
-	pi->s = h;
-	raise_error("invalid hex character", pi->str, pi->s);
-    }
-    h++;
-    b = b << 4;
-    if ('0' <= *h && *h <= '9') {
-	b += *h - '0';
-    } else if ('A' <= *h && *h <= 'F') {
-	b += *h - 'A' + 10;
-    } else if ('a' <= *h && *h <= 'f') {
-	b += *h - 'a' + 10;
+static char*
+unicode_to_chars(ParseInfo pi, char *t, uint32_t code) {
+    if (0x0000007F >= code) {
+	*t++ = (char)code;
+    } else if (0x000007FF >= code) {
+	*t++ = 0xC0 | (code >> 6);
+	*t++ = 0x80 | (0x3F & code);
+    } else if (0x0000FFFF >= code) {
+	*t++ = 0xE0 | (code >> 12);
+	*t++ = 0x80 | ((code >> 6) & 0x3F);
+	*t++ = 0x80 | (0x3F & code);
+    } else if (0x001FFFFF >= code) {
+	*t++ = 0xF0 | (code >> 18);
+	*t++ = 0x80 | ((code >> 12) & 0x3F);
+	*t++ = 0x80 | ((code >> 6) & 0x3F);
+	*t++ = 0x80 | (0x3F & code);
+    } else if (0x03FFFFFF >= code) {
+	*t++ = 0xF8 | (code >> 24);
+	*t++ = 0x80 | ((code >> 18) & 0x3F);
+	*t++ = 0x80 | ((code >> 12) & 0x3F);
+	*t++ = 0x80 | ((code >> 6) & 0x3F);
+	*t++ = 0x80 | (0x3F & code);
+    } else if (0x7FFFFFFF >= code) {
+	*t++ = 0xFC | (code >> 30);
+	*t++ = 0x80 | ((code >> 24) & 0x3F);
+	*t++ = 0x80 | ((code >> 18) & 0x3F);
+	*t++ = 0x80 | ((code >> 12) & 0x3F);
+	*t++ = 0x80 | ((code >> 6) & 0x3F);
+	*t++ = 0x80 | (0x3F & code);
     } else {
-	pi->s = h;
-	raise_error("invalid hex character", pi->str, pi->s);
+	raise_error("invalid Unicode character", pi->str, pi->s);
     }
-    return (char)b;
+    return t;
 }
 
 /* Assume the value starts immediately and goes until the quote character is
@@ -731,16 +691,31 @@ read_quoted_value(ParseInfo pi) {
 	    case '"':	*t = '"';	break;
 	    case '/':	*t = '/';	break;
 	    case '\\':	*t = '\\';	break;
-	    case 'u':
+	    case 'u': {
+		uint32_t	code;
+
 		h++;
-		*t = read_hex(pi, h);
-		h += 2;
-		if ('\0' != *t) {
-		    t++;
+		code = read_4hex(pi, h);
+		h += 3;
+		if (0x0000D800 <= code && code <= 0x0000DFFF) {
+		    uint32_t	c1 = (code - 0x0000D800) & 0x000003FF;
+		    uint32_t	c2;
+
+		    h++;
+		    if ('\\' != *h || 'u' != *(h + 1)) {
+			pi->s = h;
+			raise_error("invalid escaped character", pi->str, pi->s);
+		    }
+		    h += 2;
+		    c2 = read_4hex(pi, h);
+		    h += 3;
+		    c2 = (c2 - 0x0000DC00) & 0x000003FF;
+		    code = ((c1 << 10) | c2) + 0x00010000;
 		}
-		*t = read_hex(pi, h);
-		h++;
+		t = unicode_to_chars(pi, t, code);
+		t--;
 		break;
+	    }
 	    default:
 		pi->s = h;
 		raise_error("invalid escaped character", pi->str, pi->s);
@@ -759,15 +734,10 @@ read_quoted_value(ParseInfo pi) {
 // doc support functions
 inline static void
 doc_init(Doc doc) {
+    memset(doc, 0, sizeof(struct _Doc));
     doc->where = doc->where_path;
-    *doc->where = 0;
-    doc->data = 0;
     doc->self = Qundef;
-    doc->size = 0;
-    doc->json = 0;
     doc->batches = &doc->batch0;
-    doc->batch0.next = 0;
-    doc->batch0.next_avail = 0;
 }
 
 static void
@@ -885,7 +855,7 @@ get_doc_leaf(Doc doc, const char *path) {
 	    if (MAX_STACK <= cnt) {
 		rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
 	    }
-	    memcpy(stack, doc->where_path, sizeof(Leaf) * cnt);
+	    memcpy(stack, doc->where_path, sizeof(Leaf) * (cnt + 1));
 	    lp = stack + cnt;
 	}
 	return get_leaf(stack, lp, path);
@@ -914,7 +884,7 @@ get_leaf(Leaf *stack, Leaf *lp, const char *path) {
 	} else if (COL_VAL == leaf->value_type && 0 != leaf->elements) {
 	    Leaf	first = leaf->elements->next;
 	    Leaf	e = first;
-	    int		type = leaf->type;
+	    int		type = leaf->rtype;
 
 	    leaf = 0;
 	    if (T_ARRAY == type) {
@@ -1021,7 +991,7 @@ move_step(Doc doc, const char *path, int loc) {
 	    Leaf	first = leaf->elements->next;
 	    Leaf	e = first;
 
-	    if (T_ARRAY == leaf->type) {
+	    if (T_ARRAY == leaf->rtype) {
 		int	cnt = 0;
 
 		for (; '0' <= *path && *path <= '9'; path++) {
@@ -1046,7 +1016,7 @@ move_step(Doc doc, const char *path, int loc) {
 		    cnt--;
 		    e = e->next;
 		} while (e != first);
-	    } else if (T_HASH == leaf->type) {
+	    } else if (T_HASH == leaf->rtype) {
 		const char	*key = path;
 		const char	*slash = strchr(path, '/');
 		int		klen;
@@ -1302,7 +1272,7 @@ doc_type(int argc, VALUE *argv, VALUE self) {
 	path = StringValuePtr(*argv);
     }
     if (0 != (leaf = get_doc_leaf(doc, path))) {
-	switch (leaf->type) {
+	switch (leaf->rtype) {
 	case T_NIL:	type = rb_cNilClass;	break;
 	case T_TRUE:	type = rb_cTrueClass;	break;
 	case T_FALSE:	type = rb_cFalseClass;	break;
@@ -1375,7 +1345,7 @@ doc_each_leaf(int argc, VALUE *argv, VALUE self) {
 
 	wlen = doc->where - doc->where_path;
 	if (0 < wlen) {
-	    memcpy(save_path, doc->where_path, sizeof(Leaf) * wlen);
+	    memcpy(save_path, doc->where_path, sizeof(Leaf) * (wlen + 1));
 	}
 	if (1 <= argc) {
 	    Check_Type(*argv, T_STRING);
@@ -1386,14 +1356,14 @@ doc_each_leaf(int argc, VALUE *argv, VALUE self) {
 	    }
 	    if (0 != move_step(doc, path, 1)) {
 		if (0 < wlen) {
-		    memcpy(doc->where_path, save_path, sizeof(Leaf) * wlen);
+		    memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));
 		}
 		return Qnil;
 	    }
 	}
 	each_leaf(doc, self);
 	if (0 < wlen) {
-	    memcpy(doc->where_path, save_path, sizeof(Leaf) * wlen);
+	    memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));
 	}
     }
     return Qnil;
@@ -1451,7 +1421,7 @@ doc_each_child(int argc, VALUE *argv, VALUE self) {
 
 	wlen = doc->where - doc->where_path;
 	if (0 < wlen) {
-	    memcpy(save_path, doc->where_path, sizeof(Leaf) * wlen);
+	    memcpy(save_path, doc->where_path, sizeof(Leaf) * (wlen + 1));
 	}
 	if (1 <= argc) {
 	    Check_Type(*argv, T_STRING);
@@ -1462,7 +1432,7 @@ doc_each_child(int argc, VALUE *argv, VALUE self) {
 	    }
 	    if (0 != move_step(doc, path, 1)) {
 		if (0 < wlen) {
-		    memcpy(doc->where_path, save_path, sizeof(Leaf) * wlen);
+		    memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));
 		}
 		return Qnil;
 	    }
@@ -1479,7 +1449,7 @@ doc_each_child(int argc, VALUE *argv, VALUE self) {
 	    } while (e != first);
 	}
 	if (0 < wlen) {
-	    memcpy(doc->where_path, save_path, sizeof(Leaf) * wlen);
+	    memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));
 	}
     }
     return Qnil;

data/ext/oj/oj.h

@@ -179,9 +179,10 @@ typedef struct _StreamWriter {
 } *StreamWriter;
 
 enum {
-    STR_VAL  = 0x00,
-    COL_VAL  = 0x01,
-    RUBY_VAL = 0x02
+    NO_VAL   = 0x00,
+    STR_VAL  = 0x01,
+    COL_VAL  = 0x02,
+    RUBY_VAL = 0x03
 };
     
 typedef struct _Leaf {
@@ -191,11 +192,11 @@ typedef struct _Leaf {
 	size_t		index;	   // array index, 0 is not set
     };
     union {
-	char		*str;	   // pointer to location in json string
+	char		*str;	   // pointer to location in json string or allocated
 	struct _Leaf	*elements; // array and hash elements
 	VALUE		value;
     };
-    uint8_t		type;
+    uint8_t		rtype;
     uint8_t		parent_type;
     uint8_t		value_type;
 } *Leaf;

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '2.12.1'
+  VERSION = '2.12.2'
 end

data/test/bug.rb

@@ -1,3 +1,5 @@
+#!/usr/bin/env ruby
+
 #!/usr/bin/env ruby
 # encoding: UTF-8
 
@@ -5,60 +7,47 @@ $: << File.dirname(__FILE__)
 
 require 'helper'
 
-require 'oj'
-require 'securerandom'
-
 class Handler
-  def hash_start()      {} end
-  def hash_set(h,k,v)   h.store(k,v) end
-  def array_start()     [] end
-  def array_append(a,v) a << v end
-  def error(message, line, column)
-    raise Exception.new(message, line, column)
+  def initialize
+    @state = []
+  end
+
+  def hash_start
+    @state << {}
+    @state.last
+  end
+
+  def hash_end
+    @state.pop
+  end
+
+  def hash_set(h,k,v)
+    h.store(k,v)
   end
-end
 
-json = Oj.dump({"this"=>"object"})
-
-if true
-  name = "/tmp/#{SecureRandom.uuid}"
-  `mkfifo #{name}`
-  if fork
-    open(name, 'r+') do |read_io|
-      p "start reading #{read_io.stat.ftype}"
-      Oj.sc_parse(Handler.new, read_io) {|v| p v}
-      p "stop reading"
-    end
-  else
-    open(name, 'w+') do |write_io|
-      p "start writing #{write_io.stat.ftype}  autoclose: #{write_io.autoclose?}"
-      write_io.write json
-      write_io.write json
-      p "stop writing"
-    end
-    sleep(1) # make it obvious that there are two threads
-    open(name, 'w+') do |write_io|
-      p "start writing #{write_io.stat.ftype}"
-      write_io.write json
-      write_io.write json
-      p "stop writing"
-    end
+  def array_start
+    @state << []
+    @state.last
   end
-else
-  IO.pipe do |read_io, write_io|
-    if fork
-      write_io.close
-      p "start reading #{read_io.stat.ftype}"
-      Oj.sc_parse(Handler.new, read_io) {|v| p v}
-      p "stop reading"
-      read_io.close
-    else
-      read_io.close
-      p "start writing #{write_io.stat.ftype}"
-      write_io.write json
-      write_io.write json
-      p "stop writing"
-      write_io.close
-    end
+
+
+  def array_end
+    @state.pop
+  end
+
+  def array_append(a,v)
+    a << v
+  end
+
+  def add_value(v)
+    p v
   end
+
+  def error(message, line, column); p "ERROR: #{message}" end
 end
+
+$handler = Handler.new
+
+IO.popen("cat tst") { |p| puts Oj.sc_parse($handler, p) }
+
+#File.open('tst', 'r') { |file|  Oj.sc_parse($handler, file) }