oj 2.1.7 → 2.2.0

data/ext/oj/strict.c

@@ -52,7 +52,7 @@ add_value(ParseInfo pi, VALUE val) {
 
 static void
 add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    VALUE	rstr = rb_str_new(str, len);
+    volatile VALUE	rstr = rb_str_new(str, len);
 
     rstr = oj_encode(rstr);
     pi->stack.head->val = rstr;
@@ -70,7 +70,7 @@ start_hash(ParseInfo pi) {
 
 static VALUE
 hash_key(ParseInfo pi, const char *key, size_t klen) {
-    VALUE	rkey = rb_str_new(key, klen);
+    volatile VALUE	rkey = rb_str_new(key, klen);
 
     rkey = oj_encode(rkey);
     if (Yes == pi->options.sym_key) {
@@ -81,7 +81,7 @@ hash_key(ParseInfo pi, const char *key, size_t klen) {
 
 static void
 hash_set_cstr(ParseInfo pi, const char *key, size_t klen, const char *str, size_t len, const char *orig) {
-    VALUE	rstr = rb_str_new(str, len);
+    volatile VALUE	rstr = rb_str_new(str, len);
 
     rstr = oj_encode(rstr);
     rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), rstr);
@@ -104,7 +104,7 @@ start_array(ParseInfo pi) {
 
 static void
 array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    VALUE	rstr = rb_str_new(str, len);
+    volatile VALUE	rstr = rb_str_new(str, len);
 
     rstr = oj_encode(rstr);
     rb_ary_push(stack_peek(&pi->stack)->val, rstr);

data/ext/oj/val_stack.c

@@ -28,8 +28,38 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "oj.h"
 #include "val_stack.h"
 
+static void
+mark(void *ptr) {
+    ValStack	stack = (ValStack)ptr;
+    Val		v;
+
+    pthread_mutex_lock(&stack->mutex);
+    for (v = stack->head; v < stack->tail; v++) {
+	if (Qnil != v->val && Qundef != v->val) {
+	    rb_gc_mark(v->val);
+	}
+    }
+    pthread_mutex_unlock(&stack->mutex);
+}
+
+VALUE
+oj_stack_init(ValStack stack) {
+    pthread_mutex_init(&stack->mutex, 0);
+    stack->head = stack->base;
+    stack->end = stack->base + sizeof(stack->base) / sizeof(struct _Val);
+    stack->tail = stack->head;
+    stack->head->val = Qundef;
+    stack->head->key = 0;
+    stack->head->classname = 0;
+    stack->head->klen = 0;
+    stack->head->clen = 0;
+    stack->head->next = NEXT_NONE;
+    return Data_Wrap_Struct(oj_cstack_class, mark, 0, stack);
+}
+
 const char*
 oj_stack_next_string(ValNext n) {
     switch (n) {

data/ext/oj/val_stack.h

@@ -34,8 +34,9 @@
 #include "ruby.h"
 #include "odd.h"
 #include <stdint.h>
+#include <pthread.h>
 
-#define STACK_INC	32
+#define STACK_INC	64
 
 typedef enum {
     NEXT_NONE		= 0,
@@ -50,38 +51,27 @@ typedef enum {
 } ValNext;
 
 typedef struct _Val {
-    VALUE	val;
-    const char	*key;
+    VALUE		val;
+    const char		*key;
     union {
 	const char	*classname;
 	OddArgs		odd_args;
     };
-    uint16_t	klen;
-    uint16_t	clen;
-    char	next; // ValNext
-    char	k1;   // first original character in the key
+    uint16_t		klen;
+    uint16_t		clen;
+    char		next; // ValNext
+    char		k1;   // first original character in the key
 } *Val;
 
 typedef struct _ValStack {
-    struct _Val	base[STACK_INC];
-    Val		head;	// current stack
-    Val		end;	// stack end
-    Val		tail;	// pointer to one past last element name on stack
+    struct _Val		base[STACK_INC];
+    Val			head;	// current stack
+    Val			end;	// stack end
+    Val			tail;	// pointer to one past last element name on stack
+    pthread_mutex_t	mutex;
 } *ValStack;
 
-inline static void
-stack_init(ValStack stack) {
-    stack->head = stack->base;
-    stack->end = stack->base + sizeof(stack->base) / sizeof(struct _Val);
-    stack->tail = stack->head;
-    stack->head->val = Qundef;
-    stack->head->key = 0;
-    stack->head->classname = 0;
-    stack->head->klen = 0;
-    stack->head->clen = 0;
-    stack->head->next = NEXT_NONE;
-    //stack->head->type = TYPE_NONE;
-}
+extern VALUE	oj_stack_init(ValStack stack);
 
 inline static int
 stack_empty(ValStack stack) {
@@ -100,15 +90,21 @@ stack_push(ValStack stack, VALUE val, ValNext next) {
     if (stack->end <= stack->tail) {
 	size_t	len = stack->end - stack->head;
 	size_t	toff = stack->tail - stack->head;
+	Val	head = stack->head;
 
+	// A realloc can trigger a GC so make sure it happens outside the lock
+	// but lock before changing pointers.
 	if (stack->base == stack->head) {
-	    stack->head = ALLOC_N(struct _Val, len + STACK_INC);
-	    memcpy(stack->head, stack->base, sizeof(struct _Val) * len);
+	    head = ALLOC_N(struct _Val, len + STACK_INC);
+	    memcpy(head, stack->base, sizeof(struct _Val) * len);
 	} else {
-	    REALLOC_N(stack->head, struct _Val, len + STACK_INC);
+	    REALLOC_N(head, struct _Val, len + STACK_INC);
 	}
+	pthread_mutex_lock(&stack->mutex);
+	stack->head = head;
 	stack->tail = stack->head + toff;
 	stack->end = stack->head + len + STACK_INC;
+	pthread_mutex_unlock(&stack->mutex);
     }
     stack->tail->val = val;
     stack->tail->next = next;
@@ -116,7 +112,6 @@ stack_push(ValStack stack, VALUE val, ValNext next) {
     stack->tail->key = 0;
     stack->tail->clen = 0;
     stack->tail->klen = 0;
-    //stack->tail->type = TYPE_NONE;
     stack->tail++;
 }
 

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '2.1.7'
+  VERSION = '2.2.0'
 end

data/test/test_gc.rb

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get
+# warnings to show up the -w options is required. That can be set in the RUBYOPT
+# environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'test/unit'
+require 'oj'
+
+class Goo
+  attr_accessor :x, :child
+
+  def initialize(x, child)
+    @x = x
+    @child = child
+  end
+  
+  def to_hash()
+    { 'json_class' => "#{self.class}", 'x' => x, 'child' => child }
+  end
+
+  def self.json_create(h)
+    GC.start
+    self.new(h['x'], h['child'])
+  end
+end # Goo
+
+class GCTest < ::Test::Unit::TestCase
+
+  # if no crash then the GC marking is working
+  def test_parse_compat_gc
+    g = Goo.new(0, nil)
+    100.times { |i| g = Goo.new(i, g) }
+    json = Oj.dump(g, :mode => :compat)
+    Oj.compat_load(json)
+  end
+
+  def test_parse_object_gc
+    g = Goo.new(0, nil)
+    100.times { |i| g = Goo.new(i, g) }
+    json = Oj.dump(g, :mode => :object)
+    Oj.object_load(json)
+  end
+end

data/test/test_saj.rb

@@ -178,7 +178,7 @@ class SajTest < ::Test::Unit::TestCase
     json = %{12345xyz}
     Oj.saj_parse(handler, json)
     assert_equal([[:add_value, 12345, nil],
-                  [:error, "invalid format, extra characters at line 1, column 6 [saj.c:711]", 1, 6]], handler.calls)
+                  [:error, "invalid format, extra characters at line 1, column 6 [saj.c:705]", 1, 6]], handler.calls)
   end
 
 end

data/test/tests.rb

@@ -40,13 +40,13 @@ class Jam
   end
   alias == eql?
 
-end # Jam
+end# Jam
 
 class Jeez < Jam
   def initialize(x, y)
     super
   end
-  
+
   def to_json()
     %{{"json_class":"#{self.class}","x":#{@x},"y":#{@y}}}
   end
@@ -54,7 +54,7 @@ class Jeez < Jam
   def self.json_create(h)
     self.new(h['x'], h['y'])
   end
-end # Jeez
+end# Jeez
 
 # contributed by sauliusg to fix as_json
 class Orange < Jam
@@ -97,7 +97,7 @@ class Jazz < Jam
   def self.json_create(h)
     self.new(h['x'], h['y'])
   end
-end # Jazz
+end# Jazz
 
 class Range
   def to_hash()
@@ -129,7 +129,7 @@ class Juice < ::Test::Unit::TestCase
                    :auto_define=>false,
                    :symbol_keys=>false,
                    :class_cache=>true,
-                   :ascii_only=>false,
+                   :escape_mode=>:json,
                    :mode=>:object,
                    :time_format=>:unix,
                    :bigdecimal_as_decimal=>true,
@@ -145,7 +145,7 @@ class Juice < ::Test::Unit::TestCase
       :auto_define=>false,
       :symbol_keys=>false,
       :class_cache=>true,
-      :ascii_only=>false,
+      :escape_mode=>:ascii,
       :mode=>:object,
       :time_format=>:unix,
       :bigdecimal_as_decimal=>true,
@@ -158,7 +158,7 @@ class Juice < ::Test::Unit::TestCase
       :auto_define=>true,
       :symbol_keys=>true,
       :class_cache=>false,
-      :ascii_only=>true,
+      :escape_mode=>:json,
       :mode=>:compat,
       :time_format=>:ruby,
       :bigdecimal_as_decimal=>false,
@@ -250,6 +250,36 @@ class Juice < ::Test::Unit::TestCase
     dump_and_load([[nil], 58], false)
   end
 
+  # rails encoding tests
+  def test_does_not_escape_entities_by_default
+    # use Oj to create the hash since some Rubies don't deal nicely with unicode.
+    json = %{{"key":"I <3 this\\u2028space"}}
+    hash = Oj.load(json)
+    out = Oj.dump(hash)
+    assert_equal(json, out)
+  end
+  def test_escapes_entities_by_default_when_configured_to_do_so
+    hash = {'key' => "I <3 this"}
+    Oj.default_options = {:escape_mode => :xss_safe}
+    out = Oj.dump hash
+    assert_equal(%{{"key":"I \\u003c3 this"}}, out)
+  end
+  def test_escapes_entities_when_asked_to
+    hash = {'key' => "I <3 this"}
+    out = Oj.dump(hash, :escape_mode => :xss_safe)
+    assert_equal(%{{"key":"I \\u003c3 this"}}, out)
+  end
+  def test_does_not_escape_entities_when_not_asked_to
+    hash = {'key' => "I <3 this"}
+    out = Oj.dump(hash, :escape_mode => :json)
+    assert_equal(%{{"key":"I <3 this"}}, out)
+  end
+  def test_escapes_common_xss_vectors
+    hash = {'key' => "<script>alert(123) && formatHD()</script>"}
+    out = Oj.dump(hash, :escape_mode => :xss_safe)
+    assert_equal(%{{"key":"\\u003cscript\\u003ealert(123) \\u0026\\u0026 formatHD()\\u003c\\/script\\u003e"}}, out)
+  end
+
   # Symbol
   def test_symbol_strict
     begin
@@ -267,7 +297,7 @@ class Juice < ::Test::Unit::TestCase
   def test_symbol_compat
     json = Oj.dump(:abc, :mode => :compat)
     assert_equal('"abc"', json)
-  end    
+  end
   def test_symbol_object
     Oj.default_options = { :mode => :object }
     #dump_and_load(''.to_sym, false)
@@ -295,7 +325,7 @@ class Juice < ::Test::Unit::TestCase
     #t = Time.local(2012, 1, 5, 23, 58, 7, 123456)
     json = Oj.dump(t, :mode => :compat)
     assert_equal(%{1325775487.123456000}, json)
-  end    
+  end
   def test_unix_time_compat_precision
     t = Time.xmlschema("2012-01-05T23:58:07.123456789+09:00")
     #t = Time.local(2012, 1, 5, 23, 58, 7, 123456)
@@ -304,23 +334,23 @@ class Juice < ::Test::Unit::TestCase
     t = Time.xmlschema("2012-01-05T23:58:07.999600+09:00")
     json = Oj.dump(t, :mode => :compat, :second_precision => 3)
     assert_equal(%{1325775488.000}, json)
-  end    
+  end
   def test_unix_time_compat_early
     t = Time.xmlschema("1954-01-05T00:00:00.123456789+00:00")
     json = Oj.dump(t, :mode => :compat, :second_precision => 5)
     assert_equal(%{-504575999.87654}, json)
-  end    
+  end
   def test_unix_time_compat_1970
     t = Time.xmlschema("1970-01-01T00:00:00.123456789+00:00")
     json = Oj.dump(t, :mode => :compat, :second_precision => 5)
     assert_equal(%{0.12346}, json)
-  end    
+  end
   def test_ruby_time_compat
     t = Time.xmlschema("2012-01-05T23:58:07.123456000+09:00")
     json = Oj.dump(t, :mode => :compat, :time_format => :ruby)
     #assert_equal(%{"2012-01-05 23:58:07 +0900"}, json)
     assert_equal(%{"#{t.to_s}"}, json)
-  end    
+  end
   def test_xml_time_compat
     begin
       t = Time.new(2012, 1, 5, 23, 58, 7.123456000, 34200)
@@ -339,7 +369,7 @@ class Juice < ::Test::Unit::TestCase
       end
       assert_equal(%{"2012-01-05T23:58:07.123456000%s%02d:%02d"} % [sign, tz / 3600, tz / 60 % 60], json)
     end
-  end    
+  end
   def test_xml_time_compat_no_secs
     begin
       t = Time.new(2012, 1, 5, 23, 58, 7.0, 34200)
@@ -358,7 +388,7 @@ class Juice < ::Test::Unit::TestCase
       end
       assert_equal(%{"2012-01-05T23:58:07%s%02d:%02d"} % [sign, tz / 3600, tz / 60 % 60], json)
     end
-  end    
+  end
   def test_xml_time_compat_precision
     begin
       t = Time.new(2012, 1, 5, 23, 58, 7.123456789, 32400)
@@ -377,7 +407,7 @@ class Juice < ::Test::Unit::TestCase
       end
       assert_equal(%{"2012-01-05T23:58:07.12346%s%02d:%02d"} % [sign, tz / 3600, tz / 60 % 60], json)
     end
-  end    
+  end
   def test_xml_time_compat_precision_round
     begin
       t = Time.new(2012, 1, 5, 23, 58, 7.9996, 32400)
@@ -396,7 +426,7 @@ class Juice < ::Test::Unit::TestCase
       end
       assert_equal(%{"2012-01-05T23:58:08%s%02d:%02d"} % [sign, tz / 3600, tz / 60 % 60], json)
     end
-  end    
+  end
   def test_xml_time_compat_zulu
     begin
       t = Time.new(2012, 1, 5, 23, 58, 7.0, 0)
@@ -409,7 +439,7 @@ class Juice < ::Test::Unit::TestCase
       #tz = t.utc_offset
       assert_equal(%{"2012-01-05T23:58:07Z"}, json)
     end
-  end    
+  end
   def test_time_object
     t = Time.now()
     Oj.default_options = { :mode => :object }
@@ -437,7 +467,7 @@ class Juice < ::Test::Unit::TestCase
   def test_class_compat
     json = Oj.dump(Juice, :mode => :compat)
     assert_equal(%{"Juice"}, json)
-  end    
+  end
   def test_class_object
     Oj.default_options = { :mode => :object }
     dump_and_load(Juice, false)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 2.1.7
+  version: 2.2.0
 platform: ruby
 authors:
 - Peter Ohler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-19 00:00:00.000000000 Z
+date: 2013-11-11 00:00:00.000000000 Z
 dependencies: []
 description: 'The fastest JSON parser and object serializer. '
 email: peter@ohler.com
@@ -84,6 +84,7 @@ files:
 - test/sample_json.rb
 - test/test_compat.rb
 - test/test_fast.rb
+- test/test_gc.rb
 - test/test_mimic.rb
 - test/test_mimic_after.rb
 - test/test_object.rb