oj 2.9.9 → 2.10.0

data/ext/oj/sparse.c

@@ -49,7 +49,7 @@
 #define NUM_MAX		(FIXNUM_MAX >> 8)
 #endif
 #define EXP_MAX		1023
-#define DEC_MAX		14
+#define DEC_MAX		15
 
 static void
 skip_comment(ParseInfo pi) {
@@ -99,7 +99,7 @@ add_value(ParseInfo pi, VALUE rval) {
 	    parent->next = NEXT_ARRAY_COMMA;
 	    break;
 	case NEXT_HASH_VALUE:
-	    pi->hash_set_value(pi, parent->key, parent->klen, rval);
+	    pi->hash_set_value(pi, parent, rval);
 	    if (parent->kalloc) {
 		xfree((char*)parent->key);
 	    }
@@ -134,7 +134,7 @@ add_num_value(ParseInfo pi, NumInfo ni) {
 	    parent->next = NEXT_ARRAY_COMMA;
 	    break;
 	case NEXT_HASH_VALUE:
-	    pi->hash_set_num(pi, parent->key, parent->klen, ni);
+	    pi->hash_set_num(pi, parent, ni);
 	    if (parent->kalloc) {
 		xfree((char*)parent->key);
 	    }
@@ -302,13 +302,18 @@ read_escaped_str(ParseInfo pi) {
 	    break;
 	case NEXT_HASH_NEW:
 	case NEXT_HASH_KEY:
-	    parent->key = strdup(buf.head);
-	    parent->klen = buf_len(&buf);
+	    if (Qundef == (parent->key_val = pi->hash_key(pi, buf.head, buf_len(&buf)))) {
+		parent->key = strdup(buf.head);
+		parent->klen = buf_len(&buf);
+	    } else {
+		parent->key = "";
+		parent->klen = 0;
+	    }
 	    parent->k1 = *pi->rd.str;
 	    parent->next = NEXT_HASH_COLON;
 	    break;
 	case NEXT_HASH_VALUE:
-	    pi->hash_set_cstr(pi, parent->key, parent->klen, buf.head, buf_len(&buf), pi->rd.str);
+	    pi->hash_set_cstr(pi, parent, buf.head, buf_len(&buf), pi->rd.str);
 	    if (parent->kalloc) {
 		xfree((char*)parent->key);
 	    }
@@ -366,11 +371,12 @@ read_str(ParseInfo pi) {
 		parent->key = parent->karray;
 		parent->kalloc = 0;
 	    }
+	    parent->key_val = pi->hash_key(pi, parent->key, parent->klen);
 	    parent->k1 = *pi->rd.str;
 	    parent->next = NEXT_HASH_COLON;
 	    break;
 	case NEXT_HASH_VALUE:
-	    pi->hash_set_cstr(pi, parent->key, parent->klen, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
+	    pi->hash_set_cstr(pi, parent, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
 	    if (parent->kalloc) {
 		xfree((char*)parent->key);
 	    }

data/ext/oj/strict.c

@@ -42,6 +42,11 @@ static void
 noop_end(struct _ParseInfo *pi) {
 }
 
+static VALUE
+noop_hash_key(struct _ParseInfo *pi, const char *key, size_t klen) {
+    return Qundef;
+}
+
 static void
 add_value(ParseInfo pi, VALUE val) {
     pi->stack.head->val = val;
@@ -69,9 +74,12 @@ start_hash(ParseInfo pi) {
 }
 
 static VALUE
-hash_key(ParseInfo pi, const char *key, size_t klen) {
-    volatile VALUE	rkey = rb_str_new(key, klen);
+calc_hash_key(ParseInfo pi, Val parent) {
+    volatile VALUE	rkey = parent->key_val;
 
+    if (Qundef == rkey) {
+	rkey = rb_str_new(parent->key, parent->klen);
+    }
     rkey = oj_encode(rkey);
     if (Yes == pi->options.sym_key) {
 	rkey = rb_str_intern(rkey);
@@ -80,24 +88,24 @@ hash_key(ParseInfo pi, const char *key, size_t klen) {
 }
 
 static void
-hash_set_cstr(ParseInfo pi, const char *key, size_t klen, const char *str, size_t len, const char *orig) {
+hash_set_cstr(ParseInfo pi, Val parent, const char *str, size_t len, const char *orig) {
     volatile VALUE	rstr = rb_str_new(str, len);
 
     rstr = oj_encode(rstr);
-    rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), rstr);
+    rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), rstr);
 }
 
 static void
-hash_set_num(struct _ParseInfo *pi, const char *key, size_t klen, NumInfo ni) {
+hash_set_num(struct _ParseInfo *pi, Val parent, NumInfo ni) {
     if (ni->infinity || ni->nan) {
 	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
     }
-    rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), oj_num_as_value(ni));
+    rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), oj_num_as_value(ni));
 }
 
 static void
-hash_set_value(ParseInfo pi, const char *key, size_t klen, VALUE value) {
-    rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), value);
+hash_set_value(ParseInfo pi, Val parent, VALUE value) {
+    rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), value);
 }
 
 static VALUE
@@ -130,6 +138,7 @@ void
 oj_set_strict_callbacks(ParseInfo pi) {
     pi->start_hash = start_hash;
     pi->end_hash = noop_end;
+    pi->hash_key = noop_hash_key;
     pi->hash_set_cstr = hash_set_cstr;
     pi->hash_set_num = hash_set_num;
     pi->hash_set_value = hash_set_value;

data/ext/oj/val_stack.c

@@ -69,6 +69,7 @@ oj_stack_init(ValStack stack) {
     stack->tail = stack->head;
     stack->head->val = Qundef;
     stack->head->key = 0;
+    stack->head->key_val = Qundef;
     stack->head->classname = 0;
     stack->head->klen = 0;
     stack->head->clen = 0;

data/ext/oj/val_stack.h

@@ -53,9 +53,10 @@ typedef enum {
 } ValNext;
 
 typedef struct _Val {
-    VALUE		val;
+    volatile VALUE	val;
     const char		*key;
     char		karray[32];
+    volatile VALUE	key_val;
     union {
 	const char	*classname;
 	OddArgs		odd_args;
@@ -127,6 +128,7 @@ stack_push(ValStack stack, VALUE val, ValNext next) {
     stack->tail->next = next;
     stack->tail->classname = 0;
     stack->tail->key = 0;
+    stack->tail->key_val = Qundef;
     stack->tail->clen = 0;
     stack->tail->klen = 0;
     stack->tail->kalloc = 0;

data/lib/oj/schandler.rb

@@ -45,6 +45,7 @@ module Oj
   #
   #    def hash_start(); end
   #    def hash_end(); end
+  #    def hash_key(key); end
   #    def hash_set(h, key, value); end
   #    def array_start(); end
   #    def array_end(); end
@@ -63,6 +64,12 @@ module Oj
   #
   #    hash_end
   #
+  # When a hash key is encountered the hash_key method is called with the parsed
+  # hash value key. The return value from the call is then used as the key in
+  # the key-value pair that follows.
+  #
+  #    hash_key
+  #
   # At the end of a JSON object element the hash_end() callback is called if public.
   #
   #    hash_set
@@ -112,6 +119,10 @@ module Oj
     def hash_end()
     end
 
+    def hash_key(key)
+      key
+    end
+
     def hash_set(h, key, value)
     end
 

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '2.9.9'
+  VERSION = '2.10.0'
 end

data/test/bug.rb

@@ -1,3 +1,5 @@
+#!/usr/bin/env ruby
+
 #!/usr/bin/env ruby
 # encoding: UTF-8
 
@@ -5,60 +7,47 @@ $: << File.dirname(__FILE__)
 
 require 'helper'
 
-require 'oj'
-require 'securerandom'
-
 class Handler
-  def hash_start()      {} end
-  def hash_set(h,k,v)   h.store(k,v) end
-  def array_start()     [] end
-  def array_append(a,v) a << v end
-  def error(message, line, column)
-    raise Exception.new(message, line, column)
+  def initialize
+    @state = []
+  end
+
+  def hash_start
+    @state << {}
+    @state.last
+  end
+
+  def hash_end
+    @state.pop
+  end
+
+  def hash_set(h,k,v)
+    h.store(k,v)
   end
-end
 
-json = Oj.dump({"this"=>"object"})
-
-if true
-  name = "/tmp/#{SecureRandom.uuid}"
-  `mkfifo #{name}`
-  if fork
-    open(name, 'r+') do |read_io|
-      p "start reading #{read_io.stat.ftype}"
-      Oj.sc_parse(Handler.new, read_io) {|v| p v}
-      p "stop reading"
-    end
-  else
-    open(name, 'w+') do |write_io|
-      p "start writing #{write_io.stat.ftype}  autoclose: #{write_io.autoclose?}"
-      write_io.write json
-      write_io.write json
-      p "stop writing"
-    end
-    sleep(1) # make it obvious that there are two threads
-    open(name, 'w+') do |write_io|
-      p "start writing #{write_io.stat.ftype}"
-      write_io.write json
-      write_io.write json
-      p "stop writing"
-    end
+  def array_start
+    @state << []
+    @state.last
   end
-else
-  IO.pipe do |read_io, write_io|
-    if fork
-      write_io.close
-      p "start reading #{read_io.stat.ftype}"
-      Oj.sc_parse(Handler.new, read_io) {|v| p v}
-      p "stop reading"
-      read_io.close
-    else
-      read_io.close
-      p "start writing #{write_io.stat.ftype}"
-      write_io.write json
-      write_io.write json
-      p "stop writing"
-      write_io.close
-    end
+
+
+  def array_end
+    @state.pop
+  end
+
+  def array_append(a,v)
+    a << v
+  end
+
+  def add_value(v)
+    p v
   end
+
+  def error(message, line, column); p "ERROR: #{message}" end
 end
+
+$handler = Handler.new
+
+IO.popen("cat tst") { |p| puts Oj.sc_parse($handler, p) }
+
+#File.open('tst', 'r') { |file|  Oj.sc_parse($handler, file) }

data/test/bug2.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(__FILE__)
+
+require 'helper'
+
+

data/test/io.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.dirname(__FILE__)
+
+require 'helper'
+
+class Handler
+  def initialize
+    @state = []
+  end
+
+  def hash_start
+    @state << {}
+    @state.last
+  end
+
+  def hash_end
+    @state.pop
+  end
+
+  def hash_set(h,k,v)
+    h.store(k,v)
+  end
+
+  def array_start
+    @state << []
+    @state.last
+  end
+
+
+  def array_end
+    @state.pop
+  end
+
+  def array_append(a,v)
+    a << v
+  end
+
+  def error(message, line, column); p "ERROR: #{message}" end
+end
+
+handler = Handler.new
+def handler.add_value(v)
+  p v
+end
+
+Oj.sc_parse(handler, StringIO.new('{"a":"b","c":[1,2,{"d":"e"}]}[4,5,6]'))

data/test/struct.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
+# required. That can be set in the RUBYOPT environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+A = Struct.new(:a,:b,:c,:d)
+B = Struct.new(:e,:f)
+
+obj = [A.new(55, B.new(1, 'X'), B.new(2, 'Y'), 3)]
+
+s = Oj.dump(obj, :mode => :object)
+
+100000.times do
+  Oj.load(s, :mode => :object)
+  # ds = Oj.dump(o, :mode => :object)
+  # if ds != s
+  #   puts ds
+  #   raise "holy crap"
+  # end
+end

data/test/test_compat.rb

@@ -90,6 +90,8 @@ class CompatJuice < Minitest::Test
     dump_and_load(12345.6789, false)
     dump_and_load(70.35, false)
     dump_and_load(-54321.012, false)
+    dump_and_load(1.7775, false)
+    dump_and_load(2.5024, false)
     dump_and_load(2.48e16, false)
     dump_and_load(2.48e100 * 1.0e10, false)
     dump_and_load(-2.48e100 * 1.0e10, false)

data/test/test_file.rb

@@ -85,6 +85,8 @@ class FileJuice < Minitest::Test
     dump_and_load(12345.6789, false)
     dump_and_load(70.35, false)
     dump_and_load(-54321.012, false)
+    dump_and_load(1.7775, false)
+    dump_and_load(2.5024, false)
     dump_and_load(2.48e16, false)
     dump_and_load(2.48e100 * 1.0e10, false)
     dump_and_load(-2.48e100 * 1.0e10, false)

data/test/test_object.rb

@@ -163,6 +163,8 @@ class ObjectJuice < Minitest::Test
     dump_and_load(12345.6789, false)
     dump_and_load(70.35, false)
     dump_and_load(-54321.012, false)
+    dump_and_load(1.7775, false)
+    dump_and_load(2.5024, false)
     dump_and_load(2.48e16, false)
     dump_and_load(2.48e100 * 1.0e10, false)
     dump_and_load(-2.48e100 * 1.0e10, false)

data/test/test_scp.rb

@@ -41,6 +41,13 @@ class AllHandler < Oj::ScHandler
     @calls << [:hash_end]
   end
 
+  def hash_key(key)
+    @calls << [:hash_key, key]
+    return 'too' if 'two' == key
+    return :symbol if 'symbol' == key
+    key
+  end
+
   def array_start()
     @calls << [:array_start]
     []
@@ -159,8 +166,10 @@ class ScpTest < Minitest::Test
     json = %{{"one":true,"two":false}}
     Oj.sc_parse(handler, json)
     assert_equal([[:hash_start],
+                  [:hash_key, 'one'],
                   [:hash_set, 'one', true],
-                  [:hash_set, 'two', false],
+                  [:hash_key, 'two'],
+                  [:hash_set, 'too', false],
                   [:hash_end],
                   [:add_value, {}]], handler.calls)
   end
@@ -170,8 +179,23 @@ class ScpTest < Minitest::Test
     json = %{{"one":true,"two":false}}
     Oj.sc_parse(handler, json, :symbol_keys => true)
     assert_equal([[:hash_start],
-                  [:hash_set, :one, true],
-                  [:hash_set, :two, false],
+                  [:hash_key, 'one'],
+                  [:hash_set, 'one', true],
+                  [:hash_key, 'two'],
+                  [:hash_set, 'too', false],
+                  [:hash_end],
+                  [:add_value, {}]], handler.calls)
+  end
+
+  def test_symbol_hash_key_without_symbol_keys
+    handler = AllHandler.new()
+    json = %{{"one":true,"symbol":false}}
+    Oj.sc_parse(handler, json)
+    assert_equal([[:hash_start],
+                  [:hash_key, 'one'],
+                  [:hash_set, 'one', true],
+                  [:hash_key, 'symbol'],
+                  [:hash_set, :symbol, false],
                   [:hash_end],
                   [:add_value, {}]], handler.calls)
   end
@@ -180,12 +204,18 @@ class ScpTest < Minitest::Test
     handler = AllHandler.new()
     Oj.sc_parse(handler, $json)
     assert_equal([[:hash_start],
+                  [:hash_key, 'array'],
                   [:array_start],
                   [:hash_start],
+                  [:hash_key, 'num'],
                   [:hash_set, "num", 3],
+                  [:hash_key, 'string'],
                   [:hash_set, "string", "message"],
+                  [:hash_key, 'hash'],
                   [:hash_start],
+                  [:hash_key, 'h2'],
                   [:hash_start],
+                  [:hash_key, 'a'],
                   [:array_start],
                   [:array_append, 1],
                   [:array_append, 2],
@@ -200,6 +230,7 @@ class ScpTest < Minitest::Test
                   [:array_append, {}],
                   [:array_end],
                   [:hash_set, "array", []],
+                  [:hash_key, 'boolean'],
                   [:hash_set, "boolean", true],
                   [:hash_end],
                   [:add_value, {}]], handler.calls)
@@ -210,12 +241,16 @@ class ScpTest < Minitest::Test
     json = %{{"one":true,"two":false}{"three":true,"four":false}}
     Oj.sc_parse(handler, json)
     assert_equal([[:hash_start],
+                  [:hash_key, 'one'],
                   [:hash_set, 'one', true],
-                  [:hash_set, 'two', false],
+                  [:hash_key, 'two'],
+                  [:hash_set, 'too', false],
                   [:hash_end],
                   [:add_value, {}],
                   [:hash_start],
+                  [:hash_key, 'three'],
                   [:hash_set, 'three', true],
+                  [:hash_key, 'four'],
                   [:hash_set, 'four', false],
                   [:hash_end],
                   [:add_value, {}]], handler.calls)
@@ -253,8 +288,10 @@ class ScpTest < Minitest::Test
         Oj.sc_parse(handler, read_io) {|v| p v}
         read_io.close
         assert_equal([[:hash_start],
+                      [:hash_key, 'one'],
                       [:hash_set, 'one', true],
-                      [:hash_set, 'two', false],
+                      [:hash_key, 'two'],
+                      [:hash_set, 'too', false],
                       [:hash_end],
                       [:add_value, {}]], handler.calls)
       else