oj 2.9.0 → 2.9.1

data/ext/oj/strict.c

@@ -145,7 +145,11 @@ oj_strict_parse(int argc, VALUE *argv, VALUE self) {
     pi.options = oj_default_options;
     oj_set_strict_callbacks(&pi);
 
-    return oj_pi_parse(argc, argv, &pi, 0, 0);
+    if (T_STRING == rb_type(*argv)) {
+	return oj_pi_parse(argc, argv, &pi, 0, 0, 1);
+    } else {
+	return oj_pi_sparse(argc, argv, &pi, 0);
+    }
 }
 
 VALUE
@@ -155,5 +159,5 @@ oj_strict_parse_cstr(int argc, VALUE *argv, char *json, size_t len) {
     pi.options = oj_default_options;
     oj_set_strict_callbacks(&pi);
 
-    return oj_pi_parse(argc, argv, &pi, json, len);
+    return oj_pi_parse(argc, argv, &pi, json, len, 1);
 }

data/ext/oj/val_stack.h

@@ -55,6 +55,7 @@ typedef enum {
 typedef struct _Val {
     VALUE		val;
     const char		*key;
+    char		karray[32];
     union {
 	const char	*classname;
 	OddArgs		odd_args;
@@ -63,6 +64,7 @@ typedef struct _Val {
     uint16_t		clen;
     char		next; // ValNext
     char		k1;   // first original character in the key
+    char		kalloc;
 } *Val;
 
 typedef struct _ValStack {
@@ -127,6 +129,7 @@ stack_push(ValStack stack, VALUE val, ValNext next) {
     stack->tail->key = 0;
     stack->tail->clen = 0;
     stack->tail->klen = 0;
+    stack->tail->kalloc = 0;
     stack->tail++;
 }
 

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '2.9.0'
+  VERSION = '2.9.1'
 end

data/test/a.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << File.dirname(__FILE__)
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'pp'
+require 'oj'
+require 'perf'
+
+obj = [[1],[2],[3],[4],[5],[6],[7],[8],[9]]
+obj = [[],[],[],[],[],[],[],[],[]]
+obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [12345, nil], 3.967, ['something', false], nil]], # mix it up array
+  'e' => { 'one' => 1, 'two' => 2 }, # hash
+  'f' => nil,     # nil
+  'g' => 12345678901234567890123456789, # big number
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+json = Oj.dump(obj, mode: :compat)
+
+puts json
+#pp Oj.saj_parse(nil, json)
+pp Oj.t_parse(json)
+
+if true
+  perf = Perf.new()
+  perf.add('SAJ', 'oj') { Oj.saj_parse(nil, json) }
+  perf.add('T', 'oj') { Oj.t_parse(json) }
+  perf.add('load', 'oj') { Oj.load(json) }
+  perf.run(10000)
+end

data/test/bug.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
+# required. That can be set in the RUBYOPT environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+obj = Oj.load_file('bug.json', :mode => :object)
+
+puts Oj.dump(obj, :mode => :object, :indent => 0)

data/test/e.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+json = %{"\xc2\xa9\xc3\x98"}
+puts "original:\n#{json}"
+
+str = Oj.load(json)
+puts "out: #{str}"

data/test/foo.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
+# required. That can be set in the RUBYOPT environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+class Foo < String
+  def initialize(str, safe)
+    super(str)
+    @safe = safe
+  end
+end
+
+f = Foo.new("z")
+
+puts f

data/test/lots.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
+# required. That can be set in the RUBYOPT environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+module One
+  module Two
+    module Three
+      class Empty
+
+        def initialize()
+        end
+
+        def eql?(o)
+          self.class == o.class
+        end
+        alias == eql?
+
+        def to_hash()
+          {'json_class' => "#{self.class.name}"}
+        end
+
+        def to_json(*a)
+          %{{"json_class":"#{self.class.name}"}}
+        end
+
+        def self.json_create(h)
+          self.new()
+        end
+      end # Empty
+    end # Three
+  end # Two
+end # One
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'g' => One::Two::Three::Empty.new(),
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+$obj = [$obj]*10000
+
+Oj.default_options = { :indent => 2, :mode => :compat }
+
+$json = Oj.dump($obj, :mode => :compat)
+
+$result = nil
+100.times { |i|
+  print(".") if (0 == i % 10)
+  $result = Oj.compat_load($json)
+}
+
+

data/test/mj.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+# $: << File.join(File.dirname(__FILE__), "../../multi_json/lib")
+
+require 'multi_json'
+require 'benchmark'
+require 'yajl'
+require 'json'
+require 'oj'
+
+iter = 1_000_000
+iter = 100_000
+
+json = %({"k1":"val1","k2":"val2","k3":"val3"})
+obj = { k1: "val1", k2: "val2", k3: "val3" }
+
+puts "Benchmarks for different JSON handlers with MultiJson."
+puts "  Ruby #{RUBY_VERSION}"
+puts "  #{iter} iterations"
+
+MultiJson.engine = :oj
+dt = Benchmark.realtime { iter.times { MultiJson.decode(json) }}
+et = Benchmark.realtime { iter.times { MultiJson.encode(obj) }}
+puts "    Oj decode: #{dt}  encode: #{et}"
+
+MultiJson.engine = :yajl
+dt = Benchmark.realtime { iter.times { MultiJson.decode(json) }}
+et = Benchmark.realtime { iter.times { MultiJson.encode(obj) }}
+puts "  Yajl decode: #{dt}  encode: #{et}"
+
+MultiJson.engine = :json_gem
+dt = Benchmark.realtime { iter.times { MultiJson.decode(json) }}
+et = Benchmark.realtime { iter.times { MultiJson.encode(obj) }}
+puts "  Json decode: #{dt}  encode: #{et}"
+
+Oj.default_options = { :mode => :compat, :time_format => :ruby }
+dt = Benchmark.realtime { iter.times { Oj.load(json) }}
+et = Benchmark.realtime { iter.times { Oj.dump(obj) }}
+puts "Raw Oj decode: #{dt}  encode: #{et}"
+
+ye = Yajl::Encoder.new
+dt = Benchmark.realtime { iter.times { Yajl::Parser.parse(json) }}
+et = Benchmark.realtime { iter.times { Yajl::Encoder.encode(obj) }}
+e2 = Benchmark.realtime { iter.times { ye.encode(obj) }}
+puts "Raw Yajl decode: #{dt} encode: #{et}, encoder: #{e2}"

data/test/perf_file.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby -wW1
+
+$: << '.'
+$: << '../lib'
+$: << '../ext'
+
+if __FILE__ == $0
+  if (i = ARGV.index('-I'))
+    x,path = ARGV.slice!(i, 2)
+    $: << path
+  end
+end
+
+require 'optparse'
+require 'oj'
+require 'perf'
+
+$indent = 0
+$iter = 1
+$size = 1
+
+opts = OptionParser.new
+
+opts.on("-r", "read")                                       { do_read = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")       { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indent")          { |i| $indent = i }
+opts.on("-s", "--size [Int]", Integer, "size in Mbytes")    { |s| $size = s }
+
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'g' => 12345678901234567890123456789, #_bignum
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+json = Oj.dump($obj, :indent => $indent)
+cnt = ($size * 1024 * 1024 + json.size) / json.size
+cnt = 1 if 0 == $size
+
+filename = 'tmp.json'
+File.open(filename, "w") { |f|
+  cnt.times do
+    Oj.to_stream(f, $obj, :indent => $indent)
+  end
+}
+
+Oj.default_options = { :mode => :strict, :indent => $indent }
+
+puts '-' * 80
+puts "Read from #{cnt * json.size / (1024 * 1024)} Mb file Performance"
+perf = Perf.new()
+perf.add('Oj.load_file', '') { Oj.load_file(filename) }
+perf.add('Oj.load(string)', '') { Oj.load(File.read(filename)) }
+perf.add('Oj.load(file)', '') { File.open(filename, 'r') { |f| Oj.load(f) } }
+perf.run($iter)
+

data/test/perf_object.rb

@@ -74,7 +74,7 @@ if files.empty?
 
   $mars = Marshal.dump($obj)
   $xml = Ox.dump($obj, :indent => $indent, :circular => $circular)
-  $json = Oj.dump($obj, :indent => $indent, :circular => $circular)
+  $json = Oj.dump($obj, :indent => $indent, :circular => $circular, :mode => :object)
   File.open('sample.xml', 'w') { |f| f.write($xml) }
   File.open('sample.json', 'w') { |f| f.write($json) }
   File.open('sample.marshal', 'w') { |f| f.write($mars) }
@@ -119,6 +119,7 @@ if do_read
   puts "Read from file Performance"
   perf = Perf.new()
   perf.add('Oj', 'load') { Oj.load_file('sample.json') }
+  #perf.add('Oj', 'load') { Oj.load(File.read('sample.json')) }
   perf.add('Ox', 'load_file') { Ox.load_file('sample.xml', :mode => :object) }
   perf.add('Marshal', 'load') { Marshal.load(File.new('sample.marshal')) }
   perf.run($iter)

data/test/perf_str.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby -wW1
+
+$: << '.'
+$: << '../lib'
+$: << '../ext'
+
+if __FILE__ == $0
+  if (i = ARGV.index('-I'))
+    x,path = ARGV.slice!(i, 2)
+    $: << path
+  end
+end
+
+require 'optparse'
+require 'ox'
+require 'oj'
+require 'perf'
+
+$indent = 0
+$iter = 1000
+
+opts = OptionParser.new
+
+opts.on("-i", "--iterations [Int]", Integer, "iterations")  { |i| $iter = i }
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = "x" * 50_000
+
+Oj.default_options = { :mode => :strict, :indent => $indent }
+
+puts '-' * 80
+puts "Dump Performance"
+perf = Perf.new()
+perf.add('Oj', 'dump') { Oj.dump($obj) }
+perf.add('Ox', 'dump') { Ox.dump($obj, :indent => $indent, :circular => $circular) }
+perf.add('Marshal', 'dump') { Marshal.dump($obj) }
+perf.run($iter)

data/test/perf_strictx.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'stringio'
+require 'optparse'
+require 'perf'
+require 'oj'
+
+$verbose = false
+$indent = 0
+$iter = 20000
+$with_bignum = false
+$with_nums = true
+$size = 0
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                    { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")       { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")     { |i| $indent = i }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
+opts.on("-b", "with bignum")                                { $with_bignum = true }
+opts.on("-n", "without numbers")                            { $with_nums = false }
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+if $with_nums
+  $obj = {
+    'a' => 'Alpha', # string
+    'b' => true,    # boolean
+    'c' => 12345,   # number
+    'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+    'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+    'f' => nil,     # nil
+    'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+    'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+  }
+  $obj['g'] = 12345678901234567890123456789 if $with_bignum
+else
+  $obj = {
+    'a' => 'Alpha',
+    'b' => true,
+    'c' => '12345',
+    'd' => [ true, [false, ['12345', nil], '3.967', ['something', false], nil]],
+    'e' => { 'zero' => '0', 'one' => '1', 'two' => '2' },
+    'f' => nil,
+    'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+    'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+  }
+end
+
+Oj.default_options = { :indent => $indent, :mode => :strict }
+
+if 0 < $size
+  o = $obj
+  $obj = []
+  (4 * $size).times do
+    $obj << o
+  end
+end
+
+$json = Oj.dump($obj)
+$obj_json = Oj.dump($obj, :mode => :object)
+#puts "*** size: #{$obj_json.size}"
+$failed = {} # key is same as String used in tests later
+
+def capture_error(tag, orig, load_key, dump_key, &blk)
+  begin
+    obj = blk.call(orig)
+    raise "#{tag} #{dump_key} and #{load_key} did not return the same object as the original." unless orig == obj
+  rescue Exception => e
+    $failed[tag] = "#{e.class}: #{e.message}"
+  end
+end
+
+# Verify that all packages dump and load correctly and return the same Object as the original.
+capture_error('Oj:strict-str', $obj, 'load', 'dump') { |o| Oj.strict_load(Oj.dump(o, :mode => :strict)) }
+capture_error('Oj:strict', $obj, 'load', 'dump') { |o| Oj.strict_load(StringIO.new(Oj.dump(o, :mode => :strict))) }
+
+puts '-' * 80
+puts "Strict Parse Performance"
+perf = Perf.new()
+perf.add('Oj:strict-str', 'strict_load-str') { Oj.strict_load($json) }
+perf.add('Oj:strict-io', 'strict_load') { Oj.strict_load(StringIO.new($json)) }
+perf.run($iter)
+
+puts
+puts '-' * 80
+puts
+
+unless $failed.empty?
+  puts "The following packages were not included for the reason listed"
+  $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }
+end