oj 2.1.3 → 2.1.4

data/test/perf_fast.rb

@@ -0,0 +1,164 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'yajl'
+require 'perf'
+require 'json'
+require 'json/ext'
+require 'oj'
+
+$verbose = false
+$indent = 0
+$iter = 10000
+$gets = 0
+$fetch = false
+$write = false
+$read = false
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                  { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")     { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")   { |i| $indent = i }
+opts.on("-g", "--gets [Int]", Integer, "number of gets")  { |i| $gets = i }
+opts.on("-f", "fetch")                                    { $fetch = true }
+opts.on("-w", "write")                                    { $write = true }
+opts.on("-r", "read")                                     { $read = true }
+opts.on("-h", "--help", "Show this display")              { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+# This just navigates to each leaf of the JSON structure.
+def dig(obj, &blk)
+  case obj
+  when Array
+    obj.each { |e| dig(e, &blk) }
+  when Hash
+    obj.values.each { |e| dig(e, &blk) }
+  else
+    blk.yield(obj)
+  end
+end
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, {'12345' => 12345, 'nil' => nil}, 3.967, { 'x' => 'something', 'y' => false, 'z' => true}, nil]], # mix it up array
+  'e' => { 'one' => 1, 'two' => 2 }, # hash
+  'f' => nil,     # nil
+  'g' => 12345678901234567890123456789, # big number
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+Oj.default_options = { :indent => $indent, :mode => :compat }
+
+$json = Oj.dump($obj)
+$failed = {} # key is same as String used in tests later
+
+def capture_error(tag, orig, load_key, dump_key, &blk)
+  begin
+    obj = blk.call(orig)
+    raise "#{tag} #{dump_key} and #{load_key} did not return the same object as the original." unless orig == obj
+  rescue Exception => e
+    $failed[tag] = "#{e.class}: #{e.message}"
+  end
+end
+
+# Verify that all packages dump and load correctly and return the same Object as the original.
+capture_error('Oj::Doc', $obj, 'load', 'dump') { |o| Oj::Doc.open(Oj.dump(o, :mode => :strict)) { |f| f.fetch() } }
+capture_error('Yajl', $obj, 'encode', 'parse') { |o| Yajl::Parser.parse(Yajl::Encoder.encode(o)) }
+capture_error('JSON::Ext', $obj, 'generate', 'parse') { |o| JSON.generator = JSON::Ext::Generator; JSON::Ext::Parser.new(JSON.generate(o)).parse }
+
+if $verbose
+  puts "json:\n#{$json}\n"
+end
+
+puts '-' * 80
+puts "Parse Performance"
+perf = Perf.new()
+perf.add('Oj::Doc', 'parse') { Oj::Doc.open($json) {|f| } } unless $failed.has_key?('Oj::Doc')
+perf.add('Yajl', 'parse') { Yajl::Parser.parse($json) } unless $failed.has_key?('Yajl')
+perf.add('JSON::Ext', 'parse') { JSON::Ext::Parser.new($json).parse } unless $failed.has_key?('JSON::Ext')
+perf.run($iter)
+
+puts '-' * 80
+puts "JSON generation Performance"
+Oj::Doc.open($json) do |doc|
+  perf = Perf.new()
+  perf.add('Oj::Doc', 'dump') { doc.dump() }
+  perf.add('Yajl', 'encode') { Yajl::Encoder.encode($obj) }
+  perf.add('JSON::Ext', 'fast_generate') { JSON.fast_generate($obj) }
+  perf.before('JSON::Ext') { JSON.generator = JSON::Ext::Generator }
+  perf.run($iter)
+end
+
+if 0 < $gets
+  puts '-' * 80
+  puts "Parse and get all values Performance"
+  perf = Perf.new()
+  perf.add('Oj::Doc', 'parse') { Oj::Doc.open($json) {|f| $gets.times { f.each_value() {} } } } unless $failed.has_key?('Oj::Doc')
+  perf.add('Yajl', 'parse') { $gets.times { dig(Yajl::Parser.parse($json)) {} } } unless $failed.has_key?('Yajl')
+  perf.add('JSON::Ext', 'parse') { $gets.times { dig(JSON::Ext::Parser.new($json).parse) {} } } unless $failed.has_key?('JSON::Ext')
+  perf.run($iter)
+end
+
+if $fetch
+  puts '-' * 80
+  puts "fetch nested Performance"
+  json_hash = Oj.load($json, :mode => :strict)
+  Oj::Doc.open($json) do |fast|
+    #puts "*** C fetch: #{fast.fetch('/d/2/4/y')}"
+    #puts "*** Ruby fetch: #{json_hash.fetch('d', []).fetch(1, []).fetch(3, []).fetch('x', nil)}"
+    perf = Perf.new()
+    perf.add('Oj::Doc', 'fetch') { fast.fetch('/d/2/4/x'); fast.fetch('/h/a/b/c/d/e/f/g'); fast.fetch('/i/1/1/1/1/1/1/1') }
+    # version that fails gracefully
+    perf.add('Ruby', 'fetch') do
+      json_hash.fetch('d', []).fetch(1, []).fetch(3, []).fetch('x', nil)
+      json_hash.fetch('h', {}).fetch('a', {}).fetch('b', {}).fetch('c', {}).fetch('d', {}).fetch('e', {}).fetch('f', {}).fetch('g', {})
+      json_hash.fetch('i', []).fetch(0, []).fetch(0, []).fetch(0, []).fetch(0, []).fetch(0, []).fetch(0, []).fetch(0, nil)
+    end
+    # version that raises if the path is incorrect
+#    perf.add('Ruby', 'fetch') { $fetch.times { json_hash['d'][1][3][1] } }
+    perf.run($iter)
+  end
+end
+
+if $write
+  puts '-' * 80
+  puts "JSON write to file Performance"
+  Oj::Doc.open($json) do |doc|
+    perf = Perf.new()
+    perf.add('Oj::Doc', 'dump') { doc.dump(nil, 'oj.json') }
+    perf.add('Yajl', 'encode') { File.open('yajl.json', 'w') { |f| Yajl::Encoder.encode($obj, f) } }
+    perf.add('JSON::Ext', 'fast_generate') { File.open('json_ext.json', 'w') { |f| f.write(JSON.fast_generate($obj)) } }
+    perf.before('JSON::Ext') { JSON.generator = JSON::Ext::Generator }
+    perf.run($iter)
+  end
+end
+
+if $read
+  puts '-' * 80
+  puts "JSON read from file Performance"
+  Oj::Doc.open($json) { |doc| doc.dump(nil, 'oj.json') }
+  File.open('yajl.json', 'w') { |f| Yajl::Encoder.encode($obj, f) }
+  JSON.generator = JSON::Ext::Generator
+  File.open('json_ext.json', 'w') { |f| f.write(JSON.fast_generate($obj)) }
+  Oj::Doc.open($json) do |doc|
+    perf = Perf.new()
+    perf.add('Oj::Doc', 'open_file') { ::Oj::Doc.open_file('oj.json') }
+    perf.add('Yajl', 'decode') { Yajl::decoder.decode(File.read('yajl.json')) }
+    perf.add('JSON::Ext', '') { JSON.parse(File.read('json_ext.json')) }
+    perf.before('JSON::Ext') { JSON.parser = JSON::Ext::Parser }
+    perf.run($iter)
+  end
+end
+
+unless $failed.empty?
+  puts "The following packages were not included for the reason listed"
+  $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }
+end

data/test/perf_object.rb

@@ -0,0 +1,136 @@
+#!/usr/bin/env ruby -wW1
+
+$: << '.'
+$: << '../lib'
+$: << '../ext'
+
+if __FILE__ == $0
+  if (i = ARGV.index('-I'))
+    x,path = ARGV.slice!(i, 2)
+    $: << path
+  end
+end
+
+require 'optparse'
+require 'ox'
+require 'oj'
+require 'perf'
+require 'sample'
+require 'files'
+
+$circular = false
+$indent = 0
+
+do_sample = false
+do_files = false
+
+do_load = false
+do_dump = false
+do_read = false
+do_write = false
+$iter = 1000
+$mult = 1
+
+opts = OptionParser.new
+opts.on("-c", "circular options")                           { $circular = true }
+
+opts.on("-x", "use sample instead of files")                { do_sample = true }
+
+opts.on("-s", "load and dump as sample Ruby object")        { do_sample = true }
+opts.on("-f", "load and dump as files Ruby object")         { do_files = true }
+
+opts.on("-l", "load")                                       { do_load = true }
+opts.on("-d", "dump")                                       { do_dump = true }
+opts.on("-r", "read")                                       { do_read = true }
+opts.on("-w", "write")                                      { do_write = true }
+opts.on("-a", "load, dump, read and write")                 { do_load = true; do_dump = true; do_read = true; do_write = true }
+
+opts.on("-i", "--iterations [Int]", Integer, "iterations")  { |i| $iter = i }
+opts.on("-m", "--multiply [Int]", Integer, "multiplier")    { |i| $mult = i }
+
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = nil
+$xml = nil
+$mars = nil
+$json = nil
+
+unless do_load || do_dump || do_read || do_write
+  do_load = true
+  do_dump = true
+  do_read = true
+  do_write = true
+end
+
+# prepare all the formats for input
+if files.empty?
+  $obj = []
+  $mult.times do
+    $obj << (do_sample ? sample_doc(2) : files('..'))
+  end
+
+  $mars = Marshal.dump($obj)
+  $xml = Ox.dump($obj, :indent => $indent, :circular => $circular)
+  $json = Oj.dump($obj, :indent => $indent, :circular => $circular)
+  File.open('sample.xml', 'w') { |f| f.write($xml) }
+  File.open('sample.json', 'w') { |f| f.write($json) }
+  File.open('sample.marshal', 'w') { |f| f.write($mars) }
+else
+  puts "loading and parsing #{files}\n\n"
+  # TBD change to allow xml and json
+  data = files.map do |f|
+    $xml = File.read(f)
+    $obj = Ox.load($xml);
+    $mars = Marshal.dump($obj)
+    $json = Oj.dump($obj, :indent => $indent, :circular => $circular)
+  end
+end
+
+Oj.default_options = { :mode => :object, :indent => $indent, :circular => $circular }
+#puts "json: #{$json.size}"
+#puts "xml: #{$xml.size}"
+#puts "marshal: #{$mars.size}"
+
+
+if do_load
+  puts '-' * 80
+  puts "Load Performance"
+  perf = Perf.new()
+  perf.add('Oj.object', 'load') { Oj.object_load($json) }
+  perf.add('Ox', 'load') { Ox.load($xml, :mode => :object) }
+  perf.add('Marshal', 'load') { Marshal.load($mars) }
+  perf.run($iter)
+end
+
+if do_dump
+  puts '-' * 80
+  puts "Dump Performance"
+  perf = Perf.new()
+  perf.add('Oj', 'dump') { Oj.dump($obj) }
+  perf.add('Ox', 'dump') { Ox.dump($obj, :indent => $indent, :circular => $circular) }
+  perf.add('Marshal', 'dump') { Marshal.dump($obj) }
+  perf.run($iter)
+end
+
+if do_read
+  puts '-' * 80
+  puts "Read from file Performance"
+  perf = Perf.new()
+  perf.add('Oj', 'load') { Oj.load_file('sample.json') }
+  perf.add('Ox', 'load_file') { Ox.load_file('sample.xml', :mode => :object) }
+  perf.add('Marshal', 'load') { Marshal.load(File.new('sample.marshal')) }
+  perf.run($iter)
+end
+
+if do_write
+  puts '-' * 80
+  puts "Write to file Performance"
+  perf = Perf.new()
+  perf.add('Oj', 'to_file') { Oj.to_file('sample.json', $obj) }
+  perf.add('Ox', 'to_file') { Ox.to_file('sample.xml', $obj, :indent => $indent, :circular => $circular) }
+  perf.add('Marshal', 'dump') { Marshal.dump($obj, File.new('sample.marshal', 'w')) }
+  perf.run($iter)
+end
+
+

data/test/perf_saj.rb

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'yajl'
+require 'perf'
+require 'json'
+require 'json/ext'
+require 'oj'
+
+$verbose = false
+$indent = 0
+$iter = 10000
+$gets = 0
+$fetch = false
+$write = false
+$read = false
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                  { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")     { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")   { |i| $indent = i }
+opts.on("-g", "--gets [Int]", Integer, "number of gets")  { |i| $gets = i }
+opts.on("-f", "fetch")                                    { $fetch = true }
+opts.on("-w", "write")                                    { $write = true }
+opts.on("-r", "read")                                     { $read = true }
+opts.on("-h", "--help", "Show this display")              { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+class AllSaj < Oj::Saj
+  def initialize()
+  end
+
+  def hash_start(key)
+  end
+
+  def hash_end(key)
+  end
+
+  def array_start(key)
+  end
+
+  def array_end(key)
+  end
+
+  def add_value(value, key)
+  end
+end # AllSaj
+
+class NoSaj < Oj::Saj
+  def initialize()
+  end
+end # NoSaj
+
+saj_handler = AllSaj.new()
+no_saj = NoSaj.new()
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, {'12345' => 12345, 'nil' => nil}, 3.967, { 'x' => 'something', 'y' => false, 'z' => true}, nil]], # mix it up array
+  'e' => { 'one' => 1, 'two' => 2 }, # hash
+  'f' => nil,     # nil
+  'g' => 12345678901234567890123456789, # big number
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+Oj.default_options = { :indent => $indent, :mode => :compat }
+
+$json = Oj.dump($obj)
+$failed = {} # key is same as String used in tests later
+
+def capture_error(tag, orig, load_key, dump_key, &blk)
+  begin
+    obj = blk.call(orig)
+    raise "#{tag} #{dump_key} and #{load_key} did not return the same object as the original." unless orig == obj
+  rescue Exception => e
+    $failed[tag] = "#{e.class}: #{e.message}"
+  end
+end
+
+# Verify that all packages dump and load correctly and return the same Object as the original.
+capture_error('Yajl', $obj, 'encode', 'parse') { |o| Yajl::Parser.parse(Yajl::Encoder.encode(o)) }
+capture_error('JSON::Ext', $obj, 'generate', 'parse') { |o| JSON.generator = JSON::Ext::Generator; JSON::Ext::Parser.new(JSON.generate(o)).parse }
+
+if $verbose
+  puts "json:\n#{$json}\n"
+end
+
+
+puts '-' * 80
+puts "Parse Performance"
+perf = Perf.new()
+perf.add('Oj::Saj', 'all') { Oj.saj_parse(saj_handler, $json) }
+perf.add('Oj::Saj', 'none') { Oj.saj_parse(no_saj, $json) }
+perf.add('Yajl', 'parse') { Yajl::Parser.parse($json) } unless $failed.has_key?('Yajl')
+perf.add('JSON::Ext', 'parse') { JSON::Ext::Parser.new($json).parse } unless $failed.has_key?('JSON::Ext')
+perf.run($iter)
+
+unless $failed.empty?
+  puts "The following packages were not included for the reason listed"
+  $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }
+end

data/test/perf_scp.rb

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'yajl'
+require 'perf'
+require 'json'
+require 'json/ext'
+require 'oj'
+
+$verbose = false
+$indent = 0
+$iter = 50000
+$with_bignum = false
+$size = 0
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                  { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")     { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")   { |i| $indent = i }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
+opts.on("-b", "with bignum")                                { $with_bignum = true }
+opts.on("-h", "--help", "Show this display")              { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+$obj['g'] = 12345678901234567890123456789 if $with_bignum
+
+if 0 < $size
+  o = $obj
+  $obj = []
+  (4 * $size).times do
+    $obj << o
+  end
+end
+
+Oj.default_options = { :indent => $indent, :mode => :compat }
+
+$json = Oj.dump($obj)
+$failed = {} # key is same as String used in tests later
+
+class AllSaj < Oj::Saj
+  def initialize()
+  end
+
+  def hash_start(key)
+  end
+
+  def hash_end(key)
+  end
+
+  def array_start(key)
+  end
+
+  def array_end(key)
+  end
+
+  def add_value(value, key)
+  end
+end # AllSaj
+
+class NoSaj < Oj::Saj
+  def initialize()
+  end
+end # NoSaj
+
+class NoHandler < Oj::ScHandler
+  def initialize()
+  end
+end # NoHandler
+
+class AllHandler < Oj::ScHandler
+  def initialize()
+  end
+
+  def hash_start()
+    return nil
+  end
+
+  def hash_end()
+  end
+
+  def array_start()
+    return nil
+  end
+
+  def array_end()
+  end
+
+  def add_value(value)
+  end
+
+  def hash_set(h, key, value)
+  end
+  
+  def array_append(a, value)
+  end
+
+end # AllHandler
+
+saj_handler = AllSaj.new()
+no_saj = NoSaj.new()
+
+sc_handler = AllHandler.new()
+no_handler = NoHandler.new()
+
+def capture_error(tag, orig, load_key, dump_key, &blk)
+  begin
+    obj = blk.call(orig)
+    raise "#{tag} #{dump_key} and #{load_key} did not return the same object as the original." unless orig == obj
+  rescue Exception => e
+    $failed[tag] = "#{e.class}: #{e.message}"
+  end
+end
+
+# Verify that all packages dump and load correctly and return the same Object as the original.
+capture_error('Yajl', $obj, 'encode', 'parse') { |o| Yajl::Parser.parse(Yajl::Encoder.encode(o)) }
+capture_error('JSON::Ext', $obj, 'generate', 'parse') { |o| JSON.generator = JSON::Ext::Generator; JSON::Ext::Parser.new(JSON.generate(o)).parse }
+
+if $verbose
+  puts "json:\n#{$json}\n"
+end
+
+puts '-' * 80
+puts "Parse Performance"
+perf = Perf.new()
+perf.add('Oj::Saj', 'all') { Oj.saj_parse(saj_handler, $json) }
+perf.add('Oj::Saj', 'none') { Oj.saj_parse(no_saj, $json) }
+perf.add('Oj::Scp', 'all') { Oj.sc_parse(sc_handler, $json) }
+perf.add('Oj::Scp', 'none') { Oj.sc_parse(no_handler, $json) }
+perf.add('Yajl', 'parse') { Yajl::Parser.parse($json) } unless $failed.has_key?('Yajl')
+perf.add('JSON::Ext', 'parse') { JSON::Ext::Parser.new($json).parse } unless $failed.has_key?('JSON::Ext')
+perf.run($iter)
+
+unless $failed.empty?
+  puts "The following packages were not included for the reason listed"
+  $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }
+end