oj 3.11.0 → 3.16.5

data/test/perf_file.rb

@@ -1,58 +1,57 @@
 #!/usr/bin/env ruby -wW1
+# frozen_string_literal: true
 
-$: << '.'
-$: << '../lib'
-$: << '../ext'
+$LOAD_PATH << '.'
+$LOAD_PATH << '../lib'
+$LOAD_PATH << '../ext'
 
-if __FILE__ == $0
-  if (i = ARGV.index('-I'))
-    x,path = ARGV.slice!(i, 2)
-    $: << path
-  end
+if __FILE__ == $PROGRAM_NAME && (i = ARGV.index('-I'))
+  _, path = ARGV.slice!(i, 2)
+  $LOAD_PATH << path
 end
 
 require 'optparse'
 require 'oj'
 require 'perf'
 
-$indent = 0
-$iter = 1
-$size = 1
+@indent = 0
+@iter = 1
+@size = 1
 
 opts = OptionParser.new
 
-opts.on("-r", "read")                                       { do_read = true }
-opts.on("-c", "--count [Int]", Integer, "iterations")       { |i| $iter = i }
-opts.on("-i", "--indent [Int]", Integer, "indent")          { |i| $indent = i }
-opts.on("-s", "--size [Int]", Integer, "size in Mbytes")    { |s| $size = s }
+opts.on('-r', 'read')                                       { true }
+opts.on('-c', '--count [Int]', Integer, 'iterations')       { |v| @iter = v }
+opts.on('-i', '--indent [Int]', Integer, 'indent')          { |v| @indent = v }
+opts.on('-s', '--size [Int]', Integer, 'size in Mbytes')    { |s| @size = s }
 
-opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
-files = opts.parse(ARGV)
+opts.on('-h', '--help', 'Show this display')                { puts opts; Process.exit!(0) }
+opts.parse(ARGV)
 
-$obj = {
+@obj = {
   'a' => 'Alpha', # string
   'b' => true,    # boolean
-  'c' => 12345,   # number
-  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'c' => 12_345,   # number
+  'd' => [ true, [false, [-123_456_789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
   'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
   'f' => nil,     # nil
-  'g' => 12345678901234567890123456789, #_bignum
+  'g' => 12_345_678_901_234_567_890_123_456_789, # _bignum
   'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
   'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
 }
 
-json = Oj.dump($obj, :indent => $indent)
-cnt = ($size * 1024 * 1024 + json.size) / json.size
-cnt = 1 if 0 == $size
+json = Oj.dump(@obj, :indent => @indent)
+cnt = ((@size * 1024 * 1024) + json.size) / json.size
+cnt = 1 if 0 == @size
 
 filename = 'tmp.json'
-File.open(filename, "w") { |f|
+File.open(filename, 'w') { |f|
   cnt.times do
-    Oj.to_stream(f, $obj, :indent => $indent)
+    Oj.to_stream(f, @obj, :indent => @indent)
   end
 }
 
-Oj.default_options = { :mode => :strict, :indent => $indent }
+Oj.default_options = { :mode => :strict, :indent => @indent }
 
 puts '-' * 80
 puts "Read from #{cnt * json.size / (1024 * 1024)} Mb file Performance"
@@ -60,5 +59,4 @@ perf = Perf.new()
 perf.add('Oj.load_file', '') { Oj.load_file(filename) }
 perf.add('Oj.load(string)', '') { Oj.load(File.read(filename)) }
 perf.add('Oj.load(file)', '') { File.open(filename, 'r') { |f| Oj.load(f) } }
-perf.run($iter)
-
+perf.run(@iter)

data/test/perf_object.rb

@@ -1,14 +1,13 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-$: << '.'
-$: << '../lib'
-$: << '../ext'
+$LOAD_PATH << '.'
+$LOAD_PATH << '../lib'
+$LOAD_PATH << '../ext'
 
-if __FILE__ == $0
-  if (i = ARGV.index('-I'))
-    x,path = ARGV.slice!(i, 2)
-    $: << path
-  end
+if __FILE__ == $PROGRAM_NAME && (i = ARGV.index('-I'))
+  _, path = ARGV.slice!(i, 2)
+  $LOAD_PATH << path
 end
 
 require 'optparse'
@@ -18,9 +17,9 @@ require 'perf'
 require 'sample'
 require 'files'
 
-$circular = false
-$indent = 0
-$allow_gc = true
+@circular = false
+@indent = 0
+@allow_gc = true
 
 do_sample = false
 do_files = false
@@ -29,34 +28,34 @@ do_load = false
 do_dump = false
 do_read = false
 do_write = false
-$iter = 1000
-$mult = 1
+@iter = 1000
+@mult = 1
 
 opts = OptionParser.new
-opts.on("-c", "circular options")                           { $circular = true }
+opts.on('-c', 'circular options')                           { @circular = true }
 
-opts.on("-x", "use sample instead of files")                { do_sample = true }
-opts.on("-g", "no GC during parsing")                       { $allow_gc = false }
+opts.on('-x', 'use sample instead of files')                { do_sample = true }
+opts.on('-g', 'no GC during parsing')                       { @allow_gc = false }
 
-opts.on("-s", "load and dump as sample Ruby object")        { do_sample = true }
-opts.on("-f", "load and dump as files Ruby object")         { do_files = true }
+opts.on('-s', 'load and dump as sample Ruby object')        { do_sample = true }
+opts.on('-f', 'load and dump as files Ruby object')         { do_files = true }
 
-opts.on("-l", "load")                                       { do_load = true }
-opts.on("-d", "dump")                                       { do_dump = true }
-opts.on("-r", "read")                                       { do_read = true }
-opts.on("-w", "write")                                      { do_write = true }
-opts.on("-a", "load, dump, read and write")                 { do_load = true; do_dump = true; do_read = true; do_write = true }
+opts.on('-l', 'load')                                       { do_load = true }
+opts.on('-d', 'dump')                                       { do_dump = true }
+opts.on('-r', 'read')                                       { do_read = true }
+opts.on('-w', 'write')                                      { do_write = true }
+opts.on('-a', 'load, dump, read and write')                 { do_load = true; do_dump = true; do_read = true; do_write = true }
 
-opts.on("-i", "--iterations [Int]", Integer, "iterations")  { |i| $iter = i }
-opts.on("-m", "--multiply [Int]", Integer, "multiplier")    { |i| $mult = i }
+opts.on('-i', '--iterations [Int]', Integer, 'iterations')  { |v| @iter = v }
+opts.on('-m', '--multiply [Int]', Integer, 'multiplier')    { |v| @mult = v }
 
-opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+opts.on('-h', '--help', 'Show this display')                { puts opts; Process.exit!(0) }
 files = opts.parse(ARGV)
 
-$obj = nil
-$xml = nil
-$mars = nil
-$json = nil
+@obj = nil
+@xml = nil
+@mars = nil
+@json = nil
 
 unless do_load || do_dump || do_read || do_write
   do_load = true
@@ -67,72 +66,69 @@ end
 
 # prepare all the formats for input
 if files.empty?
-  $obj = []
-  $mult.times do
-    $obj << (do_sample ? sample_doc(2) : files('..'))
+  @obj = []
+  @mult.times do
+    @obj << (do_sample ? sample_doc(2) : files('..'))
   end
 
-  $mars = Marshal.dump($obj)
-  $xml = Ox.dump($obj, :indent => $indent, :circular => $circular)
-  $json = Oj.dump($obj, :indent => $indent, :circular => $circular, :mode => :object)
-  File.open('sample.xml', 'w') { |f| f.write($xml) }
-  File.open('sample.json', 'w') { |f| f.write($json) }
-  File.open('sample.marshal', 'w') { |f| f.write($mars) }
+  @mars = Marshal.dump(@obj)
+  @xml = Ox.dump(@obj, :indent => @indent, :circular => @circular)
+  @json = Oj.dump(@obj, :indent => @indent, :circular => @circular, :mode => :object)
+  File.write('sample.xml', @xml)
+  File.write('sample.json', @json)
+  File.write('sample.marshal', @mars)
 else
   puts "loading and parsing #{files}\n\n"
-  data = files.map do |f|
-    $xml = File.read(f)
-    $obj = Ox.load($xml);
-    $mars = Marshal.dump($obj)
-    $json = Oj.dump($obj, :indent => $indent, :circular => $circular)
+  files.map do |f|
+    @xml = File.read(f)
+    @obj = Ox.load(@xml)
+    @mars = Marshal.dump(@obj)
+    @json = Oj.dump(@obj, :indent => @indent, :circular => @circular)
   end
 end
 
-Oj.default_options = { :mode => :object, :indent => $indent, :circular => $circular, :allow_gc => $allow_gc }
-#puts "json: #{$json.size}"
-#puts "xml: #{$xml.size}"
-#puts "marshal: #{$mars.size}"
-
+Oj.default_options = { :mode => :object, :indent => @indent, :circular => @circular, :allow_gc => @allow_gc }
+# puts "json: #{@json.size}"
+# puts "xml: #{@xml.size}"
+# puts "marshal: #{@mars.size}"
 
 if do_load
   puts '-' * 80
-  puts "Load Performance"
+  puts 'Load Performance'
   perf = Perf.new()
-  perf.add('Oj.object', 'load') { Oj.object_load($json) }
-  perf.add('Ox', 'load') { Ox.load($xml, :mode => :object) }
-  perf.add('Marshal', 'load') { Marshal.load($mars) }
-  perf.run($iter)
+  perf.add('Oj.object', 'load') { Oj.object_load(@json) }
+  perf.add('Ox', 'load') { Ox.load(@xml, :mode => :object) }
+  perf.add('Marshal', 'load') { Marshal.load(@mars) }
+  perf.run(@iter)
 end
 
 if do_dump
   puts '-' * 80
-  puts "Dump Performance"
+  puts 'Dump Performance'
   perf = Perf.new()
-  perf.add('Oj', 'dump') { Oj.dump($obj) }
-  perf.add('Ox', 'dump') { Ox.dump($obj, :indent => $indent, :circular => $circular) }
-  perf.add('Marshal', 'dump') { Marshal.dump($obj) }
-  perf.run($iter)
+  perf.add('Oj', 'dump') { Oj.dump(@obj) }
+  perf.add('Ox', 'dump') { Ox.dump(@obj, :indent => @indent, :circular => @circular) }
+  perf.add('Marshal', 'dump') { Marshal.dump(@obj) }
+  perf.run(@iter)
 end
 
 if do_read
   puts '-' * 80
-  puts "Read from file Performance"
+  puts 'Read from file Performance'
   perf = Perf.new()
   perf.add('Oj', 'load') { Oj.load_file('sample.json') }
-  #perf.add('Oj', 'load') { Oj.load(File.read('sample.json')) }
+  # perf.add('Oj', 'load') { Oj.load(File.read('sample.json')) }
   perf.add('Ox', 'load_file') { Ox.load_file('sample.xml', :mode => :object) }
   perf.add('Marshal', 'load') { Marshal.load(File.new('sample.marshal')) }
-  perf.run($iter)
+  perf.run(@iter)
 end
 
 if do_write
   puts '-' * 80
-  puts "Write to file Performance"
+  puts 'Write to file Performance'
   perf = Perf.new()
-  perf.add('Oj', 'to_file') { Oj.to_file('sample.json', $obj) }
-  perf.add('Ox', 'to_file') { Ox.to_file('sample.xml', $obj, :indent => $indent, :circular => $circular) }
-  perf.add('Marshal', 'dump') { Marshal.dump($obj, File.new('sample.marshal', 'w')) }
-  perf.run($iter)
+  perf.add('Oj', 'to_file') { Oj.to_file('sample.json', @obj) }
+  perf.add('Ox', 'to_file') { Ox.to_file('sample.xml', @obj, :indent => @indent, :circular => @circular) }
+  perf.add('Marshal', 'dump') { Marshal.dump(@obj, File.new('sample.marshal', 'w')) }
+  perf.run(@iter)
 end
-
-

data/test/perf_once.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'English'
+$LOAD_PATH << '.'
+$LOAD_PATH << File.join(__dir__, '../lib')
+$LOAD_PATH << File.join(__dir__, '../ext')
+
+require 'oj'
+
+filename = 'tmp.json'
+File.open(filename, 'w') { |f|
+  cnt = 0
+  f.puts('{')
+  ('a'..'z').each { |a|
+    ('a'..'z').each { |b|
+      ('a'..'z').each { |c|
+        ('a'..'z').each { |d|
+          f.puts(%|"#{a}#{b}#{c}#{d}":#{cnt},|)
+          cnt += 1
+        }
+      }
+    }
+  }
+  f.puts('"_last":0}')
+}
+
+def mem
+  `ps -o rss= -p #{$PROCESS_ID}`.to_i
+end
+
+Oj.default_options = { mode: :strict, cache_keys: false, cache_str: -1 }
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "no cache duration: #{dur} @ #{mem}"
+
+Oj.default_options = { cache_keys: true }
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "initial cache duration: #{dur} @ #{mem}"
+
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "second cache duration: #{dur} @ #{mem}"
+
+10.times { GC.start }
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "after several GCs cache duration: #{dur} @ #{mem}"
+
+# TBD check memory use

data/test/perf_parser.rb

@@ -0,0 +1,183 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH << '.'
+$LOAD_PATH << File.join(__dir__, '../lib')
+$LOAD_PATH << File.join(__dir__, '../ext')
+
+require 'optparse'
+require 'perf'
+require 'oj'
+require 'json'
+
+$verbose = false
+$iter = 50_000
+$with_bignum = false
+$size = 1
+$cache_keys = true
+$symbol_keys = false
+
+opts = OptionParser.new
+opts.on('-v', 'verbose')                                  { $verbose = true }
+opts.on('-c', '--count [Int]', Integer, 'iterations')     { |i| $iter = i }
+opts.on('-s', '--size [Int]', Integer, 'size (~Kbytes)')  { |i| $size = i }
+opts.on('-b', 'with bignum')                              { $with_bignum = true }
+opts.on('-k', 'no cache')                                 { $cache_keys = false }
+opts.on('-sym', 'symbol keys')                            { $symbol_keys = true }
+opts.on('-h', '--help', 'Show this display')              { puts opts; Process.exit!(0) }
+opts.parse(ARGV)
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12_345,   # number
+  'd' => [ true, [false, [-123_456_789, nil], 3.9676, ['Something else.', false, 1, nil], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+$obj['g'] = 12_345_678_901_234_567_890_123_456_789 if $with_bignum
+
+if 0 < $size
+  o = $obj
+  $obj = []
+  (4 * $size).times do
+    $obj << o
+  end
+end
+
+$json = Oj.dump($obj)
+$failed = {} # key is same as String used in tests later
+Oj.default_options = {create_id: '^', create_additions: true, class_cache: true}
+Oj.default_options = if $cache_keys
+                       {cache_keys: true, cache_str: 6, symbol_keys: $symbol_keys}
+                     else
+                       {cache_keys: false, cache_str: -1, symbol_keys: $symbol_keys}
+                     end
+JSON.parser = JSON::Ext::Parser
+
+class AllSaj
+
+  def hash_start(key)
+  end
+
+  def hash_end(key)
+  end
+
+  def array_start(key)
+  end
+
+  def array_end(key)
+  end
+
+  def add_value(value, key)
+  end
+end # AllSaj
+
+no_handler = Object.new()
+all_handler = AllSaj.new()
+
+if $verbose
+  puts "json:\n#{$json}\n"
+end
+
+### Validate ######################
+p_val = Oj::Parser.new(:validate)
+
+puts '-' * 80
+puts 'Validate Performance'
+perf = Perf.new()
+perf.add('Oj::Parser.validate', 'none') { p_val.parse($json) }
+perf.add('Oj::Saj.none', 'none') { Oj.saj_parse(no_handler, $json) }
+perf.run($iter)
+
+### SAJ ######################
+p_all = Oj::Parser.new(:saj)
+p_all.handler = all_handler
+p_all.cache_keys = $cache_keys
+p_all.cache_strings = 6
+
+puts '-' * 80
+puts 'Parse Callback Performance'
+perf = Perf.new()
+perf.add('Oj::Parser.saj', 'all') { p_all.parse($json) }
+perf.add('Oj::Saj.all', 'all') { Oj.saj_parse(all_handler, $json) }
+perf.run($iter)
+
+### Usual ######################
+p_usual = Oj::Parser.new(:usual)
+p_usual.cache_keys = $cache_keys
+p_usual.cache_strings = ($cache_keys ? 6 : 0)
+p_usual.symbol_keys = $symbol_keys
+
+puts '-' * 80
+puts 'Parse Usual Performance'
+perf = Perf.new()
+perf.add('Oj::Parser.usual', '') { p_usual.parse($json) }
+perf.add('Oj::strict_load', '') { Oj.strict_load($json) }
+perf.add('JSON::Ext', 'parse') { JSON.parse($json) }
+perf.run($iter)
+
+### Usual Objects ######################
+
+# Original Oj follows the JSON gem for creating objects which uses the class
+# json_create(arg) method. Oj::Parser in usual mode supprts the same but also
+# handles populating the object variables directly which is faster.
+
+class Stuff
+  attr_accessor :alpha, :bravo, :charlie, :delta, :echo, :foxtrot, :golf, :hotel, :india, :juliet
+
+  def self.json_create(arg)
+    obj = new
+    obj.alpha = arg['alpha']
+    obj.bravo = arg['bravo']
+    obj.charlie = arg['charlie']
+    obj.delta = arg['delta']
+    obj.echo = arg['echo']
+    obj.foxtrot = arg['foxtrot']
+    obj.golf = arg['golf']
+    obj.hotel = arg['hotel']
+    obj.india = arg['india']
+    obj.juliet = arg['juliet']
+    obj
+  end
+end
+
+$obj_json = %|{
+  "alpha": [0, 1,2,3,4,5,6,7,8,9],
+  "bravo": true,
+  "charlie": 123,
+  "delta": "some string",
+  "echo": null,
+  "^": "Stuff",
+  "foxtrot": false,
+  "golf": "gulp",
+  "hotel": {"x": true, "y": false},
+  "india": [null, true, 123],
+  "juliet": "junk"
+}|
+
+p_usual = Oj::Parser.new(:usual)
+p_usual.cache_keys = $cache_keys
+p_usual.cache_strings = ($cache_keys ? 6 : 0)
+p_usual.symbol_keys = $symbol_keys
+p_usual.create_id = '^'
+p_usual.class_cache = true
+p_usual.ignore_json_create = true
+
+JSON.create_id = '^'
+
+puts '-' * 80
+puts 'Parse Usual Object Performance'
+perf = Perf.new()
+perf.add('Oj::Parser.usual', '') { p_usual.parse($obj_json) }
+perf.add('Oj::compat_load', '') { Oj.compat_load($obj_json) }
+perf.add('JSON::Ext', 'parse') { JSON.parse($obj_json) }
+
+perf.run($iter)
+
+unless $failed.empty?
+  puts 'The following packages were not included for the reason listed'
+  $failed.each { |tag, msg| puts "***** #{tag}: #{msg}" }
+end