oj 3.7.4 → 3.13.21

data/test/foo.rb

@@ -1,83 +1,77 @@
 #!/usr/bin/env ruby
-# encoding: UTF-8
 
-$: << File.dirname(__FILE__)
-$oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
-%w(lib ext).each do |dir|
-  $: << File.join($oj_dir, dir)
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require "oj"
+require "socket"
+require 'io/nonblock'
+
+#pid = spawn("nc -d 0.1 -l 5000", out: "/dev/null")
+pid = spawn("nc -i 1 -l 7777", out: "/dev/null")
+at_exit { Process.kill 9, pid }
+sleep 0.2
+s = Socket.tcp("localhost", 7777)
+s.nonblock = false
+1_000_000.times do |x|
+  Oj.to_stream(s, { x: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
 end
 
-require 'oj'
-require 'tracer'
-
-#Tracer.on
-
-class MyParser
-  attr_accessor :enum
-
-  def initialize
-    @io = StringIO.new
-    @writer = Oj::StreamWriter.new(@io)
-
-    json_string = Oj.dump({ records: 1.upto(2).map{|i| { id: i, name: "record_#{i}" }} }, mode: :strict)
-    @test_json = StringIO.new(json_string)
-
-    @enum = Enumerator.new do |yielder|
-      @yielder = yielder
-      Oj.sc_parse(self, @test_json)
+=begin
+IO.pipe do |r, w|
+  if fork
+    r.close
+    #w.nonblock = false
+    1_000_000.times do |i|
+      begin
+	Oj.to_stream(w, { x: i})
+      rescue IOError => e
+	puts "*** #{i} raised #{e.class}: #{e}"
+	IO.select(nil, [w])
+	retry
+      end
+      w.puts
     end
-  end
-
-  # Stream parsing methods
-  def hash_start
-    @writer.push_object
-  end
-
-  def hash_end
-    @writer.pop unless @io.eof
-  end
-
-  def hash_key(key)
-    @writer.push_key(key)
-  end
-
-  def hash_set(h, key, value)
-    @writer.push_value(value)
-  end
-
-  def array_start
-    @writer.push_array
-  end
-
-  def array_end
-    @writer.pop
-  end
-
-  def array_append(a, value)
-    yield_data
-  end
-
-  def add_value(value);end
-
-  def yield_data
-    @writer.pop_all
-    @yielder << @io.string
-    @io.reopen("")
-    array_start
+  else
+    w.close
+    sleep(0.1)
+    r.each_line { |b|
+      #print b
+    }
+    r.close
+    Process.exit(0)
   end
 end
-
-#puts MyParser.new.enum.to_a
-
-puts "------------"
-
-MyError = Class.new(StandardError)
-
-MyParser.new.enum.each.with_index do |r, i|
-  puts "========="
-  raise MyError.new('hello')
-  #raise StopIteration if i == 0
-  #break if i >= 4
+=end
+
+=begin
+IO.pipe do |r, w|
+  if fork
+    r.close
+    #w.nonblock = false
+    a = []
+    10_000.times do |i|
+      a << i
+    end
+    begin
+      Oj.to_stream(w, a, indent: 2)
+    rescue IOError => e
+      puts "*** raised #{e.class}: #{e}"
+      puts "*** fileno: #{w.fileno}"
+      puts "*** is an IO?: #{w.kind_of?(IO)}"
+      IO.select(nil, [w])
+      retry
+    end
+    w.puts
+  else
+    w.close
+    sleep(0.5)
+    r.each_line { |b|
+      #print b
+    }
+    r.close
+    Process.exit(0)
+  end
 end
-
-#{"records":[{"id":1,"name":"record_1"},{"id":2,"name":"record_2"},{"id":3,"name":"record_3"},{"id":4,"name":"record_4"},{"id":5,"name":"record_5"},{"id":6,"name":"record_6"},{"id":7,"name":"record_7"},{"id":8,"name":"record_8"},{"id":9,"name":"record_9"},{"id":10,"name":"record_10"},{"id":11,"name":"record_11"},{"id":12,"name":"record_12"},{"id":13,"name":"record_13"},{"id":14,"name":"record_14"},{"id":15,"name":"record_15"},{"id":16,"name":"record_16"},{"id":17,"name":"record_17"},{"id":18,"name":"record_18"},{"id":19,"name":"record_19"},{"id":20,"name":"record_20"},{"id":21,"name":"record_21"},{"id":22,"name":"record_22"},{"id":23,"name":"record_23"},{"id":24,"name":"record_24"},{"id":25,"name":"record_25"}]}
+=end

data/test/helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+#
 # Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
 # required. That can be set in the RUBYOPT environment variable.
 # export RUBYOPT=-w
@@ -16,6 +18,20 @@ require 'bigdecimal'
 require 'pp'
 require 'oj'
 
+
+def verify_gc_compaction
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  if defined?(GC.verify_compaction_references) == 'method' && !(RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/)
+    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2.0")
+      GC.verify_compaction_references(expand_heap: true, toward: :empty)
+    else
+      GC.verify_compaction_references(double_heap: true, toward: :empty)
+    end
+  end
+end
+
+
 $ruby = RUBY_DESCRIPTION.split(' ')[0]
 $ruby = 'ree' if 'ruby' == $ruby && RUBY_DESCRIPTION.include?('Ruby Enterprise Edition')
 

data/test/json_gem/json_common_interface_test.rb

@@ -15,7 +15,7 @@ class JSONCommonInterfaceTest < Test::Unit::TestCase
   def setup
     @hash = {
       'a' => 2,
-      'b' => 3.141,
+      #'b' => 5.23683071,
       'c' => 'c',
       'd' => [ 1, "b", 3.14 ],
       'e' => { 'foo' => 'bar' },
@@ -23,8 +23,13 @@ class JSONCommonInterfaceTest < Test::Unit::TestCase
       'h' => 1000.0,
       'i' => 0.001
     }
-    @json = '{"a":2,"b":3.141,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},'\
-      '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}'
+    # Tired of chasing floating point rounding and precision. Oj now uses the
+    # Ruby float parser in compat mode yet on i386 machines there are issues
+    # with this test when the float is included.
+    #@json = '{"a":2,"b":5.23683071,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},'\
+    #'"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}'
+    @json = '{"a":2,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},'\
+	    '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}'
   end
 
   def test_index

data/test/json_gem/json_generator_test.rb

@@ -72,6 +72,8 @@ EOT
     parsed_json = JSON.parse(json)
     assert_equal({"1"=>2}, parsed_json)
     assert_equal '666', JSON.pretty_generate(666)
+    json_nil_opts = JSON.pretty_generate({1=>2}, nil)
+    assert_equal json, json_nil_opts
   end
 
   def test_generate_custom
@@ -113,7 +115,7 @@ EOT
   end
 
   # TBD Implement JSON.state to return state class.
-  # set state attibutes from defaults
+  # set state attributes from defaults
   # implement methods
   # circular should use circular in defaults or maybe always set to true, allow changes with [:check_circular]=
   def test_states
@@ -136,6 +138,10 @@ EOT
 
   def test_pretty_state
     state = JSON::PRETTY_STATE_PROTOTYPE.dup
+    # In come cases in Ruby 3.0 an :escape_slash is included in the state. It
+    # seems to occur on travis but not locally.
+    actual = state.to_h
+    actual.delete(:escape_slash)
     assert_equal({
       :allow_nan             => false,
       :array_nl              => "\n",
@@ -147,11 +153,15 @@ EOT
       :object_nl             => "\n",
       :space                 => " ",
       :space_before          => "",
-    }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+    }.sort_by { |n,| n.to_s }, actual.sort_by { |n,| n.to_s })
   end
 
   def test_safe_state
     state = JSON::SAFE_STATE_PROTOTYPE.dup
+    # In come cases in Ruby 3.0 an :escape_slash is included in the state. It
+    # seems to occur on travis but not locally.
+    actual = state.to_h
+    actual.delete(:escape_slash)
     assert_equal({
       :allow_nan             => false,
       :array_nl              => "",
@@ -163,11 +173,15 @@ EOT
       :object_nl             => "",
       :space                 => "",
       :space_before          => "",
-    }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+    }.sort_by { |n,| n.to_s }, actual.sort_by { |n,| n.to_s })
   end
 
   def test_fast_state
     state = JSON::FAST_STATE_PROTOTYPE.dup
+    # In come cases in Ruby 3.0 an :escape_slash is included in the state. It
+    # seems to occur on travis but not locally.
+    actual = state.to_h
+    actual.delete(:escape_slash)
     assert_equal({
       :allow_nan             => false,
       :array_nl              => "",
@@ -179,7 +193,7 @@ EOT
       :object_nl             => "",
       :space                 => "",
       :space_before          => "",
-    }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s })
+    }.sort_by { |n,| n.to_s }, actual.sort_by { |n,| n.to_s })
   end
 
   def test_allow_nan

data/test/json_gem/json_parser_test.rb

@@ -24,6 +24,8 @@ class JSONParserTest < Test::Unit::TestCase
   end if defined?(Encoding::UTF_16)
 
   def test_error_message_encoding
+    omit 'TruffleRuby causes NameError(<uninitialized constant JSON::Ext>)' if RUBY_ENGINE == 'truffleruby'
+
     bug10705 = '[ruby-core:67386] [Bug #10705]'
     json = ".\"\xE2\x88\x9A\"".force_encoding(Encoding::UTF_8)
     e = assert_raise(JSON::ParserError) {
@@ -269,6 +271,13 @@ EOT
     assert_equal too_deep_ary, ok
     ok = JSON.parse too_deep, :max_nesting => 0
     assert_equal too_deep_ary, ok
+
+    unless ENV['REAL_JSON_GEM']
+      # max_nesting should be reset to 0 if not included in options
+      # This behavior is not compatible with Ruby standard JSON gem
+      ok = JSON.parse too_deep, {}
+      assert_equal too_deep_ary, ok
+    end
   end
 
   def test_backslash

data/test/json_gem/test_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 $: << File.dirname(__FILE__)
 $oj_dir = File.dirname(File.dirname(File.expand_path(File.dirname(__FILE__))))
 %w(lib ext).each do |dir|
@@ -12,6 +14,16 @@ if ENV['REAL_JSON_GEM']
 else
   require 'oj'
   Oj.mimic_JSON
+
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  if defined?(GC.verify_compaction_references) == 'method'
+    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2.0")
+      GC.verify_compaction_references(expand_heap: true, toward: :empty)
+    else
+      GC.verify_compaction_references(double_heap: true, toward: :empty)
+    end
+  end
 end
 
 NaN = JSON::NaN if defined?(JSON::NaN)

data/test/mem.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+Oj.default_options = { mode: :rails, cache_keys: false, cache_str: -1 }
+
+def mem
+  `ps -o rss= -p #{$$}`.to_i
+end
+
+('a'..'z').each { |a|
+  ('a'..'z').each { |b|
+    ('a'..'z').each { |c|
+      ('a'..'z').each { |d|
+	('a'..'z').each { |e|
+	  ('a'..'z').each { |f|
+	    key = "#{a}#{b}#{c}#{d}#{e}#{f}"
+	    x = Oj.load(%|{ "#{key}": 101}|)
+	    #Oj.dump(x)
+	  }
+	}
+      }
+    }
+    puts "#{a}#{b} #{mem}"
+  }
+}
+
+Oj::Parser.new(:usual)

data/test/perf.rb

@@ -17,7 +17,7 @@ class Perf
       end
     end
   end
-  
+
   def run(iter)
     base = Item.new(nil, nil) { }
     base.run(iter, 0.0)

data/test/perf_dump.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'oj'
+
+$verbose = false
+$indent = 2
+$iter = 100_000
+$size = 2
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                    { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")       { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")     { |i| $indent = i }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+Oj.default_options = { :indent => $indent, :mode => :strict }
+
+if 0 < $size
+  o = $obj
+  $obj = []
+  (4 * $size).times do
+    $obj << o
+  end
+end
+
+$json = Oj.dump($obj)
+GC.start
+start = Time.now
+$iter.times { Oj.dump($obj) }
+duration = Time.now - start
+puts "Dumped #{$json.length} byte JSON #{$iter} times in %0.3f seconds or %0.3f iteration/sec." % [duration, $iter / duration]

data/test/perf_once.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+filename = 'tmp.json'
+File.open(filename, "w") { |f|
+  cnt = 0
+  f.puts('{')
+  ('a'..'z').each { |a|
+    ('a'..'z').each { |b|
+      ('a'..'z').each { |c|
+	('a'..'z').each { |d|
+	  f.puts(%|"#{a}#{b}#{c}#{d}":#{cnt},|)
+	  cnt += 1
+	}
+      }
+    }
+  }
+  f.puts('"_last":0}')
+}
+
+def mem
+  `ps -o rss= -p #{$$}`.to_i
+end
+
+Oj.default_options = { mode: :strict, cache_keys: false, cache_str: -1 }
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "no cache duration: #{dur} @ #{mem}"
+
+Oj.default_options = { cache_keys: true }
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "initial cache duration: #{dur} @ #{mem}"
+
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "second cache duration: #{dur} @ #{mem}"
+
+10.times{ GC.start }
+start = Time.now
+Oj.load_file('tmp.json')
+dur = Time.now - start
+GC.start
+puts "after several GCs cache duration: #{dur} @ #{mem}"
+
+# TBD check memory use

data/test/perf_parser.rb

@@ -0,0 +1,189 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'perf'
+require 'oj'
+require 'json'
+
+$verbose = false
+$iter = 50_000
+$with_bignum = false
+$size = 1
+$cache_keys = true
+$symbol_keys = false
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                  { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")     { |i| $iter = i }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")  { |i| $size = i }
+opts.on("-b", "with bignum")                              { $with_bignum = true }
+opts.on("-k", "no cache")                                 { $cache_keys = false }
+opts.on("-sym", "symbol keys")                            { $symbol_keys = true }
+opts.on("-h", "--help", "Show this display")              { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false, 1, nil], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+$obj['g'] = 12345678901234567890123456789 if $with_bignum
+
+if 0 < $size
+  o = $obj
+  $obj = []
+  (4 * $size).times do
+    $obj << o
+  end
+end
+
+$json = Oj.dump($obj)
+$failed = {} # key is same as String used in tests later
+Oj.default_options = {create_id: '^', create_additions: true, class_cache: true}
+if $cache_keys
+  Oj.default_options = {cache_keys: true, cache_str: 6, symbol_keys: $symbol_keys}
+else
+  Oj.default_options = {cache_keys: false, cache_str: -1, symbol_keys: $symbol_keys}
+end
+JSON.parser = JSON::Ext::Parser
+
+class AllSaj
+  def initialize()
+  end
+
+  def hash_start(key)
+  end
+
+  def hash_end(key)
+  end
+
+  def array_start(key)
+  end
+
+  def array_end(key)
+  end
+
+  def add_value(value, key)
+  end
+end # AllSaj
+
+class NoSaj
+  def initialize()
+  end
+end # NoSaj
+
+no_handler = NoSaj.new()
+all_handler = AllSaj.new()
+
+if $verbose
+  puts "json:\n#{$json}\n"
+end
+
+### Validate ######################
+p_val = Oj::Parser.new(:validate)
+
+puts '-' * 80
+puts "Validate Performance"
+perf = Perf.new()
+perf.add('Oj::Parser.validate', 'none') { p_val.parse($json) }
+perf.add('Oj::Saj.none', 'none') { Oj.saj_parse(no_handler, $json) }
+perf.run($iter)
+
+### SAJ ######################
+p_all = Oj::Parser.new(:saj)
+p_all.handler = all_handler
+p_all.cache_keys = $cache_keys
+p_all.cache_strings = 6
+
+puts '-' * 80
+puts "Parse Callback Performance"
+perf = Perf.new()
+perf.add('Oj::Parser.saj', 'all') { p_all.parse($json) }
+perf.add('Oj::Saj.all', 'all') { Oj.saj_parse(all_handler, $json) }
+perf.run($iter)
+
+### Usual ######################
+p_usual = Oj::Parser.new(:usual)
+p_usual.cache_keys = $cache_keys
+p_usual.cache_strings = ($cache_keys ? 6 : 0)
+p_usual.symbol_keys = $symbol_keys
+
+puts '-' * 80
+puts "Parse Usual Performance"
+perf = Perf.new()
+perf.add('Oj::Parser.usual', '') { p_usual.parse($json) }
+perf.add('Oj::strict_load', '') { Oj.strict_load($json) }
+perf.add('JSON::Ext', 'parse') { JSON.load($json) }
+perf.run($iter)
+
+### Usual Objects ######################
+
+# Original Oj follows the JSON gem for creating objects which uses the class
+# json_create(arg) method. Oj::Parser in usual mode supprts the same but also
+# handles populating the object variables directly which is faster.
+
+class Stuff
+  attr_accessor :alpha, :bravo, :charlie, :delta, :echo, :foxtrot, :golf, :hotel, :india, :juliet
+  def self.json_create(arg)
+    obj = self.new
+    obj.alpha = arg["alpha"]
+    obj.bravo = arg["bravo"]
+    obj.charlie = arg["charlie"]
+    obj.delta = arg["delta"]
+    obj.echo = arg["echo"]
+    obj.foxtrot = arg["foxtrot"]
+    obj.golf = arg["golf"]
+    obj.hotel = arg["hotel"]
+    obj.india = arg["india"]
+    obj.juliet = arg["juliet"]
+    obj
+  end
+end
+
+$obj_json = %|{
+  "alpha": [0, 1,2,3,4,5,6,7,8,9],
+  "bravo": true,
+  "charlie": 123,
+  "delta": "some string",
+  "echo": null,
+  "^": "Stuff",
+  "foxtrot": false,
+  "golf": "gulp",
+  "hotel": {"x": true, "y": false},
+  "india": [null, true, 123],
+  "juliet": "junk"
+}|
+
+
+p_usual = Oj::Parser.new(:usual)
+p_usual.cache_keys = $cache_keys
+p_usual.cache_strings = ($cache_keys ? 6 : 0)
+p_usual.symbol_keys = $symbol_keys
+p_usual.create_id = '^'
+p_usual.class_cache = true
+p_usual.ignore_json_create = true
+
+JSON.create_id = '^'
+
+puts '-' * 80
+puts "Parse Usual Object Performance"
+perf = Perf.new()
+perf.add('Oj::Parser.usual', '') { p_usual.parse($obj_json) }
+perf.add('Oj::compat_load', '') { Oj.compat_load($obj_json) }
+perf.add('JSON::Ext', 'parse') { JSON.load($obj_json) }
+perf.run($iter)
+
+unless $failed.empty?
+  puts "The following packages were not included for the reason listed"
+  $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }
+end