oj 3.13.11 → 3.13.23

data/test/helper.rb

@@ -19,10 +19,16 @@ require 'pp'
 require 'oj'
 
 
-if defined?(GC.verify_compaction_references) == 'method'
+def verify_gc_compaction
   # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
   # move objects around, helping to find object movement bugs.
-  GC.verify_compaction_references(double_heap: true, toward: :empty)
+  if defined?(GC.verify_compaction_references) == 'method' && !(RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/)
+    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2.0")
+      GC.verify_compaction_references(expand_heap: true, toward: :empty)
+    else
+      GC.verify_compaction_references(double_heap: true, toward: :empty)
+    end
+  end
 end
 
 

data/test/json_gem/json_generator_test.rb

@@ -72,6 +72,8 @@ EOT
     parsed_json = JSON.parse(json)
     assert_equal({"1"=>2}, parsed_json)
     assert_equal '666', JSON.pretty_generate(666)
+    json_nil_opts = JSON.pretty_generate({1=>2}, nil)
+    assert_equal json, json_nil_opts
   end
 
   def test_generate_custom
@@ -292,7 +294,9 @@ EOT
     assert_equal '2', state.indent
   end
 
-  if defined?(JSON::Ext::Generator)
+  if defined?(JSON::Ext::Generator) && Process.respond_to?(:fork)
+    # forking to avoid modifying core class of a parent process and
+    # introducing race conditions of tests are run in parallel
     def test_broken_bignum # [ruby-core:38867]
       pid = fork do
         x = 1 << 64
@@ -309,9 +313,6 @@ EOT
       end
       _, status = Process.waitpid2(pid)
       assert status.success?
-    rescue NotImplementedError
-      # forking to avoid modifying core class of a parent process and
-      # introducing race conditions of tests are run in parallel
     end
   end
 

data/test/json_gem/json_parser_test.rb

@@ -31,7 +31,7 @@ class JSONParserTest < Test::Unit::TestCase
     }
     assert_equal(Encoding::UTF_8, e.message.encoding, bug10705)
     assert_include(e.message, json, bug10705)
-  end if defined?(Encoding::UTF_8)
+  end if defined?(Encoding::UTF_8) and defined?(JSON::Ext::Parser)
 
   def test_parsing
     parser = JSON::Parser.new('"test"')
@@ -269,6 +269,13 @@ EOT
     assert_equal too_deep_ary, ok
     ok = JSON.parse too_deep, :max_nesting => 0
     assert_equal too_deep_ary, ok
+
+    unless ENV['REAL_JSON_GEM']
+      # max_nesting should be reset to 0 if not included in options
+      # This behavior is not compatible with Ruby standard JSON gem
+      ok = JSON.parse too_deep, {}
+      assert_equal too_deep_ary, ok
+    end
   end
 
   def test_backslash

data/test/json_gem/test_helper.rb

@@ -15,10 +15,14 @@ else
   require 'oj'
   Oj.mimic_JSON
 
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
   if defined?(GC.verify_compaction_references) == 'method'
-    # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
-    # move objects around, helping to find object movement bugs.
-    GC.verify_compaction_references(double_heap: true, toward: :empty)
+    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2.0")
+      GC.verify_compaction_references(expand_heap: true, toward: :empty)
+    else
+      GC.verify_compaction_references(double_heap: true, toward: :empty)
+    end
   end
 end
 

data/test/perf_dump.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'oj'
+
+$verbose = false
+$indent = 2
+$iter = 100_000
+$size = 2
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                    { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")       { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")     { |i| $indent = i }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+Oj.default_options = { :indent => $indent, :mode => :strict }
+
+if 0 < $size
+  o = $obj
+  $obj = []
+  (4 * $size).times do
+    $obj << o
+  end
+end
+
+$json = Oj.dump($obj)
+GC.start
+start = Time.now
+$iter.times { Oj.dump($obj) }
+duration = Time.now - start
+puts "Dumped #{$json.length} byte JSON #{$iter} times in %0.3f seconds or %0.3f iteration/sec." % [duration, $iter / duration]

data/test/test_compat.rb

@@ -488,6 +488,31 @@ class CompatJuice < Minitest::Test
     assert_equal([1,2], Oj.load(s, :mode => :compat))
   end
 
+  def test_parse_large_string
+    error = assert_raises() { Oj.load(%|{"a":"aaaaaaaaaa\0aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}|) }
+    assert(error.message.include?('NULL byte in string'))
+
+    error = assert_raises() { Oj.load(%|{"a":"aaaaaaaaaaaaaaaaaaaa                       }|) }
+    assert(error.message.include?('quoted string not terminated'))
+
+    json =<<~JSON
+    {
+      "a": "\\u3074\\u30fc\\u305f\\u30fc",
+      "b": "aaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    }
+    JSON
+    assert_equal("ぴーたー", Oj.load(json)['a'])
+  end
+
+  def test_parse_large_escaped_string
+    invalid_json = %|{"a":\"aaaa\\nbbbb\\rcccc\\tddd\\feee\\bf\/\\\\\\u3074\\u30fc\\u305f\\u30fc                             }|
+    error = assert_raises() { Oj.load(invalid_json) }
+    assert(error.message.include?('quoted string not terminated'))
+
+    json = "\"aaaa\\nbbbb\\rcccc\\tddd\\feee\\bf\/\\\\\\u3074\\u30fc\\u305f\\u30fc             \""
+    assert_equal("aaaa\nbbbb\rcccc\tddd\feee\bf/\\ぴーたー             ", Oj.load(json))
+  end
+
   def dump_and_load(obj, trace=false)
     json = Oj.dump(obj)
     puts json if trace

data/test/test_custom.rb

@@ -200,6 +200,8 @@ class CustomJuice < Minitest::Test
   end
 
   def test_deep_nest
+    skip 'TruffleRuby causes SEGV' if RUBY_ENGINE == 'truffleruby'
+
     begin
       n = 10000
       Oj.strict_load('[' * n + ']' * n)
@@ -480,9 +482,18 @@ class CustomJuice < Minitest::Test
   end
 
   def test_time
+    skip 'TruffleRuby fails this spec' if RUBY_ENGINE == 'truffleruby'
+
     obj = Time.now()
-    dump_load_dump(obj, false, :time_format => :unix, :create_id => "^o", :create_additions => true)
-    dump_load_dump(obj, false, :time_format => :unix_zone, :create_id => "^o", :create_additions => true)
+    # These two forms should be able to recreate the time precisely,
+    # so we check they can load a dumped version and recreate the
+    # original object correctly.
+    dump_and_load(obj, false, :time_format => :unix, :create_id => "^o", :create_additions => true)
+    dump_and_load(obj, false, :time_format => :unix_zone, :create_id => "^o", :create_additions => true)
+    # These two forms will lose precision while dumping as they don't
+    # preserve full precision. We check that a dumped version is equal
+    # to that version loaded and dumped a second time, but don't check
+    # that the loaded Ruby objects is still the same as the original.
     dump_load_dump(obj, false, :time_format => :xmlschema, :create_id => "^o", :create_additions => true)
     dump_load_dump(obj, false, :time_format => :ruby, :create_id => "^o", :create_additions => true)
   end

data/test/test_file.rb

@@ -130,8 +130,8 @@ class FileJuice < Minitest::Test
     dump_and_load(t, false)
   end
   def test_time_object_early
-    # Windows does not support dates before 1970.
-    return if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
+    skip 'Windows does not support dates before 1970.' if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
+
     t = Time.xmlschema("1954-01-05T00:00:00.123456")
     Oj.default_options = { :mode => :object, :time_format => :unix_zone }
     dump_and_load(t, false)
@@ -162,12 +162,10 @@ class FileJuice < Minitest::Test
   def test_range_object
     Oj.default_options = { :mode => :object }
     json = Oj.dump(1..7, :mode => :object, :indent => 0)
-    if 'rubinius' == $ruby
-      assert(%{{"^O":"Range","begin":1,"end":7,"exclude_end?":false}} == json)
-    elsif 'jruby' == $ruby
-      assert(%{{"^O":"Range","begin":1,"end":7,"exclude_end?":false}} == json)
-    else
+    if $ruby == 'ruby'
       assert_equal(%{{"^u":["Range",1,7,false]}}, json)
+    else
+      assert(%{{"^O":"Range","begin":1,"end":7,"exclude_end?":false}} == json)
     end
     dump_and_load(1..7, false)
     dump_and_load(1..1, false)
@@ -212,6 +210,24 @@ class FileJuice < Minitest::Test
     dump_and_load(DateTime.new(2012, 6, 19), false)
   end
 
+  def test_load_unicode_path
+    json =<<~JSON
+    {
+      "x":true,
+      "y":58,
+      "z": [1,2,3]
+    }
+    JSON
+
+    Tempfile.create('file_test_conceição1.json') do |f|
+      f.write(json)
+      f.close
+
+      objects = Oj.load_file(f.path)
+      assert_equal(Oj.load(json), objects)
+    end
+  end
+
   def dump_and_load(obj, trace=false)
     filename = File.join(File.dirname(__FILE__), 'file_test.json')
     File.open(filename, "w") { |f|

data/test/test_gc.rb

@@ -26,10 +26,12 @@ class GCTest < Minitest::Test
 
   def setup
     @default_options = Oj.default_options
+    GC.stress = true
   end
 
   def teardown
     Oj.default_options = @default_options
+    GC.stress = false
   end
 
   # if no crash then the GC marking is working
@@ -46,4 +48,13 @@ class GCTest < Minitest::Test
     json = Oj.dump(g, :mode => :object)
     Oj.object_load(json)
   end
+
+  def test_parse_gc
+    json = '{"a":"Alpha","b":true,"c":12345,"d":[true,[false,[-123456789,null],3.9676,["Something else.",false],null]],"e":{"zero":null,"one":1,"two":2,"three":[3],"four":[0,1,2,3,4]},"f":null,"h":{"a":{"b":{"c":{"d":{"e":{"f":{"g":null}}}}}}},"i":[[[[[[[null]]]]]]]}'
+
+    50.times do
+      data = Oj.load(json)
+      assert_equal(json, Oj.dump(data))
+    end
+  end
 end

data/test/test_object.rb

@@ -821,10 +821,10 @@ class ObjectJuice < Minitest::Test
   def test_range_object
     Oj.default_options = { :mode => :object }
     json = Oj.dump(1..7, :mode => :object, :indent => 0)
-    if 'rubinius' == $ruby
-      assert(%{{"^O":"Range","begin":1,"end":7,"exclude_end?":false}} == json)
-    else
+    if 'ruby' == $ruby
       assert_equal(%{{"^u":["Range",1,7,false]}}, json)
+    else
+      assert(%{{"^O":"Range","begin":1,"end":7,"exclude_end?":false}} == json)
     end
     dump_and_load(1..7, false)
     dump_and_load(1..1, false)
@@ -948,6 +948,11 @@ class ObjectJuice < Minitest::Test
 
   def test_odd_date
     dump_and_load(Date.new(2012, 6, 19), false)
+
+    Oj.register_odd(Date, Date, :jd, :jd)
+    json = Oj.dump(Date.new(2015, 3, 7), :mode => :object)
+    assert_equal(%|{"^O":"Date","jd":2457089}|, json)
+    dump_and_load(Date.new(2012, 6, 19), false)
   end
 
   def test_odd_datetime
@@ -972,13 +977,6 @@ class ObjectJuice < Minitest::Test
     dump_and_load(s, false)
   end
 
-  def test_odd_date_replaced
-    Oj.register_odd(Date, Date, :jd, :jd)
-    json = Oj.dump(Date.new(2015, 3, 7), :mode => :object)
-    assert_equal(%|{"^O":"Date","jd":2457089}|, json)
-    dump_and_load(Date.new(2012, 6, 19), false)
-  end
-
   def test_odd_raw
     Oj.register_odd_raw(Raw, Raw, :create, :to_json)
     json = Oj.dump(Raw.new(%|{"a":1}|), :mode => :object)

data/test/test_parser.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
+# encoding: UTF-8
 
 $: << File.dirname(__FILE__)
 $oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
@@ -7,21 +7,5 @@ $oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
   $: << File.join($oj_dir, dir)
 end
 
-require 'minitest'
-require 'minitest/autorun'
-require 'stringio'
-require 'date'
-require 'bigdecimal'
-require 'oj'
-
-class ParserJuice < Minitest::Test
-
-  def test_array
-    p = Oj::Parser.new(:debug)
-    out = p.parse(%|[true, false, null, 123, -1.23, "abc"]|)
-    puts out
-    out = p.parse(%|{"abc": []}|)
-    puts out
-  end
-
-end
+require 'test_parser_usual'
+require 'test_parser_saj'

data/test/test_parser_debug.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+$: << File.dirname(__FILE__)
+$oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
+%w(lib ext).each do |dir|
+  $: << File.join($oj_dir, dir)
+end
+
+require 'minitest'
+require 'minitest/autorun'
+require 'stringio'
+require 'date'
+require 'bigdecimal'
+require 'oj'
+
+class ParserJuice < Minitest::Test
+
+  def test_array
+    p = Oj::Parser.new(:debug)
+    out = p.parse(%|[true, false, null, 123, -1.23, "abc"]|)
+    puts out
+    out = p.parse(%|{"abc": []}|)
+    puts out
+  end
+
+end

data/test/test_parser_saj.rb

@@ -5,7 +5,7 @@ $: << File.dirname(__FILE__)
 
 require 'helper'
 
-$json = %{{
+$json = %|{
   "array": [
     {
       "num"   : 3,
@@ -18,7 +18,7 @@ $json = %{{
     }
   ],
   "boolean" : true
-}}
+}|
 
 class AllSaj < Oj::Saj
   attr_accessor :calls
@@ -53,6 +53,35 @@ class AllSaj < Oj::Saj
 
 end # AllSaj
 
+class LocSaj
+  attr_accessor :calls
+
+  def initialize()
+    @calls = []
+  end
+
+  def hash_start(key, line, column)
+    @calls << [:hash_start, key, line, column]
+  end
+
+  def hash_end(key, line, column)
+    @calls << [:hash_end, key, line, column]
+  end
+
+  def array_start(key, line, column)
+    @calls << [:array_start, key, line, column]
+  end
+
+  def array_end(key, line, column)
+    @calls << [:array_end, key, line, column]
+  end
+
+  def add_value(value, key, line, column)
+    @calls << [:add_value, value, key, line, column]
+  end
+
+end # LocSaj
+
 class SajTest < Minitest::Test
 
   def test_nil
@@ -120,6 +149,43 @@ class SajTest < Minitest::Test
     assert_equal((12345.6789e7 * 10000).to_i, (handler.calls[0][1] * 10000).to_i)
   end
 
+  def test_bignum
+    handler = AllSaj.new()
+    json = %{-11.899999999999999}
+    p = Oj::Parser.new(:saj)
+    p.handler = handler
+    p.parse(json)
+    assert_equal(1, handler.calls.size)
+    assert_equal(:add_value, handler.calls[0][0])
+    assert_equal(-118999, (handler.calls[0][1] * 10000).to_i)
+  end
+
+  def test_bignum_loc
+    handler = LocSaj.new()
+    json = <<~JSON
+      {
+        "width": 192.33800000000002,
+        "xaxis": {
+          "anchor": "y"
+        }
+      }
+    JSON
+
+    p = Oj::Parser.new(:saj)
+    p.handler = handler
+    p.parse(json)
+    assert_equal(6, handler.calls.size)
+    assert_equal(1_923_380, (handler.calls[1][1] * 10000).to_i)
+    handler.calls[1][1] = 1_923_380
+    assert_equal([[:hash_start, nil, 1, 1],
+                  [:add_value, 1923380, 'width', 2, 30],
+                  [:hash_start, 'xaxis', 3, 12],
+                  [:add_value, 'y', 'anchor', 4, 17],
+                  [:hash_end, 'xaxis', 5, 3],
+                  [:hash_end, nil, 6, 1]],
+                 handler.calls)
+  end
+
   def test_array_empty
     handler = AllSaj.new()
     json = %{[]}
@@ -242,4 +308,28 @@ class SajTest < Minitest::Test
 		 ], handler.calls)
   end
 
+  def test_loc
+    handler = LocSaj.new()
+    Oj::Parser.saj.handler = handler
+    Oj::Parser.saj.parse($json)
+    assert_equal([[:hash_start, nil, 1, 1],
+                  [:array_start, 'array', 2, 12],
+                  [:hash_start, nil, 3, 5],
+                  [:add_value, 3, 'num', 4, 18],
+                  [:add_value, 'message', 'string', 5, 25],
+                  [:hash_start, 'hash', 6, 17],
+                  [:hash_start, 'h2', 7, 17],
+                  [:array_start, 'a', 8, 17],
+                  [:add_value, 1, nil, 8, 20],
+                  [:add_value, 2, nil, 8, 23],
+                  [:add_value, 3, nil, 8, 26],
+                  [:array_end, 'a', 8, 27],
+                  [:hash_end, 'h2', 9, 9],
+                  [:hash_end, 'hash', 10, 7],
+                  [:hash_end, nil, 11, 5],
+                  [:array_end, 'array', 12, 3],
+                  [:add_value, true, 'boolean', 13, 18],
+                  [:hash_end, nil, 14, 1]], handler.calls)
+  end
+
 end

data/test/test_scp.rb

@@ -320,8 +320,7 @@ class ScpTest < Minitest::Test
   end
 
   def test_pipe
-    # Windows does not support fork
-    return if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
+    skip 'needs fork' unless Process.respond_to?(:fork)
 
     handler = AllHandler.new()
     json = %{{"one":true,"two":false}}
@@ -357,8 +356,7 @@ class ScpTest < Minitest::Test
   end
 
   def test_pipe_close
-    # Windows does not support fork
-    return if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
+    skip 'needs fork' unless Process.respond_to?(:fork)
 
     json = %{{"one":true,"two":false}}
     IO.pipe do |read_io, write_io|

data/test/test_strict.rb

@@ -156,6 +156,8 @@ class StrictJuice < Minitest::Test
   end
 
   def test_deep_nest
+    skip 'TruffleRuby causes SEGV' if RUBY_ENGINE == 'truffleruby'
+
     begin
       n = 10000
       Oj.strict_load('[' * n + ']' * n)

data/test/test_various.rb

@@ -345,6 +345,12 @@ class Juice < Minitest::Test
     out = Oj.dump hash
     assert_equal(%{{"key":"I \\u003c3 this"}}, out)
   end
+  def test_escapes_slashes_by_default_when_configured_to_do_so
+    hash = {'key' => "I <3 this </script>"}
+    Oj.default_options = {:escape_mode => :slash}
+    out = Oj.dump hash
+    assert_equal(%{{"key":"I <3 this <\\/script>"}}, out)
+  end
   def test_escapes_entities_when_asked_to
     hash = {'key' => "I <3 this"}
     out = Oj.dump(hash, :escape_mode => :xss_safe)
@@ -553,9 +559,6 @@ class Juice < Minitest::Test
   end
 
   def test_io_file
-    # Windows does not support fork
-    return if RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/
-
     src = { 'x' => true, 'y' => 58, 'z' => [1, 2, 3]}
     filename = File.join(File.dirname(__FILE__), 'open_file_test.json')
     File.open(filename, "w") { |f|
@@ -568,6 +571,8 @@ class Juice < Minitest::Test
   end
 
   def test_io_stream
+    skip 'needs fork' unless Process.respond_to?(:fork)
+
     IO.pipe do |r, w|
       if fork
 	r.close

data/test/test_wab.rb

@@ -105,6 +105,8 @@ class WabJuice < Minitest::Test
   end
 
   def test_deep_nest
+    skip 'TruffleRuby causes SEGV' if RUBY_ENGINE == 'truffleruby'
+
     begin
       n = 10000
       Oj.wab_load('[' * n + ']' * n)

data/test/tests.rb

@@ -22,3 +22,12 @@ require 'test_rails'
 require 'test_wab'
 require 'test_writer'
 require 'test_integer_range'
+
+at_exit do
+  require 'helper'
+  if '3.1.0' <= RUBY_VERSION && !(RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/)
+    #Oj::debug_odd("teardown before GC.verify_compaction_references")
+    verify_gc_compaction
+    #Oj::debug_odd("teardown after GC.verify_compaction_references")
+  end
+end

data/test/tests_mimic.rb

@@ -12,3 +12,12 @@ require 'json_generator_test'
 require 'json_generic_object_test'
 require 'json_parser_test'
 require 'json_string_matching_test'
+
+at_exit do
+  require 'helper'
+  if '3.1.0' <= RUBY_VERSION && !(RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/)
+    #Oj::debug_odd("teardown before GC.verify_compaction_references")
+    verify_gc_compaction
+    #Oj::debug_odd("teardown after GC.verify_compaction_references")
+  end
+end

data/test/tests_mimic_addition.rb

@@ -5,3 +5,12 @@ $: << File.dirname(__FILE__)
 $: << File.join(File.dirname(__FILE__), 'json_gem')
 
 require 'json_addition_test'
+
+at_exit do
+  require 'helper'
+  if '3.1.0' <= RUBY_VERSION && !(RbConfig::CONFIG['host_os'] =~ /(mingw|mswin)/)
+    #Oj::debug_odd("teardown before GC.verify_compaction_references")
+    verify_gc_compaction
+    #Oj::debug_odd("teardown after GC.verify_compaction_references")
+  end
+end