oj 2.1.3 → 2.1.4

data/ext/oj/strict.c

@@ -36,6 +36,7 @@
 #include "oj.h"
 #include "err.h"
 #include "parse.h"
+#include "encode.h"
 
 // Workaround in case INFINITY is not defined in math.h or if the OS is CentOS
 #define OJ_INFINITY (1.0/0.0)
@@ -53,9 +54,7 @@ static void
 add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
     VALUE	rstr = rb_str_new(str, len);
 
-#if HAS_ENCODING_SUPPORT
-    rb_enc_associate(rstr, oj_utf8_encoding);
-#endif
+    rstr = oj_encode(rstr);
     pi->stack.head->val = rstr;
 }
 
@@ -73,9 +72,7 @@ static VALUE
 hash_key(ParseInfo pi, const char *key, size_t klen) {
     VALUE	rkey = rb_str_new(key, klen);
 
-#if HAS_ENCODING_SUPPORT
-    rb_enc_associate(rkey, oj_utf8_encoding);
-#endif
+    rkey = oj_encode(rkey);
     if (Yes == pi->options.sym_key) {
 	rkey = rb_str_intern(rkey);
     }
@@ -86,9 +83,7 @@ static void
 hash_set_cstr(ParseInfo pi, const char *key, size_t klen, const char *str, size_t len, const char *orig) {
     VALUE	rstr = rb_str_new(str, len);
 
-#if HAS_ENCODING_SUPPORT
-    rb_enc_associate(rstr, oj_utf8_encoding);
-#endif
+    rstr = oj_encode(rstr);
     rb_hash_aset(stack_peek(&pi->stack)->val, hash_key(pi, key, klen), rstr);
 }
 
@@ -111,9 +106,7 @@ static void
 array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
     VALUE	rstr = rb_str_new(str, len);
 
-#if HAS_ENCODING_SUPPORT
-    rb_enc_associate(rstr, oj_utf8_encoding);
-#endif
+    rstr = oj_encode(rstr);
     rb_ary_push(stack_peek(&pi->stack)->val, rstr);
 }
 

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module. 
-  VERSION = '2.1.3'
+  VERSION = '2.1.4'
 end

data/test/a.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << File.dirname(__FILE__)
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'pp'
+require 'oj'
+require 'perf'
+
+obj = [[1],[2],[3],[4],[5],[6],[7],[8],[9]]
+obj = [[],[],[],[],[],[],[],[],[]]
+obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [12345, nil], 3.967, ['something', false], nil]], # mix it up array
+  'e' => { 'one' => 1, 'two' => 2 }, # hash
+  'f' => nil,     # nil
+  'g' => 12345678901234567890123456789, # big number
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+json = Oj.dump(obj, mode: :compat)
+
+puts json
+#pp Oj.saj_parse(nil, json)
+pp Oj.t_parse(json)
+
+if true
+  perf = Perf.new()
+  perf.add('SAJ', 'oj') { Oj.saj_parse(nil, json) }
+  perf.add('T', 'oj') { Oj.t_parse(json) }
+  perf.add('load', 'oj') { Oj.load(json) }
+  perf.run(10000)
+end

data/test/bug.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
+# required. That can be set in the RUBYOPT environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+Oj.load_file('josh2.json', :mode => :strict)

data/test/e.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+json = %{"\xc2\xa9\xc3\x98"}
+puts "original:\n#{json}"
+
+str = Oj.load(json)
+puts "out: #{str}"

data/test/files.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby -wW2
+
+if $0 == __FILE__
+  $: << '.'
+  $: << '..'
+  $: << '../lib'
+  $: << '../ext'
+end
+
+require 'pp'
+require 'sample/file'
+require 'sample/dir'
+
+def files(dir)
+  d = ::Sample::Dir.new(dir)
+  Dir.new(dir).each do |fn|
+    next if fn.start_with?('.')
+    filename = File.join(dir, fn)
+    #filename = '.' == dir ? fn : File.join(dir, fn)
+    if File.directory?(filename)
+      d << files(filename)
+    else
+      d << ::Sample::File.new(filename)
+    end
+  end
+  #pp d
+  d
+end
+

data/test/foo.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Ubuntu does not accept arguments to ruby when called using env. To get warnings to show up the -w options is
+# required. That can be set in the RUBYOPT environment variable.
+# export RUBYOPT=-w
+
+$VERBOSE = true
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'oj'
+
+reltypes={}
+Oj::Doc.open_file('foo.json') do |doc|
+  doc.each_child do |target|
+    puts "#{target.local_key} is #{target.local_key.class}"
+    target.each_leaf do |score|
+      reltype=score.local_key
+      reltypes[reltype] = (reltypes[reltype] || 0) + 1
+    end
+  end
+end

data/test/mj.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+# $: << File.join(File.dirname(__FILE__), "../../multi_json/lib")
+
+require 'multi_json'
+require 'benchmark'
+require 'yajl'
+require 'json'
+require 'oj'
+
+iter = 1_000_000
+iter = 100_000
+
+json = %({"k1":"val1","k2":"val2","k3":"val3"})
+obj = { k1: "val1", k2: "val2", k3: "val3" }
+
+puts "Benchmarks for different JSON handlers with MultiJson."
+puts "  Ruby #{RUBY_VERSION}"
+puts "  #{iter} iterations"
+
+MultiJson.engine = :oj
+dt = Benchmark.realtime { iter.times { MultiJson.decode(json) }}
+et = Benchmark.realtime { iter.times { MultiJson.encode(obj) }}
+puts "    Oj decode: #{dt}  encode: #{et}"
+
+MultiJson.engine = :yajl
+dt = Benchmark.realtime { iter.times { MultiJson.decode(json) }}
+et = Benchmark.realtime { iter.times { MultiJson.encode(obj) }}
+puts "  Yajl decode: #{dt}  encode: #{et}"
+
+MultiJson.engine = :json_gem
+dt = Benchmark.realtime { iter.times { MultiJson.decode(json) }}
+et = Benchmark.realtime { iter.times { MultiJson.encode(obj) }}
+puts "  Json decode: #{dt}  encode: #{et}"
+
+Oj.default_options = { :mode => :compat, :time_format => :ruby }
+dt = Benchmark.realtime { iter.times { Oj.load(json) }}
+et = Benchmark.realtime { iter.times { Oj.dump(obj) }}
+puts "Raw Oj decode: #{dt}  encode: #{et}"
+
+ye = Yajl::Encoder.new
+dt = Benchmark.realtime { iter.times { Yajl::Parser.parse(json) }}
+et = Benchmark.realtime { iter.times { Yajl::Encoder.encode(obj) }}
+e2 = Benchmark.realtime { iter.times { ye.encode(obj) }}
+puts "Raw Yajl decode: #{dt} encode: #{et}, encoder: #{e2}"

data/test/perf.rb

@@ -0,0 +1,107 @@
+
+class Perf
+
+  def initialize()
+    @items = []
+  end
+
+  def add(title, op, &blk)
+    @items << Item.new(title, op, &blk)
+  end
+
+  def before(title, &blk)
+    @items.each do |i|
+      if title == i.title
+        i.set_before(&blk)
+        break
+      end
+    end
+  end
+  
+  def run(iter)
+    base = Item.new(nil, nil) { }
+    base.run(iter, 0.0)
+    @items.each do |i|
+      i.run(iter, base.duration)
+      if i.error.nil?
+        puts "#{i.title}.#{i.op} #{iter} times in %0.3f seconds or %0.3f #{i.op}/sec." % [i.duration, iter / i.duration]
+      else
+        puts "***** #{i.title}.#{i.op} failed! #{i.error}"
+      end
+    end
+    summary()
+  end
+
+  def summary()
+    fastest = nil
+    slowest = nil
+    width = 6
+    @items.each do |i|
+      next if i.duration.nil?
+      width = i.title.size if width < i.title.size
+    end
+    iva = @items.clone
+    iva.delete_if { |i| i.duration.nil? }
+    iva = iva.sort_by { |i| i.duration }
+    puts
+    puts "Summary:"
+    puts "%*s  time (secs)  rate (ops/sec)" % [width, 'System']
+    puts "#{'-' * width}  -----------  --------------"
+    iva.each do |i|
+      if i.duration.nil?
+      else
+        puts "%*s %11.3f  %14.3f" % [width, i.title, i.duration, i.rate ]
+      end
+    end
+    puts
+    puts "Comparison Matrix\n(performance factor, 2.0 means row is twice as fast as column)"
+    puts ([' ' * width] + iva.map { |i| "%*s" % [width, i.title] }).join('  ')
+    puts (['-' * width] + iva.map { |i| '-' * width }).join('  ')
+    iva.each do |i|
+      line = ["%*s" % [width, i.title]]
+      iva.each do |o|
+        line << "%*.2f" % [width, o.duration / i.duration]
+      end
+      puts line.join('  ')
+    end
+    puts
+  end
+
+  class Item
+    attr_accessor :title
+    attr_accessor :op
+    attr_accessor :blk
+    attr_accessor :duration
+    attr_accessor :rate
+    attr_accessor :error
+
+    def initialize(title, op, &blk)
+      @title = title
+      @blk = blk
+      @op = op
+      @duration = nil
+      @rate = nil
+      @error = nil
+      @before = nil
+    end
+
+    def set_before(&blk)
+      @before = blk
+    end
+
+    def run(iter, base)
+      begin
+        GC.start
+        @before.call unless @before.nil?
+        start = Time.now
+        iter.times { @blk.call }
+        @duration = Time.now - start - base
+        @duration = 0.0 if @duration < 0.0
+        @rate = iter / @duration
+      rescue Exception => e
+        @error = "#{e.class}: #{e.message}"
+      end
+    end
+
+  end # Item
+end # Perf

data/test/perf_compat.rb

@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
+
+require 'optparse'
+require 'perf'
+require 'oj'
+
+$verbose = false
+$indent = 0
+$iter = 20000
+$with_bignum = false
+$with_nums = true
+$size = 0
+
+opts = OptionParser.new
+opts.on("-v", "verbose")                                    { $verbose = true }
+opts.on("-c", "--count [Int]", Integer, "iterations")       { |i| $iter = i }
+opts.on("-i", "--indent [Int]", Integer, "indentation")     { |i| $indent = i }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+module One
+  module Two
+    module Three
+      class Empty
+
+        def initialize()
+        end
+
+        def eql?(o)
+          self.class == o.class
+        end
+        alias == eql?
+
+        def to_hash()
+          {'json_class' => "#{self.class.name}"}
+        end
+
+        def to_json(*a)
+          %{{"json_class":"#{self.class.name}"}}
+        end
+
+        def self.json_create(h)
+          self.new()
+        end
+      end # Empty
+    end # Three
+  end # Two
+end # One
+
+$obj = {
+  'a' => 'Alpha', # string
+  'b' => true,    # boolean
+  'c' => 12345,   # number
+  'd' => [ true, [false, [-123456789, nil], 3.9676, ['Something else.', false], nil]], # mix it up array
+  'e' => { 'zero' => nil, 'one' => 1, 'two' => 2, 'three' => [3], 'four' => [0, 1, 2, 3, 4] }, # hash
+  'f' => nil,     # nil
+  'g' => One::Two::Three::Empty.new(),
+  'h' => { 'a' => { 'b' => { 'c' => { 'd' => {'e' => { 'f' => { 'g' => nil }}}}}}}, # deep hash, not that deep
+  'i' => [[[[[[[nil]]]]]]]  # deep array, again, not that deep
+}
+
+Oj.default_options = { :indent => $indent, :mode => :compat }
+
+if 0 < $size
+  s = Oj.dump($obj, :mode => :compat).size + 1
+  cnt = $size * 1024 / s
+  o = $obj
+  $obj = []
+  cnt.times do
+    $obj << o
+  end
+end
+
+$json = Oj.dump($obj, :mode => :compat)
+$failed = {} # key is same as String used in tests later
+
+def capture_error(tag, orig, load_key, dump_key, &blk)
+  begin
+    obj = blk.call(orig)
+    raise "#{tag} #{dump_key} and #{load_key} did not return the same object as the original." unless orig == obj
+  rescue Exception => e
+    $failed[tag] = "#{e.class}: #{e.message}"
+  end
+end
+
+# Verify that all packages dump and load correctly and return the same Object as the original.
+capture_error('Oj:compat', $obj, 'load', 'dump') { |o| Oj.compat_load(Oj.dump(o, :mode => :compat)) }
+capture_error('JSON::Ext', $obj, 'generate', 'parse') { |o|
+  require 'json'
+  require 'json/ext'
+  JSON.generator = JSON::Ext::Generator
+  JSON.parser = JSON::Ext::Parser
+  JSON.load(JSON.generate(o))
+}
+
+if $verbose
+  puts "size: #{$json.size}"
+  puts "json:\n#{$json}\n"
+  puts "Oj:compat loaded object:\n#{Oj.compat_load($json)}\n"
+  puts "JSON loaded object:\n#{JSON::Ext::Parser.new($json).parse}\n"
+end
+
+puts '-' * 80
+puts "Compat Parse Performance"
+perf = Perf.new()
+unless $failed.has_key?('JSON::Ext')
+  perf.add('JSON::Ext', 'parse') { JSON.load($json) }
+  perf.before('JSON::Ext') { JSON.parser = JSON::Ext::Parser }
+end
+unless $failed.has_key?('Oj:compat')
+  perf.add('Oj:compat', 'compat_load') { Oj.compat_load($json) }
+end
+perf.run($iter)
+
+puts
+puts '-' * 80
+puts
+
+unless $failed.empty?
+  puts "The following packages were not included for the reason listed"
+  $failed.each { |tag,msg| puts "***** #{tag}: #{msg}" }
+end