oj 3.11.8 → 3.12.3

data/ext/oj/strict.c

@@ -8,10 +8,65 @@
 
 #include "encode.h"
 #include "err.h"
+#include "hash.h"
 #include "oj.h"
 #include "parse.h"
 #include "trace.h"
 
+VALUE oj_cstr_to_value(const char *str, size_t len, size_t cache_str) {
+    volatile VALUE rstr = Qnil;
+
+    if (len <= cache_str) {
+        VALUE *slot;
+
+        if (Qnil == (rstr = oj_str_hash_get(str, len, &slot))) {
+            rstr  = rb_str_new(str, len);
+            rstr  = oj_encode(rstr);
+            *slot = rstr;
+            rb_gc_register_address(slot);
+        }
+    } else {
+        rstr = rb_str_new(str, len);
+        rstr = oj_encode(rstr);
+    }
+    return rstr;
+}
+
+VALUE oj_calc_hash_key(ParseInfo pi, Val parent) {
+    volatile VALUE rkey = parent->key_val;
+
+    if (Qundef != rkey) {
+        return rkey;
+    }
+    if (Yes != pi->options.cache_keys) {
+        rkey = rb_str_new(parent->key, parent->klen);
+        rkey = oj_encode(rkey);
+        if (Yes == pi->options.sym_key) {
+            rkey = rb_str_intern(rkey);
+        }
+        return rkey;
+    }
+    VALUE *slot;
+
+    if (Yes == pi->options.sym_key) {
+        if (Qnil == (rkey = oj_sym_hash_get(parent->key, parent->klen, &slot))) {
+            rkey  = rb_str_new(parent->key, parent->klen);
+            rkey  = oj_encode(rkey);
+            rkey  = rb_str_intern(rkey);
+            *slot = rkey;
+            rb_gc_register_address(slot);
+        }
+    } else {
+        if (Qnil == (rkey = oj_str_hash_get(parent->key, parent->klen, &slot))) {
+            rkey  = rb_str_new(parent->key, parent->klen);
+            rkey  = oj_encode(rkey);
+            *slot = rkey;
+            rb_gc_register_address(slot);
+        }
+    }
+    return rkey;
+}
+
 static void hash_end(ParseInfo pi) {
     if (Yes == pi->options.trace) {
         oj_trace_parse_hash_end(pi, __FILE__, __LINE__);
@@ -36,9 +91,8 @@ static void add_value(ParseInfo pi, VALUE val) {
 }
 
 static void add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
 
-    rstr                = oj_encode(rstr);
     pi->stack.head->val = rstr;
     if (Yes == pi->options.trace) {
         oj_trace_parse_call("add_string", pi, __FILE__, __LINE__, rstr);
@@ -65,24 +119,12 @@ static VALUE start_hash(ParseInfo pi) {
     return rb_hash_new();
 }
 
-static VALUE calc_hash_key(ParseInfo pi, Val parent) {
-    volatile VALUE rkey = parent->key_val;
-
-    if (Qundef == rkey) {
-        rkey = rb_str_new(parent->key, parent->klen);
-    }
-    rkey = oj_encode(rkey);
-    if (Yes == pi->options.sym_key) {
-        rkey = rb_str_intern(rkey);
-    }
-    return rkey;
-}
-
 static void hash_set_cstr(ParseInfo pi, Val parent, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
 
-    rstr = oj_encode(rstr);
-    rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), rstr);
+    rb_hash_aset(stack_peek(&pi->stack)->val,
+                 oj_calc_hash_key(pi, parent),
+                 rstr);
     if (Yes == pi->options.trace) {
         oj_trace_parse_call("set_string", pi, __FILE__, __LINE__, rstr);
     }
@@ -95,14 +137,18 @@ static void hash_set_num(ParseInfo pi, Val parent, NumInfo ni) {
         oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
     }
     v = oj_num_as_value(ni);
-    rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), v);
+    rb_hash_aset(stack_peek(&pi->stack)->val,
+                 oj_calc_hash_key(pi, parent),
+                 v);
     if (Yes == pi->options.trace) {
         oj_trace_parse_call("set_number", pi, __FILE__, __LINE__, v);
     }
 }
 
 static void hash_set_value(ParseInfo pi, Val parent, VALUE value) {
-    rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), value);
+    rb_hash_aset(stack_peek(&pi->stack)->val,
+                 oj_calc_hash_key(pi, parent),
+                 value);
     if (Yes == pi->options.trace) {
         oj_trace_parse_call("set_value", pi, __FILE__, __LINE__, value);
     }
@@ -116,9 +162,8 @@ static VALUE start_array(ParseInfo pi) {
 }
 
 static void array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
 
-    rstr = oj_encode(rstr);
     rb_ary_push(stack_peek(&pi->stack)->val, rstr);
     if (Yes == pi->options.trace) {
         oj_trace_parse_call("append_string", pi, __FILE__, __LINE__, rstr);

data/ext/oj/wab.c

@@ -10,6 +10,7 @@
 #include "dump.h"
 #include "encode.h"
 #include "err.h"
+#include "hash.h"
 #include "oj.h"
 #include "parse.h"
 #include "trace.h"
@@ -292,6 +293,34 @@ void oj_dump_wab_val(VALUE obj, int depth, Out out) {
 
 ///// load functions /////
 
+static VALUE calc_hash_key(ParseInfo pi, Val parent) {
+    volatile VALUE rkey = parent->key_val;
+
+    if (Qundef != rkey) {
+        rkey = oj_encode(rkey);
+        rkey = rb_str_intern(rkey);
+
+        return rkey;
+    }
+    if (Yes != pi->options.cache_keys) {
+        rkey = rb_str_new(parent->key, parent->klen);
+        rkey = oj_encode(rkey);
+	rkey = rb_str_intern(rkey);
+
+        return rkey;
+    }
+    VALUE *slot;
+
+    if (Qnil == (rkey = oj_sym_hash_get(parent->key, parent->klen, &slot))) {
+        rkey  = rb_str_new(parent->key, parent->klen);
+        rkey  = oj_encode(rkey);
+        rkey  = rb_str_intern(rkey);
+        *slot = rkey;
+        rb_gc_register_address(slot);
+    }
+    return rkey;
+}
+
 static void hash_end(ParseInfo pi) {
     if (Yes == pi->options.trace) {
         oj_trace_parse_hash_end(pi, __FILE__, __LINE__);
@@ -432,7 +461,7 @@ static VALUE protect_uri(VALUE rstr) {
     return rb_funcall(resolve_uri_class(), oj_parse_id, 1, rstr);
 }
 
-static VALUE cstr_to_rstr(const char *str, size_t len) {
+static VALUE cstr_to_rstr(ParseInfo pi, const char *str, size_t len) {
     volatile VALUE v = Qnil;
 
     if (30 == len && '-' == str[4] && '-' == str[7] && 'T' == str[10] && ':' == str[13] &&
@@ -445,20 +474,20 @@ static VALUE cstr_to_rstr(const char *str, size_t len) {
         uuid_check(str, (int)len) && Qnil != resolve_wab_uuid_class()) {
         return rb_funcall(wab_uuid_clas, oj_new_id, 1, rb_str_new(str, len));
     }
-    v = rb_str_new(str, len);
     if (7 < len && 0 == strncasecmp("http://", str, 7)) {
         int            err = 0;
+	v = rb_str_new(str, len);
         volatile VALUE uri = rb_protect(protect_uri, v, &err);
 
         if (0 == err) {
             return uri;
         }
     }
-    return oj_encode(v);
+    return oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
 }
 
 static void add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    pi->stack.head->val = cstr_to_rstr(str, len);
+    pi->stack.head->val = cstr_to_rstr(pi, str, len);
     if (Yes == pi->options.trace) {
         oj_trace_parse_call("add_string", pi, __FILE__, __LINE__, pi->stack.head->val);
     }
@@ -484,20 +513,8 @@ static VALUE start_hash(ParseInfo pi) {
     return rb_hash_new();
 }
 
-static VALUE calc_hash_key(ParseInfo pi, Val parent) {
-    volatile VALUE rkey = parent->key_val;
-
-    if (Qundef == rkey) {
-        rkey = rb_str_new(parent->key, parent->klen);
-    }
-    rkey = oj_encode(rkey);
-    rkey = rb_str_intern(rkey);
-
-    return rkey;
-}
-
 static void hash_set_cstr(ParseInfo pi, Val parent, const char *str, size_t len, const char *orig) {
-    volatile VALUE rval = cstr_to_rstr(str, len);
+    volatile VALUE rval = cstr_to_rstr(pi, str, len);
 
     rb_hash_aset(stack_peek(&pi->stack)->val, calc_hash_key(pi, parent), rval);
     if (Yes == pi->options.trace) {
@@ -533,7 +550,7 @@ static VALUE start_array(ParseInfo pi) {
 }
 
 static void array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rval = cstr_to_rstr(str, len);
+    volatile VALUE rval = cstr_to_rstr(pi, str, len);
 
     rb_ary_push(stack_peek(&pi->stack)->val, rval);
     if (Yes == pi->options.trace) {

data/lib/oj/mimic.rb

@@ -23,6 +23,8 @@ module Oj
     bigdecimal_load: :auto,
     circular: false,
     class_cache: false,
+    cache_keys: true,
+    cache_str: 5,
     create_additions: false,
     create_id: "json_class",
     empty_string: false,

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module.
-  VERSION = '3.11.8'
+  VERSION = '3.12.3'
 end

data/pages/Modes.md

@@ -97,6 +97,8 @@ information.
 | :bigdecimal_as_decimal | Boolean |         |         |         |       3 |       x |       x |         |
 | :bigdecimal_load       | Boolean |         |         |         |         |         |       x |         |
 | :compat_bigdecimal     | Boolean |         |         |       x |         |         |       x |         |
+| :cache_keys            | Boolean |       x |       x |       x |       x |         |       x |         |
+| :cache_strings         | Fixnum  |       x |       x |       x |       x |         |       x |         |
 | :circular              | Boolean |       x |       x |       x |       x |       x |       x |         |
 | :class_cache           | Boolean |         |         |         |         |       x |       x |         |
 | :create_additions      | Boolean |         |         |       x |       x |         |       x |         |

data/pages/Options.md

@@ -70,13 +70,17 @@ This can also be set with `:decimal_class` when used as a load or
 parse option to match the JSON gem. In that case either `Float`,
 `BigDecimal`, or `nil` can be provided.
 
-### :compat_bigdecimal [Boolean]
+### :cache_keys [Boolean]
 
-Determines how to load decimals when in `:compat` mode.
+If true Hash keys are cached or interned. There are trade-offs with
+caching keys. Large caches will use more memory and in extreme cases
+(like over a million) the cache may be slower than not using
+it. Repeated parsing of similar JSON docs is where cache_keys shines.
 
- - `true` convert all decimal numbers to BigDecimal.
+### :cache_strings [Int]
 
- - `false` convert all decimal numbers to Float.
+Shorter strings can be cached for better performance. A limit,
+cache_strings, defines the upper limit on what strings are cached.
 
 ### :circular [Boolean]
 
@@ -90,6 +94,14 @@ recreate the looped references on load.
 Cache classes for faster parsing. This option should not be used if
 dynamically modifying classes or reloading classes then don't use this.
 
+### :compat_bigdecimal [Boolean]
+
+Determines how to load decimals when in `:compat` mode.
+
+ - `true` convert all decimal numbers to BigDecimal.
+
+ - `false` convert all decimal numbers to Float.
+
 ### :create_additions
 
 A flag indicating that the :create_id key, when encountered during parsing,
@@ -251,7 +263,13 @@ compatibility. Using just indent as an integer gives better performance.
 
 ### :symbol_keys [Boolean]
 
-Use symbols instead of strings for hash keys. :symbolize_names is an alias.
+Use symbols instead of strings for hash keys.
+
+### :symbolize_names [Boolean]
+
+Like :symbol_keys has keys are made into symbols but only when
+mimicing the JSON gem and then only as the JSON gem honors it so
+JSON.parse honors the option but JSON.load does not.
 
 ### :trace
 

data/test/foo.rb

@@ -7,14 +7,7 @@ $oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
 end
 
 require 'oj'
-require 'active_support'
 
-Oj.default_options = {trace: true}
+p = Oj::Parser.new(:debug)
 
-Oj::Rails.mimic_JSON
-#Oj.mimic_JSON
-begin
-  ::JSON.load('foo=&bar')
-rescue Exception => e
-  puts "*** #{e.class}: #{e.message}"
-end
+p.parse("[true, false]")

data/test/perf.rb

@@ -17,7 +17,7 @@ class Perf
       end
     end
   end
-  
+
   def run(iter)
     base = Item.new(nil, nil) { }
     base.run(iter, 0.0)

data/test/perf_scp.rb

@@ -14,16 +14,16 @@ require 'oj'
 
 $verbose = false
 $indent = 0
-$iter = 50000
+$iter = 50_000
 $with_bignum = false
-$size = 0
+$size = 1
 
 opts = OptionParser.new
 opts.on("-v", "verbose")                                  { $verbose = true }
 opts.on("-c", "--count [Int]", Integer, "iterations")     { |i| $iter = i }
 opts.on("-i", "--indent [Int]", Integer, "indentation")   { |i| $indent = i }
-opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
-opts.on("-b", "with bignum")                                { $with_bignum = true }
+opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")  { |i| $size = i }
+opts.on("-b", "with bignum")                              { $with_bignum = true }
 opts.on("-h", "--help", "Show this display")              { puts opts; Process.exit!(0) }
 files = opts.parse(ARGV)
 
@@ -47,7 +47,7 @@ if 0 < $size
   end
 end
 
-Oj.default_options = { :indent => $indent, :mode => :compat }
+Oj.default_options = { :indent => $indent, :mode => :strict, cache_keys: true, cache_str: 5 }
 
 $json = Oj.dump($obj)
 $failed = {} # key is same as String used in tests later
@@ -105,7 +105,7 @@ class AllHandler < Oj::ScHandler
 
   def hash_set(h, key, value)
   end
-  
+
   def array_append(a, value)
   end
 
@@ -137,10 +137,11 @@ end
 puts '-' * 80
 puts "Parse Performance"
 perf = Perf.new()
-perf.add('Oj::Saj', 'all') { Oj.saj_parse(saj_handler, $json) }
-perf.add('Oj::Saj', 'none') { Oj.saj_parse(no_saj, $json) }
-perf.add('Oj::Scp', 'all') { Oj.sc_parse(sc_handler, $json) }
-perf.add('Oj::Scp', 'none') { Oj.sc_parse(no_handler, $json) }
+perf.add('Oj::Saj.all', 'all') { Oj.saj_parse(saj_handler, $json) }
+perf.add('Oj::Saj.none', 'none') { Oj.saj_parse(no_saj, $json) }
+perf.add('Oj::Scp.all', 'all') { Oj.sc_parse(sc_handler, $json) }
+perf.add('Oj::Scp.none', 'none') { Oj.sc_parse(no_handler, $json) }
+perf.add('Oj::load', 'none') { Oj.wab_load($json) }
 perf.add('Yajl', 'parse') { Yajl::Parser.parse($json) } unless $failed.has_key?('Yajl')
 perf.add('JSON::Ext', 'parse') { JSON::Ext::Parser.new($json).parse } unless $failed.has_key?('JSON::Ext')
 perf.run($iter)

data/test/perf_strict.rb

@@ -15,6 +15,8 @@ $iter = 20000
 $with_bignum = false
 $with_nums = true
 $size = 0
+$symbolize = false
+$cache_keys = true
 
 opts = OptionParser.new
 opts.on("-v", "verbose")                                    { $verbose = true }
@@ -23,6 +25,8 @@ opts.on("-i", "--indent [Int]", Integer, "indentation")     { |i| $indent = i }
 opts.on("-s", "--size [Int]", Integer, "size (~Kbytes)")    { |i| $size = i }
 opts.on("-b", "with bignum")                                { $with_bignum = true }
 opts.on("-n", "without numbers")                            { $with_nums = false }
+opts.on("-z", "--symbolize", "symbolize keys")              { $symbolize = true }
+opts.on("-k", "--no-cache", "turn off key caching")         { $cache_keys = false }
 opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
 files = opts.parse(ARGV)
 
@@ -51,7 +55,7 @@ else
   }
 end
 
-Oj.default_options = { :indent => $indent, :mode => :strict }
+Oj.default_options = { :indent => $indent, :mode => :strict, cache_keys: $cache_keys, cache_str: 5 }
 
 if 0 < $size
   o = $obj
@@ -62,9 +66,6 @@ if 0 < $size
 end
 
 $json = Oj.dump($obj)
-$obj_json = Oj.dump($obj, :mode => :object)
-#puts "*** size: #{$obj_json.size}"
-#puts "*** #{$obj_json}"
 $failed = {} # key is same as String used in tests later
 
 def capture_error(tag, orig, load_key, dump_key, &blk)
@@ -77,8 +78,13 @@ def capture_error(tag, orig, load_key, dump_key, &blk)
 end
 
 # Verify that all packages dump and load correctly and return the same Object as the original.
-capture_error('Oj:strict', $obj, 'load', 'dump') { |o| Oj.strict_load(Oj.dump(o, :mode => :strict)) }
-capture_error('Yajl', $obj, 'encode', 'parse') { |o| require 'yajl'; Yajl::Parser.parse(Yajl::Encoder.encode(o)) }
+capture_error('Oj:strict', $obj, 'load', 'dump') { |o|
+  Oj.strict_load(Oj.dump(o))
+}
+capture_error('Yajl', $obj, 'encode', 'parse') { |o|
+  require 'yajl'
+  Yajl::Parser.parse(Yajl::Encoder.encode(o))
+}
 capture_error('JSON::Ext', $obj, 'generate', 'parse') { |o|
   require 'json'
   require 'json/ext'
@@ -86,16 +92,11 @@ capture_error('JSON::Ext', $obj, 'generate', 'parse') { |o|
   JSON.parser = JSON::Ext::Parser
   JSON.parse(JSON.generate(o))
 }
-capture_error('JSON::Pure', $obj, 'generate', 'parse') { |o|
-  require 'json/pure'
-  JSON.generator = JSON::Pure::Generator
-  JSON.parser = JSON::Pure::Parser
-  JSON.parse(JSON.generate(o))
-}
+
+Oj.default_options = { symbol_keys: $symbolize }
 
 if $verbose
   puts "json:\n#{$json}\n"
-  puts "object json:\n#{$obj_json}\n"
   puts "Oj loaded object:\n#{Oj.strict_load($json)}\n"
   puts "Yajl loaded object:\n#{Yajl::Parser.parse($json)}\n"
   puts "JSON loaded object:\n#{JSON::Ext::Parser.new($json).parse}\n"
@@ -105,15 +106,12 @@ puts '-' * 80
 puts "Strict Parse Performance"
 perf = Perf.new()
 unless $failed.has_key?('JSON::Ext')
-  perf.add('JSON::Ext', 'parse') { JSON.parse($json) }
+  perf.add('JSON::Ext', 'parse') { JSON.parse($json, symbolize_names: $symbolize) }
   perf.before('JSON::Ext') { JSON.parser = JSON::Ext::Parser }
 end
-unless $failed.has_key?('JSON::Pure')
-  perf.add('JSON::Pure', 'parse') { JSON.parse($json) }
-  perf.before('JSON::Pure') { JSON.parser = JSON::Pure::Parser }
-end
 unless $failed.has_key?('Oj:strict')
   perf.add('Oj:strict', 'strict_load') { Oj.strict_load($json) }
+  perf.add('Oj:wab', 'wab_load') { Oj.wab_load($json) }
 end
 perf.add('Yajl', 'parse') { Yajl::Parser.parse($json) } unless $failed.has_key?('Yajl')
 perf.run($iter)
@@ -125,12 +123,8 @@ unless $failed.has_key?('JSON::Ext')
   perf.add('JSON::Ext', 'dump') { JSON.generate($obj) }
   perf.before('JSON::Ext') { JSON.generator = JSON::Ext::Generator }
 end
-unless $failed.has_key?('JSON::Pure')
-  perf.add('JSON::Pure', 'generate') { JSON.generate($obj) }
-  perf.before('JSON::Pure') { JSON.generator = JSON::Pure::Generator }
-end
 unless $failed.has_key?('Oj:strict')
-  perf.add('Oj:strict', 'dump') { Oj.dump($obj, :mode => :strict) }
+  perf.add('Oj:strict', 'dump') { Oj.dump($obj) }
 end
 perf.add('Yajl', 'encode') { Yajl::Encoder.encode($obj) } unless $failed.has_key?('Yajl')
 perf.run($iter)