oj 3.13.3 → 3.13.7

data/ext/oj/oj.c

@@ -107,6 +107,7 @@ static VALUE bigdecimal_load_sym;
 static VALUE bigdecimal_sym;
 static VALUE cache_keys_sym;
 static VALUE cache_str_sym;
+static VALUE cache_string_sym;
 static VALUE circular_sym;
 static VALUE class_cache_sym;
 static VALUE compat_bigdecimal_sym;
@@ -287,7 +288,7 @@ struct _options oj_default_options = {
  * - *:ignore* [_nil_|_Array_] either nil or an Array of classes to ignore when dumping
  * - *:ignore_under* [_Boolean_] if true then attributes that start with _ are ignored when dumping in
  *object or custom mode.
- * - *:cache_keys* [_Boolean_] if true then hash keys are cached
+ * - *:cache_keys* [_Boolean_] if true then hash keys are cached if less than 35 bytes.
  * - *:cache_str* [_Fixnum_] maximum string value length to cache (strings less than this are cached)
  * - *:integer_range* [_Range_] Dump integers outside range as strings.
  * - *:trace* [_true,_|_false_] Trace all load and dump calls, default is false (trace is off)
@@ -692,7 +693,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts)
             sprintf(copts->float_fmt, "%%0.%dg", n);
             copts->float_prec = n;
         }
-    } else if (cache_str_sym == k) {
+    } else if (cache_str_sym == k || cache_string_sym == k) {
         int n;
 
 #ifdef RUBY_INTEGER_UNIFICATION
@@ -1920,6 +1921,8 @@ void Init_oj() {
     rb_gc_register_address(&cache_keys_sym);
     cache_str_sym = ID2SYM(rb_intern("cache_str"));
     rb_gc_register_address(&cache_str_sym);
+    cache_string_sym = ID2SYM(rb_intern("cache_string"));
+    rb_gc_register_address(&cache_string_sym);
     circular_sym = ID2SYM(rb_intern("circular"));
     rb_gc_register_address(&circular_sym);
     class_cache_sym = ID2SYM(rb_intern("class_cache"));

data/ext/oj/parser.c

@@ -610,6 +610,9 @@ static void parse(ojParser p, const byte *json) {
     printf("*** parse - mode: %c %s\n", p->map[256], (const char *)json);
 #endif
     for (; '\0' != *b; b++) {
+#if DEBUG
+        printf("*** parse - mode: %c %02x %s => %c\n", p->map[256], *b, b, p->map[*b]);
+#endif
         switch (p->map[*b]) {
         case SKIP_NEWLINE:
             p->line++;
@@ -887,13 +890,17 @@ static void parse(ojParser p, const byte *json) {
                 buf_append_string(&p->buf, (const char *)start, b - start);
             }
             if ('"' == *b) {
+                p->funcs[p->stack[p->depth]].add_str(p);
                 p->map = p->next_map;
                 break;
             }
             b--;
             break;
         case STR_SLASH: p->map = esc_map; break;
-        case STR_QUOTE: p->map = p->next_map; break;
+        case STR_QUOTE:
+            p->funcs[p->stack[p->depth]].add_str(p);
+            p->map = p->next_map;
+            break;
         case ESC_U:
             p->map   = u_map;
             p->ri    = 0;
@@ -1135,13 +1142,42 @@ extern void oj_set_parser_saj(ojParser p);
 extern void oj_set_parser_usual(ojParser p);
 extern void oj_set_parser_debug(ojParser p);
 
+static int opt_cb(VALUE rkey, VALUE value, VALUE ptr) {
+    ojParser    p   = (ojParser)ptr;
+    const char *key = NULL;
+    char        set_key[64];
+    long        klen;
+
+    switch (rb_type(rkey)) {
+    case RUBY_T_SYMBOL:
+        rkey = rb_sym2str(rkey);
+        // fall through
+    case RUBY_T_STRING:
+        key  = rb_string_value_ptr(&rkey);
+        klen = RSTRING_LEN(rkey);
+        break;
+    default: rb_raise(rb_eArgError, "option keys must be a symbol or string");
+    }
+    if ((long)sizeof(set_key) - 1 <= klen) {
+        return ST_CONTINUE;
+    }
+    memcpy(set_key, key, klen);
+    set_key[klen]     = '=';
+    set_key[klen + 1] = '\0';
+    p->option(p, set_key, value);
+
+    return ST_CONTINUE;
+}
+
 /* Document-method: new
  * call-seq: new(mode=nil)
  *
  * Creates a new Parser with the specified mode. If no mode is provided
- * validation is assumed.
+ * validation is assumed. Optional arguments can be provided that match the
+ * mode. For example with the :usual mode the call might look like
+ * Oj::Parser.new(:usual, cache_keys: true).
  */
-static VALUE parser_new(VALUE self, VALUE mode) {
+static VALUE parser_new(int argc, VALUE *argv, VALUE self) {
     ojParser p = ALLOC(struct _ojParser);
 
 #if HAVE_RB_EXT_RACTOR_SAFE
@@ -1151,33 +1187,45 @@ static VALUE parser_new(VALUE self, VALUE mode) {
     memset(p, 0, sizeof(struct _ojParser));
     buf_init(&p->key);
     buf_init(&p->buf);
-
     p->map = value_map;
-    if (Qnil == mode) {
-        oj_set_parser_validator(p);
+
+    if (argc < 1) {
+	oj_set_parser_validator(p);
     } else {
-        const char *ms = NULL;
-
-        switch (rb_type(mode)) {
-        case RUBY_T_SYMBOL:
-            mode = rb_sym2str(mode);
-            // fall through
-        case RUBY_T_STRING: ms = RSTRING_PTR(mode); break;
-        default: rb_raise(rb_eArgError, "mode must be :validate, :usual, :saj, or :object");
-        }
-        if (0 == strcmp("usual", ms) || 0 == strcmp("standard", ms) || 0 == strcmp("strict", ms) ||
-            0 == strcmp("compat", ms)) {
-            oj_set_parser_usual(p);
-        } else if (0 == strcmp("object", ms)) {
-            // TBD
-        } else if (0 == strcmp("saj", ms)) {
-            oj_set_parser_saj(p);
-        } else if (0 == strcmp("validate", ms)) {
+        VALUE mode = argv[0];
+
+        if (Qnil == mode) {
             oj_set_parser_validator(p);
-        } else if (0 == strcmp("debug", ms)) {
-            oj_set_parser_debug(p);
         } else {
-            rb_raise(rb_eArgError, "mode must be :validate, :usual, :saj, or :object");
+            const char *ms = NULL;
+
+            switch (rb_type(mode)) {
+            case RUBY_T_SYMBOL:
+                mode = rb_sym2str(mode);
+                // fall through
+            case RUBY_T_STRING: ms = RSTRING_PTR(mode); break;
+            default: rb_raise(rb_eArgError, "mode must be :validate, :usual, :saj, or :object");
+            }
+            if (0 == strcmp("usual", ms) || 0 == strcmp("standard", ms) || 0 == strcmp("strict", ms) ||
+                0 == strcmp("compat", ms)) {
+                oj_set_parser_usual(p);
+            } else if (0 == strcmp("object", ms)) {
+                // TBD
+            } else if (0 == strcmp("saj", ms)) {
+                oj_set_parser_saj(p);
+            } else if (0 == strcmp("validate", ms)) {
+                oj_set_parser_validator(p);
+            } else if (0 == strcmp("debug", ms)) {
+                oj_set_parser_debug(p);
+            } else {
+                rb_raise(rb_eArgError, "mode must be :validate, :usual, :saj, or :object");
+            }
+        }
+        if (1 < argc) {
+            VALUE ropts = argv[1];
+
+            Check_Type(ropts, T_HASH);
+            rb_hash_foreach(ropts, opt_cb, (VALUE)p);
         }
     }
     return Data_Wrap_Struct(parser_class, parser_mark, parser_free, p);
@@ -1204,7 +1252,7 @@ static VALUE parser_new(VALUE self, VALUE mode) {
  * - *:usual*
  *   - _cache_keys=_ sets the value of the _cache_keys_ flag.
  *   - _cache_keys_ returns the value of the _cache_keys_ flag.
- *   - _cache_strings=_ sets the value of the _cache_strings_ to an positive integer less than 35. Strings shorter than
+ *   - _cache_strings=_ sets the value of the _cache_strings_ to a positive integer less than 35. Strings shorter than
  * that length are cached.
  *   - _cache_strings_ returns the value of the _cache_strings_ integer value.
  *   - _cache_expunge=_ sets the value of the _cache_expunge_ where 0 never expunges, 1 expunges slowly, 2 expunges
@@ -1469,7 +1517,7 @@ static VALUE parser_validate(VALUE self) {
  */
 void oj_parser_init() {
     parser_class = rb_define_class_under(Oj, "Parser", rb_cObject);
-    rb_define_module_function(parser_class, "new", parser_new, 1);
+    rb_define_module_function(parser_class, "new", parser_new, -1);
     rb_define_method(parser_class, "parse", parser_parse, 1);
     rb_define_method(parser_class, "load", parser_load, 1);
     rb_define_method(parser_class, "file", parser_file, 1);

data/ext/oj/strict.c

@@ -16,7 +16,7 @@
 VALUE oj_cstr_to_value(const char *str, size_t len, size_t cache_str) {
     volatile VALUE rstr = Qnil;
 
-    if (len <= cache_str) {
+    if (len < cache_str) {
 	rstr = oj_str_intern(str, len);
     } else {
         rstr = rb_str_new(str, len);
@@ -37,7 +37,7 @@ VALUE oj_calc_hash_key(ParseInfo pi, Val parent) {
         } else {
             rkey = rb_str_new(parent->key, parent->klen);
             rkey = oj_encode(rkey);
-            OBJ_FREEZE(rkey);
+	    OBJ_FREEZE(rkey); // frozen when used as a Hash key anyway
         }
         return rkey;
     }

data/ext/oj/usual.c

@@ -39,7 +39,7 @@ typedef struct _col {
 typedef union _key {
     struct {
         int16_t len;
-        char    buf[22];
+        char    buf[30];
     };
     struct {
         int16_t xlen;  // should be the same as len
@@ -209,21 +209,21 @@ static void push(ojParser p, VALUE v) {
 static VALUE cache_key(ojParser p, Key kp) {
     Delegate d = (Delegate)p->ctx;
 
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return cache_intern(d->key_cache, kp->buf, kp->len);
     }
     return cache_intern(d->key_cache, kp->key, kp->len);
 }
 
 static VALUE str_key(ojParser p, Key kp) {
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return rb_str_freeze(rb_utf8_str_new(kp->buf, kp->len));
     }
     return rb_str_freeze(rb_utf8_str_new(kp->key, kp->len));
 }
 
 static VALUE sym_key(ojParser p, Key kp) {
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return rb_str_freeze(rb_str_intern(rb_utf8_str_new(kp->buf, kp->len)));
     }
     return rb_str_freeze(rb_str_intern(rb_utf8_str_new(kp->key, kp->len)));
@@ -232,7 +232,7 @@ static VALUE sym_key(ojParser p, Key kp) {
 static ID get_attr_id(ojParser p, Key kp) {
     Delegate d = (Delegate)p->ctx;
 
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return (ID)cache_intern(d->attr_cache, kp->buf, kp->len);
     }
     return (ID)cache_intern(d->attr_cache, kp->key, kp->len);
@@ -253,7 +253,7 @@ static void push_key(ojParser p) {
         d->kend  = d->khead + cap;
     }
     d->ktail->len = klen;
-    if (klen <= sizeof(d->ktail->buf) + 1) {
+    if (klen < sizeof(d->ktail->buf)) {
         memcpy(d->ktail->buf, key, klen);
         d->ktail->buf[klen] = '\0';
     } else {
@@ -336,7 +336,7 @@ static void close_object(ojParser p) {
 #if HAVE_RB_HASH_BULK_INSERT
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         *vp = d->get_key(p, kp);
-        if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+        if (sizeof(kp->buf) <= (size_t)kp->len) {
             xfree(kp->key);
         }
     }
@@ -344,7 +344,7 @@ static void close_object(ojParser p) {
 #else
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         rb_hash_aset(obj, d->get_key(p, kp), *(vp + 1));
-        if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+        if (sizeof(kp->buf) <= (size_t)kp->len) {
             xfree(kp->key);
         }
     }
@@ -368,7 +368,7 @@ static void close_object_class(ojParser p) {
 
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         rb_funcall(obj, hset_id, 2, d->get_key(p, kp), *(vp + 1));
-        if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+        if (sizeof(kp->buf) <= (size_t)kp->len) {
             xfree(kp->key);
         }
     }
@@ -396,7 +396,7 @@ static void close_object_create(ojParser p) {
 #if HAVE_RB_HASH_BULK_INSERT
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 *vp = d->get_key(p, kp);
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -404,7 +404,7 @@ static void close_object_create(ojParser p) {
 #else
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_hash_aset(obj, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -413,7 +413,7 @@ static void close_object_create(ojParser p) {
             obj = rb_class_new_instance(0, NULL, d->hash_class);
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_funcall(obj, hset_id, 2, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -428,7 +428,7 @@ static void close_object_create(ojParser p) {
 #if HAVE_RB_HASH_BULK_INSERT
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 *vp = d->get_key(p, kp);
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -436,7 +436,7 @@ static void close_object_create(ojParser p) {
 #else
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_hash_aset(arg, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -446,7 +446,7 @@ static void close_object_create(ojParser p) {
             obj = rb_class_new_instance(0, NULL, clas);
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_ivar_set(obj, get_attr_id(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module.
-  VERSION = '3.13.3'
+  VERSION = '3.13.7'
 end

data/pages/Options.md

@@ -75,12 +75,22 @@ parse option to match the JSON gem. In that case either `Float`,
 If true Hash keys are cached or interned. There are trade-offs with
 caching keys. Large caches will use more memory and in extreme cases
 (like over a million) the cache may be slower than not using
-it. Repeated parsing of similar JSON docs is where cache_keys shines.
+it. Repeated parsing of similar JSON docs is where cache_keys shines
+especially with symbol keys.
+
+There is a maximum length for cached keys. Any key longer than 34
+bytes is not cached. Everything still works but the key is not cached.
 
 ### :cache_strings [Int]
 
 Shorter strings can be cached for better performance. A limit,
-cache_strings, defines the upper limit on what strings are cached.
+cache_strings, defines the upper limit on what strings are cached. As
+with cached keys only strings less than 35 bytes are cached even if
+the limit is set higher. Setting the limit to zero effectively
+disables the caching of string values.
+
+Note that caching for strings is for string values and not Hash keys
+or Object attributes.
 
 ### :circular [Boolean]
 

data/test/bar.rb

@@ -1,35 +1,16 @@
 #!/usr/bin/env ruby
 
-$: << File.dirname(__FILE__)
-$oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
-%w(lib ext).each do |dir|
-  $: << File.join($oj_dir, dir)
-end
-
-require 'active_support'
-require "active_support/json"
-
-$s = "\u2014 & \n \u{1F618}"
-
-=begin
-def check(label)
-  puts "\n--- #{label} --------------------"
-
-  ActiveSupport::JSON::Encoding.use_standard_json_time_format = true
-  puts "with standard_json == true: t.to_json - #{$t.to_json}"
-  ActiveSupport::JSON::Encoding.use_standard_json_time_format = false
-  puts "with standard_json == false: t.to_json - #{$t.to_json}"
-end
-
-check('Before Oj')
-=end
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
 
 require 'oj'
 
-ActiveSupport::JSON::Encoding.escape_html_entities_in_json = false
-puts "ActiveSupport.encode(s) - #{ActiveSupport::JSON.encode($s)}"
+json = %|[{"x12345678901234567890": true}]|
 
-Oj.optimize_rails
-Oj.default_options = { mode: :rails }
+p = Oj::Parser.new(:usual)
+p.cache_keys = false
+p.symbol_keys = true
+x = p.parse(json)
 
-puts "Oj.dump(s) - #{Oj.dump($s)}"
+pp x

data/test/foo.rb

@@ -1,13 +1,13 @@
 #!/usr/bin/env ruby
 
-$: << File.dirname(__FILE__)
-$oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
-%w(lib ext).each do |dir|
-  $: << File.join($oj_dir, dir)
-end
-
 require 'oj'
 
-p = Oj::Parser.new(:debug)
+Oj::default_options = {cache_str: 0, cache_keys: true, mode: :strict}
+
+puts "Ruby version: #{RUBY_VERSION}"
+puts "Oj version: #{Oj::VERSION}"
+
+puts "cache_keys: #{Oj::default_options[:cache_keys]}"
+puts "cache_str: #{Oj::default_options[:cache_str]}"
 
-p.parse("[true, false]")
+Oj.load('{"":""}').each_pair {|k,v| puts "k.frozen?: #{k.frozen?}\nv.frozen?: #{v.frozen?}"}

data/test/test_parser_usual.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-# encoding: UTF-8
+# encoding: utf-8
 
 $: << File.dirname(__FILE__)
 
@@ -73,9 +73,14 @@ class UsualTest < Minitest::Test
     assert_equal({a: true, b: false}, doc)
   end
 
-  def test_capacity
+  def test_strings
     p = Oj::Parser.new(:usual)
-    p.capacity = 1000
+    doc = p.parse('{"ぴ": "", "ぴ ": "x", "c": "ぴーたー", "d": " ぴーたー "}')
+    assert_equal({'ぴ' => '', 'ぴ ' => 'x', 'c' => 'ぴーたー', 'd' => ' ぴーたー '}, doc)
+  end
+
+  def test_capacity
+    p = Oj::Parser.new(:usual, capacity: 1000)
     assert_equal(4096, p.capacity)
     p.capacity = 5000
     assert_equal(5000, p.capacity)
@@ -181,8 +186,7 @@ class UsualTest < Minitest::Test
   end
 
   def test_missing_class
-    p = Oj::Parser.new(:usual)
-    p.create_id = '^'
+    p = Oj::Parser.new(:usual, create_id: '^')
     json = '{"a":true,"^":"Auto","b":false}'
     doc = p.parse(json)
     assert_equal(Hash, doc.class)