oj 3.13.2 → 3.13.6

data/ext/oj/saj2.c

@@ -158,7 +158,7 @@ static void add_str(ojParser p) {
     const char *   str = buf_str(&p->buf);
     size_t         len = buf_len(&p->buf);
 
-    if (d->cache_str <= len) {
+    if (d->cache_str < len) {
         rstr = cache_intern(d->str_cache, str, len);
     } else {
         rstr = rb_utf8_str_new(str, len);
@@ -172,7 +172,7 @@ static void add_str_key(ojParser p) {
     const char *   str = buf_str(&p->buf);
     size_t         len = buf_len(&p->buf);
 
-    if (d->cache_str <= len) {
+    if (d->cache_str < len) {
         rstr = cache_intern(d->str_cache, str, len);
     } else {
         rstr = rb_utf8_str_new(str, len);
@@ -336,7 +336,7 @@ void oj_set_parser_saj(ojParser p) {
     d->klen      = 256;
     d->keys      = ALLOC_N(VALUE, d->klen);
     d->tail      = d->keys;
-    d->str_cache = cache_create(0, form_str, true);
+    d->str_cache = cache_create(0, form_str, true, false);
 
     p->ctx = (void *)d;
     reset(p);

data/ext/oj/strict.c

@@ -16,7 +16,7 @@
 VALUE oj_cstr_to_value(const char *str, size_t len, size_t cache_str) {
     volatile VALUE rstr = Qnil;
 
-    if (len <= cache_str) {
+    if (len < cache_str) {
 	rstr = oj_str_intern(str, len);
     } else {
         rstr = rb_str_new(str, len);
@@ -37,8 +37,8 @@ VALUE oj_calc_hash_key(ParseInfo pi, Val parent) {
         } else {
             rkey = rb_str_new(parent->key, parent->klen);
             rkey = oj_encode(rkey);
+	    OBJ_FREEZE(rkey); // frozen when used as a Hash key anyway
         }
-        OBJ_FREEZE(rkey);
         return rkey;
     }
     if (Yes == pi->options.sym_key) {
@@ -46,7 +46,6 @@ VALUE oj_calc_hash_key(ParseInfo pi, Val parent) {
     } else {
 	rkey = oj_str_intern(parent->key, parent->klen);
     }
-    OBJ_FREEZE(rkey);
     return rkey;
 }
 

data/ext/oj/usual.c

@@ -39,7 +39,7 @@ typedef struct _col {
 typedef union _key {
     struct {
         int16_t len;
-        char    buf[22];
+        char    buf[30];
     };
     struct {
         int16_t xlen;  // should be the same as len
@@ -77,6 +77,7 @@ typedef struct _delegate {
     char *  create_id;
     uint8_t create_id_len;
     uint8_t cache_str;
+    uint8_t cache_xrate;
     uint8_t miss_class;
     bool    cache_keys;
     bool    ignore_json_create;
@@ -100,7 +101,6 @@ static VALUE form_str(const char *str, size_t len) {
 }
 
 static VALUE form_sym(const char *str, size_t len) {
-    // return ID2SYM(rb_intern3(str, len, oj_utf8_encoding));
     return rb_str_intern(rb_utf8_str_new(str, len));
 }
 
@@ -209,21 +209,21 @@ static void push(ojParser p, VALUE v) {
 static VALUE cache_key(ojParser p, Key kp) {
     Delegate d = (Delegate)p->ctx;
 
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return cache_intern(d->key_cache, kp->buf, kp->len);
     }
     return cache_intern(d->key_cache, kp->key, kp->len);
 }
 
 static VALUE str_key(ojParser p, Key kp) {
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return rb_str_freeze(rb_utf8_str_new(kp->buf, kp->len));
     }
     return rb_str_freeze(rb_utf8_str_new(kp->key, kp->len));
 }
 
 static VALUE sym_key(ojParser p, Key kp) {
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return rb_str_freeze(rb_str_intern(rb_utf8_str_new(kp->buf, kp->len)));
     }
     return rb_str_freeze(rb_str_intern(rb_utf8_str_new(kp->key, kp->len)));
@@ -232,7 +232,7 @@ static VALUE sym_key(ojParser p, Key kp) {
 static ID get_attr_id(ojParser p, Key kp) {
     Delegate d = (Delegate)p->ctx;
 
-    if ((size_t)kp->len < sizeof(kp->buf) - 1) {
+    if ((size_t)kp->len < sizeof(kp->buf)) {
         return (ID)cache_intern(d->attr_cache, kp->buf, kp->len);
     }
     return (ID)cache_intern(d->attr_cache, kp->key, kp->len);
@@ -253,7 +253,7 @@ static void push_key(ojParser p) {
         d->kend  = d->khead + cap;
     }
     d->ktail->len = klen;
-    if (klen <= sizeof(d->ktail->buf) + 1) {
+    if (klen < sizeof(d->ktail->buf)) {
         memcpy(d->ktail->buf, key, klen);
         d->ktail->buf[klen] = '\0';
     } else {
@@ -323,7 +323,7 @@ static void open_array_key(ojParser p) {
 }
 
 static void close_object(ojParser p) {
-    VALUE *vp;
+    VALUE *  vp;
     Delegate d = (Delegate)p->ctx;
 
     d->ctail--;
@@ -336,7 +336,7 @@ static void close_object(ojParser p) {
 #if HAVE_RB_HASH_BULK_INSERT
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         *vp = d->get_key(p, kp);
-        if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+        if (sizeof(kp->buf) <= (size_t)kp->len) {
             xfree(kp->key);
         }
     }
@@ -344,7 +344,7 @@ static void close_object(ojParser p) {
 #else
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         rb_hash_aset(obj, d->get_key(p, kp), *(vp + 1));
-        if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+        if (sizeof(kp->buf) <= (size_t)kp->len) {
             xfree(kp->key);
         }
     }
@@ -356,7 +356,7 @@ static void close_object(ojParser p) {
 }
 
 static void close_object_class(ojParser p) {
-    VALUE *vp;
+    VALUE *  vp;
     Delegate d = (Delegate)p->ctx;
 
     d->ctail--;
@@ -368,7 +368,7 @@ static void close_object_class(ojParser p) {
 
     for (vp = head; kp < d->ktail; kp++, vp += 2) {
         rb_funcall(obj, hset_id, 2, d->get_key(p, kp), *(vp + 1));
-        if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+        if (sizeof(kp->buf) <= (size_t)kp->len) {
             xfree(kp->key);
         }
     }
@@ -379,7 +379,7 @@ static void close_object_class(ojParser p) {
 }
 
 static void close_object_create(ojParser p) {
-    VALUE *vp;
+    VALUE *  vp;
     Delegate d = (Delegate)p->ctx;
 
     d->ctail--;
@@ -396,7 +396,7 @@ static void close_object_create(ojParser p) {
 #if HAVE_RB_HASH_BULK_INSERT
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 *vp = d->get_key(p, kp);
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -404,7 +404,7 @@ static void close_object_create(ojParser p) {
 #else
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_hash_aset(obj, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -413,7 +413,7 @@ static void close_object_create(ojParser p) {
             obj = rb_class_new_instance(0, NULL, d->hash_class);
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_funcall(obj, hset_id, 2, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -428,7 +428,7 @@ static void close_object_create(ojParser p) {
 #if HAVE_RB_HASH_BULK_INSERT
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 *vp = d->get_key(p, kp);
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -436,7 +436,7 @@ static void close_object_create(ojParser p) {
 #else
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_hash_aset(arg, d->get_key(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -446,7 +446,7 @@ static void close_object_create(ojParser p) {
             obj = rb_class_new_instance(0, NULL, clas);
             for (vp = head; kp < d->ktail; kp++, vp += 2) {
                 rb_ivar_set(obj, get_attr_id(p, kp), *(vp + 1));
-                if (sizeof(kp->buf) - 1 < (size_t)kp->len) {
+                if (sizeof(kp->buf) <= (size_t)kp->len) {
                     xfree(kp->key);
                 }
             }
@@ -471,7 +471,7 @@ static void close_array(ojParser p) {
 }
 
 static void close_array_class(ojParser p) {
-    VALUE *vp;
+    VALUE *  vp;
     Delegate d = (Delegate)p->ctx;
 
     d->ctail--;
@@ -686,10 +686,10 @@ static void mark(ojParser p) {
         return;
     }
     Delegate d = (Delegate)p->ctx;
-    VALUE *vp;
+    VALUE *  vp;
 
     if (NULL == d) {
-	return;
+        return;
     }
     cache_mark(d->str_cache);
     if (NULL != d->sym_cache) {
@@ -792,6 +792,30 @@ static VALUE opt_cache_strings_set(ojParser p, VALUE value) {
     return INT2NUM((int)d->cache_str);
 }
 
+static VALUE opt_cache_expunge(ojParser p, VALUE value) {
+    Delegate d = (Delegate)p->ctx;
+
+    return INT2NUM((int)d->cache_xrate);
+}
+
+static VALUE opt_cache_expunge_set(ojParser p, VALUE value) {
+    Delegate d    = (Delegate)p->ctx;
+    int      rate = NUM2INT(value);
+
+    if (rate < 0) {
+        rate = 0;
+    } else if (3 < rate) {
+        rate = 3;
+    }
+    d->cache_xrate = (uint8_t)rate;
+    cache_set_expunge_rate(d->str_cache, rate);
+    cache_set_expunge_rate(d->attr_cache, rate);
+    if (NULL != d->sym_cache) {
+        cache_set_expunge_rate(d->sym_cache, rate);
+    }
+    return INT2NUM((int)rate);
+}
+
 static VALUE opt_capacity(ojParser p, VALUE value) {
     Delegate d = (Delegate)p->ctx;
 
@@ -830,11 +854,7 @@ static VALUE opt_class_cache_set(ojParser p, VALUE value) {
 
     if (Qtrue == value) {
         if (NULL == d->class_cache) {
-            if (MISS_AUTO == d->miss_class) {
-                d->class_cache = cache_create(0, form_class_auto, true);
-            } else {
-                d->class_cache = cache_create(0, form_class, false);
-            }
+            d->class_cache = cache_create(0, form_class_auto, MISS_AUTO == d->miss_class, false);
         }
     } else if (NULL != d->class_cache) {
         cache_free(d->class_cache);
@@ -1078,7 +1098,8 @@ static VALUE opt_symbol_keys_set(ojParser p, VALUE value) {
     Delegate d = (Delegate)p->ctx;
 
     if (Qtrue == value) {
-        d->sym_cache = cache_create(0, form_sym, true);
+        d->sym_cache = cache_create(0, form_sym, true, false);
+        cache_set_expunge_rate(d->sym_cache, d->cache_xrate);
         d->key_cache = d->sym_cache;
         if (!d->cache_keys) {
             d->get_key = sym_key;
@@ -1097,13 +1118,15 @@ static VALUE opt_symbol_keys_set(ojParser p, VALUE value) {
 
 static VALUE option(ojParser p, const char *key, VALUE value) {
     struct opt *op;
-    struct opt opts[] = {
+    struct opt  opts[] = {
         {.name = "array_class", .func = opt_array_class},
         {.name = "array_class=", .func = opt_array_class_set},
         {.name = "cache_keys", .func = opt_cache_keys},
         {.name = "cache_keys=", .func = opt_cache_keys_set},
         {.name = "cache_strings", .func = opt_cache_strings},
         {.name = "cache_strings=", .func = opt_cache_strings_set},
+        {.name = "cache_expunge", .func = opt_cache_expunge},
+        {.name = "cache_expunge=", .func = opt_cache_expunge_set},
         {.name = "capacity", .func = opt_capacity},
         {.name = "capacity=", .func = opt_capacity_set},
         {.name = "class_cache", .func = opt_class_cache},
@@ -1163,6 +1186,7 @@ void oj_set_parser_usual(ojParser p) {
     d->create_id          = NULL;
     d->create_id_len      = 0;
     d->miss_class         = MISS_IGNORE;
+    d->cache_xrate        = 1;
 
     Funcs f         = &p->funcs[TOP_FUN];
     f->add_null     = add_null;
@@ -1203,8 +1227,8 @@ void oj_set_parser_usual(ojParser p) {
     f->open_object  = open_object_key;
     f->close_object = close_object;
 
-    d->str_cache   = cache_create(0, form_str, true);
-    d->attr_cache  = cache_create(0, form_attr, false);
+    d->str_cache   = cache_create(0, form_str, true, false);
+    d->attr_cache  = cache_create(0, form_attr, false, false);
     d->sym_cache   = NULL;
     d->class_cache = NULL;
     d->key_cache   = d->str_cache;

data/ext/oj/wab.c

@@ -305,11 +305,14 @@ static VALUE calc_hash_key(ParseInfo pi, Val parent) {
     if (Yes == pi->options.cache_keys) {
         rkey = oj_sym_intern(parent->key, parent->klen);
     } else {
-        rkey = rb_str_new(parent->key, parent->klen);
-        rkey = oj_encode(rkey);
+#if HAVE_RB_ENC_INTERNED_STR
+        rkey = rb_enc_interned_str(parent->key, parent->klen, oj_utf8_encoding);
+#else
+	rkey = rb_utf8_str_new(parent->key, parent->klen);
         rkey = rb_str_intern(rkey);
+        OBJ_FREEZE(rkey);
+#endif
     }
-    OBJ_FREEZE(rkey);
     return rkey;
 }
 

data/lib/oj/state.rb

@@ -1,7 +1,7 @@
 
 module JSON
   module Ext
-    module Generator 
+    module Generator
       unless defined?(::JSON::Ext::Generator::State)
         # This class exists for json gem compatibility only. While it can be
         # used as the options for other than compatibility a simple Hash is
@@ -44,11 +44,11 @@ module JSON
           def to_h()
             return @attrs.dup
           end
-          
+
           def to_hash()
             return @attrs.dup
           end
-          
+
           def allow_nan?()
             @attrs[:allow_nan]
           end
@@ -104,7 +104,7 @@ module JSON
           def has_key?(k)
             @attrs.has_key?(key.to_sym)
           end
-          
+
           # Handles requests for Hash values. Others cause an Exception to be raised.
           # @param [Symbol|String] m method symbol
           # @return [Boolean] the value of the specified instance variable.
@@ -116,11 +116,12 @@ module JSON
               m = m.to_s[0..-2]
               m = m.to_sym
               return @attrs.store(m, args[0])
-            else
+	    end
+	    if @attrs.has_key?(m.to_sym)
               raise ArgumentError.new("wrong number of arguments (#{args.size} for 0 with #{m}) to method #{m}") unless args.nil? or args.empty?
               return @attrs[m.to_sym]
-            end
-            raise NoMethodError.new("undefined method #{m}", m)
+	    end
+	    return @attrs.send(m, *args, &block)
           end
 
         end # State

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module.
-  VERSION = '3.13.2'
+  VERSION = '3.13.6'
 end

data/pages/Options.md

@@ -75,12 +75,22 @@ parse option to match the JSON gem. In that case either `Float`,
 If true Hash keys are cached or interned. There are trade-offs with
 caching keys. Large caches will use more memory and in extreme cases
 (like over a million) the cache may be slower than not using
-it. Repeated parsing of similar JSON docs is where cache_keys shines.
+it. Repeated parsing of similar JSON docs is where cache_keys shines
+especially with symbol keys.
+
+There is a maximum length for cached keys. Any key longer than 34
+bytes is not cached. Everything still works but the key is not cached.
 
 ### :cache_strings [Int]
 
 Shorter strings can be cached for better performance. A limit,
-cache_strings, defines the upper limit on what strings are cached.
+cache_strings, defines the upper limit on what strings are cached. As
+with cached keys only strings less than 35 bytes are cached even if
+the limit is set higher. Setting the limit to zero effectively
+disables the caching of string values.
+
+Note that caching for strings is for string values and not Hash keys
+or Object attributes.
 
 ### :circular [Boolean]
 

data/test/bar.rb

@@ -1,35 +1,16 @@
 #!/usr/bin/env ruby
 
-$: << File.dirname(__FILE__)
-$oj_dir = File.dirname(File.expand_path(File.dirname(__FILE__)))
-%w(lib ext).each do |dir|
-  $: << File.join($oj_dir, dir)
-end
-
-require 'active_support'
-require "active_support/json"
-
-$s = "\u2014 & \n \u{1F618}"
-
-=begin
-def check(label)
-  puts "\n--- #{label} --------------------"
-
-  ActiveSupport::JSON::Encoding.use_standard_json_time_format = true
-  puts "with standard_json == true: t.to_json - #{$t.to_json}"
-  ActiveSupport::JSON::Encoding.use_standard_json_time_format = false
-  puts "with standard_json == false: t.to_json - #{$t.to_json}"
-end
-
-check('Before Oj')
-=end
+$: << '.'
+$: << File.join(File.dirname(__FILE__), "../lib")
+$: << File.join(File.dirname(__FILE__), "../ext")
 
 require 'oj'
 
-ActiveSupport::JSON::Encoding.escape_html_entities_in_json = false
-puts "ActiveSupport.encode(s) - #{ActiveSupport::JSON.encode($s)}"
+json = %|[{"x12345678901234567890": true}]|
 
-Oj.optimize_rails
-Oj.default_options = { mode: :rails }
+p = Oj::Parser.new(:usual)
+p.cache_keys = false
+p.symbol_keys = true
+x = p.parse(json)
 
-puts "Oj.dump(s) - #{Oj.dump($s)}"
+pp x