oj 3.12.3 → 3.13.3

data/ext/oj/extconf.rb

@@ -30,6 +30,11 @@ have_func('rb_ivar_foreach')
 have_func('rb_gc_mark_movable')
 have_func('stpcpy')
 have_func('pthread_mutex_init')
+have_func('rb_enc_associate')
+have_func('rb_enc_interned_str')
+have_func('rb_ext_ractor_safe', 'ruby.h')
+# rb_hash_bulk_insert is deep down in a header not included in normal build and that seems to fool have_func.
+have_func('rb_hash_bulk_insert', 'ruby.h') unless '2' == version[0] && '6' == version[1]
 
 dflags['OJ_DEBUG'] = true unless ENV['OJ_DEBUG'].nil?
 

data/ext/oj/fast.c

@@ -200,9 +200,7 @@ static VALUE leaf_value(Doc doc, Leaf leaf) {
             break;
         case T_ARRAY: return leaf_array_value(doc, leaf); break;
         case T_HASH: return leaf_hash_value(doc, leaf); break;
-        default:
-            rb_raise(rb_const_get_at(Oj, rb_intern("Error")), "Unexpected type %02x.", leaf->rtype);
-            break;
+        default: rb_raise(rb_const_get_at(Oj, rb_intern("Error")), "Unexpected type %02x.", leaf->rtype); break;
         }
     }
     return leaf->value;
@@ -773,7 +771,7 @@ static VALUE parse_json(VALUE clas, char *json, bool given, bool allocated) {
     pi.doc = doc;
 #if IS_WINDOWS
     // assume a 1M stack and give half to ruby
-    pi.stack_min = (void *)((char *)&pi - (512 * 1024));
+    pi.stack_min = (void*)((char*)&pi - (512 * 1024));
 #else
     {
         struct rlimit lim;
@@ -825,9 +823,7 @@ static Leaf get_doc_leaf(Doc doc, const char *path) {
             size_t cnt = doc->where - doc->where_path;
 
             if (MAX_STACK <= cnt) {
-                rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")),
-                         "Path too deep. Limit is %d levels.",
-                         MAX_STACK);
+                rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
             }
             memcpy(stack, doc->where_path, sizeof(Leaf) * (cnt + 1));
             lp = stack + cnt;
@@ -868,9 +864,7 @@ static Leaf get_leaf(Leaf *stack, Leaf *lp, const char *path) {
     Leaf leaf = *lp;
 
     if (MAX_STACK <= lp - stack) {
-        rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")),
-                 "Path too deep. Limit is %d levels.",
-                 MAX_STACK);
+        rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
     }
     if ('\0' != *path) {
         if ('.' == *path && '.' == *(path + 1)) {
@@ -946,9 +940,7 @@ static void each_leaf(Doc doc, VALUE self) {
 
             doc->where++;
             if (MAX_STACK <= doc->where - doc->where_path) {
-                rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")),
-                         "Path too deep. Limit is %d levels.",
-                         MAX_STACK);
+                rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
             }
             do {
                 *doc->where = e;
@@ -964,9 +956,7 @@ static void each_leaf(Doc doc, VALUE self) {
 
 static int move_step(Doc doc, const char *path, int loc) {
     if (MAX_STACK <= doc->where - doc->where_path) {
-        rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")),
-                 "Path too deep. Limit is %d levels.",
-                 MAX_STACK);
+        rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
     }
     if ('\0' == *path) {
         loc = 0;
@@ -1264,7 +1254,6 @@ static VALUE doc_path(VALUE self) {
     return doc_where(self);
 }
 
-
 /* @overload local_key() => String, Fixnum, nil
  *
  * Returns the final key to the current location.
@@ -1483,7 +1472,7 @@ static VALUE doc_move(VALUE self, VALUE str) {
  * to the block on yield is the Doc instance after moving to the child
  * location.
  *   @param [String] path if provided it identified the top of the branch to
- * process the chilren of
+ * process the children of
  * @yieldparam [Doc] Doc at the child location
  * @example
  *   Oj::Doc.open('[3,[2,1]]') { |doc|

data/ext/oj/intern.c

@@ -0,0 +1,281 @@
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#include "intern.h"
+
+#include <stdint.h>
+
+#if HAVE_PTHREAD_MUTEX_INIT
+#include <pthread.h>
+#endif
+#include "cache.h"
+#include "parse.h"
+
+// Only used for the class cache so 256 should be sufficient.
+#define HASH_SLOT_CNT ((uint64_t)256)
+#define HASH_MASK (HASH_SLOT_CNT - 1)
+
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
+
+typedef struct _keyVal {
+    struct _keyVal *next;
+    const char *    key;
+    size_t          len;
+    VALUE           val;
+} * KeyVal;
+
+typedef struct _hash {
+    struct _keyVal slots[HASH_SLOT_CNT];
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_t mutex;
+#else
+    VALUE mutex;
+#endif
+} * Hash;
+
+struct _hash class_hash;
+struct _hash attr_hash;
+
+static struct _cache *str_cache = NULL;
+static VALUE          str_cache_obj;
+
+static struct _cache *sym_cache = NULL;
+static VALUE          sym_cache_obj;
+
+static struct _cache *attr_cache = NULL;
+static VALUE          attr_cache_obj;
+
+static VALUE form_str(const char *str, size_t len) {
+    return rb_str_freeze(rb_utf8_str_new(str, len));
+}
+
+static VALUE form_sym(const char *str, size_t len) {
+    return rb_str_intern(rb_utf8_str_new(str, len));
+}
+
+static VALUE form_attr(const char *str, size_t len) {
+    char buf[256];
+
+    if (sizeof(buf) - 2 <= len) {
+        char *b = ALLOC_N(char, len + 2);
+        ID    id;
+
+        if ('~' == *str) {
+	    memcpy(b, str + 1, len - 1);
+	    b[len - 1] = '\0';
+	    len -= 2;
+	} else {
+	    *b = '@';
+	    memcpy(b + 1, str, len);
+	    b[len + 1] = '\0';
+	}
+        id = rb_intern3(buf, len + 1, oj_utf8_encoding);
+        xfree(b);
+        return id;
+    }
+    if ('~' == *str) {
+	memcpy(buf, str + 1, len - 1);
+	buf[len - 1] = '\0';
+	len -= 2;
+    } else {
+	*buf = '@';
+	memcpy(buf + 1, str, len);
+	buf[len + 1] = '\0';
+    }
+    return (VALUE)rb_intern3(buf, len + 1, oj_utf8_encoding);
+}
+
+void oj_hash_init() {
+    VALUE cache_class = rb_define_class_under(Oj, "Cache", rb_cObject);
+
+    str_cache     = cache_create(0, form_str, true, true);
+    str_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, str_cache);
+    rb_gc_register_address(&str_cache_obj);
+
+    sym_cache     = cache_create(0, form_sym, true, true);
+    sym_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, sym_cache);
+    rb_gc_register_address(&sym_cache_obj);
+
+    attr_cache     = cache_create(0, form_attr, false, true);
+    attr_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, attr_cache);
+    rb_gc_register_address(&attr_cache_obj);
+
+    memset(class_hash.slots, 0, sizeof(class_hash.slots));
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_init(&class_hash.mutex, NULL);
+#else
+    class_hash.mutex = rb_mutex_new();
+    rb_gc_register_address(&class_hash.mutex);
+#endif
+}
+
+VALUE
+oj_str_intern(const char *key, size_t len) {
+    return cache_intern(str_cache, key, len);
+}
+
+VALUE
+oj_sym_intern(const char *key, size_t len) {
+    return cache_intern(sym_cache, key, len);
+}
+
+ID
+oj_attr_intern(const char *key, size_t len) {
+    return cache_intern(attr_cache, key, len);
+}
+
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
+
+    while (key < endless) {
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
+
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
+    }
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
+
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
+    }
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
+}
+
+static VALUE resolve_classname(VALUE mod, const char *classname, int auto_define) {
+    VALUE clas;
+    ID    ci = rb_intern(classname);
+
+    if (rb_const_defined_at(mod, ci)) {
+        clas = rb_const_get_at(mod, ci);
+    } else if (auto_define) {
+        clas = rb_define_class_under(mod, classname, oj_bag_class);
+    } else {
+        clas = Qundef;
+    }
+    return clas;
+}
+
+static VALUE resolve_classpath(ParseInfo pi, const char *name, size_t len, int auto_define, VALUE error_class) {
+    char        class_name[1024];
+    VALUE       clas;
+    char *      end = class_name + sizeof(class_name) - 1;
+    char *      s;
+    const char *n = name;
+
+    clas = rb_cObject;
+    for (s = class_name; 0 < len; n++, len--) {
+        if (':' == *n) {
+            *s = '\0';
+            n++;
+            len--;
+            if (':' != *n) {
+                return Qundef;
+            }
+            if (Qundef == (clas = resolve_classname(clas, class_name, auto_define))) {
+                return Qundef;
+            }
+            s = class_name;
+        } else if (end <= s) {
+            return Qundef;
+        } else {
+            *s++ = *n;
+        }
+    }
+    *s = '\0';
+    if (Qundef == (clas = resolve_classname(clas, class_name, auto_define))) {
+        oj_set_error_at(pi, error_class, __FILE__, __LINE__, "class %s is not defined", name);
+        if (Qnil != error_class) {
+            pi->err_class = error_class;
+        }
+    }
+    return clas;
+}
+
+VALUE oj_class_intern(const char *key, size_t len, bool safe, ParseInfo pi, int auto_define, VALUE error_class) {
+    uint64_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
+    KeyVal   bucket = class_hash.slots + h;
+    KeyVal   b;
+
+    if (safe) {
+#if HAVE_PTHREAD_MUTEX_INIT
+        pthread_mutex_lock(&class_hash.mutex);
+#else
+        rb_mutex_lock(class_hash.mutex);
+#endif
+        if (NULL != bucket->key) {  // not the top slot
+            for (b = bucket; 0 != b; b = b->next) {
+                if (len == b->len && 0 == strncmp(b->key, key, len)) {
+#if HAVE_PTHREAD_MUTEX_INIT
+                    pthread_mutex_unlock(&class_hash.mutex);
+#else
+                    rb_mutex_unlock(class_hash.mutex);
+#endif
+                    return b->val;
+                }
+                bucket = b;
+            }
+            b            = ALLOC(struct _keyVal);
+            b->next      = NULL;
+            bucket->next = b;
+            bucket       = b;
+        }
+        bucket->key = oj_strndup(key, len);
+        bucket->len = len;
+        bucket->val = resolve_classpath(pi, key, len, auto_define, error_class);
+#if HAVE_PTHREAD_MUTEX_INIT
+        pthread_mutex_unlock(&class_hash.mutex);
+#else
+        rb_mutex_unlock(class_hash.mutex);
+#endif
+    } else {
+        if (NULL != bucket->key) {
+            for (b = bucket; 0 != b; b = b->next) {
+                if (len == b->len && 0 == strncmp(b->key, key, len)) {
+                    return (ID)b->val;
+                }
+                bucket = b;
+            }
+            b            = ALLOC(struct _keyVal);
+            b->next      = NULL;
+            bucket->next = b;
+            bucket       = b;
+        }
+        bucket->key = oj_strndup(key, len);
+        bucket->len = len;
+        bucket->val = resolve_classpath(pi, key, len, auto_define, error_class);
+    }
+    return bucket->val;
+}
+
+char *oj_strndup(const char *s, size_t len) {
+    char *d = ALLOC_N(char, len + 1);
+
+    memcpy(d, s, len);
+    d[len] = '\0';
+
+    return d;
+}
+
+void intern_cleanup() {
+    cache_free(str_cache);
+    cache_free(sym_cache);
+    cache_free(attr_cache);
+}

data/ext/oj/intern.h

@@ -0,0 +1,26 @@
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#ifndef OJ_INTERN_H
+#define OJ_INTERN_H
+
+#include <stdbool.h>
+#include <ruby.h>
+
+struct _parseInfo;
+
+extern void oj_hash_init();
+
+extern VALUE oj_str_intern(const char *key, size_t len);
+extern VALUE oj_sym_intern(const char *key, size_t len);
+extern ID    oj_attr_intern(const char *key, size_t len);
+extern VALUE oj_class_intern(const char *       key,
+                             size_t             len,
+                             bool               safe,
+                             struct _parseInfo *pi,
+                             int                auto_define,
+                             VALUE              error_class);
+
+extern char *oj_strndup(const char *s, size_t len);
+
+#endif /* OJ_INTERN_H */

data/ext/oj/mimic_json.c

@@ -682,7 +682,7 @@ static VALUE mimic_set_create_id(VALUE self, VALUE id) {
  */
 static VALUE mimic_create_id(VALUE self) {
     if (NULL != oj_default_options.create_id) {
-        return oj_encode(rb_str_new_cstr(oj_default_options.create_id));
+        return  rb_utf8_str_new(oj_default_options.create_id, oj_default_options.create_id_len);
     }
     return rb_str_new_cstr(oj_json_class);
 }
@@ -714,7 +714,7 @@ static struct _options mimic_object_to_json_options = {0,              // indent
                                                        false,          // sec_prec_set
                                                        No,             // ignore_under
                                                        Yes,            // cache_keys
-                                                       3,              // cache_str
+                                                       0,              // cache_str
                                                        0,              // int_range_min
                                                        0,              // int_range_max
                                                        oj_json_class,  // create_id

data/ext/oj/object.c

@@ -7,7 +7,7 @@
 
 #include "encode.h"
 #include "err.h"
-#include "hash.h"
+#include "intern.h"
 #include "odd.h"
 #include "oj.h"
 #include "parse.h"
@@ -30,45 +30,18 @@ inline static long read_long(const char *str, size_t len) {
 
 static VALUE calc_hash_key(ParseInfo pi, Val kval, char k1) {
     volatile VALUE rkey;
-#if 0
-    VALUE *slot;
 
     if (':' == k1) {
-        if (Qnil == (rkey = oj_sym_hash_get(kval->key + 1, kval->klen - 1, &slot))) {
-            rkey  = rb_str_new(kval->key + 1, kval->klen - 1);
-            rkey  = oj_encode(rkey);
-            rkey  = rb_str_intern(rkey);
-            *slot = rkey;
-            rb_gc_register_address(slot);
-        }
-    } else if (Yes == pi->options.sym_key) {
-        if (Qnil == (rkey = oj_sym_hash_get(kval->key, kval->klen, &slot))) {
-            rkey  = rb_str_new(kval->key, kval->klen);
-            rkey  = oj_encode(rkey);
-            rkey  = rb_str_intern(rkey);
-            *slot = rkey;
-            rb_gc_register_address(slot);
-        }
-    } else {
-        if (Qnil == (rkey = oj_str_hash_get(kval->key, kval->klen, &slot))) {
-            rkey  = rb_str_new(kval->key, kval->klen);
-            rkey  = oj_encode(rkey);
-            *slot = rkey;
-            rb_gc_register_address(slot);
-        }
+        return ID2SYM(rb_intern3(kval->key + 1, kval->klen - 1, oj_utf8_encoding));
     }
-#else
-    if (':' == k1) {
-        rkey = rb_str_new(kval->key + 1, kval->klen - 1);
-        rkey = oj_encode(rkey);
-	rkey  = rb_str_intern(rkey);
-    } else {
-        rkey = rb_str_new(kval->key, kval->klen);
-        rkey = oj_encode(rkey);
-        if (Yes == pi->options.sym_key) {
-            rkey = rb_str_intern(rkey);
-        }
+    if (Yes == pi->options.sym_key) {
+	return ID2SYM(rb_intern3(kval->key, kval->klen, oj_utf8_encoding));
     }
+#if HAVE_RB_ENC_INTERNED_STR
+    rkey = rb_enc_interned_str(kval->key, kval->klen, oj_utf8_encoding);
+#else
+    rkey = rb_utf8_str_new(kval->key, kval->klen);
+    OBJ_FREEZE(rkey);
 #endif
     return rkey;
 }
@@ -77,9 +50,7 @@ static VALUE str_to_value(ParseInfo pi, const char *str, size_t len, const char
     volatile VALUE rstr = Qnil;
 
     if (':' == *orig && 0 < len) {
-        rstr = rb_str_new(str + 1, len - 1);
-        rstr = oj_encode(rstr);
-        rstr = rb_funcall(rstr, oj_to_sym_id, 0);
+        rstr = ID2SYM(rb_intern3(str + 1, len - 1, oj_utf8_encoding));
     } else if (pi->circ_array && 3 <= len && '^' == *orig && 'r' == orig[1]) {
         long i = read_long(str + 2, len - 2);
 
@@ -89,8 +60,7 @@ static VALUE str_to_value(ParseInfo pi, const char *str, size_t len, const char
         }
         rstr = oj_circ_array_get(pi->circ_array, i);
     } else {
-        rstr = rb_str_new(str, len);
-        rstr = oj_encode(rstr);
+	rstr = rb_utf8_str_new(str, len);
     }
     return rstr;
 }
@@ -258,13 +228,10 @@ static int hat_cstr(ParseInfo pi, Val parent, Val kval, const char *str, size_t
             parent->odd_args = oj_odd_alloc_args(odd);
         } break;
         case 'm':
-            parent->val = rb_str_new(str + 1, len - 1);
-            parent->val = oj_encode(parent->val);
-            parent->val = rb_funcall(parent->val, oj_to_sym_id, 0);
+            parent->val = ID2SYM(rb_intern3(str + 1, len - 1, oj_utf8_encoding));
             break;
         case 's':
-            parent->val = rb_str_new(str, len);
-            parent->val = oj_encode(parent->val);
+	    parent->val = rb_utf8_str_new(str, len);
             break;
         case 'c':  // class
         {
@@ -305,7 +272,7 @@ static int hat_num(ParseInfo pi, Val parent, Val kval, NumInfo ni) {
                 }
                 if (86400 == ni->exp) {  // UTC time
                     parent->val = rb_time_nano_new(ni->i, (long)nsec);
-                    // Since the ruby C routines alway create local time, the
+                    // Since the ruby C routines always create local time, the
                     // offset and then a conversion to UTC keeps makes the time
                     // match the expected value.
                     parent->val = rb_funcall2(parent->val, oj_utc_id, 0, 0);
@@ -416,51 +383,7 @@ static int hat_value(ParseInfo pi, Val parent, const char *key, size_t klen, vol
 }
 
 void oj_set_obj_ivar(Val parent, Val kval, VALUE value) {
-    const char *key  = kval->key;
-    int         klen = kval->klen;
-    ID          var_id;
-    ID *        slot;
-
-#ifdef HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&oj_cache_mutex);
-#else
-    rb_mutex_lock(oj_cache_mutex);
-#endif
-    if (0 == (var_id = oj_attr_hash_get(key, klen, &slot))) {
-        char attr[256];
-
-        if ((int)sizeof(attr) <= klen + 2) {
-            char *buf = ALLOC_N(char, klen + 2);
-
-            if ('~' == *key) {
-                memcpy(buf, key + 1, klen - 1);
-                buf[klen - 1] = '\0';
-            } else {
-                *buf = '@';
-                memcpy(buf + 1, key, klen);
-                buf[klen + 1] = '\0';
-            }
-            var_id = rb_intern(buf);
-            xfree(buf);
-        } else {
-            if ('~' == *key) {
-                memcpy(attr, key + 1, klen - 1);
-                attr[klen - 1] = '\0';
-            } else {
-                *attr = '@';
-                memcpy(attr + 1, key, klen);
-                attr[klen + 1] = '\0';
-            }
-            var_id = rb_intern(attr);
-        }
-        *slot = var_id;
-    }
-#ifdef HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&oj_cache_mutex);
-#else
-    rb_mutex_unlock(oj_cache_mutex);
-#endif
-    rb_ivar_set(parent->val, var_id, value);
+    rb_ivar_set(parent->val, oj_attr_intern(kval->key, kval->klen), value);
 }
 
 static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, const char *orig) {