oj 3.13.2 → 3.13.3

data/ext/oj/intern.c

@@ -8,11 +8,16 @@
 #if HAVE_PTHREAD_MUTEX_INIT
 #include <pthread.h>
 #endif
+#include "cache.h"
 #include "parse.h"
 
-#define HASH_SLOT_CNT ((uint32_t)8192)
+// Only used for the class cache so 256 should be sufficient.
+#define HASH_SLOT_CNT ((uint64_t)256)
 #define HASH_MASK (HASH_SLOT_CNT - 1)
 
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
+
 typedef struct _keyVal {
     struct _keyVal *next;
     const char *    key;
@@ -30,256 +35,128 @@ typedef struct _hash {
 } * Hash;
 
 struct _hash class_hash;
-struct _hash str_hash;
-struct _hash sym_hash;
 struct _hash attr_hash;
 
-// almost the Murmur hash algorithm
-#define M 0x5bd1e995
-#define C1 0xCC9E2D51
-#define C2 0x1B873593
-#define N 0xE6546B64
+static struct _cache *str_cache = NULL;
+static VALUE          str_cache_obj;
 
-static uint32_t hash_calc(const uint8_t *key, size_t len) {
-    const uint8_t *end     = key + len;
-    const uint8_t *endless = key + (len & 0xFFFFFFFC);
-    uint32_t       h       = (uint32_t)len;
-    uint32_t       k;
+static struct _cache *sym_cache = NULL;
+static VALUE          sym_cache_obj;
 
-    while (key < endless) {
-        k = (uint32_t)*key++;
-        k |= (uint32_t)*key++ << 8;
-        k |= (uint32_t)*key++ << 16;
-        k |= (uint32_t)*key++ << 24;
+static struct _cache *attr_cache = NULL;
+static VALUE          attr_cache_obj;
 
-        k *= M;
-        k ^= k >> 24;
-        h *= M;
-        h ^= k * M;
-    }
-    if (1 < end - key) {
-        uint16_t k16 = (uint16_t)*key++;
+static VALUE form_str(const char *str, size_t len) {
+    return rb_str_freeze(rb_utf8_str_new(str, len));
+}
 
-        k16 |= (uint16_t)*key++ << 8;
-        h ^= k16 << 8;
+static VALUE form_sym(const char *str, size_t len) {
+    return rb_str_intern(rb_utf8_str_new(str, len));
+}
+
+static VALUE form_attr(const char *str, size_t len) {
+    char buf[256];
+
+    if (sizeof(buf) - 2 <= len) {
+        char *b = ALLOC_N(char, len + 2);
+        ID    id;
+
+        if ('~' == *str) {
+	    memcpy(b, str + 1, len - 1);
+	    b[len - 1] = '\0';
+	    len -= 2;
+	} else {
+	    *b = '@';
+	    memcpy(b + 1, str, len);
+	    b[len + 1] = '\0';
+	}
+        id = rb_intern3(buf, len + 1, oj_utf8_encoding);
+        xfree(b);
+        return id;
     }
-    if (key < end) {
-        h ^= *key;
+    if ('~' == *str) {
+	memcpy(buf, str + 1, len - 1);
+	buf[len - 1] = '\0';
+	len -= 2;
+    } else {
+	*buf = '@';
+	memcpy(buf + 1, str, len);
+	buf[len + 1] = '\0';
     }
-    h *= M;
-    h ^= h >> 13;
-    h *= M;
-    h ^= h >> 15;
-
-    return h;
+    return (VALUE)rb_intern3(buf, len + 1, oj_utf8_encoding);
 }
 
 void oj_hash_init() {
+    VALUE cache_class = rb_define_class_under(Oj, "Cache", rb_cObject);
+
+    str_cache     = cache_create(0, form_str, true, true);
+    str_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, str_cache);
+    rb_gc_register_address(&str_cache_obj);
+
+    sym_cache     = cache_create(0, form_sym, true, true);
+    sym_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, sym_cache);
+    rb_gc_register_address(&sym_cache_obj);
+
+    attr_cache     = cache_create(0, form_attr, false, true);
+    attr_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, attr_cache);
+    rb_gc_register_address(&attr_cache_obj);
+
     memset(class_hash.slots, 0, sizeof(class_hash.slots));
-    memset(str_hash.slots, 0, sizeof(str_hash.slots));
-    memset(sym_hash.slots, 0, sizeof(sym_hash.slots));
-    memset(attr_hash.slots, 0, sizeof(attr_hash.slots));
 #if HAVE_PTHREAD_MUTEX_INIT
     pthread_mutex_init(&class_hash.mutex, NULL);
-    pthread_mutex_init(&str_hash.mutex, NULL);
-    pthread_mutex_init(&sym_hash.mutex, NULL);
-    pthread_mutex_init(&attr_hash.mutex, NULL);
 #else
     class_hash.mutex = rb_mutex_new();
     rb_gc_register_address(&class_hash.mutex);
-    str_hash.mutex = rb_mutex_new();
-    rb_gc_register_address(&str_hash.mutex);
-    sym_hash.mutex = rb_mutex_new();
-    rb_gc_register_address(&sym_hash.mutex);
-    attr_hash.mutex = rb_mutex_new();
-    rb_gc_register_address(&attr_hash.mutex);
 #endif
 }
 
-void oj_hash_print() {
-    uint32_t i;
-    KeyVal   b;
-
-    for (i = 0; i < HASH_SLOT_CNT; i++) {
-        printf("%4d:", i);
-        for (b = class_hash.slots + i; 0 != b && 0 != b->key; b = b->next) {
-            printf(" %s", b->key);
-        }
-        printf("\n");
-    }
-}
-
-void oj_hash_sizes() {
-    uint32_t i;
-    KeyVal   b;
-    int      max = 0;
-    int      min = 1000000;
-
-    for (i = 0; i < HASH_SLOT_CNT; i++) {
-        int cnt = 0;
-
-        for (b = str_hash.slots + i; 0 != b && 0 != b->key; b = b->next) {
-            cnt++;
-        }
-        // printf(" %4d\n", cnt);
-        if (max < cnt) {
-            max = cnt;
-        }
-        if (cnt < min) {
-            min = cnt;
-        }
-    }
-    printf("min: %d  max: %d\n", min, max);
-}
-
 VALUE
 oj_str_intern(const char *key, size_t len) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
-    KeyVal   bucket = str_hash.slots + h;
-    KeyVal   b;
-
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&str_hash.mutex);
-#else
-    rb_mutex_lock(str_hash.mutex);
-#endif
-    if (NULL != bucket->key) {  // not the top slot
-        for (b = bucket; 0 != b; b = b->next) {
-            if (len == b->len && 0 == strncmp(b->key, key, len)) {
-#if HAVE_PTHREAD_MUTEX_INIT
-                pthread_mutex_unlock(&str_hash.mutex);
-#else
-                rb_mutex_unlock(str_hash.mutex);
-#endif
-                return b->val;
-            }
-            bucket = b;
-        }
-        b            = ALLOC(struct _keyVal);
-        b->next      = NULL;
-        bucket->next = b;
-        bucket       = b;
-    }
-    bucket->key = oj_strndup(key, len);
-    bucket->len = len;
-    bucket->val = rb_utf8_str_new(key, len);
-    bucket->val = rb_str_freeze(bucket->val);
-    rb_gc_register_address(&bucket->val);
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&str_hash.mutex);
-#else
-    rb_mutex_unlock(str_hash.mutex);
-#endif
-    return bucket->val;
+    return cache_intern(str_cache, key, len);
 }
 
 VALUE
 oj_sym_intern(const char *key, size_t len) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
-    KeyVal   bucket = sym_hash.slots + h;
-    KeyVal   b;
+    return cache_intern(sym_cache, key, len);
+}
 
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&sym_hash.mutex);
-#else
-    rb_mutex_lock(sym_hash.mutex);
-#endif
-    if (NULL != bucket->key) {  // not the top slot
-        for (b = bucket; 0 != b; b = b->next) {
-            if (len == b->len && 0 == strncmp(b->key, key, len)) {
-#if HAVE_PTHREAD_MUTEX_INIT
-                pthread_mutex_unlock(&sym_hash.mutex);
-#else
-                rb_mutex_unlock(sym_hash.mutex);
-#endif
-                return b->val;
-            }
-            bucket = b;
-        }
-        b            = ALLOC(struct _keyVal);
-        b->next      = NULL;
-        bucket->next = b;
-        bucket       = b;
-    }
-    bucket->key = oj_strndup(key, len);
-    bucket->len = len;
-    bucket->val = ID2SYM(rb_intern3(key, len, oj_utf8_encoding));
-    rb_gc_register_address(&bucket->val);
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&sym_hash.mutex);
-#else
-    rb_mutex_unlock(sym_hash.mutex);
-#endif
-    return bucket->val;
+ID
+oj_attr_intern(const char *key, size_t len) {
+    return cache_intern(attr_cache, key, len);
 }
 
-static ID form_attr(const char *key, size_t klen) {
-    char attr[256];
-    ID   var_id;
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
 
-    if ((int)sizeof(attr) <= klen + 2) {
-        char *buf = ALLOC_N(char, klen + 2);
+    while (key < endless) {
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
 
-        if ('~' == *key) {
-            memcpy(buf, key + 1, klen - 1);
-            buf[klen - 1] = '\0';
-        } else {
-            *buf = '@';
-            memcpy(buf + 1, key, klen);
-            buf[klen + 1] = '\0';
-        }
-        var_id = rb_intern(buf);
-        xfree(buf);
-    } else {
-        if ('~' == *key) {
-            memcpy(attr, key + 1, klen - 1);
-            attr[klen - 1] = '\0';
-        } else {
-            *attr = '@';
-            memcpy(attr + 1, key, klen);
-            attr[klen + 1] = '\0';
-        }
-        var_id = rb_intern(attr);
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
     }
-    return var_id;
-}
-
-ID oj_attr_intern(const char *key, size_t len) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
-    KeyVal   bucket = attr_hash.slots + h;
-    KeyVal   b;
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
 
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&attr_hash.mutex);
-#else
-    rb_mutex_lock(attr_hash.mutex);
-#endif
-    if (NULL != bucket->key) {  // not the top slot
-        for (b = bucket; 0 != b; b = b->next) {
-            if (len == b->len && 0 == strncmp(b->key, key, len)) {
-#if HAVE_PTHREAD_MUTEX_INIT
-                pthread_mutex_unlock(&attr_hash.mutex);
-#else
-                rb_mutex_unlock(attr_hash.mutex);
-#endif
-                return (ID)b->val;
-            }
-            bucket = b;
-        }
-        b            = ALLOC(struct _keyVal);
-        b->next      = NULL;
-        bucket->next = b;
-        bucket       = b;
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
     }
-    bucket->key = oj_strndup(key, len);
-    bucket->len = len;
-    bucket->val = (VALUE)form_attr(key, len);
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&attr_hash.mutex);
-#else
-    rb_mutex_unlock(attr_hash.mutex);
-#endif
-    return (ID)bucket->val;
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
 }
 
 static VALUE resolve_classname(VALUE mod, const char *classname, int auto_define) {
@@ -333,7 +210,7 @@ static VALUE resolve_classpath(ParseInfo pi, const char *name, size_t len, int a
 }
 
 VALUE oj_class_intern(const char *key, size_t len, bool safe, ParseInfo pi, int auto_define, VALUE error_class) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
+    uint64_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
     KeyVal   bucket = class_hash.slots + h;
     KeyVal   b;
 
@@ -396,3 +273,9 @@ char *oj_strndup(const char *s, size_t len) {
 
     return d;
 }
+
+void intern_cleanup() {
+    cache_free(str_cache);
+    cache_free(sym_cache);
+    cache_free(attr_cache);
+}

data/ext/oj/intern.h

@@ -21,7 +21,6 @@ extern VALUE oj_class_intern(const char *       key,
                              int                auto_define,
                              VALUE              error_class);
 
-extern void  oj_hash_print();
 extern char *oj_strndup(const char *s, size_t len);
 
 #endif /* OJ_INTERN_H */

data/ext/oj/mimic_json.c

@@ -682,7 +682,7 @@ static VALUE mimic_set_create_id(VALUE self, VALUE id) {
  */
 static VALUE mimic_create_id(VALUE self) {
     if (NULL != oj_default_options.create_id) {
-        return oj_encode(rb_str_new_cstr(oj_default_options.create_id));
+        return  rb_utf8_str_new(oj_default_options.create_id, oj_default_options.create_id_len);
     }
     return rb_str_new_cstr(oj_json_class);
 }
@@ -714,7 +714,7 @@ static struct _options mimic_object_to_json_options = {0,              // indent
                                                        false,          // sec_prec_set
                                                        No,             // ignore_under
                                                        Yes,            // cache_keys
-                                                       3,              // cache_str
+                                                       0,              // cache_str
                                                        0,              // int_range_min
                                                        0,              // int_range_max
                                                        oj_json_class,  // create_id

data/ext/oj/object.c

@@ -30,46 +30,19 @@ inline static long read_long(const char *str, size_t len) {
 
 static VALUE calc_hash_key(ParseInfo pi, Val kval, char k1) {
     volatile VALUE rkey;
-#if 0
-    VALUE *slot;
 
     if (':' == k1) {
-        if (Qnil == (rkey = oj_sym_hash_get(kval->key + 1, kval->klen - 1, &slot))) {
-            rkey  = rb_str_new(kval->key + 1, kval->klen - 1);
-            rkey  = oj_encode(rkey);
-            rkey  = rb_str_intern(rkey);
-            *slot = rkey;
-            rb_gc_register_address(slot);
-        }
-    } else if (Yes == pi->options.sym_key) {
-        if (Qnil == (rkey = oj_sym_hash_get(kval->key, kval->klen, &slot))) {
-            rkey  = rb_str_new(kval->key, kval->klen);
-            rkey  = oj_encode(rkey);
-            rkey  = rb_str_intern(rkey);
-            *slot = rkey;
-            rb_gc_register_address(slot);
-        }
-    } else {
-        if (Qnil == (rkey = oj_str_hash_get(kval->key, kval->klen, &slot))) {
-            rkey  = rb_str_new(kval->key, kval->klen);
-            rkey  = oj_encode(rkey);
-            *slot = rkey;
-            rb_gc_register_address(slot);
-        }
+        return ID2SYM(rb_intern3(kval->key + 1, kval->klen - 1, oj_utf8_encoding));
     }
-#else
-    if (':' == k1) {
-        rkey = ID2SYM(rb_intern3(kval->key + 1, kval->klen - 1, oj_utf8_encoding));
-    } else {
-        if (Yes == pi->options.sym_key) {
-            rkey = ID2SYM(rb_intern3(kval->key, kval->klen, oj_utf8_encoding));
-        } else {
-            rkey = rb_str_new(kval->key, kval->klen);
-            rkey = oj_encode(rkey);
-        }
+    if (Yes == pi->options.sym_key) {
+	return ID2SYM(rb_intern3(kval->key, kval->klen, oj_utf8_encoding));
     }
-#endif
+#if HAVE_RB_ENC_INTERNED_STR
+    rkey = rb_enc_interned_str(kval->key, kval->klen, oj_utf8_encoding);
+#else
+    rkey = rb_utf8_str_new(kval->key, kval->klen);
     OBJ_FREEZE(rkey);
+#endif
     return rkey;
 }
 
@@ -87,8 +60,7 @@ static VALUE str_to_value(ParseInfo pi, const char *str, size_t len, const char
         }
         rstr = oj_circ_array_get(pi->circ_array, i);
     } else {
-        rstr = rb_str_new(str, len);
-        rstr = oj_encode(rstr);
+	rstr = rb_utf8_str_new(str, len);
     }
     return rstr;
 }
@@ -259,8 +231,7 @@ static int hat_cstr(ParseInfo pi, Val parent, Val kval, const char *str, size_t
             parent->val = ID2SYM(rb_intern3(str + 1, len - 1, oj_utf8_encoding));
             break;
         case 's':
-            parent->val = rb_str_new(str, len);
-            parent->val = oj_encode(parent->val);
+	    parent->val = rb_utf8_str_new(str, len);
             break;
         case 'c':  // class
         {