oj 3.13.1 → 3.13.5

data/ext/oj/cache.h

@@ -11,10 +11,11 @@
 
 struct _cache;
 
-extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark);
+extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
 extern void           cache_free(struct _cache *c);
 extern void           cache_mark(struct _cache *c);
 extern void           cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
 extern VALUE          cache_intern(struct _cache *c, const char *key, size_t len);
+extern void           cache_set_expunge_rate(struct _cache *c, int rate);
 
 #endif /* CACHE_H */

data/ext/oj/compat.c

@@ -30,8 +30,7 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
                 if (Yes == pi->options.sym_key) {
                     rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
                 } else {
-                    rkey = rb_str_new(key, klen);
-                    rkey = oj_encode(rkey);
+		    rkey = rb_utf8_str_new(key, klen);
                 }
             } else if (Yes == pi->options.sym_key) {
                 rkey = oj_sym_intern(key, klen);

data/ext/oj/custom.c

@@ -955,17 +955,15 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
             }
         }
     } else {
-	//volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
-        volatile VALUE rstr = rb_str_new(str, len);
+	volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
+        //volatile VALUE rstr = rb_utf8_str_new(str, len);
 
         if (Qundef == rkey) {
             if (Yes == pi->options.sym_key) {
                 rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
             } else {
-                rkey = rb_str_new(key, klen);
-                rkey = oj_encode(rkey);
+		rkey = rb_utf8_str_new(key, klen);
             }
-            rstr = oj_encode(rstr);
         }
         if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {
             VALUE clas = oj_rxclass_match(&pi->options.str_rx, str, (int)len);
@@ -1032,7 +1030,7 @@ static void hash_set_num(struct _parseInfo *pi, Val kval, NumInfo ni) {
             }
             if (86400 == ni->exp) {  // UTC time
                 parent->val = rb_time_nano_new(ni->i, (long)nsec);
-                // Since the ruby C routines alway create local time, the
+                // Since the ruby C routines always create local time, the
                 // offset and then a conversion to UTC keeps makes the time
                 // match the expected value.
                 parent->val = rb_funcall2(parent->val, oj_utc_id, 0, 0);
@@ -1090,9 +1088,8 @@ static void array_append_num(ParseInfo pi, NumInfo ni) {
 }
 
 static void array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = rb_utf8_str_new(str, len);
 
-    rstr = oj_encode(rstr);
     if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {
         VALUE clas = oj_rxclass_match(&pi->options.str_rx, str, (int)len);
 

data/ext/oj/debug.c

@@ -109,8 +109,9 @@ static void mark(struct _ojParser *p) {
 
 void oj_set_parser_debug(ojParser p) {
     Funcs end = p->funcs + 3;
+    Funcs f;
 
-    for (Funcs f = p->funcs; f < end; f++) {
+    for (f = p->funcs; f < end; f++) {
         f->add_null     = add_null;
         f->add_true     = add_true;
         f->add_false    = add_false;

data/ext/oj/extconf.rb

@@ -31,6 +31,7 @@ have_func('rb_gc_mark_movable')
 have_func('stpcpy')
 have_func('pthread_mutex_init')
 have_func('rb_enc_associate')
+have_func('rb_enc_interned_str')
 have_func('rb_ext_ractor_safe', 'ruby.h')
 # rb_hash_bulk_insert is deep down in a header not included in normal build and that seems to fool have_func.
 have_func('rb_hash_bulk_insert', 'ruby.h') unless '2' == version[0] && '6' == version[1]

data/ext/oj/fast.c

@@ -1472,7 +1472,7 @@ static VALUE doc_move(VALUE self, VALUE str) {
  * to the block on yield is the Doc instance after moving to the child
  * location.
  *   @param [String] path if provided it identified the top of the branch to
- * process the chilren of
+ * process the children of
  * @yieldparam [Doc] Doc at the child location
  * @example
  *   Oj::Doc.open('[3,[2,1]]') { |doc|

data/ext/oj/intern.c

@@ -8,11 +8,16 @@
 #if HAVE_PTHREAD_MUTEX_INIT
 #include <pthread.h>
 #endif
+#include "cache.h"
 #include "parse.h"
 
-#define HASH_SLOT_CNT ((uint32_t)8192)
+// Only used for the class cache so 256 should be sufficient.
+#define HASH_SLOT_CNT ((uint64_t)256)
 #define HASH_MASK (HASH_SLOT_CNT - 1)
 
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
+
 typedef struct _keyVal {
     struct _keyVal *next;
     const char *    key;
@@ -30,256 +35,135 @@ typedef struct _hash {
 } * Hash;
 
 struct _hash class_hash;
-struct _hash str_hash;
-struct _hash sym_hash;
 struct _hash attr_hash;
 
-// almost the Murmur hash algorithm
-#define M 0x5bd1e995
-#define C1 0xCC9E2D51
-#define C2 0x1B873593
-#define N 0xE6546B64
+static struct _cache *str_cache = NULL;
+static VALUE          str_cache_obj;
 
-static uint32_t hash_calc(const uint8_t *key, size_t len) {
-    const uint8_t *end     = key + len;
-    const uint8_t *endless = key + (len & 0xFFFFFFFC);
-    uint32_t       h       = (uint32_t)len;
-    uint32_t       k;
+static struct _cache *sym_cache = NULL;
+static VALUE          sym_cache_obj;
 
-    while (key < endless) {
-        k = (uint32_t)*key++;
-        k |= (uint32_t)*key++ << 8;
-        k |= (uint32_t)*key++ << 16;
-        k |= (uint32_t)*key++ << 24;
+static struct _cache *attr_cache = NULL;
+static VALUE          attr_cache_obj;
 
-        k *= M;
-        k ^= k >> 24;
-        h *= M;
-        h ^= k * M;
-    }
-    if (1 < end - key) {
-        uint16_t k16 = (uint16_t)*key++;
+static VALUE form_str(const char *str, size_t len) {
+    return rb_str_freeze(rb_utf8_str_new(str, len));
+}
 
-        k16 |= (uint16_t)*key++ << 8;
-        h ^= k16 << 8;
+static VALUE form_sym(const char *str, size_t len) {
+    return rb_to_symbol(rb_str_intern(rb_utf8_str_new(str, len)));
+}
+
+static VALUE form_attr(const char *str, size_t len) {
+    char buf[256];
+
+    if (sizeof(buf) - 2 <= len) {
+        char *b = ALLOC_N(char, len + 2);
+        ID    id;
+
+        if ('~' == *str) {
+	    memcpy(b, str + 1, len - 1);
+	    b[len - 1] = '\0';
+	    len -= 2;
+	} else {
+	    *b = '@';
+	    memcpy(b + 1, str, len);
+	    b[len + 1] = '\0';
+	}
+        id = rb_intern3(buf, len + 1, oj_utf8_encoding);
+        xfree(b);
+        return id;
     }
-    if (key < end) {
-        h ^= *key;
+    if ('~' == *str) {
+	memcpy(buf, str + 1, len - 1);
+	buf[len - 1] = '\0';
+	len -= 2;
+    } else {
+	*buf = '@';
+	memcpy(buf + 1, str, len);
+	buf[len + 1] = '\0';
     }
-    h *= M;
-    h ^= h >> 13;
-    h *= M;
-    h ^= h >> 15;
-
-    return h;
+    return (VALUE)rb_intern3(buf, len + 1, oj_utf8_encoding);
 }
 
 void oj_hash_init() {
+    VALUE cache_class = rb_define_class_under(Oj, "Cache", rb_cObject);
+
+    str_cache     = cache_create(0, form_str, true, true);
+    str_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, str_cache);
+    rb_gc_register_address(&str_cache_obj);
+
+    sym_cache     = cache_create(0, form_sym, true, true);
+    sym_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, sym_cache);
+    rb_gc_register_address(&sym_cache_obj);
+
+    attr_cache     = cache_create(0, form_attr, false, true);
+    attr_cache_obj = Data_Wrap_Struct(cache_class, cache_mark, cache_free, attr_cache);
+    rb_gc_register_address(&attr_cache_obj);
+
     memset(class_hash.slots, 0, sizeof(class_hash.slots));
-    memset(str_hash.slots, 0, sizeof(str_hash.slots));
-    memset(sym_hash.slots, 0, sizeof(sym_hash.slots));
-    memset(attr_hash.slots, 0, sizeof(attr_hash.slots));
 #if HAVE_PTHREAD_MUTEX_INIT
     pthread_mutex_init(&class_hash.mutex, NULL);
-    pthread_mutex_init(&str_hash.mutex, NULL);
-    pthread_mutex_init(&sym_hash.mutex, NULL);
-    pthread_mutex_init(&attr_hash.mutex, NULL);
 #else
     class_hash.mutex = rb_mutex_new();
     rb_gc_register_address(&class_hash.mutex);
-    str_hash.mutex = rb_mutex_new();
-    rb_gc_register_address(&str_hash.mutex);
-    sym_hash.mutex = rb_mutex_new();
-    rb_gc_register_address(&sym_hash.mutex);
-    attr_hash.mutex = rb_mutex_new();
-    rb_gc_register_address(&attr_hash.mutex);
 #endif
 }
 
-void oj_hash_print() {
-    uint32_t i;
-    KeyVal   b;
-
-    for (i = 0; i < HASH_SLOT_CNT; i++) {
-        printf("%4d:", i);
-        for (b = class_hash.slots + i; 0 != b && 0 != b->key; b = b->next) {
-            printf(" %s", b->key);
-        }
-        printf("\n");
-    }
-}
-
-void oj_hash_sizes() {
-    uint32_t i;
-    KeyVal   b;
-    int      max = 0;
-    int      min = 1000000;
-
-    for (i = 0; i < HASH_SLOT_CNT; i++) {
-        int cnt = 0;
-
-        for (b = str_hash.slots + i; 0 != b && 0 != b->key; b = b->next) {
-            cnt++;
-        }
-        // printf(" %4d\n", cnt);
-        if (max < cnt) {
-            max = cnt;
-        }
-        if (cnt < min) {
-            min = cnt;
-        }
-    }
-    printf("min: %d  max: %d\n", min, max);
-}
-
 VALUE
 oj_str_intern(const char *key, size_t len) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
-    KeyVal   bucket = str_hash.slots + h;
-    KeyVal   b;
-
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&str_hash.mutex);
-#else
-    rb_mutex_lock(str_hash.mutex);
-#endif
-    if (NULL != bucket->key) {  // not the top slot
-        for (b = bucket; 0 != b; b = b->next) {
-            if (len == b->len && 0 == strncmp(b->key, key, len)) {
-#if HAVE_PTHREAD_MUTEX_INIT
-                pthread_mutex_unlock(&str_hash.mutex);
+    // For huge cache sizes over half a million the rb_enc_interned_str
+    // performs slightly better but at more "normal" size of a several
+    // thousands the cache intern performs about 20% better.
+#if HAVE_RB_ENC_INTERNED_STR && 0
+    return rb_enc_interned_str(key, len, rb_utf8_encoding());
 #else
-                rb_mutex_unlock(str_hash.mutex);
+    return cache_intern(str_cache, key, len);
 #endif
-                return b->val;
-            }
-            bucket = b;
-        }
-        b            = ALLOC(struct _keyVal);
-        b->next      = NULL;
-        bucket->next = b;
-        bucket       = b;
-    }
-    bucket->key = oj_strndup(key, len);
-    bucket->len = len;
-    bucket->val = rb_utf8_str_new(key, len);
-    bucket->val = rb_str_freeze(bucket->val);
-    rb_gc_register_address(&bucket->val);
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&str_hash.mutex);
-#else
-    rb_mutex_unlock(str_hash.mutex);
-#endif
-    return bucket->val;
 }
 
 VALUE
 oj_sym_intern(const char *key, size_t len) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
-    KeyVal   bucket = sym_hash.slots + h;
-    KeyVal   b;
+    return cache_intern(sym_cache, key, len);
+}
 
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&sym_hash.mutex);
-#else
-    rb_mutex_lock(sym_hash.mutex);
-#endif
-    if (NULL != bucket->key) {  // not the top slot
-        for (b = bucket; 0 != b; b = b->next) {
-            if (len == b->len && 0 == strncmp(b->key, key, len)) {
-#if HAVE_PTHREAD_MUTEX_INIT
-                pthread_mutex_unlock(&sym_hash.mutex);
-#else
-                rb_mutex_unlock(sym_hash.mutex);
-#endif
-                return b->val;
-            }
-            bucket = b;
-        }
-        b            = ALLOC(struct _keyVal);
-        b->next      = NULL;
-        bucket->next = b;
-        bucket       = b;
-    }
-    bucket->key = oj_strndup(key, len);
-    bucket->len = len;
-    bucket->val = ID2SYM(rb_intern3(key, len, oj_utf8_encoding));
-    rb_gc_register_address(&bucket->val);
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&sym_hash.mutex);
-#else
-    rb_mutex_unlock(sym_hash.mutex);
-#endif
-    return bucket->val;
+ID
+oj_attr_intern(const char *key, size_t len) {
+    return cache_intern(attr_cache, key, len);
 }
 
-static ID form_attr(const char *key, size_t klen) {
-    char attr[256];
-    ID   var_id;
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
 
-    if ((int)sizeof(attr) <= klen + 2) {
-        char *buf = ALLOC_N(char, klen + 2);
+    while (key < endless) {
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
 
-        if ('~' == *key) {
-            memcpy(buf, key + 1, klen - 1);
-            buf[klen - 1] = '\0';
-        } else {
-            *buf = '@';
-            memcpy(buf + 1, key, klen);
-            buf[klen + 1] = '\0';
-        }
-        var_id = rb_intern(buf);
-        xfree(buf);
-    } else {
-        if ('~' == *key) {
-            memcpy(attr, key + 1, klen - 1);
-            attr[klen - 1] = '\0';
-        } else {
-            *attr = '@';
-            memcpy(attr + 1, key, klen);
-            attr[klen + 1] = '\0';
-        }
-        var_id = rb_intern(attr);
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
     }
-    return var_id;
-}
-
-ID oj_attr_intern(const char *key, size_t len) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
-    KeyVal   bucket = attr_hash.slots + h;
-    KeyVal   b;
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
 
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_lock(&attr_hash.mutex);
-#else
-    rb_mutex_lock(attr_hash.mutex);
-#endif
-    if (NULL != bucket->key) {  // not the top slot
-        for (b = bucket; 0 != b; b = b->next) {
-            if (len == b->len && 0 == strncmp(b->key, key, len)) {
-#if HAVE_PTHREAD_MUTEX_INIT
-                pthread_mutex_unlock(&attr_hash.mutex);
-#else
-                rb_mutex_unlock(attr_hash.mutex);
-#endif
-                return (ID)b->val;
-            }
-            bucket = b;
-        }
-        b            = ALLOC(struct _keyVal);
-        b->next      = NULL;
-        bucket->next = b;
-        bucket       = b;
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
     }
-    bucket->key = oj_strndup(key, len);
-    bucket->len = len;
-    bucket->val = (VALUE)form_attr(key, len);
-#if HAVE_PTHREAD_MUTEX_INIT
-    pthread_mutex_unlock(&attr_hash.mutex);
-#else
-    rb_mutex_unlock(attr_hash.mutex);
-#endif
-    return (ID)bucket->val;
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
 }
 
 static VALUE resolve_classname(VALUE mod, const char *classname, int auto_define) {
@@ -333,7 +217,7 @@ static VALUE resolve_classpath(ParseInfo pi, const char *name, size_t len, int a
 }
 
 VALUE oj_class_intern(const char *key, size_t len, bool safe, ParseInfo pi, int auto_define, VALUE error_class) {
-    uint32_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
+    uint64_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
     KeyVal   bucket = class_hash.slots + h;
     KeyVal   b;
 
@@ -396,3 +280,9 @@ char *oj_strndup(const char *s, size_t len) {
 
     return d;
 }
+
+void intern_cleanup() {
+    cache_free(str_cache);
+    cache_free(sym_cache);
+    cache_free(attr_cache);
+}