oj 3.13.2 → 3.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6371eea98925c8d16b816b991404d643051c566f3d36e2c4cf2c69cd29ffd816
-  data.tar.gz: 9373b88d98493ddf677e5f72fd1a16f155f5a9c773f007452b1e5810c586e057
+  metadata.gz: f5350547ea8224c68aefa0d8f1539689ffb24c9addc20b3cbad3537af06b9f45
+  data.tar.gz: d27f8a3fa9311685f215f1636f7ebaf4ed3b10d03725531f8b42c4d1e64b0985
 SHA512:
-  metadata.gz: bb0fb2e41354e3f96804aa2d74f546e87f2df9076b1e3ec3815b271b0cf4b84ac4dc3ed6d49f59a11cb845c3675de84b4ac8a3b27dc3f80fd310f196437a34b4
-  data.tar.gz: ca2efef1c0e3f1e3fb36c7bf0ba5fdffcc596680f2d28923efd8a8cd7cf472178a364a3a93370f5bbefb404e9d0b338754905ed9eb4183eaa9fe0061af41bc93
+  metadata.gz: e3497f0ac6fb20ae4b0aa252855120f2a3e938bbd703dfcebc1d433ea112924b24e3e352080b3f7c09c5cae99d57dde5f26659031126bcb6855d85761e282b1e
+  data.tar.gz: c35a5190c600bcd237041cd30cbe227700fc6aed723fb2d54f13013b4af6f79b96b1c5c2dfbcb4dc9c9db30b2894d97c6242a9b0183376da2c2aaa9fb08d4f0a

data/ext/oj/cache.c

@@ -1,15 +1,32 @@
 // Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
 // Licensed under the MIT License. See LICENSE file in the project root for license details.
 
+#if HAVE_PTHREAD_MUTEX_INIT
+#include <pthread.h>
+#endif
+
 #include "cache.h"
 
-#define REHASH_LIMIT 64
+#define REHASH_LIMIT 4
 #define MIN_SHIFT 8
+#define REUSE_MAX 8192
+
+#if HAVE_PTHREAD_MUTEX_INIT
+#define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
+#define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
+#else
+#define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
+#define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
+#endif
+
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
 
 typedef struct _slot {
     struct _slot *next;
     VALUE         val;
-    uint32_t      hash;
+    uint64_t      hash;
+    uint32_t      use_cnt;
     uint8_t       klen;
     char          key[CACHE_MAX_KEY];
 } * Slot;
@@ -18,17 +35,20 @@ typedef struct _cache {
     Slot * slots;
     size_t cnt;
     VALUE (*form)(const char *str, size_t len);
-    uint32_t size;
-    uint32_t mask;
-    bool     mark;
+    uint64_t size;
+    uint64_t mask;
+    VALUE (*intern)(struct _cache *c, const char *key, size_t len);
+    Slot   reuse;
+    size_t rcnt;
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_t mutex;
+#else
+    VALUE mutex;
+#endif
+    uint8_t xrate;
+    bool    mark;
 } * Cache;
 
-// almost the Murmur hash algorithm
-#define M 0x5bd1e995
-#define C1 0xCC9E2D51
-#define C2 0x1B873593
-#define N 0xE6546B64
-
 void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
     c->form = form;
 }
@@ -36,7 +56,7 @@ void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
 #if 0
 // For debugging only.
 static void cache_print(Cache c) {
-    for (uint32_t i = 0; i < c->size; i++) {
+    for (uint64_t i = 0; i < c->size; i++) {
         printf("%4d:", i);
         for (Slot s = c->slots[i]; NULL != s; s = s->next) {
             char buf[40];
@@ -49,17 +69,17 @@ static void cache_print(Cache c) {
 }
 #endif
 
-static uint32_t hash_calc(const uint8_t *key, size_t len) {
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
     const uint8_t *end     = key + len;
     const uint8_t *endless = key + (len & 0xFFFFFFFC);
-    uint32_t       h       = (uint32_t)len;
-    uint32_t       k;
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
 
     while (key < endless) {
-        k = (uint32_t)*key++;
-        k |= (uint32_t)*key++ << 8;
-        k |= (uint32_t)*key++ << 16;
-        k |= (uint32_t)*key++ << 24;
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
 
         k *= M;
         k ^= k >> 24;
@@ -83,42 +103,23 @@ static uint32_t hash_calc(const uint8_t *key, size_t len) {
     return h;
 }
 
-Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark) {
-    Cache c     = ALLOC(struct _cache);
-    int   shift = 0;
-
-    for (; REHASH_LIMIT < size; size /= 2, shift++) {
-    }
-    if (shift < MIN_SHIFT) {
-        shift = MIN_SHIFT;
-    }
-    c->size  = 1 << shift;
-    c->mask  = c->size - 1;
-    c->slots = ALLOC_N(Slot, c->size);
-    memset(c->slots, 0, sizeof(Slot) * c->size);
-    c->form = form;
-    c->cnt  = 0;
-    c->mark = mark;
-
-    return c;
-}
-
 static void rehash(Cache c) {
-    uint32_t osize = c->size;
-    Slot *   end   = c->slots + osize;
+    uint64_t osize = c->size;
+    Slot *   end;
     Slot *   sp;
 
     c->size = osize * 4;
     c->mask = c->size - 1;
     REALLOC_N(c->slots, Slot, c->size);
     memset(c->slots + osize, 0, sizeof(Slot) * osize * 3);
+    end = c->slots + osize;
     for (sp = c->slots; sp < end; sp++) {
         Slot s    = *sp;
         Slot next = NULL;
 
         *sp = NULL;
         for (; NULL != s; s = next) {
-            uint32_t h      = s->hash & c->mask;
+            uint64_t h      = s->hash & c->mask;
             Slot *   bucket = c->slots + h;
 
             next    = s->next;
@@ -128,8 +129,153 @@ static void rehash(Cache c) {
     }
 }
 
+static VALUE lockless_intern(Cache c, const char *key, size_t len) {
+    uint64_t h      = hash_calc((const uint8_t *)key, len);
+    Slot *   bucket = c->slots + (h & c->mask);
+    Slot     b;
+
+    while (REUSE_MAX < c->rcnt) {
+        if (NULL != (b = c->reuse)) {
+            c->reuse = b->next;
+            xfree(b);
+            c->rcnt--;
+        } else {
+            // An accounting error occured somewhere so correct it.
+            c->rcnt = 0;
+        }
+    }
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            b->use_cnt += 4;
+            return b->val;
+        }
+    }
+    {
+        volatile VALUE rkey = c->form(key, len);
+
+        if (NULL == (b = c->reuse)) {
+            b = ALLOC(struct _slot);
+        } else {
+            c->reuse = b->next;
+            c->rcnt--;
+        }
+        b->hash = h;
+        memcpy(b->key, key, len);
+        b->klen     = (uint8_t)len;
+        b->key[len] = '\0';
+        b->val      = rkey;
+        b->use_cnt  = 4;
+        b->next     = *bucket;
+        *bucket     = b;
+        c->cnt++;  // Don't worry about wrapping. Worse case is the entry is removed and recreated.
+        if (REHASH_LIMIT < c->cnt / c->size) {
+            rehash(c);
+        }
+    }
+    return b->val;
+}
+
+static VALUE locking_intern(Cache c, const char *key, size_t len) {
+    uint64_t h;
+    Slot *   bucket;
+    Slot     b;
+    uint64_t old_size;
+
+    CACHE_LOCK(c);
+    while (REUSE_MAX < c->rcnt) {
+        if (NULL != (b = c->reuse)) {
+            c->reuse = b->next;
+            xfree(b);
+            c->rcnt--;
+        } else {
+            // An accounting error occured somewhere so correct it.
+            c->rcnt = 0;
+        }
+    }
+    h      = hash_calc((const uint8_t *)key, len);
+    bucket = c->slots + (h & c->mask);
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            b->use_cnt += 4;
+            CACHE_UNLOCK(c);
+            return b->val;
+        }
+    }
+    old_size = c->size;
+    // The creation of a new value may trigger a GC which be a problem if the
+    // cache is locked so make sure it is unlocked for the key value creation.
+    if (NULL == (b = c->reuse)) {
+        b = ALLOC(struct _slot);
+    } else {
+        c->reuse = b->next;
+        c->rcnt--;
+    }
+    CACHE_UNLOCK(c);
+    {
+        volatile VALUE rkey = c->form(key, len);
+
+        b->hash = h;
+        memcpy(b->key, key, len);
+        b->klen     = (uint8_t)len;
+        b->key[len] = '\0';
+        b->val      = rkey;
+        b->use_cnt  = 4;
+
+        // Lock again to add the new entry.
+        CACHE_LOCK(c);
+        if (old_size != c->size) {
+            h      = hash_calc((const uint8_t *)key, len);
+            bucket = c->slots + (h & c->mask);
+        }
+        b->next = *bucket;
+        *bucket = b;
+        c->cnt++;  // Don't worry about wrapping. Worse case is the entry is removed and recreated.
+        if (REHASH_LIMIT < c->cnt / c->size) {
+            rehash(c);
+        }
+        CACHE_UNLOCK(c);
+    }
+    return b->val;
+}
+
+Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
+    Cache c     = ALLOC(struct _cache);
+    int   shift = 0;
+
+    for (; REHASH_LIMIT < size; size /= 2, shift++) {
+    }
+    if (shift < MIN_SHIFT) {
+        shift = MIN_SHIFT;
+    }
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_init(&c->mutex, NULL);
+#else
+    c->mutex = rb_mutex_new();
+#endif
+    c->size  = 1 << shift;
+    c->mask  = c->size - 1;
+    c->slots = ALLOC_N(Slot, c->size);
+    memset(c->slots, 0, sizeof(Slot) * c->size);
+    c->form  = form;
+    c->cnt   = 0;
+    c->xrate = 1;  // low
+    c->mark  = mark;
+    c->reuse = NULL;
+    c->rcnt  = 0;
+    if (locking) {
+        c->intern = locking_intern;
+    } else {
+        c->intern = lockless_intern;
+    }
+    return c;
+}
+
+void cache_set_expunge_rate(Cache c, int rate) {
+    c->xrate = (uint8_t)rate;
+}
+
 void cache_free(Cache c) {
-    uint32_t i;
+    uint64_t i;
 
     for (i = 0; i < c->size; i++) {
         Slot next;
@@ -145,14 +291,43 @@ void cache_free(Cache c) {
 }
 
 void cache_mark(Cache c) {
-    if (c->mark) {
-	uint32_t i;
+    uint64_t i;
 
-        for (i = 0; i < c->size; i++) {
-            Slot s;
-            for (s = c->slots[i]; NULL != s; s = s->next) {
+#if !HAVE_PTHREAD_MUTEX_INIT
+    rb_gc_mark(c->mutex);
+#endif
+    if (0 == c->cnt) {
+        return;
+    }
+    for (i = 0; i < c->size; i++) {
+        Slot s;
+        Slot prev = NULL;
+        Slot next;
+
+        for (s = c->slots[i]; NULL != s; s = next) {
+            next = s->next;
+            if (0 == s->use_cnt) {
+                if (NULL == prev) {
+                    c->slots[i] = next;
+                } else {
+                    prev->next = next;
+                }
+                c->cnt--;
+                s->next  = c->reuse;
+                c->reuse = s;
+                c->rcnt++;
+                continue;
+            }
+            switch (c->xrate) {
+            case 0: break;
+            case 2: s->use_cnt -= 2; break;
+            case 3: s->use_cnt /= 2; break;
+            default: s->use_cnt--; break;
+            }
+            if (c->mark) {
                 rb_gc_mark(s->val);
             }
+            prev = s;
         }
     }
 }
@@ -162,32 +337,5 @@ cache_intern(Cache c, const char *key, size_t len) {
     if (CACHE_MAX_KEY < len) {
         return c->form(key, len);
     }
-    uint32_t h      = hash_calc((const uint8_t *)key, len);
-    Slot *   bucket = c->slots + (h & c->mask);
-    Slot     b;
-    Slot     tail = NULL;
-
-    for (b = *bucket; NULL != b; b = b->next) {
-        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
-            return b->val;
-        }
-        tail = b;
-    }
-    b       = ALLOC(struct _slot);
-    b->hash = h;
-    b->next = NULL;
-    memcpy(b->key, key, len);
-    b->klen     = (uint8_t)len;
-    b->key[len] = '\0';
-    b->val      = c->form(key, len);
-    if (NULL == tail) {
-        *bucket = b;
-    } else {
-        tail->next = b;
-    }
-    c->cnt++;
-    if (REHASH_LIMIT < c->cnt / c->size) {
-        rehash(c);
-    }
-    return b->val;
+    return c->intern(c, key, len);
 }

data/ext/oj/cache.h

@@ -11,10 +11,11 @@
 
 struct _cache;
 
-extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark);
+extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
 extern void           cache_free(struct _cache *c);
 extern void           cache_mark(struct _cache *c);
 extern void           cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
 extern VALUE          cache_intern(struct _cache *c, const char *key, size_t len);
+extern void           cache_set_expunge_rate(struct _cache *c, int rate);
 
 #endif /* CACHE_H */

data/ext/oj/compat.c

@@ -30,8 +30,7 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
                 if (Yes == pi->options.sym_key) {
                     rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
                 } else {
-                    rkey = rb_str_new(key, klen);
-                    rkey = oj_encode(rkey);
+		    rkey = rb_utf8_str_new(key, klen);
                 }
             } else if (Yes == pi->options.sym_key) {
                 rkey = oj_sym_intern(key, klen);

data/ext/oj/custom.c

@@ -956,16 +956,14 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
         }
     } else {
 	//volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
-        volatile VALUE rstr = rb_str_new(str, len);
+        volatile VALUE rstr = rb_utf8_str_new(str, len);
 
         if (Qundef == rkey) {
             if (Yes == pi->options.sym_key) {
                 rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
             } else {
-                rkey = rb_str_new(key, klen);
-                rkey = oj_encode(rkey);
+		rkey = rb_utf8_str_new(key, klen);
             }
-            rstr = oj_encode(rstr);
         }
         if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {
             VALUE clas = oj_rxclass_match(&pi->options.str_rx, str, (int)len);
@@ -1090,9 +1088,8 @@ static void array_append_num(ParseInfo pi, NumInfo ni) {
 }
 
 static void array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = rb_utf8_str_new(str, len);
 
-    rstr = oj_encode(rstr);
     if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {
         VALUE clas = oj_rxclass_match(&pi->options.str_rx, str, (int)len);
 

data/ext/oj/extconf.rb

@@ -31,6 +31,7 @@ have_func('rb_gc_mark_movable')
 have_func('stpcpy')
 have_func('pthread_mutex_init')
 have_func('rb_enc_associate')
+have_func('rb_enc_interned_str')
 have_func('rb_ext_ractor_safe', 'ruby.h')
 # rb_hash_bulk_insert is deep down in a header not included in normal build and that seems to fool have_func.
 have_func('rb_hash_bulk_insert', 'ruby.h') unless '2' == version[0] && '6' == version[1]