oj 3.12.3 → 3.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 125ec67a260b09db65d47add3487d46c55ee805ebaa721d3bf841ef27fe3ade7
-  data.tar.gz: a1cd38c40217e8c5ffd24f8759ff00f0d6770dcab56a736bd0c4ba52d5f3a03a
+  metadata.gz: f5350547ea8224c68aefa0d8f1539689ffb24c9addc20b3cbad3537af06b9f45
+  data.tar.gz: d27f8a3fa9311685f215f1636f7ebaf4ed3b10d03725531f8b42c4d1e64b0985
 SHA512:
-  metadata.gz: 63496098643062caf40131db78e9ef1ccd75f9fd56f32e49538ccb6e005a42dcc86198d202562edc0b149b7ee649492376584d21554423846ac078efec700899
-  data.tar.gz: 9d619857d1f9f217d5e7c27c8a999013516680de8e28bc835ee2e267e60eee5dacde1a69e2de694d99d8caf21eb7e5c0c5a51ab9407d03b509423af894913985
+  metadata.gz: e3497f0ac6fb20ae4b0aa252855120f2a3e938bbd703dfcebc1d433ea112924b24e3e352080b3f7c09c5cae99d57dde5f26659031126bcb6855d85761e282b1e
+  data.tar.gz: c35a5190c600bcd237041cd30cbe227700fc6aed723fb2d54f13013b4af6f79b96b1c5c2dfbcb4dc9c9db30b2894d97c6242a9b0183376da2c2aaa9fb08d4f0a

data/README.md

@@ -8,8 +8,7 @@
 
 A *fast* JSON parser and Object marshaller as a Ruby gem.
 
-Version 3.0 is out! 3.0 provides better json gem and Rails compatibility. It
-also provides additional optimization options.
+Version 3.13 is out with a much faster parser (`Oj::Parser`) and option isolation.
 
 ## Using
 
@@ -54,7 +53,7 @@ For more details on options, modes, advanced features, and more follow these
 links.
 
  - [{file:Options.md}](pages/Options.md) for parse and dump options.
- - [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicing the JSON gem, and mimicing Rails and ActiveSupport behavior.
+ - [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicking the JSON gem, and mimicking Rails and ActiveSupport behavior.
  - [{file:JsonGem.md}](pages/JsonGem.md) includes more details on json gem compatibility and use.
  - [{file:Rails.md}](pages/Rails.md) includes more details on Rails and ActiveSupport compatibility and use.
  - [{file:Custom.md}](pages/Custom.md) includes more details on Custom mode.

data/ext/oj/buf.h

@@ -19,6 +19,10 @@ inline static void buf_init(Buf buf) {
     buf->tail = buf->head;
 }
 
+inline static void buf_reset(Buf buf) {
+    buf->tail = buf->head;
+}
+
 inline static void buf_cleanup(Buf buf) {
     if (buf->base != buf->head) {
         xfree(buf->head);
@@ -29,6 +33,11 @@ inline static size_t buf_len(Buf buf) {
     return buf->tail - buf->head;
 }
 
+inline static const char *buf_str(Buf buf) {
+    *buf->tail = '\0';
+    return buf->head;
+}
+
 inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
     if (buf->end <= buf->tail + slen) {
         size_t len     = buf->end - buf->head;

data/ext/oj/cache.c

@@ -0,0 +1,341 @@
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#if HAVE_PTHREAD_MUTEX_INIT
+#include <pthread.h>
+#endif
+
+#include "cache.h"
+
+#define REHASH_LIMIT 4
+#define MIN_SHIFT 8
+#define REUSE_MAX 8192
+
+#if HAVE_PTHREAD_MUTEX_INIT
+#define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
+#define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
+#else
+#define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
+#define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
+#endif
+
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
+
+typedef struct _slot {
+    struct _slot *next;
+    VALUE         val;
+    uint64_t      hash;
+    uint32_t      use_cnt;
+    uint8_t       klen;
+    char          key[CACHE_MAX_KEY];
+} * Slot;
+
+typedef struct _cache {
+    Slot * slots;
+    size_t cnt;
+    VALUE (*form)(const char *str, size_t len);
+    uint64_t size;
+    uint64_t mask;
+    VALUE (*intern)(struct _cache *c, const char *key, size_t len);
+    Slot   reuse;
+    size_t rcnt;
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_t mutex;
+#else
+    VALUE mutex;
+#endif
+    uint8_t xrate;
+    bool    mark;
+} * Cache;
+
+void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
+    c->form = form;
+}
+
+#if 0
+// For debugging only.
+static void cache_print(Cache c) {
+    for (uint64_t i = 0; i < c->size; i++) {
+        printf("%4d:", i);
+        for (Slot s = c->slots[i]; NULL != s; s = s->next) {
+            char buf[40];
+            strncpy(buf, s->key, s->klen);
+            buf[s->klen] = '\0';
+            printf(" %s", buf);
+        }
+        printf("\n");
+    }
+}
+#endif
+
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
+
+    while (key < endless) {
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
+
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
+    }
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
+
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
+    }
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
+}
+
+static void rehash(Cache c) {
+    uint64_t osize = c->size;
+    Slot *   end;
+    Slot *   sp;
+
+    c->size = osize * 4;
+    c->mask = c->size - 1;
+    REALLOC_N(c->slots, Slot, c->size);
+    memset(c->slots + osize, 0, sizeof(Slot) * osize * 3);
+    end = c->slots + osize;
+    for (sp = c->slots; sp < end; sp++) {
+        Slot s    = *sp;
+        Slot next = NULL;
+
+        *sp = NULL;
+        for (; NULL != s; s = next) {
+            uint64_t h      = s->hash & c->mask;
+            Slot *   bucket = c->slots + h;
+
+            next    = s->next;
+            s->next = *bucket;
+            *bucket = s;
+        }
+    }
+}
+
+static VALUE lockless_intern(Cache c, const char *key, size_t len) {
+    uint64_t h      = hash_calc((const uint8_t *)key, len);
+    Slot *   bucket = c->slots + (h & c->mask);
+    Slot     b;
+
+    while (REUSE_MAX < c->rcnt) {
+        if (NULL != (b = c->reuse)) {
+            c->reuse = b->next;
+            xfree(b);
+            c->rcnt--;
+        } else {
+            // An accounting error occured somewhere so correct it.
+            c->rcnt = 0;
+        }
+    }
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            b->use_cnt += 4;
+            return b->val;
+        }
+    }
+    {
+        volatile VALUE rkey = c->form(key, len);
+
+        if (NULL == (b = c->reuse)) {
+            b = ALLOC(struct _slot);
+        } else {
+            c->reuse = b->next;
+            c->rcnt--;
+        }
+        b->hash = h;
+        memcpy(b->key, key, len);
+        b->klen     = (uint8_t)len;
+        b->key[len] = '\0';
+        b->val      = rkey;
+        b->use_cnt  = 4;
+        b->next     = *bucket;
+        *bucket     = b;
+        c->cnt++;  // Don't worry about wrapping. Worse case is the entry is removed and recreated.
+        if (REHASH_LIMIT < c->cnt / c->size) {
+            rehash(c);
+        }
+    }
+    return b->val;
+}
+
+static VALUE locking_intern(Cache c, const char *key, size_t len) {
+    uint64_t h;
+    Slot *   bucket;
+    Slot     b;
+    uint64_t old_size;
+
+    CACHE_LOCK(c);
+    while (REUSE_MAX < c->rcnt) {
+        if (NULL != (b = c->reuse)) {
+            c->reuse = b->next;
+            xfree(b);
+            c->rcnt--;
+        } else {
+            // An accounting error occured somewhere so correct it.
+            c->rcnt = 0;
+        }
+    }
+    h      = hash_calc((const uint8_t *)key, len);
+    bucket = c->slots + (h & c->mask);
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            b->use_cnt += 4;
+            CACHE_UNLOCK(c);
+            return b->val;
+        }
+    }
+    old_size = c->size;
+    // The creation of a new value may trigger a GC which be a problem if the
+    // cache is locked so make sure it is unlocked for the key value creation.
+    if (NULL == (b = c->reuse)) {
+        b = ALLOC(struct _slot);
+    } else {
+        c->reuse = b->next;
+        c->rcnt--;
+    }
+    CACHE_UNLOCK(c);
+    {
+        volatile VALUE rkey = c->form(key, len);
+
+        b->hash = h;
+        memcpy(b->key, key, len);
+        b->klen     = (uint8_t)len;
+        b->key[len] = '\0';
+        b->val      = rkey;
+        b->use_cnt  = 4;
+
+        // Lock again to add the new entry.
+        CACHE_LOCK(c);
+        if (old_size != c->size) {
+            h      = hash_calc((const uint8_t *)key, len);
+            bucket = c->slots + (h & c->mask);
+        }
+        b->next = *bucket;
+        *bucket = b;
+        c->cnt++;  // Don't worry about wrapping. Worse case is the entry is removed and recreated.
+        if (REHASH_LIMIT < c->cnt / c->size) {
+            rehash(c);
+        }
+        CACHE_UNLOCK(c);
+    }
+    return b->val;
+}
+
+Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
+    Cache c     = ALLOC(struct _cache);
+    int   shift = 0;
+
+    for (; REHASH_LIMIT < size; size /= 2, shift++) {
+    }
+    if (shift < MIN_SHIFT) {
+        shift = MIN_SHIFT;
+    }
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_init(&c->mutex, NULL);
+#else
+    c->mutex = rb_mutex_new();
+#endif
+    c->size  = 1 << shift;
+    c->mask  = c->size - 1;
+    c->slots = ALLOC_N(Slot, c->size);
+    memset(c->slots, 0, sizeof(Slot) * c->size);
+    c->form  = form;
+    c->cnt   = 0;
+    c->xrate = 1;  // low
+    c->mark  = mark;
+    c->reuse = NULL;
+    c->rcnt  = 0;
+    if (locking) {
+        c->intern = locking_intern;
+    } else {
+        c->intern = lockless_intern;
+    }
+    return c;
+}
+
+void cache_set_expunge_rate(Cache c, int rate) {
+    c->xrate = (uint8_t)rate;
+}
+
+void cache_free(Cache c) {
+    uint64_t i;
+
+    for (i = 0; i < c->size; i++) {
+        Slot next;
+        Slot s;
+
+        for (s = c->slots[i]; NULL != s; s = next) {
+            next = s->next;
+            xfree(s);
+        }
+    }
+    xfree(c->slots);
+    xfree(c);
+}
+
+void cache_mark(Cache c) {
+    uint64_t i;
+
+#if !HAVE_PTHREAD_MUTEX_INIT
+    rb_gc_mark(c->mutex);
+#endif
+    if (0 == c->cnt) {
+        return;
+    }
+    for (i = 0; i < c->size; i++) {
+        Slot s;
+        Slot prev = NULL;
+        Slot next;
+
+        for (s = c->slots[i]; NULL != s; s = next) {
+            next = s->next;
+            if (0 == s->use_cnt) {
+                if (NULL == prev) {
+                    c->slots[i] = next;
+                } else {
+                    prev->next = next;
+                }
+                c->cnt--;
+                s->next  = c->reuse;
+                c->reuse = s;
+                c->rcnt++;
+                continue;
+            }
+            switch (c->xrate) {
+            case 0: break;
+            case 2: s->use_cnt -= 2; break;
+            case 3: s->use_cnt /= 2; break;
+            default: s->use_cnt--; break;
+            }
+            if (c->mark) {
+                rb_gc_mark(s->val);
+            }
+            prev = s;
+        }
+    }
+}
+
+VALUE
+cache_intern(Cache c, const char *key, size_t len) {
+    if (CACHE_MAX_KEY < len) {
+        return c->form(key, len);
+    }
+    return c->intern(c, key, len);
+}

data/ext/oj/cache.h

@@ -0,0 +1,21 @@
+// Copyright (c) 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#ifndef CACHE_H
+#define CACHE_H
+
+#include <ruby.h>
+#include <stdbool.h>
+
+#define CACHE_MAX_KEY 35
+
+struct _cache;
+
+extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
+extern void           cache_free(struct _cache *c);
+extern void           cache_mark(struct _cache *c);
+extern void           cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
+extern VALUE          cache_intern(struct _cache *c, const char *key, size_t len);
+extern void           cache_set_expunge_rate(struct _cache *c, int rate);
+
+#endif /* CACHE_H */

data/ext/oj/compat.c

@@ -5,7 +5,7 @@
 
 #include "encode.h"
 #include "err.h"
-#include "hash.h"
+#include "intern.h"
 #include "oj.h"
 #include "parse.h"
 #include "resolve.h"
@@ -27,30 +27,15 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
 
         if (Qundef == rkey) {
             if (Yes != pi->options.cache_keys) {
-                rkey = rb_str_new(key, klen);
-                rkey = oj_encode(rkey);
                 if (Yes == pi->options.sym_key) {
-                    rkey = rb_str_intern(rkey);
-                }
-            } else {
-                VALUE *slot;
-
-                if (Yes == pi->options.sym_key) {
-                    if (Qnil == (rkey = oj_sym_hash_get(key, klen, &slot))) {
-                        rkey  = rb_str_new(key, klen);
-                        rkey  = oj_encode(rkey);
-                        rkey  = rb_str_intern(rkey);
-                        *slot = rkey;
-                        rb_gc_register_address(slot);
-                    }
+                    rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
                 } else {
-                    if (Qnil == (rkey = oj_str_hash_get(key, klen, &slot))) {
-                        rkey  = rb_str_new(key, klen);
-                        rkey  = oj_encode(rkey);
-                        *slot = rkey;
-                        rb_gc_register_address(slot);
-                    }
+		    rkey = rb_utf8_str_new(key, klen);
                 }
+            } else if (Yes == pi->options.sym_key) {
+                rkey = oj_sym_intern(key, klen);
+            } else {
+                rkey = oj_str_intern(key, klen);
             }
         }
         if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {