oj 3.12.0 → 3.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5bf6799d9923c05b62b3bde88f052bb4373e9e0f15153f59e137b1023205a23
-  data.tar.gz: 872155b5f53adb9a8dfbd20e54a9b1a329762c353b25b656eba071e85966acbd
+  metadata.gz: 659a7b0d5dc0fd786f01d8879cf94549d7ba1c7b893048222579b77455dc1654
+  data.tar.gz: 3413e8022bb3b6972a0c91acd63ca1d292bccebb9531fc8fb1f86aecfe675471
 SHA512:
-  metadata.gz: 5426970bd5c337bb387de75baa6589a82e81549d54b77c4ee7557fc52767cc933dcafb88b30e995ddf0c265ccd3e0d572c173bd4246d7c3998894879808aa68e
-  data.tar.gz: d6748332981816586e67aa48b80f0730b7bfe13b124e0afea70fae323b8908b49dede05b39f80586a017391dc7d257ad889a2f828cedfdd0ccd272835f69e2a6
+  metadata.gz: 41123b593024773a786bdb12982820387da83a7dc6e385e2cd40f3a4a8b1779bbc3d2a15b8e9f52e4a69b54c4b086c453e745a33bcf21d7d237e22f1fd9e6b04
+  data.tar.gz: 4daa8aadb603184ab90b91dd91a81a74fc1c8bfcb85d77b342ffb8054bd48bda34b94d46990ace8eb07784e4e2221f4a836d304b3a1057bcf6fdb844b89997e5

data/README.md

@@ -8,8 +8,7 @@
 
 A *fast* JSON parser and Object marshaller as a Ruby gem.
 
-Version 3.0 is out! 3.0 provides better json gem and Rails compatibility. It
-also provides additional optimization options.
+Version 3.13 is out with a much faster parser (`Oj::Parser`) and option isolation.
 
 ## Using
 

data/ext/oj/buf.h

@@ -19,6 +19,10 @@ inline static void buf_init(Buf buf) {
     buf->tail = buf->head;
 }
 
+inline static void buf_reset(Buf buf) {
+    buf->tail = buf->head;
+}
+
 inline static void buf_cleanup(Buf buf) {
     if (buf->base != buf->head) {
         xfree(buf->head);
@@ -29,6 +33,11 @@ inline static size_t buf_len(Buf buf) {
     return buf->tail - buf->head;
 }
 
+inline static const char *buf_str(Buf buf) {
+    *buf->tail = '\0';
+    return buf->head;
+}
+
 inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
     if (buf->end <= buf->tail + slen) {
         size_t len     = buf->end - buf->head;

data/ext/oj/cache.c

@@ -0,0 +1,187 @@
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#include "cache.h"
+
+#define REHASH_LIMIT 64
+#define MIN_SHIFT 8
+
+typedef struct _slot {
+    struct _slot *next;
+    VALUE         val;
+    uint32_t      hash;
+    uint8_t       klen;
+    char          key[CACHE_MAX_KEY];
+} * Slot;
+
+typedef struct _cache {
+    Slot * slots;
+    size_t cnt;
+    VALUE (*form)(const char *str, size_t len);
+    uint32_t size;
+    uint32_t mask;
+    bool     mark;
+} * Cache;
+
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
+#define C1 0xCC9E2D51
+#define C2 0x1B873593
+#define N 0xE6546B64
+
+void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
+    c->form = form;
+}
+
+#if 0
+// For debugging only.
+static void cache_print(Cache c) {
+    for (uint32_t i = 0; i < c->size; i++) {
+        printf("%4d:", i);
+        for (Slot s = c->slots[i]; NULL != s; s = s->next) {
+            char buf[40];
+            strncpy(buf, s->key, s->klen);
+            buf[s->klen] = '\0';
+            printf(" %s", buf);
+        }
+        printf("\n");
+    }
+}
+#endif
+
+static uint32_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint32_t       h       = (uint32_t)len;
+    uint32_t       k;
+
+    while (key < endless) {
+        k = (uint32_t)*key++;
+        k |= (uint32_t)*key++ << 8;
+        k |= (uint32_t)*key++ << 16;
+        k |= (uint32_t)*key++ << 24;
+
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
+    }
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
+
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
+    }
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
+}
+
+Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark) {
+    Cache c     = ALLOC(struct _cache);
+    int   shift = 0;
+
+    for (; REHASH_LIMIT < size; size /= 2, shift++) {
+    }
+    if (shift < MIN_SHIFT) {
+        shift = MIN_SHIFT;
+    }
+    c->size  = 1 << shift;
+    c->mask  = c->size - 1;
+    c->slots = ALLOC_N(Slot, c->size);
+    memset(c->slots, 0, sizeof(Slot) * c->size);
+    c->form = form;
+    c->cnt  = 0;
+    c->mark  = mark;
+
+    return c;
+}
+
+static void rehash(Cache c) {
+    uint32_t osize = c->size;
+
+    c->size = osize * 4;
+    c->mask = c->size - 1;
+    REALLOC_N(c->slots, Slot, c->size);
+    memset(c->slots + osize, 0, sizeof(Slot) * osize * 3);
+
+    Slot *end = c->slots + osize;
+    for (Slot *sp = c->slots; sp < end; sp++) {
+        Slot s    = *sp;
+        Slot next = NULL;
+
+        *sp = NULL;
+        for (; NULL != s; s = next) {
+            next = s->next;
+
+            uint32_t h      = s->hash & c->mask;
+            Slot *   bucket = c->slots + h;
+
+            s->next = *bucket;
+            *bucket = s;
+        }
+    }
+}
+
+void cache_free(Cache c) {
+    for (uint32_t i = 0; i < c->size; i++) {
+        Slot next;
+        for (Slot s = c->slots[i]; NULL != s; s = next) {
+            next = s->next;
+            xfree(s);
+        }
+    }
+    xfree(c->slots);
+    xfree(c);
+}
+
+void cache_mark(Cache c) {
+    if (c->mark) {
+        for (uint32_t i = 0; i < c->size; i++) {
+            for (Slot s = c->slots[i]; NULL != s; s = s->next) {
+		rb_gc_mark(s->val);
+            }
+        }
+    }
+}
+
+VALUE
+cache_intern(Cache c, const char *key, size_t len) {
+    if (CACHE_MAX_KEY < len) {
+        return c->form(key, len);
+    }
+    uint32_t h      = hash_calc((const uint8_t *)key, len);
+    Slot *   bucket = c->slots + (h & c->mask);
+    Slot     b;
+    Slot     tail = NULL;
+
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            return b->val;
+        }
+        tail = b;
+    }
+    b       = ALLOC(struct _slot);
+    b->hash = h;
+    b->next = NULL;
+    memcpy(b->key, key, len);
+    b->klen     = (uint8_t)len;
+    b->key[len] = '\0';
+    b->val      = c->form(key, len);
+    if (NULL == tail) {
+        *bucket = b;
+    } else {
+        tail->next = b;
+    }
+    c->cnt++;
+    if (REHASH_LIMIT < c->cnt / c->size) {
+        rehash(c);
+    }
+    return b->val;
+}

data/ext/oj/cache.h

@@ -0,0 +1,20 @@
+// Copyright (c) 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#ifndef CACHE_H
+#define CACHE_H
+
+#include <ruby.h>
+#include <stdbool.h>
+
+#define CACHE_MAX_KEY 35
+
+struct _cache;
+
+extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark);
+extern void           cache_free(struct _cache *c);
+extern void           cache_mark(struct _cache *c);
+extern void           cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
+extern VALUE          cache_intern(struct _cache *c, const char *key, size_t len);
+
+#endif /* CACHE_H */

data/ext/oj/compat.c

@@ -5,7 +5,7 @@
 
 #include "encode.h"
 #include "err.h"
-#include "hash.h"
+#include "intern.h"
 #include "oj.h"
 #include "parse.h"
 #include "resolve.h"
@@ -26,23 +26,17 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
         volatile VALUE rstr = oj_cstr_to_value(str, len, (size_t)pi->options.cache_str);
 
         if (Qundef == rkey) {
-            VALUE *slot;
-
-            if (Yes == pi->options.sym_key) {
-                if (Qnil == (rkey = oj_sym_hash_get(key, klen, &slot))) {
-                    rkey  = rb_str_new(key, klen);
-                    rkey  = oj_encode(rkey);
-                    rkey  = rb_str_intern(rkey);
-                    *slot = rkey;
-                    rb_gc_register_address(slot);
+            if (Yes != pi->options.cache_keys) {
+                if (Yes == pi->options.sym_key) {
+                    rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
+                } else {
+                    rkey = rb_str_new(key, klen);
+                    rkey = oj_encode(rkey);
                 }
+            } else if (Yes == pi->options.sym_key) {
+                rkey = oj_sym_intern(key, klen);
             } else {
-                if (Qnil == (rkey = oj_str_hash_get(key, klen, &slot))) {
-                    rkey  = rb_str_new(key, klen);
-                    rkey  = oj_encode(rkey);
-                    *slot = rkey;
-                    rb_gc_register_address(slot);
-                }
+                rkey = oj_str_intern(key, klen);
             }
         }
         if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {

data/ext/oj/custom.c

@@ -8,7 +8,7 @@
 #include "dump.h"
 #include "encode.h"
 #include "err.h"
-#include "hash.h"
+#include "intern.h"
 #include "odd.h"
 #include "oj.h"
 #include "parse.h"
@@ -31,14 +31,14 @@ static void dump_obj_str(VALUE obj, int depth, Out out) {
 
 static void dump_obj_as_str(VALUE obj, int depth, Out out) {
     volatile VALUE rstr = rb_funcall(obj, oj_to_s_id, 0);
-    const char *   str  = rb_string_value_ptr((VALUE *)&rstr);
+    const char *   str  = RSTRING_PTR(rstr);
 
     oj_dump_cstr(str, RSTRING_LEN(rstr), 0, 0, out);
 }
 
 static void bigdecimal_dump(VALUE obj, int depth, Out out) {
     volatile VALUE rstr = rb_funcall(obj, oj_to_s_id, 0);
-    const char *   str  = rb_string_value_ptr((VALUE *)&rstr);
+    const char *   str  = RSTRING_PTR(rstr);
     int            len  = (int)RSTRING_LEN(rstr);
 
     if (0 == strcasecmp("Infinity", str)) {
@@ -123,7 +123,7 @@ static void date_dump(VALUE obj, int depth, Out out) {
         case RubyTime:
         case XmlTime:
             v = rb_funcall(obj, rb_intern("iso8601"), 0);
-            oj_dump_cstr(rb_string_value_ptr((VALUE *)&v), (int)RSTRING_LEN(v), 0, 0, out);
+            oj_dump_cstr(RSTRING_PTR(v), (int)RSTRING_LEN(v), 0, 0, out);
             break;
         case UnixZTime:
             v = rb_funcall(obj, rb_intern("to_time"), 0);
@@ -420,7 +420,7 @@ static void dump_odd(VALUE obj, Odd odd, VALUE clas, int depth, Out out) {
         if (Qundef == v || T_STRING != rb_type(v)) {
             rb_raise(rb_eEncodingError, "Invalid type for raw JSON.\n");
         } else {
-            const char *s    = rb_string_value_ptr((VALUE *)&v);
+            const char *s    = RSTRING_PTR(v);
             int         len  = (int)RSTRING_LEN(v);
             const char *name = rb_id2name(*odd->attrs);
             size_t      nlen = strlen(name);
@@ -510,7 +510,7 @@ static VALUE dump_common(VALUE obj, int depth, Out out) {
         if (Yes == out->opts->trace) {
             oj_trace("to_json", obj, __FILE__, __LINE__, depth + 1, TraceRubyOut);
         }
-        s   = rb_string_value_ptr((VALUE *)&rs);
+        s   = RSTRING_PTR(rs);
         len = (int)RSTRING_LEN(rs);
 
         assure_size(out, len + 1);
@@ -537,7 +537,7 @@ static VALUE dump_common(VALUE obj, int depth, Out out) {
         if (aj == obj) {
             volatile VALUE rstr = rb_funcall(obj, oj_to_s_id, 0);
 
-            oj_dump_cstr(rb_string_value_ptr((VALUE *)&rstr),
+            oj_dump_cstr(RSTRING_PTR(rstr),
                          (int)RSTRING_LEN(rstr),
                          false,
                          false,
@@ -835,7 +835,7 @@ static void dump_struct(VALUE obj, int depth, Out out, bool as_ok) {
             if (ma != Qnil) {
                 volatile VALUE s = rb_sym_to_s(rb_ary_entry(ma, i));
 
-                name = rb_string_value_ptr((VALUE *)&s);
+                name = RSTRING_PTR(s);
                 len  = (int)RSTRING_LEN(s);
             } else {
                 len  = snprintf(num_id, sizeof(num_id), "%d", i);
@@ -959,12 +959,13 @@ static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, c
         volatile VALUE rstr = rb_str_new(str, len);
 
         if (Qundef == rkey) {
-            rkey = rb_str_new(key, klen);
-            rstr = oj_encode(rstr);
-            rkey = oj_encode(rkey);
             if (Yes == pi->options.sym_key) {
-                rkey = rb_str_intern(rkey);
+                rkey = ID2SYM(rb_intern3(key, klen, oj_utf8_encoding));
+            } else {
+                rkey = rb_str_new(key, klen);
+                rkey = oj_encode(rkey);
             }
+            rstr = oj_encode(rstr);
         }
         if (Yes == pi->options.create_ok && NULL != pi->options.str_rx.head) {
             VALUE clas = oj_rxclass_match(&pi->options.str_rx, str, (int)len);

data/ext/oj/debug.c

@@ -0,0 +1,131 @@
+// Copyright (c) 2021, Peter Ohler, All rights reserved.
+
+#include "parser.h"
+
+static void add_null(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_null at top\n"); break;
+    case ARRAY_FUN: printf("*** add_null to array\n"); break;
+    case OBJECT_FUN: printf("*** add_null with '%s'\n", buf_str(&p->key)); break;
+    }
+}
+
+static void add_true(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_true at top\n"); break;
+    case ARRAY_FUN: printf("*** add_true to array\n"); break;
+    case OBJECT_FUN: printf("*** add_true with '%s'\n", buf_str(&p->key)); break;
+    }
+}
+
+static void add_false(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_false at top\n"); break;
+    case ARRAY_FUN: printf("*** add_false to array\n"); break;
+    case OBJECT_FUN: printf("*** add_false with '%s'\n", buf_str(&p->key)); break;
+    }
+}
+
+static void add_int(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_int %lld at top\n", (long long)p->num.fixnum); break;
+    case ARRAY_FUN: printf("*** add_int %lld to array\n", (long long)p->num.fixnum); break;
+    case OBJECT_FUN:
+        printf("*** add_int %lld with '%s'\n", (long long)p->num.fixnum, buf_str(&p->key));
+        break;
+    }
+}
+
+static void add_float(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_float %Lf at top\n", p->num.dub); break;
+    case ARRAY_FUN: printf("*** add_float %Lf to array\n", p->num.dub); break;
+    case OBJECT_FUN: printf("*** add_float %Lf with '%s'\n", p->num.dub, buf_str(&p->key)); break;
+    }
+}
+
+static void add_big(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_big %s at top\n", buf_str(&p->buf)); break;
+    case ARRAY_FUN: printf("*** add_big %s to array\n", buf_str(&p->buf)); break;
+    case OBJECT_FUN:
+        printf("*** add_big %s with '%s'\n", buf_str(&p->buf), buf_str(&p->key));
+        break;
+    }
+}
+
+static void add_str(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** add_str '%s' at top\n", buf_str(&p->buf)); break;
+    case ARRAY_FUN: printf("*** add_str '%s' to array\n", buf_str(&p->buf)); break;
+    case OBJECT_FUN:
+        printf("*** add_str '%s' with '%s'\n", buf_str(&p->buf), buf_str(&p->key));
+        break;
+    }
+}
+
+static void open_array(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** open_array at top\n"); break;
+    case ARRAY_FUN: printf("*** open_array to array\n"); break;
+    case OBJECT_FUN: printf("*** open_array with '%s'\n", buf_str(&p->key)); break;
+    }
+}
+
+static void close_array(struct _ojParser *p) {
+    printf("*** close_array\n");
+}
+
+static void open_object(struct _ojParser *p) {
+    switch (p->stack[p->depth]) {
+    case TOP_FUN: printf("*** open_object at top\n"); break;
+    case ARRAY_FUN: printf("*** open_object to array\n"); break;
+    case OBJECT_FUN: printf("*** open_object with '%s'\n", buf_str(&p->key)); break;
+    }
+}
+
+static void close_object(struct _ojParser *p) {
+    printf("*** close_object\n");
+}
+
+static VALUE option(ojParser p, const char *key, VALUE value) {
+    rb_raise(rb_eArgError, "%s is not an option for the debug delegate", key);
+    return Qnil;
+}
+
+static VALUE result(struct _ojParser *p) {
+    return Qnil;
+}
+
+static void start(struct _ojParser *p) {
+    printf("*** start\n");
+}
+
+static void dfree(struct _ojParser *p) {
+}
+
+static void mark(struct _ojParser *p) {
+}
+
+void oj_set_parser_debug(ojParser p) {
+    Funcs end = p->funcs + 3;
+
+    for (Funcs f = p->funcs; f < end; f++) {
+        f->add_null     = add_null;
+        f->add_true     = add_true;
+        f->add_false    = add_false;
+        f->add_int      = add_int;
+        f->add_float    = add_float;
+        f->add_big      = add_big;
+        f->add_str      = add_str;
+        f->open_array   = open_array;
+        f->close_array  = close_array;
+        f->open_object  = open_object;
+        f->close_object = close_object;
+    }
+    p->option = option;
+    p->result = result;
+    p->free   = dfree;
+    p->mark   = mark;
+    p->start  = start;
+}