oj 3.11.5 → 3.16.5

data/ext/oj/intern.c

@@ -0,0 +1,313 @@
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#include "intern.h"
+
+#include <stdint.h>
+
+#if HAVE_PTHREAD_MUTEX_INIT
+#include <pthread.h>
+#endif
+
+#include "cache.h"
+#include "mem.h"
+#include "parse.h"
+
+// Only used for the class cache so 256 should be sufficient.
+#define HASH_SLOT_CNT ((uint64_t)256)
+#define HASH_MASK (HASH_SLOT_CNT - 1)
+
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
+
+typedef struct _keyVal {
+    struct _keyVal *next;
+    const char     *key;
+    size_t          len;
+    VALUE           val;
+} *KeyVal;
+
+typedef struct _hash {
+    struct _keyVal slots[HASH_SLOT_CNT];
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_t mutex;
+#else
+    VALUE mutex;
+#endif
+} *Hash;
+
+struct _hash class_hash;
+struct _hash attr_hash;
+
+static VALUE str_cache_obj;
+
+static VALUE sym_cache_obj;
+
+static VALUE attr_cache_obj;
+
+static VALUE form_str(const char *str, size_t len) {
+    return rb_str_freeze(rb_utf8_str_new(str, len));
+}
+
+static VALUE form_sym(const char *str, size_t len) {
+    return rb_to_symbol(rb_str_intern(rb_utf8_str_new(str, len)));
+}
+
+static VALUE form_attr(const char *str, size_t len) {
+    char buf[256];
+
+    if (sizeof(buf) - 2 <= len) {
+        char *b = OJ_R_ALLOC_N(char, len + 2);
+        ID    id;
+
+        if ('~' == *str) {
+            memcpy(b, str + 1, len - 1);
+            b[len - 1] = '\0';
+            len -= 2;
+        } else {
+            *b = '@';
+            memcpy(b + 1, str, len);
+            b[len + 1] = '\0';
+        }
+        id = rb_intern3(buf, len + 1, oj_utf8_encoding);
+        OJ_R_FREE(b);
+        return id;
+    }
+    if ('~' == *str) {
+        memcpy(buf, str + 1, len - 1);
+        buf[len - 1] = '\0';
+        len -= 2;
+    } else {
+        *buf = '@';
+        memcpy(buf + 1, str, len);
+        buf[len + 1] = '\0';
+    }
+    return (VALUE)rb_intern3(buf, len + 1, oj_utf8_encoding);
+}
+
+static const rb_data_type_t oj_cache_type = {
+    "Oj/cache",
+    {
+        cache_mark,
+        cache_free,
+        NULL,
+    },
+    0,
+    0,
+};
+
+void oj_hash_init(void) {
+    VALUE cache_class = rb_define_class_under(Oj, "Cache", rb_cObject);
+    rb_undef_alloc_func(cache_class);
+
+    struct _cache *str_cache = cache_create(0, form_str, true, true);
+    str_cache_obj            = TypedData_Wrap_Struct(cache_class, &oj_cache_type, str_cache);
+    rb_gc_register_address(&str_cache_obj);
+
+    struct _cache *sym_cache = cache_create(0, form_sym, true, true);
+    sym_cache_obj            = TypedData_Wrap_Struct(cache_class, &oj_cache_type, sym_cache);
+    rb_gc_register_address(&sym_cache_obj);
+
+    struct _cache *attr_cache = cache_create(0, form_attr, false, true);
+    attr_cache_obj            = TypedData_Wrap_Struct(cache_class, &oj_cache_type, attr_cache);
+    rb_gc_register_address(&attr_cache_obj);
+
+    memset(class_hash.slots, 0, sizeof(class_hash.slots));
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_init(&class_hash.mutex, NULL);
+#else
+    class_hash.mutex = rb_mutex_new();
+    rb_gc_register_address(&class_hash.mutex);
+#endif
+}
+
+VALUE
+oj_str_intern(const char *key, size_t len) {
+    // For huge cache sizes over half a million the rb_enc_interned_str
+    // performs slightly better but at more "normal" size of a several
+    // thousands the cache intern performs about 20% better.
+#if HAVE_RB_ENC_INTERNED_STR && 0
+    return rb_enc_interned_str(key, len, rb_utf8_encoding());
+#else
+    Cache c;
+    TypedData_Get_Struct(str_cache_obj, struct _cache, &oj_cache_type, c);
+    return cache_intern(c, key, len);
+#endif
+}
+
+VALUE
+oj_sym_intern(const char *key, size_t len) {
+    Cache c;
+    TypedData_Get_Struct(sym_cache_obj, struct _cache, &oj_cache_type, c);
+    return cache_intern(c, key, len);
+}
+
+ID oj_attr_intern(const char *key, size_t len) {
+    Cache c;
+    TypedData_Get_Struct(attr_cache_obj, struct _cache, &oj_cache_type, c);
+    return cache_intern(c, key, len);
+}
+
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
+
+    while (key < endless) {
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
+
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
+    }
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
+
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
+    }
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
+}
+
+static VALUE resolve_classname(VALUE mod, const char *classname, int auto_define) {
+    VALUE clas;
+    ID    ci = rb_intern(classname);
+
+    if (rb_const_defined_at(mod, ci)) {
+        clas = rb_const_get_at(mod, ci);
+    } else if (auto_define) {
+        clas = rb_define_class_under(mod, classname, oj_bag_class);
+    } else {
+        clas = Qundef;
+    }
+    return clas;
+}
+
+static VALUE resolve_classpath(ParseInfo pi, const char *name, size_t len, int auto_define, VALUE error_class) {
+    char        class_name[1024];
+    VALUE       clas;
+    char       *end = class_name + sizeof(class_name) - 1;
+    char       *s;
+    const char *n    = name;
+    size_t      nlen = len;
+
+    clas = rb_cObject;
+    for (s = class_name; 0 < len; n++, len--) {
+        if (':' == *n) {
+            *s = '\0';
+            n++;
+            len--;
+            if (':' != *n) {
+                return Qundef;
+            }
+            if (Qundef == (clas = resolve_classname(clas, class_name, auto_define))) {
+                return Qundef;
+            }
+            s = class_name;
+        } else if (end <= s) {
+            return Qundef;
+        } else {
+            *s++ = *n;
+        }
+    }
+    *s = '\0';
+    if (Qundef == (clas = resolve_classname(clas, class_name, auto_define))) {
+        if (sizeof(class_name) <= nlen) {
+            nlen = sizeof(class_name) - 1;
+        }
+        strncpy(class_name, name, nlen);
+        class_name[nlen] = '\0';
+        oj_set_error_at(pi, error_class, __FILE__, __LINE__, "class '%s' is not defined", class_name);
+        if (Qnil != error_class) {
+            pi->err_class = error_class;
+        }
+    }
+    return clas;
+}
+
+VALUE oj_class_intern(const char *key, size_t len, bool safe, ParseInfo pi, int auto_define, VALUE error_class) {
+    uint64_t h      = hash_calc((const uint8_t *)key, len) & HASH_MASK;
+    KeyVal   bucket = class_hash.slots + h;
+    KeyVal   b;
+
+    if (safe) {
+#if HAVE_PTHREAD_MUTEX_INIT
+        pthread_mutex_lock(&class_hash.mutex);
+#else
+        rb_mutex_lock(class_hash.mutex);
+#endif
+        if (NULL != bucket->key) {  // not the top slot
+            for (b = bucket; 0 != b; b = b->next) {
+                if (len == b->len && 0 == strncmp(b->key, key, len)) {
+#if HAVE_PTHREAD_MUTEX_INIT
+                    pthread_mutex_unlock(&class_hash.mutex);
+#else
+                    rb_mutex_unlock(class_hash.mutex);
+#endif
+                    return b->val;
+                }
+                bucket = b;
+            }
+            b            = OJ_R_ALLOC(struct _keyVal);
+            b->next      = NULL;
+            bucket->next = b;
+            bucket       = b;
+        }
+        bucket->key = oj_strndup(key, len);
+        bucket->len = len;
+        bucket->val = resolve_classpath(pi, key, len, auto_define, error_class);
+#if HAVE_PTHREAD_MUTEX_INIT
+        pthread_mutex_unlock(&class_hash.mutex);
+#else
+        rb_mutex_unlock(class_hash.mutex);
+#endif
+    } else {
+        if (NULL != bucket->key) {
+            for (b = bucket; 0 != b; b = b->next) {
+                if (len == b->len && 0 == strncmp(b->key, key, len)) {
+                    return (ID)b->val;
+                }
+                bucket = b;
+            }
+            b            = OJ_R_ALLOC(struct _keyVal);
+            b->next      = NULL;
+            bucket->next = b;
+            bucket       = b;
+        }
+        bucket->key = oj_strndup(key, len);
+        bucket->len = len;
+        bucket->val = resolve_classpath(pi, key, len, auto_define, error_class);
+    }
+    rb_gc_register_mark_object(bucket->val);
+    return bucket->val;
+}
+
+char *oj_strndup(const char *s, size_t len) {
+    char *d = OJ_R_ALLOC_N(char, len + 1);
+
+    memcpy(d, s, len);
+    d[len] = '\0';
+
+    return d;
+}
+
+/*
+void intern_cleanup(void) {
+    cache_free(str_cache);
+    cache_free(sym_cache);
+    cache_free(attr_cache);
+}
+*/

data/ext/oj/intern.h

@@ -0,0 +1,22 @@
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
+
+#ifndef OJ_INTERN_H
+#define OJ_INTERN_H
+
+#include <ruby.h>
+#include <stdbool.h>
+
+struct _parseInfo;
+
+extern void oj_hash_init(void);
+
+extern VALUE oj_str_intern(const char *key, size_t len);
+extern VALUE oj_sym_intern(const char *key, size_t len);
+extern ID    oj_attr_intern(const char *key, size_t len);
+extern VALUE
+oj_class_intern(const char *key, size_t len, bool safe, struct _parseInfo *pi, int auto_define, VALUE error_class);
+
+extern char *oj_strndup(const char *s, size_t len);
+
+#endif /* OJ_INTERN_H */

data/ext/oj/mem.c

@@ -0,0 +1,318 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#include "mem.h"
+
+#include <pthread.h>
+#include <ruby.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct _rec {
+    struct _rec *next;
+    const void  *ptr;
+    size_t       size;
+    const char  *file;
+    int          line;
+    bool         ruby;
+} *Rec;
+
+typedef struct _rep {
+    struct _rep *next;
+    size_t       size;
+    const char  *file;
+    int          line;
+    int          cnt;
+} *Rep;
+
+#ifdef MEM_DEBUG
+
+static pthread_mutex_t lock      = PTHREAD_MUTEX_INITIALIZER;
+static Rec             recs      = NULL;
+static const char      mem_pad[] = "--- This is a memory pad and should not change until being freed. ---";
+
+void *oj_malloc(size_t size, const char *file, int line) {
+    void *ptr = malloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = false;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+void *oj_realloc(void *orig, size_t size, const char *file, int line) {
+    void *ptr = realloc(orig, size + sizeof(mem_pad));
+    Rec   r;
+
+    if (NULL != ptr) {
+        strcpy(((char *)ptr) + size, mem_pad);
+        pthread_mutex_lock(&lock);
+        for (r = recs; NULL != r; r = r->next) {
+            if (orig == r->ptr) {
+                r->ptr  = ptr;
+                r->size = size;
+                r->file = file;
+                r->line = line;
+                r->ruby = false;
+                break;
+            }
+        }
+        pthread_mutex_unlock(&lock);
+        if (NULL == r) {
+            printf("Realloc at %s:%d (%p) not allocated.\n", file, line, orig);
+        }
+    }
+    return ptr;
+}
+
+void *oj_calloc(size_t count, size_t size, const char *file, int line) {
+    void *ptr;
+
+    size *= count;
+    if (NULL != (ptr = malloc(size + sizeof(mem_pad)))) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            memset(ptr, 0, size);
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = false;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+void *oj_r_alloc(size_t size, const char *file, int line) {
+    void *ptr = ruby_xmalloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = true;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+void *oj_r_realloc(void *orig, size_t size, const char *file, int line) {
+    void *ptr = ruby_xrealloc2(orig, 1, size + sizeof(mem_pad));
+    Rec   r;
+
+    if (NULL != ptr) {
+        strcpy(((char *)ptr) + size, mem_pad);
+        pthread_mutex_lock(&lock);
+        for (r = recs; NULL != r; r = r->next) {
+            if (orig == r->ptr) {
+                r->ptr  = ptr;
+                r->size = size;
+                r->file = file;
+                r->line = line;
+                r->ruby = true;
+                break;
+            }
+        }
+        pthread_mutex_unlock(&lock);
+        if (NULL == r) {
+            printf("Realloc at %s:%d (%p) not allocated.\n", file, line, orig);
+        }
+    }
+    return ptr;
+}
+
+void oj_freed(void *ptr, const char *file, int line, bool ruby) {
+    if (NULL != ptr) {
+        Rec r    = NULL;
+        Rec prev = NULL;
+
+        pthread_mutex_lock(&lock);
+        for (r = recs; NULL != r; r = r->next) {
+            if (ptr == r->ptr) {
+                if (NULL == prev) {
+                    recs = r->next;
+                } else {
+                    prev->next = r->next;
+                }
+                break;
+            }
+            prev = r;
+        }
+        pthread_mutex_unlock(&lock);
+        if (NULL == r) {
+            printf("Free at %s:%d (%p) not allocated or already freed.\n", file, line, ptr);
+        } else {
+            char *pad = (char *)r->ptr + r->size;
+
+            if (r->ruby != ruby) {
+                if (r->ruby) {
+                    printf("Memory at %s:%d (%p) allocated with Ruby allocator and freed with stdlib free.\n",
+                           file,
+                           line,
+                           ptr);
+                } else {
+                    printf("Memory at %s:%d (%p) allocated with stdlib allocator and freed with Ruby free.\n",
+                           file,
+                           line,
+                           ptr);
+                }
+            }
+            if (0 != strcmp(mem_pad, pad)) {
+                uint8_t *p;
+                uint8_t *end = (uint8_t *)pad + sizeof(mem_pad);
+
+                printf("Memory at %s:%d (%p) write outside allocated.\n", file, line, ptr);
+                for (p = (uint8_t *)pad; p < end; p++) {
+                    if (0x20 < *p && *p < 0x7f) {
+                        printf("%c  ", *p);
+                    } else {
+                        printf("%02x ", *(uint8_t *)p);
+                    }
+                }
+                printf("\n");
+            }
+            free(r);
+        }
+    }
+}
+
+void oj_r_free(void *ptr, const char *file, int line) {
+    oj_freed(ptr, file, line, true);
+    xfree(ptr);
+}
+
+void oj_free(void *ptr, const char *file, int line) {
+    oj_freed(ptr, file, line, false);
+    free(ptr);
+}
+
+char *oj_mem_strdup(const char *str, const char *file, int line) {
+    size_t size = strlen(str) + 1;
+    char  *ptr  = (char *)malloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            strcpy(ptr, str);
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = (void *)ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = false;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+#endif
+
+#ifdef MEM_DEBUG
+
+static Rep update_reps(Rep reps, Rec r) {
+    Rep rp = reps;
+
+    for (; NULL != rp; rp = rp->next) {
+        if (rp->line == r->line && (rp->file == r->file || 0 == strcmp(rp->file, r->file))) {
+            rp->size += r->size;
+            rp->cnt++;
+            break;
+        }
+    }
+    if (NULL == rp && NULL != (rp = (Rep)malloc(sizeof(struct _rep)))) {
+        rp->size = r->size;
+        rp->file = r->file;
+        rp->line = r->line;
+        rp->cnt  = 1;
+        rp->next = reps;
+        reps     = rp;
+    }
+    return reps;
+}
+
+static void print_stats() {
+    printf("\n--- Memory Usage Report --------------------------------------------------------\n");
+    pthread_mutex_lock(&lock);
+
+    if (NULL == recs) {
+        printf("No memory leaks\n");
+    } else {
+        Rep    reps = NULL;
+        Rep    rp;
+        Rec    r;
+        size_t leaked = 0;
+
+        for (r = recs; NULL != r; r = r->next) {
+            reps = update_reps(reps, r);
+        }
+        while (NULL != (rp = reps)) {
+            reps = rp->next;
+            printf("%16s:%3d %8lu bytes over %d occurances allocated and not freed.\n",
+                   rp->file,
+                   rp->line,
+                   rp->size,
+                   rp->cnt);
+            leaked += rp->size;
+            free(rp);
+        }
+        printf("%lu bytes leaked\n", leaked);
+    }
+    pthread_mutex_unlock(&lock);
+    printf("--------------------------------------------------------------------------------\n");
+}
+
+#endif
+
+void oj_mem_report(void) {
+#ifdef MEM_DEBUG
+    rb_gc();
+    print_stats();
+#endif
+}

data/ext/oj/mem.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#ifndef OJ_MEM_H
+#define OJ_MEM_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef MEM_DEBUG
+
+#define OJ_MALLOC(size) oj_malloc(size, __FILE__, __LINE__)
+#define OJ_REALLOC(ptr, size) oj_realloc(ptr, size, __FILE__, __LINE__)
+#define OJ_CALLOC(count, size) oj_calloc(count, size, __FILE__, __LINE__)
+#define OJ_FREE(ptr) oj_free(ptr, __FILE__, __LINE__)
+
+#define OJ_R_ALLOC(type) oj_r_alloc(sizeof(type), __FILE__, __LINE__)
+#define OJ_R_ALLOC_N(type, n) (type *)oj_r_alloc(sizeof(type) * (n), __FILE__, __LINE__)
+#define OJ_R_REALLOC_N(ptr, type, n) ((ptr) = (type *)oj_r_realloc(ptr, (sizeof(type) * (n)), __FILE__, __LINE__))
+#define OJ_R_FREE(ptr) oj_r_free(ptr, __FILE__, __LINE__)
+
+#define OJ_STRDUP(str) oj_mem_strdup(str, __FILE__, __LINE__)
+
+extern void *oj_malloc(size_t size, const char *file, int line);
+extern void *oj_realloc(void *ptr, size_t size, const char *file, int line);
+extern void *oj_calloc(size_t count, size_t size, const char *file, int line);
+extern void  oj_free(void *ptr, const char *file, int line);
+
+extern void *oj_r_alloc(size_t size, const char *file, int line);
+extern void *oj_r_realloc(void *ptr, size_t size, const char *file, int line);
+extern void  oj_r_free(void *ptr, const char *file, int line);
+
+extern char *oj_mem_strdup(const char *str, const char *file, int line);
+
+#else
+
+#define OJ_MALLOC(size) malloc(size)
+#define OJ_REALLOC(ptr, size) realloc(ptr, size)
+#define OJ_CALLOC(count, size) calloc(count, size)
+#define OJ_FREE(ptr) free(ptr)
+
+#define OJ_R_ALLOC(type) RB_ALLOC(type)
+#define OJ_R_ALLOC_N(type, n) RB_ALLOC_N(type, n)
+#define OJ_R_REALLOC_N(ptr, type, n) RB_REALLOC_N(ptr, type, n)
+#define OJ_R_FREE(ptr) xfree(ptr)
+
+#define OJ_STRDUP(str) strdup(str)
+
+#endif
+
+extern void oj_mem_report();
+
+#endif /* OJ_MEM_H */