oj 3.13.11 → 3.15.0

data/ext/oj/mem.c

@@ -0,0 +1,318 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#include "mem.h"
+
+#include <pthread.h>
+#include <ruby.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct _rec {
+    struct _rec *next;
+    const void  *ptr;
+    size_t       size;
+    const char  *file;
+    int          line;
+    bool         ruby;
+} *Rec;
+
+typedef struct _rep {
+    struct _rep *next;
+    size_t       size;
+    const char  *file;
+    int          line;
+    int          cnt;
+} *Rep;
+
+#ifdef MEM_DEBUG
+
+static pthread_mutex_t lock      = PTHREAD_MUTEX_INITIALIZER;
+static Rec             recs      = NULL;
+static const char      mem_pad[] = "--- This is a memory pad and should not change until being freed. ---";
+
+void *oj_malloc(size_t size, const char *file, int line) {
+    void *ptr = malloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = false;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+void *oj_realloc(void *orig, size_t size, const char *file, int line) {
+    void *ptr = realloc(orig, size + sizeof(mem_pad));
+    Rec   r;
+
+    if (NULL != ptr) {
+        strcpy(((char *)ptr) + size, mem_pad);
+        pthread_mutex_lock(&lock);
+        for (r = recs; NULL != r; r = r->next) {
+            if (orig == r->ptr) {
+                r->ptr  = ptr;
+                r->size = size;
+                r->file = file;
+                r->line = line;
+                r->ruby = false;
+                break;
+            }
+        }
+        pthread_mutex_unlock(&lock);
+        if (NULL == r) {
+            printf("Realloc at %s:%d (%p) not allocated.\n", file, line, orig);
+        }
+    }
+    return ptr;
+}
+
+void *oj_calloc(size_t count, size_t size, const char *file, int line) {
+    void *ptr;
+
+    size *= count;
+    if (NULL != (ptr = malloc(size + sizeof(mem_pad)))) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            memset(ptr, 0, size);
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = false;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+void *oj_r_alloc(size_t size, const char *file, int line) {
+    void *ptr = ruby_xmalloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = true;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+void *oj_r_realloc(void *orig, size_t size, const char *file, int line) {
+    void *ptr = ruby_xrealloc2(orig, 1, size + sizeof(mem_pad));
+    Rec   r;
+
+    if (NULL != ptr) {
+        strcpy(((char *)ptr) + size, mem_pad);
+        pthread_mutex_lock(&lock);
+        for (r = recs; NULL != r; r = r->next) {
+            if (orig == r->ptr) {
+                r->ptr  = ptr;
+                r->size = size;
+                r->file = file;
+                r->line = line;
+                r->ruby = true;
+                break;
+            }
+        }
+        pthread_mutex_unlock(&lock);
+        if (NULL == r) {
+            printf("Realloc at %s:%d (%p) not allocated.\n", file, line, orig);
+        }
+    }
+    return ptr;
+}
+
+void oj_freed(void *ptr, const char *file, int line, bool ruby) {
+    if (NULL != ptr) {
+        Rec r    = NULL;
+        Rec prev = NULL;
+
+        pthread_mutex_lock(&lock);
+        for (r = recs; NULL != r; r = r->next) {
+            if (ptr == r->ptr) {
+                if (NULL == prev) {
+                    recs = r->next;
+                } else {
+                    prev->next = r->next;
+                }
+                break;
+            }
+            prev = r;
+        }
+        pthread_mutex_unlock(&lock);
+        if (NULL == r) {
+            printf("Free at %s:%d (%p) not allocated or already freed.\n", file, line, ptr);
+        } else {
+            char *pad = (char *)r->ptr + r->size;
+
+            if (r->ruby != ruby) {
+                if (r->ruby) {
+                    printf("Memory at %s:%d (%p) allocated with Ruby allocator and freed with stdlib free.\n",
+                           file,
+                           line,
+                           ptr);
+                } else {
+                    printf("Memory at %s:%d (%p) allocated with stdlib allocator and freed with Ruby free.\n",
+                           file,
+                           line,
+                           ptr);
+                }
+            }
+            if (0 != strcmp(mem_pad, pad)) {
+                uint8_t *p;
+                uint8_t *end = (uint8_t *)pad + sizeof(mem_pad);
+
+                printf("Memory at %s:%d (%p) write outside allocated.\n", file, line, ptr);
+                for (p = (uint8_t *)pad; p < end; p++) {
+                    if (0x20 < *p && *p < 0x7f) {
+                        printf("%c  ", *p);
+                    } else {
+                        printf("%02x ", *(uint8_t *)p);
+                    }
+                }
+                printf("\n");
+            }
+            free(r);
+        }
+    }
+}
+
+void oj_r_free(void *ptr, const char *file, int line) {
+    oj_freed(ptr, file, line, true);
+    xfree(ptr);
+}
+
+void oj_free(void *ptr, const char *file, int line) {
+    oj_freed(ptr, file, line, false);
+    free(ptr);
+}
+
+char *oj_mem_strdup(const char *str, const char *file, int line) {
+    size_t size = strlen(str) + 1;
+    char  *ptr  = (char *)malloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+        Rec r = (Rec)malloc(sizeof(struct _rec));
+
+        if (NULL != r) {
+            strcpy(ptr, str);
+            strcpy(((char *)ptr) + size, mem_pad);
+            r->ptr  = (void *)ptr;
+            r->size = size;
+            r->file = file;
+            r->line = line;
+            r->ruby = false;
+            pthread_mutex_lock(&lock);
+            r->next = recs;
+            recs    = r;
+            pthread_mutex_unlock(&lock);
+        } else {
+            free(ptr);
+            ptr = NULL;
+        }
+    }
+    return ptr;
+}
+
+#endif
+
+#ifdef MEM_DEBUG
+
+static Rep update_reps(Rep reps, Rec r) {
+    Rep rp = reps;
+
+    for (; NULL != rp; rp = rp->next) {
+        if (rp->line == r->line && (rp->file == r->file || 0 == strcmp(rp->file, r->file))) {
+            rp->size += r->size;
+            rp->cnt++;
+            break;
+        }
+    }
+    if (NULL == rp && NULL != (rp = (Rep)malloc(sizeof(struct _rep)))) {
+        rp->size = r->size;
+        rp->file = r->file;
+        rp->line = r->line;
+        rp->cnt  = 1;
+        rp->next = reps;
+        reps     = rp;
+    }
+    return reps;
+}
+
+static void print_stats() {
+    printf("\n--- Memory Usage Report --------------------------------------------------------\n");
+    pthread_mutex_lock(&lock);
+
+    if (NULL == recs) {
+        printf("No memory leaks\n");
+    } else {
+        Rep    reps = NULL;
+        Rep    rp;
+        Rec    r;
+        size_t leaked = 0;
+
+        for (r = recs; NULL != r; r = r->next) {
+            reps = update_reps(reps, r);
+        }
+        while (NULL != (rp = reps)) {
+            reps = rp->next;
+            printf("%16s:%3d %8lu bytes over %d occurances allocated and not freed.\n",
+                   rp->file,
+                   rp->line,
+                   rp->size,
+                   rp->cnt);
+            leaked += rp->size;
+            free(rp);
+        }
+        printf("%lu bytes leaked\n", leaked);
+    }
+    pthread_mutex_unlock(&lock);
+    printf("--------------------------------------------------------------------------------\n");
+}
+
+#endif
+
+void oj_mem_report(void) {
+#ifdef MEM_DEBUG
+    rb_gc();
+    print_stats();
+#endif
+}

data/ext/oj/mem.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#ifndef OJ_MEM_H
+#define OJ_MEM_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef MEM_DEBUG
+
+#define OJ_MALLOC(size) oj_malloc(size, __FILE__, __LINE__)
+#define OJ_REALLOC(ptr, size) oj_realloc(ptr, size, __FILE__, __LINE__)
+#define OJ_CALLOC(count, size) oj_calloc(count, size, __FILE__, __LINE__)
+#define OJ_FREE(ptr) oj_free(ptr, __FILE__, __LINE__)
+
+#define OJ_R_ALLOC(type) oj_r_alloc(sizeof(type), __FILE__, __LINE__)
+#define OJ_R_ALLOC_N(type, n) (type *)oj_r_alloc(sizeof(type) * (n), __FILE__, __LINE__)
+#define OJ_R_REALLOC_N(ptr, type, n) ((ptr) = (type *)oj_r_realloc(ptr, (sizeof(type) * (n)), __FILE__, __LINE__))
+#define OJ_R_FREE(ptr) oj_r_free(ptr, __FILE__, __LINE__)
+
+#define OJ_STRDUP(str) oj_mem_strdup(str, __FILE__, __LINE__)
+
+extern void *oj_malloc(size_t size, const char *file, int line);
+extern void *oj_realloc(void *ptr, size_t size, const char *file, int line);
+extern void *oj_calloc(size_t count, size_t size, const char *file, int line);
+extern void  oj_free(void *ptr, const char *file, int line);
+
+extern void *oj_r_alloc(size_t size, const char *file, int line);
+extern void *oj_r_realloc(void *ptr, size_t size, const char *file, int line);
+extern void  oj_r_free(void *ptr, const char *file, int line);
+
+extern char *oj_mem_strdup(const char *str, const char *file, int line);
+
+#else
+
+#define OJ_MALLOC(size) malloc(size)
+#define OJ_REALLOC(ptr, size) realloc(ptr, size)
+#define OJ_CALLOC(count, size) calloc(count, size)
+#define OJ_FREE(ptr) free(ptr)
+
+#define OJ_R_ALLOC(type) RB_ALLOC(type)
+#define OJ_R_ALLOC_N(type, n) RB_ALLOC_N(type, n)
+#define OJ_R_REALLOC_N(ptr, type, n) RB_REALLOC_N(ptr, type, n)
+#define OJ_R_FREE(ptr) xfree(ptr)
+
+#define OJ_STRDUP(str) strdup(str)
+
+#endif
+
+extern void oj_mem_report();
+
+#endif /* OJ_MEM_H */

data/ext/oj/mimic_json.c

@@ -3,6 +3,7 @@
 
 #include "dump.h"
 #include "encode.h"
+#include "mem.h"
 #include "oj.h"
 #include "parse.h"
 
@@ -198,7 +199,6 @@ static int mimic_limit_arg(VALUE a) {
  * Returns [_String_] a JSON string.
  */
 static VALUE mimic_dump(int argc, VALUE *argv, VALUE self) {
-    char            buf[4096];
     struct _out     out;
     struct _options copts = oj_default_options;
     VALUE           rstr;
@@ -206,9 +206,9 @@ static VALUE mimic_dump(int argc, VALUE *argv, VALUE self) {
 
     copts.str_rx.head = NULL;
     copts.str_rx.tail = NULL;
-    out.buf           = buf;
-    out.end           = buf + sizeof(buf) - 10;
-    out.allocated     = false;
+
+    oj_out_init(&out);
+
     out.caller        = CALLER_DUMP;
     copts.escape_mode = JXEsc;
     copts.mode        = CompatMode;
@@ -257,9 +257,9 @@ static VALUE mimic_dump(int argc, VALUE *argv, VALUE self) {
         rb_funcall2(io, oj_write_id, 1, args);
         rstr = io;
     }
-    if (out.allocated) {
-        xfree(out.buf);
-    }
+
+    oj_out_free(&out);
+
     return rstr;
 }
 
@@ -349,26 +349,26 @@ static VALUE mimic_load(int argc, VALUE *argv, VALUE self) {
 static VALUE mimic_dump_load(int argc, VALUE *argv, VALUE self) {
     if (1 > argc) {
         rb_raise(rb_eArgError, "wrong number of arguments (0 for 1)");
-    } else if (T_STRING == rb_type(*argv)) {
+    }
+    if (T_STRING == rb_type(*argv)) {
         return mimic_load(argc, argv, self);
-    } else {
-        return mimic_dump(argc, argv, self);
     }
-    return Qnil;
+    return mimic_dump(argc, argv, self);
 }
 
 static VALUE mimic_generate_core(int argc, VALUE *argv, Options copts) {
-    char        buf[4096];
     struct _out out;
     VALUE       rstr;
 
-    memset(buf, 0, sizeof(buf));
+    if (0 == argc) {
+        rb_raise(rb_eArgError, "wrong number of arguments (0))");
+    }
+    memset(out.stack_buffer, 0, sizeof(out.stack_buffer));
+
+    oj_out_init(&out);
 
-    out.buf       = buf;
-    out.end       = buf + sizeof(buf) - 10;
-    out.allocated = false;
-    out.omit_nil  = copts->dump_opts.omit_nil;
-    out.caller    = CALLER_GENERATE;
+    out.omit_nil = copts->dump_opts.omit_nil;
+    out.caller   = CALLER_GENERATE;
     // For obj.to_json or generate nan is not allowed but if called from dump
     // it is.
     copts->dump_opts.nan_dump = RaiseNan;
@@ -388,6 +388,8 @@ static VALUE mimic_generate_core(int argc, VALUE *argv, Options copts) {
         VALUE active_hack[1];
 
         if (Qundef == state_class) {
+            rb_warn("Oj::Rails.mimic_JSON was called implicitly. "
+                    "Call it explicitly beforehand if you want to remove this warning.");
             oj_define_mimic_json(0, NULL, Qnil);
         }
         active_hack[0] = rb_funcall(state_class, oj_new_id, 0);
@@ -398,9 +400,9 @@ static VALUE mimic_generate_core(int argc, VALUE *argv, Options copts) {
     }
     rstr = rb_str_new2(out.buf);
     rstr = oj_encode(rstr);
-    if (out.allocated) {
-        xfree(out.buf);
-    }
+
+    oj_out_free(&out);
+
     return rstr;
 }
 
@@ -457,7 +459,10 @@ oj_mimic_pretty_generate(int argc, VALUE *argv, VALUE self) {
     // a Hash. I haven't dug deep enough to find out why but using a State
     // instance and not a Hash gives the desired behavior.
     *rargs = *argv;
-    if (1 == argc) {
+    if (0 == argc) {
+        rb_raise(rb_eArgError, "wrong number of arguments (0))");
+    }
+    if (1 == argc || Qnil == argv[1]) {
         h = rb_hash_new();
     } else {
         h = argv[1];
@@ -478,6 +483,8 @@ oj_mimic_pretty_generate(int argc, VALUE *argv, VALUE self) {
         rb_hash_aset(h, oj_array_nl_sym, rb_str_new2("\n"));
     }
     if (Qundef == state_class) {
+        rb_warn("Oj::Rails.mimic_JSON was called implicitly. "
+                "Call it explicitly beforehand if you want to remove this warning.");
         oj_define_mimic_json(0, NULL, Qnil);
     }
     rargs[1] = rb_funcall(state_class, oj_new_id, 1, h);
@@ -533,14 +540,6 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE info) {
         }
     } else if (oj_decimal_class_sym == k) {
         pi->options.compat_bigdec = (oj_bigdecimal_class == v);
-    } else if (oj_max_nesting_sym == k) {
-        if (Qtrue == v) {
-            pi->max_depth = 100;
-        } else if (Qfalse == v || Qnil == v) {
-            pi->max_depth = 0;
-        } else if (T_FIXNUM == rb_type(v)) {
-            pi->max_depth = NUM2INT(v);
-        }
     }
     return ST_CONTINUE;
 }
@@ -570,11 +569,21 @@ static VALUE mimic_parse_core(int argc, VALUE *argv, VALUE self, bool bang) {
     pi.max_depth             = 100;
 
     if (Qnil != ropts) {
+        VALUE v;
+
         if (T_HASH != rb_type(ropts)) {
             rb_raise(rb_eArgError, "options must be a hash.");
         }
 
         rb_hash_foreach(ropts, parse_options_cb, (VALUE)&pi);
+        v = rb_hash_lookup(ropts, oj_max_nesting_sym);
+        if (Qtrue == v) {
+            pi.max_depth = 100;
+        } else if (Qfalse == v || Qnil == v) {
+            pi.max_depth = 0;
+        } else if (T_FIXNUM == rb_type(v)) {
+            pi.max_depth = NUM2INT(v);
+        }
         oj_parse_opt_match_string(&pi.options.str_rx, ropts);
         if (Yes == pi.options.create_ok && Yes == pi.options.sym_key) {
             rb_raise(rb_eArgError, ":symbolize_names and :create_additions can not both be true.");
@@ -644,23 +653,22 @@ static VALUE mimic_recurse_proc(VALUE self, VALUE obj) {
  *
  * - *id* [_nil_|String] new create_id
  *
- * Returns [_String_] the id.
+ * Returns [_nil_|_String_] the id.
  */
 static VALUE mimic_set_create_id(VALUE self, VALUE id) {
-    Check_Type(id, T_STRING);
-
     if (NULL != oj_default_options.create_id) {
         if (oj_json_class != oj_default_options.create_id) {
-            xfree((char *)oj_default_options.create_id);
+            OJ_R_FREE((char *)oj_default_options.create_id);
         }
         oj_default_options.create_id     = NULL;
         oj_default_options.create_id_len = 0;
     }
     if (Qnil != id) {
-        size_t len = RSTRING_LEN(id) + 1;
+        const char *ptr = StringValueCStr(id);
+        size_t      len = RSTRING_LEN(id) + 1;
 
-        oj_default_options.create_id = ALLOC_N(char, len);
-        strcpy((char *)oj_default_options.create_id, StringValuePtr(id));
+        oj_default_options.create_id = OJ_R_ALLOC_N(char, len);
+        strcpy((char *)oj_default_options.create_id, ptr);
         oj_default_options.create_id_len = len - 1;
     }
     return id;
@@ -730,6 +738,7 @@ static struct _options mimic_object_to_json_options = {0,              // indent
                                                            0,         // array_size
                                                            RaiseNan,  // nan_dump
                                                            false,     // omit_nil
+                                                           false,     // omit_null_byte
                                                            100,       // max_depth
                                                        },
                                                        {
@@ -740,19 +749,18 @@ static struct _options mimic_object_to_json_options = {0,              // indent
                                                        }};
 
 static VALUE mimic_object_to_json(int argc, VALUE *argv, VALUE self) {
-    char            buf[4096];
     struct _out     out;
     VALUE           rstr;
     struct _options copts = oj_default_options;
 
     copts.str_rx.head = NULL;
     copts.str_rx.tail = NULL;
-    out.buf           = buf;
-    out.end           = buf + sizeof(buf) - 10;
-    out.allocated     = false;
-    out.omit_nil      = copts.dump_opts.omit_nil;
-    copts.mode        = CompatMode;
-    copts.to_json     = No;
+
+    oj_out_init(&out);
+
+    out.omit_nil  = copts.dump_opts.omit_nil;
+    copts.mode    = CompatMode;
+    copts.to_json = No;
     if (1 <= argc && Qnil != argv[0]) {
         oj_parse_mimic_dump_options(argv[0], &copts);
     }
@@ -765,9 +773,9 @@ static VALUE mimic_object_to_json(int argc, VALUE *argv, VALUE self) {
     }
     rstr = rb_str_new2(out.buf);
     rstr = oj_encode(rstr);
-    if (out.allocated) {
-        xfree(out.buf);
-    }
+
+    oj_out_free(&out);
+
     return rstr;
 }
 
@@ -784,28 +792,48 @@ void oj_mimic_json_methods(VALUE json) {
     VALUE json_error;
     VALUE generator;
     VALUE ext;
+    VALUE verbose;
 
+    // rb_undef_method doesn't work for modules or maybe sometimes
+    // doesn't. Anyway setting verbose should hide the warning.
+    verbose = rb_gv_get("$VERBOSE");
+    rb_gv_set("$VERBOSE", Qfalse);
+
+    rb_undef_method(json, "create_id=");
     rb_define_module_function(json, "create_id=", mimic_set_create_id, 1);
+    rb_undef_method(json, "create_id");
     rb_define_module_function(json, "create_id", mimic_create_id, 0);
 
+    rb_undef_method(json, "dump");
     rb_define_module_function(json, "dump", mimic_dump, -1);
+    rb_undef_method(json, "load");
     rb_define_module_function(json, "load", mimic_load, -1);
     rb_define_module_function(json, "restore", mimic_load, -1);
+    rb_undef_method(json, "recurse_proc");
     rb_define_module_function(json, "recurse_proc", mimic_recurse_proc, 1);
+    rb_undef_method(json, "[]");
     rb_define_module_function(json, "[]", mimic_dump_load, -1);
 
+    rb_undef_method(json, "generate");
     rb_define_module_function(json, "generate", oj_mimic_generate, -1);
+    rb_undef_method(json, "fast_generate");
     rb_define_module_function(json, "fast_generate", oj_mimic_generate, -1);
+    rb_undef_method(json, "pretty_generate");
     rb_define_module_function(json, "pretty_generate", oj_mimic_pretty_generate, -1);
     // For older versions of JSON, the deprecated unparse methods.
+    rb_undef_method(json, "unparse");
     rb_define_module_function(json, "unparse", oj_mimic_generate, -1);
     rb_define_module_function(json, "fast_unparse", oj_mimic_generate, -1);
     rb_define_module_function(json, "pretty_unparse", oj_mimic_pretty_generate, -1);
 
+    rb_undef_method(json, "parse");
     rb_define_module_function(json, "parse", oj_mimic_parse, -1);
+    rb_undef_method(json, "parse!");
     rb_define_module_function(json, "parse!", mimic_parse_bang, -1);
 
+    rb_undef_method(json, "state");
     rb_define_module_function(json, "state", mimic_state, 0);
+    rb_gv_set("$VERBOSE", verbose);
 
     if (rb_const_defined_at(json, rb_intern("JSONError"))) {
         json_error = rb_const_get(json, rb_intern("JSONError"));