oj 3.13.23 → 3.14.2

data/ext/oj/mem.c

@@ -0,0 +1,324 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <ruby.h>
+
+#include "mem.h"
+
+typedef struct _rec {
+    struct _rec	*next;
+    const void	*ptr;
+    size_t	size;
+    const char	*file;
+    int		line;
+    bool        ruby;
+} *Rec;
+
+typedef struct _rep {
+    struct _rep	*next;
+    size_t	size;
+    const char	*file;
+    int		line;
+    int		cnt;
+} *Rep;
+
+#ifdef MEM_DEBUG
+
+static pthread_mutex_t	lock = PTHREAD_MUTEX_INITIALIZER;
+static Rec		recs = NULL;
+static const char	mem_pad[] = "--- This is a memory pad and should not change until being freed. ---";
+
+void*
+oj_malloc(size_t size, const char *file, int line) {
+    void	*ptr = malloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+	Rec	r = (Rec)malloc(sizeof(struct _rec));
+
+	if (NULL != r) {
+	    strcpy(((char*)ptr) + size, mem_pad);
+	    r->ptr = ptr;
+	    r->size = size;
+	    r->file = file;
+	    r->line = line;
+            r->ruby = false;
+	    pthread_mutex_lock(&lock);
+	    r->next = recs;
+	    recs = r;
+	    pthread_mutex_unlock(&lock);
+	} else {
+	    free(ptr);
+	    ptr = NULL;
+	}
+    }
+    return ptr;
+}
+
+void*
+oj_realloc(void *orig, size_t size, const char *file, int line) {
+    void	*ptr = realloc(orig, size + sizeof(mem_pad));
+    Rec		r;
+
+    if (NULL != ptr) {
+	strcpy(((char*)ptr) + size, mem_pad);
+	pthread_mutex_lock(&lock);
+	for (r = recs; NULL != r; r = r->next) {
+	    if (orig == r->ptr) {
+		r->ptr = ptr;
+		r->size = size;
+		r->file = file;
+		r->line = line;
+                r->ruby = false;
+		break;
+	    }
+	}
+	pthread_mutex_unlock(&lock);
+	if (NULL == r) {
+	    printf("Realloc at %s:%d (%p) not allocated.\n", file, line, orig);
+	}
+    }
+    return ptr;
+}
+
+void*
+oj_calloc(size_t count, size_t size, const char *file, int line) {
+    void	*ptr;
+
+    size *= count;
+    if (NULL != (ptr = malloc(size + sizeof(mem_pad)))) {
+	Rec	r = (Rec)malloc(sizeof(struct _rec));
+
+	if (NULL != r) {
+	    memset(ptr, 0, size);
+	    strcpy(((char*)ptr) + size, mem_pad);
+	    r->ptr = ptr;
+	    r->size = size;
+	    r->file = file;
+	    r->line = line;
+            r->ruby = false;
+	    pthread_mutex_lock(&lock);
+	    r->next = recs;
+	    recs = r;
+	    pthread_mutex_unlock(&lock);
+	} else {
+	    free(ptr);
+	    ptr = NULL;
+	}
+    }
+    return ptr;
+}
+
+void*
+oj_r_alloc(size_t size, const char *file, int line) {
+    void	*ptr = ruby_xmalloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+	Rec	r = (Rec)malloc(sizeof(struct _rec));
+
+	if (NULL != r) {
+	    strcpy(((char*)ptr) + size, mem_pad);
+	    r->ptr = ptr;
+	    r->size = size;
+	    r->file = file;
+	    r->line = line;
+            r->ruby = true;
+	    pthread_mutex_lock(&lock);
+	    r->next = recs;
+	    recs = r;
+	    pthread_mutex_unlock(&lock);
+	} else {
+	    free(ptr);
+	    ptr = NULL;
+	}
+    }
+    return ptr;
+}
+
+void*
+oj_r_realloc(void *orig, size_t size, const char *file, int line) {
+    void	*ptr = ruby_xrealloc2(orig, 1, size + sizeof(mem_pad));
+    Rec		r;
+
+    if (NULL != ptr) {
+	strcpy(((char*)ptr) + size, mem_pad);
+	pthread_mutex_lock(&lock);
+	for (r = recs; NULL != r; r = r->next) {
+	    if (orig == r->ptr) {
+		r->ptr = ptr;
+		r->size = size;
+		r->file = file;
+		r->line = line;
+                r->ruby = true;
+		break;
+	    }
+	}
+	pthread_mutex_unlock(&lock);
+	if (NULL == r) {
+	    printf("Realloc at %s:%d (%p) not allocated.\n", file, line, orig);
+	}
+    }
+    return ptr;
+}
+
+
+void
+oj_freed(void *ptr, const char *file, int line, bool ruby) {
+    if (NULL != ptr) {
+	Rec	r = NULL;
+	Rec	prev = NULL;
+
+	pthread_mutex_lock(&lock);
+	for (r = recs; NULL != r; r = r->next) {
+	    if (ptr == r->ptr) {
+		if (NULL == prev) {
+		    recs = r->next;
+		} else {
+		    prev->next = r->next;
+		}
+		break;
+	    }
+	    prev = r;
+	}
+	pthread_mutex_unlock(&lock);
+	if (NULL == r) {
+	    printf("Free at %s:%d (%p) not allocated or already freed.\n", file, line, ptr);
+	} else {
+	    char	*pad = (char*)r->ptr + r->size;
+
+            if (r->ruby != ruby) {
+                if (r->ruby) {
+                    printf("Memory at %s:%d (%p) allocated with Ruby allocator and freed with stdlib free.\n", file, line, ptr);
+                } else {
+                    printf("Memory at %s:%d (%p) allocated with stdlib allocator and freed with Ruby free.\n", file, line, ptr);
+                }
+            }
+	    if (0 != strcmp(mem_pad, pad)) {
+		uint8_t	*p;
+		uint8_t	*end = (uint8_t*)pad + sizeof(mem_pad);
+
+		printf("Memory at %s:%d (%p) write outside allocated.\n", file, line, ptr);
+		for (p = (uint8_t*)pad; p < end; p++) {
+		    if (0x20 < *p && *p < 0x7f) {
+			printf("%c  ", *p);
+		    } else {
+			printf("%02x ", *(uint8_t*)p);
+		    }
+		}
+		printf("\n");
+	    }
+	    free(r);
+	}
+    }
+}
+
+void
+oj_r_free(void *ptr, const char *file, int line) {
+    oj_freed(ptr, file, line, true);
+    xfree(ptr);
+}
+
+
+void
+oj_free(void *ptr, const char *file, int line) {
+    oj_freed(ptr, file, line, false);
+    free(ptr);
+}
+
+char*
+oj_mem_strdup(const char *str, const char *file, int line) {
+    size_t	size = strlen(str) + 1;
+    char	*ptr = (char*)malloc(size + sizeof(mem_pad));
+
+    if (NULL != ptr) {
+	Rec	r = (Rec)malloc(sizeof(struct _rec));
+
+	if (NULL != r) {
+	    strcpy(ptr, str);
+	    strcpy(((char*)ptr) + size, mem_pad);
+	    r->ptr = (void*)ptr;
+	    r->size = size;
+	    r->file = file;
+	    r->line = line;
+            r->ruby = false;
+	    pthread_mutex_lock(&lock);
+	    r->next = recs;
+	    recs = r;
+	    pthread_mutex_unlock(&lock);
+	} else {
+	    free(ptr);
+	    ptr = NULL;
+	}
+    }
+    return ptr;
+}
+
+#endif
+
+#ifdef MEM_DEBUG
+
+static Rep
+update_reps(Rep reps, Rec r) {
+    Rep	rp = reps;
+
+    for (; NULL != rp; rp = rp->next) {
+	if (rp->line == r->line && (rp->file == r->file || 0 == strcmp(rp->file, r->file))) {
+	    rp->size += r->size;
+	    rp->cnt++;
+	    break;
+	}
+    }
+    if (NULL == rp &&
+	NULL != (rp = (Rep)malloc(sizeof(struct _rep)))) {
+	rp->size = r->size;
+	rp->file = r->file;
+	rp->line = r->line;
+	rp->cnt = 1;
+	rp->next = reps;
+	reps = rp;
+    }
+    return reps;
+}
+
+static void
+print_stats() {
+    printf("\n--- Memory Usage Report --------------------------------------------------------\n");
+    pthread_mutex_lock(&lock);
+
+    if (NULL == recs) {
+	printf("No memory leaks\n");
+    } else {
+	Rep	reps = NULL;
+	Rep	rp;
+	Rec	r;
+	size_t	leaked = 0;
+
+	for (r = recs; NULL != r; r = r->next) {
+	    reps = update_reps(reps, r);
+	}
+	while (NULL != (rp = reps)) {
+	    reps = rp->next;
+	    printf("%16s:%3d %8lu bytes over %d occurances allocated and not freed.\n", rp->file, rp->line, rp->size, rp->cnt);
+	    leaked += rp->size;
+	    free(rp);
+	}
+	printf("%lu bytes leaked\n", leaked);
+    }
+    pthread_mutex_unlock(&lock);
+    printf("--------------------------------------------------------------------------------\n");
+}
+
+#endif
+
+void
+oj_mem_report() {
+#ifdef MEM_DEBUG
+    rb_gc();
+    print_stats();
+#endif
+}

data/ext/oj/mem.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#ifndef OJ_MEM_H
+#define OJ_MEM_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef MEM_DEBUG
+
+#define OJ_MALLOC(size) oj_malloc(size, __FILE__, __LINE__)
+#define OJ_REALLOC(ptr, size) oj_realloc(ptr, size, __FILE__, __LINE__)
+#define OJ_CALLOC(count, size) oj_calloc(count, size, __FILE__, __LINE__)
+#define OJ_FREE(ptr) oj_free(ptr, __FILE__, __LINE__)
+
+#define OJ_R_ALLOC(type) oj_r_alloc(sizeof(type), __FILE__, __LINE__)
+#define OJ_R_ALLOC_N(type, n) (type *)oj_r_alloc(sizeof(type) * (n), __FILE__, __LINE__)
+#define OJ_R_REALLOC_N(ptr, type, n) ((ptr) = (type*)oj_r_realloc(ptr, (sizeof(type) * (n)), __FILE__, __LINE__))
+#define OJ_R_FREE(ptr) oj_r_free(ptr, __FILE__, __LINE__)
+
+#define OJ_STRDUP(str) oj_mem_strdup(str, __FILE__, __LINE__)
+
+extern void*	oj_malloc(size_t size, const char *file, int line);
+extern void*	oj_realloc(void *ptr, size_t size, const char *file, int line);
+extern void*	oj_calloc(size_t count, size_t size, const char *file, int line);
+extern void	oj_free(void *ptr, const char *file, int line);
+
+extern void*	oj_r_alloc(size_t size, const char *file, int line);
+extern void*	oj_r_realloc(void *ptr, size_t size, const char *file, int line);
+extern void	oj_r_free(void *ptr, const char *file, int line);
+
+extern char*	oj_mem_strdup(const char *str, const char *file, int line);
+
+#else
+
+#define OJ_MALLOC(size) malloc(size)
+#define OJ_REALLOC(ptr, size) realloc(ptr, size)
+#define OJ_CALLOC(count, size) calloc(count, size)
+#define OJ_FREE(ptr) free(ptr)
+
+#define OJ_R_ALLOC(type) RB_ALLOC(type)
+#define OJ_R_ALLOC_N(type, n) RB_ALLOC_N(type, n)
+#define OJ_R_REALLOC_N(ptr, type, n) RB_REALLOC_N(ptr, type, n)
+#define OJ_R_FREE(ptr) xfree(ptr)
+
+#define OJ_STRDUP(str) strdup(str)
+
+#endif
+
+extern void	oj_mem_report();
+
+#endif /* OJ_MEM_H */

data/ext/oj/mimic_json.c

@@ -1,6 +1,7 @@
 // Copyright (c) 2012, 2017 Peter Ohler. All rights reserved.
 // Licensed under the MIT License. See LICENSE file in the project root for license details.
 
+#include "mem.h"
 #include "dump.h"
 #include "encode.h"
 #include "oj.h"
@@ -664,7 +665,7 @@ static VALUE mimic_set_create_id(VALUE self, VALUE id) {
 
     if (NULL != oj_default_options.create_id) {
         if (oj_json_class != oj_default_options.create_id) {
-            xfree((char *)oj_default_options.create_id);
+            OJ_R_FREE((char *)oj_default_options.create_id);
         }
         oj_default_options.create_id     = NULL;
         oj_default_options.create_id_len = 0;
@@ -672,7 +673,7 @@ static VALUE mimic_set_create_id(VALUE self, VALUE id) {
     if (Qnil != id) {
         size_t len = RSTRING_LEN(id) + 1;
 
-        oj_default_options.create_id = ALLOC_N(char, len);
+        oj_default_options.create_id = OJ_R_ALLOC_N(char, len);
         strcpy((char *)oj_default_options.create_id, StringValuePtr(id));
         oj_default_options.create_id_len = len - 1;
     }

data/ext/oj/object.c

@@ -308,7 +308,7 @@ static int hat_value(ParseInfo pi, Val parent, const char *key, size_t klen, vol
                 oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "Invalid struct data");
                 return 1;
             }
-            e1 = *RARRAY_PTR(value);
+            e1 = *RARRAY_CONST_PTR(value);
             // check for anonymous Struct
             if (T_ARRAY == rb_type(e1)) {
                 VALUE          args[1024];
@@ -322,10 +322,10 @@ static int hat_value(ParseInfo pi, Val parent, const char *key, size_t klen, vol
                 sc = rb_funcall2(rb_cStruct, oj_new_id, cnt, args);
             } else {
                 // If struct is not defined then we let this fail and raise an exception.
-                sc = oj_name2struct(pi, *RARRAY_PTR(value), rb_eArgError);
+                sc = oj_name2struct(pi, *RARRAY_CONST_PTR(value), rb_eArgError);
             }
             if (sc == rb_cRange) {
-              parent->val = rb_class_new_instance(len - 1, RARRAY_PTR(value) + 1, rb_cRange);
+              parent->val = rb_class_new_instance(len - 1, RARRAY_CONST_PTR(value) + 1, rb_cRange);
             } else {
                 // Create a properly initialized struct instance without calling the initialize method.
                 parent->val = rb_obj_alloc(sc);
@@ -346,20 +346,20 @@ static int hat_value(ParseInfo pi, Val parent, const char *key, size_t klen, vol
                     int i;
 
                     for (i = 0; i < len - 1; i++) {
-                        rb_struct_aset(parent->val, INT2FIX(i), RARRAY_PTR(value)[i + 1]);
+                        rb_struct_aset(parent->val, INT2FIX(i), RARRAY_CONST_PTR(value)[i + 1]);
                     }
                 }
             }
             return 1;
         } else if (3 <= klen && '#' == key[1]) {
-            volatile VALUE *a;
+            volatile const VALUE *a;
 
             if (2 != len) {
                 oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid hash pair");
                 return 1;
             }
             parent->val = rb_hash_new();
-            a           = RARRAY_PTR(value);
+            a           = RARRAY_CONST_PTR(value);
             rb_hash_aset(parent->val, *a, a[1]);
 
             return 1;
@@ -546,7 +546,7 @@ WHICH_TYPE:
         } else {
             if (3 <= klen && '^' == *key && '#' == key[1] && T_ARRAY == rb_type(value)) {
                 long            len = RARRAY_LEN(value);
-                volatile VALUE *a   = RARRAY_PTR(value);
+                volatile const VALUE *a   = RARRAY_CONST_PTR(value);
 
                 if (2 != len) {
                     oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid hash pair");
@@ -609,9 +609,7 @@ WHICH_TYPE:
 }
 
 static VALUE start_hash(ParseInfo pi) {
-    if (RB_UNLIKELY(Yes == pi->options.trace)) {
-        oj_trace_parse_in("start_hash", pi, __FILE__, __LINE__);
-    }
+    TRACE_PARSE_IN(pi->options.trace, "start_hash", pi);
     return Qnil;
 }
 
@@ -627,9 +625,7 @@ static void end_hash(ParseInfo pi) {
         oj_odd_free(oa);
         parent->odd_args = NULL;
     }
-    if (RB_UNLIKELY(Yes == pi->options.trace)) {
-        oj_trace_parse_hash_end(pi, __FILE__, __LINE__);
-    }
+    TRACE_PARSE_HASH_END(pi->options.trace, pi);
 }
 
 static void array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {

data/ext/oj/odd.c

@@ -1,6 +1,7 @@
 // Copyright (c) 2011 Peter Ohler. All rights reserved.
 // Licensed under the MIT License. See LICENSE file in the project root for license details.
 
+#include "mem.h"
 #include "odd.h"
 
 #include <string.h>
@@ -68,7 +69,7 @@ void print_all_odds(const char *label) {
 }
 
 static Odd odd_create(void) {
-    Odd odd = ALLOC(struct _odd);
+    Odd odd = OJ_R_ALLOC(struct _odd);
 
     memset(odd, 0, sizeof(struct _odd));
     odd->next = odds;
@@ -172,7 +173,7 @@ Odd oj_get_oddc(const char *classname, size_t len) {
 }
 
 OddArgs oj_odd_alloc_args(Odd odd) {
-    OddArgs oa = ALLOC_N(struct _oddArgs, 1);
+    OddArgs oa = OJ_R_ALLOC_N(struct _oddArgs, 1);
     VALUE  *a;
     int     i;
 
@@ -184,7 +185,7 @@ OddArgs oj_odd_alloc_args(Odd odd) {
 }
 
 void oj_odd_free(OddArgs args) {
-    xfree(args);
+    OJ_R_FREE(args);
 }
 
 int oj_odd_set_arg(OddArgs args, const char *key, size_t klen, VALUE value) {
@@ -210,7 +211,7 @@ void oj_reg_odd(VALUE clas, VALUE create_object, VALUE create_method, int mcnt,
     odd       = odd_create();
     odd->clas = clas;
     rb_gc_register_mark_object(odd->clas);
-    if (NULL == (odd->classname = strdup(rb_class2name(clas)))) {
+    if (NULL == (odd->classname = OJ_STRDUP(rb_class2name(clas)))) {
         rb_raise(rb_eNoMemError, "for class name.");
     }
     odd->clen       = strlen(odd->classname);
@@ -224,13 +225,13 @@ void oj_reg_odd(VALUE clas, VALUE create_object, VALUE create_method, int mcnt,
         *fp = 0;
         switch (rb_type(*members)) {
         case T_STRING:
-            if (NULL == (*np = strdup(RSTRING_PTR(*members)))) {
+            if (NULL == (*np = OJ_STRDUP(RSTRING_PTR(*members)))) {
                 rb_raise(rb_eNoMemError, "for attribute name.");
             }
             break;
         case T_SYMBOL:
             // The symbol can move and invalidate the name so make a copy.
-            if (NULL == (*np = strdup(rb_id2name(SYM2ID(*members))))) {
+            if (NULL == (*np = OJ_STRDUP(rb_id2name(SYM2ID(*members))))) {
                 rb_raise(rb_eNoMemError, "for attribute name.");
             }
             break;

data/ext/oj/oj.c

@@ -11,6 +11,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include "mem.h"
 #include "dump.h"
 #include "encode.h"
 #include "intern.h"
@@ -782,7 +783,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
     } else if (create_id_sym == k) {
         if (Qnil == v) {
             if (oj_json_class != oj_default_options.create_id && NULL != copts->create_id) {
-                xfree((char *)oj_default_options.create_id);
+                OJ_R_FREE((char *)oj_default_options.create_id);
             }
             copts->create_id     = NULL;
             copts->create_id_len = 0;
@@ -791,7 +792,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
 
             len = RSTRING_LEN(v);
             if (len != copts->create_id_len || 0 != strcmp(copts->create_id, str)) {
-                copts->create_id = ALLOC_N(char, len + 1);
+                copts->create_id = OJ_R_ALLOC_N(char, len + 1);
                 strcpy((char *)copts->create_id, str);
                 copts->create_id_len = len;
             }
@@ -911,7 +912,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
             copts->array_class = v;
         }
     } else if (ignore_sym == k) {
-        xfree(copts->ignore);
+        OJ_R_FREE(copts->ignore);
         copts->ignore = NULL;
         if (Qnil != v) {
             int cnt;
@@ -921,7 +922,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
             if (0 < cnt) {
                 int i;
 
-                copts->ignore = ALLOC_N(VALUE, cnt + 1);
+                copts->ignore = OJ_R_ALLOC_N(VALUE, cnt + 1);
                 for (i = 0; i < cnt; i++) {
                     copts->ignore[i] = RARRAY_AREF(v, i);
                 }
@@ -1159,7 +1160,7 @@ static VALUE load_file(int argc, VALUE *argv, VALUE self) {
         WCHAR *wide_path;
         wide_path = rb_w32_mbstr_to_wstr(CP_UTF8, path, -1, NULL);
         fd = rb_w32_wopen(wide_path, O_RDONLY);
-        free(wide_path);
+        OJ_FREE(wide_path);
     }
 #else
     fd = open(path, O_RDONLY);
@@ -1724,6 +1725,10 @@ debug_odd(VALUE self, VALUE label) {
     return Qnil;
 }
 
+static VALUE mem_report(VALUE self) {
+    oj_mem_report();
+    return Qnil;
+}
 
 /* Document-module: Oj
  *
@@ -1812,6 +1817,8 @@ void Init_oj(void) {
 
     rb_define_module_function(Oj, "optimize_rails", oj_optimize_rails, 0);
 
+    rb_define_module_function(Oj, "mem_report", mem_report, 0);
+
     oj_add_value_id          = rb_intern("add_value");
     oj_array_append_id       = rb_intern("array_append");
     oj_array_end_id          = rb_intern("array_end");

data/ext/oj/oj.h

@@ -379,6 +379,12 @@ extern bool oj_use_hash_alt;
 extern bool oj_use_array_alt;
 extern bool string_writer_optimized;
 
+static inline VALUE oj_safe_string_convert(VALUE obj) {
+    VALUE rstr = rb_funcall(obj, oj_to_s_id, 0);
+    StringValue(rstr);
+    return rstr;
+}
+
 #define APPEND_CHARS(buffer, chars, size) \
     { \
         memcpy(buffer, chars, size); \