oj 3.7.4 → 3.11.2

data/ext/oj/dump.c

@@ -1,24 +1,19 @@
-/* dump.c
- * Copyright (c) 2012, 2017, Peter Ohler
- * All rights reserved.
- */
+// Copyright (c) 2012, 2017 Peter Ohler. All rights reserved.
 
-#include <stdlib.h>
 #include <errno.h>
-#if !IS_WINDOWS
-#include <sys/time.h>
-#endif
-#include <time.h>
+#include <math.h>
+#include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
-#include <math.h>
 #include <unistd.h>
-#include <errno.h>
 
 #include "oj.h"
 #include "cache8.h"
 #include "dump.h"
 #include "odd.h"
+#include "trace.h"
+#include "util.h"
 
 // Workaround in case INFINITY is not defined in math.h or if the OS is CentOS
 #define OJ_INFINITY (1.0/0.0)
@@ -85,6 +80,17 @@ static char	xss_friendly_chars[256] = "\
 // JSON XSS combo
 static char	hixss_friendly_chars[256] = "\
 66666666222622666666666666666666\
+11211111111111111111111111111111\
+11111111111111111111111111112111\
+11111111111111111111111111111111\
+11111111111111111111111111111111\
+11111111111111111111111111111111\
+11111111111111111111111111111111\
+11611111111111111111111111111111";
+
+// Rails XSS combo
+static char	rails_xss_friendly_chars[256] = "\
+66666666222622666666666666666666\
 11211161111111111111111111116161\
 11111111111111111111111111112111\
 11111111111111111111111111111111\
@@ -102,7 +108,7 @@ static char	rails_friendly_chars[256] = "\
 11111111111111111111111111111111\
 11111111111111111111111111111111\
 11111111111111111111111111111111\
-11611111111111111111111111111111";
+11111111111111111111111111111111";
 
 static void
 raise_strict(VALUE obj) {
@@ -158,7 +164,7 @@ hixss_friendly_size(const uint8_t *str, size_t len) {
     size_t	size = 0;
     size_t	i = len;
     bool	check = false;
-    
+
     for (; 0 < i; str++, i--) {
 	size += hixss_friendly_chars[*str];
 	if (0 != (0x80 & *str)) {
@@ -168,6 +174,22 @@ hixss_friendly_size(const uint8_t *str, size_t len) {
     return size - len * (size_t)'0' + check;
 }
 
+inline static long
+rails_xss_friendly_size(const uint8_t *str, size_t len) {
+    long	size = 0;
+    size_t	i = len;
+    uint8_t	hi = 0;
+
+    for (; 0 < i; str++, i--) {
+	size += rails_xss_friendly_chars[*str];
+	hi |= *str & 0x80;
+    }
+    if (0 == hi) {
+	return size - len * (size_t)'0';
+    }
+    return -(size - len * (size_t)'0');
+}
+
 inline static size_t
 rails_friendly_size(const uint8_t *str, size_t len) {
     size_t	size = 0;
@@ -182,7 +204,7 @@ rails_friendly_size(const uint8_t *str, size_t len) {
 const char*
 oj_nan_str(VALUE obj, int opt, int mode, bool plus, int *lenp) {
     const char	*str = NULL;
-    
+
     if (AutoNan == opt) {
 	switch (mode) {
 	case CompatMode:	opt = WordNan;	break;
@@ -232,7 +254,6 @@ dump_hex(uint8_t c, Out out) {
 
 static void
 raise_invalid_unicode(const char *str, int len, int pos) {
-    char	buf[len + 1];
     char	c;
     char	code[32];
     char	*cp = code;
@@ -251,8 +272,7 @@ raise_invalid_unicode(const char *str, int len, int pos) {
     cp--;
     *cp++ = ']';
     *cp = '\0';
-    strncpy(buf, str, len);
-    rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d in '%s'", code, pos, buf);
+    rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d", code, pos);
 }
 
 static const char*
@@ -304,7 +324,7 @@ dump_unicode(const char *str, const char *end, Out out, const char *orig) {
     *out->cur++ = 'u';
     for (i = 3; 0 <= i; i--) {
 	*out->cur++ = hex_chars[(uint8_t)(code >> (i * 4)) & 0x0F];
-    }	
+    }
     return str - 1;
 }
 
@@ -312,7 +332,7 @@ static const char*
 check_unicode(const char *str, const char *end, const char *orig) {
     uint8_t	b = *(uint8_t*)str;
     int		cnt = 0;
-    
+
     if (0xC0 == (0xE0 & b)) {
 	cnt = 1;
     } else if (0xE0 == (0xF0 & b)) {
@@ -376,11 +396,17 @@ oj_dump_time(VALUE obj, Out out, int withZone) {
     long long	nsec;
 
 #ifdef HAVE_RB_TIME_TIMESPEC
-    {
+    // rb_time_timespec as well as rb_time_timeeval have a bug that causes an
+    // exception to be raised if a time is before 1970 on 32 bit systems so
+    // check the timespec size and use the ruby calls if a 32 bit system.
+    if (16 <= sizeof(struct timespec)) {
 	struct timespec	ts = rb_time_timespec(obj);
 
 	sec = (long long)ts.tv_sec;
 	nsec = ts.tv_nsec;
+    } else {
+	sec = rb_num2ll(rb_funcall2(obj, oj_tv_sec_id, 0, 0));
+	nsec = rb_num2ll(rb_funcall2(obj, oj_tv_nsec_id, 0, 0));
     }
 #else
     sec = rb_num2ll(rb_funcall2(obj, oj_tv_sec_id, 0, 0));
@@ -465,20 +491,24 @@ oj_dump_ruby_time(VALUE obj, Out out) {
 
 void
 oj_dump_xml_time(VALUE obj, Out out) {
-    char	buf[64];
-    struct tm	*tm;
-    long	one = 1000000000;
-    time_t	sec;
-    long long	nsec;
-    long	tzsecs = NUM2LONG(rb_funcall2(obj, oj_utc_offset_id, 0, 0));
-    int		tzhour, tzmin;
-    char	tzsign = '+';
+    char		buf[64];
+    struct _timeInfo	ti;
+    long		one = 1000000000;
+    int64_t		sec;
+    long long		nsec;
+    long		tzsecs = NUM2LONG(rb_funcall2(obj, oj_utc_offset_id, 0, 0));
+    int			tzhour, tzmin;
+    char		tzsign = '+';
 
 #ifdef HAVE_RB_TIME_TIMESPEC
-    {
+    if (16 <= sizeof(struct timespec)) {
 	struct timespec	ts = rb_time_timespec(obj);
+
 	sec = ts.tv_sec;
 	nsec = ts.tv_nsec;
+    } else {
+	sec = rb_num2ll(rb_funcall2(obj, oj_tv_sec_id, 0, 0));
+	nsec = rb_num2ll(rb_funcall2(obj, oj_tv_nsec_id, 0, 0));
     }
 #else
     sec = rb_num2ll(rb_funcall2(obj, oj_tv_sec_id, 0, 0));
@@ -509,8 +539,7 @@ oj_dump_xml_time(VALUE obj, Out out) {
     // 2012-01-05T23:58:07.123456000+09:00
     //tm = localtime(&sec);
     sec += tzsecs;
-    tm = gmtime(&sec);
-#if 1
+    sec_as_time((int64_t)sec, &ti);
     if (0 > tzsecs) {
         tzsign = '-';
         tzhour = (int)(tzsecs / -3600);
@@ -519,26 +548,12 @@ oj_dump_xml_time(VALUE obj, Out out) {
         tzhour = (int)(tzsecs / 3600);
         tzmin = (int)(tzsecs / 60) - (tzhour * 60);
     }
-#else
-    if (0 > tm->tm_gmtoff) {
-        tzsign = '-';
-        tzhour = (int)(tm->tm_gmtoff / -3600);
-        tzmin = (int)(tm->tm_gmtoff / -60) - (tzhour * 60);
-    } else {
-        tzhour = (int)(tm->tm_gmtoff / 3600);
-        tzmin = (int)(tm->tm_gmtoff / 60) - (tzhour * 60);
-    }
-#endif
-    if (0 == nsec || 0 == out->opts->sec_prec) {
+    if ((0 == nsec && !out->opts->sec_prec_set) || 0 == out->opts->sec_prec) {
 	if (0 == tzsecs && rb_funcall2(obj, oj_utcq_id, 0, 0)) {
-	    sprintf(buf, "%04d-%02d-%02dT%02d:%02d:%02dZ",
-		    tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
-		    tm->tm_hour, tm->tm_min, tm->tm_sec);
+	    sprintf(buf, "%04d-%02d-%02dT%02d:%02d:%02dZ", ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec);
 	    oj_dump_cstr(buf, 20, 0, 0, out);
 	} else {
-	    sprintf(buf, "%04d-%02d-%02dT%02d:%02d:%02d%c%02d:%02d",
-		    tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
-		    tm->tm_hour, tm->tm_min, tm->tm_sec,
+	    sprintf(buf, "%04d-%02d-%02dT%02d:%02d:%02d%c%02d:%02d", ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec,
 		    tzsign, tzhour, tzmin);
 	    oj_dump_cstr(buf, 25, 0, 0, out);
 	}
@@ -550,9 +565,7 @@ oj_dump_xml_time(VALUE obj, Out out) {
 	    format[32] = '0' + out->opts->sec_prec;
 	    len -= 9 - out->opts->sec_prec;
 	}
-	sprintf(buf, format,
-		tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
-		tm->tm_hour, tm->tm_min, tm->tm_sec, (long)nsec);
+	sprintf(buf, format, ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec, (long)nsec);
 	oj_dump_cstr(buf, len, 0, 0, out);
     } else {
 	char	format[64] = "%04d-%02d-%02dT%02d:%02d:%02d.%09ld%c%02d:%02d";
@@ -562,10 +575,7 @@ oj_dump_xml_time(VALUE obj, Out out) {
 	    format[32] = '0' + out->opts->sec_prec;
 	    len -= 9 - out->opts->sec_prec;
 	}
-	sprintf(buf, format,
-		tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
-		tm->tm_hour, tm->tm_min, tm->tm_sec, (long)nsec,
-		tzsign, tzhour, tzmin);
+	sprintf(buf, format, ti.year, ti.mon, ti.day, ti.hour, ti.min, ti.sec, (long)nsec, tzsign, tzhour, tzmin);
 	oj_dump_cstr(buf, len, 0, 0, out);
     }
 }
@@ -622,7 +632,7 @@ oj_dump_obj_to_json_using_params(VALUE obj, Options copts, Out out, int argc, VA
 void
 oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
     char	buf[4096];
-    struct _Out out;
+    struct _out out;
     size_t	size;
     FILE	*f;
     int		ok;
@@ -654,7 +664,7 @@ oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
 void
 oj_write_obj_to_stream(VALUE obj, VALUE stream, Options copts) {
     char	buf[4096];
-    struct _Out out;
+    struct _out out;
     ssize_t	size;
     VALUE	clas = rb_obj_class(stream);
 #if !IS_WINDOWS
@@ -731,11 +741,36 @@ debug_raise(const char *orig, size_t cnt, int line) {
     rb_raise(oj_json_generator_error_class, "Partial character in string. %s @ %d", buf, line);
 }
 
+void
+oj_dump_raw_json(VALUE obj, int depth, Out out) {
+    if (oj_string_writer_class == rb_obj_class(obj)) {
+	StrWriter	sw = (StrWriter)DATA_PTR(obj);
+	size_t		len = sw->out.cur - sw->out.buf;
+
+	if (0 < len) {
+	    len--;
+	}
+	oj_dump_raw(sw->out.buf, len, out);
+    } else {
+	volatile VALUE	jv;
+
+	if (Yes == out->opts->trace) {
+	    oj_trace("raw_json", obj, __FILE__, __LINE__, depth + 1, TraceRubyIn);
+	}
+	jv = rb_funcall(obj, oj_raw_json_id, 2, RB_INT2NUM(depth), RB_INT2NUM(out->indent));
+	if (Yes == out->opts->trace) {
+	    oj_trace("raw_json", obj, __FILE__, __LINE__, depth + 1, TraceRubyOut);
+	}
+	oj_dump_raw(rb_string_value_ptr((VALUE*)&jv), (size_t)RSTRING_LEN(jv), out);
+    }
+}
+
 void
 oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
     size_t	size;
     char	*cmap;
     const char	*orig = str;
+    bool	has_hi = false;
 
     switch (out->opts->escape_mode) {
     case NLEsc:
@@ -754,6 +789,19 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 	cmap = hixss_friendly_chars;
 	size = hixss_friendly_size((uint8_t*)str, cnt);
 	break;
+    case RailsXEsc: {
+	long	sz;
+
+	cmap = rails_xss_friendly_chars;
+	sz = rails_xss_friendly_size((uint8_t*)str, cnt);
+	if (sz < 0) {
+	    has_hi = true;
+	    size = (size_t)-sz;
+	} else {
+	    size = (size_t)sz;
+	}
+	break;
+    }
     case RailsEsc:
 	cmap = rails_friendly_chars;
 	size = rails_friendly_size((uint8_t*)str, cnt);
@@ -777,7 +825,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 	str++;
 	is_sym = 0; // just to make sure
     }
-    if (cnt == size) {
+    if (cnt == size && !has_hi) {
 	if (is_sym) {
 	    *out->cur++ = ':';
 	}
@@ -788,14 +836,14 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
     } else {
 	const char	*end = str + cnt;
 	const char	*check_start = str;
-	
+
 	if (is_sym) {
 	    *out->cur++ = ':';
 	}
 	for (; str < end; str++) {
 	    switch (cmap[(uint8_t)*str]) {
 	    case '1':
-		if (JXEsc == out->opts->escape_mode && check_start <= str) {
+		if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && check_start <= str) {
 		    if (0 != (0x80 & (uint8_t)*str)) {
 			if (0xC0 == (0xC0 & (uint8_t)*str)) {
 			    check_start = check_unicode(str, end, orig);
@@ -819,7 +867,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 		}
 		break;
 	    case '3': // Unicode
-		if (0xe2 == (uint8_t)*str && JXEsc == out->opts->escape_mode && 2 <= end - str) {
+		if (0xe2 == (uint8_t)*str && (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 2 <= end - str) {
 		    if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
 			str = dump_unicode(str, end, out, orig);
 		    } else {
@@ -838,7 +886,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 		    *out->cur++ = '0';
 		    dump_hex((uint8_t)*str, out);
 		} else {
-		    if (0xe2 == (uint8_t)*str && JXEsc == out->opts->escape_mode && 2 <= end - str) {
+		    if (0xe2 == (uint8_t)*str && (JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 2 <= end - str) {
 			if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
 			    str = dump_unicode(str, end, out, orig);
 			} else {
@@ -854,13 +902,13 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 		break; // ignore, should never happen if the table is correct
 	    }
 	}
-	*out->cur++ = '"'; 
+	*out->cur++ = '"';
     }
-    if (JXEsc == out->opts->escape_mode && 0 < str - orig && 0 != (0x80 & *(str - 1))) {
+    if ((JXEsc == out->opts->escape_mode || RailsXEsc == out->opts->escape_mode) && 0 < str - orig && 0 != (0x80 & *(str - 1))) {
 	uint8_t	c = (uint8_t)*(str - 1);
 	int	i;
 	int	scnt = (int)(str - orig);
-	
+
 	// Last utf-8 characters must be 0x10xxxxxx. The start must be
 	// 0x110xxxxx for 2 characters, 0x1110xxxx for 3, and 0x11110xxx for
 	// 4.
@@ -926,7 +974,7 @@ oj_grow_out(Out out, size_t len) {
     size_t  size = out->end - out->buf;
     long    pos = out->cur - out->buf;
     char    *buf = out->buf;
-	
+
     size *= 2;
     if (size <= len * 2 + pos) {
 	size += len;
@@ -983,24 +1031,21 @@ oj_dump_fixnum(VALUE obj, int depth, Out out, bool as_ok) {
     char	*b = buf + sizeof(buf) - 1;
     long long	num = rb_num2ll(obj);
     int		neg = 0;
-	bool	dump_as_string = false;
+    bool	dump_as_string = false;
 
-	if (out->opts->integer_range_max != 0 && out->opts->integer_range_min != 0 &&
-		(out->opts->integer_range_max < num || out->opts->integer_range_min > num)) {
+    if (out->opts->int_range_max != 0 && out->opts->int_range_min != 0 &&
+	(out->opts->int_range_max < num || out->opts->int_range_min > num)) {
 	dump_as_string = true;
-	}
-
+    }
     if (0 > num) {
 	neg = 1;
 	num = -num;
     }
-
     *b-- = '\0';
 
-	if (dump_as_string) {
+    if (dump_as_string) {
 	*b-- = '"';
-	}
-
+    }
     if (0 < num) {
 	for (; 0 < num; num /= 10, b--) {
 	    *b = (num % 10) + '0';
@@ -1013,11 +1058,9 @@ oj_dump_fixnum(VALUE obj, int depth, Out out, bool as_ok) {
     } else {
 	*b = '0';
     }
-
-	if (dump_as_string) {
+    if (dump_as_string) {
 	*--b = '"';
-	}
-
+    }
     assure_size(out, (sizeof(buf) - (b - buf)));
     for (; '\0' != *b; b++) {
 	*out->cur++ = *b;
@@ -1029,23 +1072,20 @@ void
 oj_dump_bignum(VALUE obj, int depth, Out out, bool as_ok) {
     volatile VALUE	rs = rb_big2str(obj, 10);
     int			cnt = (int)RSTRING_LEN(rs);
-	bool		dump_as_string = false;
+    bool		dump_as_string = false;
 
-	if (out->opts->integer_range_max != 0 || out->opts->integer_range_min != 0) { // Bignum cannot be inside of Fixnum range
-	dump_as_string = true; 
+    if (out->opts->int_range_max != 0 || out->opts->int_range_min != 0) { // Bignum cannot be inside of Fixnum range
+	dump_as_string = true;
 	assure_size(out, cnt + 2);
 	*out->cur++ = '"';
-	} else {
+    } else {
 	assure_size(out, cnt);
-	}
-
+    }
     memcpy(out->cur, rb_string_value_ptr((VALUE*)&rs), cnt);
     out->cur += cnt;
-
-	if(dump_as_string) {
+    if (dump_as_string) {
 	*out->cur++ = '"';
-	}
-
+    }
     *out->cur = '\0';
 }
 
@@ -1070,7 +1110,7 @@ oj_dump_float(VALUE obj, int depth, Out out, bool as_ok) {
 	    cnt = sizeof(inf_val) - 1;
 	} else {
 	    NanDump	nd = out->opts->dump_opts.nan_dump;
-	    
+
 	    if (AutoNan == nd) {
 		switch (out->opts->mode) {
 		case CompatMode:	nd = WordNan;	break;
@@ -1105,7 +1145,7 @@ oj_dump_float(VALUE obj, int depth, Out out, bool as_ok) {
 	    cnt = sizeof(ninf_val) - 1;
 	} else {
 	    NanDump	nd = out->opts->dump_opts.nan_dump;
-	    
+
 	    if (AutoNan == nd) {
 		switch (out->opts->mode) {
 		case CompatMode:	nd = WordNan;	break;
@@ -1139,7 +1179,7 @@ oj_dump_float(VALUE obj, int depth, Out out, bool as_ok) {
 	    cnt = sizeof(ninf_val) - 1;
 	} else {
 	    NanDump	nd = out->opts->dump_opts.nan_dump;
-	    
+
 	    if (AutoNan == nd) {
 		switch (out->opts->mode) {
 		case ObjectMode:	nd = HugeNan;	break;