oj 3.16.16 → 3.17.3

data/ext/oj/oj.c

@@ -20,6 +20,8 @@
 #include "rails.h"
 #include "simd.h"
 
+#define MAX_INDENT 16
+
 typedef struct _yesNoOpt {
     VALUE sym;
     char *attr;
@@ -120,7 +122,9 @@ static VALUE create_id_sym;
 static VALUE custom_sym;
 static VALUE empty_string_sym;
 static VALUE escape_mode_sym;
+static VALUE except_sym;
 static VALUE integer_range_sym;
+static VALUE max_integer_digits_sym;
 static VALUE fast_sym;
 static VALUE float_prec_sym;
 static VALUE float_format_sym;
@@ -138,6 +142,7 @@ static VALUE null_sym;
 static VALUE object_sym;
 static VALUE omit_null_byte_sym;
 static VALUE omit_nil_sym;
+static VALUE only_sym;
 static VALUE rails_sym;
 static VALUE raise_sym;
 static VALUE ruby_sym;
@@ -204,6 +209,7 @@ struct _options oj_default_options = {
     0,              // cache_str
     0,              // int_range_min
     0,              // int_range_max
+    0,              // max_integer_digits
     oj_json_class,  // create_id
     10,             // create_id_len
     9,              // sec_prec
@@ -228,6 +234,8 @@ struct _options oj_default_options = {
         false,      // omit_nil
         false,      // omit_null_byte
         MAX_DEPTH,  // max_depth
+        NULL,       // only
+        NULL,       // except
     },
     {
         // str_rx
@@ -238,6 +246,22 @@ struct _options oj_default_options = {
     NULL,
 };
 
+static VALUE only_array_from_string(const char *str) {
+    volatile VALUE a = Qnil;
+
+    if (NULL != str && 2 < strlen(str)) {
+        str++;
+        char *cp;
+
+        a = rb_ary_new();
+        while (NULL != (cp = strchr(str, ':'))) {
+            rb_ary_push(a, rb_id2sym(rb_intern2(str, cp - str)));
+            str = cp + 1;
+        }
+    }
+    return a;
+}
+
 /* Document-method: default_options()
  *	call-seq: default_options()
  *
@@ -314,10 +338,17 @@ struct _options oj_default_options = {
  * - *:cache_str* [_Fixnum_] maximum string value length to cache (strings less
  *   than this are cached)
  * - *:integer_range* [_Range_] Dump integers outside range as strings.
+ * - *:max_integer_digits* [_Fixnum_] Maximum number of decimal digits allowed in a
+ *   parsed integer. When the limit is exceeded a parse error is raised. 0 (the
+ *   default) disables the limit. Setting a reasonable limit is recommended when
+ *   parsing untrusted input to mitigate CPU-DoS attacks. Only applies to the
+ *   legacy parsers (Oj.load, Oj::Doc, JSON.parse mimic); Oj::Parser is unaffected.
  * - *:trace* [_true,_|_false_] Trace all load and dump calls, default is false
  *   (trace is off)
  * - *:safe* [_true,_|_false_] Safe mimic breaks JSON mimic to be safer, default
  *   is false (safe is off)
+ * - *:only* [_nil,_|_Array_] A list of the fields to encode. All others are skipped.
+ * - *:except* [_nil,_|_Array_] A list of the fields to not encode. All others are encoded.
  *
  * Return [_Hash_] all current option settings.
  */
@@ -426,6 +457,7 @@ static VALUE get_def_opts(VALUE self) {
     } else {
         rb_hash_aset(opts, integer_range_sym, Qnil);
     }
+    rb_hash_aset(opts, max_integer_digits_sym, LONG2NUM((long)oj_default_options.max_integer_digits));
     switch (oj_default_options.escape_mode) {
     case NLEsc: rb_hash_aset(opts, escape_mode_sym, newline_sym); break;
     case JSONEsc: rb_hash_aset(opts, escape_mode_sym, json_sym); break;
@@ -483,6 +515,9 @@ static VALUE get_def_opts(VALUE self) {
     rb_hash_aset(opts, oj_hash_class_sym, oj_default_options.hash_class);
     rb_hash_aset(opts, oj_array_class_sym, oj_default_options.array_class);
 
+    rb_hash_aset(opts, only_sym, only_array_from_string(oj_default_options.dump_opts.only));
+    rb_hash_aset(opts, except_sym, only_array_from_string(oj_default_options.dump_opts.except));
+
     if (NULL == oj_default_options.ignore) {
         rb_hash_aset(opts, ignore_sym, Qnil);
     } else {
@@ -566,6 +601,8 @@ static VALUE get_def_opts(VALUE self) {
  *   - *:cache_keys* [_Boolean_] if true then hash keys are cached
  *   - *:cache_str* [_Fixnum_] maximum string value length to cache (strings less than this are cached)
  *   - *:integer_range* [_Range_] Dump integers outside range as strings.
+ *   - *:max_integer_digits* [_Fixnum_] Maximum decimal digits in a parsed integer
+ *     (0 = unlimited). Use to mitigate CPU-DoS via huge integer values in JSON.
  *   - *:trace* [_Boolean_] turn trace on or off.
  *   - *:safe* [_Boolean_] turn safe mimic on or off.
  */
@@ -625,6 +662,95 @@ bool set_yesno_options(VALUE key, VALUE value, Options copts) {
     return false;
 }
 
+static const char *make_only_value(VALUE v) {
+    switch (rb_type(v)) {
+    case RUBY_T_NIL:
+    case RUBY_T_NONE: return NULL;
+    case RUBY_T_ARRAY: {
+        long  len = rb_array_len(v);
+        long  i;
+        long  size = 0;
+        char *buf;
+        char *bp;
+
+        for (i = 0; i < len; i++) {
+            VALUE x = rb_ary_entry(v, i);
+
+            switch (rb_type(x)) {
+            case RUBY_T_SYMBOL:
+                size += strlen(rb_id2name(rb_sym2id(x)));
+                size++;
+                break;
+            case RUBY_T_STRING:
+                size += strlen(StringValueCStr(x));
+                size++;
+                break;
+            default: rb_raise(rb_eArgError, ":only and :except must be nil, symbol, string, or array."); break;
+            }
+        }
+        if (0 == size) {
+            return NULL;
+        }
+        buf   = OJ_R_ALLOC_N(char, size + 2);
+        bp    = buf;
+        *bp++ = ':';
+        for (i = 0; i < len; i++) {
+            VALUE       x = rb_ary_entry(v, i);
+            const char *str;
+
+            switch (rb_type(x)) {
+            case RUBY_T_SYMBOL:
+                str  = rb_id2name(rb_sym2id(x));
+                size = strlen(str);
+                memcpy(bp, str, size);
+                bp += size;
+                *bp++ = ':';
+                break;
+            case RUBY_T_STRING:
+                str  = StringValueCStr(x);
+                size = strlen(str);
+                memcpy(bp, str, size);
+                bp += size;
+                *bp++ = ':';
+                break;
+            default:
+                // ignore
+                break;
+            }
+        }
+        *bp = '\0';
+
+        return buf;
+    }
+    case RUBY_T_STRING: {
+        const char *str  = StringValueCStr(v);
+        size_t      size = strlen(str);
+        char       *buf  = OJ_R_ALLOC_N(char, size + 3);
+
+        buf[0] = ':';
+        strcpy(buf + 1, str);
+        buf[size + 1] = ':';
+        buf[size + 2] = '\0';
+
+        return buf;
+    }
+    case RUBY_T_SYMBOL: {
+        const char *str  = rb_id2name(rb_sym2id(v));
+        size_t      size = strlen(str);
+        char       *buf  = OJ_R_ALLOC_N(char, size + 3);
+
+        buf[0] = ':';
+        strcpy(buf + 1, str);
+        buf[size + 1] = ':';
+        buf[size + 2] = '\0';
+
+        return buf;
+    }
+    default: rb_raise(rb_eArgError, ":only and zzz :except must be nil, symbol, string, or array."); break;
+    }
+    return NULL;
+}
+
 static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
     Options copts = (Options)opts;
     size_t  len;
@@ -642,7 +768,10 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         case T_FIXNUM:
             copts->dump_opts.indent_size = 0;
             *copts->dump_opts.indent_str = '\0';
-            copts->indent                = FIX2INT(v);
+            if (MAX_INDENT < FIX2INT(v)) {
+                rb_raise(rb_eArgError, "indent is limited to %d characters.", MAX_INDENT);
+            }
+            copts->indent = FIX2INT(v);
             break;
         case T_STRING:
             if (sizeof(copts->dump_opts.indent_str) <= (len = RSTRING_LEN(v))) {
@@ -957,6 +1086,20 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         } else if (Qfalse != v) {
             rb_raise(rb_eArgError, ":integer_range must be a range of Fixnum.");
         }
+    } else if (max_integer_digits_sym == k) {
+        if (Qnil == v || Qfalse == v) {
+            copts->max_integer_digits = 0;
+        } else if (T_FIXNUM == rb_type(v)) {
+            long n = FIX2LONG(v);
+
+            if (n < 0) {
+                rb_raise(rb_eArgError, ":max_integer_digits must be >= 0.");
+            }
+
+            copts->max_integer_digits = (size_t)n;
+        } else {
+            rb_raise(rb_eArgError, ":max_integer_digits must be a non-negative Integer.");
+        }
     } else if (symbol_keys_sym == k || oj_symbolize_names_sym == k) {
         if (Qnil == v) {
             return ST_CONTINUE;
@@ -981,6 +1124,18 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
         strncpy(copts->float_fmt, RSTRING_PTR(v), (size_t)RSTRING_LEN(v));
         copts->float_fmt[RSTRING_LEN(v)] = '\0';
+    } else if (only_sym == k) {
+        if (NULL != copts->dump_opts.only) {
+            OJ_R_FREE((void *)copts->dump_opts.only);
+            copts->dump_opts.only = NULL;
+        }
+        copts->dump_opts.only = make_only_value(v);
+    } else if (except_sym == k) {
+        if (NULL != copts->dump_opts.except) {
+            OJ_R_FREE((void *)copts->dump_opts.except);
+            copts->dump_opts.except = NULL;
+        }
+        copts->dump_opts.except = make_only_value(v);
     }
     return ST_CONTINUE;
 }
@@ -2027,6 +2182,8 @@ void Init_oj(void) {
     rb_gc_register_address(&escape_mode_sym);
     integer_range_sym = ID2SYM(rb_intern("integer_range"));
     rb_gc_register_address(&integer_range_sym);
+    max_integer_digits_sym = ID2SYM(rb_intern("max_integer_digits"));
+    rb_gc_register_address(&max_integer_digits_sym);
     fast_sym = ID2SYM(rb_intern("fast"));
     rb_gc_register_address(&fast_sym);
     float_format_sym = ID2SYM(rb_intern("float_format"));
@@ -2133,6 +2290,10 @@ void Init_oj(void) {
     rb_gc_register_address(&xmlschema_sym);
     xss_safe_sym = ID2SYM(rb_intern("xss_safe"));
     rb_gc_register_address(&xss_safe_sym);
+    only_sym = ID2SYM(rb_intern("only"));
+    rb_gc_register_address(&only_sym);
+    except_sym = ID2SYM(rb_intern("except"));
+    rb_gc_register_address(&except_sym);
 
     oj_slash_string = rb_str_new2("/");
     rb_gc_register_address(&oj_slash_string);

data/ext/oj/oj.h

@@ -104,61 +104,64 @@ typedef enum {
 } StreamWriterType;
 
 typedef struct _dumpOpts {
-    bool    use;
-    char    indent_str[16];
-    char    before_sep[16];
-    char    after_sep[16];
-    char    hash_nl[16];
-    char    array_nl[16];
-    uint8_t indent_size;
-    uint8_t before_size;
-    uint8_t after_size;
-    uint8_t hash_size;
-    uint8_t array_size;
-    char    nan_dump;  // NanDump
-    bool    omit_nil;
-    bool    omit_null_byte;
-    int     max_depth;
+    bool        use;
+    char        indent_str[16];
+    char        before_sep[16];
+    char        after_sep[16];
+    char        hash_nl[16];
+    char        array_nl[16];
+    uint8_t     indent_size;
+    uint8_t     before_size;
+    uint8_t     after_size;
+    uint8_t     hash_size;
+    uint8_t     array_size;
+    char        nan_dump;  // NanDump
+    bool        omit_nil;
+    bool        omit_null_byte;
+    int         max_depth;
+    const char *only;
+    const char *except;
 } *DumpOpts;
 
 typedef struct _options {
-    int              indent;         // indention for dump, default 2
-    char             circular;       // YesNo
-    char             auto_define;    // YesNo
-    char             sym_key;        // YesNo
-    char             escape_mode;    // Escape_Mode
-    char             mode;           // Mode
-    char             class_cache;    // YesNo
-    char             time_format;    // TimeFormat
-    char             bigdec_as_num;  // YesNo
-    char             bigdec_load;    // BigLoad
-    char             compat_bigdec;  // boolean (0 or 1)
-    char             to_hash;        // YesNo
-    char             to_json;        // YesNo
-    char             as_json;        // YesNo
-    char             raw_json;       // YesNo
-    char             nilnil;         // YesNo
-    char             empty_string;   // YesNo
-    char             allow_gc;       // allow GC during parse
-    char             quirks_mode;    // allow single JSON values instead of documents
-    char             allow_invalid;  // YesNo - allow invalid unicode
-    char             create_ok;      // YesNo allow create_id
-    char             allow_nan;      // YEsyNo for parsing only
-    char             trace;          // YesNo
-    char             safe;           // YesNo
-    char             sec_prec_set;   // boolean (0 or 1)
-    char             ignore_under;   // YesNo - ignore attrs starting with _ if true in object and custom modes
-    char             cache_keys;     // YesNo
-    char             cache_str;      // string short than or equal to this are cache
-    int64_t          int_range_min;  // dump numbers below as string
-    int64_t          int_range_max;  // dump numbers above as string
-    const char      *create_id;      // 0 or string
-    size_t           create_id_len;  // length of create_id
-    int              sec_prec;       // second precision when dumping time
-    char             float_prec;     // float precision, linked to float_fmt
-    char             float_fmt[7];   // float format for dumping, if empty use Ruby
-    VALUE            hash_class;     // class to use in place of Hash on load
-    VALUE            array_class;    // class to use in place of Array on load
+    int              indent;              // indention for dump, default 2
+    char             circular;            // YesNo
+    char             auto_define;         // YesNo
+    char             sym_key;             // YesNo
+    char             escape_mode;         // Escape_Mode
+    char             mode;                // Mode
+    char             class_cache;         // YesNo
+    char             time_format;         // TimeFormat
+    char             bigdec_as_num;       // YesNo
+    char             bigdec_load;         // BigLoad
+    char             compat_bigdec;       // boolean (0 or 1)
+    char             to_hash;             // YesNo
+    char             to_json;             // YesNo
+    char             as_json;             // YesNo
+    char             raw_json;            // YesNo
+    char             nilnil;              // YesNo
+    char             empty_string;        // YesNo
+    char             allow_gc;            // allow GC during parse
+    char             quirks_mode;         // allow single JSON values instead of documents
+    char             allow_invalid;       // YesNo - allow invalid unicode
+    char             create_ok;           // YesNo allow create_id
+    char             allow_nan;           // YesNo for parsing only
+    char             trace;               // YesNo
+    char             safe;                // YesNo
+    char             sec_prec_set;        // boolean (0 or 1)
+    char             ignore_under;        // YesNo - ignore attrs starting with _ if true in object and custom modes
+    char             cache_keys;          // YesNo
+    char             cache_str;           // string short than or equal to this are cache
+    int64_t          int_range_min;       // dump numbers below as string
+    int64_t          int_range_max;       // dump numbers above as string
+    size_t           max_integer_digits;  // 0 = unlimited; max decimal digits for parsed integers
+    const char      *create_id;           // 0 or string
+    size_t           create_id_len;       // length of create_id
+    int              sec_prec;            // second precision when dumping time
+    char             float_prec;          // float precision, linked to float_fmt
+    char             float_fmt[7];        // float format for dumping, if empty use Ruby
+    VALUE            hash_class;          // class to use in place of Hash on load
+    VALUE            array_class;         // class to use in place of Array on load
     struct _dumpOpts dump_opts;
     struct _rxClass  str_rx;
     VALUE           *ignore;  // Qnil terminated array of classes or NULL

data/ext/oj/parse.c

@@ -209,7 +209,7 @@ static inline const char *string_scan_neon(const char *str, const char *end) {
     while (str + sizeof(uint8x16_t) <= end) {
         uint8x16_t      chunk = vld1q_u8((const uint8_t *)str);
         uint8x16_t      tmp   = vorrq_u8(vorrq_u8(vceqq_u8(chunk, null_char), vceqq_u8(chunk, backslash)),
-                                  vceqq_u8(chunk, quote));
+                                         vceqq_u8(chunk, quote));
         const uint8x8_t res   = vshrn_n_u16(vreinterpretq_u16_u8(tmp), 4);
         uint64_t        mask  = vget_lane_u64(vreinterpret_u64_u8(res), 0);
         if (mask != 0) {
@@ -285,10 +285,10 @@ static OJ_TARGET_SSE42 const char *scan_string_SSE42(const char *str, const char
     for (; str <= safe_end_16; str += 16) {
         const __m128i string = _mm_loadu_si128((const __m128i *)str);
         const int     r      = _mm_cmpestri(terminate,
-                                   3,
-                                   string,
-                                   16,
-                                   _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+                                            3,
+                                            string,
+                                            16,
+                                            _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
         if (r != 16)
             return str + r;
     }
@@ -394,7 +394,7 @@ void oj_scanner_init(void) {
 static void read_escaped_str(ParseInfo pi, const char *start) {
     struct _buf buf;
     const char *s;
-    int         cnt = (int)(pi->cur - start);
+    size_t      cnt = pi->cur - start;
     uint32_t    code;
     Val         parent = stack_peek(&pi->stack);
 
@@ -611,7 +611,7 @@ static void read_num(ParseInfo pi) {
         ni.bigdec_load = pi->options.compat_bigdec;
     } else {
         ni.no_big      = (FloatDec == pi->options.bigdec_load || FastDec == pi->options.bigdec_load ||
-                     RubyDec == pi->options.bigdec_load);
+                          RubyDec == pi->options.bigdec_load);
         ni.bigdec_load = pi->options.bigdec_load;
     }
 
@@ -669,7 +669,7 @@ static void read_num(ParseInfo pi) {
             // A trailing . is not a valid decimal but if encountered allow it
             // except when mimicking the JSON gem or in strict mode.
             if (StrictMode == pi->options.mode || CompatMode == pi->options.mode) {
-                int pos = (int)(pi->cur - ni.str);
+                size_t pos = pi->cur - ni.str;
 
                 if (1 == pos || (2 == pos && ni.neg)) {
                     oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number");
@@ -971,6 +971,20 @@ static long double exp_plus[] = {
     1.0e39, 1.0e40, 1.0e41, 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, 1.0e49,
 };
 
+static void validate_integer_size(size_t limit, NumInfo ni) {
+    size_t digit_count = ni->len - (ni->neg ? 1 : 0);
+
+    if (digit_count > limit) {
+        oj_set_error_at(ni->pi,
+                        (Qnil != ni->pi->err_class) ? ni->pi->err_class : oj_parse_error_class,
+                        __FILE__,
+                        __LINE__,
+                        "integer exceeds :max_integer_digits (%lu > %lu)",
+                        (unsigned long)digit_count,
+                        (unsigned long)limit);
+    }
+}
+
 VALUE
 oj_num_as_value(NumInfo ni) {
     VALUE rnum = Qnil;
@@ -984,6 +998,12 @@ oj_num_as_value(NumInfo ni) {
     } else if (ni->nan) {
         rnum = rb_float_new(0.0 / 0.0);
     } else if (1 == ni->div && 0 == ni->exp && !ni->has_exp) {  // fixnum
+        size_t limit = (NULL != ni->pi) ? ni->pi->options.max_integer_digits : 0;
+
+        if (0 < limit) {
+            validate_integer_size(limit, ni);
+        }
+
         if (ni->big) {
             if (256 > ni->len) {
                 char buf[256];
@@ -1193,11 +1213,19 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
             buf      = OJ_R_ALLOC_N(char, len + 1);
             pi->json = buf;
             pi->end  = buf + len;
-            if (0 >= (cnt = read(fd, (char *)pi->json, len)) || cnt != (ssize_t)len) {
-                if (0 != buf) {
-                    OJ_R_FREE(buf);
+            {
+                size_t total = 0;
+
+                while (total < len) {
+                    cnt = read(fd, (char *)pi->json + total, len - total);
+                    if (cnt <= 0) {
+                        if (0 != buf) {
+                            OJ_R_FREE(buf);
+                        }
+                        rb_raise(rb_eIOError, "failed to read from IO Object.");
+                    }
+                    total += cnt;
                 }
-                rb_raise(rb_eIOError, "failed to read from IO Object.");
             }
             ((char *)pi->json)[len] = '\0';
             /* skip UTF-8 BOM if present */