oj 3.17.1 → 3.17.6

data/ext/oj/oj.c

@@ -20,6 +20,8 @@
 #include "rails.h"
 #include "simd.h"
 
+#define MAX_INDENT 16
+
 typedef struct _yesNoOpt {
     VALUE sym;
     char *attr;
@@ -297,7 +299,8 @@ static VALUE only_array_from_string(const char *str) {
  *   floats, 0 indicates use Ruby
  * - *:float_format* [_String_] the C printf format string for printing floats.
  *   Default follows the float_precision and will be changed if float_precision is
- *   changed. The string can be no more than 6 bytes.
+ *   changed. The string can be no more than 6 bytes and can hold at most one
+ *   directive, which must be one of aAeEfgG and take no argument of its own.
  * - *:use_to_json* [_Boolean_|_nil_] call to_json() methods on dump, default is false
  * - *:use_as_json* [_Boolean_|_nil_] call as_json() methods on dump, default is false
  * - *:use_raw_json* [_Boolean_|_nil_] call raw_json() methods on dump, default is false
@@ -572,7 +575,8 @@ static VALUE get_def_opts(VALUE self) {
  *     when dumping the seconds portion of time.
  *   - *:float_format* [_String_] the C printf format string for printing floats.
  *     Default follows the float_precision and will be changed if float_precision
- *     is changed. The string can be no more than 6 bytes.
+ *     is changed. The string can be no more than 6 bytes and can hold at most one
+ *     directive, which must be one of aAeEfgG and take no argument of its own.
  *   - *:float_precision* [_Fixnum_|_nil_] number of digits of precision when dumping floats, 0 indicates use Ruby.
  *   - *:use_to_json* [_Boolean_|_nil_] call to_json() methods on dump, default is false.
  *   - *:use_as_json* [_Boolean_|_nil_] call as_json() methods on dump, default is false.
@@ -604,9 +608,52 @@ static VALUE get_def_opts(VALUE self) {
  *   - *:trace* [_Boolean_] turn trace on or off.
  *   - *:safe* [_Boolean_] turn safe mimic on or off.
  */
+static ID oj_default_options_keeper_id = 0;
+
+// The default options are a plain C global so the Ruby objects in them, the
+// classes of the hash_class, array_class, and ignore options and the regexps
+// and the classes of the match_string option, can not be reached by the GC and
+// are collected while the defaults still refer to them. Holding them in an
+// array the GC does know about keeps them alive. The array is rebuilt whenever
+// the defaults change and replaces the previous one so that only the objects
+// the defaults are currently using are held.
+static void keep_default_options(void) {
+    VALUE keep = rb_ary_new();
+
+    if (0 == oj_default_options_keeper_id) {
+        oj_default_options_keeper_id = rb_intern("@default_options_keeper");
+    }
+    if (Qnil != oj_default_options.hash_class) {
+        rb_ary_push(keep, oj_default_options.hash_class);
+    }
+    if (Qnil != oj_default_options.array_class) {
+        rb_ary_push(keep, oj_default_options.array_class);
+    }
+    if (NULL != oj_default_options.ignore) {
+        VALUE *vp;
+
+        for (vp = oj_default_options.ignore; Qnil != *vp; vp++) {
+            rb_ary_push(keep, *vp);
+        }
+    }
+    oj_rxclass_keep(&oj_default_options.str_rx, keep);
+    // An instance variable of the Oj module is a reference the GC can follow on
+    // every engine, unlike a C global.
+    rb_ivar_set(Oj, oj_default_options_keeper_id, keep);
+}
+
 static VALUE set_def_opts(VALUE self, VALUE opts) {
     Check_Type(opts, T_HASH);
+    // A :match_string option replaces the regexps instead of adding to them so
+    // the ones the defaults are holding have to be freed here, the only place
+    // that owns them. A per call options struct only aliases them.
+    if (Qnil != rb_hash_lookup(opts, match_string_sym)) {
+        oj_rxclass_cleanup(&oj_default_options.str_rx);
+        oj_default_options.str_rx.head = NULL;
+        oj_default_options.str_rx.tail = NULL;
+    }
     oj_parse_options(opts, &oj_default_options);
+    keep_default_options();
 
     return Qnil;
 }
@@ -660,6 +707,46 @@ bool set_yesno_options(VALUE key, VALUE value, Options copts) {
     return false;
 }
 
+inline static bool in_set(const char *set, char c) {
+    return '\0' != c && NULL != strchr(set, c);
+}
+
+// The format is used with one double and nothing else, so a directive that
+// asks for anything more takes it from whatever is left in the argument
+// registers. %s reads it as a pointer and %n writes through it.
+static void validate_float_format(const char *str, size_t len) {
+    const char *s   = str;
+    const char *end = str + len;
+    int         cnt = 0;
+
+    for (; s < end; s++) {
+        if ('%' != *s) {
+            continue;
+        }
+        s++;
+        if (s < end && '%' == *s) {
+            continue;
+        }
+        if (1 < ++cnt) {
+            rb_raise(rb_eArgError, ":float_format must not have more than one directive.");
+        }
+        for (; s < end && in_set("-+ #0", *s); s++) {
+        }
+        for (; s < end && '0' <= *s && *s <= '9'; s++) {
+        }
+        if (s < end && '.' == *s) {
+            for (s++; s < end && '0' <= *s && *s <= '9'; s++) {
+            }
+        }
+        if (s < end && 'l' == *s) {
+            s++;
+        }
+        if (end <= s || !in_set("aAeEfgG", *s)) {
+            rb_raise(rb_eArgError, ":float_format directive must be one of aAeEfgG and take no argument of its own.");
+        }
+    }
+}
+
 static const char *make_only_value(VALUE v) {
     switch (rb_type(v)) {
     case RUBY_T_NIL:
@@ -749,12 +836,14 @@ static const char *make_only_value(VALUE v) {
     return NULL;
 }
 
-static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
-    Options copts = (Options)opts;
-    size_t  len;
+// Apply one key and value to copts. Returns true if the key named an option Oj
+// knows and false if it did not. An unrecognized key is still ignored rather
+// than an error; the return only lets a caller tell the two cases apart.
+static bool parse_one_option(VALUE k, VALUE v, Options copts) {
+    size_t len;
 
     if (set_yesno_options(k, v, copts)) {
-        return ST_CONTINUE;
+        return true;
     }
     if (oj_indent_sym == k) {
         switch (rb_type(v)) {
@@ -766,7 +855,10 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         case T_FIXNUM:
             copts->dump_opts.indent_size = 0;
             *copts->dump_opts.indent_str = '\0';
-            copts->indent                = FIX2INT(v);
+            if (MAX_INDENT < FIX2INT(v)) {
+                rb_raise(rb_eArgError, "indent is limited to %d characters.", MAX_INDENT);
+            }
+            copts->indent = FIX2INT(v);
             break;
         case T_STRING:
             if (sizeof(copts->dump_opts.indent_str) <= (len = RSTRING_LEN(v))) {
@@ -877,7 +969,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (bigdecimal_load_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
 
         if (bigdecimal_sym == v || Qtrue == v) {
@@ -893,7 +985,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (compat_bigdecimal_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
         copts->compat_bigdec = (Qtrue == v);
     } else if (oj_decimal_class_sym == k) {
@@ -905,9 +997,13 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
             rb_raise(rb_eArgError, ":decimal_class must be BigDecimal or Float.");
         }
     } else if (create_id_sym == k) {
+        // A per call copy of the options aliases the buffer the default
+        // options own, so only the defaults may free it.
+        bool owned = (&oj_default_options == copts && NULL != copts->create_id && oj_json_class != copts->create_id);
+
         if (Qnil == v) {
-            if (oj_json_class != oj_default_options.create_id && NULL != copts->create_id) {
-                OJ_R_FREE((char *)oj_default_options.create_id);
+            if (owned) {
+                OJ_R_FREE((char *)copts->create_id);
             }
             copts->create_id     = NULL;
             copts->create_id_len = 0;
@@ -915,7 +1011,10 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
             const char *str = StringValuePtr(v);
 
             len = RSTRING_LEN(v);
-            if (len != copts->create_id_len || 0 != strcmp(copts->create_id, str)) {
+            if (NULL == copts->create_id || len != copts->create_id_len || 0 != strcmp(copts->create_id, str)) {
+                if (owned) {
+                    OJ_R_FREE((char *)copts->create_id);
+                }
                 copts->create_id = OJ_R_ALLOC_N(char, len + 1);
                 strcpy((char *)copts->create_id, str);
                 copts->create_id_len = len;
@@ -981,7 +1080,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (nan_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
         if (null_sym == v) {
             copts->dump_opts.nan_dump = NullNan;
@@ -998,7 +1097,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (omit_nil_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
         if (Qtrue == v) {
             copts->dump_opts.omit_nil = true;
@@ -1009,7 +1108,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (omit_null_byte_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
         if (Qtrue == v) {
             copts->dump_opts.omit_null_byte = true;
@@ -1047,7 +1146,11 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
             copts->array_class = v;
         }
     } else if (ignore_sym == k) {
-        OJ_R_FREE(copts->ignore);
+        // Only free when replacing the defaults; a per-call copy's ignore still
+        // aliases the default-owned buffer, which oj_free_call_options() frees.
+        if (&oj_default_options == copts) {
+            OJ_R_FREE(copts->ignore);
+        }
         copts->ignore = NULL;
         if (Qnil != v) {
             size_t cnt;
@@ -1066,7 +1169,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (integer_range_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
         if (rb_obj_class(v) == rb_cRange) {
             VALUE min = rb_funcall(v, oj_begin_id, 0);
@@ -1097,7 +1200,7 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         }
     } else if (symbol_keys_sym == k || oj_symbolize_names_sym == k) {
         if (Qnil == v) {
-            return ST_CONTINUE;
+            return true;
         }
         copts->sym_key = (Qtrue == v) ? Yes : No;
 
@@ -1117,35 +1220,180 @@ static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
         if (6 < RSTRING_LEN(v)) {
             rb_raise(rb_eArgError, ":float_format must be 6 bytes or less.");
         }
+        validate_float_format(RSTRING_PTR(v), (size_t)RSTRING_LEN(v));
         strncpy(copts->float_fmt, RSTRING_PTR(v), (size_t)RSTRING_LEN(v));
         copts->float_fmt[RSTRING_LEN(v)] = '\0';
     } else if (only_sym == k) {
-        if (NULL != copts->dump_opts.only) {
+        // Only free when replacing the defaults; a per-call copy's pointer still
+        // aliases the default-owned buffer, which oj_free_call_options() frees.
+        if (&oj_default_options == copts && NULL != copts->dump_opts.only) {
             OJ_R_FREE((void *)copts->dump_opts.only);
-            copts->dump_opts.only = NULL;
         }
         copts->dump_opts.only = make_only_value(v);
     } else if (except_sym == k) {
-        if (NULL != copts->dump_opts.except) {
+        if (&oj_default_options == copts && NULL != copts->dump_opts.except) {
             OJ_R_FREE((void *)copts->dump_opts.except);
-            copts->dump_opts.except = NULL;
         }
         copts->dump_opts.except = make_only_value(v);
+    } else {
+        return false;
+    }
+    return true;
+}
+
+// Collects the return of parse_one_option() over every key in a hash.
+struct _optsParse {
+    Options copts;
+    bool    consumed;
+};
+
+static int parse_options_cb(VALUE k, VALUE v, VALUE opts) {
+    struct _optsParse *op = (struct _optsParse *)opts;
+
+    if (parse_one_option(k, v, op->copts)) {
+        op->consumed = true;
     }
     return ST_CONTINUE;
 }
 
-void oj_parse_options(VALUE ropts, Options copts) {
+// Apply the options in ropts to copts and report whether any key was
+// recognized. A hash of nothing but unknown keys leaves copts untouched and
+// returns false.
+bool oj_parse_options_consumed(VALUE ropts, Options copts) {
+    struct _optsParse op = {copts, false};
+
     if (T_HASH != rb_type(ropts)) {
-        return;
+        return false;
+    }
+    rb_hash_foreach(ropts, parse_options_cb, (VALUE)&op);
+    if (Qnil != rb_hash_lookup(ropts, match_string_sym)) {
+        op.consumed = true;
     }
-    rb_hash_foreach(ropts, parse_options_cb, (VALUE)copts);
     oj_parse_opt_match_string(&copts->str_rx, ropts);
 
     copts->dump_opts.use = (0 < copts->dump_opts.indent_size || 0 < copts->dump_opts.after_size ||
                             0 < copts->dump_opts.before_size || 0 < copts->dump_opts.hash_size ||
                             0 < copts->dump_opts.array_size);
-    return;
+
+    return op.consumed;
+}
+
+void oj_parse_options(VALUE ropts, Options copts) {
+    oj_parse_options_consumed(ropts, copts);
+}
+
+// Free the option buffers that oj_parse_options() allocated for a single call.
+// A per-call options struct is a copy of oj_default_options, so its create_id,
+// ignore, only, and except members initially alias the ones owned by the
+// defaults. Only free a member when it differs from the default, i.e. when it
+// was allocated for this call; the default-owned buffers must be left alone.
+void oj_free_call_options(Options copts) {
+    if (NULL != copts->create_id && oj_default_options.create_id != copts->create_id) {
+        OJ_R_FREE((char *)copts->create_id);
+        copts->create_id     = NULL;
+        copts->create_id_len = 0;
+    }
+    if (NULL != copts->ignore && oj_default_options.ignore != copts->ignore) {
+        OJ_R_FREE(copts->ignore);
+        copts->ignore = NULL;
+    }
+    if (NULL != copts->dump_opts.only && oj_default_options.dump_opts.only != copts->dump_opts.only) {
+        OJ_R_FREE((void *)copts->dump_opts.only);
+        copts->dump_opts.only = NULL;
+    }
+    if (NULL != copts->dump_opts.except && oj_default_options.dump_opts.except != copts->dump_opts.except) {
+        OJ_R_FREE((void *)copts->dump_opts.except);
+        copts->dump_opts.except = NULL;
+    }
+}
+
+static char *dup_option_str(const char *str, size_t len) {
+    char *buf = OJ_R_ALLOC_N(char, len + 1);
+
+    memcpy(buf, str, len);
+    buf[len] = '\0';
+
+    return buf;
+}
+
+// Give an options struct private copies of the option buffers it still shares
+// with the defaults so that it owns all of them.
+//
+// An options struct that outlives the call it was created in can not rely on
+// the default-owned buffers it aliases: Oj.default_options= is free to replace
+// them at any time, which would leave the alias dangling. Call this once, in
+// the call that creates such an object and after oj_parse_options(), then free
+// the buffers with oj_options_release() when the object is collected.
+void oj_options_take_ownership(Options copts) {
+    if (NULL != copts->create_id && oj_default_options.create_id == copts->create_id) {
+        copts->create_id = dup_option_str(copts->create_id, copts->create_id_len);
+    }
+    if (NULL != copts->ignore && oj_default_options.ignore == copts->ignore) {
+        VALUE *vp;
+        VALUE *buf;
+        size_t cnt = 0;
+
+        for (vp = copts->ignore; Qnil != *vp; vp++) {
+            cnt++;
+        }
+        buf = OJ_R_ALLOC_N(VALUE, cnt + 1);
+        memcpy(buf, copts->ignore, sizeof(VALUE) * (cnt + 1));
+        copts->ignore = buf;
+    }
+    if (NULL != copts->dump_opts.only && oj_default_options.dump_opts.only == copts->dump_opts.only) {
+        copts->dump_opts.only = dup_option_str(copts->dump_opts.only, strlen(copts->dump_opts.only));
+    }
+    if (NULL != copts->dump_opts.except && oj_default_options.dump_opts.except == copts->dump_opts.except) {
+        copts->dump_opts.except = dup_option_str(copts->dump_opts.except, strlen(copts->dump_opts.except));
+    }
+}
+
+// Free the option buffers of an options struct that owns all of them, i.e. one
+// that oj_options_take_ownership() was called on. Unlike oj_free_call_options()
+// this never looks at the defaults, so it is safe to call from a GC free
+// function, where the defaults may have been replaced since the object was
+// created.
+void oj_options_release(Options copts) {
+    if (NULL != copts->create_id) {
+        OJ_R_FREE((char *)copts->create_id);
+        copts->create_id     = NULL;
+        copts->create_id_len = 0;
+    }
+    if (NULL != copts->ignore) {
+        OJ_R_FREE(copts->ignore);
+        copts->ignore = NULL;
+    }
+    if (NULL != copts->dump_opts.only) {
+        OJ_R_FREE((void *)copts->dump_opts.only);
+        copts->dump_opts.only = NULL;
+    }
+    if (NULL != copts->dump_opts.except) {
+        OJ_R_FREE((void *)copts->dump_opts.except);
+        copts->dump_opts.except = NULL;
+    }
+    // The str_rx chain is detached from the defaults when the owner is created
+    // so the chain, if there is one, was built for this options struct alone.
+    oj_rxclass_cleanup(&copts->str_rx);
+}
+
+// Mark the Ruby objects held by an owned options struct. A long lived object
+// such as a writer or an encoder can be the only reference to the classes in
+// the options so without this they could be collected while the owner is still
+// alive leaving the options with dangling VALUEs.
+void oj_options_mark(Options copts) {
+    if (Qnil != copts->hash_class) {
+        rb_gc_mark(copts->hash_class);
+    }
+    if (Qnil != copts->array_class) {
+        rb_gc_mark(copts->array_class);
+    }
+    if (NULL != copts->ignore) {
+        VALUE *vp;
+
+        for (vp = copts->ignore; Qnil != *vp; vp++) {
+            rb_gc_mark(*vp);
+        }
+    }
 }
 
 static int match_string_cb(VALUE key, VALUE value, VALUE rx) {
@@ -1439,6 +1687,7 @@ static VALUE dump_ensure(VALUE a) {
     volatile struct dump_arg *arg = (void *)a;
 
     oj_out_free(arg->out);
+    oj_free_call_options(arg->copts);
 
     return Qnil;
 }
@@ -1554,6 +1803,7 @@ static VALUE to_file(int argc, VALUE *argv, VALUE self) {
         oj_parse_options(argv[2], &copts);
     }
     oj_write_obj_to_file(argv[1], StringValuePtr(*argv), &copts);
+    oj_free_call_options(&copts);
 
     return Qnil;
 }
@@ -1575,6 +1825,7 @@ static VALUE to_stream(int argc, VALUE *argv, VALUE self) {
         oj_parse_options(argv[2], &copts);
     }
     oj_write_obj_to_stream(argv[1], *argv, &copts);
+    oj_free_call_options(&copts);
 
     return Qnil;
 }

data/ext/oj/oj.h

@@ -197,6 +197,7 @@ typedef struct _out {
     bool      allocated;
     bool      omit_nil;
     bool      omit_null_byte;
+    bool      key_filter_off;  // rails: suppress the :only/:except key filter for an as_json result
     int       argc;
     VALUE    *argv;
     ROptTable ropts;
@@ -256,6 +257,11 @@ extern VALUE oj_custom_parse_cstr(int argc, VALUE *argv, char *json, size_t len)
 
 extern bool oj_hash_has_key(VALUE hash, VALUE key);
 extern void oj_parse_options(VALUE ropts, Options copts);
+extern bool oj_parse_options_consumed(VALUE ropts, Options copts);
+extern void oj_free_call_options(Options copts);
+extern void oj_options_take_ownership(Options copts);
+extern void oj_options_release(Options copts);
+extern void oj_options_mark(Options copts);
 
 extern void  oj_dump_obj_to_json(VALUE obj, Options copts, Out out);
 extern void  oj_dump_obj_to_json_using_params(VALUE obj, Options copts, Out out, int argc, VALUE *argv);
@@ -321,7 +327,6 @@ extern VALUE oj_max_nesting_sym;
 extern VALUE oj_object_class_sym;
 extern VALUE oj_object_nl_sym;
 extern VALUE oj_quirks_mode_sym;
-extern VALUE oj_skip_null_byte_sym;
 extern VALUE oj_space_before_sym;
 extern VALUE oj_space_sym;
 extern VALUE oj_symbolize_names_sym;

data/ext/oj/parse.c

@@ -394,7 +394,7 @@ void oj_scanner_init(void) {
 static void read_escaped_str(ParseInfo pi, const char *start) {
     struct _buf buf;
     const char *s;
-    int         cnt = (int)(pi->cur - start);
+    size_t      cnt = pi->cur - start;
     uint32_t    code;
     Val         parent = stack_peek(&pi->stack);
 
@@ -414,6 +414,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
 
         if ('\\' == *s) {
             s++;
+            if (pi->end <= s) {
+                oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
+                buf_cleanup(&buf);
+                return;
+            }
             switch (*s) {
             case 'n': buf_append(&buf, '\n'); break;
             case 'r': buf_append(&buf, '\r'); break;
@@ -669,7 +674,7 @@ static void read_num(ParseInfo pi) {
             // A trailing . is not a valid decimal but if encountered allow it
             // except when mimicking the JSON gem or in strict mode.
             if (StrictMode == pi->options.mode || CompatMode == pi->options.mode) {
-                int pos = (int)(pi->cur - ni.str);
+                size_t pos = pi->cur - ni.str;
 
                 if (1 == pos || (2 == pos && ni.neg)) {
                     oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number");
@@ -1096,7 +1101,9 @@ void oj_set_error_at(ParseInfo pi, VALUE err_clas, const char *file, int line, c
     }
     va_end(ap);
     pi->err.clas = err_clas;
-    if (p + 3 < end) {
+    // The eight bytes below, plus the ')' and the terminator that the two bytes
+    // held back from end are for.
+    if (p + 8 <= end) {
         *p++  = ' ';
         *p++  = '(';
         *p++  = 'a';
@@ -1306,9 +1313,16 @@ CLEANUP:
         OJ_R_FREE(json);
     }
     stack_cleanup(&pi->stack);
-    if (pi->str_rx.head != oj_default_options.str_rx.head) {
-        oj_rxclass_cleanup(&pi->str_rx);
+    // The parsers match against pi->options.str_rx. A :match_string option
+    // builds a chain of its own for this call, which has to be freed. Without
+    // the option the member still aliases the chain owned by the defaults and
+    // must be left alone.
+    if (pi->options.str_rx.head != oj_default_options.str_rx.head) {
+        oj_rxclass_cleanup(&pi->options.str_rx);
+        pi->options.str_rx.head = NULL;
+        pi->options.str_rx.tail = NULL;
     }
+    oj_free_call_options(&pi->options);
     if (err_has(&pi->err)) {
         rb_set_errinfo(Qnil);
         if (Qnil != pi->err_class) {

data/ext/oj/parse.h

@@ -49,7 +49,6 @@ typedef struct _parseInfo {
     VALUE            handler;
     struct _valStack stack;
     CircArray        circ_array;
-    struct _rxClass  str_rx;
     int              expect_value;
     int              max_depth;  // just for the json gem
     VALUE            proc;