oj 3.17.1 → 3.17.6

data/ext/oj/fast.c

@@ -80,7 +80,7 @@ static void  each_leaf(Doc doc, VALUE self);
 static int   move_step(Doc doc, const char *path, int loc);
 static Leaf  get_doc_leaf(Doc doc, const char *path);
 static Leaf  get_leaf(Leaf *stack, Leaf *lp, const char *path);
-static void  each_value(Doc doc, Leaf leaf);
+static void  each_value(Doc doc, Leaf leaf, VALUE self);
 
 VALUE oj_doc_class = Qundef;
 
@@ -92,12 +92,35 @@ inline static void next_non_white(ParseInfo pi) {
         case '\f':
         case '\n':
         case '\r': break;
-        case '/': skip_comment(pi); break;
+        case '/':
+            skip_comment(pi);
+            // A comment that is not terminated by a newline ends on the null
+            // terminator. Stop here so the loop does not step past it.
+            if ('\0' == *pi->s) {
+                return;
+            }
+            break;
         default: return;
         }
     }
 }
 
+inline static bool is_value_end(char c) {
+    switch (c) {
+    case '\0':
+    case ' ':
+    case '\t':
+    case '\f':
+    case '\n':
+    case '\r':
+    case '/':
+    case ',':
+    case ']':
+    case '}': return true;
+    default: return false;
+    }
+}
+
 inline static char *ulong_fill(char *s, size_t num) {
     char  buf[32];
     char *b = buf + sizeof(buf) - 1;
@@ -210,10 +233,11 @@ static void skip_comment(ParseInfo pi) {
             if ('*' == *pi->s && '/' == *(pi->s + 1)) {
                 pi->s++;
                 return;
-            } else if ('\0' == *pi->s) {
-                raise_error("comment not terminated", pi->str, pi->s);
             }
         }
+        // The loop only ends on the null terminator so the comment was never
+        // closed.
+        raise_error("comment not terminated", pi->str, pi->s);
     } else if ('/' == *pi->s) {
         for (; 1; pi->s++) {
             switch (*pi->s) {
@@ -455,27 +479,42 @@ static Leaf read_num(ParseInfo pi) {
     char *start = pi->s;
     int   type  = T_FIXNUM;
     Leaf  leaf;
+    int   dcnt = 0;
+    int   ecnt = 1;
 
-    if ('-' == *pi->s) {
+    // read_next() sends a leading '+' here as well, and Oj.load() takes one.
+    if ('-' == *pi->s || '+' == *pi->s) {
         pi->s++;
     }
     // digits
     for (; '0' <= *pi->s && *pi->s <= '9'; pi->s++) {
+        dcnt++;
     }
     if ('.' == *pi->s) {
         type = T_FLOAT;
         pi->s++;
         for (; '0' <= *pi->s && *pi->s <= '9'; pi->s++) {
+            dcnt++;
         }
     }
     if ('e' == *pi->s || 'E' == *pi->s) {
+        // An exponent makes it a float whether or not a '.' was seen. Without
+        // this leaf_fixnum_value() stops reading at the 'e' and the exponent is
+        // dropped.
+        type = T_FLOAT;
         pi->s++;
         if ('-' == *pi->s || '+' == *pi->s) {
             pi->s++;
         }
-        for (; '0' <= *pi->s && *pi->s <= '9'; pi->s++) {
+        for (ecnt = 0; '0' <= *pi->s && *pi->s <= '9'; pi->s++) {
+            ecnt++;
         }
     }
+    // A float is converted with rb_cstr_to_dbl(), which has to be given
+    // something it can convert.
+    if (T_FLOAT == type && (0 == dcnt || 0 == ecnt || !is_value_end(*pi->s))) {
+        raise_error("invalid number", pi->str, pi->s);
+    }
     leaf      = leaf_new(pi->doc, type);
     leaf->str = start;
 
@@ -662,7 +701,11 @@ static void doc_free(Doc doc) {
 static VALUE protect_open_proc(VALUE x) {
     ParseInfo pi = (ParseInfo)x;
 
-    pi->doc->data   = read_next(pi);  // parse
+    pi->doc->data = read_next(pi);  // parse
+    // read_obj() and read_array() terminate each value where it ends. The top
+    // level value has no closing delimiter to do that on, so a number leaf runs
+    // to the end of the document unless it is terminated here.
+    *pi->s          = '\0';
     *pi->doc->where = pi->doc->data;
     pi->doc->where  = pi->doc->where_path;
     if (rb_block_given_p()) {
@@ -710,21 +753,23 @@ static void mark_doc(void *ptr) {
 }
 #ifdef HAVE_RB_GC_MARK_MOVABLE
 static void compact_leaf(Leaf leaf) {
-    switch (leaf->value_type) {
-    case COL_VAL:
-        if (NULL != leaf->elements) {
-            Leaf first = leaf->elements->next;
-            Leaf e     = first;
+    if (NULL != leaf) {
+        switch (leaf->value_type) {
+        case COL_VAL:
+            if (NULL != leaf->elements) {
+                Leaf first = leaf->elements->next;
+                Leaf e     = first;
 
-            do {
-                compact_leaf(e);
-                e = e->next;
-            } while (e != first);
-        }
-        break;
-    case RUBY_VAL: leaf->value = rb_gc_location(leaf->value); break;
+                do {
+                    compact_leaf(e);
+                    e = e->next;
+                } while (e != first);
+            }
+            break;
+        case RUBY_VAL: leaf->value = rb_gc_location(leaf->value); break;
 
-    default: break;
+        default: break;
+        }
     }
 }
 
@@ -790,14 +835,11 @@ static VALUE parse_json(VALUE clas, char *json, bool given) {
     doc->self = self;
     result    = rb_protect(protect_open_proc, (VALUE)&pi, &ex);
     if (given || 0 != ex) {
+        // The doc is detached from its wrapper so the GC will not free it.
+        // doc_free() releases both the doc and its json buffer (doc->json), so
+        // the caller must not free json separately.
         DATA_PTR(doc->self) = NULL;
-        // TBD is this needed?
-        /*
         doc_free(pi.doc);
-        if (0 != ex) {  // will jump so caller will not free
-            OJ_R_FREE(json);
-        }
-        */
     } else {
         result = doc->self;
     }
@@ -865,7 +907,7 @@ static bool key_match(const char *pat, const char *key, int plen) {
 static Leaf get_leaf(Leaf *stack, Leaf *lp, const char *path) {
     Leaf leaf = *lp;
 
-    if (MAX_STACK <= lp - stack) {
+    if (MAX_STACK <= (lp + 1) - stack) {
         rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
     }
     if ('\0' != *path) {
@@ -946,6 +988,7 @@ static void each_leaf(Doc doc, VALUE self) {
 
             doc->where++;
             if (MAX_STACK <= doc->where - doc->where_path) {
+                doc->where--;
                 rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
             }
             do {
@@ -957,11 +1000,14 @@ static void each_leaf(Doc doc, VALUE self) {
         }
     } else {
         rb_yield(self);
+        if (NULL == DATA_PTR(self)) {
+            rb_raise(rb_eIOError, "Document closed.");
+        }
     }
 }
 
 static int move_step(Doc doc, const char *path, int loc) {
-    if (MAX_STACK <= doc->where - doc->where_path) {
+    if (MAX_STACK <= (doc->where + 1) - doc->where_path) {
         rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
     }
     if ('\0' == *path) {
@@ -969,8 +1015,9 @@ static int move_step(Doc doc, const char *path, int loc) {
     } else {
         Leaf leaf;
 
+        // A document with no root (empty or comment only JSON) has no leaf to
+        // step from so the path can not be located.
         if (0 == doc->where || 0 == (leaf = *doc->where)) {
-            printf("*** Internal error at %s\n", path);
             return loc;
         }
         if ('.' == *path && '.' == *(path + 1)) {
@@ -1050,19 +1097,22 @@ static int move_step(Doc doc, const char *path, int loc) {
     return loc;
 }
 
-static void each_value(Doc doc, Leaf leaf) {
+static void each_value(Doc doc, Leaf leaf, VALUE self) {
     if (COL_VAL == leaf->value_type) {
         if (0 != leaf->elements) {
             Leaf first = leaf->elements->next;
             Leaf e     = first;
 
             do {
-                each_value(doc, e);
+                each_value(doc, e, self);
                 e = e->next;
             } while (e != first);
         }
     } else {
         rb_yield(leaf_value(doc, leaf));
+        if (NULL == DATA_PTR(self)) {
+            rb_raise(rb_eIOError, "Document closed.");
+        }
     }
 }
 
@@ -1098,12 +1148,7 @@ static VALUE doc_open(VALUE clas, VALUE str) {
 
     memcpy(json, StringValuePtr(str), len);
     obj = parse_json(clas, json, given);
-    // TBD is this needed
-    /*
-    if (given) {
-        OJ_R_FREE(json);
-    }
-    */
+    // json is owned by the doc and freed by doc_free(); do not free it here.
     return obj;
 }
 
@@ -1135,7 +1180,9 @@ static VALUE doc_open_file(VALUE clas, VALUE filename) {
     int            given = rb_block_given_p();
 
     path = StringValuePtr(filename);
-    if (0 == (f = fopen(path, "r"))) {
+    // Open in binary mode. On Windows a text mode read translates CRLF into LF
+    // so fewer bytes are read than the file size reported by ftell().
+    if (0 == (f = fopen(path, "rb"))) {
         rb_raise(rb_eIOError, "%s", strerror(errno));
     }
     fseek(f, 0, SEEK_END);
@@ -1153,12 +1200,7 @@ static VALUE doc_open_file(VALUE clas, VALUE filename) {
     fclose(f);
     json[len] = '\0';
     obj       = parse_json(clas, json, given);
-    // TBD is this needed
-    /*
-    if (given) {
-        OJ_R_FREE(json);
-    }
-    */
+    // json is owned by the doc and freed by doc_free(); do not free it here.
     return obj;
 }
 
@@ -1257,10 +1299,14 @@ static VALUE doc_path(VALUE self) {
  * #=> nil
  */
 static VALUE doc_local_key(VALUE self) {
-    Doc            doc  = self_doc(self);
-    Leaf           leaf = *doc->where;
-    volatile VALUE key  = Qnil;
+    Doc            doc = self_doc(self);
+    Leaf           leaf;
+    volatile VALUE key = Qnil;
 
+    if (NULL == doc->where || NULL == *doc->where) {
+        return Qnil;
+    }
+    leaf = *doc->where;
     if (T_HASH == leaf->parent_type) {
         key = rb_utf8_str_new_cstr(leaf->key);
     } else if (T_ARRAY == leaf->parent_type) {
@@ -1415,6 +1461,9 @@ static VALUE doc_each_leaf(int argc, VALUE *argv, VALUE self) {
                 return Qnil;
             }
         }
+        if (NULL == doc->where || NULL == *doc->where) {
+            return Qnil;
+        }
         each_leaf(doc, self);
         if (0 < wlen) {
             memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));
@@ -1498,12 +1547,19 @@ static VALUE doc_each_child(int argc, VALUE *argv, VALUE self) {
             Leaf first = (*doc->where)->elements->next;
             Leaf e     = first;
 
+            if (MAX_STACK <= (doc->where + 1) - doc->where_path) {
+                rb_raise(rb_const_get_at(Oj, rb_intern("DepthError")), "Path too deep. Limit is %d levels.", MAX_STACK);
+            }
             doc->where++;
             do {
                 *doc->where = e;
                 rb_yield(self);
+                if (NULL == DATA_PTR(self)) {
+                    rb_raise(rb_eIOError, "Document closed.");
+                }
                 e = e->next;
             } while (e != first);
+            doc->where--;
         }
         if (0 < wlen) {
             memcpy(doc->where_path, save_path, sizeof(Leaf) * (wlen + 1));
@@ -1547,7 +1603,7 @@ static VALUE doc_each_value(int argc, VALUE *argv, VALUE self) {
             path = StringValuePtr(*argv);
         }
         if (0 != (leaf = get_doc_leaf(doc, path))) {
-            each_value(doc, leaf);
+            each_value(doc, leaf, self);
         }
     }
     return Qnil;

data/ext/oj/intern.c

@@ -69,7 +69,7 @@ static VALUE form_attr(const char *str, size_t len) {
             memcpy(b + 1, str, len);
             b[len + 1] = '\0';
         }
-        id = rb_intern3(buf, len + 1, oj_utf8_encoding);
+        id = rb_intern3(b, len + 1, oj_utf8_encoding);
         OJ_R_FREE(b);
         return id;
     }
@@ -97,6 +97,8 @@ static const rb_data_type_t oj_cache_type = {
 };
 
 void oj_hash_init(void) {
+    oj_hash_seed_init();
+
     VALUE cache_class = rb_define_class_under(Oj, "Cache", rb_cObject);
     rb_undef_alloc_func(cache_class);
 
@@ -151,7 +153,7 @@ ID oj_attr_intern(const char *key, size_t len) {
 static uint64_t hash_calc(const uint8_t *key, size_t len) {
     const uint8_t *end     = key + len;
     const uint8_t *endless = key + (len & 0xFFFFFFFC);
-    uint64_t       h       = (uint64_t)len;
+    uint64_t       h       = (uint64_t)len ^ oj_hash_seed;
     uint64_t       k;
 
     while (key < endless) {
@@ -188,6 +190,9 @@ static VALUE resolve_classname(VALUE mod, const char *classname, int auto_define
 
     if (rb_const_defined_at(mod, ci)) {
         clas = rb_const_get_at(mod, ci);
+        if (!RB_TYPE_P(clas, T_CLASS) && !RB_TYPE_P(clas, T_MODULE)) {
+            clas = Qundef;
+        }
     } else if (auto_define) {
         clas = rb_define_class_under(mod, classname, oj_bag_class);
     } else {

data/ext/oj/object.c

@@ -31,7 +31,7 @@ inline static long read_long(const char *str, size_t len) {
 static VALUE calc_hash_key(ParseInfo pi, Val kval, char k1) {
     volatile VALUE rkey;
 
-    if (':' == k1) {
+    if (':' == k1 && 0 < kval->klen) {
         return ID2SYM(rb_intern3(kval->key + 1, kval->klen - 1, oj_utf8_encoding));
     }
     if (Yes == pi->options.sym_key) {
@@ -54,7 +54,7 @@ static VALUE str_to_value(ParseInfo pi, const char *str, size_t len, const char
     } else if (pi->circ_array && 3 <= len && '^' == *orig && 'r' == orig[1]) {
         long i = read_long(str + 2, len - 2);
 
-        if (0 > i) {
+        if (0 >= i) {
             oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a valid ID number");
             return Qnil;
         }
@@ -207,7 +207,7 @@ oj_parse_xml_time(const char *str, int len) {
 
 static int hat_cstr(ParseInfo pi, Val parent, Val kval, const char *str, size_t len) {
     const char *key  = kval->key;
-    int         klen = kval->klen;
+    size_t      klen = kval->klen;
 
     if (2 == klen) {
         switch (key[1]) {
@@ -230,7 +230,12 @@ static int hat_cstr(ParseInfo pi, Val parent, Val kval, const char *str, size_t
             parent->odd_args = oj_odd_alloc_args(odd);
             break;
         }
-        case 'm': parent->val = ID2SYM(rb_intern3(str + 1, len - 1, oj_utf8_encoding)); break;
+        case 'm':
+            if (0 == len) {
+                return 0;
+            }
+            parent->val = ID2SYM(rb_intern3(str + 1, len - 1, oj_utf8_encoding));
+            break;
         case 's': parent->val = rb_utf8_str_new(str, len); break;
         case 'c':  // class
         {
@@ -319,20 +324,26 @@ static int hat_value(ParseInfo pi, Val parent, const char *key, size_t klen, vol
             e1 = *RARRAY_CONST_PTR(value);
             // check for anonymous Struct
             if (T_ARRAY == rb_type(e1)) {
-                VALUE          args[1024];
-                volatile VALUE rstr;
+                // Build the member-name list in a GC-managed Ruby array. Using a
+                // fixed C buffer (VALUE args[1024]) here overflowed the stack when
+                // the JSON supplied more than 1024 member names.
+                volatile VALUE names = rb_ary_new2(RARRAY_LEN(e1));
                 size_t         i;
                 size_t         cnt = RARRAY_LEN(e1);
 
                 for (i = 0; i < cnt; i++) {
-                    rstr    = RARRAY_AREF(e1, i);
-                    args[i] = rb_funcall(rstr, oj_to_sym_id, 0);
+                    rb_ary_push(names, rb_funcall(RARRAY_AREF(e1, i), oj_to_sym_id, 0));
                 }
-                sc = rb_funcall2(rb_cStruct, oj_new_id, (int)cnt, args);
+                sc = rb_apply(rb_cStruct, oj_new_id, names);
             } else {
                 // If struct is not defined then we let this fail and raise an exception.
                 sc = oj_name2struct(pi, *RARRAY_CONST_PTR(value), rb_eArgError);
             }
+            if (Qundef == sc || Qnil == sc) {
+                // oj_name2struct already recorded the error; do not pass an
+                // unresolved (Qundef) class on to rb_obj_alloc/rb_class_new_instance.
+                return 1;
+            }
             if (sc == rb_cRange) {
                 parent->val = rb_class_new_instance((int)(len - 1), RARRAY_CONST_PTR(value) + 1, rb_cRange);
             } else {
@@ -389,7 +400,7 @@ void oj_set_obj_ivar(Val parent, Val kval, VALUE value) {
 
 static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, const char *orig) {
     const char    *key    = kval->key;
-    int            klen   = kval->klen;
+    size_t         klen   = kval->klen;
     Val            parent = stack_peek(&pi->stack);
     volatile VALUE rval   = Qnil;
 
@@ -431,7 +442,7 @@ WHICH_TYPE:
             if (0 != oj_odd_set_arg(parent->odd_args, kval->key, kval->klen, rval)) {
                 char buf[256];
 
-                if ((int)sizeof(buf) - 1 <= klen) {
+                if (sizeof(buf) - 1 <= klen) {
                     klen = sizeof(buf) - 2;
                 }
                 memcpy(buf, key, klen);
@@ -460,7 +471,7 @@ WHICH_TYPE:
 
 static void hash_set_num(ParseInfo pi, Val kval, NumInfo ni) {
     const char    *key    = kval->key;
-    int            klen   = kval->klen;
+    size_t         klen   = kval->klen;
     Val            parent = stack_peek(&pi->stack);
     volatile VALUE rval   = Qnil;
 
@@ -500,7 +511,7 @@ WHICH_TYPE:
             if (0 != oj_odd_set_arg(parent->odd_args, key, klen, rval)) {
                 char buf[256];
 
-                if ((int)sizeof(buf) - 1 <= klen) {
+                if (sizeof(buf) - 1 <= klen) {
                     klen = sizeof(buf) - 2;
                 }
                 memcpy(buf, key, klen);
@@ -529,7 +540,7 @@ WHICH_TYPE:
 
 static void hash_set_value(ParseInfo pi, Val kval, VALUE value) {
     const char *key    = kval->key;
-    int         klen   = kval->klen;
+    size_t      klen   = kval->klen;
     Val         parent = stack_peek(&pi->stack);
 
 WHICH_TYPE:
@@ -585,7 +596,7 @@ WHICH_TYPE:
         } else if (0 != oj_odd_set_arg(parent->odd_args, key, klen, value)) {
             char buf[256];
 
-            if ((int)sizeof(buf) - 1 <= klen) {
+            if (sizeof(buf) - 1 <= klen) {
                 klen = sizeof(buf) - 2;
             }
             memcpy(buf, key, klen);

data/ext/oj/odd.c

@@ -166,7 +166,11 @@ Odd oj_get_oddc(const char *classname, size_t len) {
         if (len == odd->clen && 0 == strncmp(classname, odd->classname, len)) {
             return odd;
         }
-        if (odd->is_module && 0 == strncmp(odd->classname, classname, odd->clen) && ':' == classname[odd->clen]) {
+        // The module test needs a "::" after the module name, so the name has
+        // to be shorter than what it is compared against. Without that the
+        // comparison and the character after it both run past len.
+        if (odd->is_module && odd->clen < len && 0 == strncmp(odd->classname, classname, odd->clen) &&
+            ':' == classname[odd->clen]) {
             return odd;
         }
     }
@@ -195,7 +199,7 @@ int oj_odd_set_arg(OddArgs args, const char *key, size_t klen, VALUE value) {
     int          i;
 
     for (i = args->odd->attr_cnt, np = args->odd->attr_names, vp = args->args; 0 < i; i--, np++, vp++) {
-        if (0 == strncmp(key, *np, klen) && '\0' == *((*np) + klen)) {
+        if (klen == strlen(*np) && 0 == memcmp(key, *np, klen)) {
             *vp = value;
             return 0;
         }