ox 2.14.28 → 2.14.29

data/ext/ox/parse.c

@@ -5,6 +5,7 @@
 
 #include <errno.h>
 #include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -19,9 +20,6 @@
 #include "special.h"
 
 #define MAX_ELEMENT_DEPTH 1000
-// Anything even close to the max prolog length is most likely someone trying
-// to break something.
-#define MAX_PROLOG 32767
 
 static void  mark_pi_cb(void *ptr);
 static void  read_instruction(PInfo pi);
@@ -140,115 +138,155 @@ static void mark_pi_cb(void *ptr) {
     }
 }
 
-VALUE
-ox_parse(char *xml, size_t len, ParseCallbacks pcb, char **endp, Options options, Err err) {
-    struct _pInfo  pi;
-    int            body_read   = 0;
-    int            block_given = rb_block_given_p();
-    volatile VALUE wrap;
+// State shared between the parse body and its ensure block. Bundled so the
+// whole parse can run under rb_ensure and clean up even when a callback raises.
+typedef struct _parseCtx {
+    PInfo  pi;
+    char **endp;
+    Err    err;
+    VALUE  wrap;
+    int    block_given;
+} *ParseCtx;
+
+static VALUE ox_parse_body(VALUE ctxv) {
+    ParseCtx ctx         = (ParseCtx)ctxv;
+    PInfo    pi          = ctx->pi;
+    char   **endp        = ctx->endp;
+    Err      err         = ctx->err;
+    int      block_given = ctx->block_given;
+    int      body_read   = 0;
 
-    if (0 == xml) {
-        set_error(err, "Invalid arg, xml string can not be null", xml, 0);
-        return Qnil;
-    }
-    if (DEBUG <= options->trace) {
-        printf("Parsing xml:\n%s\n", xml);
-    }
-    // initialize parse info
-    helper_stack_init(&pi.helpers);
-    // Protect against GC
-    wrap = TypedData_Wrap_Struct(rb_cObject, &ox_wrap_type, &pi);
-
-    err_init(&pi.err);
-    pi.str        = xml;
-    pi.end        = pi.str + len;
-    pi.s          = xml;
-    pi.pcb        = pcb;
-    pi.obj        = Qnil;
-    pi.circ_array = 0;
-    pi.options    = options;
-    pi.marked     = NULL;
-    pi.mark_size  = 0;
-    pi.mark_cnt   = 0;
     while (1) {
-        next_non_white(&pi);  // skip white space
-        if ('\0' == *pi.s) {
+        next_non_white(pi);  // skip white space
+        if ('\0' == *pi->s) {
             break;
         }
         if (body_read && 0 != endp) {
-            *endp = pi.s;
+            *endp = pi->s;
             break;
         }
-        if ('<' != *pi.s) {  // all top level entities start with <
-            set_error(err, "invalid format, expected <", pi.str, pi.s);
-            helper_stack_cleanup(&pi.helpers);
+        if ('<' != *pi->s) {  // all top level entities start with <
+            set_error(err, "invalid format, expected <", pi->str, pi->s);
             return Qnil;
         }
-        pi.s++;  // past <
-        switch (*pi.s) {
+        pi->s++;  // past <
+        switch (*pi->s) {
         case '?':  // processing instruction
-            pi.s++;
-            read_instruction(&pi);
+            pi->s++;
+            read_instruction(pi);
             break;
         case '!':  // comment or doctype
-            pi.s++;
-            if ('\0' == *pi.s) {
-                set_error(err, "invalid format, DOCTYPE or comment not terminated", pi.str, pi.s);
-                helper_stack_cleanup(&pi.helpers);
+            pi->s++;
+            if ('\0' == *pi->s) {
+                set_error(err, "invalid format, DOCTYPE or comment not terminated", pi->str, pi->s);
                 return Qnil;
-            } else if ('-' == *pi.s) {
-                pi.s++;  // skip -
-                if ('-' != *pi.s) {
-                    set_error(err, "invalid format, bad comment format", pi.str, pi.s);
-                    helper_stack_cleanup(&pi.helpers);
+            } else if ('-' == *pi->s) {
+                pi->s++;  // skip -
+                if ('-' != *pi->s) {
+                    set_error(err, "invalid format, bad comment format", pi->str, pi->s);
                     return Qnil;
                 } else {
-                    pi.s++;  // skip second -
-                    read_comment(&pi);
+                    pi->s++;  // skip second -
+                    read_comment(pi);
                 }
-            } else if ((TolerantEffort == options->effort) ? 0 == strncasecmp("DOCTYPE", pi.s, 7)
-                                                           : 0 == strncmp("DOCTYPE", pi.s, 7)) {
-                pi.s += 7;
-                read_doctype(&pi);
+            } else if ((TolerantEffort == pi->options->effort) ? 0 == strncasecmp("DOCTYPE", pi->s, 7)
+                                                               : 0 == strncmp("DOCTYPE", pi->s, 7)) {
+                pi->s += 7;
+                read_doctype(pi);
             } else {
-                set_error(err, "invalid format, DOCTYPE or comment expected", pi.str, pi.s);
-                helper_stack_cleanup(&pi.helpers);
+                set_error(err, "invalid format, DOCTYPE or comment expected", pi->str, pi->s);
                 return Qnil;
             }
             break;
-        case '\0':
-            set_error(err, "invalid format, document not terminated", pi.str, pi.s);
-            helper_stack_cleanup(&pi.helpers);
-            return Qnil;
+        case '\0': set_error(err, "invalid format, document not terminated", pi->str, pi->s); return Qnil;
         default:
-            read_element(&pi, 0);
+            read_element(pi, 0);
             body_read = 1;
             break;
         }
-        if (err_has(&pi.err)) {
-            *err = pi.err;
-            helper_stack_cleanup(&pi.helpers);
+        if (err_has(&pi->err)) {
+            *err = pi->err;
             return Qnil;
         }
-        if (block_given && Qnil != pi.obj && Qundef != pi.obj) {
-            if (NULL != pcb->finish) {
-                pcb->finish(&pi);
+        if (block_given && Qnil != pi->obj && Qundef != pi->obj) {
+            if (NULL != pi->pcb->finish) {
+                pi->pcb->finish(pi);
             }
-            rb_yield(pi.obj);
+            rb_yield(pi->obj);
         }
     }
-    DATA_PTR(wrap) = NULL;
-    helper_stack_cleanup(&pi.helpers);
-    if (NULL != pcb->finish) {
-        pcb->finish(&pi);
+    if (NULL != pi->pcb->finish) {
+        pi->pcb->finish(pi);
     }
-    return pi.obj;
+    return pi->obj;
+}
+
+static VALUE ox_parse_ensure(VALUE ctxv) {
+    ParseCtx ctx = (ParseCtx)ctxv;
+
+    // pi is a stack value wrapped for GC marking. Detach the wrapper before
+    // this frame is gone so a later conservative GC can not mark a dead helper
+    // stack, and free the helper stack whether the parse returned normally or a
+    // callback raised out of it.
+    if (Qnil != ctx->wrap) {
+        DATA_PTR(ctx->wrap) = NULL;
+    }
+    helper_stack_cleanup(&ctx->pi->helpers);
+    // end_element() only frees the circular reference table on a complete
+    // parse, so release it here for an error or a callback raising out.
+    if (0 != ctx->pi->circ_array) {
+        ox_circ_array_free(ctx->pi->circ_array);
+        ctx->pi->circ_array = 0;
+    }
+    // Same for hash mode's mark list, freed by finish() only if the loop ends.
+    xfree(ctx->pi->marked);
+    ctx->pi->marked    = NULL;
+    ctx->pi->mark_size = 0;
+    ctx->pi->mark_cnt  = 0;
+    return Qnil;
+}
+
+VALUE
+ox_parse(char *xml, size_t len, ParseCallbacks pcb, char **endp, Options options, Err err) {
+    struct _pInfo    pi;
+    struct _parseCtx ctx;
+
+    if (0 == xml) {
+        set_error(err, "Invalid arg, xml string can not be null", xml, 0);
+        return Qnil;
+    }
+    if (DEBUG <= options->trace) {
+        printf("Parsing xml:\n%s\n", xml);
+    }
+    // initialize parse info
+    helper_stack_init(&pi.helpers);
+    err_init(&pi.err);
+    pi.str        = xml;
+    pi.end        = pi.str + len;
+    pi.s          = xml;
+    pi.pcb        = pcb;
+    pi.obj        = Qnil;
+    pi.circ_array = 0;
+    pi.options    = options;
+    pi.marked     = NULL;
+    pi.mark_size  = 0;
+    pi.mark_cnt   = 0;
+
+    ctx.pi          = &pi;
+    ctx.endp        = endp;
+    ctx.err         = err;
+    ctx.block_given = rb_block_given_p();
+    // Protect against GC. The wrapper marks the helper stack while parsing;
+    // ox_parse_ensure detaches it on every exit, including a callback raise.
+    ctx.wrap = TypedData_Wrap_Struct(rb_cObject, &ox_wrap_type, &pi);
+
+    return rb_ensure(ox_parse_body, (VALUE)&ctx, ox_parse_ensure, (VALUE)&ctx);
 }
 
 // Entered after the "<?" sequence. Ready to read the rest.
 static void read_instruction(PInfo pi) {
     char              content[256];
-    char             *content_ptr;
+    char             *content_ptr = content;
     struct _attrStack attrs;
     char             *attr_name;
     char             *attr_value;
@@ -262,7 +300,7 @@ static void read_instruction(PInfo pi) {
     *content = '\0';
     attr_stack_init(&attrs);
     if (0 == (target = read_name_token(pi))) {
-        return;
+        goto CLEANUP;
     }
     end = pi->s;
     for (; true; pi->s++) {
@@ -273,7 +311,7 @@ static void read_instruction(PInfo pi) {
                 goto DONE;
             }
             break;
-        case '\0': set_error(&pi->err, "processing instruction not terminated", pi->str, pi->s); return;
+        case '\0': set_error(&pi->err, "processing instruction not terminated", pi->str, pi->s); goto CLEANUP;
         default: break;
         }
     }
@@ -296,21 +334,18 @@ DONE:
         while ('?' != c) {
             pi->last = 0;
             if ('\0' == *pi->s) {
-                attr_stack_cleanup(&attrs);
                 set_error(&pi->err, "invalid format, processing instruction not terminated", pi->str, pi->s);
-                return;
+                goto CLEANUP;
             }
             next_non_white(pi);
             if (0 == (attr_name = read_name_token(pi))) {
-                attr_stack_cleanup(&attrs);
-                return;
+                goto CLEANUP;
             }
             end = pi->s;
             next_non_white(pi);
             if ('\0' == *pi->s) {
-                attr_stack_cleanup(&attrs);
                 set_error(&pi->err, "invalid format, processing instruction not terminated", pi->str, pi->s);
-                return;
+                goto CLEANUP;
             }
             if ('=' != *pi->s++) {
                 attrs_ok = false;
@@ -320,8 +355,7 @@ DONE:
             // read value
             next_non_white(pi);
             if (0 == (attr_value = read_quoted_value(pi))) {
-                attr_stack_cleanup(&attrs);
-                return;
+                goto CLEANUP;
             }
             attr_stack_push(&attrs, attr_name, attr_value);
             next_non_white(pi);
@@ -339,9 +373,8 @@ DONE:
     }
     if (attrs_ok) {
         if ('>' != *pi->s++) {
-            attr_stack_cleanup(&attrs);
             set_error(&pi->err, "invalid format, processing instruction not terminated", pi->str, pi->s);
-            return;
+            goto CLEANUP;
         }
     } else {
         pi->s = cend + 1;
@@ -362,6 +395,9 @@ DONE:
             }
         }
     }
+    // Single exit, so no error return can leak the heap content buffer or the
+    // heap-grown attribute stack.
+CLEANUP:
     attr_stack_cleanup(&attrs);
     if (content_ptr != content) {
         xfree(content_ptr);
@@ -567,6 +603,7 @@ static char *read_element(PInfo pi, int depth) {
             /* read value */
             next_non_white(pi);
             if (0 == (attr_value = read_quoted_value(pi))) {
+                attr_stack_cleanup(&attrs);
                 return 0;
             }
             if (pi->options->convert_special && 0 != strchr(attr_value, '&')) {
@@ -597,7 +634,7 @@ static char *read_element(PInfo pi, int depth) {
             if (OffSkip == pi->options->skip && start < pi->s && '<' == *pi->s) {
                 c      = *pi->s;
                 *pi->s = '\0';
-                pi->pcb->add_text(pi, start, 1);
+                pi->pcb->add_text(pi, start, (size_t)(pi->s - start), 1);
                 *pi->s = c;
             }
             c = *pi->s++;
@@ -679,7 +716,9 @@ static char *read_element(PInfo pi, int depth) {
                         default: break;
                         }
                         if ('\0' != *start) {
-                            pi->pcb->add_text(pi, start, 1);
+                            // The skip handling above moves the terminator, so
+                            // the length is whatever survived it.
+                            pi->pcb->add_text(pi, start, strlen(start), 1);
                         }
                     }
                     pi->s++;
@@ -711,6 +750,7 @@ static char *read_element(PInfo pi, int depth) {
                             return name;
                         }
                     } else if (err_has(&pi->err)) {
+                        attr_stack_cleanup(&attrs);
                         return 0;
                     }
                     break;
@@ -748,15 +788,109 @@ static char *read_element(PInfo pi, int depth) {
     return 0;
 }
 
+#define TEXT_ONES 0x0101010101010101ULL
+#define TEXT_HIGH 0x8080808080808080ULL
+
+/* Sets the high bit of every byte of a word that read_text can not copy
+ * through as is, that is a byte less than or equal to 0x20, a '&', or a '<'.
+ * Bytes with the high bit set are never flagged because ~v clears them, which
+ * matches the byte loop where a signed char with the high bit set is negative
+ * and so falls to the plain copy.
+ *
+ * A borrow out of one byte can also flag the byte above it, but a borrow is
+ * only produced by a byte that matched, so the lowest flagged byte is always a
+ * real match and there are never false negatives.
+ */
+inline static uint64_t text_bytes_of_interest(uint64_t v) {
+    uint64_t a = v ^ (TEXT_ONES * (uint64_t)'&');
+    uint64_t l = v ^ (TEXT_ONES * (uint64_t)'<');
+
+    return (((v - TEXT_ONES * 0x21) & ~v) | ((a - TEXT_ONES) & ~a) | ((l - TEXT_ONES) & ~l)) & TEXT_HIGH;
+}
+
+/* Offset of the lowest flagged byte, which is the first byte of the word that
+ * has to go through the switch below. Only ever called with a non-zero mask.
+ */
+inline static int text_first_of_interest(const char *s, uint64_t mask) {
+#if defined(__GNUC__) || defined(__clang__)
+#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+    (void)s;
+    return (int)(__builtin_clzll(mask) >> 3);
+#else
+    (void)s;
+    return (int)(__builtin_ctzll(mask) >> 3);
+#endif
+#else
+    int i;
+
+    (void)mask;
+    for (i = 0; i < 8; i++) {
+        unsigned char u = (unsigned char)s[i];
+
+        if (u <= 0x20 || '&' == u || '<' == u) {
+            break;
+        }
+    }
+    return i;
+#endif
+}
+
 static void read_text(PInfo pi) {
-    char  buf[MAX_TEXT_LEN];
-    char *b         = buf;
-    char *alloc_buf = 0;
-    char *end       = b + sizeof(buf) - 2;
-    char  c;
-    int   done = 0;
+    char   buf[MAX_TEXT_LEN];
+    char  *b         = buf;
+    char  *alloc_buf = 0;
+    char  *end       = b + sizeof(buf) - 2;
+    char  *text;
+    size_t len;
+    char   c;
+    int    done = 0;
+    /* fix_newlines only rewrites '\r'. SpcSkip folds every '\r' to a space, so
+     * the only way one reaches buf under it is an entity like &#13;. Any other
+     * skip mode can keep a '\r', so assume one might be present there and run
+     * fix_newlines as before. SpcSkip text with no such entity, the common
+     * case, then skips the fix_newlines scan and second pass entirely.
+     */
+    int maybe_cr = (SpcSkip != pi->options->skip);
 
     while (!done) {
+        /* A byte above 0x20 that is neither '&' nor '<' is copied through
+         * unchanged for every skip mode and every effort, so runs of them can
+         * be moved a word at a time instead of one switch dispatch per byte.
+         * pi->end addresses the terminating '\0' so the loads never leave the
+         * document, which keeps the loop clean under ASan.
+         */
+        while (pi->s + 8 <= pi->end && b + 8 <= end) {
+            uint64_t v;
+            uint64_t mask;
+
+            memcpy(&v, pi->s, 8);
+            mask = text_bytes_of_interest(v);
+            /* b + 8 <= end so the whole word can be stored even when only part
+             * of it is kept. The bytes past the kept prefix are overwritten by
+             * the next copy or cut off by the terminating '\0'.
+             */
+            memcpy(b, pi->s, 8);
+            if (0 != mask) {
+                int n = text_first_of_interest(pi->s, mask);
+
+                b += n;
+                pi->s += n;
+                break;
+            }
+            b += 8;
+            pi->s += 8;
+        }
+        /* Only reached when a bound stopped the word loop, or for the byte the
+         * word loop stopped on.
+         */
+        while (pi->s < pi->end && b < end) {
+            unsigned char u = (unsigned char)*pi->s;
+
+            if (u <= 0x20 || '&' == u || '<' == u) {
+                break;
+            }
+            *b++ = *pi->s++;
+        }
         c = *pi->s++;
         switch (c) {
         case '<':
@@ -766,7 +900,7 @@ static void read_text(PInfo pi) {
         case '\0':
             pi->s--;
             set_error(&pi->err, "invalid format, document not terminated", pi->str, pi->s);
-            return;
+            goto CLEANUP;
         default:
             if (end <= (b + (('&' == c) ? 7 : 0))) { /* extra 8 for special just in case it is sequence of bytes */
                 unsigned long size;
@@ -787,13 +921,15 @@ static void read_text(PInfo pi) {
             }
             if ('&' == c) {
                 if (0 == (b = read_coded_chars(pi, b))) {
-                    return;
+                    goto CLEANUP;
                 }
+                /* An entity such as &#13; can decode to '\r'; be conservative. */
+                maybe_cr = 1;
             } else {
                 if (0 <= c && c <= 0x20) {
                     if (StrictEffort == pi->options->effort && 'x' == xml_valid_lower_chars[(unsigned char)c]) {
                         set_error(&pi->err, "invalid character", pi->str, pi->s);
-                        return;
+                        goto CLEANUP;
                     }
                     switch (pi->options->skip) {
                     case CrSkip:
@@ -823,14 +959,21 @@ static void read_text(PInfo pi) {
             break;
         }
     }
-    *b = '\0';
+    *b   = '\0';
+    text = (0 != alloc_buf) ? alloc_buf : buf;
+    len  = (size_t)(b - text);
+    if (maybe_cr) {
+        // fix_newlines can fold "\r\n" to "\n" and shorten the text, so the
+        // length is only known after it runs. When it was skipped the text is
+        // exactly b - text, so add_text avoids a strlen on the common path.
+        fix_newlines(text);
+        len = strlen(text);
+    }
+    pi->pcb->add_text(pi, text, len, ('/' == *(pi->s + 1)));
+    // Single exit, so no error return can leak the heap-grown text buffer.
+CLEANUP:
     if (0 != alloc_buf) {
-        fix_newlines(alloc_buf);
-        pi->pcb->add_text(pi, alloc_buf, ('/' == *(pi->s + 1)));
         xfree(alloc_buf);
-    } else {
-        fix_newlines(buf);
-        pi->pcb->add_text(pi, buf, ('/' == *(pi->s + 1)));
     }
 }
 

data/ext/ox/sax.c

@@ -197,7 +197,7 @@ static void comment(SaxDrive dr, long pos, long line, long col) {
         Hint h      = ox_hint_find(dr->options.hints, "!--");
 
         if (NULL == parent || NULL == parent->hint || OffOverlay != parent->hint->overlay ||
-            (NULL != h && (ActiveOverlay == h->overlay || ActiveOverlay == h->overlay))) {
+            (NULL != h && (ActiveOverlay == h->overlay || NestOverlay == h->overlay))) {
             VALUE arg = rb_str_new2(dr->buf.str);
 
             if (0 != dr->encoding) {
@@ -581,7 +581,7 @@ static char read_instruction(SaxDrive dr) {
     return c;
 }
 
-static char read_delimited(SaxDrive dr, char end) {
+static char read_delimited(SaxDrive dr, char end, int depth) {
     char c;
 
     if ('"' == end || '\'' == end) {
@@ -591,6 +591,9 @@ static char read_delimited(SaxDrive dr, char end) {
                 return c;
             }
         }
+    } else if (MAX_PROLOG <= depth) {
+        rb_raise(ox_parse_error_class, "prolog (doctype) too long.\n");
+        return '\0';
     } else {
         while (1) {
             c = buf_get(&dr->buf);
@@ -599,10 +602,10 @@ static char read_delimited(SaxDrive dr, char end) {
             }
             switch (c) {
             case '\0': ox_sax_drive_error(dr, NO_TERM "doctype not terminated"); return c;
-            case '"': c = read_delimited(dr, c); break;
-            case '\'': c = read_delimited(dr, c); break;
-            case '[': c = read_delimited(dr, ']'); break;
-            case '<': c = read_delimited(dr, '>'); break;
+            case '"': c = read_delimited(dr, c, depth + 1); break;
+            case '\'': c = read_delimited(dr, c, depth + 1); break;
+            case '[': c = read_delimited(dr, ']', depth + 1); break;
+            case '<': c = read_delimited(dr, '>', depth + 1); break;
             default: break;
             }
         }
@@ -621,7 +624,7 @@ static char read_doctype(SaxDrive dr) {
 
     buf_backup(&dr->buf); /* back up to the start in case the doctype is empty */
     buf_protect(&dr->buf);
-    read_delimited(dr, '>');
+    read_delimited(dr, '>', 0);
     if (dr->options.smart && 0 == dr->options.hints) {
         for (s = dr->buf.str; is_white(*s); s++) {
         }
@@ -876,16 +879,17 @@ static char read_element_start(SaxDrive dr) {
             }
         }
     }
-    name = str2sym(dr, dr->buf.str, nlen, &ename);
-    if (NULL == ename) {
-        if (sizeof(ebuf) <= nlen) {
-            ename = ox_strndup(dr->buf.str, nlen);
-            efree = true;
-        } else {
-            memcpy(ebuf, dr->buf.str, nlen);
-            ebuf[nlen] = '\0';
-            ename      = ebuf;
-        }
+    name = str2sym(dr, dr->buf.str, nlen, NULL);
+    // ename lives until stack_push() copies it, across the start_element
+    // callback and read_attrs. The cache's key is inside a Slot, which a GC can
+    // retire and a later intern free, so copy it instead.
+    if (sizeof(ebuf) <= nlen) {
+        ename = ox_strndup(dr->buf.str, nlen);
+        efree = true;
+    } else {
+        memcpy(ebuf, dr->buf.str, nlen);
+        ebuf[nlen] = '\0';
+        ename      = ebuf;
     }
     if (dr->has_start_element && 0 >= dr->blocked &&
         (NULL == h || ActiveOverlay == h->overlay || NestOverlay == h->overlay)) {
@@ -930,7 +934,7 @@ static char read_element_start(SaxDrive dr) {
         stack_push(&dr->stack, ename, nlen, name, h);
     }
     if (efree) {
-        free((char *)ename);
+        xfree((char *)ename);
     }
     if ('>' != c) {
         ox_sax_drive_error(dr, WRONG_CHAR "element not closed");