ox 2.12.1 → 2.14.0

data/ext/ox/ox.h

@@ -16,20 +16,15 @@ extern "C" {
 #define RSTRING_NOT_MODIFIED
 
 #include "ruby.h"
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 #include "ruby/encoding.h"
 #endif
 
-#ifdef RUBINIUS_RUBY
-#undef T_COMPLEX
-enum st_retval {ST_CONTINUE = 0, ST_STOP = 1, ST_DELETE = 2, ST_CHECK};
+#if HAVE_RUBY_ST_H
+#include "ruby/st.h"
 #else
-#if HAS_TOP_LEVEL_ST_H
-/* Only on travis, local is where it is for all others. Seems to vary depending on the travis machine picked up. */
+// Only on travis, local is where it is for all others. Seems to vary depending on the travis machine picked up.
 #include "st.h"
-#else
-#include "ruby/st.h"
-#endif
 #endif
 
 #include "cache.h"
@@ -141,15 +136,14 @@ typedef struct _options {
     char		convert_special;// boolean true or false
     char		allow_invalid;	// YesNo
     char		no_empty;	// boolean - no empty elements when dumping
+    char		with_cdata;	// boolean - hash_load should include cdata
     char		inv_repl[12];	// max 10 valid characters, first character is the length
     char		strip_ns[64];	// namespace to strip, \0 is no-strip, \* is all, else only matches
     struct _hints	*html_hints;	// html hints
     VALUE		attr_key_mod;
     VALUE		element_key_mod;
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
     rb_encoding		*rb_enc;
-#elif HAS_PRIVATE_ENCODING
-    VALUE		rb_enc;
 #else
     void		*rb_enc;
 #endif
@@ -236,10 +230,8 @@ extern ID	ox_tv_nsec_id;
 extern ID	ox_tv_usec_id;
 extern ID	ox_value_id;
 
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 extern rb_encoding	*ox_utf8_encoding;
-#elif HAS_PRIVATE_ENCODING
-extern VALUE		ox_utf8_encoding;
 #else
 extern void		*ox_utf8_encoding;
 #endif

data/ext/ox/parse.c

@@ -441,7 +441,6 @@ read_element(PInfo pi) {
 	// empty element, no attributes and no children
 	pi->s++;
 	if ('>' != *pi->s) {
-	    /*printf("*** '%s' ***\n", pi->s); */
 	    attr_stack_cleanup(&attrs);
 	    set_error(&pi->err, "invalid format, element not closed", pi->str, pi->s);
 	    return 0;
@@ -479,8 +478,8 @@ read_element(PInfo pi) {
 	    pi->s++;
 	    pi->pcb->add_element(pi, ename, attrs.head, hasChildren);
 	    pi->pcb->end_element(pi, ename);
-
 	    attr_stack_cleanup(&attrs);
+
 	    return 0;
 	case '>':
 	    /* has either children or a value */
@@ -544,6 +543,12 @@ read_element(PInfo pi) {
 	while (!done) {
 	    start = pi->s;
 	    next_non_white(pi);
+	    if (OffSkip == pi->options->skip && start < pi->s && '<' == *pi->s) {
+		c = *pi->s;
+		*pi->s = '\0';
+		pi->pcb->add_text(pi, start, 1);
+		*pi->s = c;
+	    }
 	    c = *pi->s++;
 	    if ('\0' == c) {
 		attr_stack_cleanup(&attrs);
@@ -1000,11 +1005,13 @@ read_coded_chars(PInfo pi, char *text) {
     char	*b, buf[32];
     char	*end = buf + sizeof(buf) - 1;
     char	*s;
+    long	blen = 0;
 
     for (b = buf, s = pi->s; b < end; b++, s++) {
 	*b = *s;
 	if (';' == *s) {
 	    *(b + 1) = '\0';
+	    blen = b - buf;
 	    s++;
 	    break;
 	}
@@ -1025,18 +1032,9 @@ read_coded_chars(PInfo pi, char *text) {
 	} else {
 	    if (u <= 0x000000000000007FULL) {
 		*text++ = (char)u;
-#if HAS_PRIVATE_ENCODING
-	    } else if (ox_utf8_encoding == pi->options->rb_enc ||
-		       0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(pi->options->rb_enc)))) {
-#else
 	    } else if (ox_utf8_encoding == pi->options->rb_enc) {
-#endif
 		text = ox_ucs_to_utf8_chars(text, u);
-#if HAS_PRIVATE_ENCODING
-	    } else if (Qnil == pi->options->rb_enc) {
-#else
 	    } else if (0 == pi->options->rb_enc) {
-#endif
 		pi->options->rb_enc = ox_utf8_encoding;
 		text = ox_ucs_to_utf8_chars(text, u);
 	    } else if (TolerantEffort == pi->options->effort) {
@@ -1047,30 +1045,20 @@ read_coded_chars(PInfo pi, char *text) {
 	    } else {
 		/*set_error(&pi->err, "Invalid encoding, need UTF-8 or UTF-16 encoding to parse &#nnnn; character sequences.", pi->str, pi->s); */
 		set_error(&pi->err, "Invalid encoding, need UTF-8 encoding to parse &#nnnn; character sequences.", pi->str, pi->s);
-		return 0;
+		return NULL;
 	    }
 	    pi->s = s;
 	}
-    } else if (0 == strcasecmp(buf, "nbsp;")) {
-	pi->s = s;
-	*text++ = ' ';
-    } else if (0 == strcasecmp(buf, "lt;")) {
-	pi->s = s;
-	*text++ = '<';
-    } else if (0 == strcasecmp(buf, "gt;")) {
-	pi->s = s;
-	*text++ = '>';
-    } else if (0 == strcasecmp(buf, "amp;")) {
-	pi->s = s;
-	*text++ = '&';
-    } else if (0 == strcasecmp(buf, "quot;")) {
-	pi->s = s;
-	*text++ = '"';
-    } else if (0 == strcasecmp(buf, "apos;")) {
-	pi->s = s;
-	*text++ = '\'';
     } else {
-	*text++ = '&';
+	char	*t2;
+
+	buf[blen] = '\0';
+	if (NULL == (t2 = ox_entity_lookup(text, buf))) {
+	    *text++ = '&';
+	} else {
+	    text = t2;
+	    pi->s = s;
+	}
     }
     return text;
 }
@@ -1112,19 +1100,10 @@ collapse_special(PInfo pi, char *str) {
 		}
 		if (u <= 0x000000000000007FULL) {
 		    *b++ = (char)u;
-#if HAS_PRIVATE_ENCODING
-		} else if (ox_utf8_encoding == pi->options->rb_enc ||
-			   0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(pi->options->rb_enc)))) {
-#else
 		} else if (ox_utf8_encoding == pi->options->rb_enc) {
-#endif
 		    b = ox_ucs_to_utf8_chars(b, u);
 		    /* TBD support UTF-16 */
-#if HAS_PRIVATE_ENCODING
-		} else if (Qnil == pi->options->rb_enc) {
-#else
 		} else if (0 == pi->options->rb_enc) {
-#endif
 		    pi->options->rb_enc = ox_utf8_encoding;
 		    b = ox_ucs_to_utf8_chars(b, u);
 		} else {
@@ -1153,16 +1132,30 @@ collapse_special(PInfo pi, char *str) {
 		    *b++ = '&';
 		    continue;
 		} else {
-		    c = '?';
+		    char	key[16];
+		    char	*k = key;
+		    char	*kend = key + sizeof(key) - 1;
+
+		    *k++ = *s;
 		    while (';' != *s++) {
 			if ('\0' == *s) {
 			    set_error(&pi->err, "Invalid format, special character does not end with a semicolon", pi->str, pi->s);
 			    return EDOM;
 			}
+			if (kend <= k) {
+			    k = key;
+			    break;
+			}
+			*k++ = *s;
 		    }
-		    s++;
-		    set_error(&pi->err, "Invalid format, invalid special character sequence", pi->str, pi->s);
-		    return 0;
+		    k--;
+		    *k = '\0';
+		    if ('\0' == *key || NULL == (b = ox_entity_lookup(b, key))) {
+			set_error(&pi->err, "Invalid format, invalid special character sequence", pi->str, pi->s);
+			c = '?';
+			return 0;
+		    }
+		    continue;
 		}
 		*b++ = (char)c;
 	    }

data/ext/ox/sax.c

@@ -9,13 +9,16 @@
 #include <stdio.h>
 #include <strings.h>
 #include <sys/types.h>
-#if NEEDS_UIO
+#if HAVE_SYS_UIO_H
 #include <sys/uio.h>
 #endif
 #include <unistd.h>
 #include <time.h>
 
 #include "ruby.h"
+#if HAVE_RB_ENC_ASSOCIATE
+#include "ruby/encoding.h"
+#endif
 #include "ox.h"
 #include "sax.h"
 #include "sax_stack.h"
@@ -68,9 +71,9 @@ static VALUE protect_parse(VALUE drp) {
     return Qnil;
 }
 
-#if HAS_ENCODING_SUPPORT || HAS_PRIVATE_ENCODING
+#if HAVE_RB_ENC_ASSOCIATE
 static int
-strIsAscii(const char *s) {
+str_is_ascii(const char *s) {
     for (; '\0' != *s; s++) {
 	if (*s < ' ' || '~' < *s) {
 	    return 0;
@@ -87,8 +90,8 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 
     if (dr->options.symbolize) {
 	if (Qundef == (sym = ox_cache_get(ox_symbol_cache, str, &slot, strp))) {
-#if HAS_ENCODING_SUPPORT
-	    if (0 != dr->encoding && !strIsAscii(str)) {
+#if HAVE_RB_ENC_ASSOCIATE
+	    if (0 != dr->encoding && !str_is_ascii(str)) {
 		VALUE	rstr = rb_str_new2(str);
 
 		// TBD if sym can be pinned down then use this all the time
@@ -99,20 +102,6 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 		sym = ID2SYM(rb_intern(str));
 		*slot = sym;
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding && !strIsAscii(str)) {
-		VALUE	rstr = rb_str_new2(str);
-
-		rb_funcall(rstr, ox_force_encoding_id, 1, dr->encoding);
-		sym = rb_funcall(rstr, ox_to_sym_id, 0);
-		// Needed for Ruby 2.2 to get around the GC of symbols created
-		// with to_sym which is needed for encoded symbols.
-		rb_ary_push(ox_sym_bank, sym);
-		*slot = Qundef;
-	    } else {
-		sym = ID2SYM(rb_intern(str));
-		*slot = sym;
-	    }
 #else
 	    sym = ID2SYM(rb_intern(str));
 	    *slot = sym;
@@ -120,14 +109,10 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 	}
     } else {
 	sym = rb_str_new2(str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	if (0 != dr->encoding) {
 	    rb_enc_associate(sym, dr->encoding);
 	}
-#elif HAS_PRIVATE_ENCODING
-	if (Qnil != dr->encoding) {
-	    rb_funcall(sym, ox_force_encoding_id, 1, dr->encoding);
-	}
 #endif
 	if (0 != strp) {
 	    *strp = StringValuePtr(sym);
@@ -182,7 +167,7 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, SaxOptions options) {
     dr->blocked = 0;
     dr->abort = false;
     has_init(&dr->has, handler);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
     if ('\0' == *ox_default_options.encoding) {
 	VALUE	encoding;
 
@@ -196,18 +181,6 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, SaxOptions options) {
     } else {
         dr->encoding = rb_enc_find(ox_default_options.encoding);
     }
-#elif HAS_PRIVATE_ENCODING
-    if ('\0' == *ox_default_options.encoding) {
-	VALUE	encoding;
-
-	if (rb_respond_to(io, ox_external_encoding_id) && Qnil != (encoding = rb_funcall(io, ox_external_encoding_id, 0))) {
-	    dr->encoding = encoding;
-	} else {
-	    dr->encoding = Qnil;
-	}
-    } else {
-        dr->encoding = rb_str_new2(ox_default_options.encoding);
-    }
 #else
     dr->encoding = 0;
 #endif
@@ -255,9 +228,7 @@ skipBOM(SaxDrive dr) {
 
     if (0xEF == (uint8_t)c) { /* only UTF8 is supported */
 	if (0xBB == (uint8_t)buf_get(&dr->buf) && 0xBF == (uint8_t)buf_get(&dr->buf)) {
-#if HAS_ENCODING_SUPPORT
-	    dr->encoding = ox_utf8_encoding;
-#elif HAS_PRIVATE_ENCODING
+#if HAVE_RB_ENC_FIND
 	    dr->encoding = ox_utf8_encoding;
 #else
 	    dr->encoding = UTF8_STR;
@@ -364,14 +335,10 @@ parse(SaxDrive dr) {
 		    off_t	col = dr->buf.col - 1;
 
 		    args[0] = rb_str_new2("");
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 		    if (0 != dr->encoding) {
 			rb_enc_associate(args[0], dr->encoding);
 		    }
-#elif HAS_PRIVATE_ENCODING
-		    if (Qnil != dr->encoding) {
-			rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-		    }
 #endif
 		    if (dr->has.pos) {
 			rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -526,14 +493,10 @@ read_instruction(SaxDrive dr) {
 		ox_sax_collapse_special(dr, content, (int)pos, (int)line, (int)col);
 	    }
 	    args[0] = rb_str_new2(content);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.line) {
 		rb_ivar_set(dr->handler, ox_at_line_id, LONG2NUM(line));
@@ -732,14 +695,10 @@ read_cdata(SaxDrive dr) {
 	    VALUE       args[1];
 
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -826,14 +785,10 @@ read_comment(SaxDrive dr) {
 	    (NULL != h && (ActiveOverlay == h->overlay || ActiveOverlay == h->overlay))) {
 
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1158,14 +1113,10 @@ read_text(SaxDrive dr) {
 	    ((NoSkip == dr->options.skip && !isEnd) ||
 	     (OffSkip == dr->options.skip))) {
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1213,14 +1164,10 @@ read_text(SaxDrive dr) {
 		break;
 	    }
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1299,14 +1246,10 @@ read_jump(SaxDrive dr, const char *pat) {
     // TBD check parent overlay
     if (dr->has.text && !dr->blocked) {
         args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
         if (0 != dr->encoding) {
             rb_enc_associate(args[0], dr->encoding);
         }
-#elif HAS_PRIVATE_ENCODING
-        if (Qnil != dr->encoding) {
-	    rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-        }
 #endif
 	if (dr->has.pos) {
 	    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1377,10 +1320,8 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req,
 	    c = read_quoted_value(dr);
 	    attr_value = dr->buf.str;
 	    if (is_encoding) {
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
 		dr->encoding = rb_enc_find(dr->buf.str);
-#elif HAS_PRIVATE_ENCODING
-		dr->encoding = rb_str_new2(dr->buf.str);
 #else
 		dr->encoding = dr->buf.str;
 #endif
@@ -1411,14 +1352,10 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req,
 		    ox_sax_collapse_special(dr, dr->buf.str, pos, line, col);
 		}
 		args[1] = rb_str_new2(attr_value);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 		if (0 != dr->encoding) {
 		    rb_enc_associate(args[1], dr->encoding);
 		}
-#elif HAS_PRIVATE_ENCODING
-		if (Qnil != dr->encoding) {
-		    rb_funcall(args[1], ox_force_encoding_id, 1, dr->encoding);
-		}
 #endif
 		if (dr->has.pos) {
 		    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1614,19 +1551,12 @@ ox_sax_collapse_special(SaxDrive dr, char *str, long pos, long line, long col) {
 		}
 		if (u <= 0x000000000000007FULL) {
 		    *b++ = (char)u;
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
 		} else if (ox_utf8_encoding == dr->encoding) {
 		    b = ox_ucs_to_utf8_chars(b, u);
 		} else if (0 == dr->encoding) {
 		    dr->encoding = ox_utf8_encoding;
 		    b = ox_ucs_to_utf8_chars(b, u);
-#elif HAS_PRIVATE_ENCODING
-		} else if (ox_utf8_encoding == dr->encoding ||
-			   0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(dr->encoding)))) {
-		    b = ox_ucs_to_utf8_chars(b, u);
-		} else if (Qnil == dr->encoding) {
-		    dr->encoding = ox_utf8_encoding;
-		    b = ox_ucs_to_utf8_chars(b, u);
 #else
 		} else if (0 == dr->encoding) {
 		    dr->encoding = UTF8_STR;
@@ -1668,8 +1598,28 @@ ox_sax_collapse_special(SaxDrive dr, char *str, long pos, long line, long col) {
                 c = '\'';
                 s += 5;
             } else {
-		ox_sax_drive_error_at(dr, INVALID_FORMAT "Invalid special character sequence", pos, line, col);
-		c = '&';
+		char	key[16];
+		char	*k = key;
+		char	*kend = key + sizeof(key) - 1;
+		char	*bn;
+		char	*s2 = s;
+
+		for (; ';' != *s2 && '\0' != *s2; s2++, k++) {
+		    if (kend <= k) {
+			k = key;
+			break;
+		    }
+		    *k = *s2;
+		}
+		*k = '\0';
+		if ('\0' == *key || NULL == (bn = ox_entity_lookup(b, key))) {
+		    ox_sax_drive_error_at(dr, INVALID_FORMAT "Invalid special character sequence", pos, line, col);
+		    c = '&';
+		} else {
+		    b = bn;
+		    s = s2 + 1;
+		    continue;
+		}
             }
             *b++ = (char)c;
 	    col++;