ox 2.11.0 → 2.13.3

data/ext/ox/sax.c

@@ -9,13 +9,16 @@
 #include <stdio.h>
 #include <strings.h>
 #include <sys/types.h>
-#if NEEDS_UIO
+#if HAVE_SYS_UIO_H
 #include <sys/uio.h>
 #endif
 #include <unistd.h>
 #include <time.h>
 
 #include "ruby.h"
+#if HAVE_RB_ENC_ASSOCIATE
+#include "ruby/encoding.h"
+#endif
 #include "ox.h"
 #include "sax.h"
 #include "sax_stack.h"
@@ -55,7 +58,7 @@ static char		read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml,
 static char		read_name_token(SaxDrive dr);
 static char		read_quoted_value(SaxDrive dr);
 
-static void		end_element_cb(SaxDrive dr, VALUE name, int pos, int line, int col, Hint h);
+static void		end_element_cb(SaxDrive dr, VALUE name, long pos, long line, long col, Hint h);
 
 static void		hint_clear_empty(SaxDrive dr);
 static Nv		hint_try_close(SaxDrive dr, const char *name);
@@ -68,9 +71,9 @@ static VALUE protect_parse(VALUE drp) {
     return Qnil;
 }
 
-#if HAS_ENCODING_SUPPORT || HAS_PRIVATE_ENCODING
+#if HAVE_RB_ENC_ASSOCIATE
 static int
-strIsAscii(const char *s) {
+str_is_ascii(const char *s) {
     for (; '\0' != *s; s++) {
 	if (*s < ' ' || '~' < *s) {
 	    return 0;
@@ -87,8 +90,8 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 
     if (dr->options.symbolize) {
 	if (Qundef == (sym = ox_cache_get(ox_symbol_cache, str, &slot, strp))) {
-#if HAS_ENCODING_SUPPORT
-	    if (0 != dr->encoding && !strIsAscii(str)) {
+#if HAVE_RB_ENC_ASSOCIATE
+	    if (0 != dr->encoding && !str_is_ascii(str)) {
 		VALUE	rstr = rb_str_new2(str);
 
 		// TBD if sym can be pinned down then use this all the time
@@ -99,20 +102,6 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 		sym = ID2SYM(rb_intern(str));
 		*slot = sym;
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding && !strIsAscii(str)) {
-		VALUE	rstr = rb_str_new2(str);
-
-		rb_funcall(rstr, ox_force_encoding_id, 1, dr->encoding);
-		sym = rb_funcall(rstr, ox_to_sym_id, 0);
-		// Needed for Ruby 2.2 to get around the GC of symbols created
-		// with to_sym which is needed for encoded symbols.
-		rb_ary_push(ox_sym_bank, sym);
-		*slot = Qundef;
-	    } else {
-		sym = ID2SYM(rb_intern(str));
-		*slot = sym;
-	    }
 #else
 	    sym = ID2SYM(rb_intern(str));
 	    *slot = sym;
@@ -120,14 +109,10 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 	}
     } else {
 	sym = rb_str_new2(str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	if (0 != dr->encoding) {
 	    rb_enc_associate(sym, dr->encoding);
 	}
-#elif HAS_PRIVATE_ENCODING
-	if (Qnil != dr->encoding) {
-	    rb_funcall(sym, ox_force_encoding_id, 1, dr->encoding);
-	}
 #endif
 	if (0 != strp) {
 	    *strp = StringValuePtr(sym);
@@ -182,7 +167,7 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, SaxOptions options) {
     dr->blocked = 0;
     dr->abort = false;
     has_init(&dr->has, handler);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
     if ('\0' == *ox_default_options.encoding) {
 	VALUE	encoding;
 
@@ -196,18 +181,6 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, SaxOptions options) {
     } else {
         dr->encoding = rb_enc_find(ox_default_options.encoding);
     }
-#elif HAS_PRIVATE_ENCODING
-    if ('\0' == *ox_default_options.encoding) {
-	VALUE	encoding;
-
-	if (rb_respond_to(io, ox_external_encoding_id) && Qnil != (encoding = rb_funcall(io, ox_external_encoding_id, 0))) {
-	    dr->encoding = encoding;
-	} else {
-	    dr->encoding = Qnil;
-	}
-    } else {
-        dr->encoding = rb_str_new2(ox_default_options.encoding);
-    }
 #else
     dr->encoding = 0;
 #endif
@@ -221,7 +194,7 @@ ox_sax_drive_cleanup(SaxDrive dr) {
 }
 
 static void
-ox_sax_drive_error_at(SaxDrive dr, const char *msg, int pos, int line, int col) {
+ox_sax_drive_error_at(SaxDrive dr, const char *msg, off_t pos, off_t line, off_t col) {
     if (dr->has.error) {
         VALUE   args[3];
 
@@ -255,9 +228,7 @@ skipBOM(SaxDrive dr) {
 
     if (0xEF == (uint8_t)c) { /* only UTF8 is supported */
 	if (0xBB == (uint8_t)buf_get(&dr->buf) && 0xBF == (uint8_t)buf_get(&dr->buf)) {
-#if HAS_ENCODING_SUPPORT
-	    dr->encoding = ox_utf8_encoding;
-#elif HAS_PRIVATE_ENCODING
+#if HAVE_RB_ENC_FIND
 	    dr->encoding = ox_utf8_encoding;
 #else
 	    dr->encoding = UTF8_STR;
@@ -301,11 +272,11 @@ parse(SaxDrive dr) {
 		    }
 		    c = read_comment(dr);
 		} else {
-		    int	i;
-		    int	spaced = 0;
-		    int	pos = dr->buf.pos + 1;
-		    int	line = dr->buf.line;
-		    int	col = dr->buf.col + 1;
+		    int		i;
+		    int		spaced = 0;
+		    off_t	pos = dr->buf.pos + 1;
+		    off_t	line = dr->buf.line;
+		    off_t	col = dr->buf.col + 1;
 
 		    if (is_white(c)) {
 			spaced = 1;
@@ -359,19 +330,15 @@ parse(SaxDrive dr) {
 		parent = stack_peek(&dr->stack);
 		if (0 != parent && 0 == parent->childCnt && dr->has.text && !dr->blocked) {
 		    VALUE	args[1];
-		    int		pos = dr->buf.pos;
-		    int		line = dr->buf.line;
-		    int		col = dr->buf.col - 1;
+		    off_t	pos = dr->buf.pos;
+		    off_t	line = dr->buf.line;
+		    off_t	col = dr->buf.col - 1;
 
 		    args[0] = rb_str_new2("");
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 		    if (0 != dr->encoding) {
 			rb_enc_associate(args[0], dr->encoding);
 		    }
-#elif HAS_PRIVATE_ENCODING
-		    if (Qnil != dr->encoding) {
-			rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-		    }
 #endif
 		    if (dr->has.pos) {
 			rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -479,9 +446,9 @@ read_instruction(SaxDrive dr) {
     int		coff;
     VALUE	target = Qnil;
     int		is_xml;
-    int		pos = dr->buf.pos - 1;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    off_t	pos = dr->buf.pos - 1;
+    off_t	line = dr->buf.line;
+    off_t	col = dr->buf.col - 1;
 
     buf_protect(&dr->buf);
     if ('\0' == (c = read_name_token(dr))) {
@@ -511,7 +478,7 @@ read_instruction(SaxDrive dr) {
     line = dr->buf.line;
     col = dr->buf.col;
     read_content(dr, content, sizeof(content) - 1);
-    coff = dr->buf.tail - dr->buf.head;
+    coff = (int)(dr->buf.tail - dr->buf.head);
     buf_reset(&dr->buf);
     dr->err = 0;
     c = read_attrs(dr, c, '?', '?', is_xml, 1, NULL);
@@ -523,17 +490,13 @@ read_instruction(SaxDrive dr) {
 	    VALUE   args[1];
 
 	    if (dr->options.convert_special) {
-		ox_sax_collapse_special(dr, content, pos, line, col);
+		ox_sax_collapse_special(dr, content, (int)pos, (int)line, (int)col);
 	    }
 	    args[0] = rb_str_new2(content);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.line) {
 		rb_ivar_set(dr->handler, ox_at_line_id, LONG2NUM(line));
@@ -627,9 +590,9 @@ read_delimited(SaxDrive dr, char end) {
  */
 static char
 read_doctype(SaxDrive dr) {
-    int		pos = dr->buf.pos - 9;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 9;
+    long	pos = (long)(dr->buf.pos - 9);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 9);
     char	*s;
     Nv		parent = stack_peek(&dr->stack);
 
@@ -673,9 +636,9 @@ read_cdata(SaxDrive dr) {
     char        	c;
     char        	zero = '\0';
     int         	end = 0;
-    int			pos = dr->buf.pos - 9;
-    int			line = dr->buf.line;
-    int			col = dr->buf.col - 9;
+    long		pos = (long)(dr->buf.pos - 9);
+    long		line = (long)(dr->buf.line);
+    long		col = (long)(dr->buf.col - 9);
     struct _checkPt	cp = CHECK_PT_INIT;
     Nv			parent = stack_peek(&dr->stack);
 
@@ -732,14 +695,10 @@ read_cdata(SaxDrive dr) {
 	    VALUE       args[1];
 
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -768,9 +727,9 @@ read_comment(SaxDrive dr) {
     char        	c;
     char        	zero = '\0';
     int         	end = 0;
-    int			pos = dr->buf.pos - 4;
-    int			line = dr->buf.line;
-    int			col = dr->buf.col - 4;
+    long		pos = (long)(dr->buf.pos - 4);
+    long		line = (long)(dr->buf.line);
+    long		col = (long)(dr->buf.col - 4);
     struct _checkPt	cp = CHECK_PT_INIT;
 
     buf_backup(&dr->buf); /* back up to the start in case the cdata is empty */
@@ -826,14 +785,10 @@ read_comment(SaxDrive dr) {
 	    (NULL != h && (ActiveOverlay == h->overlay || ActiveOverlay == h->overlay))) {
 
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -864,9 +819,9 @@ read_element_start(SaxDrive dr) {
     volatile VALUE	name = Qnil;
     char        	c;
     int			closed;
-    int			pos = dr->buf.pos;
-    int			line = dr->buf.line;
-    int			col = dr->buf.col;
+    long		pos = (long)(dr->buf.pos);
+    long		line = (long)(dr->buf.line);
+    long		col = (long)(dr->buf.col);
     Hint		h = NULL;
     int			stackless = 0;
     Nv			parent = stack_peek(&dr->stack);
@@ -1020,9 +975,9 @@ static char
 read_element_end(SaxDrive dr) {
     VALUE       name = Qnil;
     char        c;
-    int		pos = dr->buf.pos - 1;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    long	pos = (long)(dr->buf.pos - 1);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 1);
     Nv		nv;
     Hint	h = NULL;
 
@@ -1118,9 +1073,9 @@ static char
 read_text(SaxDrive dr) {
     VALUE	args[1];
     char        c;
-    int		pos = dr->buf.pos;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    long	pos = (long)(dr->buf.pos);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 1);
     Nv		parent = stack_peek(&dr->stack);
     int		allWhite = 1;
 
@@ -1158,14 +1113,10 @@ read_text(SaxDrive dr) {
 	    ((NoSkip == dr->options.skip && !isEnd) ||
 	     (OffSkip == dr->options.skip))) {
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1213,14 +1164,10 @@ read_text(SaxDrive dr) {
 		break;
 	    }
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1266,9 +1213,9 @@ static char
 read_jump(SaxDrive dr, const char *pat) {
     VALUE	args[1];
     char        c;
-    int		pos = dr->buf.pos;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    long	pos = (long)(dr->buf.pos);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 1);
     Nv		parent = stack_peek(&dr->stack);
 
     buf_protect(&dr->buf);
@@ -1299,14 +1246,10 @@ read_jump(SaxDrive dr, const char *pat) {
     // TBD check parent overlay
     if (dr->has.text && !dr->blocked) {
         args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
         if (0 != dr->encoding) {
             rb_enc_associate(args[0], dr->encoding);
         }
-#elif HAS_PRIVATE_ENCODING
-        if (Qnil != dr->encoding) {
-	    rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-        }
 #endif
 	if (dr->has.pos) {
 	    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1330,9 +1273,9 @@ static char
 read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req, Hint h) {
     VALUE       name = Qnil;
     int         is_encoding = 0;
-    int		pos;
-    int		line;
-    int		col;
+    off_t	pos;
+    off_t	line;
+    off_t	col;
     char	*attr_value;
 
     // already protected by caller
@@ -1377,10 +1320,8 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req,
 	    c = read_quoted_value(dr);
 	    attr_value = dr->buf.str;
 	    if (is_encoding) {
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
 		dr->encoding = rb_enc_find(dr->buf.str);
-#elif HAS_PRIVATE_ENCODING
-		dr->encoding = rb_str_new2(dr->buf.str);
 #else
 		dr->encoding = dr->buf.str;
 #endif
@@ -1411,14 +1352,10 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req,
 		    ox_sax_collapse_special(dr, dr->buf.str, pos, line, col);
 		}
 		args[1] = rb_str_new2(attr_value);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 		if (0 != dr->encoding) {
 		    rb_enc_associate(args[1], dr->encoding);
 		}
-#elif HAS_PRIVATE_ENCODING
-		if (Qnil != dr->encoding) {
-		    rb_funcall(args[1], ox_force_encoding_id, 1, dr->encoding);
-		}
 #endif
 		if (dr->has.pos) {
 		    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1580,7 +1517,7 @@ read_10_uint64(char *b, uint64_t *up) {
 }
 
 int
-ox_sax_collapse_special(SaxDrive dr, char *str, int pos, int line, int col) {
+ox_sax_collapse_special(SaxDrive dr, char *str, long pos, long line, long col) {
     char        *s = str;
     char        *b = str;
 
@@ -1614,19 +1551,12 @@ ox_sax_collapse_special(SaxDrive dr, char *str, int pos, int line, int col) {
 		}
 		if (u <= 0x000000000000007FULL) {
 		    *b++ = (char)u;
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
 		} else if (ox_utf8_encoding == dr->encoding) {
 		    b = ox_ucs_to_utf8_chars(b, u);
 		} else if (0 == dr->encoding) {
 		    dr->encoding = ox_utf8_encoding;
 		    b = ox_ucs_to_utf8_chars(b, u);
-#elif HAS_PRIVATE_ENCODING
-		} else if (ox_utf8_encoding == dr->encoding ||
-			   0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(dr->encoding)))) {
-		    b = ox_ucs_to_utf8_chars(b, u);
-		} else if (Qnil == dr->encoding) {
-		    dr->encoding = ox_utf8_encoding;
-		    b = ox_ucs_to_utf8_chars(b, u);
 #else
 		} else if (0 == dr->encoding) {
 		    dr->encoding = UTF8_STR;
@@ -1668,8 +1598,28 @@ ox_sax_collapse_special(SaxDrive dr, char *str, int pos, int line, int col) {
                 c = '\'';
                 s += 5;
             } else {
-		ox_sax_drive_error_at(dr, INVALID_FORMAT "Invalid special character sequence", pos, line, col);
-		c = '&';
+		char	key[16];
+		char	*k = key;
+		char	*kend = key + sizeof(key) - 1;
+		char	*bn;
+		char	*s2 = s;
+
+		for (; ';' != *s2 && '\0' != *s2; s2++, k++) {
+		    if (kend <= k) {
+			k = key;
+			break;
+		    }
+		    *k = *s2;
+		}
+		*k = '\0';
+		if ('\0' == *key || NULL == (bn = ox_entity_lookup(b, key))) {
+		    ox_sax_drive_error_at(dr, INVALID_FORMAT "Invalid special character sequence", pos, line, col);
+		    c = '&';
+		} else {
+		    b = bn;
+		    s = s2 + 1;
+		    continue;
+		}
             }
             *b++ = (char)c;
 	    col++;
@@ -1731,7 +1681,7 @@ hint_try_close(SaxDrive dr, const char *name) {
 }
 
 static void
-end_element_cb(SaxDrive dr, VALUE name, int pos, int line, int col, Hint h) {
+end_element_cb(SaxDrive dr, VALUE name, long pos, long line, long col, Hint h) {
     if (dr->has.end_element && 0 >= dr->blocked && (NULL == h || ActiveOverlay == h->overlay || NestOverlay == h->overlay)) {
 	if (dr->has.pos) {
 	    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));