ox 2.12.0 → 2.13.4

data/ext/ox/ox.h

@@ -16,20 +16,15 @@ extern "C" {
 #define RSTRING_NOT_MODIFIED
 
 #include "ruby.h"
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 #include "ruby/encoding.h"
 #endif
 
-#ifdef RUBINIUS_RUBY
-#undef T_COMPLEX
-enum st_retval {ST_CONTINUE = 0, ST_STOP = 1, ST_DELETE = 2, ST_CHECK};
+#if HAVE_RUBY_ST_H
+#include "ruby/st.h"
 #else
-#if HAS_TOP_LEVEL_ST_H
-/* Only on travis, local is where it is for all others. Seems to vary depending on the travis machine picked up. */
+// Only on travis, local is where it is for all others. Seems to vary depending on the travis machine picked up.
 #include "st.h"
-#else
-#include "ruby/st.h"
-#endif
 #endif
 
 #include "cache.h"
@@ -146,16 +141,14 @@ typedef struct _options {
     struct _hints	*html_hints;	// html hints
     VALUE		attr_key_mod;
     VALUE		element_key_mod;
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
     rb_encoding		*rb_enc;
-#elif HAS_PRIVATE_ENCODING
-    VALUE		rb_enc;
 #else
     void		*rb_enc;
 #endif
 } *Options;
 
-/* parse information structure */
+// parse information structure
 struct _pInfo {
     struct _helperStack	helpers;
     struct _err		err;
@@ -167,6 +160,9 @@ struct _pInfo {
     CircArray		circ_array;
     unsigned long	id;		// set for text types when cirs_array is set
     Options		options;
+    VALUE		*marked;
+    int			mark_size;	// allocated size
+    int			mark_cnt;
     char		last;		// last character read, rarely set
 };
 
@@ -233,10 +229,8 @@ extern ID	ox_tv_nsec_id;
 extern ID	ox_tv_usec_id;
 extern ID	ox_value_id;
 
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 extern rb_encoding	*ox_utf8_encoding;
-#elif HAS_PRIVATE_ENCODING
-extern VALUE		ox_utf8_encoding;
 #else
 extern void		*ox_utf8_encoding;
 #endif

data/ext/ox/parse.c

@@ -121,7 +121,7 @@ ox_parse(char *xml, size_t len, ParseCallbacks pcb, char **endp, Options options
     if (DEBUG <= options->trace) {
 	printf("Parsing xml:\n%s\n", xml);
     }
-    /* initialize parse info */
+    // initialize parse info
     helper_stack_init(&pi.helpers);
     // Protect against GC
     wrap = Data_Wrap_Struct(rb_cObject, mark_pi_cb, NULL, &pi);
@@ -134,8 +134,11 @@ ox_parse(char *xml, size_t len, ParseCallbacks pcb, char **endp, Options options
     pi.obj = Qnil;
     pi.circ_array = 0;
     pi.options = options;
+    pi.marked = NULL;
+    pi.mark_size = 0;
+    pi.mark_cnt = 0;
     while (1) {
-	next_non_white(&pi);	/* skip white space */
+	next_non_white(&pi);	// skip white space
 	if ('\0' == *pi.s) {
 	    break;
 	}
@@ -143,31 +146,31 @@ ox_parse(char *xml, size_t len, ParseCallbacks pcb, char **endp, Options options
 	    *endp = pi.s;
 	    break;
 	}
-	if ('<' != *pi.s) {		/* all top level entities start with < */
+	if ('<' != *pi.s) {		// all top level entities start with <
 	    set_error(err, "invalid format, expected <", pi.str, pi.s);
 	    helper_stack_cleanup(&pi.helpers);
 	    return Qnil;
 	}
-	pi.s++;		/* past < */
+	pi.s++;		// past <
 	switch (*pi.s) {
-	case '?':	/* processing instruction */
+	case '?':	// processing instruction
 	    pi.s++;
 	    read_instruction(&pi);
 	    break;
-	case '!':	/* comment or doctype */
+	case '!':	// comment or doctype
 	    pi.s++;
 	    if ('\0' == *pi.s) {
 		set_error(err, "invalid format, DOCTYPE or comment not terminated", pi.str, pi.s);
 		helper_stack_cleanup(&pi.helpers);
 		return Qnil;
 	    } else if ('-' == *pi.s) {
-		pi.s++;	/* skip - */
+		pi.s++;	// skip -
 		if ('-' != *pi.s) {
 		    set_error(err, "invalid format, bad comment format", pi.str, pi.s);
 		    helper_stack_cleanup(&pi.helpers);
 		    return Qnil;
 		} else {
-		    pi.s++;	/* skip second - */
+		    pi.s++;	// skip second -
 		    read_comment(&pi);
 		}
 	    } else if ((TolerantEffort == options->effort) ? 0 == strncasecmp("DOCTYPE", pi.s, 7) : 0 == strncmp("DOCTYPE", pi.s, 7)) {
@@ -229,8 +232,7 @@ gather_content(const char *src, char *content, size_t len) {
     return 0;
 }
 
-/* Entered after the "<?" sequence. Ready to read the rest.
- */
+// Entered after the "<?" sequence. Ready to read the rest.
 static void
 read_instruction(PInfo pi) {
     char		content[1024];
@@ -255,7 +257,7 @@ read_instruction(PInfo pi) {
     }
     next_non_white(pi);
     c = *pi->s;
-    *end = '\0'; /* terminate name */
+    *end = '\0'; // terminate name
     if ('?' != c) {
 	while ('?' != c) {
 	    pi->last = 0;
@@ -275,8 +277,8 @@ read_instruction(PInfo pi) {
 		attrs_ok = 0;
 		break;
 	    }
-	    *end = '\0'; /* terminate name */
-	    /* read value */
+	    *end = '\0'; // terminate name
+	    // read value
 	    next_non_white(pi);
 	    if (0 == (attr_value = read_quoted_value(pi))) {
 		attr_stack_cleanup(&attrs);
@@ -355,9 +357,8 @@ read_delimited(PInfo pi, char end) {
     }
 }
 
-/* Entered after the "<!DOCTYPE" sequence plus the first character after
- * that. Ready to read the rest.
- */
+// Entered after the "<!DOCTYPE" sequence plus the first character after
+// that. Ready to read the rest.
 static void
 read_doctype(PInfo pi) {
     char	*docType;
@@ -376,8 +377,7 @@ read_doctype(PInfo pi) {
     }
 }
 
-/* Entered after "<!--". Returns error code.
- */
+// Entered after "<!--". Returns error code.
 static void
 read_comment(PInfo pi) {
     char	*end;
@@ -406,16 +406,15 @@ read_comment(PInfo pi) {
 	    break;
 	}
     }
-    *end = '\0'; /* in case the comment was blank */
+    *end = '\0'; // in case the comment was blank
     pi->s = end + 3;
     if (0 != pi->pcb->add_comment) {
 	pi->pcb->add_comment(pi, comment);
     }
 }
 
-/* Entered after the '<' and the first character after that. Returns status
- * code.
- */
+// Entered after the '<' and the first character after that. Returns stat
+// code.
 static char*
 read_element(PInfo pi) {
     struct _attrStack	attrs;
@@ -439,10 +438,9 @@ read_element(PInfo pi) {
     c = *pi->s;
     *end = '\0';
     if ('/' == c) {
-	/* empty element, no attributes and no children */
+	// empty element, no attributes and no children
 	pi->s++;
 	if ('>' != *pi->s) {
-	    /*printf("*** '%s' ***\n", pi->s); */
 	    attr_stack_cleanup(&attrs);
 	    set_error(&pi->err, "invalid format, element not closed", pi->str, pi->s);
 	    return 0;
@@ -480,8 +478,8 @@ read_element(PInfo pi) {
 	    pi->s++;
 	    pi->pcb->add_element(pi, ename, attrs.head, hasChildren);
 	    pi->pcb->end_element(pi, ename);
-
 	    attr_stack_cleanup(&attrs);
+
 	    return 0;
 	case '>':
 	    /* has either children or a value */
@@ -545,6 +543,12 @@ read_element(PInfo pi) {
 	while (!done) {
 	    start = pi->s;
 	    next_non_white(pi);
+	    if (OffSkip == pi->options->skip && start < pi->s && '<' == *pi->s) {
+		c = *pi->s;
+		*pi->s = '\0';
+		pi->pcb->add_text(pi, start, 1);
+		*pi->s = c;
+	    }
 	    c = *pi->s++;
 	    if ('\0' == c) {
 		attr_stack_cleanup(&attrs);
@@ -1001,11 +1005,13 @@ read_coded_chars(PInfo pi, char *text) {
     char	*b, buf[32];
     char	*end = buf + sizeof(buf) - 1;
     char	*s;
+    long	blen = 0;
 
     for (b = buf, s = pi->s; b < end; b++, s++) {
 	*b = *s;
 	if (';' == *s) {
 	    *(b + 1) = '\0';
+	    blen = b - buf;
 	    s++;
 	    break;
 	}
@@ -1026,18 +1032,9 @@ read_coded_chars(PInfo pi, char *text) {
 	} else {
 	    if (u <= 0x000000000000007FULL) {
 		*text++ = (char)u;
-#if HAS_PRIVATE_ENCODING
-	    } else if (ox_utf8_encoding == pi->options->rb_enc ||
-		       0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(pi->options->rb_enc)))) {
-#else
 	    } else if (ox_utf8_encoding == pi->options->rb_enc) {
-#endif
 		text = ox_ucs_to_utf8_chars(text, u);
-#if HAS_PRIVATE_ENCODING
-	    } else if (Qnil == pi->options->rb_enc) {
-#else
 	    } else if (0 == pi->options->rb_enc) {
-#endif
 		pi->options->rb_enc = ox_utf8_encoding;
 		text = ox_ucs_to_utf8_chars(text, u);
 	    } else if (TolerantEffort == pi->options->effort) {
@@ -1048,30 +1045,20 @@ read_coded_chars(PInfo pi, char *text) {
 	    } else {
 		/*set_error(&pi->err, "Invalid encoding, need UTF-8 or UTF-16 encoding to parse &#nnnn; character sequences.", pi->str, pi->s); */
 		set_error(&pi->err, "Invalid encoding, need UTF-8 encoding to parse &#nnnn; character sequences.", pi->str, pi->s);
-		return 0;
+		return NULL;
 	    }
 	    pi->s = s;
 	}
-    } else if (0 == strcasecmp(buf, "nbsp;")) {
-	pi->s = s;
-	*text++ = ' ';
-    } else if (0 == strcasecmp(buf, "lt;")) {
-	pi->s = s;
-	*text++ = '<';
-    } else if (0 == strcasecmp(buf, "gt;")) {
-	pi->s = s;
-	*text++ = '>';
-    } else if (0 == strcasecmp(buf, "amp;")) {
-	pi->s = s;
-	*text++ = '&';
-    } else if (0 == strcasecmp(buf, "quot;")) {
-	pi->s = s;
-	*text++ = '"';
-    } else if (0 == strcasecmp(buf, "apos;")) {
-	pi->s = s;
-	*text++ = '\'';
     } else {
-	*text++ = '&';
+	char	*t2;
+
+	buf[blen] = '\0';
+	if (NULL == (t2 = ox_entity_lookup(text, buf))) {
+	    *text++ = '&';
+	} else {
+	    text = t2;
+	    pi->s = s;
+	}
     }
     return text;
 }
@@ -1113,19 +1100,10 @@ collapse_special(PInfo pi, char *str) {
 		}
 		if (u <= 0x000000000000007FULL) {
 		    *b++ = (char)u;
-#if HAS_PRIVATE_ENCODING
-		} else if (ox_utf8_encoding == pi->options->rb_enc ||
-			   0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(pi->options->rb_enc)))) {
-#else
 		} else if (ox_utf8_encoding == pi->options->rb_enc) {
-#endif
 		    b = ox_ucs_to_utf8_chars(b, u);
 		    /* TBD support UTF-16 */
-#if HAS_PRIVATE_ENCODING
-		} else if (Qnil == pi->options->rb_enc) {
-#else
 		} else if (0 == pi->options->rb_enc) {
-#endif
 		    pi->options->rb_enc = ox_utf8_encoding;
 		    b = ox_ucs_to_utf8_chars(b, u);
 		} else {
@@ -1154,16 +1132,30 @@ collapse_special(PInfo pi, char *str) {
 		    *b++ = '&';
 		    continue;
 		} else {
-		    c = '?';
+		    char	key[16];
+		    char	*k = key;
+		    char	*kend = key + sizeof(key) - 1;
+
+		    *k++ = *s;
 		    while (';' != *s++) {
 			if ('\0' == *s) {
 			    set_error(&pi->err, "Invalid format, special character does not end with a semicolon", pi->str, pi->s);
 			    return EDOM;
 			}
+			if (kend <= k) {
+			    k = key;
+			    break;
+			}
+			*k++ = *s;
 		    }
-		    s++;
-		    set_error(&pi->err, "Invalid format, invalid special character sequence", pi->str, pi->s);
-		    return 0;
+		    k--;
+		    *k = '\0';
+		    if ('\0' == *key || NULL == (b = ox_entity_lookup(b, key))) {
+			set_error(&pi->err, "Invalid format, invalid special character sequence", pi->str, pi->s);
+			c = '?';
+			return 0;
+		    }
+		    continue;
 		}
 		*b++ = (char)c;
 	    }

data/ext/ox/sax.c

@@ -9,13 +9,16 @@
 #include <stdio.h>
 #include <strings.h>
 #include <sys/types.h>
-#if NEEDS_UIO
+#if HAVE_SYS_UIO_H
 #include <sys/uio.h>
 #endif
 #include <unistd.h>
 #include <time.h>
 
 #include "ruby.h"
+#if HAVE_RB_ENC_ASSOCIATE
+#include "ruby/encoding.h"
+#endif
 #include "ox.h"
 #include "sax.h"
 #include "sax_stack.h"
@@ -55,7 +58,7 @@ static char		read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml,
 static char		read_name_token(SaxDrive dr);
 static char		read_quoted_value(SaxDrive dr);
 
-static void		end_element_cb(SaxDrive dr, VALUE name, int pos, int line, int col, Hint h);
+static void		end_element_cb(SaxDrive dr, VALUE name, long pos, long line, long col, Hint h);
 
 static void		hint_clear_empty(SaxDrive dr);
 static Nv		hint_try_close(SaxDrive dr, const char *name);
@@ -68,9 +71,9 @@ static VALUE protect_parse(VALUE drp) {
     return Qnil;
 }
 
-#if HAS_ENCODING_SUPPORT || HAS_PRIVATE_ENCODING
+#if HAVE_RB_ENC_ASSOCIATE
 static int
-strIsAscii(const char *s) {
+str_is_ascii(const char *s) {
     for (; '\0' != *s; s++) {
 	if (*s < ' ' || '~' < *s) {
 	    return 0;
@@ -87,8 +90,8 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 
     if (dr->options.symbolize) {
 	if (Qundef == (sym = ox_cache_get(ox_symbol_cache, str, &slot, strp))) {
-#if HAS_ENCODING_SUPPORT
-	    if (0 != dr->encoding && !strIsAscii(str)) {
+#if HAVE_RB_ENC_ASSOCIATE
+	    if (0 != dr->encoding && !str_is_ascii(str)) {
 		VALUE	rstr = rb_str_new2(str);
 
 		// TBD if sym can be pinned down then use this all the time
@@ -99,20 +102,6 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 		sym = ID2SYM(rb_intern(str));
 		*slot = sym;
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding && !strIsAscii(str)) {
-		VALUE	rstr = rb_str_new2(str);
-
-		rb_funcall(rstr, ox_force_encoding_id, 1, dr->encoding);
-		sym = rb_funcall(rstr, ox_to_sym_id, 0);
-		// Needed for Ruby 2.2 to get around the GC of symbols created
-		// with to_sym which is needed for encoded symbols.
-		rb_ary_push(ox_sym_bank, sym);
-		*slot = Qundef;
-	    } else {
-		sym = ID2SYM(rb_intern(str));
-		*slot = sym;
-	    }
 #else
 	    sym = ID2SYM(rb_intern(str));
 	    *slot = sym;
@@ -120,14 +109,10 @@ str2sym(SaxDrive dr, const char *str, const char **strp) {
 	}
     } else {
 	sym = rb_str_new2(str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	if (0 != dr->encoding) {
 	    rb_enc_associate(sym, dr->encoding);
 	}
-#elif HAS_PRIVATE_ENCODING
-	if (Qnil != dr->encoding) {
-	    rb_funcall(sym, ox_force_encoding_id, 1, dr->encoding);
-	}
 #endif
 	if (0 != strp) {
 	    *strp = StringValuePtr(sym);
@@ -182,7 +167,7 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, SaxOptions options) {
     dr->blocked = 0;
     dr->abort = false;
     has_init(&dr->has, handler);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
     if ('\0' == *ox_default_options.encoding) {
 	VALUE	encoding;
 
@@ -196,18 +181,6 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, SaxOptions options) {
     } else {
         dr->encoding = rb_enc_find(ox_default_options.encoding);
     }
-#elif HAS_PRIVATE_ENCODING
-    if ('\0' == *ox_default_options.encoding) {
-	VALUE	encoding;
-
-	if (rb_respond_to(io, ox_external_encoding_id) && Qnil != (encoding = rb_funcall(io, ox_external_encoding_id, 0))) {
-	    dr->encoding = encoding;
-	} else {
-	    dr->encoding = Qnil;
-	}
-    } else {
-        dr->encoding = rb_str_new2(ox_default_options.encoding);
-    }
 #else
     dr->encoding = 0;
 #endif
@@ -221,7 +194,7 @@ ox_sax_drive_cleanup(SaxDrive dr) {
 }
 
 static void
-ox_sax_drive_error_at(SaxDrive dr, const char *msg, int pos, int line, int col) {
+ox_sax_drive_error_at(SaxDrive dr, const char *msg, off_t pos, off_t line, off_t col) {
     if (dr->has.error) {
         VALUE   args[3];
 
@@ -255,9 +228,7 @@ skipBOM(SaxDrive dr) {
 
     if (0xEF == (uint8_t)c) { /* only UTF8 is supported */
 	if (0xBB == (uint8_t)buf_get(&dr->buf) && 0xBF == (uint8_t)buf_get(&dr->buf)) {
-#if HAS_ENCODING_SUPPORT
-	    dr->encoding = ox_utf8_encoding;
-#elif HAS_PRIVATE_ENCODING
+#if HAVE_RB_ENC_FIND
 	    dr->encoding = ox_utf8_encoding;
 #else
 	    dr->encoding = UTF8_STR;
@@ -301,11 +272,11 @@ parse(SaxDrive dr) {
 		    }
 		    c = read_comment(dr);
 		} else {
-		    int	i;
-		    int	spaced = 0;
-		    int	pos = dr->buf.pos + 1;
-		    int	line = dr->buf.line;
-		    int	col = dr->buf.col + 1;
+		    int		i;
+		    int		spaced = 0;
+		    off_t	pos = dr->buf.pos + 1;
+		    off_t	line = dr->buf.line;
+		    off_t	col = dr->buf.col + 1;
 
 		    if (is_white(c)) {
 			spaced = 1;
@@ -359,19 +330,15 @@ parse(SaxDrive dr) {
 		parent = stack_peek(&dr->stack);
 		if (0 != parent && 0 == parent->childCnt && dr->has.text && !dr->blocked) {
 		    VALUE	args[1];
-		    int		pos = dr->buf.pos;
-		    int		line = dr->buf.line;
-		    int		col = dr->buf.col - 1;
+		    off_t	pos = dr->buf.pos;
+		    off_t	line = dr->buf.line;
+		    off_t	col = dr->buf.col - 1;
 
 		    args[0] = rb_str_new2("");
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 		    if (0 != dr->encoding) {
 			rb_enc_associate(args[0], dr->encoding);
 		    }
-#elif HAS_PRIVATE_ENCODING
-		    if (Qnil != dr->encoding) {
-			rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-		    }
 #endif
 		    if (dr->has.pos) {
 			rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -479,9 +446,9 @@ read_instruction(SaxDrive dr) {
     int		coff;
     VALUE	target = Qnil;
     int		is_xml;
-    int		pos = dr->buf.pos - 1;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    off_t	pos = dr->buf.pos - 1;
+    off_t	line = dr->buf.line;
+    off_t	col = dr->buf.col - 1;
 
     buf_protect(&dr->buf);
     if ('\0' == (c = read_name_token(dr))) {
@@ -511,7 +478,7 @@ read_instruction(SaxDrive dr) {
     line = dr->buf.line;
     col = dr->buf.col;
     read_content(dr, content, sizeof(content) - 1);
-    coff = dr->buf.tail - dr->buf.head;
+    coff = (int)(dr->buf.tail - dr->buf.head);
     buf_reset(&dr->buf);
     dr->err = 0;
     c = read_attrs(dr, c, '?', '?', is_xml, 1, NULL);
@@ -523,17 +490,13 @@ read_instruction(SaxDrive dr) {
 	    VALUE   args[1];
 
 	    if (dr->options.convert_special) {
-		ox_sax_collapse_special(dr, content, pos, line, col);
+		ox_sax_collapse_special(dr, content, (int)pos, (int)line, (int)col);
 	    }
 	    args[0] = rb_str_new2(content);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.line) {
 		rb_ivar_set(dr->handler, ox_at_line_id, LONG2NUM(line));
@@ -627,9 +590,9 @@ read_delimited(SaxDrive dr, char end) {
  */
 static char
 read_doctype(SaxDrive dr) {
-    int		pos = dr->buf.pos - 9;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 9;
+    long	pos = (long)(dr->buf.pos - 9);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 9);
     char	*s;
     Nv		parent = stack_peek(&dr->stack);
 
@@ -673,9 +636,9 @@ read_cdata(SaxDrive dr) {
     char        	c;
     char        	zero = '\0';
     int         	end = 0;
-    int			pos = dr->buf.pos - 9;
-    int			line = dr->buf.line;
-    int			col = dr->buf.col - 9;
+    long		pos = (long)(dr->buf.pos - 9);
+    long		line = (long)(dr->buf.line);
+    long		col = (long)(dr->buf.col - 9);
     struct _checkPt	cp = CHECK_PT_INIT;
     Nv			parent = stack_peek(&dr->stack);
 
@@ -732,14 +695,10 @@ read_cdata(SaxDrive dr) {
 	    VALUE       args[1];
 
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -768,9 +727,9 @@ read_comment(SaxDrive dr) {
     char        	c;
     char        	zero = '\0';
     int         	end = 0;
-    int			pos = dr->buf.pos - 4;
-    int			line = dr->buf.line;
-    int			col = dr->buf.col - 4;
+    long		pos = (long)(dr->buf.pos - 4);
+    long		line = (long)(dr->buf.line);
+    long		col = (long)(dr->buf.col - 4);
     struct _checkPt	cp = CHECK_PT_INIT;
 
     buf_backup(&dr->buf); /* back up to the start in case the cdata is empty */
@@ -826,14 +785,10 @@ read_comment(SaxDrive dr) {
 	    (NULL != h && (ActiveOverlay == h->overlay || ActiveOverlay == h->overlay))) {
 
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -864,9 +819,9 @@ read_element_start(SaxDrive dr) {
     volatile VALUE	name = Qnil;
     char        	c;
     int			closed;
-    int			pos = dr->buf.pos;
-    int			line = dr->buf.line;
-    int			col = dr->buf.col;
+    long		pos = (long)(dr->buf.pos);
+    long		line = (long)(dr->buf.line);
+    long		col = (long)(dr->buf.col);
     Hint		h = NULL;
     int			stackless = 0;
     Nv			parent = stack_peek(&dr->stack);
@@ -1020,9 +975,9 @@ static char
 read_element_end(SaxDrive dr) {
     VALUE       name = Qnil;
     char        c;
-    int		pos = dr->buf.pos - 1;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    long	pos = (long)(dr->buf.pos - 1);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 1);
     Nv		nv;
     Hint	h = NULL;
 
@@ -1118,9 +1073,9 @@ static char
 read_text(SaxDrive dr) {
     VALUE	args[1];
     char        c;
-    int		pos = dr->buf.pos;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    long	pos = (long)(dr->buf.pos);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 1);
     Nv		parent = stack_peek(&dr->stack);
     int		allWhite = 1;
 
@@ -1158,14 +1113,10 @@ read_text(SaxDrive dr) {
 	    ((NoSkip == dr->options.skip && !isEnd) ||
 	     (OffSkip == dr->options.skip))) {
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1213,14 +1164,10 @@ read_text(SaxDrive dr) {
 		break;
 	    }
 	    args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 	    if (0 != dr->encoding) {
 		rb_enc_associate(args[0], dr->encoding);
 	    }
-#elif HAS_PRIVATE_ENCODING
-	    if (Qnil != dr->encoding) {
-		rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-	    }
 #endif
 	    if (dr->has.pos) {
 		rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1266,9 +1213,9 @@ static char
 read_jump(SaxDrive dr, const char *pat) {
     VALUE	args[1];
     char        c;
-    int		pos = dr->buf.pos;
-    int		line = dr->buf.line;
-    int		col = dr->buf.col - 1;
+    long	pos = (long)(dr->buf.pos);
+    long	line = (long)(dr->buf.line);
+    long	col = (long)(dr->buf.col - 1);
     Nv		parent = stack_peek(&dr->stack);
 
     buf_protect(&dr->buf);
@@ -1299,14 +1246,10 @@ read_jump(SaxDrive dr, const char *pat) {
     // TBD check parent overlay
     if (dr->has.text && !dr->blocked) {
         args[0] = rb_str_new2(dr->buf.str);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
         if (0 != dr->encoding) {
             rb_enc_associate(args[0], dr->encoding);
         }
-#elif HAS_PRIVATE_ENCODING
-        if (Qnil != dr->encoding) {
-	    rb_funcall(args[0], ox_force_encoding_id, 1, dr->encoding);
-        }
 #endif
 	if (dr->has.pos) {
 	    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1330,9 +1273,9 @@ static char
 read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req, Hint h) {
     VALUE       name = Qnil;
     int         is_encoding = 0;
-    int		pos;
-    int		line;
-    int		col;
+    off_t	pos;
+    off_t	line;
+    off_t	col;
     char	*attr_value;
 
     // already protected by caller
@@ -1377,10 +1320,8 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req,
 	    c = read_quoted_value(dr);
 	    attr_value = dr->buf.str;
 	    if (is_encoding) {
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
 		dr->encoding = rb_enc_find(dr->buf.str);
-#elif HAS_PRIVATE_ENCODING
-		dr->encoding = rb_str_new2(dr->buf.str);
 #else
 		dr->encoding = dr->buf.str;
 #endif
@@ -1411,14 +1352,10 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml, int eq_req,
 		    ox_sax_collapse_special(dr, dr->buf.str, pos, line, col);
 		}
 		args[1] = rb_str_new2(attr_value);
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_ASSOCIATE
 		if (0 != dr->encoding) {
 		    rb_enc_associate(args[1], dr->encoding);
 		}
-#elif HAS_PRIVATE_ENCODING
-		if (Qnil != dr->encoding) {
-		    rb_funcall(args[1], ox_force_encoding_id, 1, dr->encoding);
-		}
 #endif
 		if (dr->has.pos) {
 		    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));
@@ -1580,7 +1517,7 @@ read_10_uint64(char *b, uint64_t *up) {
 }
 
 int
-ox_sax_collapse_special(SaxDrive dr, char *str, int pos, int line, int col) {
+ox_sax_collapse_special(SaxDrive dr, char *str, long pos, long line, long col) {
     char        *s = str;
     char        *b = str;
 
@@ -1614,19 +1551,12 @@ ox_sax_collapse_special(SaxDrive dr, char *str, int pos, int line, int col) {
 		}
 		if (u <= 0x000000000000007FULL) {
 		    *b++ = (char)u;
-#if HAS_ENCODING_SUPPORT
+#if HAVE_RB_ENC_FIND
 		} else if (ox_utf8_encoding == dr->encoding) {
 		    b = ox_ucs_to_utf8_chars(b, u);
 		} else if (0 == dr->encoding) {
 		    dr->encoding = ox_utf8_encoding;
 		    b = ox_ucs_to_utf8_chars(b, u);
-#elif HAS_PRIVATE_ENCODING
-		} else if (ox_utf8_encoding == dr->encoding ||
-			   0 == strcasecmp(rb_str_ptr(rb_String(ox_utf8_encoding)), rb_str_ptr(rb_String(dr->encoding)))) {
-		    b = ox_ucs_to_utf8_chars(b, u);
-		} else if (Qnil == dr->encoding) {
-		    dr->encoding = ox_utf8_encoding;
-		    b = ox_ucs_to_utf8_chars(b, u);
 #else
 		} else if (0 == dr->encoding) {
 		    dr->encoding = UTF8_STR;
@@ -1668,8 +1598,28 @@ ox_sax_collapse_special(SaxDrive dr, char *str, int pos, int line, int col) {
                 c = '\'';
                 s += 5;
             } else {
-		ox_sax_drive_error_at(dr, INVALID_FORMAT "Invalid special character sequence", pos, line, col);
-		c = '&';
+		char	key[16];
+		char	*k = key;
+		char	*kend = key + sizeof(key) - 1;
+		char	*bn;
+		char	*s2 = s;
+
+		for (; ';' != *s2 && '\0' != *s2; s2++, k++) {
+		    if (kend <= k) {
+			k = key;
+			break;
+		    }
+		    *k = *s2;
+		}
+		*k = '\0';
+		if ('\0' == *key || NULL == (bn = ox_entity_lookup(b, key))) {
+		    ox_sax_drive_error_at(dr, INVALID_FORMAT "Invalid special character sequence", pos, line, col);
+		    c = '&';
+		} else {
+		    b = bn;
+		    s = s2 + 1;
+		    continue;
+		}
             }
             *b++ = (char)c;
 	    col++;
@@ -1731,7 +1681,7 @@ hint_try_close(SaxDrive dr, const char *name) {
 }
 
 static void
-end_element_cb(SaxDrive dr, VALUE name, int pos, int line, int col, Hint h) {
+end_element_cb(SaxDrive dr, VALUE name, long pos, long line, long col, Hint h) {
     if (dr->has.end_element && 0 >= dr->blocked && (NULL == h || ActiveOverlay == h->overlay || NestOverlay == h->overlay)) {
 	if (dr->has.pos) {
 	    rb_ivar_set(dr->handler, ox_at_pos_id, LONG2NUM(pos));