ox 1.9.2 → 1.9.3

data/README.md

@@ -34,9 +34,22 @@ A fast XML parser and Object marshaller as a Ruby gem.
 
 ## <a name="release">Release Notes</a>
 
-### Release 1.9.2
+### Release 1.9.3
 
- - Fixed bug in the sax element name check that cause a memory write error.
+ - mcarpenter fixed a compile problem with Cygwin.
+
+ - Now more tolerant when the :effort is set to :tolerant. Ox will let all sorts
+   of errors typical in HTML documents pass. The result may not be perfect but
+   at least parsed results are returned.
+   
+   - Attribute values need not be quoted or they can be quoted with single
+     quotes or there can be no =value are all.
+
+   - Elements not terminated will be terminated by the next element
+     termination. This effect goes up until a match is found on the element
+     name.
+
+ - SAX parser also given a :tolerant option with the same tolerance as the string parser.
 
 ## <a name="description">Description</a>
 

data/ext/ox/extconf.rb

@@ -20,7 +20,7 @@ dflags = {
   'HAS_RB_TIME_TIMESPEC' => ('ruby' == type && ('1.9.3' == RUBY_VERSION)) ? 1 : 0,
   #'HAS_RB_TIME_TIMESPEC' => ('ruby' == type && ('1.9.3' == RUBY_VERSION || '2' <= version[0])) ? 1 : 0,
   'HAS_TM_GMTOFF' => ('ruby' == type && (('1' == version[0] && '9' == version[1]) || '2' <= version[0]) && 
-                      !(platform.include?('solaris') || platform.include?('linux') || RUBY_PLATFORM =~ /(win|w)32$/)) ? 1 : 0,
+                      !(platform.include?('cygwin') || platform.include?('solaris') || platform.include?('linux') || RUBY_PLATFORM =~ /(win|w)32$/)) ? 1 : 0,
   'HAS_ENCODING_SUPPORT' => (('ruby' == type || 'rubinius' == type) &&
                              (('1' == version[0] && '9' == version[1]) || '2' <= version[0])) ? 1 : 0,
   'HAS_PRIVATE_ENCODING' => ('jruby' == type && '1' == version[0] && '9' == version[1]) ? 1 : 0,

data/ext/ox/ox.c

@@ -435,7 +435,7 @@ static VALUE
 load(char *xml, int argc, VALUE *argv, VALUE self, VALUE encoding) {
     VALUE		obj;
     struct _Options	options = ox_default_options;
-    
+
     if (1 == argc && rb_cHash == rb_obj_class(*argv)) {
 	VALUE	h = *argv;
 	VALUE	v;
@@ -622,11 +622,13 @@ load_file(int argc, VALUE *argv, VALUE self) {
  * @param [Ox::Sax] handler SAX (responds to OX::Sax methods) like handler
  * @param [IO|String] io IO Object to read from
  * @param [Hash] options parse options
- * @param [true|false] :convert_special flag indicating special special characters like < are converted
+ * @param [true|false] :convert_special flag indicating special characters like < are converted
+ * @param [true|false] :tolerant flag indicating the parser should be tolerant of XML errors
  */
 static VALUE
 sax_parse(int argc, VALUE *argv, VALUE self) {
     int convert = 0;
+    int tolerant = 0;
 
     if (argc < 2) {
 	rb_raise(ox_parse_error_class, "Wrong number of arguments to sax_parse.\n");
@@ -638,8 +640,11 @@ sax_parse(int argc, VALUE *argv, VALUE self) {
 	if (Qnil != (v = rb_hash_lookup(h, convert_special_sym))) {
 	    convert = (Qtrue == v);
 	}
+	if (Qnil != (v = rb_hash_lookup(h, tolerant_sym))) {
+	    tolerant = (Qtrue == v);
+	}
     }
-    ox_sax_parse(argv[0], argv[1], convert);
+    ox_sax_parse(argv[0], argv[1], convert, tolerant);
 
     return Qnil;
 }

data/ext/ox/ox.h

@@ -206,15 +206,15 @@ struct _PInfo {
     CircArray		circ_array;
     unsigned long	id;		/* set for text types when cirs_array is set */
     Options		options;
+    char		last;		/* last character read, rarely set */
 };
 
 extern VALUE	ox_parse(char *xml, ParseCallbacks pcb, char **endp, Options options);
 extern void	_ox_raise_error(const char *msg, const char *xml, const char *current, const char* file, int line);
 
-extern void	ox_sax_parse(VALUE handler, VALUE io, int convert);
+extern void	ox_sax_parse(VALUE handler, VALUE io, int convert, int tolerant);
 extern void	ox_sax_define(void);
 
-
 extern char*	ox_write_obj_to_str(VALUE obj, Options copts);
 extern void	ox_write_obj_to_file(VALUE obj, const char *path, Options copts);
 

data/ext/ox/parse.c

@@ -39,7 +39,7 @@
 static void	read_instruction(PInfo pi);
 static void	read_doctype(PInfo pi);
 static void	read_comment(PInfo pi);
-static void	read_element(PInfo pi);
+static char*	read_element(PInfo pi);
 static void	read_text(PInfo pi);
 /*static void	  read_reduced_text(PInfo pi); */
 static void	read_cdata(PInfo pi);
@@ -147,7 +147,7 @@ ox_parse(char *xml, ParseCallbacks pcb, char **endp, Options options) {
 		    pi.s++;	/* skip second - */
 		    read_comment(&pi);
 		}
-	    } else if (0 == strncmp("DOCTYPE", pi.s, 7)) {
+	    } else if ((TolerantEffort == options->effort) ? 0 == strncasecmp("DOCTYPE", pi.s, 7) : 0 == strncmp("DOCTYPE", pi.s, 7)) {
 		pi.s += 7;
 		read_doctype(&pi);
 	    } else {
@@ -210,7 +210,8 @@ read_instruction(PInfo pi) {
     c = *pi->s;
     *end = '\0'; /* terminate name */
     if ('?' != c) {
-	while ('?' != *pi->s) {
+	while ('?' != c) {
+	    pi->last = 0;
 	    if ('\0' == *pi->s) {
 		raise_error("invalid format, processing instruction not terminated", pi->str, pi->s);
 	    }
@@ -232,6 +233,11 @@ read_instruction(PInfo pi) {
 		break;
 	    }
 	    next_non_white(pi);
+	    if ('\0' == pi->last) {
+		c = *pi->s;
+	    } else {
+		c = pi->last;
+	    }
 	}
 	if ('?' == *pi->s) {
 	    pi->s++;
@@ -326,7 +332,7 @@ read_comment(PInfo pi) {
 /* Entered after the '<' and the first character after that. Returns status
  * code.
  */
-static void
+static char*
 read_element(PInfo pi) {
     struct _Attr	attrs[MAX_ATTRS];
     Attr		ap = attrs;
@@ -356,7 +362,7 @@ read_element(PInfo pi) {
 	pi->pcb->add_element(pi, ename, attrs, hasChildren);
 	pi->pcb->end_element(pi, ename);
 
-	return;
+	return 0;
     }
     /* read attribute names until the close (/ or >) is reached */
     while (!done) {
@@ -364,6 +370,7 @@ read_element(PInfo pi) {
 	    next_non_white(pi);
 	    c = *pi->s;
 	}
+	pi->last = 0;
 	switch (c) {
 	case '\0':
 	    raise_error("invalid format, document not terminated", pi->str, pi->s);
@@ -378,7 +385,7 @@ read_element(PInfo pi) {
 	    pi->pcb->add_element(pi, ename, attrs, hasChildren);
 	    pi->pcb->end_element(pi, ename);
 
-	    return;
+	    return 0;
 	case '>':
 	    /* has either children or a value */
 	    pi->s++;
@@ -394,7 +401,19 @@ read_element(PInfo pi) {
 	    end = pi->s;
 	    next_non_white(pi);
 	    if ('=' != *pi->s++) {
-		raise_error("invalid format, no attribute value", pi->str, pi->s);
+		if (TolerantEffort == pi->options->effort) {
+		    pi->s--;
+		    pi->last = *pi->s;
+		    *end = '\0'; /* terminate name */
+		    ap->value = "";
+		    ap++;
+		    if (MAX_ATTRS <= (ap - attrs)) {
+			raise_error("too many attributes", pi->str, pi->s);
+		    }
+		    break;
+		} else {
+		    raise_error("invalid format, no attribute value", pi->str, pi->s);
+		}
 	    }
 	    *end = '\0'; /* terminate name */
 	    /* read value */
@@ -411,7 +430,12 @@ read_element(PInfo pi) {
 	    }
 	    break;
 	}
-	c = '\0';
+	if ('\0' == pi->last) {
+	    c = '\0';
+	} else {
+	    c = pi->last;
+	    pi->last = '\0';
+	}
     }
     if (hasChildren) {
 	char	*start;
@@ -435,7 +459,9 @@ read_element(PInfo pi) {
 		    if ('-' == *pi->s && '-' == *(pi->s + 1)) {
 			pi->s += 2;
 			read_comment(pi);
-		    } else if (0 == strncmp("[CDATA[", pi->s, 7)) {
+		    } else if ((TolerantEffort == pi->options->effort) ?
+			       0 == strncasecmp("[CDATA[", pi->s, 7) :
+			       0 == strncmp("[CDATA[", pi->s, 7)) {
 			pi->s += 7;
 			read_cdata(pi);
 		    } else {
@@ -455,7 +481,12 @@ read_element(PInfo pi) {
 		    c = *pi->s;
 		    *end = '\0';
 		    if (0 != strcmp(name, ename)) {
-			raise_error("invalid format, elements overlap", pi->str, pi->s);
+			if (TolerantEffort == pi->options->effort) {
+			    pi->pcb->end_element(pi, ename);
+			    return name;
+			} else {
+			    raise_error("invalid format, elements overlap", pi->str, pi->s);
+			}
 		    }
 		    if ('>' != c) {
 			raise_error("invalid format, element not closed", pi->str, pi->s);
@@ -467,13 +498,27 @@ read_element(PInfo pi) {
 		    }
 		    pi->s++;
 		    pi->pcb->end_element(pi, ename);
-		    return;
+		    return 0;
 		case '\0':
-		    raise_error("invalid format, document not terminated", pi->str, pi->s);
+		    if (TolerantEffort == pi->options->effort) {
+			return 0;
+		    } else {
+			raise_error("invalid format, document not terminated", pi->str, pi->s);
+		    }
 		default:
 		    first = 0;
 		    /* a child element */
-		    read_element(pi);
+		    // Child closed with mismatched name.
+		    if (0 != (name = read_element(pi))) {
+			if (0 == strcmp(name, ename)) {
+			    pi->s++;
+			    pi->pcb->end_element(pi, ename);
+			    return 0;
+			} else { // not the correct element yet
+			    pi->pcb->end_element(pi, ename);
+			    return name;
+			}
+		    }
 		    break;
 		}
 	    } else {	/* read as TEXT */
@@ -489,11 +534,12 @@ read_element(PInfo pi) {
 		    /* close tag after text so treat as a value */
 		    pi->s += elen + 3;
 		    pi->pcb->end_element(pi, ename);
-		    return;
+		    return 0;
 		}
 	    }
 	}
     }
+    return 0;
 }
 
 static void
@@ -697,10 +743,31 @@ read_quoted_value(PInfo pi) {
                 raise_error("invalid format, document not terminated", pi->str, pi->s);
             }
         }
-        *pi->s = '\0'; /* terminate value */
-        pi->s++;	   /* move past quote */
+        *pi->s = '\0';	/* terminate value */
+        pi->s++;	/* move past quote */
     } else if (StrictEffort == pi->options->effort) {
 	raise_error("invalid format, expected a quote character", pi->str, pi->s);
+    } else if (TolerantEffort == pi->options->effort) {
+        value = pi->s;
+        for (; 1; pi->s++) {
+	    switch (*pi->s) {
+	    case '\0':
+                raise_error("invalid format, document not terminated", pi->str, pi->s);
+	    case ' ':
+	    case '/':
+	    case '>':
+	    case '?': // for instructions
+	    case '\t':
+	    case '\n':
+	    case '\r':
+		pi->last = *pi->s;
+		*pi->s = '\0';	/* terminate value */
+		pi->s++;
+		return value;
+	    default:
+		break;
+            }
+        }
     } else {
         value = pi->s;
         next_white(pi);
@@ -812,7 +879,7 @@ read_coded_chars(PInfo pi, char *text) {
 	}
     }
     if (b > end) {
-	*text++ = *pi->s;
+	*text++ = '&';
     } else if ('#' == *buf) {
 	uint64_t	u = 0;
 	
@@ -823,9 +890,8 @@ read_coded_chars(PInfo pi, char *text) {
 	    b = read_10_uint64(b, &u);
 	}
 	if (0 == b) {
-	    *text++ = *pi->s;
+	    *text++ = '&';
 	} else {
-	    pi->s = s;
 	    if (u <= 0x000000000000007FULL) {
 		*text++ = (char)u;
 #if HAS_PRIVATE_ENCODING
@@ -842,10 +908,14 @@ read_coded_chars(PInfo pi, char *text) {
 #endif
 		pi->options->rb_enc = ox_utf8_encoding;
 		text = ucs_to_utf8_chars(text, u);
+	    } else if (TolerantEffort == pi->options->effort) {
+		*text++ = '&';
+		return text;
 	    } else {
 		/*raise_error("Invalid encoding, need UTF-8 or UTF-16 encoding to parse &#nnnn; character sequences.", pi->str, pi->s); */
 		raise_error("Invalid encoding, need UTF-8 encoding to parse &#nnnn; character sequences.", pi->str, pi->s);
 	    }
+	    pi->s = s;
 	}
     } else if (0 == strcasecmp(buf, "nbsp;")) {
 	pi->s = s;
@@ -866,7 +936,7 @@ read_coded_chars(PInfo pi, char *text) {
 	pi->s = s;
 	*text++ = '\'';
     } else {
-	*text++ = *pi->s;
+	*text++ = '&';
     }
     return text;
 }
@@ -884,15 +954,26 @@ collapse_special(PInfo pi, char *str) {
 	    s++;
 	    if ('#' == *s) {
 		uint64_t	u = 0;
+		char		x;
 
 		s++;
 		if ('x' == *s || 'X' == *s) {
+		    x = *s;
 		    s++;
 		    end = read_hex_uint64(s, &u);
 		} else {
+		    x = '\0';
 		    end = read_10_uint64(s, &u);
 		}
 		if (0 == end) {
+		    if (TolerantEffort == pi->options->effort) {
+			*b++ = '&';
+			*b++ = '#';
+			if ('\0' != x) {
+			    *b++ = x;
+			}
+			continue;
+		    }
 		    return EDOM;
 		}
 		if (u <= 0x000000000000007FULL) {
@@ -933,6 +1014,9 @@ collapse_special(PInfo pi, char *str) {
 		} else if (0 == strncasecmp(s, "apos;", 5)) {
 		    c = '\'';
 		    s += 5;
+		} else if (TolerantEffort == pi->options->effort) {
+		    *b++ = '&';
+		    continue;
 		} else {
 		    c = '?';
 		    while (';' != *s++) {

data/ext/ox/sax.c

@@ -42,6 +42,8 @@
 #include "ruby.h"
 #include "ox.h"
 
+#define NAME_MISMATCH	1
+
 typedef struct _SaxDrive {
     char        base_buf[0x00010000];
     char        *buf;
@@ -55,6 +57,7 @@ typedef struct _SaxDrive {
     VALUE	value_obj;
     int         (*read_func)(struct _SaxDrive *dr);
     int         convert_special;
+    int		tolerant;
     union {
         int     	fd;
         VALUE   	io;
@@ -84,7 +87,7 @@ typedef struct _SaxDrive {
 #ifdef NEEDS_STPCPY
 char *stpncpy(char *dest, const char *src, size_t n);
 #endif
-static void		sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert);
+static void		sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert, int tolerant);
 static void		sax_drive_cleanup(SaxDrive dr);
 static int		sax_drive_read(SaxDrive dr);
 static void		sax_drive_error(SaxDrive dr, const char *msg, int critical);
@@ -98,8 +101,8 @@ static int		read_element(SaxDrive dr);
 static int		read_text(SaxDrive dr);
 static const char*	read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml);
 static char		read_name_token(SaxDrive dr);
-static int		read_quoted_value(SaxDrive dr);
-static int		collapse_special(char *str);
+static int		read_quoted_value(SaxDrive dr, char *last);
+static int		collapse_special(char *str, int tolerant);
 
 static VALUE		rescue_cb(VALUE rdr, VALUE err);
 static VALUE		io_cb(VALUE rdr);
@@ -251,10 +254,10 @@ str2sym(const char *str, SaxDrive dr, char **strp) {
 
 
 void
-ox_sax_parse(VALUE handler, VALUE io, int convert) {
+ox_sax_parse(VALUE handler, VALUE io, int convert, int tolerant) {
     struct _SaxDrive    dr;
     
-    sax_drive_init(&dr, handler, io, convert);
+    sax_drive_init(&dr, handler, io, convert, tolerant);
 #if 0
     printf("*** sax_parse with these flags\n");
     printf("    has_instruct = %s\n", dr.has_instruct ? "true" : "false");
@@ -293,7 +296,7 @@ respond_to(VALUE obj, ID method) {
 }
 
 static void
-sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert) {
+sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert, int tolerant) {
     if (ox_stringio_class == rb_obj_class(io)) {
 	VALUE	s = rb_funcall2(io, ox_string_id, 0, 0);
 
@@ -344,6 +347,7 @@ sax_drive_init(SaxDrive dr, VALUE handler, VALUE io, int convert) {
     dr->value_obj = rb_data_object_alloc(sax_value_class, dr, 0, 0);
     rb_gc_register_address(&dr->value_obj);
     dr->convert_special = convert;
+    dr->tolerant = tolerant;
     dr->has_instruct = respond_to(handler, ox_instruct_id);
     dr->has_end_instruct = respond_to(handler, ox_end_instruct_id);
     dr->has_attr = respond_to(handler, ox_attr_id);
@@ -485,7 +489,7 @@ read_children(SaxDrive dr, int first) {
         if ('\0' == c || (is_white(c) && '\0' == (c = next_non_white(dr)))) {
             if (!first) {
                 sax_drive_error(dr, "invalid format, element not terminated", 1);
-                err = 1;
+                err = -1;
             }
             break; /* normal completion if first */
         }
@@ -525,12 +529,12 @@ read_children(SaxDrive dr, int first) {
                 for (i = 7; 0 < i; i--) {
                     sax_drive_get(dr);
                 }
-                if (0 == strncmp("DOCTYPE", dr->str, 7)) {
+                if ((dr->tolerant) ? 0 == strncasecmp("DOCTYPE", dr->str, 7) : 0 == strncmp("DOCTYPE", dr->str, 7)) {
                     if (element_read || !first) {
                         sax_drive_error(dr, "invalid format, DOCTYPE can not come after an element", 0);
                     }
                     err = read_doctype(dr);
-                } else if (0 == strncmp("[CDATA[", dr->str, 7)) {
+                } else if ((dr->tolerant) ? 0 == strncasecmp("[CDATA[", dr->str, 7) : 0 == strncmp("[CDATA[", dr->str, 7)) {
                     err = read_cdata(dr);
                 } else {
                     sax_drive_error(dr, "invalid format, DOCTYPE or comment expected", 1);
@@ -552,7 +556,7 @@ read_children(SaxDrive dr, int first) {
             break;
 	default:
 	    backup(dr); /* safe since no read occurred after getting last character */
-            if (first && element_read) {
+            if (first && element_read && !dr->tolerant) {
                 sax_drive_error(dr, "invalid format, multiple top level elements", 0);
             }
 	    err = read_element(dr);
@@ -632,7 +636,7 @@ read_instruction(SaxDrive dr) {
 	    VALUE   args[1];
 
 	    if (dr->convert_special) {
-		if (0 != collapse_special(content)) {
+		if (0 != collapse_special(content, dr->tolerant)) {
 		    sax_drive_error(dr, "invalid format, special character does not end with a semicolon", 0);
 		}
 	    }
@@ -836,6 +840,7 @@ read_element(SaxDrive dr) {
     int         closed;
     int		line = dr->line;
     int		col = dr->col - 1;
+    int		e;
 
     if ('\0' == (c = read_name_token(dr))) {
         return -1;
@@ -887,8 +892,28 @@ read_element(SaxDrive dr) {
             rb_funcall2(dr->handler, ox_end_element_id, 1, args);
         }
     } else {
-        if (0 != read_children(dr, 0)) {
-            return -1;
+        if (0 != (e = read_children(dr, 0))) {
+	    if (NAME_MISMATCH == e) {
+		if (0 != dr->has_end_element) {
+		    VALUE       args[1];
+
+		    if (dr->has_line) {
+			rb_ivar_set(dr->handler, ox_at_line_id, INT2FIX(line));
+		    }
+		    if (dr->has_column) {
+			rb_ivar_set(dr->handler, ox_at_column_id, INT2FIX(col - 2));
+		    }
+		    args[0] = name;
+		    rb_funcall2(dr->handler, ox_end_element_id, 1, args);
+		}
+		if (0 == strcmp(dr->str, ename)) {
+		    return 0;
+		} else {
+		    return NAME_MISMATCH;
+		}
+	    } else {
+		return -1;
+	    }
         }
 	line = dr->line;
 	col = dr->col;
@@ -902,8 +927,24 @@ read_element(SaxDrive dr) {
 		rb_ivar_set(dr->handler, ox_at_column_id, INT2FIX(col));
 	    }
 	    //printf("*** ename: %s  close: %s\n", ename, dr->str);
-            sax_drive_error(dr, "invalid format, element start and end names do not match", 1);
-	    return -1;
+	    if (dr->tolerant) {
+		if (0 != dr->has_end_element) {
+		    VALUE       args[1];
+
+		    if (dr->has_line) {
+			rb_ivar_set(dr->handler, ox_at_line_id, INT2FIX(line));
+		    }
+		    if (dr->has_column) {
+			rb_ivar_set(dr->handler, ox_at_column_id, INT2FIX(col - 2));
+		    }
+		    args[0] = name;
+		    rb_funcall2(dr->handler, ox_end_element_id, 1, args);
+		}
+		return NAME_MISMATCH; // dr->str is still the name
+	    } else {
+		sax_drive_error(dr, "invalid format, element start and end names do not match", 1);
+		return -1;
+	    }
 	}
         if (0 != dr->has_end_element) {
             VALUE       args[1];
@@ -953,7 +994,7 @@ read_text(SaxDrive dr) {
         VALUE   args[1];
 
         if (dr->convert_special) {
-            if (0 != collapse_special(dr->str) && 0 != strchr(dr->str, '&')) {
+            if (0 != collapse_special(dr->str, dr->tolerant) && 0 != strchr(dr->str, '&')) {
                 sax_drive_error(dr, "invalid format, special character does not end with a semicolon", 0);
             }
         }
@@ -986,22 +1027,24 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml) {
     int         is_encoding = 0;
     int		line;
     int		col;
+    char	last;
+    char	*attr_value;
     
     dr->str = dr->cur; /* lock it down */
     if (is_white(c)) {
         c = next_non_white(dr);
     }
     while (termc != c && term2 != c) {
-        backup(dr);
+	backup(dr);
 	line = dr->line;
 	col = dr->col;
         if ('\0' == c) {
 	    return "invalid format, attributes not terminated";
         }
         if ('\0' == (c = read_name_token(dr))) {
-            return "error reading tolen";
+            return "error reading token";
         }
-        if (is_xml && 0 == strcmp("encoding", dr->str)) {
+        if (is_xml && 0 == strcasecmp("encoding", dr->str)) {
             is_encoding = 1;
         }
 	/* TBD use symbol cache */
@@ -1011,20 +1054,28 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml) {
         if (is_white(c)) {
             c = next_non_white(dr);
         }
+	last = '\0';
         if ('=' != c) {
-            return "invalid format, no attribute value";
-        }
-        if (0 != read_quoted_value(dr)) {
-            return "error reading quoted value";
-        }
-        if (is_encoding) {
+	    if (dr->tolerant) {
+		attr_value = (char*)"";
+		last = c;
+	    } else {
+		return "invalid format, no attribute value";
+	    }
+        } else {
+	    if (0 != read_quoted_value(dr, &last)) {
+		return "error reading quoted value";
+	    }
+	    attr_value = dr->str;
+	    if (is_encoding) {
 #if HAS_ENCODING_SUPPORT
-            dr->encoding = rb_enc_find(dr->str);
+		dr->encoding = rb_enc_find(dr->str);
 #elif HAS_PRIVATE_ENCODING
-	    dr->encoding = rb_str_new2(dr->str);
+		dr->encoding = rb_str_new2(dr->str);
 #endif
-            is_encoding = 0;
-        }
+		is_encoding = 0;
+	    }
+	}
         if (dr->has_attr_value) {
             VALUE       args[2];
 
@@ -1041,10 +1092,10 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml) {
             VALUE       args[2];
 
             args[0] = name;
-            if (0 != collapse_special(dr->str) && 0 != strchr(dr->str, '&')) {
+            if (0 != collapse_special(dr->str, dr->tolerant) && 0 != strchr(dr->str, '&')) {
                 sax_drive_error(dr, "invalid format, special character does not end with a semicolon", 0);
             }
-            args[1] = rb_str_new2(dr->str);
+            args[1] = rb_str_new2(attr_value);
 #if HAS_ENCODING_SUPPORT
             if (0 != dr->encoding) {
                 rb_enc_associate(args[1], dr->encoding);
@@ -1062,7 +1113,11 @@ read_attrs(SaxDrive dr, char c, char termc, char term2, int is_xml) {
 	    }
             rb_funcall2(dr->handler, ox_attr_id, 2, args);
         }
-        c = next_non_white(dr);
+	if ('\0' != last) {
+	    c = last;
+	} else {
+	    c = next_non_white(dr);
+	}
     }
     dr->str = 0;
 
@@ -1105,7 +1160,7 @@ read_name_token(SaxDrive dr) {
 }
 
 static int
-read_quoted_value(SaxDrive dr) {
+read_quoted_value(SaxDrive dr, char *last) {
     char        c;
 
     dr->str = dr->cur;
@@ -1123,6 +1178,26 @@ read_quoted_value(SaxDrive dr) {
                 return -1;
             }
         }
+    } else if (dr->tolerant) {
+	dr->str = dr->cur - 1;
+	while ('\0' != (c = sax_drive_get(dr))) {
+	    switch (c) {
+	    case '\0':
+                sax_drive_error(dr, "invalid format, non quoted value not terminated", 1);
+	    case ' ':
+	    case '/':
+	    case '>':
+	    case '?': // for instructions
+	    case '\t':
+	    case '\n':
+	    case '\r':
+		*last = c;
+		*(dr->cur - 1) = '\0'; /* terminate value */
+		return 0;
+	    default:
+		break;
+            }
+        }
     } else {
         dr->str = dr->cur - 1;
         if ('\0' == (c = next_white(dr))) {
@@ -1240,7 +1315,7 @@ read_from_str(SaxDrive dr) {
 }
 
 static int
-collapse_special(char *str) {
+collapse_special(char *str, int tolerant) {
     char        *s = str;
     char        *b = str;
 
@@ -1248,17 +1323,27 @@ collapse_special(char *str) {
         if ('&' == *s) {
             int         c;
             char        *end;
+	    int		x = 0;
             
             s++;
             if ('#' == *s) {
                 s++;
 		if ('x' == *s || 'X' == *s) {
 		    s++;
+		    x = 1;
 		    c = (int)strtol(s, &end, 16);
 		} else {
 		    c = (int)strtol(s, &end, 10);
 		}
                 if (';' != *end) {
+		    if (tolerant) {
+			*b++ = '&';
+			*b++ = '#';
+			if (x) {
+			    *b++ = *(s - 1);
+			}
+			continue;
+		    }
                     return EDOM;
                 }
                 s = end + 1;
@@ -1277,6 +1362,9 @@ collapse_special(char *str) {
             } else if (0 == strncasecmp(s, "apos;", 5)) {
                 c = '\'';
                 s += 5;
+            } else if (tolerant) {
+		*b++ = '&';
+		continue;
             } else {
                 c = '?';
                 while (';' != *s++) {
@@ -1397,7 +1485,7 @@ sax_value_as_s(VALUE self) {
 	return Qnil;
     }
     if (dr->convert_special) {
-	if (0 != collapse_special(dr->str) && 0 != strchr(dr->str, '&')) {
+	if (0 != collapse_special(dr->str, dr->tolerant) && 0 != strchr(dr->str, '&')) {
 	    sax_drive_error(dr, "invalid format, special character does not end with a semicolon", 0);
 	}
     }

data/lib/ox/version.rb

@@ -1,5 +1,5 @@
 
 module Ox
   # Current version of the module. 
-  VERSION = '1.9.2'
+  VERSION = '1.9.3'
 end

metadata

@@ -1,19 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: ox
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.9.3
+  prerelease: 
 platform: ruby
 authors:
 - Peter Ohler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-09 00:00:00.000000000 Z
+date: 2013-03-22 00:00:00.000000000 Z
 dependencies: []
-description: "A fast XML parser and object serializer that uses only standard C lib.\n
-  \           \nOptimized XML (Ox), as the name implies was written to provide speed
-  optimized\nXML handling. It was designed to be an alternative to Nokogiri and other
-  Ruby\nXML parsers for generic XML parsing and as an alternative to Marshal for Object\nserialization. "
+description: ! "A fast XML parser and object serializer that uses only standard C
+  lib.\n            \nOptimized XML (Ox), as the name implies was written to provide
+  speed optimized\nXML handling. It was designed to be an alternative to Nokogiri
+  and other Ruby\nXML parsers for generic XML parsing and as an alternative to Marshal
+  for Object\nserialization. "
 email: peter@ohler.com
 executables: []
 extensions:
@@ -54,7 +56,6 @@ files:
 - README.md
 homepage: http://www.ohler.com/ox
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options:
 - --main
@@ -63,20 +64,22 @@ require_paths:
 - lib
 - ext
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: ox
-rubygems_version: 2.0.0
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: A fast XML parser and object serializer.
 test_files: []
 has_rdoc: true

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: be07680b4d502d3d808f0a3b7ad63e3a66685211
-  data.tar.gz: d83f1a8a18ab5f30c9aceb1d46aea0e51dc30298
-SHA512:
-  metadata.gz: 63813c9d94cb42e86b47c57b66f6d0fc5b0d6150717b85cc43b1021b4cfa1729602f2d0f7b085767ddf28a609bd8ec182a5edd3a5090819cf85da3e46d518a7b
-  data.tar.gz: 82374d21d291afbc5222b6458f7ef1fce2f5e661c28a89a0b1633c9ea93e4659fc7100809722aa3c4bacc0a58387a5edc4481bd8fde6cddac21d31281ab0a625