ox 1.2.15 → 1.3.0

data/{README.rdoc → README.md}

@@ -1,17 +1,38 @@
-= Ox: A fast XML parser and Object marshaller.
+# Ox gem
+A fast XML parser and Object marshaller as a Ruby gem.
+
+## <a name="installation">Installation</a>
+    gem install ox
+
+## <a name="source">Source</a>
 
 *GitHub* *repo*: https://github.com/ohler55/ox
 
 *RubyGems* *repo*: https://rubygems.org/gems/ox
 
-=== Description:
+## <a name="follow">Follow @oxgem on Twitter</a>
+
+[Follow @oxgem on Twitter](http://twitter.com/#!/oxgem) for announcements and news about the Ox gem.
+
+## <a name="build_status">Build Status</a>
+
+[![Build Status](http://travis-ci.org/ohler55/ox.png)](http://travis-ci.org/ohler55/ox)
+
+## <a name="release">Release Notes</a>
+
+### Release 1.3.0
+
+ - fixed Mutex dump bug
+ - added SAX parser, 30+ times faster than Nokogiri and 10+ times faster than LibXML
+
+## <a name="description">Description</a>
 
 Optimized XML (Ox), as the name implies was written to provide speed optimized
 XML handling. It was designed to be an alternative to Nokogiri in generic XML
 parsing and as an alternative to Marshal for Object serialization. 
 
-Nokogiri relies on libXml while Ox is self contained. Ox uses nothing other
-than standard C libraries so version issues with libXml are not an issue.
+Unlike Nokogiri Ox is self contained. Ox uses nothing other than standard C
+libraries so version issues with libXml are not an issue.
 
 Marshal uses a binary format for serializing Objects. That binary format
 changes with releases making Marshal dumped Object incompatible between some
@@ -25,9 +46,9 @@ It is possible to write an XML serialization gem with Nokogiri but writing
 such a package in Ruby results in a module significantly slower than
 Marshal. This is what triggered the start of Ox development.
 
-Ox handles XML documents in two ways. It is a generic XML parser and writer as
-well as a fast Object / XML marshaller. Ox was written for speed as a
-replacement for Nokogiri and for Marshal.
+Ox handles XML documents in three ways. It is a generic XML parser and writer,
+a fast Object / XML marshaller, and a stream SAX parser. Ox was written for
+speed as a replacement for Nokogiri, Ruby LibXML, and for Marshal.
 
 As an XML parser it is 2 or more times faster than Nokogiri and as a generic
 XML writer it is as much as 20 times faster than Nokogiri. Of course different
@@ -36,53 +57,60 @@ files may result in slightly different times.
 As an Object serializer Ox is up to 6 times faster than the standard Ruby
 Marshal.dump() and up to 3 times faster than Marshal.load().
 
-
-=== Object Dump Sample:
-
-  require 'ox'
-
-  class Sample
-    attr_accessor :a, :b, :c
-
-    def initialize(a, b, c)
-      @a = a
-      @b = b
-      @c = c
+The SAX like stream parser is over 30 times faster than Nokogiri and more than
+10 times faster than LibXML when using a trivial Ruby side set of
+callbacks. Unlike Nokogiri and LibXML, Ox can be tuned to use only the SAX
+callbacks that are of interest to the caller. (See the perf_sax.rb file for an
+example.)
+
+Ox is compatible with Ruby 1.8.7, 1.9.2, JRuby, and RBX.
+
+### Object Dump Sample:
+
+    require 'ox'
+  
+    class Sample
+      attr_accessor :a, :b, :c
+  
+      def initialize(a, b, c)
+        @a = a
+        @b = b
+        @c = c
+      end
     end
-  end
-
-  # Create Object
-  obj = Sample.new(1, "bee", ['x', :y, 7.0])
-  # Now dump the Object to an XML String.
-  xml = Ox.dump(obj)
-  # Convert the object back into a Sample Object.
-  obj2 = Ox.parse_obj(xml)
-
-=== Generic XML Writing and Parsing:
-
-  require 'ox'
-
-  doc = Ox::Document.new(:version => '1.0')
-
-  top = Ox::Element.new('top')
-  top[:name] = 'sample'
-  doc << top
-
-  mid = Ox::Element.new('middle')
-  mid[:name] = 'second'
-  top << mid
-
-  bot = Ox::Element.new('bottom')
-  bot[:name] = 'third'
-  mid << bot
-
-  xml = Ox.dump(doc)
-  puts xml
-  doc2 = Ox.parse(xml)
-  puts "Same? #{doc == doc2}"
-
-
-=== Object XML format
+  
+    # Create Object
+    obj = Sample.new(1, "bee", ['x', :y, 7.0])
+    # Now dump the Object to an XML String.
+    xml = Ox.dump(obj)
+    # Convert the object back into a Sample Object.
+    obj2 = Ox.parse_obj(xml)
+
+### Generic XML Writing and Parsing:
+
+    require 'ox'
+  
+    doc = Ox::Document.new(:version => '1.0')
+  
+    top = Ox::Element.new('top')
+    top[:name] = 'sample'
+    doc << top
+  
+    mid = Ox::Element.new('middle')
+    mid[:name] = 'second'
+    top << mid
+  
+    bot = Ox::Element.new('bottom')
+    bot[:name] = 'third'
+    mid << bot
+  
+    xml = Ox.dump(doc)
+    puts xml
+    doc2 = Ox.parse(xml)
+    puts "Same? #{doc == doc2}"
+
+
+### Object XML format
 
 The XML format used for Object encoding follows the structure of the
 Object. Each XML element is encoded so that the XML element name is a type
@@ -127,25 +155,25 @@ interpreter.)
 Values are encoded as the text portion of an element or in the sub-elements
 of the principle. For example, a Fixnum is encoded as:
 
-  <i>123</i>
+    <i>123</i>
 
 An Array has sub-elements and is encoded similar to this example.
 
-  <a>
-    <i>1</i>
-    <s>abc</s>
-  </a>
+    <a>
+      <i>1</i>
+      <s>abc</s>
+    </a>
 
 A Hash is encoded with an even number of elements where the first element is
 the key and the second is the value. This is repeated for each entry in the
 Hash. An example is of { 1 => 'one', 2 => 'two' } encoding is:
 
-  <h>
-    <i>1</i>
-    <s>one</s>
-    <i>2</i>
-    <s>two</s>
-  </h>
+    <h>
+      <i>1</i>
+      <s>one</s>
+      <i>2</i>
+      <s>two</s>
+    </h>
 
 Strings with characters not allowed in XML are base64 encoded amd will be
 converted back into a String when loaded.

data/ext/ox/dump.c

@@ -829,14 +829,14 @@ static int
 dump_var(ID key, VALUE value, Out out) {
     if (T_DATA == rb_type(value) && rb_cTime != rb_obj_class(value)) {
         /* There is a secret recipe that keeps Exception mesg attributes as a
-         * T_DATA until it is needed. StringValue() makes the value needed and
-         * it is converted to a regular Ruby Object. It might seem reasonable
-         * to expect that this would be done before calling the foreach
-         * callback but it isn't. A slight hack fixes the inconsistency. If
-         * the var is not something that can be represented as a String then
-         * this will fail.
+         * T_DATA until it is needed. StringValue() or a safer method of
+         * calling to_s() makes the value needed and it is converted to a
+         * regular Ruby Object. It might seem reasonable to expect that this
+         * would be done before calling the foreach callback but it isn't. A
+         * slight hack fixes the inconsistency. If the var is not something
+         * that can be represented as a String then this will fail.
          */
-        StringValue(value);
+        rb_funcall(value, to_s_id, 0);
     }
     dump_obj(key, value, out->depth, out);
 

data/ext/ox/ox.c

@@ -48,15 +48,24 @@ VALUE    Ox = Qnil;
 ID      at_id;
 ID      attributes_id;
 ID      beg_id;
+ID      cdata_id;
+ID      comment_id;
 ID      den_id;
+ID      doctype_id;
+ID      end_element_id;
 ID      end_id;
+ID      error_id;
 ID      excl_id;
 ID      inspect_id;
+ID      instruct_id;
 ID      keys_id;
 ID      local_id;
 ID      nodes_id;
 ID      num_id;
 ID      parse_id;
+ID      readpartial_id;
+ID      start_element_id;
+ID      text_id;
 ID      to_c_id;
 ID      to_s_id;
 ID      tv_sec_id;
@@ -140,7 +149,7 @@ static VALUE
 get_def_opts(VALUE self) {
     VALUE       opts = rb_hash_new();
     int         elen = (int)strlen(default_options.encoding);
-    
+
     rb_hash_aset(opts, encoding_sym, (0 == elen) ? Qnil : rb_str_new(default_options.encoding, elen));
     rb_hash_aset(opts, indent_sym, INT2FIX(default_options.indent));
     rb_hash_aset(opts, trace_sym, INT2FIX(default_options.trace));
@@ -434,6 +443,20 @@ load_file(int argc, VALUE *argv, VALUE self) {
     return load(xml, argc - 1, argv + 1, self);
 }
 
+/* call-seq: sax_parse(handler, io)
+ *
+ * Parses an IO stream or file containing an XML document. Raises an exception
+ * if the XML is malformed or the classes specified are not valid.
+ * @param [Ox::Sax] handler SAX (responds to OX::Sax methods) like handler
+ * @param [IO|String] io IO Object to read from
+ */
+static VALUE
+sax_parse(VALUE self, VALUE handler, VALUE io) {
+    ox_sax_parse(handler, io);
+
+    return Qnil;
+}
+
 static void
 parse_dump_options(VALUE ropts, Options copts) {
     struct _YesNoOpt    ynos[] = {
@@ -583,6 +606,7 @@ void Init_ox() {
     rb_define_module_function(Ox, "parse_obj", to_obj, 1);
     rb_define_module_function(Ox, "parse", to_gen, 1);
     rb_define_module_function(Ox, "load", load_str, -1);
+    rb_define_module_function(Ox, "sax_parse", sax_parse, 2);
 
     rb_define_module_function(Ox, "to_xml", dump, -1);
     rb_define_module_function(Ox, "dump", dump, -1);
@@ -591,23 +615,32 @@ void Init_ox() {
     rb_define_module_function(Ox, "to_file", to_file, -1);
 
     rb_require("time");
-    parse_id = rb_intern("parse");
-    local_id = rb_intern("local");
     at_id = rb_intern("at");
-    inspect_id = rb_intern("inspect");
+    attributes_id = rb_intern("@attributes");
     beg_id = rb_intern("@beg");
-    end_id = rb_intern("@end");
+    cdata_id = rb_intern("cdata");
+    comment_id = rb_intern("comment");
     den_id = rb_intern("@den");
+    doctype_id = rb_intern("doctype");
+    end_element_id = rb_intern("end_element");
+    end_id = rb_intern("@end");
+    error_id = rb_intern("error");
     excl_id = rb_intern("@excl");
-    value_id = rb_intern("@value");
+    inspect_id = rb_intern("inspect");
+    instruct_id = rb_intern("instruct");
+    keys_id = rb_intern("keys");
+    local_id = rb_intern("local");
     nodes_id = rb_intern("@nodes");
     num_id = rb_intern("@num");
-    attributes_id = rb_intern("@attributes");
-    keys_id = rb_intern("keys");
-    tv_sec_id = rb_intern("tv_sec");
-    tv_usec_id = rb_intern("tv_usec");
+    parse_id = rb_intern("parse");
+    readpartial_id = rb_intern("readpartial");
+    start_element_id = rb_intern("start_element");
+    text_id = rb_intern("text");
     to_c_id = rb_intern("to_c");
     to_s_id = rb_intern("to_s");
+    tv_sec_id = rb_intern("tv_sec");
+    tv_usec_id = rb_intern("tv_usec");
+    value_id = rb_intern("@value");
 
     time_class = rb_const_get(rb_cObject, rb_intern("Time"));
     struct_class = rb_const_get(rb_cObject, rb_intern("Struct"));

data/ext/ox/ox.h

@@ -198,6 +198,8 @@ typedef struct _Options {
 extern VALUE    parse(char *xml, ParseCallbacks pcb, char **endp, int trace, Effort effort);
 extern void     _raise_error(const char *msg, const char *xml, const char *current, const char* file, int line);
 
+extern void     ox_sax_parse(VALUE handler, VALUE io);
+
 extern char*    write_obj_to_str(VALUE obj, Options copts);
 extern void     write_obj_to_file(VALUE obj, const char *path, Options copts);
 
@@ -206,15 +208,24 @@ extern VALUE    Ox;
 extern ID       at_id;
 extern ID       attributes_id;
 extern ID       beg_id;
+extern ID       cdata_id;
+extern ID       comment_id;
 extern ID       den_id;
+extern ID       doctype_id;
+extern ID       end_element_id;
 extern ID       end_id;
+extern ID       error_id;
 extern ID       excl_id;
 extern ID       inspect_id;
+extern ID       instruct_id;
 extern ID       keys_id;
 extern ID       local_id;
 extern ID       nodes_id;
 extern ID       num_id;
 extern ID       parse_id;
+extern ID       readpartial_id;
+extern ID       start_element_id;
+extern ID       text_id;
 extern ID       to_c_id;
 extern ID       to_s_id;
 extern ID       tv_sec_id;

data/ext/ox/sax.c

@@ -0,0 +1,758 @@
+/* sax.c
+ * Copyright (c) 2011, Peter Ohler
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  - Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ *  - Neither the name of Peter Ohler nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+#include "ruby.h"
+#include "ox.h"
+
+typedef struct _SaxDrive {
+    char        base_buf[0x00010000];
+    //char        base_buf[0x00000010];
+    char        *buf;
+    char        *buf_end;
+    char        *cur;
+    char        *read_end;      // one past last character read
+    char        *str;           // start of current string being read
+    int         line;
+    int         col;
+    VALUE       handler;
+    int         (*read_func)(struct _SaxDrive *dr);
+    union {
+        int     fd;
+        VALUE   io;
+    };
+    int         has_instruct;
+    int         has_doctype;
+    int         has_comment;
+    int         has_cdata;
+    int         has_text;
+    int         has_start_element;
+    int         has_end_element;
+    int         has_error;
+    rb_encoding *encoding;
+} *SaxDrive;
+
+static void     sax_drive_init(SaxDrive dr, VALUE handler, VALUE io);
+static void     sax_drive_cleanup(SaxDrive dr);
+static int      sax_drive_read(SaxDrive dr);
+static void     sax_drive_error(SaxDrive dr, const char *msg, int critical);
+
+static int      read_children(SaxDrive dr, int first);
+static int      read_instruction(SaxDrive dr);
+static int      read_doctype(SaxDrive dr);
+static int      read_cdata(SaxDrive dr);
+static int      read_comment(SaxDrive dr);
+static int      read_element(SaxDrive dr);
+static int      read_text(SaxDrive dr);
+static int      read_attrs(SaxDrive dr, VALUE *attrs, char c, char termc, char term2);
+static char     read_name_token(SaxDrive dr);
+static int      read_quoted_value(SaxDrive dr);
+
+static VALUE    io_cb(VALUE rdr);
+static int      read_from_io(SaxDrive dr);
+static int      read_from_fd(SaxDrive dr);
+
+static inline char
+sax_drive_get(SaxDrive dr) {
+    if (dr->read_end <= dr->cur) {
+        if (0 != sax_drive_read(dr)) {
+            return 0;
+        }
+    }
+    if ('\n' == *dr->cur) {
+        dr->line++;
+        dr->col = 0;
+    }
+    dr->col++;
+    
+    return *dr->cur++;
+}
+
+/* Starts by reading a character so it is safe to use with an empty or
+ * compacted buffer.
+ */
+inline static char
+next_non_white(SaxDrive dr) {
+    char        c;
+
+    while ('\0' != (c = sax_drive_get(dr))) {
+	switch(c) {
+	case ' ':
+	case '\t':
+	case '\f':
+	case '\n':
+	case '\r':
+	    break;
+	default:
+	    return c;
+	}
+    }
+    return '\0';
+}
+
+inline static int
+is_white(char c) {
+    switch(c) {
+    case ' ':
+    case '\t':
+    case '\f':
+    case '\n':
+    case '\r':
+        return 1;
+    default:
+        break;
+    }
+    return 0;
+}
+
+inline static VALUE
+str2sym(const char *str) {
+    VALUE       *slot;
+    VALUE       sym;
+
+    if (Qundef == (sym = ox_cache_get(symbol_cache, str, &slot))) {
+        sym = ID2SYM(rb_intern(str));
+        *slot = sym;
+    }
+    return sym;
+}
+
+
+void
+ox_sax_parse(VALUE handler, VALUE io) {
+    struct _SaxDrive    dr;
+    
+    sax_drive_init(&dr, handler, io);
+#if 0
+    printf("*** sax_parse with these flags\n");
+    printf("    has_instruct = %s\n", dr.has_instruct ? "true" : "false");
+    printf("    has_doctype = %s\n", dr.has_doctype ? "true" : "false");
+    printf("    has_comment = %s\n", dr.has_comment ? "true" : "false");
+    printf("    has_cdata = %s\n", dr.has_cdata ? "true" : "false");
+    printf("    has_text = %s\n", dr.has_text ? "true" : "false");
+    printf("    has_start_element = %s\n", dr.has_start_element ? "true" : "false");
+    printf("    has_end_element = %s\n", dr.has_end_element ? "true" : "false");
+    printf("    has_error = %s\n", dr.has_error ? "true" : "false");
+#endif
+    read_children(&dr, 1);
+    sax_drive_cleanup(&dr);
+}
+
+static void
+sax_drive_init(SaxDrive dr, VALUE handler, VALUE io) {
+    if (rb_respond_to(io, readpartial_id)) {
+        VALUE   rfd;
+
+        if (rb_respond_to(io, rb_intern("fileno")) && Qnil != (rfd = rb_funcall(io, rb_intern("fileno"), 0))) {
+            dr->read_func = read_from_fd;
+            dr->fd = FIX2INT(rfd);
+        } else {
+            dr->read_func = read_from_io;
+            dr->io = io;
+        }
+    } else {
+        rb_raise(rb_eArgError, "sax_parser io argument must respond to readpartial().\n");
+    }
+    dr->buf = dr->base_buf;
+    *dr->buf = '\0';
+    dr->buf_end = dr->buf + sizeof(dr->base_buf) - 1; // 1 less to make debugging easier
+    dr->cur = dr->buf;
+    dr->read_end = dr->buf;
+    dr->str = 0;
+    dr->line = 1;
+    dr->col = 0;
+    dr->handler = handler;
+    dr->has_instruct = rb_respond_to(handler, instruct_id);
+    dr->has_doctype = rb_respond_to(handler, doctype_id);
+    dr->has_comment = rb_respond_to(handler, comment_id);
+    dr->has_cdata = rb_respond_to(handler, cdata_id);
+    dr->has_text = rb_respond_to(handler, text_id);
+    dr->has_start_element = rb_respond_to(handler, start_element_id);
+    dr->has_end_element = rb_respond_to(handler, end_element_id);
+    dr->has_error = rb_respond_to(handler, error_id);
+    dr->encoding = 0;
+}
+
+static void
+sax_drive_cleanup(SaxDrive dr) {
+    if (dr->base_buf != dr->buf) {
+        free(dr->buf);
+    }
+}
+
+static int
+sax_drive_read(SaxDrive dr) {
+    int         err;
+    size_t      shift = 0;
+    
+    if (dr->buf < dr->cur) {
+        if (0 == dr->str) {
+            shift = dr->cur - dr->buf;
+        } else {
+            shift = dr->str - dr->buf;
+        }
+        //printf("\n*** shift: %lu\n", shift);
+        if (0 == shift) { // no space left so allocate more
+            char        *old = dr->buf;
+            size_t      size = dr->buf_end - dr->buf;
+        
+            if (dr->buf == dr->base_buf) {
+                if (0 == (dr->buf = (char*)malloc(size * 2))) {
+                    rb_raise(rb_eNoMemError, "Could not allocate memory for large element.\n");
+                }
+                memcpy(dr->buf, old, size);
+            } else {
+                if (0 == (dr->buf = (char*)realloc(dr->buf, size * 2))) {
+                    rb_raise(rb_eNoMemError, "Could not allocate memory for large element.\n");
+                }
+            }
+            dr->buf_end = dr->buf + size * 2;
+            dr->cur = dr->buf + (dr->cur - old);
+            dr->read_end = dr->buf + (dr->read_end - old);
+            if (0 != dr->str) {
+                dr->str = dr->buf + (dr->str - old);
+            }
+        } else {
+            memmove(dr->buf, dr->buf + shift, dr->read_end - (dr->buf + shift));
+            dr->cur -= shift;
+            dr->read_end -= shift;
+            if (0 != dr->str) {
+                dr->str -= shift;
+            }
+        }
+    }
+    err = dr->read_func(dr);
+    *dr->read_end = '\0';
+
+    return err;
+}
+
+static void
+sax_drive_error(SaxDrive dr, const char *msg, int critical) {
+    if (dr->has_error) {
+        VALUE   args[3];
+
+        args[0] = rb_str_new2(msg);
+        args[1] = INT2FIX(dr->line);
+        args[2] = INT2FIX(dr->col);
+        rb_funcall2(dr->handler, error_id, 3, args);
+    } else if (critical) {
+        sax_drive_cleanup(dr);
+        rb_raise(rb_eSyntaxError, "%s at line %d, column %d\n", msg, dr->line, dr->col);
+    }
+}
+
+static int
+read_children(SaxDrive dr, int first) {
+    int         err = 0;
+    int         element_read = !first;
+    int         doctype_read = !first;
+    char        c;
+    
+    while (!err) {
+        dr->str = dr->cur; // protect the start
+        if ('\0' == (c = next_non_white(dr))) {
+            if (!first) {
+                sax_drive_error(dr, "invalid format, element not terminated", 1);
+                err = 1;
+            }
+            break; // normal completion if first
+        }
+	if ('<' != c) {
+            if (first) { // all top level entities start with <
+                sax_drive_error(dr, "invalid format, expected <", 1);
+                break; // unrecoverable
+            }
+            if (0 != (err = read_text(dr))) { // finished when < is reached
+                break;
+            }
+	}
+        dr->str = dr->cur; // protect the start for elements
+        c = sax_drive_get(dr);
+	switch (c) {
+	case '?': // instructions (xml or otherwise)
+            if (!first || element_read || doctype_read) {
+                sax_drive_error(dr, "invalid format, instruction must come before elements", 0);
+            }
+	    err = read_instruction(dr);
+	    break;
+	case '!': // comment or doctype
+            dr->str = dr->cur;
+            c = sax_drive_get(dr);
+	    if ('\0' == c) {
+                sax_drive_error(dr, "invalid format, DOCTYPE or comment not terminated", 1);
+                err = 1;
+	    } else if ('-' == c) {
+                c = sax_drive_get(dr); // skip first - and get next character
+		if ('-' != c) {
+                    sax_drive_error(dr, "invalid format, bad comment format", 1);
+                    err = 1;
+		} else {
+                    c = sax_drive_get(dr); // skip second -
+		    err = read_comment(dr);
+		}
+            } else {
+                int     i;
+
+                for (i = 7; 0 < i; i--) {
+                    sax_drive_get(dr);
+                }
+                if (0 == strncmp("DOCTYPE", dr->str, 7)) {
+                    if (element_read || !first) {
+                        sax_drive_error(dr, "invalid format, DOCTYPE can not come after an element", 0);
+                    }
+                    doctype_read = 1;
+                    err = read_doctype(dr);
+                } else if (0 == strncmp("[CDATA[", dr->str, 7)) {
+                    err = read_cdata(dr);
+                } else {
+                    sax_drive_error(dr, "invalid format, DOCTYPE or comment expected", 1);
+                    err = 1;
+                }
+	    }
+	    break;
+	case '/': // element end
+            return ('\0' == read_name_token(dr));
+            break;
+	case '\0':
+            sax_drive_error(dr, "invalid format, document not terminated", 1);
+            err = 1;
+            break;
+	default:
+            dr->cur--; // safe since no read occurred after getting last character
+            if (first && element_read) {
+                sax_drive_error(dr, "invalid format, multiple top level elements", 0);
+            }
+	    err = read_element(dr);
+            element_read = 1;
+	    break;
+	}
+    }
+    return err;
+}
+
+/* Entered after the "<?" sequence. Ready to read the rest.
+ */
+static int
+read_instruction(SaxDrive dr) {
+    VALUE       target = Qnil;
+    VALUE       attrs = Qnil;
+    char        c;
+
+    if ('\0' == (c = read_name_token(dr))) {
+        return -1;
+    }
+    if (dr->has_instruct) {
+        target = rb_str_new2(dr->str);
+    }
+    if (0 != read_attrs(dr, &attrs, c, '?', '?')) {
+        return -1;
+    }
+    c = next_non_white(dr);
+    if ('>' != c) {
+        sax_drive_error(dr, "invalid format, instruction not terminated", 1);
+        return -1;
+    }
+    if (0 != dr->has_instruct) {
+        VALUE       args[2];
+
+        args[0] = target;
+        args[1] = attrs;
+        rb_funcall2(dr->handler, instruct_id, 2, args);
+    }
+    dr->str = 0;
+
+    return 0;
+}
+
+/* Entered after the "<!DOCTYPE" sequence. Ready to read the rest.
+ */
+static int
+read_doctype(SaxDrive dr) {
+    char        c;
+
+    dr->str = dr->cur - 1; // mark the start
+    while ('>' != (c = sax_drive_get(dr))) {
+        if ('\0' == c) {
+            sax_drive_error(dr, "invalid format, doctype terminated unexpectedly", 1);
+            return -1;
+        }
+    }
+    *(dr->cur - 1) = '\0';
+    if (dr->has_doctype) {
+        VALUE       args[1];
+
+        args[0] = rb_str_new2(dr->str);
+        rb_funcall2(dr->handler, doctype_id, 1, args);
+    }
+    dr->str = 0;
+
+    return 0;
+}
+
+/* Entered after the "<![CDATA[" sequence. Ready to read the rest.
+ */
+static int
+read_cdata(SaxDrive dr) {
+    char        c;
+    int         end = 0;
+
+    dr->str = dr->cur - 1; // mark the start
+    while (1) {
+        c = sax_drive_get(dr);
+        if (']' == c) {
+            end++;
+        } else if ('>' == c) {
+            if (2 <= end) {
+                *(dr->cur - 3) = '\0';
+                break;
+            }
+            end = 0;
+        } else if ('\0' == c) {
+            sax_drive_error(dr, "invalid format, cdata terminated unexpectedly", 1);
+            return -1;
+        } else {
+            end = 0;
+        }
+    }
+    if (dr->has_cdata) {
+        VALUE       args[1];
+
+        args[0] = rb_str_new2(dr->str);
+#ifdef HAVE_RUBY_ENCODING_H
+        if (0 != dr->encoding) {
+            rb_enc_associate(args[0], dr->encoding);
+        }
+#endif
+        rb_funcall2(dr->handler, cdata_id, 1, args);
+    }
+    dr->str = 0;
+
+    return 0;
+}
+
+/* Entered after the "<!--" sequence. Ready to read the rest.
+ */
+static int
+read_comment(SaxDrive dr) {
+    char        c;
+    int         end = 0;
+
+    dr->str = dr->cur - 1; // mark the start
+    while (1) {
+        c = sax_drive_get(dr);
+        if ('-' == c) {
+            if (end) {
+                *(dr->cur - 2) = '\0';
+                break;
+            } else {
+                end = 1;
+            }
+        } else if ('\0' == c) {
+            sax_drive_error(dr, "invalid format, comment terminated unexpectedly", 1);
+            return -1;
+        } else {
+            end = 0;
+        }
+    }
+    c = sax_drive_get(dr);
+    if ('>' != c) {
+        sax_drive_error(dr, "invalid format, comment terminated unexpectedly", 1);
+    }
+    if (dr->has_comment) {
+        VALUE       args[1];
+
+        args[0] = rb_str_new2(dr->str);
+#ifdef HAVE_RUBY_ENCODING_H
+        if (0 != dr->encoding) {
+            rb_enc_associate(args[0], dr->encoding);
+        }
+#endif
+        rb_funcall2(dr->handler, comment_id, 1, args);
+    }
+    dr->str = 0;
+
+    return 0;
+}
+
+/* Entered after the '<' and the first character after that. Returns status
+ * code.
+ */
+static int
+read_element(SaxDrive dr) {
+    VALUE       name = Qnil;
+    VALUE       attrs = Qnil;
+    char        c;
+    int         closed;
+
+    if ('\0' == (c = read_name_token(dr))) {
+        return -1;
+    }
+    name = str2sym(dr->str);
+    if ('/' == c) {
+        closed = 1;
+    } else if ('>' == c) {
+        closed = 0;
+    } else {
+        if (0 != read_attrs(dr, &attrs, c, '/', '>')) {
+            return -1;
+        }
+        closed = ('/' == *(dr->cur - 1));
+    }
+    if (closed) {
+        c = next_non_white(dr);
+        if ('>' != c) {
+            sax_drive_error(dr, "invalid format, element not closed", 1);
+            return -1;
+        }
+    }
+    if (0 != dr->has_start_element) {
+        VALUE       args[2];
+
+        args[0] = name;
+        args[1] = attrs;
+        rb_funcall2(dr->handler, start_element_id, 2, args);
+        if (closed && dr->has_end_element) {
+            rb_funcall2(dr->handler, end_element_id, 1, args);
+        }
+    }
+    if (!closed) {
+        if (0 != read_children(dr, 0)) {
+            return -1;
+        }
+        if (0 != strcmp(dr->str, rb_id2name(SYM2ID(name)))) {
+            sax_drive_error(dr, "invalid format, element start and end names do not match", 1);
+            return -1;
+        }
+        if (0 != dr->has_end_element) {
+            VALUE       args[1];
+
+            args[0] = name;
+            rb_funcall2(dr->handler, end_element_id, 1, args);
+        }
+    }
+    dr->str = 0;
+
+    return 0;
+}
+
+static int
+read_text(SaxDrive dr) {
+    char        c;
+
+    dr->str = dr->cur - 1; // mark the start
+    while ('<' != (c = sax_drive_get(dr))) {
+        if ('\0' == c) {
+            sax_drive_error(dr, "invalid format, text terminated unexpectedly", 1);
+            return -1;
+        }
+    }
+    *(dr->cur - 1) = '\0';
+    if (dr->has_text) {
+        VALUE       args[1];
+
+        args[0] = rb_str_new2(dr->str);
+#ifdef HAVE_RUBY_ENCODING_H
+        if (0 != dr->encoding) {
+            rb_enc_associate(args[0], dr->encoding);
+        }
+#endif
+        rb_funcall2(dr->handler, text_id, 1, args);
+    }
+    return 0;
+}
+
+static int
+read_attrs(SaxDrive dr, VALUE *attrs, char c, char termc, char term2) {
+    VALUE       name = Qnil;
+    int         is_encoding = 0;
+    
+    dr->str = dr->cur; // lock it down
+    if (is_white(c)) {
+        c = next_non_white(dr);
+    }
+    while (termc != c && term2 != c) {
+        dr->cur--;
+        if ('\0' == c) {
+            sax_drive_error(dr, "invalid format, processing instruction not terminated", 1);
+            return -1;
+        }
+        if ('\0' == (c = read_name_token(dr))) {
+            return -1;
+        }
+        if ('?' == termc && 0 == strcmp("encoding", dr->str)) {
+            is_encoding = 1;
+        }
+        if (dr->has_instruct) {
+            name = str2sym(dr->str);
+        }
+        if (is_white(c)) {
+            c = next_non_white(dr);
+        }
+        if ('=' != c) {
+            sax_drive_error(dr, "invalid format, no attribute value", 1);
+            return -1;
+        }
+        if (0 != read_quoted_value(dr)) {
+            return -1;
+        }
+#ifdef HAVE_RUBY_ENCODING_H
+        if (is_encoding) {
+            dr->encoding = rb_enc_find(dr->str);
+        }
+#endif
+        if (dr->has_instruct) {
+            VALUE       rstr = rb_str_new2(dr->str);
+            
+            if (Qnil == *attrs) {
+                *attrs = rb_hash_new();
+            }
+#ifdef HAVE_RUBY_ENCODING_H
+            if (0 != dr->encoding) {
+                rb_enc_associate(rstr, dr->encoding);
+            }
+#endif
+            rb_hash_aset(*attrs, name, rstr);
+        }
+        c = next_non_white(dr);
+    }
+    return 0;
+}
+
+static char
+read_name_token(SaxDrive dr) {
+    char        c;
+
+    dr->str = dr->cur; // make sure the start doesn't get compacted out
+    c = sax_drive_get(dr);
+    if (is_white(c)) {
+        c = next_non_white(dr);
+        dr->str = dr->cur - 1;
+    }
+    while (1) {
+	switch (c) {
+	case ' ':
+	case '\t':
+	case '\f':
+	case '?':
+	case '=':
+	case '/':
+	case '>':
+	case '\n':
+	case '\r':
+            *(dr->cur - 1) = '\0';
+	    return c;
+	case '\0':
+            // documents never terminate after a name token
+            sax_drive_error(dr, "invalid format, document not terminated", 1);
+            return '\0';
+	default:
+	    break;
+	}
+        c = sax_drive_get(dr);
+    }
+    return '\0';
+}
+
+static int
+read_quoted_value(SaxDrive dr) {
+    char        c;
+
+    dr->str = dr->cur;
+    c = sax_drive_get(dr);
+    if (is_white(c)) {
+        c = next_non_white(dr);
+    }
+    if ('"' != c) {
+        sax_drive_error(dr, "invalid format, attibute value not in quotes", 1);
+        return -1;
+    }
+    dr->str = dr->cur;
+    while ('"' != (c = sax_drive_get(dr))) {
+        if ('\0' == c) {
+            sax_drive_error(dr, "invalid format, quoted value not terminated", 1);
+            return -1;
+        }
+    }
+    *(dr->cur - 1) = '\0'; // terminate value
+
+    return 0;
+}
+
+static int
+read_from_io(SaxDrive dr) {
+    int ex = 0;
+
+    rb_protect(io_cb, (VALUE)dr, &ex);
+    // printf("*** io_cb exception = %d\n", ex);
+    // An error code of 6 is always returned not matter what kind of Exception is raised.
+    return ex;
+}
+
+static VALUE
+io_cb(VALUE rdr) {
+    SaxDrive    dr = (SaxDrive)rdr;
+    VALUE       args[1];
+    VALUE       rstr;
+    char        *str;
+    size_t      cnt;
+
+    args[0] = SIZET2NUM(dr->buf_end - dr->cur);
+    rstr = rb_funcall2(dr->io, readpartial_id, 1, args);
+    str = StringValuePtr(rstr);
+    cnt = strlen(str);
+    //printf("*** read %lu bytes, str: '%s'\n", cnt, str);
+    strcpy(dr->cur, str);
+    dr->read_end = dr->cur + cnt;
+
+    return Qnil;
+}
+
+static int
+read_from_fd(SaxDrive dr) {
+    ssize_t     cnt;
+    size_t      max = dr->buf_end - dr->cur;
+
+    cnt = read(dr->fd, dr->cur, max);
+    if (cnt < 0) {
+        sax_drive_error(dr, "failed to read from file", 1);
+        return -1;
+    } else if (0 != cnt) {
+        dr->read_end = dr->cur + cnt;
+    }
+    return 0;
+}