nokogiri 1.5.0-java → 1.5.1-java

data/ext/java/nokogiri/internals/XmlDomParser.java

@@ -35,8 +35,14 @@ package nokogiri.internals;
 import java.io.IOException;
 
 import nokogiri.XmlDocument;
+import nokogiri.internals.ParserContext.Options;
 
 import org.apache.xerces.parsers.DOMParser;
+import org.apache.xerces.parsers.XIncludeParserConfiguration;
+import org.apache.xerces.xni.XMLResourceIdentifier;
+import org.apache.xerces.xni.XNIException;
+import org.apache.xerces.xni.parser.XMLEntityResolver;
+import org.apache.xerces.xni.parser.XMLInputSource;
 import org.apache.xerces.xni.parser.XMLParserConfiguration;
 import org.cyberneko.dtd.DTDConfiguration;
 import org.w3c.dom.Document;
@@ -51,14 +57,21 @@ import org.xml.sax.SAXException;
  */
 public class XmlDomParser extends DOMParser {
     DOMParser dtd;
+    ParserContext.Options options;
 
-    public XmlDomParser() {
+    public XmlDomParser(ParserContext.Options options) {
         super();
+        this.options = options;
 
         DTDConfiguration dtdConfig = new DTDConfiguration();
         dtd = new DOMParser(dtdConfig);
 
-        XMLParserConfiguration config = getXMLParserConfiguration();
+        XMLParserConfiguration config;
+        if (options.xInclude) {
+            config = new XIncludeParserConfiguration();
+        } else {
+            config = getXMLParserConfiguration();
+        }
         config.setDTDHandler(dtdConfig);
         config.setDTDContentModelHandler(dtdConfig);
     }
@@ -66,6 +79,9 @@ public class XmlDomParser extends DOMParser {
     @Override
     public void parse(InputSource source) throws SAXException, IOException {
         dtd.reset();
+        if (options.xInclude) {
+            super.setEntityResolver(new NokogiriXInlcudeEntityResolver(source));
+        }
         super.parse(source);
         Document doc = getDocument();
         if (doc == null)
@@ -73,4 +89,19 @@ public class XmlDomParser extends DOMParser {
 
         doc.setUserData(XmlDocument.DTD_RAW_DOCUMENT, dtd.getDocument(), null);
     }
+    
+    private class NokogiriXInlcudeEntityResolver implements org.xml.sax.EntityResolver {
+        InputSource source;
+        private NokogiriXInlcudeEntityResolver(InputSource source) {
+            this.source = source;
+        }
+
+        @Override
+        public InputSource resolveEntity(String publicId, String systemId)
+                throws SAXException, IOException {
+            if (systemId != null) source.setSystemId(systemId);
+            if (publicId != null) source.setPublicId(publicId);
+            return source;
+        }
+    }
 }

data/ext/java/nokogiri/internals/XmlDomParserContext.java

@@ -34,7 +34,6 @@ package nokogiri.internals;
 
 import static nokogiri.internals.NokogiriHelpers.getNokogiriClass;
 import static nokogiri.internals.NokogiriHelpers.rubyStringToString;
-
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -73,6 +72,8 @@ public class XmlDomParserContext extends ParserContext {
     protected static final String FEATURE_INCLUDE_IGNORABLE_WHITESPACE =
         "http://apache.org/xml/features/dom/include-ignorable-whitespace";
     protected static final String FEATURE_VALIDATION = "http://xml.org/sax/features/validation";
+    private static final String XINCLUDE_FEATURE_ID = "http://apache.org/xml/features/xinclude";
+    private static final String SECURITY_MANAGER = "http://apache.org/xml/properties/security-manager";
 
     protected ParserContext.Options options;
     protected DOMParser parser;
@@ -102,9 +103,17 @@ public class XmlDomParserContext extends ParserContext {
     }
 
     protected void initParser(Ruby runtime) {
-        parser = new XmlDomParser();
+        if (options.xInclude) {
+            System.setProperty("org.apache.xerces.xni.parser.XMLParserConfiguration",
+                    "org.apache.xerces.parsers.XIncludeParserConfiguration");
+        }
+        
+        parser = new XmlDomParser(options);
         parser.setErrorHandler(errorHandler);
 
+        // Fix for Issue#586.  This limits entity expansion up to 100000 and nodes up to 3000.
+        setProperty(SECURITY_MANAGER, new org.apache.xerces.util.SecurityManager());        
+        
         if (options.noBlanks) {
             setFeature(FEATURE_INCLUDE_IGNORABLE_WHITESPACE, false);
         }
@@ -160,20 +169,30 @@ public class XmlDomParserContext extends ParserContext {
         doc.setInstanceVariable("@errors", errors);
     }
 
-    public XmlDocument getDocumentWithErrorsOrRaiseException(ThreadContext context, Exception ex) {
+    public XmlDocument getDocumentWithErrorsOrRaiseException(ThreadContext context, RubyClass klazz, Exception ex) {
         if (options.recover) {
-            XmlDocument doc = this.getNewEmptyDocument(context);
-            this.addErrorsIfNecessary(context, doc);
+            XmlDocument xmlDocument = getInterruptedOrNewXmlDocument(context, klazz);
+            this.addErrorsIfNecessary(context, xmlDocument);
             XmlSyntaxError xmlSyntaxError = (XmlSyntaxError) NokogiriService.XML_SYNTAXERROR_ALLOCATOR.allocate(context.getRuntime(), getNokogiriClass(context.getRuntime(), "Nokogiri::XML::SyntaxError"));
             xmlSyntaxError.setException(ex);
-            ((RubyArray) doc.getInstanceVariable("@errors")).append(xmlSyntaxError);
-            return doc;
+            ((RubyArray) xmlDocument.getInstanceVariable("@errors")).append(xmlSyntaxError);
+            return xmlDocument;
         } else {
             XmlSyntaxError xmlSyntaxError = (XmlSyntaxError) NokogiriService.XML_SYNTAXERROR_ALLOCATOR.allocate(context.getRuntime(), getNokogiriClass(context.getRuntime(), "Nokogiri::XML::SyntaxError"));
             xmlSyntaxError.setException(ex);
             throw new RaiseException(xmlSyntaxError);
         }
     }
+    
+    private XmlDocument getInterruptedOrNewXmlDocument(ThreadContext context, RubyClass klazz) {
+        Document document = parser.getDocument();
+        XmlDocument xmlDocument = (XmlDocument) NokogiriService.XML_DOCUMENT_ALLOCATOR.allocate(context.getRuntime(), klazz);
+        if (document != null) {
+            xmlDocument.setDocumentNode(context, document);
+        }
+        xmlDocument.setEncoding(ruby_encoding);
+        return xmlDocument;
+    }
 
     protected XmlDocument getNewEmptyDocument(ThreadContext context) {
         IRubyObject[] args = new IRubyObject[0];
@@ -188,7 +207,7 @@ public class XmlDomParserContext extends ParserContext {
                                        RubyClass klazz,
                                        Document doc) {
         XmlDocument xmlDocument = (XmlDocument) NokogiriService.XML_DOCUMENT_ALLOCATOR.allocate(context.getRuntime(), klazz);
-        xmlDocument.setNode(context, doc);
+        xmlDocument.setDocumentNode(context, doc);
         xmlDocument.setEncoding(ruby_encoding);
 
         if (options.dtdLoad) {
@@ -202,18 +221,19 @@ public class XmlDomParserContext extends ParserContext {
      * Must call setInputSource() before this method.
      */
     public XmlDocument parse(ThreadContext context,
-                             IRubyObject klass,
+                             IRubyObject klazz,
                              IRubyObject url) {
+        XmlDocument xmlDoc;
         try {
             Document doc = do_parse();
-            XmlDocument xmlDoc = wrapDocument(context, (RubyClass)klass, doc);
+            xmlDoc = wrapDocument(context, (RubyClass)klazz, doc);
             xmlDoc.setUrl(url);
             addErrorsIfNecessary(context, xmlDoc);
             return xmlDoc;
         } catch (SAXException e) {
-            return getDocumentWithErrorsOrRaiseException(context, e);
+            return getDocumentWithErrorsOrRaiseException(context, (RubyClass)klazz, e);
         } catch (IOException e) {
-            return getDocumentWithErrorsOrRaiseException(context, e);
+            return getDocumentWithErrorsOrRaiseException(context, (RubyClass)klazz, e);
         }
     }
 

data/ext/nokogiri/extconf.rb

@@ -17,7 +17,7 @@ end
 $CFLAGS << " #{ENV["CFLAGS"]}"
 $LIBS << " #{ENV["LIBS"]}"
 
-if RbConfig::CONFIG['target_os'] == 'mingw32' || RbConfig::CONFIG['target_os'] =~ /mswin32/
+if RbConfig::CONFIG['target_os'] == 'mingw32' || RbConfig::CONFIG['target_os'] =~ /mswin/
   $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF"
 elsif RbConfig::CONFIG['target_os'] =~ /solaris/
   $CFLAGS << " -DUSE_INCLUDED_VASPRINTF"
@@ -30,10 +30,11 @@ if RbConfig::MAKEFILE_CONFIG['CC'] =~ /mingw/
 end
 
 if RbConfig::MAKEFILE_CONFIG['CC'] =~ /gcc/
-  $CFLAGS << " -O3 -Wall -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
+  $CFLAGS << " -O3" unless $CFLAGS[/-O\d/]
+  $CFLAGS << " -Wall -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
 end
 
-if RbConfig::CONFIG['target_os'] =~ /mswin32/
+if RbConfig::CONFIG['target_os'] =~ /mswin/
   lib_prefix = 'lib'
 
   # There's no default include/lib dir on Windows. Let's just add the Ruby ones
@@ -80,6 +81,13 @@ else
     '/usr/local/include/libxml2',
     File.join(INCLUDEDIR, "libxml2")
   ] + HEADER_DIRS
+
+  # If the user has homebrew installed, use the libxml2 inside homebrew
+  brew_prefix = `brew --prefix libxml2 2> /dev/null`.chomp
+  unless brew_prefix.empty?
+    LIB_DIRS.unshift File.join(brew_prefix, 'lib')
+    XML2_HEADER_DIRS.unshift File.join(brew_prefix, 'include/libxml2')
+  end
 end
 
 dir_config('zlib', HEADER_DIRS, LIB_DIRS)

data/ext/nokogiri/html_document.c

@@ -1,5 +1,7 @@
 #include <html_document.h>
 
+static ID id_encoding_found;
+
 /*
  * call-seq:
  *  new
@@ -56,6 +58,18 @@ static VALUE read_io( VALUE klass,
   );
   xmlSetStructuredErrorFunc(NULL, NULL);
 
+  /*
+   * If EncodingFound has occurred in EncodingReader, make sure to do
+   * a cleanup and propagate the error.
+   */
+  if (rb_respond_to(io, id_encoding_found)) {
+    VALUE encoding_found = rb_funcall(io, id_encoding_found, 0);
+    if (!NIL_P(encoding_found)) {
+      xmlFreeDoc(doc);
+      rb_exc_raise(encoding_found);
+    }
+  }
+
   if(doc == NULL) {
     xmlErrorPtr error;
 
@@ -151,4 +165,6 @@ void init_html_document()
   rb_define_singleton_method(klass, "new", new, -1);
 
   rb_define_method(klass, "type", type, 0);
+
+  id_encoding_found = rb_intern("encoding_found");
 }

data/ext/nokogiri/html_sax_parser_context.c

@@ -13,31 +13,35 @@ static void deallocate(xmlParserCtxtPtr ctxt)
   NOKOGIRI_DEBUG_END(handler);
 }
 
-static VALUE parse_memory(VALUE klass, VALUE data, VALUE encoding)
+static VALUE
+parse_memory(VALUE klass, VALUE data, VALUE encoding)
 {
-  htmlParserCtxtPtr ctxt;
-
-  if(NIL_P(data)) rb_raise(rb_eArgError, "data cannot be nil");
-  if(!(int)RSTRING_LEN(data))
-    rb_raise(rb_eRuntimeError, "data cannot be empty");
-
-  ctxt = htmlCreateMemoryParserCtxt(
-      StringValuePtr(data),
-      (int)RSTRING_LEN(data)
-  );
+    htmlParserCtxtPtr ctxt;
+
+    if (NIL_P(data))
+	rb_raise(rb_eArgError, "data cannot be nil");
+    if (!(int)RSTRING_LEN(data))
+	rb_raise(rb_eRuntimeError, "data cannot be empty");
+
+    ctxt = htmlCreateMemoryParserCtxt(StringValuePtr(data),
+				      (int)RSTRING_LEN(data));
+    if (ctxt->sax) {
+	xmlFree(ctxt->sax);
+	ctxt->sax = NULL;
+    }
 
-  if(RTEST(encoding)) {
-    xmlCharEncodingHandlerPtr enc = xmlFindCharEncodingHandler(StringValuePtr(encoding));
-    if(enc != NULL) {
-      xmlSwitchToEncoding(ctxt, enc);
-      if(ctxt->errNo == XML_ERR_UNSUPPORTED_ENCODING) {
-        rb_raise(rb_eRuntimeError, "Unsupported encoding %s",
-            StringValuePtr(encoding));
-      }
+    if (RTEST(encoding)) {
+	xmlCharEncodingHandlerPtr enc = xmlFindCharEncodingHandler(StringValuePtr(encoding));
+	if (enc != NULL) {
+	    xmlSwitchToEncoding(ctxt, enc);
+	    if (ctxt->errNo == XML_ERR_UNSUPPORTED_ENCODING) {
+		rb_raise(rb_eRuntimeError, "Unsupported encoding %s",
+			 StringValuePtr(encoding));
+	    }
+	}
     }
-  }
 
-  return Data_Wrap_Struct(klass, NULL, deallocate, ctxt);
+    return Data_Wrap_Struct(klass, NULL, deallocate, ctxt);
 }
 
 static VALUE parse_file(VALUE klass, VALUE filename, VALUE encoding)
@@ -49,30 +53,48 @@ static VALUE parse_file(VALUE klass, VALUE filename, VALUE encoding)
   return Data_Wrap_Struct(klass, NULL, deallocate, ctxt);
 }
 
-static VALUE parse_with(VALUE self, VALUE sax_handler)
+static VALUE
+parse_doc(VALUE ctxt_val)
+{
+    htmlParserCtxtPtr ctxt = (htmlParserCtxtPtr)ctxt_val;
+    htmlParseDocument(ctxt);
+    return Qnil;
+}
+
+static VALUE
+parse_doc_finalize(VALUE ctxt_val)
 {
-  htmlParserCtxtPtr ctxt;
-  htmlSAXHandlerPtr sax;
+    htmlParserCtxtPtr ctxt = (htmlParserCtxtPtr)ctxt_val;
+
+    if (ctxt->myDoc)
+	xmlFreeDoc(ctxt->myDoc);
 
-  if(!rb_obj_is_kind_of(sax_handler, cNokogiriXmlSaxParser))
-    rb_raise(rb_eArgError, "argument must be a Nokogiri::XML::SAX::Parser");
+    NOKOGIRI_SAX_TUPLE_DESTROY(ctxt->userData);
+    return Qnil;
+}
+
+static VALUE
+parse_with(VALUE self, VALUE sax_handler)
+{
+    htmlParserCtxtPtr ctxt;
+    htmlSAXHandlerPtr sax;
 
-  Data_Get_Struct(self, htmlParserCtxt, ctxt);
-  Data_Get_Struct(sax_handler, htmlSAXHandler, sax);
+    if (!rb_obj_is_kind_of(sax_handler, cNokogiriXmlSaxParser))
+	rb_raise(rb_eArgError, "argument must be a Nokogiri::XML::SAX::Parser");
 
-  /* Free the sax handler since we'll assign our own */
-  if(ctxt->sax && ctxt->sax != (xmlSAXHandlerPtr)&xmlDefaultSAXHandler)
-    xmlFree(ctxt->sax);
+    Data_Get_Struct(self, htmlParserCtxt, ctxt);
+    Data_Get_Struct(sax_handler, htmlSAXHandler, sax);
 
-  ctxt->sax = sax;
-  ctxt->userData = (void *)NOKOGIRI_SAX_TUPLE_NEW(ctxt, sax_handler);
+    /* Free the sax handler since we'll assign our own */
+    if (ctxt->sax && ctxt->sax != (xmlSAXHandlerPtr)&xmlDefaultSAXHandler)
+	xmlFree(ctxt->sax);
 
-  htmlParseDocument(ctxt);
+    ctxt->sax = sax;
+    ctxt->userData = (void *)NOKOGIRI_SAX_TUPLE_NEW(ctxt, sax_handler);
 
-  if(NULL != ctxt->myDoc) xmlFreeDoc(ctxt->myDoc);
+    rb_ensure(parse_doc, (VALUE)ctxt, parse_doc_finalize, (VALUE)ctxt);
 
-  NOKOGIRI_SAX_TUPLE_DESTROY(ctxt->userData);
-  return self;
+    return self;
 }
 
 void init_html_sax_parser_context()

data/ext/nokogiri/html_sax_push_parser.c

@@ -0,0 +1,87 @@
+#include <html_sax_push_parser.h>
+
+/*
+ * call-seq:
+ *  native_write(chunk, last_chunk)
+ *
+ * Write +chunk+ to PushParser. +last_chunk+ triggers the end_document handle
+ */
+static VALUE native_write(VALUE self, VALUE _chunk, VALUE _last_chunk)
+{
+  xmlParserCtxtPtr ctx;
+  const char * chunk  = NULL;
+  int size            = 0;
+
+
+  Data_Get_Struct(self, xmlParserCtxt, ctx);
+
+  if(Qnil != _chunk) {
+    chunk = StringValuePtr(_chunk);
+    size = (int)RSTRING_LEN(_chunk);
+  }
+
+  if(htmlParseChunk(ctx, chunk, size, Qtrue == _last_chunk ? 1 : 0)) {
+    if (!(ctx->options & XML_PARSE_RECOVER)) {
+      xmlErrorPtr e = xmlCtxtGetLastError(ctx);
+      Nokogiri_error_raise(NULL, e);
+    }
+  }
+
+  return self;
+}
+
+/*
+ * call-seq:
+ *  initialize_native(xml_sax, filename)
+ *
+ * Initialize the push parser with +xml_sax+ using +filename+
+ */
+static VALUE initialize_native(VALUE self, VALUE _xml_sax, VALUE _filename,
+			       VALUE encoding)
+{
+  htmlSAXHandlerPtr sax;
+  const char * filename = NULL;
+  htmlParserCtxtPtr ctx;
+  xmlCharEncoding enc = XML_CHAR_ENCODING_NONE;
+
+  Data_Get_Struct(_xml_sax, xmlSAXHandler, sax);
+
+  if(_filename != Qnil) filename = StringValuePtr(_filename);
+
+  if (!NIL_P(encoding)) {
+    enc = xmlParseCharEncoding(StringValuePtr(encoding));
+    if (enc == XML_CHAR_ENCODING_ERROR)
+      rb_raise(rb_eArgError, "Unsupported Encoding");
+  }
+
+  ctx = htmlCreatePushParserCtxt(
+      sax,
+      NULL,
+      NULL,
+      0,
+      filename,
+      enc
+  );
+  if(ctx == NULL)
+    rb_raise(rb_eRuntimeError, "Could not create a parser context");
+
+  ctx->userData = NOKOGIRI_SAX_TUPLE_NEW(ctx, self);
+
+  ctx->sax2 = 1;
+  DATA_PTR(self) = ctx;
+  return self;
+}
+
+VALUE cNokogiriHtmlSaxPushParser;
+void init_html_sax_push_parser()
+{
+  VALUE nokogiri = rb_define_module("Nokogiri");
+  VALUE html = rb_define_module_under(nokogiri, "HTML");
+  VALUE sax = rb_define_module_under(html, "SAX");
+  VALUE klass = rb_define_class_under(sax, "PushParser", cNokogiriXmlSaxPushParser);
+
+  cNokogiriHtmlSaxPushParser = klass;
+
+  rb_define_private_method(klass, "initialize_native", initialize_native, 3);
+  rb_define_private_method(klass, "native_write", native_write, 2);
+}