nokogiri 1.5.2-java → 1.5.3-java

data/ext/java/nokogiri/internals/HtmlDomParserContext.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -135,6 +135,7 @@ public class HtmlDomParserContext extends XmlDomParserContext {
             }
         }
         htmlDocument.setEncoding(ruby_encoding);
+        htmlDocument.setParsedEncoding(java_encoding);
         return htmlDocument;
     }
     

data/ext/java/nokogiri/internals/NokogiriHelpers.java

@@ -34,6 +34,8 @@ package nokogiri.internals;
 
 import java.io.File;
 import java.io.UnsupportedEncodingException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.CharacterCodingException;
@@ -42,6 +44,8 @@ import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CharsetEncoder;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.SortedMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -255,6 +259,7 @@ public class NokogiriHelpers {
      * this for us.
      */
     public static String rubyStringToString(IRubyObject str) {
+        if (str.isNil()) return null;
         //return rubyStringToString(str.convertToString());
         return toJavaString(str.convertToString());
     }
@@ -656,13 +661,29 @@ public class NokogiriHelpers {
         return n;
     }
     
-    public static String guessEncoding(Ruby ruby) {
+    public static String getValidEncoding(Ruby runtime, IRubyObject encoding) {
+        if (encoding.isNil()) {
+            return guessEncoding();
+        } else {
+            return ignoreInvalidEncoding(runtime, encoding);
+        }
+    }
+    
+    private static String guessEncoding() {
         String name = null;
         if (name == null) name = System.getProperty("file.encoding");
         if (name == null) name = "UTF-8";
         return name;
     }
     
+    private static Set<String> charsetNames = ((SortedMap<String, Charset>)Charset.availableCharsets()).keySet();
+    
+    private static String ignoreInvalidEncoding(Ruby runtime, IRubyObject encoding) {
+        String givenEncoding = rubyStringToString(encoding);
+        if (charsetNames.contains(givenEncoding)) return givenEncoding;
+        else return guessEncoding();
+    }
+    
     public static String adjustSystemIdIfNecessary(String currentDir, String scriptFileName, String baseURI, String systemId) {
         if (systemId == null) return systemId;
         File file = new File(systemId);
@@ -709,4 +730,71 @@ public class NokogiriHelpers {
         return bytes;
     }
     
+    public static String convertEncodingByNKFIfNecessary(Ruby runtime, XmlDocument doc, String thing) {
+        if (!(doc instanceof HtmlDocument)) return thing;
+        String parsed_encoding = ((HtmlDocument)doc).getPraedEncoding();
+        if (parsed_encoding == null) return thing;
+        String ruby_encoding = rubyStringToString(doc.getEncoding());
+        if (ruby_encoding == null) return thing;
+        if (Charset.forName(parsed_encoding).compareTo(Charset.forName(ruby_encoding)) == 0) {
+            return thing;
+        } else {
+            return NokogiriHelpers.nkf(runtime, ruby_encoding, thing);
+        }
+        
+    }
+
+    // This method is used from HTML documents. HTML meta tag with encoding specification
+    // might appear after non-ascii characters are used. For example, a title tag before
+    // a meta tag. In such a case, Xerces encodes characters in UTF-8 without seeing meta tag.
+    // Nokogiri uses NKF library to convert characters correct encoding. This means the method
+    // works only for JIS/Shift_JIS/EUC-JP.
+    public static String nkf(Ruby runtime, String ruby_encoding, String thing) {
+        StringBuffer sb = new StringBuffer("-");
+        Charset that = Charset.forName(ruby_encoding);
+        if (NokogiriHelpers.shift_jis.compareTo(that) == 0) {
+            sb.append("S");
+        } else if (NokogiriHelpers.jis.compareTo(that) == 0) {
+            sb.append("J");
+        } else if (NokogiriHelpers.euc_jp.compareTo(that) == 0) {
+            sb.append("E");
+        } else {
+            // should not come here. should be treated before this method.
+            sb.append("W");
+        }
+        sb.append("w");
+        Class nkfClass = null;
+        try {
+            // JRuby 1.7 and later
+            nkfClass = runtime.getClassLoader().loadClass("org.jruby.ext.nkf.RubyNKF");
+        } catch (ClassNotFoundException e1) {
+            try {
+                // Before JRuby 1.7
+                nkfClass = runtime.getClassLoader().loadClass("org.jruby.RubyNKF");  
+            } catch (ClassNotFoundException e2) {
+                return thing;
+            }
+        }
+        Method nkf_method;
+        try {
+            nkf_method = nkfClass.getMethod("nkf", ThreadContext.class, IRubyObject.class, IRubyObject.class, IRubyObject.class);
+            RubyString r_str = 
+                (RubyString)nkf_method.invoke(null, runtime.getCurrentContext(), null, runtime.newString(new String(sb)), runtime.newString(thing));
+            return NokogiriHelpers.rubyStringToString(r_str);
+        } catch (SecurityException e) {
+            return thing;
+        } catch (NoSuchMethodException e) {
+            return thing;
+        } catch (IllegalArgumentException e) {
+            return thing;
+        } catch (IllegalAccessException e) {
+            return thing;
+        } catch (InvocationTargetException e) {
+            return thing;
+        }
+    }
+    
+    private static Charset shift_jis = Charset.forName("Shift_JIS");
+    private static Charset jis = Charset.forName("ISO-2022-JP");
+    private static Charset euc_jp = Charset.forName("EUC-JP");
 }

data/ext/java/nokogiri/internals/ParserContext.java

@@ -40,6 +40,9 @@ import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
 
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
@@ -159,9 +162,27 @@ public class ParserContext extends RubyObject {
             }
         }
         if (stringData != null) {
+            String encName = null;
+            if (stringData.encoding(context) != null) {
+                encName = stringData.encoding(context).toString();
+            }
+            Charset charset = null;
+            if (encName != null) {
+                try {
+                    charset = Charset.forName(encName);
+                } catch (UnsupportedCharsetException e) {
+                    // do nothing;
+                }
+            }
             ByteList bytes = stringData.getByteList();
-            stringDataSize = bytes.length() - bytes.begin();
-            source = new InputSource(new ByteArrayInputStream(bytes.unsafeBytes(), bytes.begin(), bytes.length()));
+            if (charset != null) {
+                StringReader reader = new StringReader(new String(bytes.unsafeBytes(), bytes.begin(), bytes.length(), charset));
+                source = new InputSource(reader);
+                source.setEncoding(charset.name());
+            } else {
+                stringDataSize = bytes.length() - bytes.begin();
+                source = new InputSource(new ByteArrayInputStream(bytes.unsafeBytes(), bytes.begin(), bytes.length()));
+            }
         }
     }
     

data/ext/java/nokogiri/internals/SaveContextVisitor.java

@@ -45,6 +45,8 @@ import java.util.Deque;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.cyberneko.html.HTMLElements;
 import org.w3c.dom.Attr;
@@ -252,12 +254,27 @@ public class SaveContextVisitor {
         if (!asHtml || !isHtmlBooleanAttr(name)) {
             buffer.append("=");
             buffer.append("\"");
-            buffer.append(serializeAttrTextContent(attr.getValue(), htmlDoc));
+            String value = replaceCharsetIfNecessary(attr);
+            buffer.append(serializeAttrTextContent(value, htmlDoc));
             buffer.append("\"");
         }
         return true;
     }
     
+    private static Pattern p = 
+        Pattern.compile("charset(()|\\s+)=(()|\\s+)(\\w|\\_|\\.|\\-)+", Pattern.CASE_INSENSITIVE);
+    
+    private String replaceCharsetIfNecessary(Attr attr) {
+        String value = attr.getValue();
+        if (encoding == null) return value;   // unable to replace in any case
+        if (!"content".equals(attr.getName().toLowerCase())) return value;  // must be content attr
+        if (!"meta".equals(attr.getOwnerElement().getNodeName().toLowerCase())) return value;        
+        Matcher m = p.matcher(value);
+        if (!m.find()) return value;
+        if (value.contains(encoding)) return value;  // no need to replace
+        return value.replace(m.group(), "charset=" + encoding);
+    }
+    
     public static final String[] HTML_BOOLEAN_ATTRS = {
         "checked", "compact", "declare", "defer", "disabled", "ismap",
         "multiple", "nohref", "noresize", "noshade", "nowrap", "readonly",

data/ext/java/nokogiri/internals/XmlDomParserContext.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -33,7 +33,6 @@
 package nokogiri.internals;
 
 import static nokogiri.internals.NokogiriHelpers.getNokogiriClass;
-import static nokogiri.internals.NokogiriHelpers.rubyStringToString;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -88,7 +87,7 @@ public class XmlDomParserContext extends ParserContext {
     public XmlDomParserContext(Ruby runtime, IRubyObject encoding, IRubyObject options) {
         super(runtime);
         this.options = new ParserContext.Options((Long)options.toJava(Long.class));
-        this.java_encoding = encoding.isNil() ? NokogiriHelpers.guessEncoding(runtime) : rubyStringToString(encoding);
+        java_encoding = NokogiriHelpers.getValidEncoding(runtime, encoding);
         ruby_encoding = encoding;
         initErrorHandler();
         initParser(runtime);

data/ext/nokogiri/extconf.rb

@@ -104,7 +104,7 @@ pkg_config('libxslt') if RUBY_PLATFORM =~ /mingw/
 asplode "libxml2"  unless find_header('libxml/parser.h')
 asplode "libxslt"  unless find_header('libxslt/xslt.h')
 asplode "libexslt" unless find_header('libexslt/exslt.h')
-asplode "libiconv" unless have_func('iconv_open', 'iconv.h') or have_library('iconv', 'iconv_open', 'iconv.h')
+asplode "libiconv" unless have_func('iconv_open', 'iconv.h') or have_library('iconv', 'iconv_open', 'iconv.h') or find_library('iconv', 'iconv_open', 'iconv.h')
 asplode "libxml2"  unless find_library("#{lib_prefix}xml2", 'xmlParseDoc')
 asplode "libxslt"  unless find_library("#{lib_prefix}xslt", 'xsltParseStylesheetDoc')
 asplode "libexslt" unless find_library("#{lib_prefix}exslt", 'exsltFuncRegister')

data/ext/nokogiri/xml_io.c

@@ -25,7 +25,7 @@ int io_read_callback(void * ctx, char * buffer, int len) {
   safe_len = str_len > (size_t)len ? (size_t)len : str_len;
   memcpy(buffer, StringValuePtr(string), safe_len);
 
-  return safe_len;
+  return (int)safe_len;
 }
 
 VALUE write_check(VALUE *args) {

data/ext/nokogiri/xml_namespace.c

@@ -11,13 +11,10 @@ VALUE cNokogiriXmlNamespace ;
 static VALUE prefix(VALUE self)
 {
   xmlNsPtr ns;
-  xmlDocPtr doc;
 
   Data_Get_Struct(self, xmlNs, ns);
   if(!ns->prefix) return Qnil;
 
-  Data_Get_Struct(rb_iv_get(self, "@document"), xmlDoc, doc);
-
   return NOKOGIRI_STR_NEW2(ns->prefix);
 }
 
@@ -30,13 +27,10 @@ static VALUE prefix(VALUE self)
 static VALUE href(VALUE self)
 {
   xmlNsPtr ns;
-  xmlDocPtr doc;
 
   Data_Get_Struct(self, xmlNs, ns);
   if(!ns->href) return Qnil;
 
-  Data_Get_Struct(rb_iv_get(self, "@document"), xmlDoc, doc);
-
   return NOKOGIRI_STR_NEW2(ns->href);
 }
 

data/ext/nokogiri/xml_node.c

@@ -1219,7 +1219,7 @@ static VALUE process_xincludes(VALUE self, VALUE options)
 /* TODO: DOCUMENT ME */
 static VALUE in_context(VALUE self, VALUE _str, VALUE _options)
 {
-    xmlNodePtr node, list, child_iter, tmp, node_children, doc_children;
+    xmlNodePtr node, list = 0, child_iter, node_children, doc_children;
     xmlNodeSetPtr set;
     xmlParserErrors error;
     VALUE doc, err;
@@ -1257,8 +1257,8 @@ static VALUE in_context(VALUE self, VALUE _str, VALUE _options)
      * is because if there were errors, it's possible for the child pointers
      * to be manipulated. */
     if (error != XML_ERR_OK) {
-	node->doc->children = doc_children;
-	node->children = node_children;
+      node->doc->children = doc_children;
+      node->children = node_children;
     }
 
     /* make sure parent/child pointers are coherent so an unlink will work
@@ -1266,9 +1266,9 @@ static VALUE in_context(VALUE self, VALUE _str, VALUE _options)
      */
     child_iter = node->doc->children ;
     while (child_iter) {
-	if (child_iter->parent != (xmlNodePtr)node->doc)
-	    child_iter->parent = (xmlNodePtr)node->doc;
-	child_iter = child_iter->next;
+      if (child_iter->parent != (xmlNodePtr)node->doc)
+        child_iter->parent = (xmlNodePtr)node->doc;
+      child_iter = child_iter->next;
     }
 
 #ifndef HTML_PARSE_NOIMPLIED
@@ -1285,12 +1285,12 @@ static VALUE in_context(VALUE self, VALUE _str, VALUE _options)
      * https://bugzilla.gnome.org/show_bug.cgi?id=668155
      */
     if (error != XML_ERR_OK && doc_is_empty && node->doc->children != NULL) {
-	tmp = node;
-	while (tmp->parent)
-	    tmp = tmp->parent;
+      child_iter = node;
+      while (child_iter->parent)
+        child_iter = child_iter->parent;
 	
-	if (tmp->type == XML_DOCUMENT_FRAG_NODE)
-	    node->doc->children = NULL;
+      if (child_iter->type == XML_DOCUMENT_FRAG_NODE)
+        node->doc->children = NULL;
     }
 
     /* FIXME: This probably needs to handle more constants... */

data/ext/nokogiri/xml_node_set.c

@@ -427,7 +427,7 @@ VALUE Nokogiri_wrap_xml_node_set(xmlNodeSetPtr node_set, VALUE document)
     rb_funcall(document, decorate, 1, new_set);
   }
 
-  if (node_set->nodeTab) {
+  if (node_set && node_set->nodeTab) {
     for (i = 0; i < node_set->nodeNr; i++) {
       cur = node_set->nodeTab[i];
       if (cur && cur->type == XML_NAMESPACE_DECL) {

data/ext/nokogiri/xml_xpath_context.c

@@ -49,26 +49,19 @@ static VALUE register_variable(VALUE self, VALUE name, VALUE value)
    return self;
 }
 
-static void ruby_funcall(xmlXPathParserContextPtr ctx, int nargs)
+void Nokogiri_marshal_xpath_funcall_and_return_values(xmlXPathParserContextPtr ctx, int nargs, VALUE handler, const char* function_name)
 {
-  VALUE xpath_handler = Qnil;
-  VALUE result;
+  int i;
+  VALUE result, doc;
   VALUE *argv;
-  VALUE doc;
   VALUE node_set = Qnil;
   xmlNodeSetPtr xml_node_set = NULL;
   xmlXPathObjectPtr obj;
-  int i;
   nokogiriNodeSetTuple *node_set_tuple;
 
-  assert(ctx);
-  assert(ctx->context);
-  assert(ctx->context->userData);
   assert(ctx->context->doc);
   assert(DOC_RUBY_OBJECT_TEST(ctx->context->doc));
 
-  xpath_handler = (VALUE)(ctx->context->userData);
-
   argv = (VALUE *)calloc((size_t)nargs, sizeof(VALUE));
   for (i = 0 ; i < nargs ; ++i) {
     rb_gc_register_address(&argv[i]);
@@ -100,12 +93,7 @@ static void ruby_funcall(xmlXPathParserContextPtr ctx, int nargs)
     } while(i-- > 0);
   }
 
-  result = rb_funcall2(
-      xpath_handler,
-      rb_intern((const char *)ctx->context->function),
-      nargs,
-      argv
-  );
+  result = rb_funcall2(handler, rb_intern((const char*)function_name), nargs, argv);
 
   for (i = 0 ; i < nargs ; ++i) {
     rb_gc_unregister_address(&argv[i]);
@@ -156,6 +144,22 @@ static void ruby_funcall(xmlXPathParserContextPtr ctx, int nargs)
   }
 }
 
+static void ruby_funcall(xmlXPathParserContextPtr ctx, int nargs)
+{
+  VALUE handler = Qnil;
+  const char *function = NULL ;
+
+  assert(ctx);
+  assert(ctx->context);
+  assert(ctx->context->userData);
+  assert(ctx->context->function);
+
+  handler = (VALUE)(ctx->context->userData);
+  function = (const char*)(ctx->context->function);
+
+  Nokogiri_marshal_xpath_funcall_and_return_values(ctx, nargs, handler, function);
+}
+
 static xmlXPathFunction lookup( void *ctx,
                                 const xmlChar * name,
                                 const xmlChar* ns_uri )

data/ext/nokogiri/xml_xpath_context.h

@@ -4,6 +4,7 @@
 #include <nokogiri.h>
 
 void init_xml_xpath_context();
+void Nokogiri_marshal_xpath_funcall_and_return_values(xmlXPathParserContextPtr ctx, int nargs, VALUE handler, const char* function_name) ;
 
 extern VALUE cNokogiriXmlXpathContext;
 #endif

data/ext/nokogiri/xslt_stylesheet.c

@@ -168,74 +168,17 @@ static VALUE transform(int argc, VALUE* argv, VALUE self)
 
 static void method_caller(xmlXPathParserContextPtr ctxt, int nargs)
 {
-    const xmlChar * function;
-    const xmlChar * functionURI;
-    size_t i, count;
-
+    VALUE handler;
+    const char *function_name;
     xsltTransformContextPtr transform;
-    xmlXPathObjectPtr xpath;
-    VALUE obj;
-    VALUE *args;
-    VALUE result;
+    const xmlChar *functionURI;
 
     transform = xsltXPathGetTransformContext(ctxt);
-
-    function = ctxt->context->function;
     functionURI = ctxt->context->functionURI;
-    obj = (VALUE)xsltGetExtData(transform, functionURI);
-
-    count = (size_t)ctxt->valueNr;
-    args = calloc(count, sizeof(VALUE *));
-
-    for(i = 0; i < count; i++) {
-	VALUE thing;
-
-	xpath = valuePop(ctxt);
-	switch(xpath->type) {
-	    case XPATH_STRING:
-		thing = NOKOGIRI_STR_NEW2(xpath->stringval);
-		break;
-	    case XPATH_NODESET:
-		if(NULL == xpath->nodesetval) {
-		    thing = Nokogiri_wrap_xml_node_set(
-			    xmlXPathNodeSetCreate(NULL),
-			    DOC_RUBY_OBJECT(ctxt->context->doc));
-		} else {
-		    thing = Nokogiri_wrap_xml_node_set(xpath->nodesetval,
-			    DOC_RUBY_OBJECT(ctxt->context->doc));
-		}
-		break;
-	    default:
-		rb_raise(rb_eRuntimeError, "do not handle type: %d", xpath->type);
-	}
-	args[i] = thing;
-	xmlFree(xpath);
-    }
-    result = rb_funcall3(obj, rb_intern((const char *)function), (int)count, args);
-    free(args);
-    switch(TYPE(result)) {
-	case T_FLOAT:
-	case T_BIGNUM:
-	case T_FIXNUM:
-	    xmlXPathReturnNumber(ctxt, NUM2DBL(result));
-	    break;
-	case T_STRING:
-	    xmlXPathReturnString(
-		    ctxt,
-		    xmlStrdup((xmlChar *)StringValuePtr(result))
-		    );
-	    break;
-	case T_TRUE:
-	    xmlXPathReturnTrue(ctxt);
-	    break;
-	case T_FALSE:
-	    xmlXPathReturnFalse(ctxt);
-	    break;
-	case T_NIL:
-	    break;
-	default:
-	    rb_raise(rb_eRuntimeError, "Invalid return type");
-    }
+    handler = (VALUE)xsltGetExtData(transform, functionURI);
+    function_name = (const char*)(ctxt->context->function);
+
+    Nokogiri_marshal_xpath_funcall_and_return_values(ctxt, nargs, handler, (const char*)function_name);
 }
 
 static void * initFunc(xsltTransformContextPtr ctxt, const xmlChar *uri)