nokogiri 1.5.3.rc5-java → 1.5.3.rc6-java

data/CHANGELOG.ja.rdoc

@@ -20,11 +20,12 @@
   * Nokogiri::XML::Attribute が JRuby 上で　nil namespace を返す　#647
   * Nokogiri::XML::Node#namespace= メソッドが JRuby　上で prefix
   　が無い namespace を設定できない #648
-  * JRuby 1.9 モードで rake を実行するとデッドロックを引き起こす #571
+  * [JRuby] 1.9 モードで rake を実行するとデッドロックを引き起こす #571
   * HTML::Document#meta_encoding does not raise exception on docs with
     malformed content-type. #655
   * Fixing segfault related to unsupported encodings in in-context
     parsing on 1.8.7. #643
+  * [JRuby] Concurrency issue in XPath parsing. #682
 
 
 == 1.5.2 / 2012-03-09

data/CHANGELOG.rdoc

@@ -20,11 +20,12 @@
   * Nokogiri::XML::Attribute on JRuby returns a nil namespace #647
   * Nokogiri::XML::Node#namespace= cannot set a namespace without a
     prefix on JRuby #648
-  * JRuby 1.9 mode causes dead lock while running rake #571
+  * [JRuby] 1.9 mode causes dead lock while running rake #571
   * HTML::Document#meta_encoding does not raise exception on docs with
     malformed content-type. #655
   * Fixing segfault related to unsupported encodings in in-context
     parsing on 1.8.7. #643
+  * [JRuby] Concurrency issue in XPath parsing. #682
 
 
 == 1.5.2 / 2012-03-09

data/README.rdoc

@@ -91,7 +91,7 @@ encoded like the source document.
 Some documents declare one particular encoding, but use a different
 one. So, which encoding should the parser choose?
 
-Remember that data is just a stream of bytes. Only us humans add
+Remember that data is just a stream of bytes. Only we humans add
 meaning to that stream. Any particular set of bytes could be valid
 characters in multiple encodings, so detecting encoding with 100%
 accuracy is not possible. libxml2 does its best, but it can't be right

data/ROADMAP.md

@@ -14,6 +14,15 @@
 * see fairy wing throwdown - SAX parsing is wicked slow.
 
 
+## Node should not be Enumerable; and should have a better attributes API
+
+* https://github.com/tenderlove/nokogiri/issues/679
+  Mixing in Enumerable has some unintended consequences; plus we want to improve the attributes API
+
+* (closed) https://github.com/tenderlove/nokogiri/issues/666
+  Some ideas for a better attributes API?
+
+
 ## improve CSS query parsing
 
 * https://github.com/tenderlove/nokogiri/issues/528
@@ -27,6 +36,7 @@
   * https://github.com/tenderlove/nokogiri/issues/342
   * https://github.com/tenderlove/nokogiri/issues/628
   * https://github.com/tenderlove/nokogiri/issues/652
+  * https://github.com/tenderlove/nokogiri/issues/688
 
 * https://github.com/tenderlove/nokogiri/issues/394
   nth-of-type is wrong, and possibly other selectors as well

data/ext/java/nokogiri/HtmlDocument.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -55,6 +55,7 @@ import org.w3c.dom.NodeList;
  */
 @JRubyClass(name="Nokogiri::HTML::Document", parent="Nokogiri::XML::Document")
 public class HtmlDocument extends XmlDocument {
+    private String parsed_encoding = null;
 
     public HtmlDocument(Ruby ruby, RubyClass klazz) {
         super(ruby, klazz);
@@ -123,6 +124,14 @@ public class HtmlDocument extends XmlDocument {
             stabilzeAttrValue(children.item(i));
         }
     }
+    
+    public void setParsedEncoding(String encoding) {
+        parsed_encoding = encoding;
+    }
+    
+    public String getPraedEncoding() {
+        return parsed_encoding;
+    }
 
     /*
      * call-seq:

data/ext/java/nokogiri/XmlDocument.java

@@ -128,6 +128,10 @@ public class XmlDocument extends XmlNode {
         this.encoding = encoding;
     }
     
+    public IRubyObject getEncoding() {
+        return encoding;
+    }
+    
     // not sure, but like attribute values, text value will be lost
     // unless it is referred once before this document is used.
     // this seems to happen only when the fragment is parsed from Node#in_context.

data/ext/java/nokogiri/XmlNode.java

@@ -42,13 +42,8 @@ import static nokogiri.internals.NokogiriHelpers.stringOrNil;
 
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -767,6 +762,7 @@ public class XmlNode extends RubyObject {
         } else {
             textContent = this.node.getTextContent();
         }
+        textContent = NokogiriHelpers.convertEncodingByNKFIfNecessary(context.getRuntime(), (XmlDocument)document(context), textContent);
         String decodedText = null;
         if (textContent != null) decodedText = NokogiriHelpers.decodeJavaString(textContent);
         return stringOrNil(context.getRuntime(), decodedText);

data/ext/java/nokogiri/XmlSyntaxError.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -35,6 +35,7 @@ package nokogiri;
 import static nokogiri.internals.NokogiriHelpers.getNokogiriClass;
 import static nokogiri.internals.NokogiriHelpers.stringOrNil;
 
+import org.jruby.CompatVersion;
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
 import org.jruby.RubyException;
@@ -108,12 +109,23 @@ public class XmlSyntaxError extends RubyException {
         return new XmlSyntaxError(runtime, klazz, e);
     }
 
-    @Override
-    @JRubyMethod(name = "to_s")
+    //@Override
+    //"to_s" method was branched in 1.8 and 1.9 since JRuby 1.6.6
+    // to support older version of JRuby, the annotation is commented out
+    @JRubyMethod(name = "to_s", compat = CompatVersion.RUBY1_8)
     public IRubyObject to_s(ThreadContext context) {
         if (exception != null && exception.getMessage() != null)
             return context.getRuntime().newString(exception.getMessage());
         else
             return super.to_s(context);  
     }
+    
+    //@Override
+    //"to_s" method was branched in 1.8 and 1.9 since JRuby 1.6.6
+    // to support older version of JRuby, the annotation is commented out
+    @JRubyMethod(name = "to_s", compat = CompatVersion.RUBY1_9)
+    public IRubyObject to_s19(ThreadContext context) {
+        return this.to_s(context);
+    }
+    
 }

data/ext/java/nokogiri/XmlXpathContext.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -53,7 +53,6 @@ import org.jruby.RubyBoolean;
 import org.jruby.RubyClass;
 import org.jruby.RubyException;
 import org.jruby.RubyFloat;
-import org.jruby.RubyNumeric;
 import org.jruby.RubyObject;
 import org.jruby.RubyString;
 import org.jruby.anno.JRubyClass;
@@ -72,7 +71,7 @@ import org.w3c.dom.NodeList;
 @JRubyClass(name="Nokogiri::XML::XPathContext")
 public class XmlXpathContext extends RubyObject {
     private XmlNode context;
-    private static final XPath xpath = XPathFactory.newInstance().newXPath();;
+    private XPath xpath;
     
     public XmlXpathContext(Ruby ruby, RubyClass rubyClass) {
         super(ruby, rubyClass);
@@ -98,6 +97,7 @@ public class XmlXpathContext extends RubyObject {
     public static IRubyObject rbNew(ThreadContext context, IRubyObject klazz, IRubyObject node) {
         XmlNode xmlNode = (XmlNode)node;
         XmlXpathContext xmlXpathContext = (XmlXpathContext) NokogiriService.XML_XPATHCONTEXT_ALLOCATOR.allocate(context.getRuntime(), (RubyClass)klazz);
+        xmlXpathContext.xpath = XPathFactory.newInstance().newXPath();
         xmlXpathContext.setNode(xmlNode);
         return xmlXpathContext;
     }

data/ext/java/nokogiri/internals/HtmlDomParserContext.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -135,6 +135,7 @@ public class HtmlDomParserContext extends XmlDomParserContext {
             }
         }
         htmlDocument.setEncoding(ruby_encoding);
+        htmlDocument.setParsedEncoding(java_encoding);
         return htmlDocument;
     }
     

data/ext/java/nokogiri/internals/NokogiriHelpers.java

@@ -34,6 +34,8 @@ package nokogiri.internals;
 
 import java.io.File;
 import java.io.UnsupportedEncodingException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.CharacterCodingException;
@@ -42,6 +44,8 @@ import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CharsetEncoder;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.SortedMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -657,13 +661,29 @@ public class NokogiriHelpers {
         return n;
     }
     
-    public static String guessEncoding(Ruby ruby) {
+    public static String getValidEncoding(Ruby runtime, IRubyObject encoding) {
+        if (encoding.isNil()) {
+            return guessEncoding();
+        } else {
+            return ignoreInvalidEncoding(runtime, encoding);
+        }
+    }
+    
+    private static String guessEncoding() {
         String name = null;
         if (name == null) name = System.getProperty("file.encoding");
         if (name == null) name = "UTF-8";
         return name;
     }
     
+    private static Set<String> charsetNames = ((SortedMap<String, Charset>)Charset.availableCharsets()).keySet();
+    
+    private static String ignoreInvalidEncoding(Ruby runtime, IRubyObject encoding) {
+        String givenEncoding = rubyStringToString(encoding);
+        if (charsetNames.contains(givenEncoding)) return givenEncoding;
+        else return guessEncoding();
+    }
+    
     public static String adjustSystemIdIfNecessary(String currentDir, String scriptFileName, String baseURI, String systemId) {
         if (systemId == null) return systemId;
         File file = new File(systemId);
@@ -710,4 +730,71 @@ public class NokogiriHelpers {
         return bytes;
     }
     
+    public static String convertEncodingByNKFIfNecessary(Ruby runtime, XmlDocument doc, String thing) {
+        if (!(doc instanceof HtmlDocument)) return thing;
+        String parsed_encoding = ((HtmlDocument)doc).getPraedEncoding();
+        if (parsed_encoding == null) return thing;
+        String ruby_encoding = rubyStringToString(doc.getEncoding());
+        if (ruby_encoding == null) return thing;
+        if (Charset.forName(parsed_encoding).compareTo(Charset.forName(ruby_encoding)) == 0) {
+            return thing;
+        } else {
+            return NokogiriHelpers.nkf(runtime, ruby_encoding, thing);
+        }
+        
+    }
+
+    // This method is used from HTML documents. HTML meta tag with encoding specification
+    // might appear after non-ascii characters are used. For example, a title tag before
+    // a meta tag. In such a case, Xerces encodes characters in UTF-8 without seeing meta tag.
+    // Nokogiri uses NKF library to convert characters correct encoding. This means the method
+    // works only for JIS/Shift_JIS/EUC-JP.
+    public static String nkf(Ruby runtime, String ruby_encoding, String thing) {
+        StringBuffer sb = new StringBuffer("-");
+        Charset that = Charset.forName(ruby_encoding);
+        if (NokogiriHelpers.shift_jis.compareTo(that) == 0) {
+            sb.append("S");
+        } else if (NokogiriHelpers.jis.compareTo(that) == 0) {
+            sb.append("J");
+        } else if (NokogiriHelpers.euc_jp.compareTo(that) == 0) {
+            sb.append("E");
+        } else {
+            // should not come here. should be treated before this method.
+            sb.append("W");
+        }
+        sb.append("w");
+        Class nkfClass = null;
+        try {
+            // JRuby 1.7 and later
+            nkfClass = runtime.getClassLoader().loadClass("org.jruby.ext.nkf.RubyNKF");
+        } catch (ClassNotFoundException e1) {
+            try {
+                // Before JRuby 1.7
+                nkfClass = runtime.getClassLoader().loadClass("org.jruby.RubyNKF");  
+            } catch (ClassNotFoundException e2) {
+                return thing;
+            }
+        }
+        Method nkf_method;
+        try {
+            nkf_method = nkfClass.getMethod("nkf", ThreadContext.class, IRubyObject.class, IRubyObject.class, IRubyObject.class);
+            RubyString r_str = 
+                (RubyString)nkf_method.invoke(null, runtime.getCurrentContext(), null, runtime.newString(new String(sb)), runtime.newString(thing));
+            return NokogiriHelpers.rubyStringToString(r_str);
+        } catch (SecurityException e) {
+            return thing;
+        } catch (NoSuchMethodException e) {
+            return thing;
+        } catch (IllegalArgumentException e) {
+            return thing;
+        } catch (IllegalAccessException e) {
+            return thing;
+        } catch (InvocationTargetException e) {
+            return thing;
+        }
+    }
+    
+    private static Charset shift_jis = Charset.forName("Shift_JIS");
+    private static Charset jis = Charset.forName("ISO-2022-JP");
+    private static Charset euc_jp = Charset.forName("EUC-JP");
 }

data/ext/java/nokogiri/internals/ParserContext.java

@@ -40,6 +40,9 @@ import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
 
 import org.jruby.Ruby;
 import org.jruby.RubyClass;
@@ -159,9 +162,27 @@ public class ParserContext extends RubyObject {
             }
         }
         if (stringData != null) {
+            String encName = null;
+            if (stringData.encoding(context) != null) {
+                encName = stringData.encoding(context).toString();
+            }
+            Charset charset = null;
+            if (encName != null) {
+                try {
+                    charset = Charset.forName(encName);
+                } catch (UnsupportedCharsetException e) {
+                    // do nothing;
+                }
+            }
             ByteList bytes = stringData.getByteList();
-            stringDataSize = bytes.length() - bytes.begin();
-            source = new InputSource(new ByteArrayInputStream(bytes.unsafeBytes(), bytes.begin(), bytes.length()));
+            if (charset != null) {
+                StringReader reader = new StringReader(new String(bytes.unsafeBytes(), bytes.begin(), bytes.length(), charset));
+                source = new InputSource(reader);
+                source.setEncoding(charset.name());
+            } else {
+                stringDataSize = bytes.length() - bytes.begin();
+                source = new InputSource(new ByteArrayInputStream(bytes.unsafeBytes(), bytes.begin(), bytes.length()));
+            }
         }
     }
     

data/ext/java/nokogiri/internals/SaveContextVisitor.java

@@ -45,6 +45,8 @@ import java.util.Deque;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.cyberneko.html.HTMLElements;
 import org.w3c.dom.Attr;
@@ -252,12 +254,27 @@ public class SaveContextVisitor {
         if (!asHtml || !isHtmlBooleanAttr(name)) {
             buffer.append("=");
             buffer.append("\"");
-            buffer.append(serializeAttrTextContent(attr.getValue(), htmlDoc));
+            String value = replaceCharsetIfNecessary(attr);
+            buffer.append(serializeAttrTextContent(value, htmlDoc));
             buffer.append("\"");
         }
         return true;
     }
     
+    private static Pattern p = 
+        Pattern.compile("charset(()|\\s+)=(()|\\s+)(\\w|\\_|\\.|\\-)+", Pattern.CASE_INSENSITIVE);
+    
+    private String replaceCharsetIfNecessary(Attr attr) {
+        String value = attr.getValue();
+        if (encoding == null) return value;   // unable to replace in any case
+        if (!"content".equals(attr.getName().toLowerCase())) return value;  // must be content attr
+        if (!"meta".equals(attr.getOwnerElement().getNodeName().toLowerCase())) return value;        
+        Matcher m = p.matcher(value);
+        if (!m.find()) return value;
+        if (value.contains(encoding)) return value;  // no need to replace
+        return value.replace(m.group(), "charset=" + encoding);
+    }
+    
     public static final String[] HTML_BOOLEAN_ATTRS = {
         "checked", "compact", "declare", "defer", "disabled", "ismap",
         "multiple", "nohref", "noresize", "noshade", "nowrap", "readonly",

data/ext/java/nokogiri/internals/XmlDomParserContext.java

@@ -1,7 +1,7 @@
 /**
  * (The MIT License)
  *
- * Copyright (c) 2008 - 2011:
+ * Copyright (c) 2008 - 2012:
  *
  * * {Aaron Patterson}[http://tenderlovemaking.com]
  * * {Mike Dalessio}[http://mike.daless.io]
@@ -33,7 +33,6 @@
 package nokogiri.internals;
 
 import static nokogiri.internals.NokogiriHelpers.getNokogiriClass;
-import static nokogiri.internals.NokogiriHelpers.rubyStringToString;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -88,7 +87,7 @@ public class XmlDomParserContext extends ParserContext {
     public XmlDomParserContext(Ruby runtime, IRubyObject encoding, IRubyObject options) {
         super(runtime);
         this.options = new ParserContext.Options((Long)options.toJava(Long.class));
-        this.java_encoding = encoding.isNil() ? NokogiriHelpers.guessEncoding(runtime) : rubyStringToString(encoding);
+        java_encoding = NokogiriHelpers.getValidEncoding(runtime, encoding);
         ruby_encoding = encoding;
         initErrorHandler();
         initParser(runtime);

data/lib/nokogiri/version.rb

@@ -1,6 +1,6 @@
 module Nokogiri
   # The version of Nokogiri you are using
-  VERSION = '1.5.3.rc5'
+  VERSION = '1.5.3.rc6'
 
   class VersionInfo # :nodoc:
     def jruby?

data/lib/nokogiri/xml/sax/document.rb

@@ -33,7 +33,7 @@ module Nokogiri
     #   parser = Nokogiri::XML::SAX::Parser.new(MyDocument.new)
     #
     #   # Feed the parser some XML
-    #   parser.parse(File.read(ARGV[0], 'rb'))
+    #   parser.parse(File.open(ARGV[0]))
     #
     # Now my document handler will be called when each node starts, and when
     # then document ends.  To see what kinds of events are available, take

data/lib/nokogiri/xml/sax/parser.rb

@@ -25,7 +25,7 @@ module Nokogiri
       #   parser = Nokogiri::XML::SAX::Parser.new(MyDoc.new)
       #
       #   # Send some XML to the parser
-      #   parser.parse(File.read(ARGV[0]))
+      #   parser.parse(File.open(ARGV[0]))
       #
       # For more information about SAX parsers, see Nokogiri::XML::SAX.  Also
       # see Nokogiri::XML::SAX::Document for the available events.

data/test/xml/test_document_encoding.rb

@@ -18,7 +18,9 @@ module Nokogiri
         end
 
         def test_dotted_version
-          assert_equal 'UTF-8', Nokogiri::LIBXML_VERSION.encoding.name
+          if Nokogiri.uses_libxml?
+            assert_equal 'UTF-8', Nokogiri::LIBXML_VERSION.encoding.name
+          end
         end
       end
     end

metadata

@@ -2,7 +2,7 @@
 name: nokogiri
 version: !ruby/object:Gem::Version 
   prerelease: 6
-  version: 1.5.3.rc5
+  version: 1.5.3.rc6
 platform: java
 authors: 
   - Aaron Patterson
@@ -12,7 +12,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-27 00:00:00 Z
+date: 2012-05-30 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: hoe-bundler