nokogiri 1.10.9-java → 1.11.0.rc4-java

data/README.md

@@ -11,7 +11,7 @@ or CSS3 selectors.
 
 * https://nokogiri.org
 * [Installation Help](https://nokogiri.org/tutorials/installing_nokogiri.html)
-* [Tutorials](https://nokogiri.org)
+* [Tutorials](https://nokogiri.org/tutorials/toc.html)
 * [Cheat Sheet](https://github.com/sparklemotion/nokogiri/wiki/Cheat-sheet)
 * [GitHub](https://github.com/sparklemotion/nokogiri)
 * [Mailing List](https://groups.google.com/group/nokogiri-talk)
@@ -20,14 +20,14 @@ or CSS3 selectors.
 
 ## Status
 
-[![Concourse CI](https://ci.nokogiri.org/api/v1/teams/nokogiri-core/pipelines/nokogiri/jobs/ruby-2.4-system/badge)](https://ci.nokogiri.org/teams/nokogiri-core/pipelines/nokogiri)
+[![Concourse CI](https://ci.nokogiri.org/api/v1/teams/nokogiri-core/pipelines/nokogiri/jobs/cruby-2.7/badge)](https://ci.nokogiri.org/teams/nokogiri-core/pipelines/nokogiri)
 [![Appveyor CI](https://ci.appveyor.com/api/projects/status/xj2pqwvlxwuwgr06/branch/master?svg=true)](https://ci.appveyor.com/project/flavorjones/nokogiri/branch/master)
 [![Code Climate](https://codeclimate.com/github/sparklemotion/nokogiri.svg)](https://codeclimate.com/github/sparklemotion/nokogiri)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/59c67b0e8976027a45ad/test_coverage)](https://codeclimate.com/github/sparklemotion/nokogiri/test_coverage)
 
 [![Gem Version](https://badge.fury.io/rb/nokogiri.svg)](https://rubygems.org/gems/nokogiri)
-[![SemVer compatibility](https://api.dependabot.com/badges/compatibility_score?dependency-name=nokogiri&package-manager=bundler&version-scheme=semver)](https://dependabot.com/compatibility-score.html?dependency-name=nokogiri&package-manager=bundler&version-scheme=semver)
-[![Tidelift dependencies](https://tidelift.com/badges/github/sparklemotion/nokogiri)](https://tidelift.com/subscription/pkg/rubygems-nokogiri?utm_source=rubygems-nokogiri&utm_medium=referral&utm_campaign=readme)
+[![SemVer compatibility](https://api.dependabot.com/badges/compatibility_score?dependency-name=nokogiri&package-manager=bundler&version-scheme=semver)](https://dependabot.com/compatibility-score/?dependency-name=nokogiri&package-manager=bundler)
+[![Tidelift dependencies](https://tidelift.com/badges/package/rubygems/nokogiri)](https://tidelift.com/subscription/pkg/rubygems-nokogiri?utm_source=rubygems-nokogiri&utm_medium=referral&utm_campaign=readme)
 
 
 ## Features
@@ -71,7 +71,7 @@ help you directly and improve the documentation.
 Binary packages are available for some distributions.
 
 * Debian: https://packages.debian.org/sid/ruby-nokogiri
-* SuSE: https://download.opensuse.org/repositories/devel:/languages:/ruby:/extensions/
+* openSUSE/SLE: https://download.opensuse.org/repositories/devel:/languages:/ruby:/extensions/
 * Fedora: http://s390.koji.fedoraproject.org/koji/packageinfo?packageID=6756
 
 
@@ -108,7 +108,7 @@ require 'nokogiri'
 require 'open-uri'
 
 # Fetch and parse HTML document
-doc = Nokogiri::HTML(open('https://nokogiri.org/tutorials/installing_nokogiri.html'))
+doc = Nokogiri::HTML(URI.open('https://nokogiri.org/tutorials/installing_nokogiri.html'))
 
 puts "### Search for nodes by css"
 doc.css('nav ul.menu li a', 'article h2').each do |link|
@@ -129,27 +129,23 @@ end
 
 ## Requirements
 
-* Ruby 2.3.0 or higher, including any development packages necessary
-  to compile native extensions.
+Ruby 2.5.0 or higher, including any development packages necessary to compile native extensions.
 
-* In Nokogiri 1.6.0 and later libxml2 and libxslt are bundled with the
-  gem, but if you want to use the system versions:
+In Nokogiri 1.6.0 and later libxml2 and libxslt are bundled with the gem, but if you want to use the system versions:
 
-  * First, check out [the long list](http://www.xmlsoft.org/news.html)
-    of fixes and changes between releases before deciding to use any
-    version older than is bundled with Nokogiri.
+* First, check out [the long list](http://www.xmlsoft.org/news.html)
+  of fixes and changes between releases before deciding to use any
+  version older than is bundled with Nokogiri.
 
-  * At install time, set the environment variable
-    `NOKOGIRI_USE_SYSTEM_LIBRARIES` or else use the
-    `--use-system-libraries` argument. (See
-    https://nokogiri.org/tutorials/installing_nokogiri.html#install-with-system-libraries
-    for specifics.)
+* At install time, set the environment variable
+  `NOKOGIRI_USE_SYSTEM_LIBRARIES` or else use the
+  `--use-system-libraries` argument. (See
+  https://nokogiri.org/tutorials/installing_nokogiri.html#install-with-system-libraries
+  for specifics.)
 
-  * libxml2 >=2.6.21 with iconv support
-    (libxml2-dev/-devel is also required)
+* libxml2 >=2.6.21 with iconv support (libxml2-dev/-devel is also required)
 
-  * libxslt, built with and supported by the given libxml2
-    (libxslt-dev/-devel is also required)
+* libxslt, built with and supported by the given libxml2 (libxslt-dev/-devel is also required)
 
 
 ## Encoding
@@ -191,6 +187,12 @@ explicitly setting the encoding to EUC-JP on the parser:
 We've adopted the Contributor Covenant code of conduct, which you can read in full in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md).
 
 
+## Semantic Versioning
+
+[![SemVer compatibility](https://api.dependabot.com/badges/compatibility_score?dependency-name=nokogiri&package-manager=bundler&version-scheme=semver)](https://dependabot.com/compatibility-score/?dependency-name=nokogiri&package-manager=bundler)
+
+Nokogiri follows [Semantic Versioning](https://semver.org/). See [`CHANGELOG.md`](CHANGELOG.md) for more details.
+
 ## License
 
 This project is licensed under the terms of the MIT license.

data/ext/java/nokogiri/HtmlDocument.java

@@ -36,7 +36,6 @@ import org.jruby.Ruby;
 import org.jruby.RubyClass;
 import org.jruby.anno.JRubyClass;
 import org.jruby.anno.JRubyMethod;
-import org.jruby.runtime.Arity;
 import org.jruby.runtime.Helpers;
 import org.jruby.runtime.ThreadContext;
 import org.jruby.runtime.builtin.IRubyObject;
@@ -48,6 +47,8 @@ import org.w3c.dom.NodeList;
 
 import nokogiri.internals.HtmlDomParserContext;
 
+import static nokogiri.internals.NokogiriHelpers.getNokogiriClass;
+
 /**
  * Class for Nokogiri::HTML::Document.
  *
@@ -65,21 +66,25 @@ public class HtmlDocument extends XmlDocument {
     public HtmlDocument(Ruby ruby, RubyClass klazz) {
         super(ruby, klazz);
     }
-    
+
+    public HtmlDocument(Ruby runtime, Document document) {
+        this(runtime, getNokogiriClass(runtime, "Nokogiri::XML::Document"), document);
+    }
+
     public HtmlDocument(Ruby ruby, RubyClass klazz, Document doc) {
         super(ruby, klazz, doc);
     }
 
     @JRubyMethod(name="new", meta = true, rest = true, required=0)
-    public static IRubyObject rbNew(ThreadContext context, IRubyObject klazz,
-                                    IRubyObject[] args) {
+    public static IRubyObject rbNew(ThreadContext context, IRubyObject klazz, IRubyObject[] args) {
+        final Ruby runtime = context.runtime;
         HtmlDocument htmlDocument;
         try {
-            Document docNode = createNewDocument();
-            htmlDocument = (HtmlDocument) NokogiriService.HTML_DOCUMENT_ALLOCATOR.allocate(context.getRuntime(), (RubyClass) klazz);
-            htmlDocument.setDocumentNode(context, docNode);
+            Document docNode = createNewDocument(runtime);
+            htmlDocument = (HtmlDocument) NokogiriService.HTML_DOCUMENT_ALLOCATOR.allocate(runtime, (RubyClass) klazz);
+            htmlDocument.setDocumentNode(context.runtime, docNode);
         } catch (Exception ex) {
-            throw context.getRuntime().newRuntimeError("couldn't create document: " + ex);
+            throw asRuntimeError(runtime, "couldn't create document: ", ex);
         }
 
         Helpers.invoke(context, htmlDocument, "initialize", args);
@@ -109,46 +114,29 @@ public class HtmlDocument extends XmlDocument {
         return internalSubset;
     }
 
-    public static IRubyObject do_parse(ThreadContext context,
-                                       IRubyObject klass,
-                                       IRubyObject[] args) {
-        Ruby ruby = context.getRuntime();
-        Arity.checkArgumentCount(ruby, args, 4, 4);
-        HtmlDomParserContext ctx =
-            new HtmlDomParserContext(ruby, args[2], args[3]);
-        ctx.setInputSource(context, args[0], args[1]);
-        return ctx.parse(context, klass, args[1]);
-    }
-    
-    public void setDocumentNode(ThreadContext context, Node node) {
-        super.setNode(context, node);
-        Ruby runtime = context.getRuntime();
-        if (node != null) {
-            Document document = (Document)node;
-            document.normalize();
-            stabilzeAttrValue(document.getDocumentElement());
-        }
+    @Override
+    void init(Ruby runtime, Document document) {
+        stabilizeTextContent(document);
+        document.normalize();
         setInstanceVariable("@decorators", runtime.getNil());
+        if (document.getDocumentElement() != null) {
+            stabilizeAttrs(document.getDocumentElement());
+        }
     }
-    
-    private void stabilzeAttrValue(Node node) {
-        if (node == null) return;
+
+    private static void stabilizeAttrs(Node node) {
         if (node.hasAttributes()) {
             NamedNodeMap nodeMap = node.getAttributes();
             for (int i=0; i<nodeMap.getLength(); i++) {
                 Node n = nodeMap.item(i);
                 if (n instanceof Attr) {
-                    Attr attr = (Attr)n;
-                    String attrName = attr.getName();
-                    // not sure, but need to get value always before document is referred.
-                    // or lose attribute value
-                    String attrValue = attr.getValue(); // don't delete this line
+                    stabilizeAttr((Attr) n);
                 }
             }
         }
         NodeList children = node.getChildNodes();
         for (int i=0; i<children.getLength(); i++) {
-            stabilzeAttrValue(children.item(i));
+            stabilizeAttrs(children.item(i));
         }
     }
     
@@ -167,11 +155,11 @@ public class HtmlDocument extends XmlDocument {
      * Read the HTML document from +io+ with given +url+, +encoding+,
      * and +options+.  See Nokogiri::HTML.parse
      */
-    @JRubyMethod(meta = true, rest = true)
-    public static IRubyObject read_io(ThreadContext context,
-                                      IRubyObject cls,
-                                      IRubyObject[] args) {
-        return do_parse(context, cls, args);
+    @JRubyMethod(meta = true, required = 4)
+    public static IRubyObject read_io(ThreadContext context, IRubyObject klass, IRubyObject[] args) {
+        HtmlDomParserContext ctx = new HtmlDomParserContext(context.runtime, args[2], args[3]);
+        ctx.setIOInputSource(context, args[0], args[1]);
+        return ctx.parse(context, (RubyClass) klass, args[1]);
     }
 
     /*
@@ -181,10 +169,10 @@ public class HtmlDocument extends XmlDocument {
      * Read the HTML document contained in +string+ with given +url+, +encoding+,
      * and +options+.  See Nokogiri::HTML.parse
      */
-    @JRubyMethod(meta = true, rest = true)
-    public static IRubyObject read_memory(ThreadContext context,
-                                          IRubyObject cls,
-                                          IRubyObject[] args) {
-        return do_parse(context, cls, args);
+    @JRubyMethod(meta = true, required = 4)
+    public static IRubyObject read_memory(ThreadContext context, IRubyObject klass, IRubyObject[] args) {
+        HtmlDomParserContext ctx = new HtmlDomParserContext(context.runtime, args[2], args[3]);
+        ctx.setStringInputSource(context, args[0], args[1]);
+        return ctx.parse(context, (RubyClass) klass, args[1]);
     }
 }

data/ext/java/nokogiri/HtmlSaxParserContext.java

@@ -32,28 +32,29 @@
 
 package nokogiri;
 
-import static nokogiri.internals.NokogiriHelpers.rubyStringToString;
-
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.UnsupportedCharsetException;
-import java.util.EnumSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import nokogiri.internals.NokogiriHandler;
-
 import org.apache.xerces.parsers.AbstractSAXParser;
 import org.cyberneko.html.parsers.SAXParser;
-import org.jruby.*;
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyFixnum;
+import org.jruby.RubyString;
 import org.jruby.anno.JRubyClass;
 import org.jruby.anno.JRubyMethod;
 import org.jruby.runtime.ThreadContext;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.xml.sax.SAXException;
 
+import nokogiri.internals.NokogiriHandler;
+import static nokogiri.internals.NokogiriHelpers.rubyStringToString;
+
 /**
  * Class for Nokogiri::HTML::SAX::ParserContext.
  *
@@ -65,10 +66,16 @@ import org.xml.sax.SAXException;
 @JRubyClass(name="Nokogiri::HTML::SAX::ParserContext", parent="Nokogiri::XML::SAX::ParserContext")
 public class HtmlSaxParserContext extends XmlSaxParserContext {
 
+    static HtmlSaxParserContext newInstance(final Ruby runtime, final RubyClass klazz) {
+        HtmlSaxParserContext instance = new HtmlSaxParserContext(runtime, klazz);
+        instance.initialize(runtime);
+        return instance;
+    }
+
     public HtmlSaxParserContext(Ruby ruby, RubyClass rubyClass) {
         super(ruby, rubyClass);
     }
-    
+
     @Override
     protected AbstractSAXParser createParser() throws SAXException {
         SAXParser parser = new SAXParser();
@@ -78,6 +85,11 @@ public class HtmlSaxParserContext extends XmlSaxParserContext {
                 "http://cyberneko.org/html/properties/names/elems", "lower");
             parser.setProperty(
                 "http://cyberneko.org/html/properties/names/attrs", "lower");
+
+            // NekoHTML should not try to guess the encoding based on the meta
+            // tags or other information in the document.  This is already
+            // handled by the EncodingReader.
+            parser.setFeature("http://cyberneko.org/html/features/scanner/ignore-specified-charset", true);
             return parser;
         } catch(SAXException ex) {
             throw new SAXException(
@@ -90,18 +102,17 @@ public class HtmlSaxParserContext extends XmlSaxParserContext {
                                            IRubyObject klazz,
                                            IRubyObject data,
                                            IRubyObject encoding) {
-        HtmlSaxParserContext ctx = (HtmlSaxParserContext) NokogiriService.HTML_SAXPARSER_CONTEXT_ALLOCATOR.allocate(context.getRuntime(), (RubyClass)klazz);
-        ctx.initialize(context.getRuntime());
-        String javaEncoding = findEncoding(context, encoding);
+        HtmlSaxParserContext ctx = HtmlSaxParserContext.newInstance(context.runtime, (RubyClass) klazz);
+        String javaEncoding = findEncodingName(context, encoding);
         if (javaEncoding != null) {
-            String input = applyEncoding(rubyStringToString(data), javaEncoding);
-            ByteArrayInputStream istream = new ByteArrayInputStream(input.getBytes());
+            CharSequence input = applyEncoding(rubyStringToString(data.convertToString()), javaEncoding);
+            ByteArrayInputStream istream = new ByteArrayInputStream(input.toString().getBytes());
             ctx.setInputSource(istream);
             ctx.getInputSource().setEncoding(javaEncoding);
         }
         return ctx;
     }
-    
+
     public enum EncodingType {
         NONE(0, "NONE"),
         UTF_8(1, "UTF-8"),
@@ -142,23 +153,38 @@ public class HtmlSaxParserContext extends XmlSaxParserContext {
         public String toString() {
             return name;
         }
-    }
-    
-    private static String findName(final int value) {
-        for (EncodingType type : EncodingType.values()) {
-            if (type.getValue() == value) return type.toString();
+
+        private static transient EncodingType[] values;
+
+        // NOTE: assuming ordinal == value
+        static EncodingType get(final int ordinal) {
+            EncodingType[] values = EncodingType.values;
+            if (values == null) {
+                values = EncodingType.values();
+                EncodingType.values = values;
+            }
+            if (ordinal >= 0 && ordinal < values.length) {
+                return values[ordinal];
+            }
+            return null;
         }
-        return null;
+
     }
-    
-    private static String findEncoding(ThreadContext context, IRubyObject encoding) {
+
+    private static String findEncodingName(final int value) {
+        EncodingType type = EncodingType.get(value);
+        if (type == null) return null;
+        assert type.value == value;
+        return type.name;
+    }
+
+    private static String findEncodingName(ThreadContext context, IRubyObject encoding) {
         String rubyEncoding = null;
         if (encoding instanceof RubyString) {
-            rubyEncoding = rubyStringToString(encoding);
+            rubyEncoding = rubyStringToString((RubyString) encoding);
         }
         else if (encoding instanceof RubyFixnum) {
-            int value = RubyFixnum.fix2int((RubyFixnum) encoding);
-            rubyEncoding = findName(value);
+            rubyEncoding = findEncodingName(RubyFixnum.fix2int((RubyFixnum) encoding));
         }
         if (rubyEncoding == null) return null;
         try {
@@ -168,39 +194,56 @@ public class HtmlSaxParserContext extends XmlSaxParserContext {
             throw context.getRuntime().newEncodingCompatibilityError(rubyEncoding + "is not supported");
         }
         catch (IllegalCharsetNameException e) {
-            throw context.getRuntime().newInvalidEncoding(e.getMessage());
+            throw context.getRuntime().newEncodingError(e.getMessage());
         }
     }
 
-    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset(()|\\s)=(()|\\s)([a-z]|-|_|\\d)+");
+    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset(()|\\s)=(()|\\s)([a-z]|-|_|\\d)+", Pattern.CASE_INSENSITIVE);
 
-    private static String applyEncoding(String input, String enc) {
-        String str = input.toLowerCase();
-        int start_pos = 0;
-        int end_pos = 0;
-        if (input.contains("meta") && input.contains("charset")) {
-            Matcher m = CHARSET_PATTERN.matcher(str);
+    private static CharSequence applyEncoding(final String input, final String enc) {
+        int start_pos = 0; int end_pos = 0;
+        if (containsIgnoreCase(input, "charset")) {
+            Matcher m = CHARSET_PATTERN.matcher(input);
             while (m.find()) {
                 start_pos = m.start();
                 end_pos = m.end();
             }
         }
         if (start_pos != end_pos) {
-            String substr = input.substring(start_pos, end_pos);
-            input = input.replace(substr, "charset=" + enc);
+            return new StringBuilder(input).replace(start_pos, end_pos, "charset=" + enc);
         }
         return input;
     }
 
+    private static boolean containsIgnoreCase(final String str, final String sub) {
+        final int len = sub.length();
+        final int max = str.length() - len;
+
+        if (len == 0) return true;
+        final char c0Lower = Character.toLowerCase(sub.charAt(0));
+        final char c0Upper = Character.toUpperCase(sub.charAt(0));
+
+        for (int i = 0; i <= max; i++) {
+            final char ch = str.charAt(i);
+            if (ch != c0Lower && Character.toLowerCase(ch) != c0Lower && Character.toUpperCase(ch) != c0Upper) {
+                continue; // first char doesn't match
+            }
+
+            if (str.regionMatches(true, i + 1, sub, 0 + 1, len - 1)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @JRubyMethod(name="file", meta=true)
     public static IRubyObject parse_file(ThreadContext context,
-                                         IRubyObject klazz,
+                                         IRubyObject klass,
                                          IRubyObject data,
                                          IRubyObject encoding) {
-        HtmlSaxParserContext ctx = (HtmlSaxParserContext) NokogiriService.HTML_SAXPARSER_CONTEXT_ALLOCATOR.allocate(context.getRuntime(), (RubyClass)klazz);
-        ctx.initialize(context.getRuntime());
+        HtmlSaxParserContext ctx = HtmlSaxParserContext.newInstance(context.runtime, (RubyClass) klass);
         ctx.setInputSourceFile(context, data);
-        String javaEncoding = findEncoding(context, encoding);
+        String javaEncoding = findEncodingName(context, encoding);
         if (javaEncoding != null) {
             ctx.getInputSource().setEncoding(javaEncoding);
         }
@@ -209,13 +252,12 @@ public class HtmlSaxParserContext extends XmlSaxParserContext {
 
     @JRubyMethod(name="io", meta=true)
     public static IRubyObject parse_io(ThreadContext context,
-                                       IRubyObject klazz,
+                                       IRubyObject klass,
                                        IRubyObject data,
                                        IRubyObject encoding) {
-        HtmlSaxParserContext ctx = (HtmlSaxParserContext) NokogiriService.HTML_SAXPARSER_CONTEXT_ALLOCATOR.allocate(context.getRuntime(), (RubyClass)klazz);
-        ctx.initialize(context.getRuntime());
-        ctx.setInputSource(context, data, context.getRuntime().getNil());
-        String javaEncoding = findEncoding(context, encoding);
+        HtmlSaxParserContext ctx = HtmlSaxParserContext.newInstance(context.runtime, (RubyClass) klass);
+        ctx.setIOInputSource(context, data, context.nil);
+        String javaEncoding = findEncodingName(context, encoding);
         if (javaEncoding != null) {
             ctx.getInputSource().setEncoding(javaEncoding);
         }
@@ -226,27 +268,15 @@ public class HtmlSaxParserContext extends XmlSaxParserContext {
      * Create a new parser context that will read from a raw input stream.
      * Meant to be run in a separate thread by HtmlSaxPushParser.
      */
-    static HtmlSaxParserContext parse_stream(final Ruby runtime, RubyClass klazz, InputStream stream) {
-        HtmlSaxParserContext ctx = (HtmlSaxParserContext) NokogiriService.HTML_SAXPARSER_CONTEXT_ALLOCATOR.allocate(runtime, klazz);
-        ctx.initialize(runtime);
+    static HtmlSaxParserContext parse_stream(final Ruby runtime, RubyClass klass, InputStream stream) {
+        HtmlSaxParserContext ctx = HtmlSaxParserContext.newInstance(runtime, klass);
         ctx.setInputSource(stream);
         return ctx;
     }
 
     @Override
     protected void preParse(final Ruby runtime, IRubyObject handlerRuby, NokogiriHandler handler) {
-        // final String path = "Nokogiri::XML::FragmentHandler";
-        // final String docFrag =
-        //     "http://cyberneko.org/html/features/balance-tags/document-fragment";
-        // RubyObjectAdapter adapter = JavaEmbedUtils.newObjectAdapter();
-        // IRubyObject doc = adapter.getInstanceVariable(handlerRuby, "@document");
-        // RubyModule mod = runtime.getClassFromPath(path);
-        // try {
-        //     if (doc != null && !doc.isNil() && adapter.isKindOf(doc, mod))
-        //         parser.setFeature(docFrag, true);
-        // } catch (Exception e) {
-        //     // ignore
-        // }
+        // this function is meant to be empty.  It overrides the one in XmlSaxParserContext
     }
 
 }