nokogiri 1.13.8 → 1.15.7

data/lib/nokogiri/html5.rb

@@ -227,250 +227,164 @@ module Nokogiri
   #
   # Since v1.12.0
   module HTML5
-    # HTML uses the XHTML namespace.
-    HTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
-    MATHML_NAMESPACE = "http://www.w3.org/1998/Math/MathML"
-    SVG_NAMESPACE = "http://www.w3.org/2000/svg"
-    XLINK_NAMESPACE = "http://www.w3.org/1999/xlink"
-    XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
-    XMLNS_NAMESPACE = "http://www.w3.org/2000/xmlns/"
+    class << self
+      # Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
+      def parse(string, url = nil, encoding = nil, **options, &block)
+        Document.parse(string, url, encoding, **options, &block)
+      end
 
-    # Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
-    def self.parse(string, url = nil, encoding = nil, **options, &block)
-      Document.parse(string, url, encoding, **options, &block)
-    end
+      # Parse a fragment from +string+. Convenience method for
+      # {Nokogiri::HTML5::DocumentFragment.parse}.
+      def fragment(string, encoding = nil, **options)
+        DocumentFragment.parse(string, encoding, options)
+      end
 
-    # Parse a fragment from +string+. Convenience method for
-    # {Nokogiri::HTML5::DocumentFragment.parse}.
-    def self.fragment(string, encoding = nil, **options)
-      DocumentFragment.parse(string, encoding, options)
-    end
+      # Fetch and parse a HTML document from the web, following redirects,
+      # handling https, and determining the character encoding using HTML5
+      # rules.  +uri+ may be a +String+ or a +URI+.  +options+ contains
+      # http headers and special options.  Everything which is not a
+      # special option is considered a header.  Special options include:
+      #  * :follow_limit => number of redirects which are followed
+      #  * :basic_auth => [username, password]
+      def get(uri, options = {})
+        # TODO: deprecate
+        warn(
+          "Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.",
+          uplevel: 1,
+          category: :deprecated,
+        )
+        get_impl(uri, options)
+      end
 
-    # Fetch and parse a HTML document from the web, following redirects,
-    # handling https, and determining the character encoding using HTML5
-    # rules.  +uri+ may be a +String+ or a +URI+.  +options+ contains
-    # http headers and special options.  Everything which is not a
-    # special option is considered a header.  Special options include:
-    #  * :follow_limit => number of redirects which are followed
-    #  * :basic_auth => [username, password]
-    def self.get(uri, options = {})
-      # TODO: deprecate
-      warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.",
-        uplevel: 1, category: :deprecated)
-      get_impl(uri, options)
-    end
+      # :nodoc:
+      def read_and_encode(string, encoding)
+        # Read the string with the given encoding.
+        if string.respond_to?(:read)
+          string = if encoding.nil?
+            string.read
+          else
+            string.read(encoding: encoding)
+          end
+        else
+          # Otherwise the string has the given encoding.
+          string = string.to_s
+          if encoding
+            string = string.dup
+            string.force_encoding(encoding)
+          end
+        end
 
-    private
+        # convert to UTF-8
+        if string.encoding != Encoding::UTF_8
+          string = reencode(string)
+        end
+        string
+      end
 
-    def self.get_impl(uri, options = {})
-      headers = options.clone
-      headers = { follow_limit: headers } if Numeric === headers # deprecated
-      limit = headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
+      private
 
-      require "net/http"
-      uri = URI(uri) unless URI === uri
+      def get_impl(uri, options = {})
+        headers = options.clone
+        headers = { follow_limit: headers } if Numeric === headers # deprecated
+        limit = headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
 
-      http = Net::HTTP.new(uri.host, uri.port)
+        require "net/http"
+        uri = URI(uri) unless URI === uri
 
-      # TLS / SSL support
-      http.use_ssl = true if uri.scheme == "https"
+        http = Net::HTTP.new(uri.host, uri.port)
 
-      # Pass through Net::HTTP override values, which currently include:
-      #   :ca_file, :ca_path, :cert, :cert_store, :ciphers,
-      #   :close_on_empty_response, :continue_timeout, :key, :open_timeout,
-      #   :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
-      #   :verify_callback, :verify_depth, :verify_mode
-      options.each do |key, _value|
-        http.send("#{key}=", headers.delete(key)) if http.respond_to?("#{key}=")
-      end
+        # TLS / SSL support
+        http.use_ssl = true if uri.scheme == "https"
 
-      request = Net::HTTP::Get.new(uri.request_uri)
+        # Pass through Net::HTTP override values, which currently include:
+        #   :ca_file, :ca_path, :cert, :cert_store, :ciphers,
+        #   :close_on_empty_response, :continue_timeout, :key, :open_timeout,
+        #   :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
+        #   :verify_callback, :verify_depth, :verify_mode
+        options.each do |key, _value|
+          http.send("#{key}=", headers.delete(key)) if http.respond_to?("#{key}=")
+        end
 
-      # basic authentication
-      auth = headers.delete(:basic_auth)
-      auth ||= [uri.user, uri.password] if uri.user && uri.password
-      request.basic_auth(auth.first, auth.last) if auth
+        request = Net::HTTP::Get.new(uri.request_uri)
 
-      # remaining options are treated as headers
-      headers.each { |key, value| request[key.to_s] = value.to_s }
+        # basic authentication
+        auth = headers.delete(:basic_auth)
+        auth ||= [uri.user, uri.password] if uri.user && uri.password
+        request.basic_auth(auth.first, auth.last) if auth
 
-      response = http.request(request)
+        # remaining options are treated as headers
+        headers.each { |key, value| request[key.to_s] = value.to_s }
 
-      case response
-      when Net::HTTPSuccess
-        doc = parse(reencode(response.body, response["content-type"]), options)
-        doc.instance_variable_set("@response", response)
-        doc.class.send(:attr_reader, :response)
-        doc
-      when Net::HTTPRedirection
-        response.value if limit <= 1
-        location = URI.join(uri, response["location"])
-        get_impl(location, options.merge(follow_limit: limit - 1))
-      else
-        response.value
-      end
-    end
+        response = http.request(request)
 
-    def self.read_and_encode(string, encoding)
-      # Read the string with the given encoding.
-      if string.respond_to?(:read)
-        string = if encoding.nil?
-          string.read
+        case response
+        when Net::HTTPSuccess
+          doc = parse(reencode(response.body, response["content-type"]), options)
+          doc.instance_variable_set(:@response, response)
+          doc.class.send(:attr_reader, :response)
+          doc
+        when Net::HTTPRedirection
+          response.value if limit <= 1
+          location = URI.join(uri, response["location"])
+          get_impl(location, options.merge(follow_limit: limit - 1))
         else
-          string.read(encoding: encoding)
-        end
-      else
-        # Otherwise the string has the given encoding.
-        string = string.to_s
-        if encoding
-          string = string.dup
-          string.force_encoding(encoding)
+          response.value
         end
       end
 
-      # convert to UTF-8
-      if string.encoding != Encoding::UTF_8
-        string = reencode(string)
-      end
-      string
-    end
-
-    # Charset sniffing is a complex and controversial topic that understandably isn't done _by
-    # default_ by the Ruby Net::HTTP library.  This being said, it is a very real problem for
-    # consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
-    # the Gumbo parser *only* supports utf-8.
-    #
-    # Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection.  Following
-    # this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
-    # the HTML5 standard.
-    #
-    # http://bugs.ruby-lang.org/issues/2567
-    # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
-    #
-    def self.reencode(body, content_type = nil)
-      if body.encoding == Encoding::ASCII_8BIT
-        encoding = nil
-
-        # look for a Byte Order Mark (BOM)
-        initial_bytes = body[0..2].bytes
-        if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
-          encoding = Encoding::UTF_8
-        elsif initial_bytes[0..1] == [0xFE, 0xFF]
-          encoding = Encoding::UTF_16BE
-        elsif initial_bytes[0..1] == [0xFF, 0xFE]
-          encoding = Encoding::UTF_16LE
-        end
-
-        # look for a charset in a content-encoding header
-        if content_type
-          encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
-        end
-
-        # look for a charset in a meta tag in the first 1024 bytes
-        unless encoding
-          data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, "")
-          data.scan(/<meta.*?>/m).each do |meta|
-            encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
+      # Charset sniffing is a complex and controversial topic that understandably isn't done _by
+      # default_ by the Ruby Net::HTTP library.  This being said, it is a very real problem for
+      # consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
+      # the Gumbo parser *only* supports utf-8.
+      #
+      # Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection.  Following
+      # this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
+      # the HTML5 standard.
+      #
+      # http://bugs.ruby-lang.org/issues/2567
+      # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
+      #
+      def reencode(body, content_type = nil)
+        if body.encoding == Encoding::ASCII_8BIT
+          encoding = nil
+
+          # look for a Byte Order Mark (BOM)
+          initial_bytes = body[0..2].bytes
+          if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
+            encoding = Encoding::UTF_8
+          elsif initial_bytes[0..1] == [0xFE, 0xFF]
+            encoding = Encoding::UTF_16BE
+          elsif initial_bytes[0..1] == [0xFF, 0xFE]
+            encoding = Encoding::UTF_16LE
           end
-        end
-
-        # if all else fails, default to the official default encoding for HTML
-        encoding ||= Encoding::ISO_8859_1
-
-        # change the encoding to match the detected or inferred encoding
-        body = body.dup
-        begin
-          body.force_encoding(encoding)
-        rescue ArgumentError
-          body.force_encoding(Encoding::ISO_8859_1)
-        end
-      end
 
-      body.encode(Encoding::UTF_8)
-    end
+          # look for a charset in a content-encoding header
+          if content_type
+            encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
+          end
 
-    def self.serialize_node_internal(current_node, io, encoding, options)
-      case current_node.type
-      when XML::Node::ELEMENT_NODE
-        ns = current_node.namespace
-        ns_uri = ns.nil? ? nil : ns.href
-        # XXX(sfc): attach namespaces to all nodes, even html?
-        tagname = if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE
-          current_node.name
-        else
-          "#{ns.prefix}:#{current_node.name}"
-        end
-        io << "<" << tagname
-        current_node.attribute_nodes.each do |attr|
-          attr_ns = attr.namespace
-          if attr_ns.nil?
-            attr_name = attr.name
-          else
-            ns_uri = attr_ns.href
-            attr_name = if ns_uri == XML_NAMESPACE
-              "xml:" + attr.name.sub(/^[^:]*:/, "")
-            elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, "") == "xmlns"
-              "xmlns"
-            elsif ns_uri == XMLNS_NAMESPACE
-              "xmlns:" + attr.name.sub(/^[^:]*:/, "")
-            elsif ns_uri == XLINK_NAMESPACE
-              "xlink:" + attr.name.sub(/^[^:]*:/, "")
-            else
-              "#{attr_ns.prefix}:#{attr.name}"
+          # look for a charset in a meta tag in the first 1024 bytes
+          unless encoding
+            data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, "")
+            data.scan(/<meta.*?>/im).each do |meta|
+              encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
             end
           end
-          io << " " << attr_name << '="' << escape_text(attr.content, encoding, true) << '"'
-        end
-        io << ">"
-        unless ["area", "base", "basefont", "bgsound", "br", "col", "embed", "frame", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"].include?(current_node.name)
-          io << "\n" if options[:preserve_newline] && prepend_newline?(current_node)
-          current_node.children.each do |child|
-            # XXX(sfc): Templates handled specially?
-            serialize_node_internal(child, io, encoding, options)
+
+          # if all else fails, default to the official default encoding for HTML
+          encoding ||= Encoding::ISO_8859_1
+
+          # change the encoding to match the detected or inferred encoding
+          body = body.dup
+          begin
+            body.force_encoding(encoding)
+          rescue ArgumentError
+            body.force_encoding(Encoding::ISO_8859_1)
           end
-          io << "</" << tagname << ">"
-        end
-      when XML::Node::TEXT_NODE
-        parent = current_node.parent
-        io << if parent.element? && ["style", "script", "xmp", "iframe", "noembed", "noframes", "plaintext", "noscript"].include?(parent.name)
-          current_node.content
-        else
-          escape_text(current_node.content, encoding, false)
-        end
-      when XML::Node::CDATA_SECTION_NODE
-        io << "<![CDATA[" << current_node.content << "]]>"
-      when XML::Node::COMMENT_NODE
-        io << "<!--" << current_node.content << "-->"
-      when XML::Node::PI_NODE
-        io << "<?" << current_node.content << ">"
-      when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE
-        io << "<!DOCTYPE " << current_node.name << ">"
-      when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE
-        current_node.children.each do |child|
-          serialize_node_internal(child, io, encoding, options)
         end
-      else
-        raise "Unexpected node '#{current_node.name}' of type #{current_node.type}"
-      end
-    end
 
-    def self.escape_text(text, encoding, attribute_mode)
-      text = if attribute_mode
-        text.gsub(/[&\u00a0"]/,
-          "&" => "&amp;", "\u00a0" => "&nbsp;", '"' => "&quot;")
-      else
-        text.gsub(/[&\u00a0<>]/,
-          "&" => "&amp;", "\u00a0" => "&nbsp;", "<" => "&lt;", ">" => "&gt;")
+        body.encode(Encoding::UTF_8)
       end
-      # Not part of the standard
-      text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" })
-    end
-
-    def self.prepend_newline?(node)
-      return false unless ["pre", "textarea", "listing"].include?(node.name) && !node.children.empty?
-
-      first_child = node.children[0]
-      first_child.text? && first_child.content.start_with?("\n")
     end
   end
 end

data/lib/nokogiri/jruby/dependencies.rb

@@ -1,21 +1,3 @@
 # frozen_string_literal: true
 
-# The line below caused a problem on non-GAE rack environment.
-# unless defined?(JRuby::Rack::VERSION) || defined?(AppEngine::ApiProxy)
-#
-# However, simply cutting defined?(JRuby::Rack::VERSION) off resulted in
-# an unable-to-load-nokogiri problem. Thus, now, Nokogiri checks the presense
-# of appengine-rack.jar in $LOAD_PATH. If Nokogiri is on GAE, Nokogiri
-# should skip loading xml jars. This is because those are in WEB-INF/lib and
-# already set in the classpath.
-unless $LOAD_PATH.to_s.include?("appengine-rack")
-  require "stringio"
-  require "isorelax.jar"
-  require "jing.jar"
-  require "nekohtml.jar"
-  require "nekodtd.jar"
-  require "xercesImpl.jar"
-  require "serializer.jar"
-  require "xalan.jar"
-  require "xml-apis.jar"
-end
+require_relative "nokogiri_jars"

data/lib/nokogiri/jruby/nokogiri_jars.rb

@@ -0,0 +1,43 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'xalan/serializer/2.7.3/serializer-2.7.3.jar'
+  require 'net/sourceforge/htmlunit/neko-htmlunit/2.63.0/neko-htmlunit-2.63.0.jar'
+  require 'nu/validator/jing/20200702VNU/jing-20200702VNU.jar'
+  require 'xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar'
+  require 'net/sf/saxon/Saxon-HE/9.6.0-4/Saxon-HE-9.6.0-4.jar'
+  require 'xalan/xalan/2.7.3/xalan-2.7.3.jar'
+  require 'xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar'
+  require 'org/nokogiri/nekodtd/0.1.11.noko2/nekodtd-0.1.11.noko2.jar'
+  require 'isorelax/isorelax/20030108/isorelax-20030108.jar'
+end
+
+if defined? Jars
+  require_jar 'xalan', 'serializer', '2.7.3'
+  require_jar 'net.sourceforge.htmlunit', 'neko-htmlunit', '2.63.0'
+  require_jar 'nu.validator', 'jing', '20200702VNU'
+  require_jar 'xerces', 'xercesImpl', '2.12.2'
+  require_jar 'net.sf.saxon', 'Saxon-HE', '9.6.0-4'
+  require_jar 'xalan', 'xalan', '2.7.3'
+  require_jar 'xml-apis', 'xml-apis', '1.4.01'
+  require_jar 'org.nokogiri', 'nekodtd', '0.1.11.noko2'
+  require_jar 'isorelax', 'isorelax', '20030108'
+end
+
+module Nokogiri
+  # generated by the :vendor_jars rake task
+  JAR_DEPENDENCIES = {
+    "isorelax:isorelax" => "20030108",
+    "net.sf.saxon:Saxon-HE" => "9.6.0-4",
+    "net.sourceforge.htmlunit:neko-htmlunit" => "2.63.0",
+    "nu.validator:jing" => "20200702VNU",
+    "org.nokogiri:nekodtd" => "0.1.11.noko2",
+    "xalan:serializer" => "2.7.3",
+    "xalan:xalan" => "2.7.3",
+    "xerces:xercesImpl" => "2.12.2",
+    "xml-apis:xml-apis" => "1.4.01",
+  }.freeze
+  XERCES_VERSION = JAR_DEPENDENCIES["xerces:xercesImpl"]
+  NEKO_VERSION = JAR_DEPENDENCIES["net.sourceforge.htmlunit:neko-htmlunit"]
+end

data/lib/nokogiri/version/constant.rb

@@ -2,5 +2,5 @@
 
 module Nokogiri
   # The version of Nokogiri you are using
-  VERSION = "1.13.8"
+  VERSION = "1.15.7"
 end

data/lib/nokogiri/version/info.rb

@@ -105,16 +105,16 @@ module Nokogiri
             if libxml2_using_packaged?
               cppflags << "-I#{File.join(header_directory, "include").shellescape}"
               cppflags << "-I#{File.join(header_directory, "include/libxml2").shellescape}"
+            end
 
-              if windows?
-                # on windows, nokogumbo needs to link against nokogiri.so to resolve symbols. see #2167
-                lib_directory = File.expand_path(File.join(File.dirname(__FILE__), "../#{ruby_minor}"))
-                unless File.exist?(lib_directory)
-                  lib_directory = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-                end
-                ldflags << "-L#{lib_directory.shellescape}"
-                ldflags << "-l:nokogiri.so"
+            if windows?
+              # on windows, nokogumbo needs to link against nokogiri.so to resolve symbols. see #2167
+              lib_directory = File.expand_path(File.join(File.dirname(__FILE__), "../#{ruby_minor}"))
+              unless File.exist?(lib_directory)
+                lib_directory = File.expand_path(File.join(File.dirname(__FILE__), ".."))
               end
+              ldflags << "-L#{lib_directory.shellescape}"
+              ldflags << "-l:nokogiri.so"
             end
 
             nokogiri["cppflags"] = cppflags
@@ -169,8 +169,9 @@ module Nokogiri
           vi["other_libraries"] = Hash[*Nokogiri::OTHER_LIBRARY_VERSIONS.split(/[,:]/)]
         elsif jruby?
           vi["other_libraries"] = {}.tap do |ol|
-            ol["xerces"] = Nokogiri::XERCES_VERSION
-            ol["nekohtml"] = Nokogiri::NEKO_VERSION
+            Nokogiri::JAR_DEPENDENCIES.each do |k, v|
+              ol[k] = v
+            end
           end
         end
       end

data/lib/nokogiri/xml/attr.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 # frozen_string_literal: true
 
 module Nokogiri
@@ -7,6 +8,54 @@ module Nokogiri
       alias_method :to_s, :content
       alias_method :content=, :value=
 
+      #
+      #  :call-seq: deconstruct_keys(array_of_names) → Hash
+      #
+      #  Returns a hash describing the Attr, to use in pattern matching.
+      #
+      #  Valid keys and their values:
+      #  - +name+ → (String) The name of the attribute.
+      #  - +value+ → (String) The value of the attribute.
+      #  - +namespace+ → (Namespace, nil) The Namespace of the attribute, or +nil+ if there is no namespace.
+      #
+      #  ⚡ This is an experimental feature, available since v1.14.0
+      #
+      #  *Example*
+      #
+      #    doc = Nokogiri::XML.parse(<<~XML)
+      #      <?xml version="1.0"?>
+      #      <root xmlns="http://nokogiri.org/ns/default" xmlns:noko="http://nokogiri.org/ns/noko">
+      #        <child1 foo="abc" noko:bar="def"/>
+      #      </root>
+      #    XML
+      #
+      #    attributes = doc.root.elements.first.attribute_nodes
+      #    # => [#(Attr:0x35c { name = "foo", value = "abc" }),
+      #    #     #(Attr:0x370 {
+      #    #       name = "bar",
+      #    #       namespace = #(Namespace:0x384 {
+      #    #         prefix = "noko",
+      #    #         href = "http://nokogiri.org/ns/noko"
+      #    #         }),
+      #    #       value = "def"
+      #    #       })]
+      #
+      #    attributes.first.deconstruct_keys([:name, :value, :namespace])
+      #    # => {:name=>"foo", :value=>"abc", :namespace=>nil}
+      #
+      #    attributes.last.deconstruct_keys([:name, :value, :namespace])
+      #    # => {:name=>"bar",
+      #    #     :value=>"def",
+      #    #     :namespace=>
+      #    #      #(Namespace:0x384 {
+      #    #        prefix = "noko",
+      #    #        href = "http://nokogiri.org/ns/noko"
+      #    #        })}
+      #
+      def deconstruct_keys(keys)
+        { name: name, value: value, namespace: namespace }
+      end
+
       private
 
       def inspect_attributes

data/lib/nokogiri/xml/attribute_decl.rb

@@ -12,8 +12,10 @@ module Nokogiri
       undef_method :namespace_definitions
       undef_method :line if method_defined?(:line)
 
-      def inspect
-        "#<#{self.class.name}:#{format("0x%x", object_id)} #{to_s.inspect}>"
+      private
+
+      def inspect_attributes
+        [:to_s]
       end
     end
   end

data/lib/nokogiri/xml/builder.rb

@@ -234,7 +234,7 @@ module Nokogiri
     #
     # == Document Types
     #
-    # To create a document type (DTD), access use the Builder#doc method to get
+    # To create a document type (DTD), use the Builder#doc method to get
     # the current context document.  Then call Node#create_internal_subset to
     # create the DTD node.
     #