nokogiri 1.11.7 → 1.12.0.rc1

data/lib/nokogiri/html4.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+module Nokogiri
+  class << self
+    ###
+    # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse
+    def HTML4(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
+      Nokogiri::HTML4::Document.parse(input, url, encoding, options, &block)
+    end
+  end
+
+  # @since v1.12.0
+  # @note Before v1.12.0, {Nokogiri::HTML4} did not exist, and {Nokogiri::HTML} was the module/namespace for parsing HTML.
+  module HTML4
+    class << self
+      ###
+      # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse
+      def parse(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
+        Document.parse(input, url, encoding, options, &block)
+      end
+
+      ####
+      # Parse a fragment from +string+ in to a NodeSet.
+      def fragment(string, encoding = nil)
+        HTML4::DocumentFragment.parse(string, encoding)
+      end
+    end
+
+    # Instance of Nokogiri::HTML4::EntityLookup
+    NamedCharacters = EntityLookup.new
+  end
+end
+
+require_relative "html4/entity_lookup"
+require_relative "html4/document"
+require_relative "html4/document_fragment"
+require_relative "html4/sax/parser_context"
+require_relative "html4/sax/parser"
+require_relative "html4/sax/push_parser"
+require_relative "html4/element_description"
+require_relative "html4/element_description_defaults"

data/lib/nokogiri/{html → html4}/builder.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     ###
     # Nokogiri HTML builder is used for building HTML documents.  It is very
     # similar to the Nokogiri::XML::Builder.  In fact, you should go read the
@@ -12,7 +12,7 @@ module Nokogiri
     # Create an HTML document with a body that has an onload attribute, and a
     # span tag with a class of "bold" that has content of "Hello world".
     #
-    #   builder = Nokogiri::HTML::Builder.new do |doc|
+    #   builder = Nokogiri::HTML4::Builder.new do |doc|
     #     doc.html {
     #       doc.body(:onload => 'some_func();') {
     #         doc.span.bold {

data/lib/nokogiri/{html → html4}/document.rb

@@ -3,7 +3,7 @@
 require 'pathname'
 
 module Nokogiri
-  module HTML
+  module HTML4
     class Document < Nokogiri::XML::Document
       ###
       # Get the meta tag encoding for this document.  If there is no meta tag,
@@ -268,12 +268,12 @@ module Nokogiri
             m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
               return m[4]
             catch(:encoding_found) {
-              Nokogiri::HTML::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+              Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
               nil
             }
           else
             handler = SAXHandler.new
-            parser = Nokogiri::HTML::SAX::PushParser.new(handler)
+            parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
             parser << chunk rescue Nokogiri::SyntaxError
             handler.encoding
           end
@@ -286,7 +286,7 @@ module Nokogiri
         end
 
         # This method is used by the C extension so that
-        # Nokogiri::HTML::Document#read_io() does not leak memory when
+        # Nokogiri::HTML4::Document#read_io() does not leak memory when
         # EncodingFound is raised.
         attr_reader :encoding_found
 

data/lib/nokogiri/{html → html4}/document_fragment.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     class DocumentFragment < Nokogiri::XML::DocumentFragment
       ####
       # Create a Nokogiri::XML::DocumentFragment from +tags+, using +encoding+
       def self.parse(tags, encoding = nil)
-        doc = HTML::Document.new
+        doc = HTML4::Document.new
 
         encoding ||= if tags.respond_to?(:encoding)
           encoding = tags.encoding
@@ -39,7 +39,7 @@ module Nokogiri
             "/html/body/node()"
           end
 
-          temp_doc = HTML::Document.parse("<html><body>#{tags}", nil, document.encoding)
+          temp_doc = HTML4::Document.parse("<html><body>#{tags}", nil, document.encoding)
           temp_doc.xpath(path).each { |child| child.parent = self }
           self.errors = temp_doc.errors
         end

data/lib/nokogiri/{html → html4}/element_description.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     class ElementDescription
       ###
       # Is this element a block element?

data/lib/nokogiri/{html → html4}/element_description_defaults.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     class ElementDescription
 
       # Methods are defined protected by method_defined? because at

data/lib/nokogiri/{html → html4}/entity_lookup.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     class EntityDescription < Struct.new(:value, :name, :description); end
 
     class EntityLookup

data/lib/nokogiri/{html → html4}/sax/parser.rb

@@ -1,18 +1,15 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     ###
-    # Nokogiri lets you write a SAX parser to process HTML but get HTML
-    # correction features.
+    # Nokogiri lets you write a SAX parser to process HTML but get HTML correction features.
     #
-    # See Nokogiri::HTML::SAX::Parser for a basic example of using a
-    # SAX parser with HTML.
+    # See Nokogiri::HTML4::SAX::Parser for a basic example of using a SAX parser with HTML.
     #
     # For more information on SAX parsers, see Nokogiri::XML::SAX
     module SAX
       ###
-      # This class lets you perform SAX style parsing on HTML with HTML
-      # error correction.
+      # This class lets you perform SAX style parsing on HTML with HTML error correction.
       #
       # Here is a basic usage example:
       #
@@ -22,40 +19,40 @@ module Nokogiri
       #     end
       #   end
       #
-      #   parser = Nokogiri::HTML::SAX::Parser.new(MyDoc.new)
+      #   parser = Nokogiri::HTML4::SAX::Parser.new(MyDoc.new)
       #   parser.parse(File.read(ARGV[0], mode: 'rb'))
       #
       # For more information on SAX parsers, see Nokogiri::XML::SAX
       class Parser < Nokogiri::XML::SAX::Parser
         ###
         # Parse html stored in +data+ using +encoding+
-        def parse_memory data, encoding = 'UTF-8'
+        def parse_memory(data, encoding = "UTF-8")
           raise ArgumentError unless data
           return unless data.length > 0
           ctx = ParserContext.memory(data, encoding)
           yield ctx if block_given?
-          ctx.parse_with self
+          ctx.parse_with(self)
         end
 
         ###
         # Parse given +io+
-        def parse_io io, encoding = 'UTF-8'
+        def parse_io(io, encoding = "UTF-8")
           check_encoding(encoding)
           @encoding = encoding
           ctx = ParserContext.io(io, ENCODINGS[encoding])
           yield ctx if block_given?
-          ctx.parse_with self
+          ctx.parse_with(self)
         end
 
         ###
         # Parse a file with +filename+
-        def parse_file filename, encoding = 'UTF-8'
+        def parse_file(filename, encoding = "UTF-8")
           raise ArgumentError unless filename
           raise Errno::ENOENT unless File.exist?(filename)
           raise Errno::EISDIR if File.directory?(filename)
           ctx = ParserContext.file(filename, encoding)
           yield ctx if block_given?
-          ctx.parse_with self
+          ctx.parse_with(self)
         end
       end
     end

data/lib/nokogiri/html4/sax/parser_context.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+module Nokogiri
+  module HTML4
+    module SAX
+      ###
+      # Context for HTML SAX parsers. This class is usually not instantiated by the user. Instead,
+      # you should be looking at Nokogiri::HTML4::SAX::Parser
+      class ParserContext < Nokogiri::XML::SAX::ParserContext
+        def self.new(thing, encoding = "UTF-8")
+          if [:read, :close].all? { |x| thing.respond_to?(x) }
+            super
+          else
+            memory(thing, encoding)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/{html → html4}/sax/push_parser.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 module Nokogiri
-  module HTML
+  module HTML4
     module SAX
       class PushParser
         
-        # The Nokogiri::HTML::SAX::Document on which the PushParser will be
+        # The Nokogiri::HTML4::SAX::Document on which the PushParser will be
         # operating
         attr_accessor :document
         
-        def initialize(doc = HTML::SAX::Document.new, file_name = nil, encoding = 'UTF-8')
+        def initialize(doc = HTML4::SAX::Document.new, file_name = nil, encoding = 'UTF-8')
           @document = doc
           @encoding = encoding
-          @sax_parser = HTML::SAX::Parser.new(doc, @encoding)
+          @sax_parser = HTML4::SAX::Parser.new(doc, @encoding)
 
           ## Create our push parser context
           initialize_native(@sax_parser, file_name, encoding)
@@ -27,7 +27,7 @@ module Nokogiri
 
         ###
         # Finish the parsing.  This method is only necessary for
-        # Nokogiri::HTML::SAX::Document#end_document to be called.
+        # Nokogiri::HTML4::SAX::Document#end_document to be called.
         def finish
           write '', true
         end

data/lib/nokogiri/html5.rb

@@ -0,0 +1,473 @@
+# coding: utf-8
+# frozen_string_literal: true
+#
+#  Copyright 2013-2021 Sam Ruby, Stephen Checkoway
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+require_relative 'html5/document'
+require_relative 'html5/document_fragment'
+require_relative 'html5/node'
+
+module Nokogiri
+  # @since v1.12.0
+  # @note HTML5 functionality is not available when running JRuby.
+  # Parse an HTML5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
+  def self.HTML5(input, url = nil, encoding = nil, **options, &block)
+    Nokogiri::HTML5::Document.parse(input, url, encoding, **options, &block)
+  end
+
+  # == Usage
+  #
+  # Parse an HTML5 document:
+  #
+  #   doc = Nokogiri.HTML5(string)
+  #
+  # Parse an HTML5 fragment:
+  #
+  #   fragment = Nokogiri::HTML5.fragment(string)
+  #
+  # == Parsing options
+  #
+  # The document and fragment parsing methods support options that are different from Nokogiri's.
+  #
+  # - <tt>Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})</tt>
+  # - <tt>Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})</tt>
+  # - <tt>Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})</tt>
+  # - <tt>Nokogiri::HTML5.fragment(html, encoding = nil, options = {})</tt>
+  # - <tt>Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})</tt>
+  #
+  # The three currently supported options are +:max_errors+, +:max_tree_depth+ and
+  # +:max_attributes+, described below.
+  #
+  # === Error reporting
+  #
+  # Nokogumbo contains an experimental parse error reporting facility. By default, no parse errors
+  # are reported but this can be configured by passing the +:max_errors+ option to {HTML5.parse} or
+  # {HTML5.fragment}.
+  #
+  # For example, this script:
+  #
+  #   doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
+  #   doc.errors.each do |err|
+  #     puts(err)
+  #   end
+  #
+  # Emits:
+  #
+  #   1:1: ERROR: Expected a doctype token
+  #   <span/>Hi there!</span foo=bar />
+  #   ^
+  #   1:1: ERROR: Start tag of nonvoid HTML element ends with '/>', use '>'.
+  #   <span/>Hi there!</span foo=bar />
+  #   ^
+  #   1:17: ERROR: End tag ends with '/>', use '>'.
+  #   <span/>Hi there!</span foo=bar />
+  #                   ^
+  #   1:17: ERROR: End tag contains attributes.
+  #   <span/>Hi there!</span foo=bar />
+  #                   ^
+  #
+  # Using <tt>max_errors: -1</tt> results in an unlimited number of errors being returned.
+  #
+  # The errors returned by {HTML5::Document#errors} are instances of {Nokogiri::XML::SyntaxError}.
+  #
+  # The {https://html.spec.whatwg.org/multipage/parsing.html#parse-errors HTML standard} defines a
+  # number of standard parse error codes. These error codes only cover the "tokenization" stage of
+  # parsing HTML. The parse errors in the "tree construction" stage do not have standardized error
+  # codes (yet).
+  #
+  # As a convenience to Nokogumbo users, the defined error codes are available via
+  # {Nokogiri::XML::SyntaxError#str1} method.
+  #
+  #   doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
+  #   doc.errors.each do |err|
+  #     puts("#{err.line}:#{err.column}: #{err.str1}")
+  #   end
+  #   # => 1:1: generic-parser
+  #   #    1:1: non-void-html-element-start-tag-with-trailing-solidus
+  #   #    1:17: end-tag-with-trailing-solidus
+  #   #    1:17: end-tag-with-attributes
+  #
+  # Note that the first error is +generic-parser+ because it's an error from the tree construction
+  # stage and doesn't have a standardized error code.
+  #
+  # For the purposes of semantic versioning, the error messages, error locations, and error codes
+  # are not part of Nokogumbo's public API. That is, these are subject to change without Nokogumbo's
+  # major version number changing. These may be stabilized in the future.
+  #
+  # === Maximum tree depth
+  #
+  # The maximum depth of the DOM tree parsed by the various parsing methods is configurable by the
+  # +:max_tree_depth+ option. If the depth of the tree would exceed this limit, then an
+  # {::ArgumentError} is thrown.
+  #
+  # This limit (which defaults to <tt>Nokogumbo::DEFAULT_MAX_TREE_DEPTH = 400</tt>) can be removed
+  # by giving the option <tt>max_tree_depth: -1</tt>.
+  #
+  #   html = '<!DOCTYPE html>' + '<div>' * 1000
+  #   doc = Nokogiri.HTML5(html)
+  #   # raises ArgumentError: Document tree depth limit exceeded
+  #   doc = Nokogiri.HTML5(html, max_tree_depth: -1)
+  #
+  # === Attribute limit per element
+  #
+  # The maximum number of attributes per DOM element is configurable by the +:max_attributes+
+  # option. If a given element would exceed this limit, then an {::ArgumentError} is thrown.
+  #
+  # This limit (which defaults to <tt>Nokogumbo::DEFAULT_MAX_ATTRIBUTES = 400</tt>) can be removed
+  # by giving the option <tt>max_attributes: -1</tt>.
+  #
+  #   html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
+  #   # "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
+  #   doc = Nokogiri.HTML5(html)
+  #   # raises ArgumentError: Attributes per element limit exceeded
+  #   doc = Nokogiri.HTML5(html, max_attributes: -1)
+  #
+  # == HTML Serialization
+  #
+  # After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization
+  # methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will
+  # serialize a given node and its children. (This is the equivalent of JavaScript's
+  # +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given
+  # node. (This is the equivalent of JavaScript's +Element.innerHTML+.)
+  #
+  #   doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>")
+  #   puts doc.serialize
+  #   # => <!DOCTYPE html><html><head></head><body><span>Hello world!</span></body></html>
+  #
+  # Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be
+  # serialized and then re-parsed, resulting in a different DOM.  Mostly, this happens with DOMs
+  # produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and
+  # re-parsing.
+  #
+  # In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by
+  # the parser.
+  #
+  #   doc = Nokogiri::HTML5(<<-EOF)
+  #   <!DOCTYPE html>
+  #   <pre>
+  #   Content</pre>
+  #   EOF
+  #   puts doc.at('/html/body/pre').serialize
+  #   # => <pre>Content</pre>
+  #
+  # In this case, the original HTML is semantically equivalent to the serialized version. If the
+  # +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be
+  # stripped on the first parse and the second newline will be stripped on the second, leading to
+  # semantically different DOMs. Passing the parameter <tt>preserve_newline: true</tt> will cause
+  # two or more newlines to be preserved. (A single leading newline will still be removed.)
+  #
+  #   doc = Nokogiri::HTML5(<<-EOF)
+  #   <!DOCTYPE html>
+  #   <listing>
+  #
+  #   Content</listing>
+  #   EOF
+  #   puts doc.at('/html/body/listing').serialize(preserve_newline: true)
+  #   # => <listing>
+  #   #
+  #   #    Content</listing>
+  #
+  # == Encodings
+  #
+  # Nokogumbo always parses HTML using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the
+  # encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is
+  # most useful when the input comes not from a string but from an IO object.
+  #
+  # When serializing a document or node, the encoding of the output string can be specified via the
+  # +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded
+  # as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric
+  # entities}.
+  #
+  #   frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>')
+  #   html = frag.serialize(encoding: 'US-ASCII')
+  #   puts html
+  #   # => <span>&#xc544;&#xb294; &#xae38;&#xb3c4; &#xbb3c;&#xc5b4;&#xac00;&#xb77c;</span>
+  #   frag = Nokogiri::HTML5.fragment(html)
+  #   puts frag.serialize
+  #   # => <span>아는 길도 물어가라</span>
+  #
+  # (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that
+  # can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only
+  # encoding I'm aware of that has this bug is <tt>'ISO-2022-JP'</tt>. We recommend avoiding this
+  # encoding.)
+  #
+  # == Notes
+  #
+  # * The {Nokogiri::HTML5.fragment} function takes a string and parses it
+  #   as a HTML5 document.  The +<html>+, +<head>+, and +<body>+ elements are
+  #   removed from this document, and any children of these elements that remain
+  #   are returned as a {Nokogiri::HTML5::DocumentFragment}.
+  #
+  # * The {Nokogiri::HTML5.parse} function takes a string and passes it to the
+  #   <code>gumbo_parse_with_options</code> method, using the default options.
+  #   The resulting Gumbo parse tree is then walked.
+  #
+  # * Instead of uppercase element names, lowercase element names are produced.
+  #
+  # * Instead of returning +unknown+ as the element name for unknown tags, the
+  #   original tag name is returned verbatim.
+  #
+  # @since v1.12.0
+  # @note HTML5 functionality is not available when running JRuby.
+  module HTML5
+    # HTML uses the XHTML namespace.
+    HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml'.freeze
+    MATHML_NAMESPACE = 'http://www.w3.org/1998/Math/MathML'.freeze
+    SVG_NAMESPACE = 'http://www.w3.org/2000/svg'.freeze
+    XLINK_NAMESPACE = 'http://www.w3.org/1999/xlink'.freeze
+    XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace'.freeze
+    XMLNS_NAMESPACE = 'http://www.w3.org/2000/xmlns/'.freeze
+
+    # Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
+    def self.parse(string, url = nil, encoding = nil, **options, &block)
+      Document.parse(string, url, encoding, **options, &block)
+    end
+
+    # Parse a fragment from +string+. Convenience method for
+    # {Nokogiri::HTML5::DocumentFragment.parse}.
+    def self.fragment(string, encoding = nil, **options)
+      DocumentFragment.parse(string, encoding, options)
+    end
+
+    # Fetch and parse a HTML document from the web, following redirects,
+    # handling https, and determining the character encoding using HTML5
+    # rules.  +uri+ may be a +String+ or a +URI+.  +options+ contains
+    # http headers and special options.  Everything which is not a
+    # special option is considered a header.  Special options include:
+    #  * :follow_limit => number of redirects which are followed
+    #  * :basic_auth => [username, password]
+    def self.get(uri, options={})
+      warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.",
+           uplevel: 1, category: :deprecated)
+      get_impl(uri, options)
+    end
+
+    private
+
+    def self.get_impl(uri, options={})
+      headers = options.clone
+      headers = {:follow_limit => headers} if Numeric === headers # deprecated
+      limit=headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
+
+      require 'net/http'
+      uri = URI(uri) unless URI === uri
+
+      http = Net::HTTP.new(uri.host, uri.port)
+
+      # TLS / SSL support
+      http.use_ssl = true if uri.scheme == 'https'
+
+      # Pass through Net::HTTP override values, which currently include:
+      #   :ca_file, :ca_path, :cert, :cert_store, :ciphers,
+      #   :close_on_empty_response, :continue_timeout, :key, :open_timeout,
+      #   :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
+      #   :verify_callback, :verify_depth, :verify_mode
+      options.each do |key, value|
+        http.send "#{key}=", headers.delete(key) if http.respond_to? "#{key}="
+      end
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+
+      # basic authentication
+      auth = headers.delete(:basic_auth)
+      auth ||= [uri.user, uri.password] if uri.user && uri.password
+      request.basic_auth auth.first, auth.last if auth
+
+      # remaining options are treated as headers
+      headers.each {|key, value| request[key.to_s] = value.to_s}
+
+      response = http.request(request)
+
+      case response
+      when Net::HTTPSuccess
+        doc = parse(reencode(response.body, response['content-type']), options)
+        doc.instance_variable_set('@response', response)
+        doc.class.send(:attr_reader, :response)
+        doc
+      when Net::HTTPRedirection
+        response.value if limit <= 1
+        location = URI.join(uri, response['location'])
+        get_impl(location, options.merge(:follow_limit => limit-1))
+      else
+        response.value
+      end
+    end
+
+    def self.read_and_encode(string, encoding)
+      # Read the string with the given encoding.
+      if string.respond_to?(:read)
+        if encoding.nil?
+          string = string.read
+        else
+          string = string.read(encoding: encoding)
+        end
+      else
+        # Otherwise the string has the given encoding.
+        string = string.to_s
+        if encoding
+          string = string.dup
+          string.force_encoding(encoding)
+        end
+      end
+
+      # convert to UTF-8
+      if string.encoding != Encoding::UTF_8
+        string = reencode(string)
+      end
+      string
+    end
+
+    # Charset sniffing is a complex and controversial topic that understandably isn't done _by
+    # default_ by the Ruby Net::HTTP library.  This being said, it is a very real problem for
+    # consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
+    # the Gumbo parser *only* supports utf-8.
+    #
+    # Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection.  Following
+    # this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
+    # the HTML5 standard.
+    #
+    # http://bugs.ruby-lang.org/issues/2567
+    # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
+    #
+    def self.reencode(body, content_type=nil)
+      if body.encoding == Encoding::ASCII_8BIT
+        encoding = nil
+
+        # look for a Byte Order Mark (BOM)
+        initial_bytes = body[0..2].bytes
+        if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
+          encoding = Encoding::UTF_8
+        elsif initial_bytes[0..1] == [0xFE, 0xFF]
+          encoding = Encoding::UTF_16BE
+        elsif initial_bytes[0..1] == [0xFF, 0xFE]
+          encoding = Encoding::UTF_16LE
+        end
+
+        # look for a charset in a content-encoding header
+        if content_type
+          encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
+        end
+
+        # look for a charset in a meta tag in the first 1024 bytes
+        if not encoding
+          data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, '')
+          data.scan(/<meta.*?>/m).each do |meta|
+            encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
+          end
+        end
+
+        # if all else fails, default to the official default encoding for HTML
+        encoding ||= Encoding::ISO_8859_1
+
+        # change the encoding to match the detected or inferred encoding
+        body = body.dup
+        begin
+          body.force_encoding(encoding)
+        rescue ArgumentError
+          body.force_encoding(Encoding::ISO_8859_1)
+        end
+      end
+
+      body.encode(Encoding::UTF_8)
+    end
+
+    def self.serialize_node_internal(current_node, io, encoding, options)
+      case current_node.type
+      when XML::Node::ELEMENT_NODE
+        ns = current_node.namespace
+        ns_uri = ns.nil? ? nil : ns.href
+        # XXX(sfc): attach namespaces to all nodes, even html?
+        if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE
+          tagname = current_node.name
+        else
+          tagname = "#{ns.prefix}:#{current_node.name}"
+        end
+        io << '<' << tagname
+        current_node.attribute_nodes.each do |attr|
+          attr_ns = attr.namespace
+          if attr_ns.nil?
+            attr_name = attr.name
+          else
+            ns_uri = attr_ns.href
+            if ns_uri == XML_NAMESPACE
+              attr_name = 'xml:' + attr.name.sub(/^[^:]*:/, '')
+            elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, '') == 'xmlns'
+              attr_name = 'xmlns'
+            elsif ns_uri == XMLNS_NAMESPACE
+              attr_name = 'xmlns:' + attr.name.sub(/^[^:]*:/, '')
+            elsif ns_uri == XLINK_NAMESPACE
+              attr_name = 'xlink:' + attr.name.sub(/^[^:]*:/, '')
+            else
+              attr_name = "#{attr_ns.prefix}:#{attr.name}"
+            end
+          end
+          io << ' ' << attr_name << '="' << escape_text(attr.content, encoding, true) << '"'
+        end
+        io << '>'
+        if !%w[area base basefont bgsound br col embed frame hr img input keygen
+               link meta param source track wbr].include?(current_node.name)
+          io << "\n" if options[:preserve_newline] && prepend_newline?(current_node)
+          current_node.children.each do |child|
+            # XXX(sfc): Templates handled specially?
+            serialize_node_internal(child, io, encoding, options)
+          end
+          io << '</' << tagname << '>'
+        end
+      when XML::Node::TEXT_NODE
+        parent = current_node.parent
+        if parent.element? && %w[style script xmp iframe noembed noframes plaintext noscript].include?(parent.name)
+          io << current_node.content
+        else
+          io << escape_text(current_node.content, encoding, false)
+        end
+      when XML::Node::CDATA_SECTION_NODE
+        io << '<![CDATA[' << current_node.content << ']]>'
+      when XML::Node::COMMENT_NODE
+        io << '<!--' << current_node.content << '-->'
+      when XML::Node::PI_NODE
+        io << '<?' << current_node.content << '>'
+      when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE
+          io << '<!DOCTYPE ' << current_node.name << '>'
+      when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE
+        current_node.children.each do |child|
+          serialize_node_internal(child, io, encoding, options)
+        end
+      else
+        raise "Unexpected node '#{current_node.name}' of type #{current_node.type}"
+      end
+    end
+
+    def self.escape_text(text, encoding, attribute_mode)
+      if attribute_mode
+        text = text.gsub(/[&\u00a0"]/,
+                           '&' => '&amp;', "\u00a0" => '&nbsp;', '"' => '&quot;')
+      else
+        text = text.gsub(/[&\u00a0<>]/,
+                           '&' => '&amp;', "\u00a0" => '&nbsp;',  '<' => '&lt;', '>' => '&gt;')
+      end
+      # Not part of the standard
+      text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" })
+    end
+
+    def self.prepend_newline?(node)
+      return false unless %w[pre textarea listing].include?(node.name) && !node.children.empty?
+      first_child = node.children[0]
+      first_child.text? && first_child.content.start_with?("\n")
+    end
+  end
+end
+
+require_relative 'gumbo'