nokogiri 1.5.10 → 1.13.0

data/lib/nokogiri/html.rb

@@ -1,37 +1,48 @@
-require 'nokogiri/html/entity_lookup'
-require 'nokogiri/html/document'
-require 'nokogiri/html/document_fragment'
-require 'nokogiri/html/sax/parser_context'
-require 'nokogiri/html/sax/parser'
-require 'nokogiri/html/sax/push_parser'
-require 'nokogiri/html/element_description'
-require 'nokogiri/html/element_description_defaults'
+# coding: utf-8
+# frozen_string_literal: true
+
+require_relative "html4"
 
 module Nokogiri
-  class << self
-    ###
-    # Parse HTML.  Convenience method for Nokogiri::HTML::Document.parse
-    def HTML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
-      Nokogiri::HTML::Document.parse(thing, url, encoding, options, &block)
-    end
-  end
+  # Alias for Nokogiri::HTML4
+  HTML = Nokogiri::HTML4
+
+  # :singleton-method: HTML
+  # :call-seq: HTML(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) → Nokogiri::HTML4::Document
+  #
+  # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse
 
+  # :nodoc:
+  define_singleton_method(:HTML, Nokogiri.method(:HTML4))
+
+  # 💡 This module/namespace is an alias for Nokogiri::HTML4 as of v1.12.0. Before v1.12.0,
+  #   Nokogiri::HTML4 did not exist, and this was the module/namespace for all HTML-related
+  #   classes.
   module HTML
-    class << self
-      ###
-      # Parse HTML.  Convenience method for Nokogiri::HTML::Document.parse
-      def parse thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
-        Document.parse(thing, url, encoding, options, &block)
+    # 💡 This class is an alias for Nokogiri::HTML4::Document as of v1.12.0.
+    class Document < Nokogiri::XML::Document
+    end
+
+    # 💡 This class is an alias for Nokogiri::HTML4::DocumentFragment as of v1.12.0.
+    class DocumentFragment < Nokogiri::XML::DocumentFragment
+    end
+
+    # 💡 This class is an alias for Nokogiri::HTML4::Builder as of v1.12.0.
+    class Builder < Nokogiri::XML::Builder
+    end
+
+    module SAX
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::Parser as of v1.12.0.
+      class Parser < Nokogiri::XML::SAX::Parser
       end
 
-      ####
-      # Parse a fragment from +string+ in to a NodeSet.
-      def fragment string, encoding = nil
-        HTML::DocumentFragment.parse string, encoding
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::ParserContext as of v1.12.0.
+      class ParserContext < Nokogiri::XML::SAX::ParserContext
       end
-    end
 
-    # Instance of Nokogiri::HTML::EntityLookup
-    NamedCharacters = EntityLookup.new
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::PushParser as of v1.12.0.
+      class PushParser
+      end
+    end
   end
 end

data/lib/nokogiri/{html → html4}/builder.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Nokogiri
-  module HTML
+  module HTML4
     ###
     # Nokogiri HTML builder is used for building HTML documents.  It is very
     # similar to the Nokogiri::XML::Builder.  In fact, you should go read the
@@ -11,7 +13,7 @@ module Nokogiri
     # Create an HTML document with a body that has an onload attribute, and a
     # span tag with a class of "bold" that has content of "Hello world".
     #
-    #   builder = Nokogiri::HTML::Builder.new do |doc|
+    #   builder = Nokogiri::HTML4::Builder.new do |doc|
     #     doc.html {
     #       doc.body(:onload => 'some_func();') {
     #         doc.span.bold {

data/lib/nokogiri/html4/document.rb

@@ -0,0 +1,331 @@
+# coding: utf-8
+# frozen_string_literal: true
+
+require "pathname"
+
+module Nokogiri
+  module HTML4
+    class Document < Nokogiri::XML::Document
+      ###
+      # Get the meta tag encoding for this document.  If there is no meta tag,
+      # then nil is returned.
+      def meta_encoding
+        if (meta = at_xpath("//meta[@charset]"))
+          meta[:charset]
+        elsif (meta = meta_content_type)
+          meta["content"][/charset\s*=\s*([\w-]+)/i, 1]
+        end
+      end
+
+      ###
+      # Set the meta tag encoding for this document.
+      #
+      # If an meta encoding tag is already present, its content is
+      # replaced with the given text.
+      #
+      # Otherwise, this method tries to create one at an appropriate
+      # place supplying head and/or html elements as necessary, which
+      # is inside a head element if any, and before any text node or
+      # content element (typically <body>) if any.
+      #
+      # The result when trying to set an encoding that is different
+      # from the document encoding is undefined.
+      #
+      # Beware in CRuby, that libxml2 automatically inserts a meta tag
+      # into a head element.
+      def meta_encoding=(encoding)
+        if (meta = meta_content_type)
+          meta["content"] = format("text/html; charset=%s", encoding)
+          encoding
+        elsif (meta = at_xpath("//meta[@charset]"))
+          meta["charset"] = encoding
+        else
+          meta = XML::Node.new("meta", self)
+          if (dtd = internal_subset) && dtd.html5_dtd?
+            meta["charset"] = encoding
+          else
+            meta["http-equiv"] = "Content-Type"
+            meta["content"] = format("text/html; charset=%s", encoding)
+          end
+
+          if (head = at_xpath("//head"))
+            head.prepend_child(meta)
+          else
+            set_metadata_element(meta)
+          end
+          encoding
+        end
+      end
+
+      def meta_content_type
+        xpath("//meta[@http-equiv and boolean(@content)]").find do |node|
+          node["http-equiv"] =~ /\AContent-Type\z/i
+        end
+      end
+      private :meta_content_type
+
+      ###
+      # Get the title string of this document.  Return nil if there is
+      # no title tag.
+      def title
+        (title = at_xpath("//title")) && title.inner_text
+      end
+
+      ###
+      # Set the title string of this document.
+      #
+      # If a title element is already present, its content is replaced
+      # with the given text.
+      #
+      # Otherwise, this method tries to create one at an appropriate
+      # place supplying head and/or html elements as necessary, which
+      # is inside a head element if any, right after a meta
+      # encoding/charset tag if any, and before any text node or
+      # content element (typically <body>) if any.
+      def title=(text)
+        tnode = XML::Text.new(text, self)
+        if (title = at_xpath("//title"))
+          title.children = tnode
+          return text
+        end
+
+        title = XML::Node.new("title", self) << tnode
+        if (head = at_xpath("//head"))
+          head << title
+        elsif (meta = (at_xpath("//meta[@charset]") || meta_content_type))
+          # better put after charset declaration
+          meta.add_next_sibling(title)
+        else
+          set_metadata_element(title)
+        end
+      end
+
+      def set_metadata_element(element) # rubocop:disable Naming/AccessorMethodName
+        if (head = at_xpath("//head"))
+          head << element
+        elsif (html = at_xpath("//html"))
+          head = html.prepend_child(XML::Node.new("head", self))
+          head.prepend_child(element)
+        elsif (first = children.find do |node|
+                 case node
+                 when XML::Element, XML::Text
+                   true
+                 end
+               end)
+          # We reach here only if the underlying document model
+          # allows <html>/<head> elements to be omitted and does not
+          # automatically supply them.
+          first.add_previous_sibling(element)
+        else
+          html = add_child(XML::Node.new("html", self))
+          head = html.add_child(XML::Node.new("head", self))
+          head.prepend_child(element)
+        end
+      end
+      private :set_metadata_element
+
+      ####
+      # Serialize Node using +options+.  Save options can also be set using a
+      # block. See SaveOptions.
+      #
+      # These two statements are equivalent:
+      #
+      #  node.serialize(:encoding => 'UTF-8', :save_with => FORMAT | AS_XML)
+      #
+      # or
+      #
+      #   node.serialize(:encoding => 'UTF-8') do |config|
+      #     config.format.as_xml
+      #   end
+      #
+      def serialize(options = {})
+        options[:save_with] ||= XML::Node::SaveOptions::DEFAULT_HTML
+        super
+      end
+
+      ####
+      # Create a Nokogiri::XML::DocumentFragment from +tags+
+      def fragment(tags = nil)
+        DocumentFragment.new(self, tags, root)
+      end
+
+      # :call-seq:
+      #   xpath_doctype() → Nokogiri::CSS::XPathVisitor::DoctypeConfig
+      #
+      # [Returns] The document type which determines CSS-to-XPath translation.
+      #
+      # See XPathVisitor for more information.
+      def xpath_doctype
+        Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML4
+      end
+
+      class << self
+        ###
+        # Parse HTML.  +string_or_io+ may be a String, or any object that
+        # responds to _read_ and _close_ such as an IO, or StringIO.
+        # +url+ is resource where this document is located.  +encoding+ is the
+        # encoding that should be used when processing the document. +options+
+        # is a number that sets options in the parser, such as
+        # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
+        # Nokogiri::XML::ParseOptions.
+        def parse(string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML)
+          options = Nokogiri::XML::ParseOptions.new(options) if Integer === options
+          yield options if block_given?
+
+          url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+
+          if string_or_io.respond_to?(:encoding)
+            unless string_or_io.encoding.name == "ASCII-8BIT"
+              encoding ||= string_or_io.encoding.name
+            end
+          end
+
+          if string_or_io.respond_to?(:read)
+            if string_or_io.is_a?(Pathname)
+              # resolve the Pathname to the file and open it as an IO object, see #2110
+              string_or_io = string_or_io.expand_path.open
+              url ||= string_or_io.path
+            end
+
+            unless encoding
+              # Libxml2's parser has poor support for encoding
+              # detection.  First, it does not recognize the HTML5
+              # style meta charset declaration.  Secondly, even if it
+              # successfully detects an encoding hint, it does not
+              # re-decode or re-parse the preceding part which may be
+              # garbled.
+              #
+              # EncodingReader aims to perform advanced encoding
+              # detection beyond what Libxml2 does, and to emulate
+              # rewinding of a stream and make Libxml2 redo parsing
+              # from the start when an encoding hint is found.
+              string_or_io = EncodingReader.new(string_or_io)
+              begin
+                return read_io(string_or_io, url, encoding, options.to_i)
+              rescue EncodingFound => e
+                encoding = e.found_encoding
+              end
+            end
+            return read_io(string_or_io, url, encoding, options.to_i)
+          end
+
+          # read_memory pukes on empty docs
+          if string_or_io.nil? || string_or_io.empty?
+            return encoding ? new.tap { |i| i.encoding = encoding } : new
+          end
+
+          encoding ||= EncodingReader.detect_encoding(string_or_io)
+
+          read_memory(string_or_io, url, encoding, options.to_i)
+        end
+      end
+
+      class EncodingFound < StandardError # :nodoc: all
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super(format("encoding found: %s", encoding))
+        end
+      end
+
+      # :nodoc: all
+      class EncodingReader
+        class SAXHandler < Nokogiri::XML::SAX::Document
+          attr_reader :encoding
+
+          def initialize
+            @encoding = nil
+            super()
+          end
+
+          def start_element(name, attrs = [])
+            return unless name == "meta"
+            attr = Hash[attrs]
+            (charset = attr["charset"]) &&
+              (@encoding = charset)
+            (http_equiv = attr["http-equiv"]) &&
+              http_equiv.match(/\AContent-Type\z/i) &&
+              (content = attr["content"]) &&
+              (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
+              (@encoding = m[1])
+          end
+        end
+
+        class JumpSAXHandler < SAXHandler
+          def initialize(jumptag)
+            @jumptag = jumptag
+            super()
+          end
+
+          def start_element(name, attrs = [])
+            super
+            throw(@jumptag, @encoding) if @encoding
+            throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
+          end
+        end
+
+        def self.detect_encoding(chunk)
+          (m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/)) &&
+            (return Nokogiri.XML(m[1]).encoding)
+
+          if Nokogiri.jruby?
+            (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
+              (return m[4])
+            catch(:encoding_found) do
+              Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+              nil
+            end
+          else
+            handler = SAXHandler.new
+            parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
+            begin
+              parser << chunk
+            rescue
+              Nokogiri::SyntaxError
+            end
+            handler.encoding
+          end
+        end
+
+        def initialize(io)
+          @io = io
+          @firstchunk = nil
+          @encoding_found = nil
+        end
+
+        # This method is used by the C extension so that
+        # Nokogiri::HTML4::Document#read_io() does not leak memory when
+        # EncodingFound is raised.
+        attr_reader :encoding_found
+
+        def read(len)
+          # no support for a call without len
+
+          unless @firstchunk
+            (@firstchunk = @io.read(len)) || (return nil)
+
+            # This implementation expects that the first call from
+            # htmlReadIO() is made with a length long enough (~1KB) to
+            # achieve advanced encoding detection.
+            if (encoding = EncodingReader.detect_encoding(@firstchunk))
+              # The first chunk is stored for the next read in retry.
+              raise @encoding_found = EncodingFound.new(encoding)
+            end
+          end
+          @encoding_found = nil
+
+          ret = @firstchunk.slice!(0, len)
+          if (len -= ret.length) > 0
+            (rest = @io.read(len)) && ret << (rest)
+          end
+          if ret.empty?
+            nil
+          else
+            ret
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4/document_fragment.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    class DocumentFragment < Nokogiri::XML::DocumentFragment
+      ####
+      # Create a Nokogiri::XML::DocumentFragment from +tags+, using +encoding+
+      def self.parse(tags, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
+        doc = HTML4::Document.new
+
+        encoding ||= if tags.respond_to?(:encoding)
+          encoding = tags.encoding
+          if encoding == ::Encoding::ASCII_8BIT
+            "UTF-8"
+          else
+            encoding.name
+          end
+        else
+          "UTF-8"
+        end
+
+        doc.encoding = encoding
+
+        new(doc, tags, nil, options, &block)
+      end
+
+      def initialize(document, tags = nil, ctx = nil, options = XML::ParseOptions::DEFAULT_HTML)
+        return self unless tags
+
+        options = Nokogiri::XML::ParseOptions.new(options) if Integer === options
+        yield options if block_given?
+
+        if ctx
+          preexisting_errors = document.errors.dup
+          node_set = ctx.parse("<div>#{tags}</div>", options)
+          node_set.first.children.each { |child| child.parent = self } unless node_set.empty?
+          self.errors = document.errors - preexisting_errors
+        else
+          # This is a horrible hack, but I don't care
+          path = if /^\s*?<body/i.match?(tags)
+            "/html/body"
+          else
+            "/html/body/node()"
+          end
+
+          temp_doc = HTML4::Document.parse("<html><body>#{tags}", nil, document.encoding, options)
+          temp_doc.xpath(path).each { |child| child.parent = self }
+          self.errors = temp_doc.errors
+        end
+        children
+      end
+    end
+  end
+end

data/lib/nokogiri/{html → html4}/element_description.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Nokogiri
-  module HTML
+  module HTML4
     class ElementDescription
       ###
       # Is this element a block element?