nokogiri 1.12.5 → 1.14.3

data/lib/nokogiri/html4/encoding_reader.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    # Libxml2's parser has poor support for encoding detection.  First, it does not recognize the
+    # HTML5 style meta charset declaration.  Secondly, even if it successfully detects an encoding
+    # hint, it does not re-decode or re-parse the preceding part which may be garbled.
+    #
+    # EncodingReader aims to perform advanced encoding detection beyond what Libxml2 does, and to
+    # emulate rewinding of a stream and make Libxml2 redo parsing from the start when an encoding
+    # hint is found.
+
+    # :nodoc: all
+    class EncodingReader
+      class EncodingFound < StandardError
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super(format("encoding found: %s", encoding))
+        end
+      end
+
+      class SAXHandler < Nokogiri::XML::SAX::Document
+        attr_reader :encoding
+
+        def initialize
+          @encoding = nil
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          return unless name == "meta"
+
+          attr = Hash[attrs]
+          (charset = attr["charset"]) &&
+            (@encoding = charset)
+          (http_equiv = attr["http-equiv"]) &&
+            http_equiv.match(/\AContent-Type\z/i) &&
+            (content = attr["content"]) &&
+            (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
+            (@encoding = m[1])
+        end
+      end
+
+      class JumpSAXHandler < SAXHandler
+        def initialize(jumptag)
+          @jumptag = jumptag
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          super
+          throw(@jumptag, @encoding) if @encoding
+          throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
+        end
+      end
+
+      def self.detect_encoding(chunk)
+        (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
+          (return Nokogiri.XML(m[1]).encoding)
+
+        if Nokogiri.jruby?
+          (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
+            (return m[4])
+          catch(:encoding_found) do
+            Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+            nil
+          end
+        else
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
+          begin
+            parser << chunk
+          rescue
+            Nokogiri::SyntaxError
+          end
+          handler.encoding
+        end
+      end
+
+      def initialize(io)
+        @io = io
+        @firstchunk = nil
+        @encoding_found = nil
+      end
+
+      # This method is used by the C extension so that
+      # Nokogiri::HTML4::Document#read_io() does not leak memory when
+      # EncodingFound is raised.
+      attr_reader :encoding_found
+
+      def read(len)
+        # no support for a call without len
+
+        unless @firstchunk
+          (@firstchunk = @io.read(len)) || (return nil)
+
+          # This implementation expects that the first call from
+          # htmlReadIO() is made with a length long enough (~1KB) to
+          # achieve advanced encoding detection.
+          if (encoding = EncodingReader.detect_encoding(@firstchunk))
+            # The first chunk is stored for the next read in retry.
+            raise @encoding_found = EncodingFound.new(encoding)
+          end
+        end
+        @encoding_found = nil
+
+        ret = @firstchunk.slice!(0, len)
+        if (len -= ret.length) > 0
+          (rest = @io.read(len)) && ret << (rest)
+        end
+        if ret.empty?
+          nil
+        else
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4/entity_lookup.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module HTML4
     class EntityDescription < Struct.new(:value, :name, :description); end
@@ -6,7 +7,7 @@ module Nokogiri
     class EntityLookup
       ###
       # Look up entity with +name+
-      def [] name
+      def [](name)
         (val = get(name)) && val.value
       end
     end

data/lib/nokogiri/html4/sax/parser.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module HTML4
     ###
@@ -27,8 +28,9 @@ module Nokogiri
         ###
         # Parse html stored in +data+ using +encoding+
         def parse_memory(data, encoding = "UTF-8")
-          raise ArgumentError unless data
-          return unless data.length > 0
+          raise TypeError unless String === data
+          return if data.empty?
+
           ctx = ParserContext.memory(data, encoding)
           yield ctx if block_given?
           ctx.parse_with(self)
@@ -50,6 +52,7 @@ module Nokogiri
           raise ArgumentError unless filename
           raise Errno::ENOENT unless File.exist?(filename)
           raise Errno::EISDIR if File.directory?(filename)
+
           ctx = ParserContext.file(filename, encoding)
           yield ctx if block_given?
           ctx.parse_with(self)

data/lib/nokogiri/html4/sax/parser_context.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module HTML4
     module SAX

data/lib/nokogiri/html4/sax/push_parser.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module HTML4
     module SAX
       class PushParser
-        
         # The Nokogiri::HTML4::SAX::Document on which the PushParser will be
         # operating
         attr_accessor :document
-        
-        def initialize(doc = HTML4::SAX::Document.new, file_name = nil, encoding = 'UTF-8')
+
+        def initialize(doc = HTML4::SAX::Document.new, file_name = nil, encoding = "UTF-8")
           @document = doc
           @encoding = encoding
           @sax_parser = HTML4::SAX::Parser.new(doc, @encoding)
@@ -16,20 +16,20 @@ module Nokogiri
           ## Create our push parser context
           initialize_native(@sax_parser, file_name, encoding)
         end
-        
+
         ###
         # Write a +chunk+ of HTML to the PushParser.  Any callback methods
         # that can be called will be called immediately.
-        def write chunk, last_chunk = false
+        def write(chunk, last_chunk = false)
           native_write(chunk, last_chunk)
         end
-        alias :<< :write
+        alias_method :<<, :write
 
         ###
         # Finish the parsing.  This method is only necessary for
         # Nokogiri::HTML4::SAX::Document#end_document to be called.
         def finish
-          write '', true
+          write("", true)
         end
       end
     end

data/lib/nokogiri/html4.rb

@@ -1,15 +1,21 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 module Nokogiri
   class << self
-    ###
+    # :call-seq:
+    #   HTML4(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) → Nokogiri::HTML4::Document
+    #
     # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse
     def HTML4(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
       Nokogiri::HTML4::Document.parse(input, url, encoding, options, &block)
     end
   end
 
-  # @since v1.12.0
-  # @note Before v1.12.0, {Nokogiri::HTML4} did not exist, and {Nokogiri::HTML} was the module/namespace for parsing HTML.
+  # Since v1.12.0
+  #
+  # 💡 Before v1.12.0, Nokogiri::HTML4 did not exist, and Nokogiri::HTML was the module/namespace
+  # for parsing HTML.
   module HTML4
     class << self
       ###
@@ -20,8 +26,8 @@ module Nokogiri
 
       ####
       # Parse a fragment from +string+ in to a NodeSet.
-      def fragment(string, encoding = nil)
-        HTML4::DocumentFragment.parse(string, encoding)
+      def fragment(string, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
+        HTML4::DocumentFragment.parse(string, encoding, options, &block)
       end
     end
 
@@ -33,6 +39,7 @@ end
 require_relative "html4/entity_lookup"
 require_relative "html4/document"
 require_relative "html4/document_fragment"
+require_relative "html4/encoding_reader"
 require_relative "html4/sax/parser_context"
 require_relative "html4/sax/parser"
 require_relative "html4/sax/push_parser"

data/lib/nokogiri/html5/document.rb

@@ -1,4 +1,6 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 #
 #  Copyright 2013-2021 Sam Ruby, Stephen Checkoway
 #
@@ -19,55 +21,147 @@ require_relative "../html4/document"
 
 module Nokogiri
   module HTML5
-    # @since v1.12.0
-    # @note HTML5 functionality is not available when running JRuby.
+    # Enum for the HTML5 parser quirks mode values. Values returned by HTML5::Document#quirks_mode
+    #
+    # See https://dom.spec.whatwg.org/#concept-document-quirks for more information on HTML5 quirks
+    # mode.
+    #
+    # Since v1.14.0
+    module QuirksMode
+      NO_QUIRKS = 0 # The document was parsed in "no-quirks" mode
+      QUIRKS = 1 # The document was parsed in "quirks" mode
+      LIMITED_QUIRKS = 2 # The document was parsed in "limited-quirks" mode
+    end
+
+    # Since v1.12.0
+    #
+    # 💡 HTML5 functionality is not available when running JRuby.
     class Document < Nokogiri::HTML4::Document
-      def self.parse(string_or_io, url = nil, encoding = nil, **options, &block)
-        yield options if block_given?
-	string_or_io = '' unless string_or_io
+      # Get the url name for this document, as passed into Document.parse, Document.read_io, or
+      # Document.read_memory
+      attr_reader :url
+
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::Document.new`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
+
+      class << self
+        # :call-seq:
+        #   parse(input)
+        #   parse(input, url=nil, encoding=nil, **options)
+        #   parse(input, url=nil, encoding=nil) { |options| ... }
+        #
+        # Parse HTML5 input.
+        #
+        # [Parameters]
+        # - +input+ may be a String, or any object that responds to _read_ and _close_ such as an
+        #   IO, or StringIO.
+        #
+        # - +url+ (optional) is a String indicating the canonical URI where this document is located.
+        #
+        # - +encoding+ (optional) is the encoding that should be used when processing
+        #   the document.
+        #
+        # - +options+ (optional) is a configuration Hash (or keyword arguments) to set options
+        #   during parsing. The three currently supported options are +:max_errors+,
+        #   +:max_tree_depth+ and +:max_attributes+, described at Nokogiri::HTML5.
+        #
+        #   ⚠ Note that these options are different than those made available by
+        #   Nokogiri::XML::Document and Nokogiri::HTML4::Document.
+        #
+        # - +block+ (optional) is passed a configuration Hash on which parse options may be set. See
+        #   Nokogiri::HTML5 for more information and usage.
+        #
+        # [Returns] Nokogiri::HTML5::Document
+        #
+        def parse(string_or_io, url = nil, encoding = nil, **options, &block)
+          yield options if block
+          string_or_io = "" unless string_or_io
+
+          if string_or_io.respond_to?(:encoding) && string_or_io.encoding != Encoding::ASCII_8BIT
+            encoding ||= string_or_io.encoding.name
+          end
+
+          if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
+            url ||= string_or_io.path
+          end
+          unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+            raise ArgumentError, "not a string or IO object"
+          end
 
-        if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != 'ASCII-8BIT'
-          encoding ||= string_or_io.encoding.name
+          do_parse(string_or_io, url, encoding, options)
         end
 
-        if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
-          url ||= string_or_io.path
+        # Create a new document from an IO object.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_io(io, url = nil, encoding = nil, **options)
+          raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+
+          do_parse(io, url, encoding, options)
         end
-        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
-          raise ArgumentError.new("not a string or IO object")
+
+        # Create a new document from a String.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_memory(string, url = nil, encoding = nil, **options)
+          raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
+
+          do_parse(string, url, encoding, options)
         end
-        do_parse(string_or_io, url, encoding, options)
-      end
 
-      def self.read_io(io, url = nil, encoding = nil, **options)
-        raise ArgumentError.new("io object doesn't respond to :read") unless io.respond_to?(:read)
-        do_parse(io, url, encoding, options)
+        private
+
+        def do_parse(string_or_io, url, encoding, options)
+          string = HTML5.read_and_encode(string_or_io, encoding)
+          max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
+          max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
+          max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
+          doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth, self)
+          doc.encoding = "UTF-8"
+          doc
+        end
       end
 
-      def self.read_memory(string, url = nil, encoding = nil, **options)
-        raise ArgumentError.new("string object doesn't respond to :to_str") unless string.respond_to?(:to_str)
-        do_parse(string, url, encoding, options)
+      def initialize(*args) # :nodoc:
+        super
+        @url = nil
+        @quirks_mode = nil
       end
 
-      def fragment(tags = nil)
-        DocumentFragment.new(self, tags, self.root)
+      # :call-seq:
+      #   fragment() → Nokogiri::HTML5::DocumentFragment
+      #   fragment(markup) → Nokogiri::HTML5::DocumentFragment
+      #
+      # Parse a HTML5 document fragment from +markup+, returning a Nokogiri::HTML5::DocumentFragment.
+      #
+      # [Properties]
+      # - +markup+ (String) The HTML5 markup fragment to be parsed
+      #
+      # [Returns]
+      #   Nokogiri::HTML5::DocumentFragment. This object's children will be empty if `markup` is not passed, is empty, or is `nil`.
+      #
+      def fragment(markup = nil)
+        DocumentFragment.new(self, markup)
       end
 
-      def to_xml(options = {}, &block)
+      def to_xml(options = {}, &block) # :nodoc:
         # Bypass XML::Document#to_xml which doesn't add
         # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does.
-        XML::Node.instance_method(:to_xml).bind(self).call(options, &block)
+        XML::Node.instance_method(:to_xml).bind_call(self, options, &block)
       end
 
-      private
-      def self.do_parse(string_or_io, url, encoding, options)
-        string = HTML5.read_and_encode(string_or_io, encoding)
-        max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
-        max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
-        max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth)
-        doc.encoding = 'UTF-8'
-        doc
+      # :call-seq:
+      #   xpath_doctype() → Nokogiri::CSS::XPathVisitor::DoctypeConfig
+      #
+      # [Returns] The document type which determines CSS-to-XPath translation.
+      #
+      # See CSS::XPathVisitor for more information.
+      def xpath_doctype
+        Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML5
       end
     end
   end

data/lib/nokogiri/html5/document_fragment.rb

@@ -1,4 +1,6 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 #
 #  Copyright 2013-2021 Sam Ruby, Stephen Checkoway
 #
@@ -19,12 +21,20 @@ require_relative "../html4/document_fragment"
 
 module Nokogiri
   module HTML5
-    # @since v1.12.0
-    # @note HTML5 functionality is not available when running JRuby.
+    # Since v1.12.0
+    #
+    # 💡 HTML5 functionality is not available when running JRuby.
     class DocumentFragment < Nokogiri::HTML4::DocumentFragment
       attr_accessor :document
       attr_accessor :errors
 
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::DocumentFragment.new(doc)`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
+
       # Create a document fragment.
       def initialize(doc, tags = nil, ctx = nil, options = {})
         self.document = doc
@@ -38,10 +48,10 @@ module Nokogiri
         Nokogiri::Gumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
 
-      def serialize(options = {}, &block)
+      def serialize(options = {}, &block) # :nodoc:
         # Bypass XML::Document.serialize which doesn't support options even
         # though XML::Node.serialize does!
-        XML::Node.instance_method(:serialize).bind(self).call(options, &block)
+        XML::Node.instance_method(:serialize).bind_call(self, options, &block)
       end
 
       # Parse a document fragment from +tags+, returning a Nodeset.

data/lib/nokogiri/html5/node.rb

@@ -1,4 +1,6 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 #
 #  Copyright 2013-2021 Sam Ruby, Stephen Checkoway
 #
@@ -19,18 +21,21 @@ require_relative "../xml/node"
 
 module Nokogiri
   module HTML5
-    # @since v1.12.0
-    # @note HTML5 functionality is not available when running JRuby.
+    # Since v1.12.0
+    #
+    # 💡 HTML5 functionality is not available when running JRuby.
     module Node
       def inner_html(options = {})
         return super(options) unless document.is_a?(HTML5::Document)
-        result = options[:preserve_newline] && HTML5.prepend_newline?(self) ? String.new("\n") : String.new
+
+        result = options[:preserve_newline] && prepend_newline? ? +"\n" : +""
         result << children.map { |child| child.to_html(options) }.join
         result
       end
 
       def write_to(io, *options)
         return super(io, *options) unless document.is_a?(HTML5::Document)
+
         options = options.first.is_a?(Hash) ? options.shift : {}
         encoding = options[:encoding] || options[0]
         if Nokogiri.jruby?
@@ -51,16 +56,15 @@ module Nokogiri
           native_write_to(io, encoding, indent_string, config_options)
         else
           # Serialize including the current node.
+          html = html_standard_serialize(options[:preserve_newline] || false)
           encoding ||= document.encoding || Encoding::UTF_8
-          internal_ops = {
-            preserve_newline: options[:preserve_newline] || false,
-          }
-          HTML5.serialize_node_internal(self, io, encoding, internal_ops)
+          io << html.encode(encoding, fallback: lambda { |c| "&#x#{c.ord.to_s(16)};" })
         end
       end
 
       def fragment(tags)
         return super(tags) unless document.is_a?(HTML5::Document)
+
         DocumentFragment.new(document, tags, self)
       end
 
@@ -73,6 +77,7 @@ module Nokogiri
       # actually create the xml namespace if it doesn't exist already.
       def add_child_node_and_reparent_attrs(node)
         return super(node) unless document.is_a?(HTML5::Document)
+
         # I'm not sure what this method is supposed to do. Reparenting
         # namespaces is handled by libxml2, including child namespaces which
         # this method wouldn't handle.