nokogiri 1.13.10 → 1.14.0.rc1

data/lib/nokogiri/html4/encoding_reader.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    # Libxml2's parser has poor support for encoding detection.  First, it does not recognize the
+    # HTML5 style meta charset declaration.  Secondly, even if it successfully detects an encoding
+    # hint, it does not re-decode or re-parse the preceding part which may be garbled.
+    #
+    # EncodingReader aims to perform advanced encoding detection beyond what Libxml2 does, and to
+    # emulate rewinding of a stream and make Libxml2 redo parsing from the start when an encoding
+    # hint is found.
+
+    # :nodoc: all
+    class EncodingReader
+      class EncodingFound < StandardError
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super(format("encoding found: %s", encoding))
+        end
+      end
+
+      class SAXHandler < Nokogiri::XML::SAX::Document
+        attr_reader :encoding
+
+        def initialize
+          @encoding = nil
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          return unless name == "meta"
+
+          attr = Hash[attrs]
+          (charset = attr["charset"]) &&
+            (@encoding = charset)
+          (http_equiv = attr["http-equiv"]) &&
+            http_equiv.match(/\AContent-Type\z/i) &&
+            (content = attr["content"]) &&
+            (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
+            (@encoding = m[1])
+        end
+      end
+
+      class JumpSAXHandler < SAXHandler
+        def initialize(jumptag)
+          @jumptag = jumptag
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          super
+          throw(@jumptag, @encoding) if @encoding
+          throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
+        end
+      end
+
+      def self.detect_encoding(chunk)
+        (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
+          (return Nokogiri.XML(m[1]).encoding)
+
+        if Nokogiri.jruby?
+          (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
+            (return m[4])
+          catch(:encoding_found) do
+            Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+            nil
+          end
+        else
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
+          begin
+            parser << chunk
+          rescue
+            Nokogiri::SyntaxError
+          end
+          handler.encoding
+        end
+      end
+
+      def initialize(io)
+        @io = io
+        @firstchunk = nil
+        @encoding_found = nil
+      end
+
+      # This method is used by the C extension so that
+      # Nokogiri::HTML4::Document#read_io() does not leak memory when
+      # EncodingFound is raised.
+      attr_reader :encoding_found
+
+      def read(len)
+        # no support for a call without len
+
+        unless @firstchunk
+          (@firstchunk = @io.read(len)) || (return nil)
+
+          # This implementation expects that the first call from
+          # htmlReadIO() is made with a length long enough (~1KB) to
+          # achieve advanced encoding detection.
+          if (encoding = EncodingReader.detect_encoding(@firstchunk))
+            # The first chunk is stored for the next read in retry.
+            raise @encoding_found = EncodingFound.new(encoding)
+          end
+        end
+        @encoding_found = nil
+
+        ret = @firstchunk.slice!(0, len)
+        if (len -= ret.length) > 0
+          (rest = @io.read(len)) && ret << (rest)
+        end
+        if ret.empty?
+          nil
+        else
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4.rb

@@ -39,6 +39,7 @@ end
 require_relative "html4/entity_lookup"
 require_relative "html4/document"
 require_relative "html4/document_fragment"
+require_relative "html4/encoding_reader"
 require_relative "html4/sax/parser_context"
 require_relative "html4/sax/parser"
 require_relative "html4/sax/push_parser"

data/lib/nokogiri/html5/document.rb

@@ -21,48 +21,137 @@ require_relative "../html4/document"
 
 module Nokogiri
   module HTML5
+    # Enum for the HTML5 parser quirks mode values. Values returned by HTML5::Document#quirks_mode
+    #
+    # See https://dom.spec.whatwg.org/#concept-document-quirks for more information on HTML5 quirks
+    # mode.
+    #
+    # Since v1.14.0
+    module QuirksMode
+      NO_QUIRKS = 0 # The document was parsed in "no-quirks" mode
+      QUIRKS = 1 # The document was parsed in "quirks" mode
+      LIMITED_QUIRKS = 2 # The document was parsed in "limited-quirks" mode
+    end
+
     # Since v1.12.0
     #
     # 💡 HTML5 functionality is not available when running JRuby.
     class Document < Nokogiri::HTML4::Document
-      def self.parse(string_or_io, url = nil, encoding = nil, **options, &block)
-        yield options if block
-        string_or_io = "" unless string_or_io
+      # Get the url name for this document, as passed into Document.parse, Document.read_io, or
+      # Document.read_memory
+      attr_reader :url
 
-        if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != "ASCII-8BIT"
-          encoding ||= string_or_io.encoding.name
-        end
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::Document.new`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
 
-        if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
-          url ||= string_or_io.path
+      class << self
+        # :call-seq:
+        #   parse(input)
+        #   parse(input, url=nil, encoding=nil, **options)
+        #   parse(input, url=nil, encoding=nil) { |options| ... }
+        #
+        # Parse HTML5 input.
+        #
+        # [Parameters]
+        # - +input+ may be a String, or any object that responds to _read_ and _close_ such as an
+        #   IO, or StringIO.
+        #
+        # - +url+ (optional) is a String indicating the canonical URI where this document is located.
+        #
+        # - +encoding+ (optional) is the encoding that should be used when processing
+        #   the document.
+        #
+        # - +options+ (optional) is a configuration Hash (or keyword arguments) to set options
+        #   during parsing. The three currently supported options are +:max_errors+,
+        #   +:max_tree_depth+ and +:max_attributes+, described at Nokogiri::HTML5.
+        #
+        #   ⚠ Note that these options are different than those made available by
+        #   Nokogiri::XML::Document and Nokogiri::HTML4::Document.
+        #
+        # - +block+ (optional) is passed a configuration Hash on which parse options may be set. See
+        #   Nokogiri::HTML5 for more information and usage.
+        #
+        # [Returns] Nokogiri::HTML5::Document
+        #
+        def parse(string_or_io, url = nil, encoding = nil, **options, &block)
+          yield options if block
+          string_or_io = "" unless string_or_io
+
+          if string_or_io.respond_to?(:encoding) && string_or_io.encoding != Encoding::ASCII_8BIT
+            encoding ||= string_or_io.encoding.name
+          end
+
+          if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
+            url ||= string_or_io.path
+          end
+          unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+            raise ArgumentError, "not a string or IO object"
+          end
+
+          do_parse(string_or_io, url, encoding, options)
         end
-        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
-          raise ArgumentError, "not a string or IO object"
+
+        # Create a new document from an IO object.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_io(io, url = nil, encoding = nil, **options)
+          raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+
+          do_parse(io, url, encoding, options)
         end
 
-        do_parse(string_or_io, url, encoding, options)
-      end
+        # Create a new document from a String.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_memory(string, url = nil, encoding = nil, **options)
+          raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
 
-      def self.read_io(io, url = nil, encoding = nil, **options)
-        raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+          do_parse(string, url, encoding, options)
+        end
 
-        do_parse(io, url, encoding, options)
-      end
+        private
 
-      def self.read_memory(string, url = nil, encoding = nil, **options)
-        raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
+        def do_parse(string_or_io, url, encoding, options)
+          string = HTML5.read_and_encode(string_or_io, encoding)
+          max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
+          max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
+          max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
+          doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth, self)
+          doc.encoding = "UTF-8"
+          doc
+        end
+      end
 
-        do_parse(string, url, encoding, options)
+      def initialize(*args) # :nodoc:
+        super
+        @url = nil
+        @quirks_mode = nil
       end
 
-      def fragment(tags = nil)
-        DocumentFragment.new(self, tags, root)
+      # :call-seq:
+      #   fragment() → Nokogiri::HTML5::DocumentFragment
+      #   fragment(markup) → Nokogiri::HTML5::DocumentFragment
+      #
+      # Parse a HTML5 document fragment from +markup+, returning a Nokogiri::HTML5::DocumentFragment.
+      #
+      # [Properties]
+      # - +markup+ (String) The HTML5 markup fragment to be parsed
+      #
+      # [Returns]
+      #   Nokogiri::HTML5::DocumentFragment. This object's children will be empty if `markup` is not passed, is empty, or is `nil`.
+      #
+      def fragment(markup = nil)
+        DocumentFragment.new(self, markup)
       end
 
-      def to_xml(options = {}, &block)
+      def to_xml(options = {}, &block) # :nodoc:
         # Bypass XML::Document#to_xml which doesn't add
         # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does.
-        XML::Node.instance_method(:to_xml).bind(self).call(options, &block)
+        XML::Node.instance_method(:to_xml).bind_call(self, options, &block)
       end
 
       # :call-seq:
@@ -70,22 +159,10 @@ module Nokogiri
       #
       # [Returns] The document type which determines CSS-to-XPath translation.
       #
-      # See XPathVisitor for more information.
+      # See CSS::XPathVisitor for more information.
       def xpath_doctype
         Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML5
       end
-
-      private
-
-      def self.do_parse(string_or_io, url, encoding, options)
-        string = HTML5.read_and_encode(string_or_io, encoding)
-        max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
-        max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
-        max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth)
-        doc.encoding = "UTF-8"
-        doc
-      end
     end
   end
 end

data/lib/nokogiri/html5/document_fragment.rb

@@ -28,6 +28,13 @@ module Nokogiri
       attr_accessor :document
       attr_accessor :errors
 
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::DocumentFragment.new(doc)`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
+
       # Create a document fragment.
       def initialize(doc, tags = nil, ctx = nil, options = {})
         self.document = doc
@@ -41,10 +48,10 @@ module Nokogiri
         Nokogiri::Gumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
 
-      def serialize(options = {}, &block)
+      def serialize(options = {}, &block) # :nodoc:
         # Bypass XML::Document.serialize which doesn't support options even
         # though XML::Node.serialize does!
-        XML::Node.instance_method(:serialize).bind(self).call(options, &block)
+        XML::Node.instance_method(:serialize).bind_call(self, options, &block)
       end
 
       # Parse a document fragment from +tags+, returning a Nodeset.

data/lib/nokogiri/html5/node.rb

@@ -28,7 +28,7 @@ module Nokogiri
       def inner_html(options = {})
         return super(options) unless document.is_a?(HTML5::Document)
 
-        result = options[:preserve_newline] && HTML5.prepend_newline?(self) ? +"\n" : +""
+        result = options[:preserve_newline] && prepend_newline? ? +"\n" : +""
         result << children.map { |child| child.to_html(options) }.join
         result
       end
@@ -56,11 +56,9 @@ module Nokogiri
           native_write_to(io, encoding, indent_string, config_options)
         else
           # Serialize including the current node.
+          html = html_standard_serialize(options[:preserve_newline] || false)
           encoding ||= document.encoding || Encoding::UTF_8
-          internal_ops = {
-            preserve_newline: options[:preserve_newline] || false,
-          }
-          HTML5.serialize_node_internal(self, io, encoding, internal_ops)
+          io << html.encode(encoding, fallback: lambda { |c| "&#x#{c.ord.to_s(16)};" })
         end
       end