nokogiri 1.13.10-java → 1.14.0.rc1-java

data/lib/nokogiri/html4/document.rb

@@ -176,7 +176,7 @@ module Nokogiri
           url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
 
           if string_or_io.respond_to?(:encoding)
-            unless string_or_io.encoding.name == "ASCII-8BIT"
+            unless string_or_io.encoding == Encoding::ASCII_8BIT
               encoding ||= string_or_io.encoding.name
             end
           end
@@ -189,21 +189,10 @@ module Nokogiri
             end
 
             unless encoding
-              # Libxml2's parser has poor support for encoding
-              # detection.  First, it does not recognize the HTML5
-              # style meta charset declaration.  Secondly, even if it
-              # successfully detects an encoding hint, it does not
-              # re-decode or re-parse the preceding part which may be
-              # garbled.
-              #
-              # EncodingReader aims to perform advanced encoding
-              # detection beyond what Libxml2 does, and to emulate
-              # rewinding of a stream and make Libxml2 redo parsing
-              # from the start when an encoding hint is found.
               string_or_io = EncodingReader.new(string_or_io)
               begin
                 return read_io(string_or_io, url, encoding, options.to_i)
-              rescue EncodingFound => e
+              rescue EncodingReader::EncodingFound => e
                 encoding = e.found_encoding
               end
             end
@@ -220,114 +209,6 @@ module Nokogiri
           read_memory(string_or_io, url, encoding, options.to_i)
         end
       end
-
-      class EncodingFound < StandardError # :nodoc: all
-        attr_reader :found_encoding
-
-        def initialize(encoding)
-          @found_encoding = encoding
-          super(format("encoding found: %s", encoding))
-        end
-      end
-
-      # :nodoc: all
-      class EncodingReader
-        class SAXHandler < Nokogiri::XML::SAX::Document
-          attr_reader :encoding
-
-          def initialize
-            @encoding = nil
-            super()
-          end
-
-          def start_element(name, attrs = [])
-            return unless name == "meta"
-
-            attr = Hash[attrs]
-            (charset = attr["charset"]) &&
-              (@encoding = charset)
-            (http_equiv = attr["http-equiv"]) &&
-              http_equiv.match(/\AContent-Type\z/i) &&
-              (content = attr["content"]) &&
-              (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
-              (@encoding = m[1])
-          end
-        end
-
-        class JumpSAXHandler < SAXHandler
-          def initialize(jumptag)
-            @jumptag = jumptag
-            super()
-          end
-
-          def start_element(name, attrs = [])
-            super
-            throw(@jumptag, @encoding) if @encoding
-            throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
-          end
-        end
-
-        def self.detect_encoding(chunk)
-          (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
-            (return Nokogiri.XML(m[1]).encoding)
-
-          if Nokogiri.jruby?
-            (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
-              (return m[4])
-            catch(:encoding_found) do
-              Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
-              nil
-            end
-          else
-            handler = SAXHandler.new
-            parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
-            begin
-              parser << chunk
-            rescue
-              Nokogiri::SyntaxError
-            end
-            handler.encoding
-          end
-        end
-
-        def initialize(io)
-          @io = io
-          @firstchunk = nil
-          @encoding_found = nil
-        end
-
-        # This method is used by the C extension so that
-        # Nokogiri::HTML4::Document#read_io() does not leak memory when
-        # EncodingFound is raised.
-        attr_reader :encoding_found
-
-        def read(len)
-          # no support for a call without len
-
-          unless @firstchunk
-            (@firstchunk = @io.read(len)) || (return nil)
-
-            # This implementation expects that the first call from
-            # htmlReadIO() is made with a length long enough (~1KB) to
-            # achieve advanced encoding detection.
-            if (encoding = EncodingReader.detect_encoding(@firstchunk))
-              # The first chunk is stored for the next read in retry.
-              raise @encoding_found = EncodingFound.new(encoding)
-            end
-          end
-          @encoding_found = nil
-
-          ret = @firstchunk.slice!(0, len)
-          if (len -= ret.length) > 0
-            (rest = @io.read(len)) && ret << (rest)
-          end
-          if ret.empty?
-            nil
-          else
-            ret
-          end
-        end
-      end
     end
   end
 end

data/lib/nokogiri/html4/element_description_defaults.rb

@@ -25,43 +25,37 @@ module Nokogiri
 
       unless method_defined?(:implied_start_tag?)
         def implied_start_tag?
-          d = default_desc
-          d ? d.startTag : nil
+          default_desc&.startTag
         end
       end
 
       unless method_defined?(:implied_end_tag?)
         def implied_end_tag?
-          d = default_desc
-          d ? d.endTag : nil
+          default_desc&.endTag
         end
       end
 
       unless method_defined?(:save_end_tag?)
         def save_end_tag?
-          d = default_desc
-          d ? d.saveEndTag : nil
+          default_desc&.saveEndTag
         end
       end
 
       unless method_defined?(:deprecated?)
         def deprecated?
-          d = default_desc
-          d ? d.depr : nil
+          default_desc&.depr
         end
       end
 
       unless method_defined?(:description)
         def description
-          d = default_desc
-          d ? d.desc : nil
+          default_desc&.desc
         end
       end
 
       unless method_defined?(:default_sub_element)
         def default_sub_element
-          d = default_desc
-          d ? d.defaultsubelt : nil
+          default_desc&.defaultsubelt
         end
       end
 

data/lib/nokogiri/html4/encoding_reader.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    # Libxml2's parser has poor support for encoding detection.  First, it does not recognize the
+    # HTML5 style meta charset declaration.  Secondly, even if it successfully detects an encoding
+    # hint, it does not re-decode or re-parse the preceding part which may be garbled.
+    #
+    # EncodingReader aims to perform advanced encoding detection beyond what Libxml2 does, and to
+    # emulate rewinding of a stream and make Libxml2 redo parsing from the start when an encoding
+    # hint is found.
+
+    # :nodoc: all
+    class EncodingReader
+      class EncodingFound < StandardError
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super(format("encoding found: %s", encoding))
+        end
+      end
+
+      class SAXHandler < Nokogiri::XML::SAX::Document
+        attr_reader :encoding
+
+        def initialize
+          @encoding = nil
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          return unless name == "meta"
+
+          attr = Hash[attrs]
+          (charset = attr["charset"]) &&
+            (@encoding = charset)
+          (http_equiv = attr["http-equiv"]) &&
+            http_equiv.match(/\AContent-Type\z/i) &&
+            (content = attr["content"]) &&
+            (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
+            (@encoding = m[1])
+        end
+      end
+
+      class JumpSAXHandler < SAXHandler
+        def initialize(jumptag)
+          @jumptag = jumptag
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          super
+          throw(@jumptag, @encoding) if @encoding
+          throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
+        end
+      end
+
+      def self.detect_encoding(chunk)
+        (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
+          (return Nokogiri.XML(m[1]).encoding)
+
+        if Nokogiri.jruby?
+          (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
+            (return m[4])
+          catch(:encoding_found) do
+            Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+            nil
+          end
+        else
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
+          begin
+            parser << chunk
+          rescue
+            Nokogiri::SyntaxError
+          end
+          handler.encoding
+        end
+      end
+
+      def initialize(io)
+        @io = io
+        @firstchunk = nil
+        @encoding_found = nil
+      end
+
+      # This method is used by the C extension so that
+      # Nokogiri::HTML4::Document#read_io() does not leak memory when
+      # EncodingFound is raised.
+      attr_reader :encoding_found
+
+      def read(len)
+        # no support for a call without len
+
+        unless @firstchunk
+          (@firstchunk = @io.read(len)) || (return nil)
+
+          # This implementation expects that the first call from
+          # htmlReadIO() is made with a length long enough (~1KB) to
+          # achieve advanced encoding detection.
+          if (encoding = EncodingReader.detect_encoding(@firstchunk))
+            # The first chunk is stored for the next read in retry.
+            raise @encoding_found = EncodingFound.new(encoding)
+          end
+        end
+        @encoding_found = nil
+
+        ret = @firstchunk.slice!(0, len)
+        if (len -= ret.length) > 0
+          (rest = @io.read(len)) && ret << (rest)
+        end
+        if ret.empty?
+          nil
+        else
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4.rb

@@ -39,6 +39,7 @@ end
 require_relative "html4/entity_lookup"
 require_relative "html4/document"
 require_relative "html4/document_fragment"
+require_relative "html4/encoding_reader"
 require_relative "html4/sax/parser_context"
 require_relative "html4/sax/parser"
 require_relative "html4/sax/push_parser"

data/lib/nokogiri/html5/document.rb

@@ -21,48 +21,137 @@ require_relative "../html4/document"
 
 module Nokogiri
   module HTML5
+    # Enum for the HTML5 parser quirks mode values. Values returned by HTML5::Document#quirks_mode
+    #
+    # See https://dom.spec.whatwg.org/#concept-document-quirks for more information on HTML5 quirks
+    # mode.
+    #
+    # Since v1.14.0
+    module QuirksMode
+      NO_QUIRKS = 0 # The document was parsed in "no-quirks" mode
+      QUIRKS = 1 # The document was parsed in "quirks" mode
+      LIMITED_QUIRKS = 2 # The document was parsed in "limited-quirks" mode
+    end
+
     # Since v1.12.0
     #
     # 💡 HTML5 functionality is not available when running JRuby.
     class Document < Nokogiri::HTML4::Document
-      def self.parse(string_or_io, url = nil, encoding = nil, **options, &block)
-        yield options if block
-        string_or_io = "" unless string_or_io
+      # Get the url name for this document, as passed into Document.parse, Document.read_io, or
+      # Document.read_memory
+      attr_reader :url
 
-        if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != "ASCII-8BIT"
-          encoding ||= string_or_io.encoding.name
-        end
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::Document.new`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
 
-        if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
-          url ||= string_or_io.path
+      class << self
+        # :call-seq:
+        #   parse(input)
+        #   parse(input, url=nil, encoding=nil, **options)
+        #   parse(input, url=nil, encoding=nil) { |options| ... }
+        #
+        # Parse HTML5 input.
+        #
+        # [Parameters]
+        # - +input+ may be a String, or any object that responds to _read_ and _close_ such as an
+        #   IO, or StringIO.
+        #
+        # - +url+ (optional) is a String indicating the canonical URI where this document is located.
+        #
+        # - +encoding+ (optional) is the encoding that should be used when processing
+        #   the document.
+        #
+        # - +options+ (optional) is a configuration Hash (or keyword arguments) to set options
+        #   during parsing. The three currently supported options are +:max_errors+,
+        #   +:max_tree_depth+ and +:max_attributes+, described at Nokogiri::HTML5.
+        #
+        #   ⚠ Note that these options are different than those made available by
+        #   Nokogiri::XML::Document and Nokogiri::HTML4::Document.
+        #
+        # - +block+ (optional) is passed a configuration Hash on which parse options may be set. See
+        #   Nokogiri::HTML5 for more information and usage.
+        #
+        # [Returns] Nokogiri::HTML5::Document
+        #
+        def parse(string_or_io, url = nil, encoding = nil, **options, &block)
+          yield options if block
+          string_or_io = "" unless string_or_io
+
+          if string_or_io.respond_to?(:encoding) && string_or_io.encoding != Encoding::ASCII_8BIT
+            encoding ||= string_or_io.encoding.name
+          end
+
+          if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
+            url ||= string_or_io.path
+          end
+          unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+            raise ArgumentError, "not a string or IO object"
+          end
+
+          do_parse(string_or_io, url, encoding, options)
         end
-        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
-          raise ArgumentError, "not a string or IO object"
+
+        # Create a new document from an IO object.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_io(io, url = nil, encoding = nil, **options)
+          raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+
+          do_parse(io, url, encoding, options)
         end
 
-        do_parse(string_or_io, url, encoding, options)
-      end
+        # Create a new document from a String.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_memory(string, url = nil, encoding = nil, **options)
+          raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
 
-      def self.read_io(io, url = nil, encoding = nil, **options)
-        raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+          do_parse(string, url, encoding, options)
+        end
 
-        do_parse(io, url, encoding, options)
-      end
+        private
 
-      def self.read_memory(string, url = nil, encoding = nil, **options)
-        raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
+        def do_parse(string_or_io, url, encoding, options)
+          string = HTML5.read_and_encode(string_or_io, encoding)
+          max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
+          max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
+          max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
+          doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth, self)
+          doc.encoding = "UTF-8"
+          doc
+        end
+      end
 
-        do_parse(string, url, encoding, options)
+      def initialize(*args) # :nodoc:
+        super
+        @url = nil
+        @quirks_mode = nil
       end
 
-      def fragment(tags = nil)
-        DocumentFragment.new(self, tags, root)
+      # :call-seq:
+      #   fragment() → Nokogiri::HTML5::DocumentFragment
+      #   fragment(markup) → Nokogiri::HTML5::DocumentFragment
+      #
+      # Parse a HTML5 document fragment from +markup+, returning a Nokogiri::HTML5::DocumentFragment.
+      #
+      # [Properties]
+      # - +markup+ (String) The HTML5 markup fragment to be parsed
+      #
+      # [Returns]
+      #   Nokogiri::HTML5::DocumentFragment. This object's children will be empty if `markup` is not passed, is empty, or is `nil`.
+      #
+      def fragment(markup = nil)
+        DocumentFragment.new(self, markup)
       end
 
-      def to_xml(options = {}, &block)
+      def to_xml(options = {}, &block) # :nodoc:
         # Bypass XML::Document#to_xml which doesn't add
         # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does.
-        XML::Node.instance_method(:to_xml).bind(self).call(options, &block)
+        XML::Node.instance_method(:to_xml).bind_call(self, options, &block)
       end
 
       # :call-seq:
@@ -70,22 +159,10 @@ module Nokogiri
       #
       # [Returns] The document type which determines CSS-to-XPath translation.
       #
-      # See XPathVisitor for more information.
+      # See CSS::XPathVisitor for more information.
       def xpath_doctype
         Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML5
       end
-
-      private
-
-      def self.do_parse(string_or_io, url, encoding, options)
-        string = HTML5.read_and_encode(string_or_io, encoding)
-        max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
-        max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
-        max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth)
-        doc.encoding = "UTF-8"
-        doc
-      end
     end
   end
 end

data/lib/nokogiri/html5/document_fragment.rb

@@ -28,6 +28,13 @@ module Nokogiri
       attr_accessor :document
       attr_accessor :errors
 
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::DocumentFragment.new(doc)`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
+
       # Create a document fragment.
       def initialize(doc, tags = nil, ctx = nil, options = {})
         self.document = doc
@@ -41,10 +48,10 @@ module Nokogiri
         Nokogiri::Gumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
 
-      def serialize(options = {}, &block)
+      def serialize(options = {}, &block) # :nodoc:
         # Bypass XML::Document.serialize which doesn't support options even
         # though XML::Node.serialize does!
-        XML::Node.instance_method(:serialize).bind(self).call(options, &block)
+        XML::Node.instance_method(:serialize).bind_call(self, options, &block)
       end
 
       # Parse a document fragment from +tags+, returning a Nodeset.

data/lib/nokogiri/html5/node.rb

@@ -28,7 +28,7 @@ module Nokogiri
       def inner_html(options = {})
         return super(options) unless document.is_a?(HTML5::Document)
 
-        result = options[:preserve_newline] && HTML5.prepend_newline?(self) ? +"\n" : +""
+        result = options[:preserve_newline] && prepend_newline? ? +"\n" : +""
         result << children.map { |child| child.to_html(options) }.join
         result
       end
@@ -56,11 +56,9 @@ module Nokogiri
           native_write_to(io, encoding, indent_string, config_options)
         else
           # Serialize including the current node.
+          html = html_standard_serialize(options[:preserve_newline] || false)
           encoding ||= document.encoding || Encoding::UTF_8
-          internal_ops = {
-            preserve_newline: options[:preserve_newline] || false,
-          }
-          HTML5.serialize_node_internal(self, io, encoding, internal_ops)
+          io << html.encode(encoding, fallback: lambda { |c| "&#x#{c.ord.to_s(16)};" })
         end
       end