nokogiri 1.18.0.rc1-x86_64-linux-gnu

data/lib/nokogiri/html4/encoding_reader.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    # Libxml2's parser has poor support for encoding detection.  First, it does not recognize the
+    # HTML5 style meta charset declaration.  Secondly, even if it successfully detects an encoding
+    # hint, it does not re-decode or re-parse the preceding part which may be garbled.
+    #
+    # EncodingReader aims to perform advanced encoding detection beyond what Libxml2 does, and to
+    # emulate rewinding of a stream and make Libxml2 redo parsing from the start when an encoding
+    # hint is found.
+
+    # :nodoc: all
+    class EncodingReader
+      class EncodingFound < StandardError
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super(format("encoding found: %s", encoding))
+        end
+      end
+
+      class SAXHandler < Nokogiri::XML::SAX::Document
+        attr_reader :encoding
+
+        def initialize
+          @encoding = nil
+          super
+        end
+
+        def start_element(name, attrs = [])
+          return unless name == "meta"
+
+          attr = Hash[attrs]
+          (charset = attr["charset"]) &&
+            (@encoding = charset)
+          (http_equiv = attr["http-equiv"]) &&
+            http_equiv.match(/\AContent-Type\z/i) &&
+            (content = attr["content"]) &&
+            (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
+            (@encoding = m[1])
+        end
+      end
+
+      class JumpSAXHandler < SAXHandler
+        def initialize(jumptag)
+          @jumptag = jumptag
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          super
+          throw(@jumptag, @encoding) if @encoding
+          throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
+        end
+      end
+
+      def self.detect_encoding(chunk)
+        (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
+          (return Nokogiri.XML(m[1]).encoding)
+
+        if Nokogiri.jruby?
+          (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
+            (return m[4])
+          catch(:encoding_found) do
+            Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+            nil
+          end
+        else
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
+          begin
+            parser << chunk
+          rescue
+            Nokogiri::SyntaxError
+          end
+          handler.encoding
+        end
+      end
+
+      def initialize(io)
+        @io = io
+        @firstchunk = nil
+        @encoding_found = nil
+      end
+
+      # This method is used by the C extension so that
+      # Nokogiri::HTML4::Document#read_io() does not leak memory when
+      # EncodingFound is raised.
+      attr_reader :encoding_found
+
+      def read(len)
+        # no support for a call without len
+
+        unless @firstchunk
+          (@firstchunk = @io.read(len)) || return
+
+          # This implementation expects that the first call from
+          # htmlReadIO() is made with a length long enough (~1KB) to
+          # achieve advanced encoding detection.
+          if (encoding = EncodingReader.detect_encoding(@firstchunk))
+            # The first chunk is stored for the next read in retry.
+            raise @encoding_found = EncodingFound.new(encoding)
+          end
+        end
+        @encoding_found = nil
+
+        ret = @firstchunk.slice!(0, len)
+        if (len -= ret.length) > 0
+          (rest = @io.read(len)) && ret << (rest)
+        end
+        if ret.empty?
+          nil
+        else
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4/entity_lookup.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    class EntityDescription < Struct.new(:value, :name, :description); end
+
+    class EntityLookup
+      ###
+      # Look up entity with +name+
+      def [](name)
+        (val = get(name)) && val.value
+      end
+    end
+  end
+end

data/lib/nokogiri/html4/sax/parser.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    ###
+    # Nokogiri provides a SAX parser to process HTML4 which will provide HTML recovery
+    # ("autocorrection") features.
+    #
+    # See Nokogiri::HTML4::SAX::Parser for a basic example of using a SAX parser with HTML.
+    #
+    # For more information on SAX parsers, see Nokogiri::XML::SAX
+    #
+    module SAX
+      ###
+      # This parser is a SAX style parser that reads its input as it deems necessary. The parser
+      # takes a Nokogiri::XML::SAX::Document, an optional encoding, then given an HTML input, sends
+      # messages to the Nokogiri::XML::SAX::Document.
+      #
+      # ⚠ This is an HTML4 parser and so may not support some HTML5 features and behaviors.
+      #
+      # Here is a basic usage example:
+      #
+      #   class MyHandler < Nokogiri::XML::SAX::Document
+      #     def start_element name, attributes = []
+      #       puts "found a #{name}"
+      #     end
+      #   end
+      #
+      #   parser = Nokogiri::HTML4::SAX::Parser.new(MyHandler.new)
+      #
+      #   # Hand an IO object to the parser, which will read the HTML from the IO.
+      #   File.open(path_to_html) do |f|
+      #     parser.parse(f)
+      #   end
+      #
+      # For more information on \SAX parsers, see Nokogiri::XML::SAX or the parent class
+      # Nokogiri::XML::SAX::Parser.
+      #
+      # Also see Nokogiri::XML::SAX::Document for the available events.
+      #
+      class Parser < Nokogiri::XML::SAX::Parser
+        # this class inherits its behavior from Nokogiri::XML::SAX::Parser, but note that superclass
+        # uses Nokogiri::ClassResolver to use HTML4::SAX::ParserContext as the context class for
+        # this class, which is where the real behavioral differences are implemented.
+      end
+    end
+  end
+end

data/lib/nokogiri/html4/sax/parser_context.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    module SAX
+      ###
+      # Context object to invoke the HTML4 SAX parser on the SAX::Document handler.
+      #
+      # 💡 This class is usually not instantiated by the user. Use Nokogiri::HTML4::SAX::Parser
+      # instead.
+      class ParserContext < Nokogiri::XML::SAX::ParserContext
+      end
+    end
+  end
+end

data/lib/nokogiri/html4/sax/push_parser.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    module SAX
+      class PushParser
+        # The Nokogiri::HTML4::SAX::Document on which the PushParser will be
+        # operating
+        attr_accessor :document
+
+        def initialize(doc = HTML4::SAX::Document.new, file_name = nil, encoding = "UTF-8")
+          @document = doc
+          @encoding = encoding
+          @sax_parser = HTML4::SAX::Parser.new(doc, @encoding)
+
+          ## Create our push parser context
+          initialize_native(@sax_parser, file_name, encoding)
+        end
+
+        ###
+        # Write a +chunk+ of HTML to the PushParser.  Any callback methods
+        # that can be called will be called immediately.
+        def write(chunk, last_chunk = false)
+          native_write(chunk, last_chunk)
+        end
+        alias_method :<<, :write
+
+        ###
+        # Finish the parsing.  This method is only necessary for
+        # Nokogiri::HTML4::SAX::Document#end_document to be called.
+        def finish
+          write("", true)
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4.rb

@@ -0,0 +1,42 @@
+# coding: utf-8
+# frozen_string_literal: true
+
+module Nokogiri
+  class << self
+    # Convenience method for Nokogiri::HTML4::Document.parse
+    def HTML4(...)
+      Nokogiri::HTML4::Document.parse(...)
+    end
+  end
+
+  # Since v1.12.0
+  #
+  # 💡 Before v1.12.0, Nokogiri::HTML4 did not exist, and Nokogiri::HTML was the module/namespace
+  # for parsing HTML.
+  module HTML4
+    class << self
+      # Convenience method for Nokogiri::HTML4::Document.parse
+      def parse(...)
+        Document.parse(...)
+      end
+
+      # Convenience method for Nokogiri::HTML4::DocumentFragment.parse
+      def fragment(...)
+        HTML4::DocumentFragment.parse(...)
+      end
+    end
+
+    # Instance of Nokogiri::HTML4::EntityLookup
+    NamedCharacters = EntityLookup.new
+  end
+end
+
+require_relative "html4/entity_lookup"
+require_relative "html4/document"
+require_relative "html4/document_fragment"
+require_relative "html4/encoding_reader"
+require_relative "html4/sax/parser_context"
+require_relative "html4/sax/parser"
+require_relative "html4/sax/push_parser"
+require_relative "html4/element_description"
+require_relative "html4/element_description_defaults"

data/lib/nokogiri/html5/builder.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML5
+    ###
+    # Nokogiri HTML5 builder is used for building HTML documents. It is very similar to the
+    # Nokogiri::XML::Builder.  In fact, you should go read the documentation for
+    # Nokogiri::XML::Builder before reading this documentation.
+    #
+    # The construction behavior is identical to HTML4::Builder, but HTML5 documents implement the
+    # [HTML5 standard's serialization
+    # algorithm](https://www.w3.org/TR/2008/WD-html5-20080610/serializing.html).
+    #
+    # == Synopsis:
+    #
+    # Create an HTML5 document with a body that has an onload attribute, and a
+    # span tag with a class of "bold" that has content of "Hello world".
+    #
+    #   builder = Nokogiri::HTML5::Builder.new do |doc|
+    #     doc.html {
+    #       doc.body(:onload => 'some_func();') {
+    #         doc.span.bold {
+    #           doc.text "Hello world"
+    #         }
+    #       }
+    #     }
+    #   end
+    #   puts builder.to_html
+    #
+    # The HTML5 builder inherits from the XML builder, so make sure to read the
+    # Nokogiri::XML::Builder documentation.
+    class Builder < Nokogiri::XML::Builder
+      ###
+      # Convert the builder to HTML
+      def to_html
+        @doc.to_html
+      end
+    end
+  end
+end

data/lib/nokogiri/html5/document.rb

@@ -0,0 +1,199 @@
+# coding: utf-8
+# frozen_string_literal: true
+
+#
+#  Copyright 2013-2021 Sam Ruby, Stephen Checkoway
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+require_relative "../html4/document"
+
+module Nokogiri
+  module HTML5
+    # Enum for the HTML5 parser quirks mode values. Values returned by HTML5::Document#quirks_mode
+    #
+    # See https://dom.spec.whatwg.org/#concept-document-quirks for more information on HTML5 quirks
+    # mode.
+    #
+    # Since v1.14.0
+    module QuirksMode
+      NO_QUIRKS = 0 # The document was parsed in "no-quirks" mode
+      QUIRKS = 1 # The document was parsed in "quirks" mode
+      LIMITED_QUIRKS = 2 # The document was parsed in "limited-quirks" mode
+    end
+
+    # Since v1.12.0
+    #
+    # 💡 HTML5 functionality is not available when running JRuby.
+    class Document < Nokogiri::HTML4::Document
+      # Get the url name for this document, as passed into Document.parse, Document.read_io, or
+      # Document.read_memory
+      attr_reader :url
+
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns +nil+ if the parser was not invoked (e.g., Nokogiri::HTML5::Document.new).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
+
+      class << self
+        # :call-seq:
+        #   parse(input) { |options| ... } → HTML5::Document
+        #   parse(input, url: encoding:) { |options| ... } → HTML5::Document
+        #   parse(input, **options) → HTML5::Document
+        #
+        # Parse \HTML input with a parser compliant with the HTML5 spec. This method uses the
+        # encoding of +input+ if it can be determined, or else falls back to the +encoding:+
+        # parameter.
+        #
+        # [Required Parameters]
+        # - +input+ (String | IO) the \HTML content to be parsed.
+        #
+        # [Optional Parameters]
+        # - +url:+ (String) the base URI of the document.
+        #
+        # [Optional Keyword Arguments]
+        # - +encoding:+ (Encoding) The name of the encoding that should be used when processing the
+        #   document. When not provided, the encoding will be determined based on the document
+        #   content.
+        #
+        # - +max_errors:+ (Integer) The maximum number of parse errors to record. (default
+        #   +Nokogiri::Gumbo::DEFAULT_MAX_ERRORS+ which is currently 0)
+        #
+        # - +max_tree_depth:+ (Integer) The maximum depth of the parse tree. (default
+        #   +Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH+)
+        #
+        # - +max_attributes:+ (Integer) The maximum number of attributes allowed on an
+        #   element. (default +Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES+)
+        #
+        # - +parse_noscript_content_as_text:+ (Boolean) Whether to parse the content of +noscript+
+        #   elements as text. (default +false+)
+        #
+        # See rdoc-ref:HTML5@Parsing+options for a complete description of these parsing options.
+        #
+        # [Yields]
+        #   If present, the block will be passed a Hash object to modify with parse options before the
+        #   input is parsed. See rdoc-ref:HTML5@Parsing+options for a list of available options.
+        #
+        #   ⚠ Note that +url:+ and +encoding:+ cannot be set by the configuration block.
+        #
+        # [Returns] Nokogiri::HTML5::Document
+        #
+        # *Example:* Parse a string with a specific encoding and custom max errors limit.
+        #
+        #   Nokogiri::HTML5::Document.parse(socket, encoding: "ISO-8859-1", max_errors: 10)
+        #
+        # *Example:* Parse a string setting the +:parse_noscript_content_as_text+ option using the
+        # configuration block parameter.
+        #
+        #   Nokogiri::HTML5::Document.parse(input) { |c| c[:parse_noscript_content_as_text] = true }
+        #
+        def parse(
+          string_or_io,
+          url_ = nil, encoding_ = nil,
+          url: url_, encoding: encoding_,
+          **options, &block
+        )
+          yield options if block
+          string_or_io = "" unless string_or_io
+
+          if string_or_io.respond_to?(:encoding) && string_or_io.encoding != Encoding::ASCII_8BIT
+            encoding ||= string_or_io.encoding.name
+          end
+
+          if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
+            url ||= string_or_io.path
+          end
+          unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+            raise ArgumentError, "not a string or IO object"
+          end
+
+          do_parse(string_or_io, url, encoding, **options)
+        end
+
+        # Create a new document from an IO object.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_io(io, url_ = nil, encoding_ = nil, url: url_, encoding: encoding_, **options)
+          raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+
+          do_parse(io, url, encoding, **options)
+        end
+
+        # Create a new document from a String.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_memory(string, url_ = nil, encoding_ = nil, url: url_, encoding: encoding_, **options)
+          raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
+
+          do_parse(string, url, encoding, **options)
+        end
+
+        private
+
+        def do_parse(string_or_io, url, encoding, **options)
+          string = HTML5.read_and_encode(string_or_io, encoding)
+
+          options[:max_attributes] ||= Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
+          options[:max_errors] ||= options.delete(:max_parse_errors) || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
+          options[:max_tree_depth] ||= Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
+
+          doc = Nokogiri::Gumbo.parse(string, url, self, **options)
+          doc.encoding = "UTF-8"
+          doc
+        end
+      end
+
+      def initialize(*args) # :nodoc:
+        super
+        @url = nil
+        @quirks_mode = nil
+      end
+
+      # :call-seq:
+      #   fragment() → Nokogiri::HTML5::DocumentFragment
+      #   fragment(markup) → Nokogiri::HTML5::DocumentFragment
+      #
+      # Parse a HTML5 document fragment from +markup+, returning a Nokogiri::HTML5::DocumentFragment.
+      #
+      # [Properties]
+      # - +markup+ (String) The HTML5 markup fragment to be parsed
+      #
+      # [Returns]
+      #   Nokogiri::HTML5::DocumentFragment. This object's children will be empty if +markup+ is not
+      #   passed, is empty, or is +nil+.
+      #
+      def fragment(markup = nil)
+        DocumentFragment.new(self, markup)
+      end
+
+      def to_xml(options = {}, &block) # :nodoc:
+        # Bypass XML::Document#to_xml which doesn't add
+        # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does.
+        XML::Node.instance_method(:to_xml).bind_call(self, options, &block)
+      end
+
+      # :call-seq:
+      #   xpath_doctype() → Nokogiri::CSS::XPathVisitor::DoctypeConfig
+      #
+      # [Returns] The document type which determines CSS-to-XPath translation.
+      #
+      # See CSS::XPathVisitor for more information.
+      def xpath_doctype
+        Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML5
+      end
+    end
+  end
+end