nokogiri 1.10.9 → 1.18.3

data/lib/nokogiri/css.rb

@@ -1,27 +1,132 @@
-require 'nokogiri/css/node'
-require 'nokogiri/css/xpath_visitor'
-x = $-w
-$-w = false
-require 'nokogiri/css/parser'
-$-w = x
-
-require 'nokogiri/css/tokenizer'
-require 'nokogiri/css/syntax_error'
+# coding: utf-8
+# frozen_string_literal: true
 
 module Nokogiri
+  # Translate a CSS selector into an XPath 1.0 query
   module CSS
     class << self
-      ###
-      # Parse this CSS selector in +selector+.  Returns an AST.
-      def parse selector
-        Parser.new.parse selector
+      # TODO: Deprecate this method ahead of 2.0 and delete it in 2.0.
+      # It is not used by Nokogiri and shouldn't be part of the public API.
+      def parse(selector) # :nodoc:
+        warn("Nokogiri::CSS.parse is deprecated and will be removed in a future version of Nokogiri. Use Nokogiri::CSS::Parser#parse instead.", uplevel: 1, category: :deprecated)
+        Parser.new.parse(selector)
       end
 
-      ###
-      # Get the XPath for +selector+.
-      def xpath_for selector, options={}
-        Parser.new(options[:ns] || {}).xpath_for selector, options
+      # :call-seq:
+      #   xpath_for(selector_list) → Array<String>
+      #   xpath_for(selector_list [, prefix:] [, ns:] [, visitor:] [, cache:]) → Array<String>
+      #
+      # Translate a CSS selector list to the equivalent XPath expressions.
+      #
+      # 💡 Note that translated queries are cached by default for performance concerns.
+      #
+      # ⚠ Users should prefer Nokogiri::XML::Searchable#css, which is mixed into all document and
+      # node classes, for querying documents with CSS selectors. This method is the underlying
+      # mechanism used by XML::Searchable and is provided solely for advanced users to translate
+      # \CSS selectors to XPath directly.
+      #
+      # Also see Nokogiri::XML::Searchable#css for documentation on supported CSS selector features,
+      # some extended syntax that Nokogiri supports, and advanced CSS features like pseudo-class
+      # functions.
+      #
+      # [Parameters]
+      # - +selector_list+ (String)
+      #
+      #   The CSS selector to be translated into XPath. This is always a String, but that string
+      #   value may be a {selector list}[https://www.w3.org/TR/selectors-4/#grouping] (see
+      #   examples).
+      #
+      # [Keyword arguments]
+      # - +prefix:+ (String)
+      #
+      #   The XPath expression prefix which determines the search context. See Nokogiri::XML::XPath
+      #   for standard options. Default is +XPath::GLOBAL_SEARCH_PREFIX+.
+      #
+      # - +ns:+ (Hash<String ⇒ String>, nil)
+      #
+      #   Namespaces that are referenced in the query, if any. This is a hash where the keys are the
+      #   namespace prefix and the values are the namespace URIs. Default is +nil+ indicating an
+      #   empty set of namespaces.
+      #
+      # - +visitor:+ (Nokogiri::CSS::XPathVisitor)
+      #
+      #   Use this XPathVisitor object to transform the CSS AST into XPath expressions. See
+      #   Nokogiri::CSS::XPathVisitor for more information on some of the complex behavior that can
+      #   be customized for your document type. Default is +Nokogiri::CSS::XPathVisitor.new+.
+      #
+      #   ⚠ Note that this option is mutually exclusive with +prefix+ and +ns+. If +visitor+ is
+      #   provided, +prefix+ and +ns+ must not be present.
+      #
+      # - +cache:+ (Boolean)
+      #
+      #   Whether to use the SelectorCache for the translated query to ensure that repeated queries
+      #   don't incur the overhead of re-parsing the selector. Default is +true+.
+      #
+      # [Returns] (Array<String>) The equivalent set of XPath expressions for +selector_list+
+      #
+      # *Example* with a simple selector:
+      #
+      #   Nokogiri::CSS.xpath_for("div") # => ["//div"]
+      #
+      # *Example* with a compound selector:
+      #
+      #   Nokogiri::CSS.xpath_for("div.xl") # => ["//div[contains(concat(' ',normalize-space(@class),' '),' xl ')]"]
+      #
+      # *Example* with a complex selector:
+      #
+      #   Nokogiri::CSS.xpath_for("h1 + div") # => ["//h1/following-sibling::*[1]/self::div"]
+      #
+      # *Example* with a selector list:
+      #
+      #   Nokogiri::CSS.xpath_for("h1, h2, h3") # => ["//h1", "//h2", "//h3"]
+      #
+      def xpath_for(
+        selector, options = nil,
+        prefix: options&.delete(:prefix),
+        visitor: options&.delete(:visitor),
+        ns: options&.delete(:ns),
+        cache: true
+      )
+        unless options.nil?
+          warn("Nokogiri::CSS.xpath_for: Passing options as an explicit hash is deprecated. Use keyword arguments instead. This will become an error in a future release.", uplevel: 1, category: :deprecated)
+        end
+
+        raise(TypeError, "no implicit conversion of #{selector.inspect} to String") unless selector.respond_to?(:to_str)
+
+        selector = selector.to_str
+        raise(Nokogiri::CSS::SyntaxError, "empty CSS selector") if selector.empty?
+
+        if visitor
+          raise ArgumentError, "cannot provide both :prefix and :visitor" if prefix
+          raise ArgumentError, "cannot provide both :ns and :visitor" if ns
+        end
+
+        visitor ||= begin
+          visitor_kw = {}
+          visitor_kw[:prefix] = prefix if prefix
+          visitor_kw[:namespaces] = ns if ns
+
+          Nokogiri::CSS::XPathVisitor.new(**visitor_kw)
+        end
+
+        if cache
+          key = SelectorCache.key(selector: selector, visitor: visitor)
+          SelectorCache[key] ||= Parser.new.xpath_for(selector, visitor)
+        else
+          Parser.new.xpath_for(selector, visitor)
+        end
       end
     end
   end
 end
+
+require_relative "css/selector_cache"
+require_relative "css/node"
+require_relative "css/xpath_visitor"
+x = $-w
+$-w = false
+require_relative "css/parser"
+$-w = x
+
+require_relative "css/tokenizer"
+require_relative "css/syntax_error"

data/lib/nokogiri/decorators/slop.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Nokogiri
   module Decorators
     ###
@@ -9,31 +11,29 @@ module Nokogiri
 
       ###
       # look for node with +name+.  See Nokogiri.Slop
-      def method_missing name, *args, &block
+      def method_missing(name, *args, &block)
         if args.empty?
-          list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, '')}")
-        elsif args.first.is_a? Hash
+          list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, "")}")
+        elsif args.first.is_a?(Hash)
           hash = args.first
           if hash[:css]
             list = css("#{name}#{hash[:css]}")
           elsif hash[:xpath]
-            conds = Array(hash[:xpath]).join(' and ')
+            conds = Array(hash[:xpath]).join(" and ")
             list = xpath("#{XPATH_PREFIX}#{name}[#{conds}]")
           end
         else
-          CSS::Parser.without_cache do
-            list = xpath(
-              *CSS.xpath_for("#{name}#{args.first}", :prefix => XPATH_PREFIX)
-            )
-          end
+          list = xpath(
+            *CSS.xpath_for("#{name}#{args.first}", prefix: XPATH_PREFIX, cache: false),
+          )
         end
 
         super if list.empty?
         list.length == 1 ? list.first : list
       end
 
-      def respond_to_missing? name, include_private = false
-        list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, '')}")
+      def respond_to_missing?(name, include_private = false)
+        list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, "")}")
 
         !list.empty?
       end

data/lib/nokogiri/encoding_handler.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Nokogiri
+  class EncodingHandler
+    # Popular encoding aliases not known by all iconv implementations that Nokogiri should support.
+    USEFUL_ALIASES = {
+      # alias_name => true_name
+      "ISO-2022-JP" => "ISO-2022-JP", # only for JRuby tests, this is a no-op in CRuby
+      "NOKOGIRI-SENTINEL" => "ISO-2022-JP", # indicating the Nokogiri has installed aliases
+      "Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932.
+    }
+
+    class << self
+      def install_default_aliases
+        USEFUL_ALIASES.each do |alias_name, name|
+          EncodingHandler.alias(name, alias_name) if EncodingHandler[alias_name].nil?
+        end
+      end
+    end
+
+    # :stopdoc:
+    if Nokogiri.jruby?
+      class << self
+        def [](name)
+          storage.key?(name) ? new(storage[name]) : nil
+        end
+
+        def alias(name, alias_name)
+          storage[alias_name] = name
+        end
+
+        def delete(name)
+          storage.delete(name)
+        end
+
+        def clear_aliases!
+          storage.clear
+        end
+
+        private
+
+        def storage
+          @storage ||= {}
+        end
+      end
+
+      def initialize(name)
+        @name = name
+      end
+
+      attr_reader :name
+    end
+  end
+end
+
+Nokogiri::EncodingHandler.install_default_aliases

data/lib/nokogiri/extension.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# load the C or Java extension
+begin
+  # native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version>
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "#{Regexp.last_match(1)}/nokogiri"
+rescue LoadError => e
+  if e.message.include?("GLIBC")
+    warn(<<~EOM)
+
+      ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system
+             with an unsupported version of glibc.
+
+        #{e.message}
+
+        If that's the case, then please install Nokogiri via the `ruby` platform gem:
+            gem install nokogiri --platform=ruby
+        or:
+            bundle config set force_ruby_platform true
+
+        Please visit https://nokogiri.org/tutorials/installing_nokogiri.html for more help.
+
+    EOM
+    raise e
+  end
+
+  # use "require" instead of "require_relative" because non-native gems will place C extension files
+  # in Gem::BasicSpecification#extension_dir after compilation (during normal installation), which
+  # is in $LOAD_PATH but not necessarily relative to this file (see #2300)
+  require "nokogiri/nokogiri"
+end

data/lib/nokogiri/gumbo.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module Gumbo
+    # The default maximum number of attributes per element.
+    DEFAULT_MAX_ATTRIBUTES = 400
+
+    # The default maximum number of errors for parsing a document or a fragment.
+    DEFAULT_MAX_ERRORS = 0
+
+    # The default maximum depth of the DOM tree produced by parsing a document
+    # or fragment.
+    DEFAULT_MAX_TREE_DEPTH = 400
+  end
+end

data/lib/nokogiri/html.rb

@@ -1,37 +1,48 @@
-require 'nokogiri/html/entity_lookup'
-require 'nokogiri/html/document'
-require 'nokogiri/html/document_fragment'
-require 'nokogiri/html/sax/parser_context'
-require 'nokogiri/html/sax/parser'
-require 'nokogiri/html/sax/push_parser'
-require 'nokogiri/html/element_description'
-require 'nokogiri/html/element_description_defaults'
+# coding: utf-8
+# frozen_string_literal: true
+
+require_relative "html4"
 
 module Nokogiri
-  class << self
-    ###
-    # Parse HTML.  Convenience method for Nokogiri::HTML::Document.parse
-    def HTML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
-      Nokogiri::HTML::Document.parse(thing, url, encoding, options, &block)
-    end
-  end
+  # Alias for Nokogiri::HTML4
+  HTML = Nokogiri::HTML4
+
+  # :singleton-method: HTML
+  # :call-seq: HTML(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) → Nokogiri::HTML4::Document
+  #
+  # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse
 
+  # :nodoc:
+  define_singleton_method(:HTML, Nokogiri.method(:HTML4))
+
+  # 💡 This module/namespace is an alias for Nokogiri::HTML4 as of v1.12.0. Before v1.12.0,
+  #   Nokogiri::HTML4 did not exist, and this was the module/namespace for all HTML-related
+  #   classes.
   module HTML
-    class << self
-      ###
-      # Parse HTML.  Convenience method for Nokogiri::HTML::Document.parse
-      def parse thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
-        Document.parse(thing, url, encoding, options, &block)
+    # 💡 This class is an alias for Nokogiri::HTML4::Document as of v1.12.0.
+    class Document < Nokogiri::XML::Document
+    end
+
+    # 💡 This class is an alias for Nokogiri::HTML4::DocumentFragment as of v1.12.0.
+    class DocumentFragment < Nokogiri::XML::DocumentFragment
+    end
+
+    # 💡 This class is an alias for Nokogiri::HTML4::Builder as of v1.12.0.
+    class Builder < Nokogiri::XML::Builder
+    end
+
+    module SAX
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::Parser as of v1.12.0.
+      class Parser < Nokogiri::XML::SAX::Parser
       end
 
-      ####
-      # Parse a fragment from +string+ in to a NodeSet.
-      def fragment string, encoding = nil
-        HTML::DocumentFragment.parse string, encoding
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::ParserContext as of v1.12.0.
+      class ParserContext < Nokogiri::XML::SAX::ParserContext
       end
-    end
 
-    # Instance of Nokogiri::HTML::EntityLookup
-    NamedCharacters = EntityLookup.new
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::PushParser as of v1.12.0.
+      class PushParser
+      end
+    end
   end
 end

data/lib/nokogiri/{html → html4}/builder.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Nokogiri
-  module HTML
+  module HTML4
     ###
     # Nokogiri HTML builder is used for building HTML documents.  It is very
     # similar to the Nokogiri::XML::Builder.  In fact, you should go read the
@@ -11,7 +13,7 @@ module Nokogiri
     # Create an HTML document with a body that has an onload attribute, and a
     # span tag with a class of "bold" that has content of "Hello world".
     #
-    #   builder = Nokogiri::HTML::Builder.new do |doc|
+    #   builder = Nokogiri::HTML4::Builder.new do |doc|
     #     doc.html {
     #       doc.body(:onload => 'some_func();') {
     #         doc.span.bold {

data/lib/nokogiri/html4/document.rb

@@ -0,0 +1,235 @@
+# coding: utf-8
+# frozen_string_literal: true
+
+require "pathname"
+
+module Nokogiri
+  module HTML4
+    class Document < Nokogiri::XML::Document
+      ###
+      # Get the meta tag encoding for this document.  If there is no meta tag,
+      # then nil is returned.
+      def meta_encoding
+        if (meta = at_xpath("//meta[@charset]"))
+          meta[:charset]
+        elsif (meta = meta_content_type)
+          meta["content"][/charset\s*=\s*([\w-]+)/i, 1]
+        end
+      end
+
+      ###
+      # Set the meta tag encoding for this document.
+      #
+      # If an meta encoding tag is already present, its content is
+      # replaced with the given text.
+      #
+      # Otherwise, this method tries to create one at an appropriate
+      # place supplying head and/or html elements as necessary, which
+      # is inside a head element if any, and before any text node or
+      # content element (typically <body>) if any.
+      #
+      # The result when trying to set an encoding that is different
+      # from the document encoding is undefined.
+      #
+      # Beware in CRuby, that libxml2 automatically inserts a meta tag
+      # into a head element.
+      def meta_encoding=(encoding)
+        if (meta = meta_content_type)
+          meta["content"] = format("text/html; charset=%s", encoding)
+          encoding
+        elsif (meta = at_xpath("//meta[@charset]"))
+          meta["charset"] = encoding
+        else
+          meta = XML::Node.new("meta", self)
+          if (dtd = internal_subset) && dtd.html5_dtd?
+            meta["charset"] = encoding
+          else
+            meta["http-equiv"] = "Content-Type"
+            meta["content"] = format("text/html; charset=%s", encoding)
+          end
+
+          if (head = at_xpath("//head"))
+            head.prepend_child(meta)
+          else
+            set_metadata_element(meta)
+          end
+          encoding
+        end
+      end
+
+      def meta_content_type
+        xpath("//meta[@http-equiv and boolean(@content)]").find do |node|
+          node["http-equiv"] =~ /\AContent-Type\z/i
+        end
+      end
+      private :meta_content_type
+
+      ###
+      # Get the title string of this document.  Return nil if there is
+      # no title tag.
+      def title
+        (title = at_xpath("//title")) && title.inner_text
+      end
+
+      ###
+      # Set the title string of this document.
+      #
+      # If a title element is already present, its content is replaced
+      # with the given text.
+      #
+      # Otherwise, this method tries to create one at an appropriate
+      # place supplying head and/or html elements as necessary, which
+      # is inside a head element if any, right after a meta
+      # encoding/charset tag if any, and before any text node or
+      # content element (typically <body>) if any.
+      def title=(text)
+        tnode = XML::Text.new(text, self)
+        if (title = at_xpath("//title"))
+          title.children = tnode
+          return text
+        end
+
+        title = XML::Node.new("title", self) << tnode
+        if (head = at_xpath("//head"))
+          head << title
+        elsif (meta = at_xpath("//meta[@charset]") || meta_content_type)
+          # better put after charset declaration
+          meta.add_next_sibling(title)
+        else
+          set_metadata_element(title)
+        end
+      end
+
+      def set_metadata_element(element) # rubocop:disable Naming/AccessorMethodName
+        if (head = at_xpath("//head"))
+          head << element
+        elsif (html = at_xpath("//html"))
+          head = html.prepend_child(XML::Node.new("head", self))
+          head.prepend_child(element)
+        elsif (first = children.find do |node|
+                 case node
+                 when XML::Element, XML::Text
+                   true
+                 end
+               end)
+          # We reach here only if the underlying document model
+          # allows <html>/<head> elements to be omitted and does not
+          # automatically supply them.
+          first.add_previous_sibling(element)
+        else
+          html = add_child(XML::Node.new("html", self))
+          head = html.add_child(XML::Node.new("head", self))
+          head.prepend_child(element)
+        end
+      end
+      private :set_metadata_element
+
+      ####
+      # Serialize Node using +options+. Save options can also be set using a block.
+      #
+      # See also Nokogiri::XML::Node::SaveOptions and Node@Serialization+and+Generating+Output.
+      #
+      # These two statements are equivalent:
+      #
+      #  node.serialize(:encoding => 'UTF-8', :save_with => FORMAT | AS_XML)
+      #
+      # or
+      #
+      #   node.serialize(:encoding => 'UTF-8') do |config|
+      #     config.format.as_xml
+      #   end
+      #
+      def serialize(options = {})
+        options[:save_with] ||= XML::Node::SaveOptions::DEFAULT_HTML
+        super
+      end
+
+      ####
+      # Create a Nokogiri::XML::DocumentFragment from +tags+
+      def fragment(tags = nil)
+        DocumentFragment.new(self, tags, root)
+      end
+
+      # :call-seq:
+      #   xpath_doctype() → Nokogiri::CSS::XPathVisitor::DoctypeConfig
+      #
+      # [Returns] The document type which determines CSS-to-XPath translation.
+      #
+      # See XPathVisitor for more information.
+      def xpath_doctype
+        Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML4
+      end
+
+      class << self
+        # :call-seq:
+        #   parse(input) { |options| ... } => Nokogiri::HTML4::Document
+        #   parse(input, url:, encoding:, options:) => Nokogiri::HTML4::Document
+        #
+        # Parse \HTML4 input from a String or IO object, and return a new HTML4::Document.
+        #
+        # [Required Parameters]
+        # - +input+ (String | IO) The content to be parsed.
+        #
+        # [Optional Keyword Arguments]
+        # - +url:+ (String) The base URI for this document.
+        #
+        # - +encoding:+ (String) The name of the encoding that should be used when processing the
+        #   document. When not provided, the encoding will be determined based on the document
+        #   content.
+        #
+        # - +options:+ (Nokogiri::XML::ParseOptions) Configuration object that determines some
+        #   behaviors during parsing. See ParseOptions for more information. The default value is
+        #   +ParseOptions::DEFAULT_HTML+.
+        #
+        # [Yields]
+        #   If a block is given, a Nokogiri::XML::ParseOptions object is yielded to the block which
+        #   can be configured before parsing. See Nokogiri::XML::ParseOptions for more information.
+        #
+        # [Returns] Nokogiri::HTML4::Document
+        def parse(
+          input,
+          url_ = nil, encoding_ = nil, options_ = XML::ParseOptions::DEFAULT_HTML,
+          url: url_, encoding: encoding_, options: options_
+        )
+          options = Nokogiri::XML::ParseOptions.new(options) if Integer === options
+          yield options if block_given?
+
+          url ||= input.respond_to?(:path) ? input.path : nil
+
+          if input.respond_to?(:encoding)
+            unless input.encoding == Encoding::ASCII_8BIT
+              encoding ||= input.encoding.name
+            end
+          end
+
+          if input.respond_to?(:read)
+            if input.is_a?(Pathname)
+              # resolve the Pathname to the file and open it as an IO object, see #2110
+              input = input.expand_path.open
+              url ||= input.path
+            end
+
+            unless encoding
+              input = EncodingReader.new(input)
+              begin
+                return read_io(input, url, encoding, options.to_i)
+              rescue EncodingReader::EncodingFound => e
+                encoding = e.found_encoding
+              end
+            end
+            return read_io(input, url, encoding, options.to_i)
+          end
+
+          # read_memory pukes on empty docs
+          if input.nil? || input.empty?
+            return encoding ? new.tap { |i| i.encoding = encoding } : new
+          end
+
+          encoding ||= EncodingReader.detect_encoding(input)
+
+          read_memory(input, url, encoding, options.to_i)
+        end
+      end
+    end
+  end
+end