nokogiri 1.12.5 → 1.13.9

data/lib/nokogiri/css/xpath_visitor.rb

@@ -1,42 +1,116 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 module Nokogiri
   module CSS
-    class XPathVisitor # :nodoc:
-      def visit_function node
-        msg = :"visit_function_#{node.value.first.gsub(/[(]/, '')}"
-        return self.send(msg, node) if self.respond_to?(msg)
+    # When translating CSS selectors to XPath queries with Nokogiri::CSS.xpath_for, the XPathVisitor
+    # class allows for changing some of the behaviors related to builtin xpath functions and quirks
+    # of HTML5.
+    class XPathVisitor
+      WILDCARD_NAMESPACES = Nokogiri.libxml2_patches.include?("0009-allow-wildcard-namespaces.patch") # :nodoc:
+
+      # Enum to direct XPathVisitor when to use Nokogiri builtin XPath functions.
+      module BuiltinsConfig
+        # Never use Nokogiri builtin functions, always generate vanilla XPath 1.0 queries. This is
+        # the default when calling Nokogiri::CSS.xpath_for directly.
+        NEVER = :never
+
+        # Always use Nokogiri builtin functions whenever possible. This is probably only useful for testing.
+        ALWAYS = :always
+
+        # Only use Nokogiri builtin functions when they will be faster than vanilla XPath. This is
+        # the behavior chosen when searching for CSS selectors on a Nokogiri document, fragment, or
+        # node.
+        OPTIMAL = :optimal
+
+        # :nodoc: array of values for validation
+        VALUES = [NEVER, ALWAYS, OPTIMAL]
+      end
+
+      # Enum to direct XPathVisitor when to tweak the XPath query to suit the nature of the document
+      # being searched. Note that searches for CSS selectors from a Nokogiri document, fragment, or
+      # node will choose the correct option automatically.
+      module DoctypeConfig
+        # The document being searched is an XML document. This is the default.
+        XML = :xml
+
+        # The document being searched is an HTML4 document.
+        HTML4 = :html4
+
+        # The document being searched is an HTML5 document.
+        HTML5 = :html5
+
+        # :nodoc: array of values for validation
+        VALUES = [XML, HTML4, HTML5]
+      end
+
+      # :call-seq:
+      #   new() → XPathVisitor
+      #   new(builtins:, doctype:) → XPathVisitor
+      #
+      # [Parameters]
+      # - +builtins:+ (BuiltinsConfig) Determine when to use Nokogiri's built-in xpath functions for performance improvements.
+      # - +doctype:+ (DoctypeConfig) Make document-type-specific accommodations for CSS queries.
+      #
+      # [Returns] XPathVisitor
+      #
+      def initialize(builtins: BuiltinsConfig::NEVER, doctype: DoctypeConfig::XML)
+        unless BuiltinsConfig::VALUES.include?(builtins)
+          raise(ArgumentError, "Invalid values #{builtins.inspect} for builtins: keyword parameter")
+        end
+        unless DoctypeConfig::VALUES.include?(doctype)
+          raise(ArgumentError, "Invalid values #{doctype.inspect} for doctype: keyword parameter")
+        end
+
+        @builtins = builtins
+        @doctype = doctype
+      end
+
+      # :call-seq: config() → Hash
+      #
+      # [Returns]
+      #   a Hash representing the configuration of the XPathVisitor, suitable for use as
+      #   part of the CSS cache key.
+      def config
+        { builtins: @builtins, doctype: @doctype }
+      end
+
+      # :stopdoc:
+      def visit_function(node)
+        msg = :"visit_function_#{node.value.first.gsub(/[(]/, "")}"
+        return send(msg, node) if respond_to?(msg)
 
         case node.value.first
         when /^text\(/
-          'child::text()'
+          "child::text()"
         when /^self\(/
           "self::#{node.value[1]}"
         when /^eq\(/
           "position()=#{node.value[1]}"
         when /^(nth|nth-of-type)\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
             nth(node.value[1])
           else
             "position()=#{node.value[1]}"
           end
         when /^nth-child\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
-            nth(node.value[1], :child => true)
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1], child: true)
           else
-            "count(preceding-sibling::*)=#{node.value[1].to_i-1}"
+            "count(preceding-sibling::*)=#{node.value[1].to_i - 1}"
           end
         when /^nth-last-of-type\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
-            nth(node.value[1], :last => true)
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1], last: true)
           else
             index = node.value[1].to_i - 1
             index == 0 ? "position()=last()" : "position()=last()-#{index}"
           end
         when /^nth-last-child\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
-            nth(node.value[1], :last => true, :child => true)
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1], last: true, child: true)
           else
-            "count(following-sibling::*)=#{node.value[1].to_i-1}"
+            "count(following-sibling::*)=#{node.value[1].to_i - 1}"
           end
         when /^(first|first-of-type)\(/
           "position()=1"
@@ -52,15 +126,18 @@ module Nokogiri
           "comment()"
         when /^has\(/
           is_direct = node.value[1].value[0].nil? # e.g. "has(> a)", "has(~ a)", "has(+ a)"
-          ".#{"//" if !is_direct}#{node.value[1].accept(self)}"
+          ".#{"//" unless is_direct}#{node.value[1].accept(self)}"
         else
-          # non-standard. this looks like a function call.
-          args = ['.'] + node.value[1..-1]
-          "#{node.value.first}#{args.join(',')})"
+          # xpath function call, let's marshal those arguments
+          args = ["."]
+          args += node.value[1..-1].map do |n|
+            n.is_a?(Nokogiri::CSS::Node) ? n.accept(self) : n
+          end
+          "#{node.value.first}#{args.join(",")})"
         end
       end
 
-      def visit_not node
+      def visit_not(node)
         child = node.value.first
         if :ELEMENT_NAME == child.type
           "not(self::#{child.accept(self)})"
@@ -69,29 +146,20 @@ module Nokogiri
         end
       end
 
-      def visit_id node
+      def visit_id(node)
         node.value.first =~ /^#(.*)$/
-        "@id='#{$1}'"
+        "@id='#{Regexp.last_match(1)}'"
       end
 
-      def visit_attribute_condition node
-        attribute = if (node.value.first.type == :FUNCTION) or (node.value.first.value.first =~ /::/)
-                      ''
-                    else
-                      '@'
-                    end
-        attribute += node.value.first.accept(self)
-
-        # non-standard. attributes starting with '@'
-        attribute.gsub!(/^@@/, '@')
-
-        return attribute unless node.value.length == 3
+      def visit_attribute_condition(node)
+        attribute = node.value.first.accept(self)
+        return attribute if node.value.length == 1
 
         value = node.value.last
-        value = "'#{value}'" if value !~ /^['"]/
+        value = "'#{value}'" unless /^['"]/.match?(value)
 
         # quoted values - see test_attribute_value_with_quotes in test/css/test_parser.rb
-        if (value[0]==value[-1]) && %q{"'}.include?(value[0])
+        if (value[0] == value[-1]) && %q{"'}.include?(value[0])
           str_value = value[1..-2]
           if str_value.include?(value[0])
             value = 'concat("' + str_value.split('"', -1).join(%q{",'"',"}) + '","")'
@@ -100,9 +168,9 @@ module Nokogiri
 
         case node.value[1]
         when :equal
-          attribute + "=" + "#{value}"
+          attribute + "=" + value.to_s
         when :not_equal
-          attribute + "!=" + "#{value}"
+          attribute + "!=" + value.to_s
         when :substring_match
           "contains(#{attribute},#{value})"
         when :prefix_match
@@ -115,16 +183,16 @@ module Nokogiri
         when :suffix_match
           "substring(#{attribute},string-length(#{attribute})-string-length(#{value})+1,string-length(#{value}))=#{value}"
         else
-          attribute + " #{node.value[1]} " + "#{value}"
+          attribute + " #{node.value[1]} " + value.to_s
         end
       end
 
-      def visit_pseudo_class node
-        if node.value.first.is_a?(Nokogiri::CSS::Node) and node.value.first.type == :FUNCTION
+      def visit_pseudo_class(node)
+        if node.value.first.is_a?(Nokogiri::CSS::Node) && (node.value.first.type == :FUNCTION)
           node.value.first.accept(self)
         else
-          msg = :"visit_pseudo_class_#{node.value.first.gsub(/[(]/, '')}"
-          return self.send(msg, node) if self.respond_to?(msg)
+          msg = :"visit_pseudo_class_#{node.value.first.gsub(/[(]/, "")}"
+          return send(msg, node) if respond_to?(msg)
 
           case node.value.first
           when "first" then "position()=1"
@@ -144,24 +212,24 @@ module Nokogiri
         end
       end
 
-      def visit_class_condition node
+      def visit_class_condition(node)
         css_class("@class", node.value.first)
       end
 
-      def visit_combinator node
+      def visit_combinator(node)
         if is_of_type_pseudo_class?(node.value.last)
-          "#{node.value.first.accept(self) if node.value.first}][#{node.value.last.accept(self)}"
+          "#{node.value.first&.accept(self)}][#{node.value.last.accept(self)}"
         else
-          "#{node.value.first.accept(self) if node.value.first} and #{node.value.last.accept(self)}"
+          "#{node.value.first&.accept(self)} and #{node.value.last.accept(self)}"
         end
       end
 
       {
-        'direct_adjacent_selector'  => "/following-sibling::*[1]/self::",
-        'following_selector'        => "/following-sibling::",
-        'descendant_selector'       => '//',
-        'child_selector'            => '/',
-      }.each do |k,v|
+        "direct_adjacent_selector" => "/following-sibling::*[1]/self::",
+        "following_selector" => "/following-sibling::",
+        "descendant_selector" => "//",
+        "child_selector" => "/",
+      }.each do |k, v|
         class_eval %{
           def visit_#{k} node
             "\#{node.value.first.accept(self) if node.value.first}#{v}\#{node.value.last.accept(self)}"
@@ -169,25 +237,50 @@ module Nokogiri
         }
       end
 
-      def visit_conditional_selector node
-        node.value.first.accept(self) + '[' +
-        node.value.last.accept(self) + ']'
+      def visit_conditional_selector(node)
+        node.value.first.accept(self) + "[" +
+          node.value.last.accept(self) + "]"
+      end
+
+      def visit_element_name(node)
+        if @doctype == DoctypeConfig::HTML5 && html5_element_name_needs_namespace_handling(node)
+          # HTML5 has namespaces that should be ignored in CSS queries
+          # https://github.com/sparklemotion/nokogiri/issues/2376
+          if @builtins == BuiltinsConfig::ALWAYS || (@builtins == BuiltinsConfig::OPTIMAL && Nokogiri.uses_libxml?)
+            if WILDCARD_NAMESPACES
+              "*:#{node.value.first}"
+            else
+              "*[nokogiri-builtin:local-name-is('#{node.value.first}')]"
+            end
+          else
+            "*[local-name()='#{node.value.first}']"
+          end
+        else
+          node.value.first
+        end
       end
 
-      def visit_element_name node
-        node.value.first
+      def visit_attrib_name(node)
+        "@#{node.value.first}"
       end
 
-      def accept node
+      def accept(node)
         node.accept(self)
       end
 
       private
 
-      def nth node, options={}
+      def html5_element_name_needs_namespace_handling(node)
+        # if this is the wildcard selector "*", use it as normal
+        node.value.first != "*" &&
+          # if there is already a namespace (i.e., it is a prefixed QName), use it as normal
+          !node.value.first.include?(":")
+      end
+
+      def nth(node, options = {})
         raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
 
-        a, b = read_a_and_positive_b node.value
+        a, b = read_a_and_positive_b(node.value)
         position = if options[:child]
           options[:last] ? "(count(following-sibling::*)+1)" : "(count(preceding-sibling::*)+1)"
         else
@@ -206,7 +299,7 @@ module Nokogiri
         end
       end
 
-      def read_a_and_positive_b values
+      def read_a_and_positive_b(values)
         op = values[2]
         if op == "+"
           a = values[0].to_i
@@ -220,9 +313,9 @@ module Nokogiri
         [a, b]
       end
 
-      def is_of_type_pseudo_class? node
-        if node.type==:PSEUDO_CLASS
-          if node.value[0].is_a?(Nokogiri::CSS::Node) and node.value[0].type == :FUNCTION
+      def is_of_type_pseudo_class?(node) # rubocop:disable Naming/PredicateName
+        if node.type == :PSEUDO_CLASS
+          if node.value[0].is_a?(Nokogiri::CSS::Node) && (node.value[0].type == :FUNCTION)
             node.value[0].value[0]
           else
             node.value[0]
@@ -230,30 +323,34 @@ module Nokogiri
         end
       end
 
-      # use only ordinary xpath functions
-      def css_class_standard(hay, needle)
-        "contains(concat(' ',normalize-space(#{hay}),' '),' #{needle} ')"
-      end
-
-      # use the builtin implementation
-      def css_class_builtin(hay, needle)
-        "nokogiri-builtin:css-class(#{hay},'#{needle}')"
+      def css_class(hay, needle)
+        if @builtins == BuiltinsConfig::ALWAYS || (@builtins == BuiltinsConfig::OPTIMAL && Nokogiri.uses_libxml?)
+          # use the builtin implementation
+          "nokogiri-builtin:css-class(#{hay},'#{needle}')"
+        else
+          # use only ordinary xpath functions
+          "contains(concat(' ',normalize-space(#{hay}),' '),' #{needle} ')"
+        end
       end
-
-      alias_method :css_class, :css_class_standard
     end
 
-    class XPathVisitorAlwaysUseBuiltins < XPathVisitor # :nodoc:
-      private
-      alias_method :css_class, :css_class_builtin
+    module XPathVisitorAlwaysUseBuiltins # :nodoc:
+      def self.new
+        warn(
+          "Nokogiri::CSS::XPathVisitorAlwaysUseBuiltins is deprecated and will be removed in a future version of Nokogiri",
+          { uplevel: 1 },
+        )
+        XPathVisitor.new(builtins: :always)
+      end
     end
 
-    class XPathVisitorOptimallyUseBuiltins < XPathVisitor # :nodoc:
-      private
-      if Nokogiri.uses_libxml?
-        alias_method :css_class, :css_class_builtin
-      else
-        alias_method :css_class, :css_class_standard
+    module XPathVisitorOptimallyUseBuiltins # :nodoc:
+      def self.new
+        warn(
+          "Nokogiri::CSS::XPathVisitorOptimallyUseBuiltins is deprecated and will be removed in a future version of Nokogiri",
+          { uplevel: 1 },
+        )
+        XPathVisitor.new(builtins: :optimal)
       end
     end
   end

data/lib/nokogiri/css.rb

@@ -1,17 +1,49 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 module Nokogiri
+  # Translate a CSS selector into an XPath 1.0 query
   module CSS
     class << self
-      ###
-      # Parse this CSS selector in +selector+.  Returns an AST.
-      def parse(selector)
+      # TODO: Deprecate this method ahead of 2.0 and delete it in 2.0.
+      # It is not used by Nokogiri and shouldn't be part of the public API.
+      def parse(selector) # :nodoc:
         Parser.new.parse(selector)
       end
 
-      ###
-      # Get the XPath for +selector+.
+      # :call-seq:
+      #   xpath_for(selector) → String
+      #   xpath_for(selector [, prefix:] [, visitor:] [, ns:]) → String
+      #
+      # Translate a CSS selector to the equivalent XPath query.
+      #
+      # [Parameters]
+      # - +selector+ (String) The CSS selector to be translated into XPath
+      #
+      # - +prefix:+ (String)
+      #
+      #   The XPath prefix for the query, see Nokogiri::XML::XPath for some options. Default is
+      #   +XML::XPath::GLOBAL_SEARCH_PREFIX+.
+      #
+      # - +visitor:+ (Nokogiri::CSS::XPathVisitor)
+      #
+      #   The visitor class to use to transform the AST into XPath. Default is
+      #   +Nokogiri::CSS::XPathVisitor.new+.
+      #
+      # - +ns:+ (Hash<String ⇒ String>)
+      #
+      #   The namespaces that are referenced in the query, if any. This is a hash where the keys are
+      #   the namespace prefix and the values are the namespace URIs. Default is an empty Hash.
+      #
+      # [Returns] (String) The equivalent XPath query for +selector+
+      #
+      # 💡 Note that translated queries are cached for performance concerns.
+      #
       def xpath_for(selector, options = {})
-        Parser.new(options[:ns] || {}).xpath_for(selector, options)
+        prefix = options.fetch(:prefix, Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX)
+        visitor = options.fetch(:visitor) { Nokogiri::CSS::XPathVisitor.new }
+        ns = options.fetch(:ns, {})
+        Parser.new(ns).xpath_for(selector, prefix, visitor)
       end
     end
   end

data/lib/nokogiri/decorators/slop.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module Decorators
     ###
@@ -10,21 +11,21 @@ module Nokogiri
 
       ###
       # look for node with +name+.  See Nokogiri.Slop
-      def method_missing name, *args, &block
+      def method_missing(name, *args, &block)
         if args.empty?
-          list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, '')}")
-        elsif args.first.is_a? Hash
+          list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, "")}")
+        elsif args.first.is_a?(Hash)
           hash = args.first
           if hash[:css]
             list = css("#{name}#{hash[:css]}")
           elsif hash[:xpath]
-            conds = Array(hash[:xpath]).join(' and ')
+            conds = Array(hash[:xpath]).join(" and ")
             list = xpath("#{XPATH_PREFIX}#{name}[#{conds}]")
           end
         else
           CSS::Parser.without_cache do
             list = xpath(
-              *CSS.xpath_for("#{name}#{args.first}", :prefix => XPATH_PREFIX)
+              *CSS.xpath_for("#{name}#{args.first}", prefix: XPATH_PREFIX)
             )
           end
         end
@@ -33,8 +34,8 @@ module Nokogiri
         list.length == 1 ? list.first : list
       end
 
-      def respond_to_missing? name, include_private = false
-        list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, '')}")
+      def respond_to_missing?(name, include_private = false)
+        list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, "")}")
 
         !list.empty?
       end

data/lib/nokogiri/extension.rb

@@ -6,7 +6,7 @@ begin
   ::RUBY_VERSION =~ /(\d+\.\d+)/
   require_relative "#{Regexp.last_match(1)}/nokogiri"
 rescue LoadError => e
-  if e.message =~ /GLIBC/
+  if /GLIBC/.match?(e.message)
     warn(<<~EOM)
 
       ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system with glibc < 2.17:

data/lib/nokogiri/gumbo.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module Gumbo
     # The default maximum number of attributes per element.

data/lib/nokogiri/html.rb

@@ -1,40 +1,46 @@
+# coding: utf-8
 # frozen_string_literal: true
+
 require_relative "html4"
 
 module Nokogiri
+  # Alias for Nokogiri::HTML4
   HTML = Nokogiri::HTML4
 
-  # @!method HTML(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
+  # :singleton-method: HTML
+  # :call-seq: HTML(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) → Nokogiri::HTML4::Document
+  #
   # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse
-  # @!scope class
+
+  # :nodoc:
   define_singleton_method(:HTML, Nokogiri.method(:HTML4))
 
-  # @note This module/namespace is an alias for {Nokogiri::HTML4} as of v1.12.0. Before v1.12.0,
-  #   {Nokogiri::HTML4} did not exist, and this was the module/namespace for all HTML-related
+  # 💡 This module/namespace is an alias for Nokogiri::HTML4 as of v1.12.0. Before v1.12.0,
+  #   Nokogiri::HTML4 did not exist, and this was the module/namespace for all HTML-related
   #   classes.
   module HTML
-    # @note This class is an alias for {Nokogiri::HTML4::Document} as of v1.12.0.
+    # 💡 This class is an alias for Nokogiri::HTML4::Document as of v1.12.0.
     class Document < Nokogiri::XML::Document
     end
 
-    # @note This class is an alias for {Nokogiri::HTML4::DocumentFragment} as of v1.12.0.
+    # 💡 This class is an alias for Nokogiri::HTML4::DocumentFragment as of v1.12.0.
     class DocumentFragment < Nokogiri::XML::DocumentFragment
     end
 
-    # @note This class is an alias for {Nokogiri::HTML4::Builder} as of v1.12.0.
+    # 💡 This class is an alias for Nokogiri::HTML4::Builder as of v1.12.0.
     class Builder < Nokogiri::XML::Builder
     end
 
     module SAX
-      # @note This class is an alias for {Nokogiri::HTML4::SAX::Parser} as of v1.12.0.
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::Parser as of v1.12.0.
       class Parser < Nokogiri::XML::SAX::Parser
       end
 
-      # @note This class is an alias for {Nokogiri::HTML4::SAX::ParserContext} as of v1.12.0.
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::ParserContext as of v1.12.0.
       class ParserContext < Nokogiri::XML::SAX::ParserContext
       end
 
-      # @note This class is an alias for {Nokogiri::HTML4::SAX::PushParser} as of v1.12.0.
+      # 💡 This class is an alias for Nokogiri::HTML4::SAX::PushParser as of v1.12.0.
       class PushParser
       end
     end

data/lib/nokogiri/html4/builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Nokogiri
   module HTML4
     ###