nokogiri 1.6.0 → 1.13.2

data/lib/nokogiri/css/xpath_visitor.rb

@@ -1,52 +1,143 @@
+# coding: utf-8
+# frozen_string_literal: true
+
 module Nokogiri
   module CSS
-    class XPathVisitor # :nodoc:
-      def visit_function node
-        #  note that nth-child and nth-last-child are preprocessed in css/node.rb.
-        msg = :"visit_function_#{node.value.first.gsub(/[(]/, '')}"
-        return self.send(msg, node) if self.respond_to?(msg)
+    # When translating CSS selectors to XPath queries with Nokogiri::CSS.xpath_for, the XPathVisitor
+    # class allows for changing some of the behaviors related to builtin xpath functions and quirks
+    # of HTML5.
+    class XPathVisitor
+      WILDCARD_NAMESPACES = Nokogiri.libxml2_patches.include?("0009-allow-wildcard-namespaces.patch") # :nodoc:
+
+      # Enum to direct XPathVisitor when to use Nokogiri builtin XPath functions.
+      module BuiltinsConfig
+        # Never use Nokogiri builtin functions, always generate vanilla XPath 1.0 queries. This is
+        # the default when calling Nokogiri::CSS.xpath_for directly.
+        NEVER = :never
+
+        # Always use Nokogiri builtin functions whenever possible. This is probably only useful for testing.
+        ALWAYS = :always
+
+        # Only use Nokogiri builtin functions when they will be faster than vanilla XPath. This is
+        # the behavior chosen when searching for CSS selectors on a Nokogiri document, fragment, or
+        # node.
+        OPTIMAL = :optimal
+
+        # :nodoc: array of values for validation
+        VALUES = [NEVER, ALWAYS, OPTIMAL]
+      end
+
+      # Enum to direct XPathVisitor when to tweak the XPath query to suit the nature of the document
+      # being searched. Note that searches for CSS selectors from a Nokogiri document, fragment, or
+      # node will choose the correct option automatically.
+      module DoctypeConfig
+        # The document being searched is an XML document. This is the default.
+        XML = :xml
+
+        # The document being searched is an HTML4 document.
+        HTML4 = :html4
+
+        # The document being searched is an HTML5 document.
+        HTML5 = :html5
+
+        # :nodoc: array of values for validation
+        VALUES = [XML, HTML4, HTML5]
+      end
+
+      # :call-seq:
+      #   new() → XPathVisitor
+      #   new(builtins:, doctype:) → XPathVisitor
+      #
+      # [Parameters]
+      # - +builtins:+ (BuiltinsConfig) Determine when to use Nokogiri's built-in xpath functions for performance improvements.
+      # - +doctype:+ (DoctypeConfig) Make document-type-specific accommodations for CSS queries.
+      #
+      # [Returns] XPathVisitor
+      #
+      def initialize(builtins: BuiltinsConfig::NEVER, doctype: DoctypeConfig::XML)
+        unless BuiltinsConfig::VALUES.include?(builtins)
+          raise(ArgumentError, "Invalid values #{builtins.inspect} for builtins: keyword parameter")
+        end
+        unless DoctypeConfig::VALUES.include?(doctype)
+          raise(ArgumentError, "Invalid values #{doctype.inspect} for doctype: keyword parameter")
+        end
+
+        @builtins = builtins
+        @doctype = doctype
+      end
+
+      # :call-seq: config() → Hash
+      #
+      # [Returns]
+      #   a Hash representing the configuration of the XPathVisitor, suitable for use as
+      #   part of the CSS cache key.
+      def config
+        { builtins: @builtins, doctype: @doctype }
+      end
+
+      # :stopdoc:
+      def visit_function(node)
+        msg = :"visit_function_#{node.value.first.gsub(/[(]/, "")}"
+        return send(msg, node) if respond_to?(msg)
 
         case node.value.first
         when /^text\(/
-          'child::text()'
+          "child::text()"
         when /^self\(/
           "self::#{node.value[1]}"
         when /^eq\(/
-          "position() = #{node.value[1]}"
-        when /^(nth|nth-of-type|nth-child)\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
-            an_plus_b(node.value[1])
+          "position()=#{node.value[1]}"
+        when /^(nth|nth-of-type)\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1])
+          else
+            "position()=#{node.value[1]}"
+          end
+        when /^nth-child\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1], child: true)
           else
-            "position() = #{node.value[1]}"
+            "count(preceding-sibling::*)=#{node.value[1].to_i - 1}"
           end
-        when /^(nth-last-child|nth-last-of-type)\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
-            an_plus_b(node.value[1], :last => true)
+        when /^nth-last-of-type\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1], last: true)
           else
             index = node.value[1].to_i - 1
-            index == 0 ? "position() = last()" : "position() = last() - #{index}"
+            index == 0 ? "position()=last()" : "position()=last()-#{index}"
+          end
+        when /^nth-last-child\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) && (node.value[1].type == :NTH)
+            nth(node.value[1], last: true, child: true)
+          else
+            "count(following-sibling::*)=#{node.value[1].to_i - 1}"
           end
         when /^(first|first-of-type)\(/
-          "position() = 1"
+          "position()=1"
         when /^(last|last-of-type)\(/
-          "position() = last()"
+          "position()=last()"
         when /^contains\(/
-          "contains(., #{node.value[1]})"
+          "contains(.,#{node.value[1]})"
         when /^gt\(/
-          "position() > #{node.value[1]}"
+          "position()>#{node.value[1]}"
         when /^only-child\(/
-          "last() = 1"
+          "last()=1"
         when /^comment\(/
           "comment()"
         when /^has\(/
-          node.value[1].accept(self)
+          is_direct = node.value[1].value[0].nil? # e.g. "has(> a)", "has(~ a)", "has(+ a)"
+          ".#{"//" unless is_direct}#{node.value[1].accept(self)}"
         else
-          args = ['.'] + node.value[1..-1]
-          "#{node.value.first}#{args.join(', ')})"
+          # xpath function call, let's marshal those arguments
+          args = ["."]
+          args += node.value[1..-1].map do |n|
+            n.is_a?(Nokogiri::CSS::Node) ? n.accept(self) : n
+          end
+          "#{node.value.first}#{args.join(",")})"
         end
       end
 
-      def visit_not node
+      def visit_not(node)
         child = node.value.first
         if :ELEMENT_NAME == child.type
           "not(self::#{child.accept(self)})"
@@ -55,61 +146,63 @@ module Nokogiri
         end
       end
 
-      def visit_id node
+      def visit_id(node)
         node.value.first =~ /^#(.*)$/
-        "@id = '#{$1}'"
+        "@id='#{Regexp.last_match(1)}'"
       end
 
-      def visit_attribute_condition node
-         attribute = if (node.value.first.type == :FUNCTION) or (node.value.first.value.first =~ /::/)
-                       ''
-                     else
-                       '@'
-                     end
-        attribute += node.value.first.accept(self)
-
-        # Support non-standard css
-        attribute.gsub!(/^@@/, '@')
-
-        return attribute unless node.value.length == 3
+      def visit_attribute_condition(node)
+        attribute = node.value.first.accept(self)
+        return attribute if node.value.length == 1
 
         value = node.value.last
-        value = "'#{value}'" if value !~ /^['"]/
+        value = "'#{value}'" unless /^['"]/.match?(value)
+
+        # quoted values - see test_attribute_value_with_quotes in test/css/test_parser.rb
+        if (value[0] == value[-1]) && %q{"'}.include?(value[0])
+          str_value = value[1..-2]
+          if str_value.include?(value[0])
+            value = 'concat("' + str_value.split('"', -1).join(%q{",'"',"}) + '","")'
+          end
+        end
 
         case node.value[1]
         when :equal
-          attribute + " = " + "#{value}"
+          attribute + "=" + value.to_s
         when :not_equal
-          attribute + " != " + "#{value}"
+          attribute + "!=" + value.to_s
         when :substring_match
-          "contains(#{attribute}, #{value})"
+          "contains(#{attribute},#{value})"
         when :prefix_match
-          "starts-with(#{attribute}, #{value})"
+          "starts-with(#{attribute},#{value})"
         when :dash_match
-          "#{attribute} = #{value} or starts-with(#{attribute}, concat(#{value}, '-'))"
+          "#{attribute}=#{value} or starts-with(#{attribute},concat(#{value},'-'))"
         when :includes
-          "contains(concat(\" \", #{attribute}, \" \"),concat(\" \", #{value}, \" \"))"
+          value = value[1..-2] # strip quotes
+          css_class(attribute, value)
         when :suffix_match
-          "substring(#{attribute}, string-length(#{attribute}) - " +
-            "string-length(#{value}) + 1, string-length(#{value})) = #{value}"
+          "substring(#{attribute},string-length(#{attribute})-string-length(#{value})+1,string-length(#{value}))=#{value}"
         else
-          attribute + " #{node.value[1]} " + "#{value}"
+          attribute + " #{node.value[1]} " + value.to_s
         end
       end
 
-      def visit_pseudo_class node
-        if node.value.first.is_a?(Nokogiri::CSS::Node) and node.value.first.type == :FUNCTION
+      def visit_pseudo_class(node)
+        if node.value.first.is_a?(Nokogiri::CSS::Node) && (node.value.first.type == :FUNCTION)
           node.value.first.accept(self)
         else
-          msg = :"visit_pseudo_class_#{node.value.first.gsub(/[(]/, '')}"
-          return self.send(msg, node) if self.respond_to?(msg)
+          msg = :"visit_pseudo_class_#{node.value.first.gsub(/[(]/, "")}"
+          return send(msg, node) if respond_to?(msg)
 
           case node.value.first
-          when "first", "first-child" then "position() = 1"
-          when "last", "last-child" then "position() = last()"
-          when "first-of-type" then "position() = 1"
-          when "last-of-type" then "position() = last()"
-          when "only-of-type" then "last() = 1"
+          when "first" then "position()=1"
+          when "first-child" then "count(preceding-sibling::*)=0"
+          when "last" then "position()=last()"
+          when "last-child" then "count(following-sibling::*)=0"
+          when "first-of-type" then "position()=1"
+          when "last-of-type" then "position()=last()"
+          when "only-child" then "count(preceding-sibling::*)=0 and count(following-sibling::*)=0"
+          when "only-of-type" then "last()=1"
           when "empty" then "not(node())"
           when "parent" then "node()"
           when "root" then "not(parent::*)"
@@ -119,17 +212,24 @@ module Nokogiri
         end
       end
 
-      def visit_class_condition node
-        "contains(concat(' ', normalize-space(@class), ' '), ' #{node.value.first} ')"
+      def visit_class_condition(node)
+        css_class("@class", node.value.first)
+      end
+
+      def visit_combinator(node)
+        if is_of_type_pseudo_class?(node.value.last)
+          "#{node.value.first&.accept(self)}][#{node.value.last.accept(self)}"
+        else
+          "#{node.value.first&.accept(self)} and #{node.value.last.accept(self)}"
+        end
       end
 
       {
-        'combinator'                => ' and ',
-        'direct_adjacent_selector'  => "/following-sibling::*[1]/self::",
-        'following_selector'        => "/following-sibling::",
-        'descendant_selector'       => '//',
-        'child_selector'            => '/',
-      }.each do |k,v|
+        "direct_adjacent_selector" => "/following-sibling::*[1]/self::",
+        "following_selector" => "/following-sibling::",
+        "descendant_selector" => "//",
+        "child_selector" => "/",
+      }.each do |k, v|
         class_eval %{
           def visit_#{k} node
             "\#{node.value.first.accept(self) if node.value.first}#{v}\#{node.value.last.accept(self)}"
@@ -137,35 +237,121 @@ module Nokogiri
         }
       end
 
-      def visit_conditional_selector node
-        node.value.first.accept(self) + '[' +
-        node.value.last.accept(self) + ']'
+      def visit_conditional_selector(node)
+        node.value.first.accept(self) + "[" +
+          node.value.last.accept(self) + "]"
+      end
+
+      def visit_element_name(node)
+        if @doctype == DoctypeConfig::HTML5 && html5_element_name_needs_namespace_handling(node)
+          # HTML5 has namespaces that should be ignored in CSS queries
+          # https://github.com/sparklemotion/nokogiri/issues/2376
+          if @builtins == BuiltinsConfig::ALWAYS || (@builtins == BuiltinsConfig::OPTIMAL && Nokogiri.uses_libxml?)
+            if WILDCARD_NAMESPACES
+              "*:#{node.value.first}"
+            else
+              "*[nokogiri-builtin:local-name-is('#{node.value.first}')]"
+            end
+          else
+            "*[local-name()='#{node.value.first}']"
+          end
+        else
+          node.value.first
+        end
       end
 
-      def visit_element_name node
-        node.value.first
+      def visit_attrib_name(node)
+        "@#{node.value.first}"
       end
 
-      def accept node
+      def accept(node)
         node.accept(self)
       end
 
-    private
-      def an_plus_b node, options={}
+      private
+
+      def html5_element_name_needs_namespace_handling(node)
+        # if this is the wildcard selector "*", use it as normal
+        node.value.first != "*" &&
+          # if there is already a namespace (i.e., it is a prefixed QName), use it as normal
+          !node.value.first.include?(":")
+      end
+
+      def nth(node, options = {})
         raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
 
-        a = node.value[0].to_i
-        b = node.value[3].to_i
-        position = options[:last] ? "(last()-position()+1)" : "position()"
+        a, b = read_a_and_positive_b(node.value)
+        position = if options[:child]
+          options[:last] ? "(count(following-sibling::*)+1)" : "(count(preceding-sibling::*)+1)"
+        else
+          options[:last] ? "(last()-position()+1)" : "position()"
+        end
 
-        if (b == 0)
-          return "(#{position} mod #{a}) = 0"
+        if b.zero?
+          "(#{position} mod #{a})=0"
         else
-          compare = (a < 0) ? "<=" : ">="
-          return "(#{position} #{compare} #{b}) and (((#{position}-#{b}) mod #{a.abs}) = 0)"
+          compare = a < 0 ? "<=" : ">="
+          if a.abs == 1
+            "#{position}#{compare}#{b}"
+          else
+            "(#{position}#{compare}#{b}) and (((#{position}-#{b}) mod #{a.abs})=0)"
+          end
         end
       end
 
+      def read_a_and_positive_b(values)
+        op = values[2]
+        if op == "+"
+          a = values[0].to_i
+          b = values[3].to_i
+        elsif op == "-"
+          a = values[0].to_i
+          b = a - (values[3].to_i % a)
+        else
+          raise ArgumentError, "expected an+b node to have either + or - as the operator, but is #{op.inspect}"
+        end
+        [a, b]
+      end
+
+      def is_of_type_pseudo_class?(node) # rubocop:disable Naming/PredicateName
+        if node.type == :PSEUDO_CLASS
+          if node.value[0].is_a?(Nokogiri::CSS::Node) && (node.value[0].type == :FUNCTION)
+            node.value[0].value[0]
+          else
+            node.value[0]
+          end =~ /(nth|first|last|only)-of-type(\()?/
+        end
+      end
+
+      def css_class(hay, needle)
+        if @builtins == BuiltinsConfig::ALWAYS || (@builtins == BuiltinsConfig::OPTIMAL && Nokogiri.uses_libxml?)
+          # use the builtin implementation
+          "nokogiri-builtin:css-class(#{hay},'#{needle}')"
+        else
+          # use only ordinary xpath functions
+          "contains(concat(' ',normalize-space(#{hay}),' '),' #{needle} ')"
+        end
+      end
+    end
+
+    module XPathVisitorAlwaysUseBuiltins # :nodoc:
+      def self.new
+        warn(
+          "Nokogiri::CSS::XPathVisitorAlwaysUseBuiltins is deprecated and will be removed in a future version of Nokogiri",
+          { uplevel: 1 },
+        )
+        XPathVisitor.new(builtins: :always)
+      end
+    end
+
+    module XPathVisitorOptimallyUseBuiltins # :nodoc:
+      def self.new
+        warn(
+          "Nokogiri::CSS::XPathVisitorOptimallyUseBuiltins is deprecated and will be removed in a future version of Nokogiri",
+          { uplevel: 1 },
+        )
+        XPathVisitor.new(builtins: :optimal)
+      end
     end
   end
 end

data/lib/nokogiri/css.rb

@@ -1,27 +1,60 @@
-require 'nokogiri/css/node'
-require 'nokogiri/css/xpath_visitor'
-x = $-w
-$-w = false
-require 'nokogiri/css/parser'
-$-w = x
-
-require 'nokogiri/css/tokenizer'
-require 'nokogiri/css/syntax_error'
+# coding: utf-8
+# frozen_string_literal: true
 
 module Nokogiri
+  # Translate a CSS selector into an XPath 1.0 query
   module CSS
     class << self
-      ###
-      # Parse this CSS selector in +selector+.  Returns an AST.
-      def parse selector
-        Parser.new.parse selector
+      # TODO: Deprecate this method ahead of 2.0 and delete it in 2.0.
+      # It is not used by Nokogiri and shouldn't be part of the public API.
+      def parse(selector) # :nodoc:
+        Parser.new.parse(selector)
       end
 
-      ###
-      # Get the XPath for +selector+.
-      def xpath_for selector, options={}
-        Parser.new(options[:ns] || {}).xpath_for selector, options
+      # :call-seq:
+      #   xpath_for(selector) → String
+      #   xpath_for(selector [, prefix:] [, visitor:] [, ns:]) → String
+      #
+      # Translate a CSS selector to the equivalent XPath query.
+      #
+      # [Parameters]
+      # - +selector+ (String) The CSS selector to be translated into XPath
+      #
+      # - +prefix:+ (String)
+      #
+      #   The XPath prefix for the query, see Nokogiri::XML::XPath for some options. Default is
+      #   +XML::XPath::GLOBAL_SEARCH_PREFIX+.
+      #
+      # - +visitor:+ (Nokogiri::CSS::XPathVisitor)
+      #
+      #   The visitor class to use to transform the AST into XPath. Default is
+      #   +Nokogiri::CSS::XPathVisitor.new+.
+      #
+      # - +ns:+ (Hash<String ⇒ String>)
+      #
+      #   The namespaces that are referenced in the query, if any. This is a hash where the keys are
+      #   the namespace prefix and the values are the namespace URIs. Default is an empty Hash.
+      #
+      # [Returns] (String) The equivalent XPath query for +selector+
+      #
+      # 💡 Note that translated queries are cached for performance concerns.
+      #
+      def xpath_for(selector, options = {})
+        prefix = options.fetch(:prefix, Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX)
+        visitor = options.fetch(:visitor) { Nokogiri::CSS::XPathVisitor.new }
+        ns = options.fetch(:ns, {})
+        Parser.new(ns).xpath_for(selector, prefix, visitor)
       end
     end
   end
 end
+
+require_relative "css/node"
+require_relative "css/xpath_visitor"
+x = $-w
+$-w = false
+require_relative "css/parser"
+$-w = x
+
+require_relative "css/tokenizer"
+require_relative "css/syntax_error"

data/lib/nokogiri/decorators/slop.rb

@@ -1,28 +1,31 @@
+# frozen_string_literal: true
+
 module Nokogiri
   module Decorators
     ###
     # The Slop decorator implements method missing such that a methods may be
     # used instead of XPath or CSS.  See Nokogiri.Slop
     module Slop
+      # The default XPath search context for Slop
+      XPATH_PREFIX = "./"
+
       ###
       # look for node with +name+.  See Nokogiri.Slop
-      def method_missing name, *args, &block
-        prefix = implied_xpath_context
-
+      def method_missing(name, *args, &block)
         if args.empty?
-          list = xpath("#{prefix}#{name.to_s.sub(/^_/, '')}")
-        elsif args.first.is_a? Hash
+          list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, "")}")
+        elsif args.first.is_a?(Hash)
           hash = args.first
           if hash[:css]
             list = css("#{name}#{hash[:css]}")
           elsif hash[:xpath]
-            conds = Array(hash[:xpath]).join(' and ')
-            list = xpath("#{prefix}#{name}[#{conds}]")
+            conds = Array(hash[:xpath]).join(" and ")
+            list = xpath("#{XPATH_PREFIX}#{name}[#{conds}]")
           end
         else
           CSS::Parser.without_cache do
             list = xpath(
-              *CSS.xpath_for("#{name}#{args.first}", :prefix => prefix)
+              *CSS.xpath_for("#{name}#{args.first}", prefix: XPATH_PREFIX)
             )
           end
         end
@@ -30,6 +33,12 @@ module Nokogiri
         super if list.empty?
         list.length == 1 ? list.first : list
       end
+
+      def respond_to_missing?(name, include_private = false)
+        list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, "")}")
+
+        !list.empty?
+      end
     end
   end
 end

data/lib/nokogiri/extension.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# load the C or Java extension
+begin
+  # native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version>
+  ::RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "#{Regexp.last_match(1)}/nokogiri"
+rescue LoadError => e
+  if /GLIBC/.match?(e.message)
+    warn(<<~EOM)
+
+      ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system with glibc < 2.17:
+
+        #{e.message}
+
+        If that's the case, then please install Nokogiri via the `ruby` platform gem:
+            gem install nokogiri --platform=ruby
+        or:
+            bundle config set force_ruby_platform true
+
+        Please visit https://nokogiri.org/tutorials/installing_nokogiri.html for more help.
+
+    EOM
+    raise e
+  end
+
+  # use "require" instead of "require_relative" because non-native gems will place C extension files
+  # in Gem::BasicSpecification#extension_dir after compilation (during normal installation), which
+  # is in $LOAD_PATH but not necessarily relative to this file (see #2300)
+  require "nokogiri/nokogiri"
+end

data/lib/nokogiri/gumbo.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module Gumbo
+    # The default maximum number of attributes per element.
+    DEFAULT_MAX_ATTRIBUTES = 400
+
+    # The default maximum number of errors for parsing a document or a fragment.
+    DEFAULT_MAX_ERRORS = 0
+
+    # The default maximum depth of the DOM tree produced by parsing a document
+    # or fragment.
+    DEFAULT_MAX_TREE_DEPTH = 400
+  end
+end