nokogiri 1.11.0.rc1 → 1.11.1

data/lib/nokogiri/css/parser.y

@@ -88,7 +88,7 @@ rule
         )
       }
     | LSQUARE NUMBER RSQUARE {
-        # Non standard, but hpricot supports it.
+        # non-standard, from hpricot
         result = Node.new(:PSEUDO_CLASS,
           [Node.new(:FUNCTION, ['nth-child(', val[1]])]
         )
@@ -139,7 +139,7 @@ rule
         when 'n'
           result = Node.new(:NTH, ['1','n','+','0'])
         else
-          # This is not CSS standard.  It allows us to support this:
+          # non-standard to support custom functions:
           # assert_xpath("//a[foo(., @href)]", @parser.parse('a:foo(@href)'))
           # assert_xpath("//a[foo(., @a, b)]", @parser.parse('a:foo(@a, b)'))
           # assert_xpath("//a[foo(., a, 10)]", @parser.parse('a:foo(a, 10)'))

data/lib/nokogiri/css/parser_extras.rb

@@ -1,64 +1,66 @@
 # frozen_string_literal: true
-require 'thread'
+require "thread"
 
 module Nokogiri
   module CSS
     class Parser < Racc::Parser
-      @cache_on = true
-      @cache    = {}
-      @mutex    = Mutex.new
+      CACHE_SWITCH_NAME = :nokogiri_css_parser_cache_is_off
+
+      @cache = {}
+      @mutex = Mutex.new
 
       class << self
-        # Turn on CSS parse caching
-        attr_accessor :cache_on
-        alias :cache_on? :cache_on
-        alias :set_cache :cache_on=
+        # Return a thread-local boolean indicating whether the CSS-to-XPath cache is active. (Default is `true`.)
+        def cache_on?
+          !Thread.current[CACHE_SWITCH_NAME]
+        end
+
+        # Set a thread-local boolean to turn cacheing on and off. Truthy values turn the cache on, falsey values turn the cache off.
+        def set_cache(value)
+          Thread.current[CACHE_SWITCH_NAME] = !value
+        end
 
         # Get the css selector in +string+ from the cache
-        def [] string
-          return unless @cache_on
+        def [](string)
+          return unless cache_on?
           @mutex.synchronize { @cache[string] }
         end
 
         # Set the css selector in +string+ in the cache to +value+
-        def []= string, value
-          return value unless @cache_on
+        def []=(string, value)
+          return value unless cache_on?
           @mutex.synchronize { @cache[string] = value }
         end
 
         # Clear the cache
-        def clear_cache
-          @mutex.synchronize { @cache = {} }
+        def clear_cache(create_new_object = false)
+          @mutex.synchronize do
+            if create_new_object
+              @cache = {}
+            else
+              @cache.clear
+            end
+          end
         end
 
         # Execute +block+ without cache
-        def without_cache &block
-          tmp = @cache_on
-          @cache_on = false
+        def without_cache(&block)
+          original_cache_setting = cache_on?
+          set_cache false
           block.call
-          @cache_on = tmp
-        end
-
-        ###
-        # Parse this CSS selector in +selector+.  Returns an AST.
-        def parse selector
-          @warned ||= false
-          unless @warned
-            $stderr.puts('Nokogiri::CSS::Parser.parse is deprecated, call Nokogiri::CSS.parse(), this will be removed August 1st or version 1.4.0 (whichever is first)')
-            @warned = true
-          end
-          new.parse selector
+        ensure
+          set_cache original_cache_setting
         end
       end
 
       # Create a new CSS parser with respect to +namespaces+
-      def initialize namespaces = {}
-        @tokenizer  = Tokenizer.new
+      def initialize(namespaces = {})
+        @tokenizer = Tokenizer.new
         @namespaces = namespaces
         super()
       end
 
-      def parse string
+      def parse(string)
         @tokenizer.scan_setup string
         do_parse
       end
@@ -68,14 +70,14 @@ module Nokogiri
       end
 
       # Get the xpath for +string+ using +options+
-      def xpath_for string, options={}
+      def xpath_for(string, options = {})
         key = "#{string}#{options[:ns]}#{options[:prefix]}"
         v = self.class[key]
         return v if v
 
         args = [
-          options[:prefix] || '//',
-          options[:visitor] || XPathVisitor.new
+          options[:prefix] || "//",
+          options[:visitor] || XPathVisitor.new,
         ]
         self.class[key] = parse(string).map { |ast|
           ast.to_xpath(*args)
@@ -83,7 +85,7 @@ module Nokogiri
       end
 
       # On CSS parser error, raise an exception
-      def on_error error_token_id, error_value, value_stack
+      def on_error(error_token_id, error_value, value_stack)
         after = value_stack.compact.last
         raise SyntaxError.new("unexpected '#{error_value}' after '#{after}'")
       end

data/lib/nokogiri/css/xpath_visitor.rb

@@ -3,7 +3,6 @@ module Nokogiri
   module CSS
     class XPathVisitor # :nodoc:
       def visit_function node
-
         msg = :"visit_function_#{node.value.first.gsub(/[(]/, '')}"
         return self.send(msg, node) if self.respond_to?(msg)
 
@@ -13,50 +12,51 @@ module Nokogiri
         when /^self\(/
           "self::#{node.value[1]}"
         when /^eq\(/
-          "position() = #{node.value[1]}"
+          "position()=#{node.value[1]}"
         when /^(nth|nth-of-type)\(/
           if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
             nth(node.value[1])
           else
-            "position() = #{node.value[1]}"
+            "position()=#{node.value[1]}"
           end
         when /^nth-child\(/
           if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
             nth(node.value[1], :child => true)
           else
-            "count(preceding-sibling::*) = #{node.value[1].to_i-1}"
+            "count(preceding-sibling::*)=#{node.value[1].to_i-1}"
           end
         when /^nth-last-of-type\(/
           if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
             nth(node.value[1], :last => true)
           else
             index = node.value[1].to_i - 1
-            index == 0 ? "position() = last()" : "position() = last() - #{index}"
+            index == 0 ? "position()=last()" : "position()=last()-#{index}"
           end
         when /^nth-last-child\(/
           if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
             nth(node.value[1], :last => true, :child => true)
           else
-            "count(following-sibling::*) = #{node.value[1].to_i-1}"
+            "count(following-sibling::*)=#{node.value[1].to_i-1}"
           end
         when /^(first|first-of-type)\(/
-          "position() = 1"
+          "position()=1"
         when /^(last|last-of-type)\(/
-          "position() = last()"
+          "position()=last()"
         when /^contains\(/
-          "contains(., #{node.value[1]})"
+          "contains(.,#{node.value[1]})"
         when /^gt\(/
-          "position() > #{node.value[1]}"
+          "position()>#{node.value[1]}"
         when /^only-child\(/
-          "last() = 1"
+          "last()=1"
         when /^comment\(/
           "comment()"
         when /^has\(/
           is_direct = node.value[1].value[0].nil? # e.g. "has(> a)", "has(~ a)", "has(+ a)"
           ".#{"//" if !is_direct}#{node.value[1].accept(self)}"
         else
+          # non-standard. this looks like a function call.
           args = ['.'] + node.value[1..-1]
-          "#{node.value.first}#{args.join(', ')})"
+          "#{node.value.first}#{args.join(',')})"
         end
       end
 
@@ -71,18 +71,18 @@ module Nokogiri
 
       def visit_id node
         node.value.first =~ /^#(.*)$/
-        "@id = '#{$1}'"
+        "@id='#{$1}'"
       end
 
       def visit_attribute_condition node
-         attribute = if (node.value.first.type == :FUNCTION) or (node.value.first.value.first =~ /::/)
-                       ''
-                     else
-                       '@'
-                     end
+        attribute = if (node.value.first.type == :FUNCTION) or (node.value.first.value.first =~ /::/)
+                      ''
+                    else
+                      '@'
+                    end
         attribute += node.value.first.accept(self)
 
-        # Support non-standard css
+        # non-standard. attributes starting with '@'
         attribute.gsub!(/^@@/, '@')
 
         return attribute unless node.value.length == 3
@@ -90,29 +90,30 @@ module Nokogiri
         value = node.value.last
         value = "'#{value}'" if value !~ /^['"]/
 
+        # quoted values - see test_attribute_value_with_quotes in test/css/test_parser.rb
         if (value[0]==value[-1]) && %q{"'}.include?(value[0])
           str_value = value[1..-2]
           if str_value.include?(value[0])
-            value = 'concat("' + str_value.split('"', -1).join(%q{", '"', "}) + '", "")'
+            value = 'concat("' + str_value.split('"', -1).join(%q{",'"',"}) + '","")'
           end
         end
 
         case node.value[1]
         when :equal
-          attribute + " = " + "#{value}"
+          attribute + "=" + "#{value}"
         when :not_equal
-          attribute + " != " + "#{value}"
+          attribute + "!=" + "#{value}"
         when :substring_match
-          "contains(#{attribute}, #{value})"
+          "contains(#{attribute},#{value})"
         when :prefix_match
-          "starts-with(#{attribute}, #{value})"
+          "starts-with(#{attribute},#{value})"
         when :dash_match
-          "#{attribute} = #{value} or starts-with(#{attribute}, concat(#{value}, '-'))"
+          "#{attribute}=#{value} or starts-with(#{attribute},concat(#{value},'-'))"
         when :includes
-          "contains(concat(\" \", #{attribute}, \" \"),concat(\" \", #{value}, \" \"))"
+          value = value[1..-2] # strip quotes
+          css_class(attribute, value)
         when :suffix_match
-          "substring(#{attribute}, string-length(#{attribute}) - " +
-            "string-length(#{value}) + 1, string-length(#{value})) = #{value}"
+          "substring(#{attribute},string-length(#{attribute})-string-length(#{value})+1,string-length(#{value}))=#{value}"
         else
           attribute + " #{node.value[1]} " + "#{value}"
         end
@@ -126,14 +127,14 @@ module Nokogiri
           return self.send(msg, node) if self.respond_to?(msg)
 
           case node.value.first
-          when "first" then "position() = 1"
-          when "first-child" then "count(preceding-sibling::*) = 0"
-          when "last" then "position() = last()"
-          when "last-child" then "count(following-sibling::*) = 0"
-          when "first-of-type" then "position() = 1"
-          when "last-of-type" then "position() = last()"
-          when "only-child" then "count(preceding-sibling::*) = 0 and count(following-sibling::*) = 0"
-          when "only-of-type" then "last() = 1"
+          when "first" then "position()=1"
+          when "first-child" then "count(preceding-sibling::*)=0"
+          when "last" then "position()=last()"
+          when "last-child" then "count(following-sibling::*)=0"
+          when "first-of-type" then "position()=1"
+          when "last-of-type" then "position()=last()"
+          when "only-child" then "count(preceding-sibling::*)=0 and count(following-sibling::*)=0"
+          when "only-of-type" then "last()=1"
           when "empty" then "not(node())"
           when "parent" then "node()"
           when "root" then "not(parent::*)"
@@ -144,7 +145,7 @@ module Nokogiri
       end
 
       def visit_class_condition node
-        "contains(concat(' ', normalize-space(@class), ' '), ' #{node.value.first} ')"
+        css_class("@class", node.value.first)
       end
 
       def visit_combinator node
@@ -181,25 +182,26 @@ module Nokogiri
         node.accept(self)
       end
 
-    private
+      private
+
       def nth node, options={}
         raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
 
         a, b = read_a_and_positive_b node.value
         position = if options[:child]
-          options[:last] ? "(count(following-sibling::*) + 1)" : "(count(preceding-sibling::*) + 1)"
+          options[:last] ? "(count(following-sibling::*)+1)" : "(count(preceding-sibling::*)+1)"
         else
           options[:last] ? "(last()-position()+1)" : "position()"
         end
 
         if b.zero?
-          "(#{position} mod #{a}) = 0"
+          "(#{position} mod #{a})=0"
         else
           compare = a < 0 ? "<=" : ">="
           if a.abs == 1
-            "#{position} #{compare} #{b}"
+            "#{position}#{compare}#{b}"
           else
-            "(#{position} #{compare} #{b}) and (((#{position}-#{b}) mod #{a.abs}) = 0)"
+            "(#{position}#{compare}#{b}) and (((#{position}-#{b}) mod #{a.abs})=0)"
           end
         end
       end
@@ -227,6 +229,32 @@ module Nokogiri
           end =~ /(nth|first|last|only)-of-type(\()?/
         end
       end
+
+      # use only ordinary xpath functions
+      def css_class_standard(hay, needle)
+        "contains(concat(' ',normalize-space(#{hay}),' '),' #{needle} ')"
+      end
+
+      # use the builtin implementation
+      def css_class_builtin(hay, needle)
+        "nokogiri-builtin:css-class(#{hay},'#{needle}')"
+      end
+
+      alias_method :css_class, :css_class_standard
+    end
+
+    class XPathVisitorAlwaysUseBuiltins < XPathVisitor # :nodoc:
+      private
+      alias_method :css_class, :css_class_builtin
+    end
+
+    class XPathVisitorOptimallyUseBuiltins < XPathVisitor # :nodoc:
+      private
+      if Nokogiri.uses_libxml?
+        alias_method :css_class, :css_class_builtin
+      else
+        alias_method :css_class, :css_class_standard
+      end
     end
   end
 end

data/lib/nokogiri/html/document.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
+
+require 'pathname'
+
 module Nokogiri
   module HTML
     class Document < Nokogiri::XML::Document
@@ -161,11 +164,12 @@ module Nokogiri
         # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
         # Nokogiri::XML::ParseOptions.
         def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML
-
           options = Nokogiri::XML::ParseOptions.new(options) if Integer === options
-          # Give the options to the user
+
           yield options if block_given?
 
+          url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+
           if string_or_io.respond_to?(:encoding)
             unless string_or_io.encoding.name == "ASCII-8BIT"
               encoding ||= string_or_io.encoding.name
@@ -173,7 +177,12 @@ module Nokogiri
           end
 
           if string_or_io.respond_to?(:read)
-            url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+            if string_or_io.is_a?(Pathname)
+              # resolve the Pathname to the file and open it as an IO object, see #2110
+              string_or_io = string_or_io.expand_path.open
+              url ||= string_or_io.path
+            end
+
             unless encoding
               # Libxml2's parser has poor support for encoding
               # detection.  First, it does not recognize the HTML5
@@ -252,9 +261,6 @@ module Nokogiri
         end
 
         def self.detect_encoding(chunk)
-          if Nokogiri.jruby? && EncodingReader.is_jruby_without_fix?
-            return EncodingReader.detect_encoding_for_jruby_without_fix(chunk)
-          end
           m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
             return Nokogiri.XML(m[1]).encoding
 
@@ -273,26 +279,6 @@ module Nokogiri
           end
         end
 
-        def self.is_jruby_without_fix?
-          JRUBY_VERSION.split('.').join.to_i < 165
-        end
-
-        def self.detect_encoding_for_jruby_without_fix(chunk)
-          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
-            return Nokogiri.XML(m[1]).encoding
-
-          m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
-            return m[4]
-
-          catch(:encoding_found) {
-            Nokogiri::HTML::SAX::Parser.new(JumpSAXHandler.new(:encoding_found.to_s)).parse(chunk)
-            nil
-          }
-        rescue Nokogiri::SyntaxError, RuntimeError
-          # Ignore parser errors that nokogiri may raise
-          nil
-        end
-
         def initialize(io)
           @io = io
           @firstchunk = nil

data/lib/nokogiri/version.rb

@@ -1,149 +1,3 @@
 # frozen_string_literal: true
-module Nokogiri
-  # The version of Nokogiri you are using
-  VERSION = "1.11.0.rc1"
-
-  class VersionInfo # :nodoc:
-    def jruby?
-      ::JRUBY_VERSION if RUBY_PLATFORM == "java"
-    end
-
-    def engine
-      defined?(RUBY_ENGINE) ? RUBY_ENGINE : "mri"
-    end
-
-    def loaded_libxml_version
-      Gem::Version.new(LIBXML_LOADED_VERSION.
-        scan(/^(\d+)(\d\d)(\d\d)(?!\d)/).first.
-        collect(&:to_i).
-        join("."))
-    end
-
-    def compiled_libxml_version
-      Gem::Version.new LIBXML_COMPILED_VERSION
-    end
-
-    def loaded_libxslt_version
-      Gem::Version.new(LIBXSLT_LOADED_VERSION.
-        scan(/^(\d+)(\d\d)(\d\d)(?!\d)/).first.
-        collect(&:to_i).
-        join("."))
-    end
-
-    def compiled_libxslt_version
-      Gem::Version.new LIBXSLT_COMPILED_VERSION
-    end
-
-    def libxml2?
-      defined?(LIBXML_COMPILED_VERSION)
-    end
-
-    def libxml2_using_system?
-      !libxml2_using_packaged?
-    end
-
-    def libxml2_using_packaged?
-      NOKOGIRI_USE_PACKAGED_LIBRARIES
-    end
-
-    def warnings
-      warnings = []
-
-      if libxml2?
-        if compiled_libxml_version != loaded_libxml_version
-          warnings << "Nokogiri was built against libxml version #{compiled_libxml_version}, but has dynamically loaded #{loaded_libxml_version}"
-        end
-
-        if compiled_libxslt_version != loaded_libxslt_version
-          warnings << "Nokogiri was built against libxslt version #{compiled_libxslt_version}, but has dynamically loaded #{loaded_libxslt_version}"
-        end
-      end
-
-      warnings
-    end
-
-    def to_hash
-      {}.tap do |vi|
-        vi["warnings"] = []
-        vi["nokogiri"] = Nokogiri::VERSION
-        vi["ruby"] = {}.tap do |ruby|
-          ruby["version"] = ::RUBY_VERSION
-          ruby["platform"] = ::RUBY_PLATFORM
-          ruby["description"] = ::RUBY_DESCRIPTION
-          ruby["engine"] = engine
-          ruby["jruby"] = jruby? if jruby?
-        end
-
-        if libxml2?
-          vi["libxml"] = {}.tap do |libxml|
-            if libxml2_using_packaged?
-              libxml["source"] = "packaged"
-              libxml["patches"] = NOKOGIRI_LIBXML2_PATCHES
-            else
-              libxml["source"] = "system"
-            end
-            libxml["compiled"] = compiled_libxml_version.to_s
-            libxml["loaded"] = loaded_libxml_version.to_s
-          end
-
-          vi["libxslt"] = {}.tap do |libxslt|
-            if libxml2_using_packaged?
-              libxslt["source"] = "packaged"
-              libxslt["patches"] = NOKOGIRI_LIBXSLT_PATCHES
-            else
-              libxslt["source"] = "system"
-            end
-            libxslt["compiled"] = compiled_libxslt_version.to_s
-            libxslt["loaded"] = loaded_libxslt_version.to_s
-          end
-
-          vi["warnings"] = warnings
-        elsif jruby?
-          vi["xerces"] = Nokogiri::XERCES_VERSION
-          vi["nekohtml"] = Nokogiri::NEKO_VERSION
-        end
-      end
-    end
-
-    def to_markdown
-      begin
-        require "psych"
-      rescue LoadError
-      end
-      require "yaml"
-      "# Nokogiri (#{Nokogiri::VERSION})\n" +
-      YAML.dump(to_hash).each_line.map { |line| "    #{line}" }.join
-    end
-
-    # FIXME: maybe switch to singleton?
-    @@instance = new
-    @@instance.warnings.each do |warning|
-      warn "WARNING: #{warning}"
-    end
-    def self.instance; @@instance; end
-  end
-
-  def self.uses_libxml?(requirement = nil) # :nodoc:
-    return false unless VersionInfo.instance.libxml2?
-    return true unless requirement
-    return Gem::Requirement.new(requirement).satisfied_by?(VersionInfo.instance.loaded_libxml_version)
-  end
-
-  def self.jruby? # :nodoc:
-    VersionInfo.instance.jruby?
-  end
-
-  # Ensure constants used in this file are loaded
-  if Nokogiri.jruby?
-    require "nokogiri/jruby/dependencies"
-  end
-  begin
-    RUBY_VERSION =~ /(\d+\.\d+)/
-    require "nokogiri/#{$1}/nokogiri"
-  rescue LoadError
-    require "nokogiri/nokogiri"
-  end
-
-  # More complete version information about libxml
-  VERSION_INFO = VersionInfo.instance.to_hash
-end
+require_relative "version/constant"
+require_relative "version/info"