nokogiri 1.0.0 → 1.6.8.1

data/lib/nokogiri/css/xpath_visitor.rb

@@ -1,8 +1,8 @@
 module Nokogiri
   module CSS
-    class XPathVisitor
+    class XPathVisitor # :nodoc:
       def visit_function node
-        #  note that nth-child and nth-last-child are preprocessed in css/node.rb.
+
         msg = :"visit_function_#{node.value.first.gsub(/[(]/, '')}"
         return self.send(msg, node) if self.respond_to?(msg)
 
@@ -11,44 +11,60 @@ module Nokogiri
           'child::text()'
         when /^self\(/
           "self::#{node.value[1]}"
-        when /^(eq|nth|nth-of-type|nth-child)\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
-            an_plus_b(node.value[1])
+        when /^eq\(/
+          "position() = #{node.value[1]}"
+        when /^(nth|nth-of-type)\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
+            nth(node.value[1])
+          else
+            "position() = #{node.value[1]}"
+          end
+        when /^nth-child\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
+            nth(node.value[1], :child => true)
+          else
+            "count(preceding-sibling::*) = #{node.value[1].to_i-1}"
+          end
+        when /^nth-last-of-type\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
+            nth(node.value[1], :last => true)
+          else
+            index = node.value[1].to_i - 1
+            index == 0 ? "position() = last()" : "position() = last() - #{index}"
+          end
+        when /^nth-last-child\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :NTH
+            nth(node.value[1], :last => true, :child => true)
           else
-            "position() = " + node.value[1]
+            "count(following-sibling::*) = #{node.value[1].to_i-1}"
           end
         when /^(first|first-of-type)\(/
           "position() = 1"
         when /^(last|last-of-type)\(/
           "position() = last()"
-        when /^(nth-last-child|nth-last-of-type)\(/
-          "position() = last() - #{node.value[1]}"
         when /^contains\(/
           "contains(., #{node.value[1]})"
         when /^gt\(/
           "position() > #{node.value[1]}"
         when /^only-child\(/
           "last() = 1"
+        when /^comment\(/
+          "comment()"
+        when /^has\(/
+          node.value[1].accept(self)
         else
-          node.value.first + ')'
+          args = ['.'] + node.value[1..-1]
+          "#{node.value.first}#{args.join(', ')})"
         end
       end
 
       def visit_not node
-        'not(' + node.value.first.accept(self) + ')'
-      end
-
-      def visit_preceding_selector node
-        node.value.last.accept(self) +
-          '[preceding-sibling::' +
-          node.value.first.accept(self) +
-          ']'
-      end
-
-      def visit_direct_adjacent_selector node
-        node.value.first.accept(self) +
-          "/following-sibling::*[1]/self::" +
-           node.value.last.accept(self)
+        child = node.value.first
+        if :ELEMENT_NAME == child.type
+          "not(self::#{child.accept(self)})"
+        else
+          "not(#{child.accept(self)})"
+        end
       end
 
       def visit_id node
@@ -73,15 +89,19 @@ module Nokogiri
         value = "'#{value}'" if value !~ /^['"]/
 
         case node.value[1]
-        when '*='
+        when :equal
+          attribute + " = " + "#{value}"
+        when :not_equal
+          attribute + " != " + "#{value}"
+        when :substring_match
           "contains(#{attribute}, #{value})"
-        when '^='
+        when :prefix_match
           "starts-with(#{attribute}, #{value})"
-        when '|='
+        when :dash_match
           "#{attribute} = #{value} or starts-with(#{attribute}, concat(#{value}, '-'))"
-        when '~='
+        when :includes
           "contains(concat(\" \", #{attribute}, \" \"),concat(\" \", #{value}, \" \"))"
-        when '$='
+        when :suffix_match
           "substring(#{attribute}, string-length(#{attribute}) - " +
             "string-length(#{value}) + 1, string-length(#{value})) = #{value}"
         else
@@ -98,26 +118,45 @@ module Nokogiri
 
           case node.value.first
           when "first" then "position() = 1"
+          when "first-child" then "count(preceding-sibling::*) = 0"
           when "last" then "position() = last()"
+          when "last-child" then "count(following-sibling::*) = 0"
           when "first-of-type" then "position() = 1"
           when "last-of-type" then "position() = last()"
+          when "only-child" then "count(preceding-sibling::*) = 0 and count(following-sibling::*) = 0"
           when "only-of-type" then "last() = 1"
           when "empty" then "not(node())"
           when "parent" then "node()"
           when "root" then "not(parent::*)"
           else
-            '1 = 1'
+            node.value.first + "(.)"
           end
         end
       end
 
       def visit_class_condition node
-        "contains(concat(' ', @class, ' '), ' #{node.value.first} ')"
+        "contains(concat(' ', normalize-space(@class), ' '), ' #{node.value.first} ')"
       end
 
       def visit_combinator node
-        node.value.first.accept(self) + ' and ' +
-        node.value.last.accept(self)
+        if is_of_type_pseudo_class?(node.value.last)
+          "#{node.value.first.accept(self) if node.value.first}][#{node.value.last.accept(self)}"
+        else
+          "#{node.value.first.accept(self) if node.value.first} and #{node.value.last.accept(self)}"
+        end
+      end
+      
+      {
+        'direct_adjacent_selector'  => "/following-sibling::*[1]/self::",
+        'following_selector'        => "/following-sibling::",
+        'descendant_selector'       => '//',
+        'child_selector'            => '/',
+      }.each do |k,v|
+        class_eval %{
+          def visit_#{k} node
+            "\#{node.value.first.accept(self) if node.value.first}#{v}\#{node.value.last.accept(self)}"
+          end
+        }
       end
 
       def visit_conditional_selector node
@@ -125,18 +164,6 @@ module Nokogiri
         node.value.last.accept(self) + ']'
       end
 
-      def visit_descendant_selector node
-        node.value.first.accept(self) +
-        '//' +
-        node.value.last.accept(self)
-      end
-
-      def visit_child_selector node
-        node.value.first.accept(self) +
-        '/' +
-        node.value.last.accept(self)
-      end
-
       def visit_element_name node
         node.value.first
       end
@@ -146,20 +173,51 @@ module Nokogiri
       end
 
     private
-      def an_plus_b node
+      def nth node, options={}
         raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
 
-        a = node.value[0].to_i
-        b = node.value[3].to_i
+        a, b = read_a_and_positive_b node.value
+        position = if options[:child]
+          options[:last] ? "(count(following-sibling::*) + 1)" : "(count(preceding-sibling::*) + 1)"
+        else
+          options[:last] ? "(last()-position()+1)" : "position()"
+        end
 
-        if (b == 0)
-          return "(position() mod #{a}) = 0"
+        if b.zero?
+          "(#{position} mod #{a}) = 0"
         else
-          compare = (a < 0) ? "<=" : ">="
-          return "(position() #{compare} #{b}) and (((position()-#{b}) mod #{a.abs}) = 0)"
+          compare = a < 0 ? "<=" : ">="
+          if a.abs == 1
+            "#{position} #{compare} #{b}"
+          else
+            "(#{position} #{compare} #{b}) and (((#{position}-#{b}) mod #{a.abs}) = 0)"
+          end
         end
       end
 
+      def read_a_and_positive_b values
+        op = values[2]
+        if op == "+"
+          a = values[0].to_i
+          b = values[3].to_i
+        elsif op == "-"
+          a = values[0].to_i
+          b = a - (values[3].to_i % a)
+        else
+          raise ArgumentError, "expected an+b node to have either + or - as the operator, but is #{op.inspect}"
+        end
+        [a, b]
+      end
+      
+      def is_of_type_pseudo_class? node
+        if node.type==:PSEUDO_CLASS
+          if node.value[0].is_a?(Nokogiri::CSS::Node) and node.value[0].type == :FUNCTION
+            node.value[0].value[0]
+          else
+            node.value[0]
+          end =~ /(nth|first|last|only)-of-type(\()?/
+        end   
+      end
     end
   end
 end

data/lib/nokogiri/css.rb

@@ -1,6 +1,27 @@
 require 'nokogiri/css/node'
 require 'nokogiri/css/xpath_visitor'
-require 'nokogiri/css/generated_tokenizer'
-require 'nokogiri/css/generated_parser'
-require 'nokogiri/css/tokenizer'
+x = $-w
+$-w = false
 require 'nokogiri/css/parser'
+$-w = x
+
+require 'nokogiri/css/tokenizer'
+require 'nokogiri/css/syntax_error'
+
+module Nokogiri
+  module CSS
+    class << self
+      ###
+      # Parse this CSS selector in +selector+.  Returns an AST.
+      def parse selector
+        Parser.new.parse selector
+      end
+
+      ###
+      # Get the XPath for +selector+.
+      def xpath_for selector, options={}
+        Parser.new(options[:ns] || {}).xpath_for selector, options
+      end
+    end
+  end
+end

data/lib/nokogiri/decorators/slop.rb

@@ -0,0 +1,42 @@
+module Nokogiri
+  module Decorators
+    ###
+    # The Slop decorator implements method missing such that a methods may be
+    # used instead of XPath or CSS.  See Nokogiri.Slop
+    module Slop
+      # The default XPath search context for Slop
+      XPATH_PREFIX = "./"
+
+      ###
+      # look for node with +name+.  See Nokogiri.Slop
+      def method_missing name, *args, &block
+        if args.empty?
+          list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, '')}")
+        elsif args.first.is_a? Hash
+          hash = args.first
+          if hash[:css]
+            list = css("#{name}#{hash[:css]}")
+          elsif hash[:xpath]
+            conds = Array(hash[:xpath]).join(' and ')
+            list = xpath("#{XPATH_PREFIX}#{name}[#{conds}]")
+          end
+        else
+          CSS::Parser.without_cache do
+            list = xpath(
+              *CSS.xpath_for("#{name}#{args.first}", :prefix => XPATH_PREFIX)
+            )
+          end
+        end
+
+        super if list.empty?
+        list.length == 1 ? list.first : list
+      end
+
+      def respond_to_missing? name, include_private = false
+        list = xpath("#{XPATH_PREFIX}#{name.to_s.sub(/^_/, '')}")
+
+        !list.empty?
+      end
+    end
+  end
+end

data/lib/nokogiri/html/builder.rb

@@ -1,6 +1,32 @@
 module Nokogiri
   module HTML
-    class Builder < XML::Builder
+    ###
+    # Nokogiri HTML builder is used for building HTML documents.  It is very
+    # similar to the Nokogiri::XML::Builder.  In fact, you should go read the
+    # documentation for Nokogiri::XML::Builder before reading this
+    # documentation.
+    #
+    # == Synopsis:
+    #
+    # Create an HTML document with a body that has an onload attribute, and a
+    # span tag with a class of "bold" that has content of "Hello world".
+    #
+    #   builder = Nokogiri::HTML::Builder.new do |doc|
+    #     doc.html {
+    #       doc.body(:onload => 'some_func();') {
+    #         doc.span.bold {
+    #           doc.text "Hello world"
+    #         }
+    #       }
+    #     }
+    #   end
+    #   puts builder.to_html
+    #
+    # The HTML builder inherits from the XML builder, so make sure to read the
+    # Nokogiri::XML::Builder documentation.
+    class Builder < Nokogiri::XML::Builder
+      ###
+      # Convert the builder to HTML
       def to_html
         @doc.to_html
       end

data/lib/nokogiri/html/document.rb

@@ -1,8 +1,334 @@
 module Nokogiri
   module HTML
-    class Document < XML::Document
-      def to_html
-        serialize
+    class Document < Nokogiri::XML::Document
+      ###
+      # Get the meta tag encoding for this document.  If there is no meta tag,
+      # then nil is returned.
+      def meta_encoding
+        case
+        when meta = at('//meta[@charset]')
+          meta[:charset]
+        when meta = meta_content_type
+          meta['content'][/charset\s*=\s*([\w-]+)/i, 1]
+        end
+      end
+
+      ###
+      # Set the meta tag encoding for this document.
+      #
+      # If an meta encoding tag is already present, its content is
+      # replaced with the given text.
+      #
+      # Otherwise, this method tries to create one at an appropriate
+      # place supplying head and/or html elements as necessary, which
+      # is inside a head element if any, and before any text node or
+      # content element (typically <body>) if any.
+      #
+      # The result when trying to set an encoding that is different
+      # from the document encoding is undefined.
+      #
+      # Beware in CRuby, that libxml2 automatically inserts a meta tag
+      # into a head element.
+      def meta_encoding= encoding
+        case
+        when meta = meta_content_type
+          meta['content'] = 'text/html; charset=%s' % encoding
+          encoding
+        when meta = at('//meta[@charset]')
+          meta['charset'] = encoding
+        else
+          meta = XML::Node.new('meta', self)
+          if dtd = internal_subset and dtd.html5_dtd?
+            meta['charset'] = encoding
+          else
+            meta['http-equiv'] = 'Content-Type'
+            meta['content'] = 'text/html; charset=%s' % encoding
+          end
+
+          case
+          when head = at('//head')
+            head.prepend_child(meta)
+          else
+            set_metadata_element(meta)
+          end
+          encoding
+        end
+      end
+
+      def meta_content_type
+        xpath('//meta[@http-equiv and boolean(@content)]').find { |node|
+          node['http-equiv'] =~ /\AContent-Type\z/i
+        }
+      end
+      private :meta_content_type
+
+      ###
+      # Get the title string of this document.  Return nil if there is
+      # no title tag.
+      def title
+        title = at('//title') and title.inner_text
+      end
+
+      ###
+      # Set the title string of this document.
+      #
+      # If a title element is already present, its content is replaced
+      # with the given text.
+      #
+      # Otherwise, this method tries to create one at an appropriate
+      # place supplying head and/or html elements as necessary, which
+      # is inside a head element if any, right after a meta
+      # encoding/charset tag if any, and before any text node or
+      # content element (typically <body>) if any.
+      def title=(text)
+        tnode = XML::Text.new(text, self)
+        if title = at('//title')
+          title.children = tnode
+          return text
+        end
+
+        title = XML::Node.new('title', self) << tnode
+        case
+        when head = at('//head')
+          head << title
+        when meta = at('//meta[@charset]') || meta_content_type
+          # better put after charset declaration
+          meta.add_next_sibling(title)
+        else
+          set_metadata_element(title)
+        end
+        text
+      end
+
+      def set_metadata_element(element)
+        case
+        when head = at('//head')
+          head << element
+        when html = at('//html')
+          head = html.prepend_child(XML::Node.new('head', self))
+          head.prepend_child(element)
+        when first = children.find { |node|
+            case node
+            when XML::Element, XML::Text
+              true
+            end
+          }
+          # We reach here only if the underlying document model
+          # allows <html>/<head> elements to be omitted and does not
+          # automatically supply them.
+          first.add_previous_sibling(element)
+        else
+          html = add_child(XML::Node.new('html', self))
+          head = html.add_child(XML::Node.new('head', self))
+          head.prepend_child(element)
+        end
+      end
+      private :set_metadata_element
+
+      ####
+      # Serialize Node using +options+.  Save options can also be set using a
+      # block. See SaveOptions.
+      #
+      # These two statements are equivalent:
+      #
+      #  node.serialize(:encoding => 'UTF-8', :save_with => FORMAT | AS_XML)
+      #
+      # or
+      #
+      #   node.serialize(:encoding => 'UTF-8') do |config|
+      #     config.format.as_xml
+      #   end
+      #
+      def serialize options = {}
+        options[:save_with] ||= XML::Node::SaveOptions::DEFAULT_HTML
+        super
+      end
+
+      ####
+      # Create a Nokogiri::XML::DocumentFragment from +tags+
+      def fragment tags = nil
+        DocumentFragment.new(self, tags, self.root)
+      end
+
+      class << self
+        ###
+        # Parse HTML.  +string_or_io+ may be a String, or any object that
+        # responds to _read_ and _close_ such as an IO, or StringIO.
+        # +url+ is resource where this document is located.  +encoding+ is the
+        # encoding that should be used when processing the document. +options+
+        # is a number that sets options in the parser, such as
+        # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
+        # Nokogiri::XML::ParseOptions.
+        def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML
+
+          options = Nokogiri::XML::ParseOptions.new(options) if Fixnum === options
+          # Give the options to the user
+          yield options if block_given?
+
+          if string_or_io.respond_to?(:encoding)
+            unless string_or_io.encoding.name == "ASCII-8BIT"
+              encoding ||= string_or_io.encoding.name
+            end
+          end
+
+          if string_or_io.respond_to?(:read)
+            url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+            unless encoding
+              # Libxml2's parser has poor support for encoding
+              # detection.  First, it does not recognize the HTML5
+              # style meta charset declaration.  Secondly, even if it
+              # successfully detects an encoding hint, it does not
+              # re-decode or re-parse the preceding part which may be
+              # garbled.
+              #
+              # EncodingReader aims to perform advanced encoding
+              # detection beyond what Libxml2 does, and to emulate
+              # rewinding of a stream and make Libxml2 redo parsing
+              # from the start when an encoding hint is found.
+              string_or_io = EncodingReader.new(string_or_io)
+              begin
+                return read_io(string_or_io, url, encoding, options.to_i)
+              rescue EncodingFound => e
+                encoding = e.found_encoding
+              end
+            end
+            return read_io(string_or_io, url, encoding, options.to_i)
+          end
+
+          # read_memory pukes on empty docs
+          if string_or_io.nil? or string_or_io.empty?
+            return encoding ? new.tap { |i| i.encoding = encoding } : new
+          end
+
+          encoding ||= EncodingReader.detect_encoding(string_or_io)
+
+          read_memory(string_or_io, url, encoding, options.to_i)
+        end
+      end
+
+      class EncodingFound < StandardError # :nodoc:
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super("encoding found: %s" % encoding)
+        end
+      end
+
+      class EncodingReader # :nodoc:
+        class SAXHandler < Nokogiri::XML::SAX::Document # :nodoc:
+          attr_reader :encoding
+          
+          def initialize
+            @encoding = nil
+            super()
+          end
+    
+          def start_element(name, attrs = [])
+            return unless name == 'meta'
+            attr = Hash[attrs]
+            charset = attr['charset'] and
+              @encoding = charset
+            http_equiv = attr['http-equiv'] and
+              http_equiv.match(/\AContent-Type\z/i) and
+              content = attr['content'] and
+              m = content.match(/;\s*charset\s*=\s*([\w-]+)/) and
+              @encoding = m[1]
+          end
+        end
+        
+        class JumpSAXHandler < SAXHandler
+          def initialize(jumptag)
+            @jumptag = jumptag
+            super()
+          end
+
+          def start_element(name, attrs = [])
+            super
+            throw @jumptag, @encoding if @encoding
+            throw @jumptag, nil if name =~ /\A(?:div|h1|img|p|br)\z/
+          end
+        end
+
+        def self.detect_encoding(chunk)
+          if Nokogiri.jruby? && EncodingReader.is_jruby_without_fix?
+            return EncodingReader.detect_encoding_for_jruby_without_fix(chunk)
+          end
+          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
+            return Nokogiri.XML(m[1]).encoding
+
+          if Nokogiri.jruby?
+            m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
+              return m[4]
+            catch(:encoding_found) {
+              Nokogiri::HTML::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+              nil
+            }
+          else
+            handler = SAXHandler.new
+            parser = Nokogiri::HTML::SAX::PushParser.new(handler)
+            parser << chunk rescue Nokogiri::SyntaxError
+            handler.encoding
+          end
+        end
+
+        def self.is_jruby_without_fix?
+          JRUBY_VERSION.split('.').join.to_i < 165
+        end
+
+        def self.detect_encoding_for_jruby_without_fix(chunk)
+          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
+            return Nokogiri.XML(m[1]).encoding
+
+          m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
+            return m[4]
+
+          catch(:encoding_found) {
+            Nokogiri::HTML::SAX::Parser.new(JumpSAXHandler.new(:encoding_found.to_s)).parse(chunk)
+            nil
+          }
+        rescue Nokogiri::SyntaxError, RuntimeError
+          # Ignore parser errors that nokogiri may raise
+          nil
+        end
+
+        def initialize(io)
+          @io = io
+          @firstchunk = nil
+          @encoding_found = nil
+        end
+
+        # This method is used by the C extension so that
+        # Nokogiri::HTML::Document#read_io() does not leak memory when
+        # EncodingFound is raised.
+        attr_reader :encoding_found
+
+        def read(len)
+          # no support for a call without len
+
+          if !@firstchunk
+            @firstchunk = @io.read(len) or return nil
+
+            # This implementation expects that the first call from
+            # htmlReadIO() is made with a length long enough (~1KB) to
+            # achieve advanced encoding detection.
+            if encoding = EncodingReader.detect_encoding(@firstchunk)
+              # The first chunk is stored for the next read in retry.
+              raise @encoding_found = EncodingFound.new(encoding)
+            end
+          end
+          @encoding_found = nil
+
+          ret = @firstchunk.slice!(0, len)
+          if (len -= ret.length) > 0
+            rest = @io.read(len) and ret << rest
+          end
+          if ret.empty?
+            nil
+          else
+            ret
+          end
+        end
       end
     end
   end