nokogiri 1.4.7 → 1.5.0.beta.1

data/lib/nokogiri/css/parser.y

@@ -1,4 +1,4 @@
-class Nokogiri::CSS::Parser
+class Nokogiri::CSS::GeneratedParser
 
 token FUNCTION INCLUDES DASHMATCH LBRACE HASH PLUS GREATER S STRING IDENT
 token COMMA NUMBER PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL
@@ -39,9 +39,6 @@ rule
         result = Node.new(:CONDITIONAL_SELECTOR, val)
       }
     | function
-    | function pseudo {
-        result = Node.new(:CONDITIONAL_SELECTOR, val)
-      }
     | function attrib {
         result = Node.new(:CONDITIONAL_SELECTOR, val)
       }
@@ -233,5 +230,3 @@ end
 
 ---- header
 
-require 'nokogiri/css/parser_extras'
-

data/lib/nokogiri/css/tokenizer.rb

@@ -1,152 +1,7 @@
-#--
-# DO NOT MODIFY!!!!
-# This file is automatically generated by rex 1.0.5
-# from lexical definition file "lib/nokogiri/css/tokenizer.rex".
-#++
-
 module Nokogiri
-module CSS
-class Tokenizer
-  require 'strscan'
-
-  class ScanError < StandardError ; end
-
-  attr_reader   :lineno
-  attr_reader   :filename
-  attr_accessor :state
-
-  def scan_setup(str)
-    @ss = StringScanner.new(str)
-    @lineno =  1
-    @state  = nil
-  end
-
-  def action
-    yield
-  end
-
-  def scan_str(str)
-    scan_setup(str)
-    do_parse
-  end
-  alias :scan :scan_str
-
-  def load_file( filename )
-    @filename = filename
-    open(filename, "r") do |f|
-      scan_setup(f.read)
+  module CSS
+    class Tokenizer < GeneratedTokenizer
+      alias :scan :scan_setup
     end
   end
-
-  def scan_file( filename )
-    load_file(filename)
-    do_parse
-  end
-
-
-  def next_token
-    return if @ss.eos?
-    
-    # skips empty actions
-    until token = _next_token or @ss.eos?; end
-    token
-  end
-
-  def _next_token
-    text = @ss.peek(1)
-    @lineno  +=  1  if text == "\n"
-    token = case @state
-    when nil
-      case
-      when (text = @ss.scan(/has\([\s]*/))
-         action { [:HAS, text] }
-
-      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\([\s]*/))
-         action { [:FUNCTION, text] }
-
-      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
-         action { [:IDENT, text] }
-
-      when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
-         action { [:HASH, text] }
-
-      when (text = @ss.scan(/[\s]*~=[\s]*/))
-         action { [:INCLUDES, text] }
-
-      when (text = @ss.scan(/[\s]*\|=[\s]*/))
-         action { [:DASHMATCH, text] }
-
-      when (text = @ss.scan(/[\s]*\^=[\s]*/))
-         action { [:PREFIXMATCH, text] }
-
-      when (text = @ss.scan(/[\s]*\$=[\s]*/))
-         action { [:SUFFIXMATCH, text] }
-
-      when (text = @ss.scan(/[\s]*\*=[\s]*/))
-         action { [:SUBSTRINGMATCH, text] }
-
-      when (text = @ss.scan(/[\s]*!=[\s]*/))
-         action { [:NOT_EQUAL, text] }
-
-      when (text = @ss.scan(/[\s]*=[\s]*/))
-         action { [:EQUAL, text] }
-
-      when (text = @ss.scan(/[\s]*\)/))
-         action { [:RPAREN, text] }
-
-      when (text = @ss.scan(/[\s]*\[[\s]*/))
-         action { [:LSQUARE, text] }
-
-      when (text = @ss.scan(/[\s]*\]/))
-         action { [:RSQUARE, text] }
-
-      when (text = @ss.scan(/[\s]*\+[\s]*/))
-         action { [:PLUS, text] }
-
-      when (text = @ss.scan(/[\s]*>[\s]*/))
-         action { [:GREATER, text] }
-
-      when (text = @ss.scan(/[\s]*,[\s]*/))
-         action { [:COMMA, text] }
-
-      when (text = @ss.scan(/[\s]*~[\s]*/))
-         action { [:TILDE, text] }
-
-      when (text = @ss.scan(/\:not\([\s]*/))
-         action { [:NOT, text] }
-
-      when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
-         action { [:NUMBER, text] }
-
-      when (text = @ss.scan(/[\s]*\/\/[\s]*/))
-         action { [:DOUBLESLASH, text] }
-
-      when (text = @ss.scan(/[\s]*\/[\s]*/))
-         action { [:SLASH, text] }
-
-      when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
-         action {[:UNICODE_RANGE, text] }
-
-      when (text = @ss.scan(/[\s]+/))
-         action { [:S, text] }
-
-      when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*'/))
-         action { [:STRING, text] }
-
-      when (text = @ss.scan(/./))
-         action { [text, text] }
-
-      else
-        text = @ss.string[@ss.pos .. -1]
-        raise  ScanError, "can not match: '" + text + "'"
-      end  # if
-
-    else
-      raise  ScanError, "undefined state: '" + state.to_s + "'"
-    end  # case state
-    token
-  end  # def _next_token
-
-end # class
-end
 end

data/lib/nokogiri/css/tokenizer.rex

@@ -1,6 +1,6 @@
 module Nokogiri
 module CSS
-class Tokenizer
+class GeneratedTokenizer < GeneratedParser
 
 macro
   nl        \n|\r\n|\r|\f

data/lib/nokogiri/css/xpath_visitor.rb

@@ -11,25 +11,18 @@ module Nokogiri
           'child::text()'
         when /^self\(/
           "self::#{node.value[1]}"
-        when /^eq\(/
-          "position() = #{node.value[1]}"
-        when /^(nth|nth-of-type|nth-child)\(/
+        when /^(eq|nth|nth-of-type|nth-child)\(/
           if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
             an_plus_b(node.value[1])
           else
-            "position() = #{node.value[1]}"
-          end
-        when /^(nth-last-child|nth-last-of-type)\(/
-          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
-            an_plus_b(node.value[1], :last => true)
-          else
-            index = node.value[1].to_i - 1
-            index == 0 ? "position() = last()" : "position() = last() - #{index}"
+            "position() = " + node.value[1]
           end
         when /^(first|first-of-type)\(/
           "position() = 1"
         when /^(last|last-of-type)\(/
           "position() = last()"
+        when /^(nth-last-child|nth-last-of-type)\(/
+          "position() = last() - #{node.value[1]}"
         when /^contains\(/
           "contains(., #{node.value[1]})"
         when /^gt\(/
@@ -55,6 +48,13 @@ module Nokogiri
         end
       end
 
+      def visit_preceding_selector node
+        node.value.last.accept(self) +
+          '[preceding-sibling::' +
+          node.value.first.accept(self) +
+          ']'
+      end
+
       def visit_id node
         node.value.first =~ /^#(.*)$/
         "@id = '#{$1}'"
@@ -126,7 +126,6 @@ module Nokogiri
       {
         'combinator'                => ' and ',
         'direct_adjacent_selector'  => "/following-sibling::*[1]/self::",
-        'preceding_selector'        => "/following-sibling::",
         'descendant_selector'       => '//',
         'child_selector'            => '/',
       }.each do |k,v|
@@ -151,18 +150,17 @@ module Nokogiri
       end
 
     private
-      def an_plus_b node, options={}
+      def an_plus_b node
         raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
 
         a = node.value[0].to_i
         b = node.value[3].to_i
-        position = options[:last] ? "(last()-position()+1)" : "position()"
 
         if (b == 0)
-          return "(#{position} mod #{a}) = 0"
+          return "(position() mod #{a}) = 0"
         else
           compare = (a < 0) ? "<=" : ">="
-          return "(#{position} #{compare} #{b}) and (((#{position}-#{b}) mod #{a.abs}) = 0)"
+          return "(position() #{compare} #{b}) and (((position()-#{b}) mod #{a.abs}) = 0)"
         end
       end
 

data/lib/nokogiri/decorators/slop.rb

@@ -7,22 +7,20 @@ module Nokogiri
       ###
       # look for node with +name+.  See Nokogiri.Slop
       def method_missing name, *args, &block
-        prefix = implied_xpath_context
-
         if args.empty?
-          list = xpath("#{prefix}#{name.to_s.sub(/^_/, '')}")
+          list = xpath("./#{name}")
         elsif args.first.is_a? Hash
           hash = args.first
           if hash[:css]
             list = css("#{name}#{hash[:css]}")
           elsif hash[:xpath]
             conds = Array(hash[:xpath]).join(' and ')
-            list = xpath("#{prefix}#{name}[#{conds}]")
+            list = xpath("./#{name}[#{conds}]")
           end
         else
           CSS::Parser.without_cache do
             list = xpath(
-              *CSS.xpath_for("#{name}#{args.first}", :prefix => prefix)
+              *CSS.xpath_for("#{name}#{args.first}", :prefix => "./")
             )
           end
         end

data/lib/nokogiri/html.rb

@@ -4,6 +4,7 @@ require 'nokogiri/html/document_fragment'
 require 'nokogiri/html/sax/parser_context'
 require 'nokogiri/html/sax/parser'
 require 'nokogiri/html/element_description'
+require 'nokogiri/html/element_description_defaults'
 
 module Nokogiri
   class << self
@@ -24,8 +25,8 @@ module Nokogiri
 
       ####
       # Parse a fragment from +string+ in to a NodeSet.
-      def fragment string, encoding = nil
-        HTML::DocumentFragment.parse string, encoding
+      def fragment string
+        HTML::DocumentFragment.parse(string)
       end
     end
 

data/lib/nokogiri/html/document.rb

@@ -3,44 +3,25 @@ module Nokogiri
     class Document < Nokogiri::XML::Document
       ###
       # Get the meta tag encoding for this document.  If there is no meta tag,
-      # then nil is returned.
+      # then nil is returned
       def meta_encoding
-        meta = meta_content_type and
-          /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
+        return nil unless meta = css('meta').find { |node|
+          node['http-equiv'] =~ /Content-Type/i
+        }
+
+        /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
       end
 
       ###
-      # Set the meta tag encoding for this document.  If there is no meta
-      # content tag, the encoding is not set.
+      # Set the meta tag encoding for this document.  If there is no meta 
+      # content tag, nil is returned and the encoding is not set.
       def meta_encoding= encoding
-        meta = meta_content_type and
-          meta['content'] = "text/html; charset=%s" % encoding
-      end
-
-      def meta_content_type
-        css('meta[@http-equiv]').find { |node|
-          node['http-equiv'] =~ /\AContent-Type\z/i
+        return nil unless meta = css('meta').find { |node|
+          node['http-equiv'] =~ /Content-Type/i
         }
-      end
-      private :meta_content_type
 
-      ###
-      # Get the title string of this document.  Return nil if there is
-      # no title tag.
-      def title
-        title = at('title') and title.inner_text
-      end
-
-      ###
-      # Set the title string of this document.  If there is no head
-      # element, the title is not set.
-      def title=(text)
-        unless title = at('title')
-          head = at('head') or return nil
-          title = Nokogiri::XML::Node.new('title', self)
-          head << title
-        end
-        title.children = XML::Text.new(text, self)
+        meta['content'] = "text/html; charset=%s" % encoding
+        encoding
       end
 
       ####
@@ -57,8 +38,11 @@ module Nokogiri
       #     config.format.as_xml
       #   end
       #
-      def serialize options = {}
-        options[:save_with] ||= XML::Node::SaveOptions::DEFAULT_HTML
+      def serialize options = {}, &block
+        options[:save_with] ||= XML::Node::SaveOptions::FORMAT |
+            XML::Node::SaveOptions::AS_HTML |
+            XML::Node::SaveOptions::NO_DECLARATION |
+            XML::Node::SaveOptions::NO_EMPTY_TAGS
         super
       end
 
@@ -77,7 +61,7 @@ module Nokogiri
         # is a number that sets options in the parser, such as
         # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
         # Nokogiri::XML::ParseOptions.
-        def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML
+        def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
 
           options = Nokogiri::XML::ParseOptions.new(options) if Fixnum === options
           # Give the options to the user
@@ -91,116 +75,16 @@ module Nokogiri
 
           if string_or_io.respond_to?(:read)
             url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
-            if !encoding
-              # Perform advanced encoding detection that libxml2 does
-              # not do.
-              string_or_io = EncodingReader.new(string_or_io)
-              begin
-                return read_io(string_or_io, url, encoding, options.to_i)
-              rescue EncodingFoundException => e
-                # A retry is required because libxml2 has a problem in
-                # that it cannot switch encoding well in the middle of
-                # parsing, especially if it has already seen a
-                # non-ASCII character when it finds an encoding hint.
-                encoding = e.encoding
-              end
-            end
             return read_io(string_or_io, url, encoding, options.to_i)
           end
 
           # read_memory pukes on empty docs
           return new if string_or_io.nil? or string_or_io.empty?
 
-          if !encoding
-            encoding = EncodingReader.detect_encoding(string_or_io)
-          end
-
           read_memory(string_or_io, url, encoding, options.to_i)
         end
       end
 
-      class EncodingFoundException < Exception # :nodoc:
-        attr_reader :encoding
-
-        def initialize(encoding)
-          @encoding = encoding
-          super("encoding found: %s" % encoding)
-        end
-      end
-
-      class EncodingReader # :nodoc:
-        class SAXHandler < Nokogiri::XML::SAX::Document # :nodoc:
-          attr_reader :encoding
-
-          def found(encoding)
-            @encoding = encoding
-            throw :found
-          end
-
-          def not_found(encoding)
-            found nil
-          end
-
-          def start_element(name, attrs = [])
-            case name
-            when /\A(?:div|h1|img|p|br)\z/
-              not_found
-            when 'meta'
-              attr = Hash[attrs]
-              http_equiv = attr['http-equiv'] and
-                http_equiv.match(/\AContent-Type\z/i) and
-                content = attr['content'] and
-                m = content.match(/;\s*charset\s*=\s*([\w-]+)/) and
-                found m[1]
-            end
-          end
-        end
-
-        def self.detect_encoding(chunk)
-          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
-            return Nokogiri.XML(m[1]).encoding
-
-          handler = SAXHandler.new
-          parser = Nokogiri::HTML::SAX::Parser.new(handler)
-          catch(:found) {
-            parser.parse(chunk)
-          }
-          handler.encoding
-        rescue => e
-          nil
-        end
-
-        def initialize(io)
-          @io = io
-          @firstchunk = nil
-        end
-
-        def read(len)
-          # no support for a call without len
-
-          if !@firstchunk
-            @firstchunk = @io.read(len) or return nil
-
-            # This implementation expects that the first call from
-            # htmlReadIO() is made with a length long enough (~1KB) to
-            # achieve advanced encoding detection.
-            if encoding = EncodingReader.detect_encoding(@firstchunk)
-              # The first chunk is stored for the next read in retry.
-              raise EncodingFoundException, encoding
-            end
-          end
-
-          ret = @firstchunk.slice!(0, len)
-          if (len -= ret.length) > 0
-            rest = @io.read(len) and ret << rest
-          end
-          if ret.empty?
-            nil
-          else
-            ret
-          end
-        end
-      end
     end
   end
 end