nokogiri 1.4.3.1-java → 1.5.0-java

data/lib/nokogiri/css/parser.y

@@ -1,4 +1,4 @@
-class Nokogiri::CSS::GeneratedParser
+class Nokogiri::CSS::Parser
 
 token FUNCTION INCLUDES DASHMATCH LBRACE HASH PLUS GREATER S STRING IDENT
 token COMMA NUMBER PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL
@@ -233,3 +233,5 @@ end
 
 ---- header
 
+require 'nokogiri/css/parser_extras'
+

data/lib/nokogiri/css/parser_extras.rb

@@ -0,0 +1,91 @@
+require 'thread'
+
+module Nokogiri
+  module CSS
+    class Parser < Racc::Parser
+      @cache_on = true
+      @cache    = {}
+      @mutex    = Mutex.new
+
+      class << self
+        # Turn on CSS parse caching
+        attr_accessor :cache_on
+        alias :cache_on? :cache_on
+        alias :set_cache :cache_on=
+
+        # Get the css selector in +string+ from the cache
+        def [] string
+          return unless @cache_on
+          @mutex.synchronize { @cache[string] }
+        end
+
+        # Set the css selector in +string+ in the cache to +value+
+        def []= string, value
+          return value unless @cache_on
+          @mutex.synchronize { @cache[string] = value }
+        end
+
+        # Clear the cache
+        def clear_cache
+          @mutex.synchronize { @cache = {} }
+        end
+
+        # Execute +block+ without cache
+        def without_cache &block
+          tmp = @cache_on
+          @cache_on = false
+          block.call
+          @cache_on = tmp
+        end
+
+        ###
+        # Parse this CSS selector in +selector+.  Returns an AST.
+        def parse selector
+          @warned ||= false
+          unless @warned
+            $stderr.puts('Nokogiri::CSS::Parser.parse is deprecated, call Nokogiri::CSS.parse(), this will be removed August 1st or version 1.4.0 (whichever is first)')
+            @warned = true
+          end
+          new.parse selector
+        end
+      end
+
+      # Create a new CSS parser with respect to +namespaces+
+      def initialize namespaces = {}
+        @tokenizer  = Tokenizer.new
+        @namespaces = namespaces
+        super()
+      end
+
+      def parse string
+        @tokenizer.scan_setup string
+        do_parse
+      end
+
+      def next_token
+        @tokenizer.next_token
+      end
+
+      # Get the xpath for +string+ using +options+
+      def xpath_for string, options={}
+        key = "#{string}#{options[:ns]}#{options[:prefix]}"
+        v = self.class[key]
+        return v if v
+
+        args = [
+          options[:prefix] || '//',
+          options[:visitor] || XPathVisitor.new
+        ]
+        self.class[key] = parse(string).map { |ast|
+          ast.to_xpath(*args)
+        }
+      end
+
+      # On CSS parser error, raise an exception
+      def on_error error_token_id, error_value, value_stack
+        after = value_stack.compact.last
+        raise SyntaxError.new("unexpected '#{error_value}' after '#{after}'")
+      end
+    end
+  end
+end

data/lib/nokogiri/css/tokenizer.rb

@@ -1,7 +1,152 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "lib/nokogiri/css/tokenizer.rex".
+#++
+
 module Nokogiri
-  module CSS
-    class Tokenizer < GeneratedTokenizer
-      alias :scan :scan_setup
+module CSS
+class Tokenizer
+  require 'strscan'
+
+  class ScanError < StandardError ; end
+
+  attr_reader   :lineno
+  attr_reader   :filename
+  attr_accessor :state
+
+  def scan_setup(str)
+    @ss = StringScanner.new(str)
+    @lineno =  1
+    @state  = nil
+  end
+
+  def action
+    yield
+  end
+
+  def scan_str(str)
+    scan_setup(str)
+    do_parse
+  end
+  alias :scan :scan_str
+
+  def load_file( filename )
+    @filename = filename
+    open(filename, "r") do |f|
+      scan_setup(f.read)
     end
   end
+
+  def scan_file( filename )
+    load_file(filename)
+    do_parse
+  end
+
+
+  def next_token
+    return if @ss.eos?
+    
+    # skips empty actions
+    until token = _next_token or @ss.eos?; end
+    token
+  end
+
+  def _next_token
+    text = @ss.peek(1)
+    @lineno  +=  1  if text == "\n"
+    token = case @state
+    when nil
+      case
+      when (text = @ss.scan(/has\([\s]*/))
+         action { [:HAS, text] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\([\s]*/))
+         action { [:FUNCTION, text] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
+         action { [:IDENT, text] }
+
+      when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
+         action { [:HASH, text] }
+
+      when (text = @ss.scan(/[\s]*~=[\s]*/))
+         action { [:INCLUDES, text] }
+
+      when (text = @ss.scan(/[\s]*\|=[\s]*/))
+         action { [:DASHMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\^=[\s]*/))
+         action { [:PREFIXMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\$=[\s]*/))
+         action { [:SUFFIXMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\*=[\s]*/))
+         action { [:SUBSTRINGMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*!=[\s]*/))
+         action { [:NOT_EQUAL, text] }
+
+      when (text = @ss.scan(/[\s]*=[\s]*/))
+         action { [:EQUAL, text] }
+
+      when (text = @ss.scan(/[\s]*\)/))
+         action { [:RPAREN, text] }
+
+      when (text = @ss.scan(/[\s]*\[[\s]*/))
+         action { [:LSQUARE, text] }
+
+      when (text = @ss.scan(/[\s]*\]/))
+         action { [:RSQUARE, text] }
+
+      when (text = @ss.scan(/[\s]*\+[\s]*/))
+         action { [:PLUS, text] }
+
+      when (text = @ss.scan(/[\s]*>[\s]*/))
+         action { [:GREATER, text] }
+
+      when (text = @ss.scan(/[\s]*,[\s]*/))
+         action { [:COMMA, text] }
+
+      when (text = @ss.scan(/[\s]*~[\s]*/))
+         action { [:TILDE, text] }
+
+      when (text = @ss.scan(/\:not\([\s]*/))
+         action { [:NOT, text] }
+
+      when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
+         action { [:NUMBER, text] }
+
+      when (text = @ss.scan(/[\s]*\/\/[\s]*/))
+         action { [:DOUBLESLASH, text] }
+
+      when (text = @ss.scan(/[\s]*\/[\s]*/))
+         action { [:SLASH, text] }
+
+      when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
+         action {[:UNICODE_RANGE, text] }
+
+      when (text = @ss.scan(/[\s]+/))
+         action { [:S, text] }
+
+      when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*'/))
+         action { [:STRING, text] }
+
+      when (text = @ss.scan(/./))
+         action { [text, text] }
+
+      else
+        text = @ss.string[@ss.pos .. -1]
+        raise  ScanError, "can not match: '" + text + "'"
+      end  # if
+
+    else
+      raise  ScanError, "undefined state: '" + state.to_s + "'"
+    end  # case state
+    token
+  end  # def _next_token
+
+end # class
+end
 end

data/lib/nokogiri/css/tokenizer.rex

@@ -1,6 +1,6 @@
 module Nokogiri
 module CSS
-class GeneratedTokenizer < GeneratedParser
+class Tokenizer
 
 macro
   nl        \n|\r\n|\r|\f

data/lib/nokogiri/css/xpath_visitor.rb

@@ -11,18 +11,25 @@ module Nokogiri
           'child::text()'
         when /^self\(/
           "self::#{node.value[1]}"
-        when /^(eq|nth|nth-of-type|nth-child)\(/
+        when /^eq\(/
+          "position() = #{node.value[1]}"
+        when /^(nth|nth-of-type|nth-child)\(/
           if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
             an_plus_b(node.value[1])
           else
-            "position() = " + node.value[1]
+            "position() = #{node.value[1]}"
+          end
+        when /^(nth-last-child|nth-last-of-type)\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
+            an_plus_b(node.value[1], :last => true)
+          else
+            index = node.value[1].to_i - 1
+            index == 0 ? "position() = last()" : "position() = last() - #{index}"
           end
         when /^(first|first-of-type)\(/
           "position() = 1"
         when /^(last|last-of-type)\(/
           "position() = last()"
-        when /^(nth-last-child|nth-last-of-type)\(/
-          "position() = last() - #{node.value[1]}"
         when /^contains\(/
           "contains(., #{node.value[1]})"
         when /^gt\(/
@@ -144,17 +151,18 @@ module Nokogiri
       end
 
     private
-      def an_plus_b node
+      def an_plus_b node, options={}
         raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
 
         a = node.value[0].to_i
         b = node.value[3].to_i
+        position = options[:last] ? "(last()-position()+1)" : "position()"
 
         if (b == 0)
-          return "(position() mod #{a}) = 0"
+          return "(#{position} mod #{a}) = 0"
         else
           compare = (a < 0) ? "<=" : ">="
-          return "(position() #{compare} #{b}) and (((position()-#{b}) mod #{a.abs}) = 0)"
+          return "(#{position} #{compare} #{b}) and (((#{position}-#{b}) mod #{a.abs}) = 0)"
         end
       end
 

data/lib/nokogiri/css.rb

@@ -1,9 +1,11 @@
 require 'nokogiri/css/node'
 require 'nokogiri/css/xpath_visitor'
-require 'nokogiri/css/generated_parser'
-require 'nokogiri/css/generated_tokenizer'
-require 'nokogiri/css/tokenizer'
+x = $-w
+$-w = false
 require 'nokogiri/css/parser'
+$-w = x
+
+require 'nokogiri/css/tokenizer'
 require 'nokogiri/css/syntax_error'
 
 module Nokogiri

data/lib/nokogiri/decorators/slop.rb

@@ -7,20 +7,22 @@ module Nokogiri
       ###
       # look for node with +name+.  See Nokogiri.Slop
       def method_missing name, *args, &block
+        prefix = implied_xpath_context
+
         if args.empty?
-          list = xpath("./#{name}")
+          list = xpath("#{prefix}#{name.to_s.sub(/^_/, '')}")
         elsif args.first.is_a? Hash
           hash = args.first
           if hash[:css]
             list = css("#{name}#{hash[:css]}")
           elsif hash[:xpath]
             conds = Array(hash[:xpath]).join(' and ')
-            list = xpath("./#{name}[#{conds}]")
+            list = xpath("#{prefix}#{name}[#{conds}]")
           end
         else
           CSS::Parser.without_cache do
             list = xpath(
-              *CSS.xpath_for("#{name}#{args.first}", :prefix => "./")
+              *CSS.xpath_for("#{name}#{args.first}", :prefix => prefix)
             )
           end
         end

data/lib/nokogiri/html/document.rb

@@ -3,25 +3,44 @@ module Nokogiri
     class Document < Nokogiri::XML::Document
       ###
       # Get the meta tag encoding for this document.  If there is no meta tag,
-      # then nil is returned
+      # then nil is returned.
       def meta_encoding
-        return nil unless meta = css('meta').find { |node|
-          node['http-equiv'] =~ /Content-Type/i
-        }
-
-        /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
+        meta = meta_content_type and
+          /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
       end
 
       ###
-      # Set the meta tag encoding for this document.  If there is no meta 
-      # content tag, nil is returned and the encoding is not set.
+      # Set the meta tag encoding for this document.  If there is no meta
+      # content tag, the encoding is not set.
       def meta_encoding= encoding
-        return nil unless meta = css('meta').find { |node|
-          node['http-equiv'] =~ /Content-Type/i
+        meta = meta_content_type and
+          meta['content'] = "text/html; charset=%s" % encoding
+      end
+
+      def meta_content_type
+        css('meta[@http-equiv]').find { |node|
+          node['http-equiv'] =~ /\AContent-Type\z/i
         }
+      end
+      private :meta_content_type
 
-        meta['content'] = "text/html; charset=%s" % encoding
-        encoding
+      ###
+      # Get the title string of this document.  Return nil if there is
+      # no title tag.
+      def title
+        title = at('title') and title.inner_text
+      end
+
+      ###
+      # Set the title string of this document.  If there is no head
+      # element, the title is not set.
+      def title=(text)
+        unless title = at('title')
+          head = at('head') or return nil
+          title = Nokogiri::XML::Node.new('title', self)
+          head << title
+        end
+        title.children = XML::Text.new(text, self)
       end
 
       ####
@@ -38,11 +57,8 @@ module Nokogiri
       #     config.format.as_xml
       #   end
       #
-      def serialize options = {}, &block
-        options[:save_with] ||= XML::Node::SaveOptions::FORMAT |
-            XML::Node::SaveOptions::AS_HTML |
-            XML::Node::SaveOptions::NO_DECLARATION |
-            XML::Node::SaveOptions::NO_EMPTY_TAGS
+      def serialize options = {}
+        options[:save_with] ||= XML::Node::SaveOptions::DEFAULT_HTML
         super
       end
 
@@ -54,14 +70,14 @@ module Nokogiri
 
       class << self
         ###
-        # Parse HTML.  +thing+ may be a String, or any object that
+        # Parse HTML.  +string_or_io+ may be a String, or any object that
         # responds to _read_ and _close_ such as an IO, or StringIO.
         # +url+ is resource where this document is located.  +encoding+ is the
         # encoding that should be used when processing the document. +options+
         # is a number that sets options in the parser, such as
         # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
         # Nokogiri::XML::ParseOptions.
-        def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
+        def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML
 
           options = Nokogiri::XML::ParseOptions.new(options) if Fixnum === options
           # Give the options to the user
@@ -75,16 +91,123 @@ module Nokogiri
 
           if string_or_io.respond_to?(:read)
             url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+            if !encoding
+              # Perform advanced encoding detection that libxml2 does
+              # not do.
+              string_or_io = EncodingReader.new(string_or_io)
+              begin
+                return read_io(string_or_io, url, encoding, options.to_i)
+              rescue EncodingFoundException => e
+                # A retry is required because libxml2 has a problem in
+                # that it cannot switch encoding well in the middle of
+                # parsing, especially if it has already seen a
+                # non-ASCII character when it finds an encoding hint.
+                encoding = e.encoding
+              end
+            end
             return read_io(string_or_io, url, encoding, options.to_i)
           end
 
           # read_memory pukes on empty docs
           return new if string_or_io.nil? or string_or_io.empty?
 
+          if !encoding
+            encoding = EncodingReader.detect_encoding(string_or_io)
+          end
+
           read_memory(string_or_io, url, encoding, options.to_i)
         end
       end
 
+      class EncodingFoundException < Exception # :nodoc:
+        attr_reader :encoding
+
+        def initialize(encoding)
+          @encoding = encoding
+          super("encoding found: %s" % encoding)
+        end
+      end
+
+      class EncodingReader # :nodoc:
+        class SAXHandler < Nokogiri::XML::SAX::Document # :nodoc:
+          attr_reader :encoding
+
+          def found(encoding)
+            @encoding = encoding
+            throw :found
+          end
+
+          def not_found(encoding)
+            found nil
+          end
+
+          def start_element(name, attrs = [])
+            case name
+            when /\A(?:div|h1|img|p|br)\z/
+              not_found
+            when 'meta'
+              attr = Hash[attrs]
+              charset = attr['charset'] and
+                found charset
+              http_equiv = attr['http-equiv'] and
+                http_equiv.match(/\AContent-Type\z/i) and
+                content = attr['content'] and
+                m = content.match(/;\s*charset\s*=\s*([\w-]+)/) and
+                found m[1]
+            end
+          end
+        end
+
+        def self.detect_encoding(chunk)
+          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
+            return Nokogiri.XML(m[1]).encoding
+
+          if Nokogiri.jruby?
+            m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
+              return m[4]
+          end
+
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML::SAX::Parser.new(handler)
+          catch(:found) {
+            parser.parse(chunk)
+          }
+          handler.encoding
+        rescue
+          nil
+        end
+
+        def initialize(io)
+          @io = io
+          @firstchunk = nil
+        end
+
+        def read(len)
+          # no support for a call without len
+
+          if !@firstchunk
+            @firstchunk = @io.read(len) or return nil
+
+            # This implementation expects that the first call from
+            # htmlReadIO() is made with a length long enough (~1KB) to
+            # achieve advanced encoding detection.
+            if encoding = EncodingReader.detect_encoding(@firstchunk)
+              # The first chunk is stored for the next read in retry.
+              raise EncodingFoundException, encoding
+            end
+          end
+
+          ret = @firstchunk.slice!(0, len)
+          if (len -= ret.length) > 0
+            rest = @io.read(len) and ret << rest
+          end
+          if ret.empty?
+            nil
+          else
+            ret
+          end
+        end
+      end
     end
   end
 end

data/lib/nokogiri/html/document_fragment.rb

@@ -1,6 +1,8 @@
 module Nokogiri
   module HTML
     class DocumentFragment < Nokogiri::XML::DocumentFragment
+      attr_accessor :errors
+
       ####
       # Create a Nokogiri::XML::DocumentFragment from +tags+, using +encoding+
       def self.parse tags, encoding = nil
@@ -15,24 +17,24 @@ module Nokogiri
       def initialize document, tags = nil, ctx = nil
         return self unless tags
 
-        children = if ctx
-                     ctx.parse("<div>#{tags.strip}</div>").first.children
-                   else
-                     ###
-                     # This is a horrible hack, but I don't care
-                     if tags.strip =~ /^<body/i
-                       path = "/html/body"
-                     else
-                       path = "/html/body/node()"
-                     end
+        if ctx
+          preexisting_errors = document.errors.dup
+          node_set = ctx.parse("<div>#{tags}</div>")
+          node_set.first.children.each { |child| child.parent = self } unless node_set.empty?
+          self.errors = document.errors - preexisting_errors
+        else
+          # This is a horrible hack, but I don't care
+          if tags.strip =~ /^<body/i
+            path = "/html/body"
+          else
+            path = "/html/body/node()"
+          end
 
-                     HTML::Document.parse(
-                       "<html><body>#{tags.strip}</body></html>",
-                       nil,
-                       document.encoding
-                     ).xpath(path)
-                   end
-        children.each { |child| child.parent = self }
+          temp_doc = HTML::Document.parse "<html><body>#{tags}", nil, document.encoding
+          temp_doc.xpath(path).each { |child| child.parent = self }
+          self.errors = temp_doc.errors
+        end
+        children
       end
     end
   end