nokogiri 1.5.0.beta.3 → 1.5.0.beta.4

data/lib/nokogiri/css/parser.y

@@ -1,4 +1,4 @@
-class Nokogiri::CSS::GeneratedParser
+class Nokogiri::CSS::Parser
 
 token FUNCTION INCLUDES DASHMATCH LBRACE HASH PLUS GREATER S STRING IDENT
 token COMMA NUMBER PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL
@@ -233,3 +233,5 @@ end
 
 ---- header
 
+require 'nokogiri/css/parser_extras'
+

data/lib/nokogiri/css/parser_extras.rb

@@ -0,0 +1,91 @@
+require 'thread'
+
+module Nokogiri
+  module CSS
+    class Parser < Racc::Parser
+      @cache_on = true
+      @cache    = {}
+      @mutex    = Mutex.new
+
+      class << self
+        # Turn on CSS parse caching
+        attr_accessor :cache_on
+        alias :cache_on? :cache_on
+        alias :set_cache :cache_on=
+
+        # Get the css selector in +string+ from the cache
+        def [] string
+          return unless @cache_on
+          @mutex.synchronize { @cache[string] }
+        end
+
+        # Set the css selector in +string+ in the cache to +value+
+        def []= string, value
+          return value unless @cache_on
+          @mutex.synchronize { @cache[string] = value }
+        end
+
+        # Clear the cache
+        def clear_cache
+          @mutex.synchronize { @cache = {} }
+        end
+
+        # Execute +block+ without cache
+        def without_cache &block
+          tmp = @cache_on
+          @cache_on = false
+          block.call
+          @cache_on = tmp
+        end
+
+        ###
+        # Parse this CSS selector in +selector+.  Returns an AST.
+        def parse selector
+          @warned ||= false
+          unless @warned
+            $stderr.puts('Nokogiri::CSS::Parser.parse is deprecated, call Nokogiri::CSS.parse(), this will be removed August 1st or version 1.4.0 (whichever is first)')
+            @warned = true
+          end
+          new.parse selector
+        end
+      end
+
+      # Create a new CSS parser with respect to +namespaces+
+      def initialize namespaces = {}
+        @tokenizer  = Tokenizer.new
+        @namespaces = namespaces
+        super()
+      end
+
+      def parse string
+        @tokenizer.scan_setup string
+        do_parse
+      end
+
+      def next_token
+        @tokenizer.next_token
+      end
+
+      # Get the xpath for +string+ using +options+
+      def xpath_for string, options={}
+        key = "#{string}#{options[:ns]}#{options[:prefix]}"
+        v = self.class[key]
+        return v if v
+
+        args = [
+          options[:prefix] || '//',
+          options[:visitor] || XPathVisitor.new
+        ]
+        self.class[key] = parse(string).map { |ast|
+          ast.to_xpath(*args)
+        }
+      end
+
+      # On CSS parser error, raise an exception
+      def on_error error_token_id, error_value, value_stack
+        after = value_stack.compact.last
+        raise SyntaxError.new("unexpected '#{error_value}' after '#{after}'")
+      end
+    end
+  end
+end

data/lib/nokogiri/css/tokenizer.rb

@@ -1,7 +1,152 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "lib/nokogiri/css/tokenizer.rex".
+#++
+
 module Nokogiri
-  module CSS
-    class Tokenizer < GeneratedTokenizer
-      alias :scan :scan_setup
+module CSS
+class Tokenizer
+  require 'strscan'
+
+  class ScanError < StandardError ; end
+
+  attr_reader   :lineno
+  attr_reader   :filename
+  attr_accessor :state
+
+  def scan_setup(str)
+    @ss = StringScanner.new(str)
+    @lineno =  1
+    @state  = nil
+  end
+
+  def action
+    yield
+  end
+
+  def scan_str(str)
+    scan_setup(str)
+    do_parse
+  end
+  alias :scan :scan_str
+
+  def load_file( filename )
+    @filename = filename
+    open(filename, "r") do |f|
+      scan_setup(f.read)
     end
   end
+
+  def scan_file( filename )
+    load_file(filename)
+    do_parse
+  end
+
+
+  def next_token
+    return if @ss.eos?
+    
+    # skips empty actions
+    until token = _next_token or @ss.eos?; end
+    token
+  end
+
+  def _next_token
+    text = @ss.peek(1)
+    @lineno  +=  1  if text == "\n"
+    token = case @state
+    when nil
+      case
+      when (text = @ss.scan(/has\([\s]*/))
+         action { [:HAS, text] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\([\s]*/))
+         action { [:FUNCTION, text] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
+         action { [:IDENT, text] }
+
+      when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
+         action { [:HASH, text] }
+
+      when (text = @ss.scan(/[\s]*~=[\s]*/))
+         action { [:INCLUDES, text] }
+
+      when (text = @ss.scan(/[\s]*\|=[\s]*/))
+         action { [:DASHMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\^=[\s]*/))
+         action { [:PREFIXMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\$=[\s]*/))
+         action { [:SUFFIXMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\*=[\s]*/))
+         action { [:SUBSTRINGMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*!=[\s]*/))
+         action { [:NOT_EQUAL, text] }
+
+      when (text = @ss.scan(/[\s]*=[\s]*/))
+         action { [:EQUAL, text] }
+
+      when (text = @ss.scan(/[\s]*\)/))
+         action { [:RPAREN, text] }
+
+      when (text = @ss.scan(/[\s]*\[[\s]*/))
+         action { [:LSQUARE, text] }
+
+      when (text = @ss.scan(/[\s]*\]/))
+         action { [:RSQUARE, text] }
+
+      when (text = @ss.scan(/[\s]*\+[\s]*/))
+         action { [:PLUS, text] }
+
+      when (text = @ss.scan(/[\s]*>[\s]*/))
+         action { [:GREATER, text] }
+
+      when (text = @ss.scan(/[\s]*,[\s]*/))
+         action { [:COMMA, text] }
+
+      when (text = @ss.scan(/[\s]*~[\s]*/))
+         action { [:TILDE, text] }
+
+      when (text = @ss.scan(/\:not\([\s]*/))
+         action { [:NOT, text] }
+
+      when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
+         action { [:NUMBER, text] }
+
+      when (text = @ss.scan(/[\s]*\/\/[\s]*/))
+         action { [:DOUBLESLASH, text] }
+
+      when (text = @ss.scan(/[\s]*\/[\s]*/))
+         action { [:SLASH, text] }
+
+      when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
+         action {[:UNICODE_RANGE, text] }
+
+      when (text = @ss.scan(/[\s]+/))
+         action { [:S, text] }
+
+      when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*'/))
+         action { [:STRING, text] }
+
+      when (text = @ss.scan(/./))
+         action { [text, text] }
+
+      else
+        text = @ss.string[@ss.pos .. -1]
+        raise  ScanError, "can not match: '" + text + "'"
+      end  # if
+
+    else
+      raise  ScanError, "undefined state: '" + state.to_s + "'"
+    end  # case state
+    token
+  end  # def _next_token
+
+end # class
+end
 end

data/lib/nokogiri/css/tokenizer.rex

@@ -1,6 +1,6 @@
 module Nokogiri
 module CSS
-class GeneratedTokenizer < GeneratedParser
+class Tokenizer
 
 macro
   nl        \n|\r\n|\r|\f

data/lib/nokogiri/html/document.rb

@@ -3,25 +3,44 @@ module Nokogiri
     class Document < Nokogiri::XML::Document
       ###
       # Get the meta tag encoding for this document.  If there is no meta tag,
-      # then nil is returned
+      # then nil is returned.
       def meta_encoding
-        return nil unless meta = css('meta').find { |node|
-          node['http-equiv'] =~ /Content-Type/i
-        }
-
-        /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
+        meta = meta_content_type and
+          /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
       end
 
       ###
       # Set the meta tag encoding for this document.  If there is no meta
-      # content tag, nil is returned and the encoding is not set.
+      # content tag, the encoding is not set.
       def meta_encoding= encoding
-        return nil unless meta = css('meta').find { |node|
-          node['http-equiv'] =~ /Content-Type/i
+        meta = meta_content_type and
+          meta['content'] = "text/html; charset=%s" % encoding
+      end
+
+      def meta_content_type
+        css('meta[@http-equiv]').find { |node|
+          node['http-equiv'] =~ /\AContent-Type\z/i
         }
+      end
+      private :meta_content_type
 
-        meta['content'] = "text/html; charset=%s" % encoding
-        encoding
+      ###
+      # Get the title string of this document.  Return nil if there is
+      # no title tag.
+      def title
+        title = at('title') and title.inner_text
+      end
+
+      ###
+      # Set the title string of this document.  If there is no head
+      # element, the title is not set.
+      def title=(text)
+        unless title = at('title')
+          head = at('head') or return nil
+          title = Nokogiri::XML::Node.new('title', self)
+          head << title
+        end
+        title.children = XML::Text.new(text, self)
       end
 
       ####
@@ -75,16 +94,124 @@ module Nokogiri
 
           if string_or_io.respond_to?(:read)
             url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+            if !encoding
+              # Perform further encoding detection that libxml2 does
+              # not do.
+              string_or_io = EncodingReader.new(string_or_io)
+              begin
+                return read_io(string_or_io, url, encoding, options.to_i)
+              rescue EncodingFoundException => e
+                # A retry is required because libxml2 has a problem in
+                # that it cannot switch encoding well in the middle of
+                # parsing, especially if it has already seen a
+                # non-ASCII character when it finds an encoding hint.
+                encoding = e.encoding
+              end
+            end
             return read_io(string_or_io, url, encoding, options.to_i)
           end
 
           # read_memory pukes on empty docs
           return new if string_or_io.nil? or string_or_io.empty?
 
+          if !encoding
+            encoding = EncodingReader.detect_encoding(string_or_io)
+          end
+
           read_memory(string_or_io, url, encoding, options.to_i)
         end
       end
 
+      class EncodingFoundException < Exception # :nodoc:
+        attr_reader :encoding
+
+        def initialize(encoding)
+          @encoding = encoding
+          super("encoding found: %s" % encoding)
+        end
+      end
+
+      class EncodingReader # :nodoc:
+        class SAXHandler < Nokogiri::XML::SAX::Document # :nodoc:
+          attr_reader :encoding
+
+          def found(encoding)
+            @encoding = encoding
+            throw :found
+          end
+
+          def not_found(encoding)
+            found nil
+          end
+
+          def start_element(name, attrs = [])
+            case name
+            when /\A(?:div|h1|img|p|br)\z/
+              not_found
+            when 'meta'
+              attr = Hash[attrs]
+              http_equiv = attr['http-equiv'] and
+                http_equiv.match(/\AContent-Type\z/i) and
+                content = attr['content'] and
+                m = content.match(/;\s*charset\s*=\s*([\w-]+)/) and
+                found m[1]
+            end
+          end
+        end
+
+        def self.detect_encoding(chunk)
+          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
+            return Nokogiri.XML(m[1]).encoding
+
+          if Nokogiri.jruby?
+            m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
+              return m[4]
+          end
+
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML::SAX::Parser.new(handler)
+          catch(:found) {
+            parser.parse(chunk)
+          }
+          handler.encoding
+        rescue => e
+          nil
+        end
+
+        def initialize(io)
+          @io = io
+          @firstchunk = nil
+        end
+
+        def read(len)
+          # no support for a call without len
+
+          if !@firstchunk
+            @firstchunk = @io.read(len) or return nil
+
+            # This implementation expects and assumes that the first
+            # call from htmlReadIO() is made with a length long enough
+            # (~1KB) to achieve further encoding detection that
+            # libxml2 does not do.
+            if encoding = EncodingReader.detect_encoding(@firstchunk)
+              raise EncodingFoundException, encoding
+            end
+
+            # This chunk is stored for the next read in retry.
+            return @firstchunk
+          end
+
+          ret = @firstchunk.slice!(0, len)
+          if (len -= ret.length) > 0
+            rest = @io.read(len) and ret << rest
+          end
+          if ret.empty?
+            nil
+          else
+            ret
+          end
+        end
+      end
     end
   end
 end

data/lib/nokogiri/html/sax/parser.rb

@@ -31,7 +31,9 @@ module Nokogiri
         def parse_memory data, encoding = 'UTF-8'
           raise ArgumentError unless data
           return unless data.length > 0
-          ParserContext.memory(data, encoding).parse_with self
+          ctx = ParserContext.memory(data, encoding)
+          yield ctx if block_given?
+          ctx.parse_with self
         end
 
         ###
@@ -40,7 +42,9 @@ module Nokogiri
           raise ArgumentError unless filename
           raise Errno::ENOENT unless File.exists?(filename)
           raise Errno::EISDIR if File.directory?(filename)
-          ParserContext.file(filename, encoding).parse_with self
+          ctx = ParserContext.file(filename, encoding)
+          yield ctx if block_given?
+          ctx.parse_with self
         end
       end
     end