tenderlove-mechanize 0.9.3.20090623142847 → 0.9.3.20090911221705

data/lib/{www/mechanize → mechanize}/monkey_patch.rb

@@ -15,13 +15,12 @@ end
 
 # Monkey patch for ruby 1.8.4
 unless RUBY_VERSION > "1.8.4"
-module Net # :nodoc:
-  class HTTPResponse # :nodoc:
-    CODE_TO_OBJ['500'] = HTTPInternalServerError
+  module Net # :nodoc:
+    class HTTPResponse # :nodoc:
+      CODE_TO_OBJ['500'] = HTTPInternalServerError
+    end
   end
-end
 else
-module WWW
   class Mechanize
     class Form
       alias :inspect  :pretty_inspect
@@ -34,4 +33,3 @@ module WWW
     end
   end
 end
-end

data/lib/mechanize/page.rb

@@ -0,0 +1,206 @@
+require 'mechanize/page/link'
+require 'mechanize/page/meta'
+require 'mechanize/page/base'
+require 'mechanize/page/frame'
+require 'mechanize/page/image'
+require 'mechanize/page/label'
+require 'mechanize/headers'
+
+class Mechanize
+  # = Synopsis
+  # This class encapsulates an HTML page.  If Mechanize finds a content
+  # type of 'text/html', this class will be instantiated and returned.
+  #
+  # == Example
+  #  require 'rubygems'
+  #  require 'mechanize'
+  #
+  #  agent = Mechanize.new
+  #  agent.get('http://google.com/').class  #=> Mechanize::Page
+  #
+  class Page < Mechanize::File
+    extend Forwardable
+
+    attr_accessor :mech
+
+    def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
+      @encoding = nil
+
+      method = response.respond_to?(:each_header) ? :each_header : :each
+      response.send(method) do |header,v|
+        next unless v =~ /charset/i
+        encoding = v.split('=').last.strip
+        @encoding = encoding unless encoding == 'none'
+      end
+
+      # Force the encoding to be 8BIT so we can perform regular expressions.
+      # We'll set it to the detected encoding later
+      body.force_encoding('ASCII-8BIT') if defined?(Encoding) && body
+
+      @encoding ||= Util.detect_charset(body)
+
+      super(uri, response, body, code)
+      @mech           ||= mech
+
+      @encoding = nil if html_body =~ /<meta[^>]*charset[^>]*>/i
+
+      raise Mechanize::ContentTypeError.new(response['content-type']) unless
+        response['content-type'] =~ /^(text\/html)|(application\/xhtml\+xml)/i
+      @parser = @links = @forms = @meta = @bases = @frames = @iframes = nil
+    end
+
+    def title
+      @title ||= if parser && search('title').inner_text.length > 0
+                   search('title').inner_text
+                 end
+    end
+
+    def encoding=(encoding)
+      @encoding = encoding
+
+      if @parser
+        parser_encoding = @parser.encoding
+        if (parser_encoding && parser_encoding.downcase) != (encoding && encoding.downcase)
+          # lazy reinitialize the parser with the new encoding
+          @parser = nil
+        end
+      end
+
+      encoding
+    end
+
+    def encoding
+      parser.respond_to?(:encoding) ? parser.encoding : nil
+    end
+
+    def parser
+      return @parser if @parser
+
+      if body && response
+        if mech.html_parser == Nokogiri::HTML
+          @parser = mech.html_parser.parse(html_body, nil, @encoding)
+        else
+          @parser = mech.html_parser.parse(html_body)
+        end
+      end
+
+      @parser
+    end
+    alias :root :parser
+
+    # Get the content type
+    def content_type
+      response['content-type']
+    end
+
+    # Search through the page like HPricot
+    def_delegator :parser, :search, :search
+    def_delegator :parser, :/, :/
+    def_delegator :parser, :at, :at
+
+    # Find a form matching +criteria+.
+    # Example:
+    #   page.form_with(:action => '/post/login.php') do |f|
+    #     ...
+    #   end
+    [:form, :link, :base, :frame, :iframe].each do |type|
+      eval(<<-eomethod)
+          def #{type}s_with(criteria)
+            criteria = {:name => criteria} if String === criteria
+            f = #{type}s.find_all do |thing|
+              criteria.all? { |k,v| v === thing.send(k) }
+            end
+            yield f if block_given?
+            f
+          end
+
+          def #{type}_with(criteria)
+            f = #{type}s_with(criteria).first
+            yield f if block_given?
+            f
+          end
+          alias :#{type} :#{type}_with
+        eomethod
+      end
+
+      def links
+        @links ||= %w{ a area }.map do |tag|
+          search(tag).map do |node|
+            Link.new(node, @mech, self)
+          end
+        end.flatten
+      end
+
+      def forms
+        @forms ||= search('form').map do |html_form|
+          form = Form.new(html_form, @mech, self)
+          form.action ||= @uri.to_s
+          form
+        end
+      end
+
+      def meta
+        @meta ||= search('meta').map do |node|
+          next unless node['http-equiv'] && node['content']
+          (equiv, content) = node['http-equiv'], node['content']
+          if equiv && equiv.downcase == 'refresh'
+            Meta.parse(content, uri) do |delay, href|
+              node['delay'] = delay
+              node['href'] = href
+              Meta.new(node, @mech, self)
+            end
+          end
+        end.compact
+      end
+
+      def bases
+        @bases ||=
+          search('base').map { |node| Base.new(node, @mech, self) }
+      end
+
+      def frames
+        @frames ||=
+          search('frame').map { |node| Frame.new(node, @mech, self) }
+      end
+
+      def iframes
+        @iframes ||=
+          search('iframe').map { |node| Frame.new(node, @mech, self) }
+      end
+
+      def images
+        @images ||=
+          search('img').map { |node| Image.new(node, self) }
+      end
+
+      def image_urls
+        @image_urls ||= images.map(&:url).uniq
+      end
+
+      def labels
+        @labels ||=
+          search('label').map { |node| Label.new(node, self) }
+      end
+
+      def labels_hash
+        unless @labels_hash
+          hash = {}
+          labels.each do |label|
+            hash[label.node['for']] = label if label.for
+          end
+          @labels_hash = hash
+        end
+        return @labels_hash
+      end
+
+      private
+
+      def html_body
+        if body
+          body.length > 0 ? body : '<html></html>'
+        else
+          ''
+        end
+      end
+    end
+  end

data/lib/mechanize/page/base.rb

@@ -0,0 +1,8 @@
+class Mechanize
+  class Page < Mechanize::File
+    # This class encapsulates a Base tag.  Mechanize treats base tags just
+    # like 'a' tags.  Base objects will contain links, but most likely will
+    # have no text.
+    class Base < Link; end
+  end
+end

data/lib/mechanize/page/frame.rb

@@ -0,0 +1,20 @@
+class Mechanize
+  class Page < Mechanize::File
+    # This class encapsulates a 'frame' tag.  Frame objects can be treated
+    # just like Link objects.  They contain src, the link they refer to,
+    # name, the name of the frame.  'src' and 'name' are aliased to 'href'
+    # and 'text' respectively so that a Frame object can be treated just
+    # like a Link.
+    class Frame < Link
+      alias :src :href
+      alias :name :text
+
+      def initialize(node, mech, referer)
+        super(node, mech, referer)
+        @node = node
+        @text = node['name']
+        @href = node['src']
+      end
+    end
+  end
+end

data/lib/mechanize/page/image.rb

@@ -0,0 +1,26 @@
+class Mechanize
+  class Page < Mechanize::File
+    class Image
+      attr_reader :node
+      attr_reader :page
+
+      def initialize(node, page)
+        @node = node
+        @page = page
+      end
+
+      def src
+        @node['src']
+      end
+
+      def url
+        case src
+        when %r{^https?://}
+          src
+        else
+          (page.uri + src).to_s
+        end
+      end
+    end
+  end
+end

data/lib/mechanize/page/label.rb

@@ -0,0 +1,20 @@
+class Mechanize
+  class Page < Mechanize::File
+    class Label
+      attr_reader :node
+      attr_reader :text
+      attr_reader :page
+      alias :to_s :text
+
+      def initialize(node, page)
+        @node = node
+        @text = node.inner_text
+        @page = page
+      end
+
+      def for
+        (id = @node['for']) && page.search("##{id}") || nil
+      end
+    end
+  end
+end

data/lib/mechanize/page/link.rb

@@ -0,0 +1,48 @@
+class Mechanize
+  class Page < Mechanize::File
+    # This class encapsulates links.  It contains the text and the URI for
+    # 'a' tags parsed out of an HTML page.  If the link contains an image,
+    # the alt text will be used for that image.
+    #
+    # For example, the text for the following links with both be 'Hello World':
+    #
+    # <a href="http://rubyforge.org">Hello World</a>
+    # <a href="http://rubyforge.org"><img src="test.jpg" alt="Hello World"></a>
+    class Link
+      attr_reader :node
+      attr_reader :href
+      attr_reader :text
+      attr_reader :attributes
+      attr_reader :page
+      alias :to_s :text
+      alias :referer :page
+
+      def initialize(node, mech, page)
+        @node = node
+        @href = node['href']
+        @text = node.inner_text
+        @page = page
+        @mech = mech
+        @attributes = node
+
+        # If there is no text, try to find an image and use it's alt text
+        if (@text.nil? || @text.length == 0) && node.search('img').length > 0
+          @text = ''
+          node.search('img').each do |e|
+            @text << ( e['alt'] || '')
+          end
+        end
+
+      end
+
+      def uri
+        @href && URI.parse(@href)
+      end
+
+      # Click on this link
+      def click
+        @mech.click self
+      end
+    end
+  end
+end

data/lib/mechanize/page/meta.rb

@@ -0,0 +1,50 @@
+class Mechanize
+  class Page < Mechanize::File
+    # This class encapsulates a Meta tag.  Mechanize treats meta tags just
+    # like 'a' tags.  Meta objects will contain links, but most likely will
+    # have no text.
+    class Meta < Link
+
+      # Matches the content attribute of a meta tag.  After the match:
+      #
+      #   $1:: delay
+      #   $3:: url
+      #
+      CONTENT_REGEXP = /^\s*(\d+\.?\d*)(;|;\s*url=\s*['"]?(\S*?)['"]?)?\s*$/i
+
+      class << self
+        # Parses the delay and url from the content attribute of a meta tag.
+        # Parse requires the uri of the current page to infer a url when no
+        # url is specified.  If a block is given, the parsed delay and url
+        # will be passed to it for further processing.
+        #
+        # Returns nil if the delay and url cannot be parsed.
+        #
+        #   # <meta http-equiv="refresh" content="5;url=http://example.com/" />
+        #   uri = URI.parse('http://current.com/')
+        #
+        #   Meta.parse("5;url=http://example.com/", uri)  # => ['5', 'http://example.com/']
+        #   Meta.parse("5;url=", uri)                     # => ['5', 'http://current.com/']
+        #   Meta.parse("5", uri)                          # => ['5', 'http://current.com/']
+        #   Meta.parse("invalid content", uri)            # => nil
+        #
+        def parse(content, uri)
+          if content && content =~ CONTENT_REGEXP
+            delay, url = $1, $3
+
+            url = case url
+                  when nil, "" then uri.to_s
+                  when /^http/i then url
+                  else "http://#{uri.host}#{url}"
+                  end
+
+            block_given? ? yield(delay, url) : [delay, url]
+          else
+            nil
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/mechanize/pluggable_parsers.rb

@@ -0,0 +1,101 @@
+require 'mechanize/file'
+require 'mechanize/file_saver'
+require 'mechanize/page'
+
+class Mechanize
+  # = Synopsis
+  # This class is used to register and maintain pluggable parsers for
+  # Mechanize to use.
+  #
+  # A Pluggable Parser is a parser that Mechanize uses for any particular
+  # content type.  Mechanize will ask PluggableParser for the class it
+  # should initialize given any content type.  This class allows users to
+  # register their own pluggable parsers, or modify existing pluggable
+  # parsers.
+  #
+  # PluggableParser returns a Mechanize::File object for content types
+  # that it does not know how to handle.  Mechanize::File provides
+  # basic functionality for any content type, so it is a good class to
+  # extend when building your own parsers.
+  # == Example
+  # To create your own parser, just create a class that takes four
+  # parameters in the constructor.  Here is an example of registering
+  # a pluggable parser that handles CSV files:
+  #  class CSVParser < Mechanize::File
+  #    attr_reader :csv
+  #    def initialize(uri=nil, response=nil, body=nil, code=nil)
+  #      super(uri, response, body, code)
+  #      @csv = CSV.parse(body)
+  #    end
+  #  end
+  #  agent = Mechanize.new
+  #  agent.pluggable_parser.csv = CSVParser
+  #  agent.get('http://example.com/test.csv')  # => CSVParser
+  # Now any page that returns the content type of 'text/csv' will initialize
+  # a CSVParser and return that object to the caller.
+  #
+  # To register a pluggable parser for a content type that pluggable parser
+  # does not know about, just use the hash syntax:
+  #  agent.pluggable_parser['text/something'] = SomeClass
+  #
+  # To set the default parser, just use the 'defaut' method:
+  #  agent.pluggable_parser.default = SomeClass
+  # Now all unknown content types will be instances of SomeClass.
+  class PluggableParser
+    CONTENT_TYPES = {
+      :html => 'text/html',
+      :wap  => 'application/vnd.wap.xhtml+xml',
+      :xhtml => 'application/xhtml+xml',
+      :pdf  => 'application/pdf',
+      :csv  => 'text/csv',
+      :xml  => 'text/xml',
+    }
+
+    attr_accessor :default
+
+    def initialize
+      @parsers = { CONTENT_TYPES[:html]   => Page,
+        CONTENT_TYPES[:xhtml]  => Page,
+        CONTENT_TYPES[:wap]    => Page,
+      }
+      @default = File
+    end
+
+    def parser(content_type)
+      content_type.nil? ? default : @parsers[content_type] || default
+    end
+
+    def register_parser(content_type, klass)
+      @parsers[content_type] = klass
+    end
+
+    def html=(klass)
+      register_parser(CONTENT_TYPES[:html], klass)
+      register_parser(CONTENT_TYPES[:xhtml], klass)
+    end
+
+    def xhtml=(klass)
+      register_parser(CONTENT_TYPES[:xhtml], klass)
+    end
+
+    def pdf=(klass)
+      register_parser(CONTENT_TYPES[:pdf], klass)
+    end
+
+    def csv=(klass)
+      register_parser(CONTENT_TYPES[:csv], klass)
+    end
+
+    def xml=(klass)
+      register_parser(CONTENT_TYPES[:xml], klass)
+    end
+
+    def [](content_type)
+      @parsers[content_type]
+    end
+
+    def []=(content_type, klass)
+      @parsers[content_type] = klass
+    end
+  end
+end