knu-mechanize 0.9.3.20090623142847

data/lib/www/mechanize/monkey_patch.rb

@@ -0,0 +1,37 @@
+module Net
+  class HTTP
+    alias :old_keep_alive? :keep_alive?
+    def keep_alive?(req, res)
+      return false if /close/i =~ req['connection'].to_s
+      return false if @seems_1_0_server
+      return false if /close/i      =~ res['connection'].to_s
+      return true  if /keep-alive/i =~ res['connection'].to_s
+      return false if /close/i      =~ res['proxy-connection'].to_s
+      return true  if /keep-alive/i =~ res['proxy-connection'].to_s
+      (@curr_http_version == '1.1')
+    end
+  end
+end
+
+# Monkey patch for ruby 1.8.4
+unless RUBY_VERSION > "1.8.4"
+module Net # :nodoc:
+  class HTTPResponse # :nodoc:
+    CODE_TO_OBJ['500'] = HTTPInternalServerError
+  end
+end
+else
+module WWW
+  class Mechanize
+    class Form
+      alias :inspect  :pretty_inspect
+    end
+    class Page
+      alias :inspect  :pretty_inspect
+      class Link
+        alias :inspect  :pretty_inspect
+      end
+    end
+  end
+end
+end

data/lib/www/mechanize/page.rb

@@ -0,0 +1,181 @@
+require 'www/mechanize/page/link'
+require 'www/mechanize/page/meta'
+require 'www/mechanize/page/base'
+require 'www/mechanize/page/frame'
+require 'www/mechanize/headers'
+
+module WWW
+  class Mechanize
+    # = Synopsis
+    # This class encapsulates an HTML page.  If Mechanize finds a content
+    # type of 'text/html', this class will be instantiated and returned.
+    #
+    # == Example
+    #  require 'rubygems'
+    #  require 'mechanize'
+    #
+    #  agent = WWW::Mechanize.new
+    #  agent.get('http://google.com/').class  #=> WWW::Mechanize::Page
+    #
+    class Page < WWW::Mechanize::File
+      extend Forwardable
+
+      attr_accessor :mech
+
+      def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
+        @encoding = nil
+
+        method = response.respond_to?(:each_header) ? :each_header : :each
+        response.send(method) do |header,v|
+          next unless v =~ /charset/i
+          encoding = v.split('=').last.strip
+          @encoding = encoding unless encoding == 'none'
+        end
+
+        # Force the encoding to be 8BIT so we can perform regular expressions.
+        # We'll set it to the detected encoding later
+        body.force_encoding('ASCII-8BIT') if defined?(Encoding) && body
+
+        @encoding ||= Util.detect_charset(body)
+
+        super(uri, response, body, code)
+        @mech           ||= mech
+
+        @encoding = nil if html_body =~ /<meta[^>]*charset[^>]*>/i
+
+        raise Mechanize::ContentTypeError.new(response['content-type']) unless
+           response['content-type'] =~ /^(text\/html)|(application\/xhtml\+xml)/i
+        @parser = @links = @forms = @meta = @bases = @frames = @iframes = nil
+      end
+
+      def title
+        @title ||= if parser && search('title').inner_text.length > 0
+          search('title').inner_text
+        end
+      end
+
+      def encoding=(encoding)
+        @encoding = encoding
+
+        if @parser
+          parser_encoding = @parser.encoding
+          if (parser_encoding && parser_encoding.downcase) != (encoding && encoding.downcase)
+            # lazy reinitialize the parser with the new encoding
+            @parser = nil
+          end
+        end
+
+        encoding
+      end
+
+      def encoding
+        parser.respond_to?(:encoding) ? parser.encoding : nil
+      end
+
+      def parser
+        return @parser if @parser
+
+        if body && response
+          if mech.html_parser == Nokogiri::HTML
+            @parser = mech.html_parser.parse(html_body, nil, @encoding)
+          else
+            @parser = mech.html_parser.parse(html_body)
+          end
+        end
+
+        @parser
+      end
+      alias :root :parser
+
+      # Get the content type
+      def content_type
+        response['content-type']
+      end
+
+      # Search through the page like HPricot
+      def_delegator :parser, :search, :search
+      def_delegator :parser, :/, :/
+      def_delegator :parser, :at, :at
+
+      # Find a form matching +criteria+.
+      # Example:
+      #   page.form_with(:action => '/post/login.php') do |f|
+      #     ...
+      #   end
+      [:form, :link, :base, :frame, :iframe].each do |type|
+        eval(<<-eomethod)
+          def #{type}s_with(criteria)
+            criteria = {:name => criteria} if String === criteria
+            f = #{type}s.find_all do |thing|
+              criteria.all? { |k,v| v === thing.send(k) }
+            end
+            yield f if block_given?
+            f
+          end
+
+          def #{type}_with(criteria)
+            f = #{type}s_with(criteria).first
+            yield f if block_given?
+            f
+          end
+          alias :#{type} :#{type}_with
+        eomethod
+      end
+    
+      def links
+        @links ||= %w{ a area }.map do |tag|
+          search(tag).map do |node|
+            Link.new(node, @mech, self)
+          end
+        end.flatten
+      end
+
+      def forms
+        @forms ||= search('form').map do |html_form|
+          form = Form.new(html_form, @mech, self)
+          form.action ||= @uri.to_s
+          form
+        end
+      end
+
+      def meta
+        @meta ||= search('meta').map do |node|
+          next unless node['http-equiv'] && node['content']
+          (equiv, content) = node['http-equiv'], node['content']
+          if equiv && equiv.downcase == 'refresh'
+            Meta.parse(content, uri) do |delay, href|
+              node['delay'] = delay
+              node['href'] = href
+              Meta.new(node, @mech, self)
+            end
+          end
+        end.compact
+      end
+
+      def bases
+        @bases ||=
+          search('base').map { |node| Base.new(node, @mech, self) }
+      end
+
+      def frames
+        @frames ||=
+          search('frame').map { |node| Frame.new(node, @mech, self) }
+      end
+
+      def iframes
+        @iframes ||= 
+          search('iframe').map { |node| Frame.new(node, @mech, self) }
+      end
+
+      private
+
+      def html_body
+        if body
+          body.length > 0 ? body : '<html></html>'
+        else
+          ''
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/page/base.rb

@@ -0,0 +1,10 @@
+module WWW
+  class Mechanize
+    class Page < WWW::Mechanize::File
+      # This class encapsulates a Base tag.  Mechanize treats base tags just
+      # like 'a' tags.  Base objects will contain links, but most likely will
+      # have no text.
+      class Base < Link; end
+    end
+  end
+end

data/lib/www/mechanize/page/frame.rb

@@ -0,0 +1,22 @@
+module WWW
+  class Mechanize
+    class Page < WWW::Mechanize::File
+      # This class encapsulates a 'frame' tag.  Frame objects can be treated
+      # just like Link objects.  They contain src, the link they refer to,
+      # name, the name of the frame.  'src' and 'name' are aliased to 'href'
+      # and 'text' respectively so that a Frame object can be treated just
+      # like a Link.
+      class Frame < Link
+        alias :src :href
+        alias :name :text
+
+        def initialize(node, mech, referer)
+          super(node, mech, referer)
+          @node = node
+          @text = node['name']
+          @href = node['src']
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/page/link.rb

@@ -0,0 +1,50 @@
+module WWW
+  class Mechanize
+    class Page < WWW::Mechanize::File
+      # This class encapsulates links.  It contains the text and the URI for
+      # 'a' tags parsed out of an HTML page.  If the link contains an image,
+      # the alt text will be used for that image.
+      #
+      # For example, the text for the following links with both be 'Hello World':
+      #
+      # <a href="http://rubyforge.org">Hello World</a>
+      # <a href="http://rubyforge.org"><img src="test.jpg" alt="Hello World"></a>
+      class Link
+        attr_reader :node
+        attr_reader :href
+        attr_reader :text
+        attr_reader :attributes
+        attr_reader :page
+        alias :to_s :text
+        alias :referer :page
+      
+        def initialize(node, mech, page)
+          @node = node
+          @href = node['href'] 
+          @text = node.inner_text
+          @page = page
+          @mech = mech
+          @attributes = node
+
+          # If there is no text, try to find an image and use it's alt text
+          if (@text.nil? || @text.length == 0) && node.search('img').length > 0
+            @text = ''
+            node.search('img').each do |e|
+              @text << ( e['alt'] || '')
+            end
+          end
+
+        end
+
+        def uri
+          @href && URI.parse(@href)
+        end
+
+        # Click on this link
+        def click
+          @mech.click self
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/page/meta.rb

@@ -0,0 +1,51 @@
+module WWW
+  class Mechanize
+    class Page < WWW::Mechanize::File
+      # This class encapsulates a Meta tag.  Mechanize treats meta tags just
+      # like 'a' tags.  Meta objects will contain links, but most likely will
+      # have no text.
+      class Meta < Link
+        
+        # Matches the content attribute of a meta tag.  After the match:
+        #
+        #   $1:: delay
+        #   $3:: url
+        #
+        CONTENT_REGEXP = /^\s*(\d+\.?\d*)(;|;\s*url=\s*['"]?(\S*?)['"]?)?\s*$/i
+        
+        class << self
+          # Parses the delay and url from the content attribute of a meta tag.
+          # Parse requires the uri of the current page to infer a url when no
+          # url is specified.  If a block is given, the parsed delay and url
+          # will be passed to it for further processing.
+          #
+          # Returns nil if the delay and url cannot be parsed.
+          #
+          #   # <meta http-equiv="refresh" content="5;url=http://example.com/" />
+          #   uri = URI.parse('http://current.com/')
+          #
+          #   Meta.parse("5;url=http://example.com/", uri)  # => ['5', 'http://example.com/']
+          #   Meta.parse("5;url=", uri)                     # => ['5', 'http://current.com/']
+          #   Meta.parse("5", uri)                          # => ['5', 'http://current.com/']
+          #   Meta.parse("invalid content", uri)            # => nil
+          #
+          def parse(content, uri)
+            if content && content =~ CONTENT_REGEXP
+              delay, url = $1, $3
+
+              url = case url
+              when nil, "" then uri.to_s
+              when /^http/i then url
+              else "http://#{uri.host}#{url}"
+              end
+
+              block_given? ? yield(delay, url) : [delay, url]
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/pluggable_parsers.rb

@@ -0,0 +1,103 @@
+require 'www/mechanize/file'
+require 'www/mechanize/file_saver'
+require 'www/mechanize/page'
+
+module WWW
+  class Mechanize
+    # = Synopsis
+    # This class is used to register and maintain pluggable parsers for
+    # Mechanize to use.
+    #
+    # A Pluggable Parser is a parser that Mechanize uses for any particular
+    # content type.  Mechanize will ask PluggableParser for the class it
+    # should initialize given any content type.  This class allows users to
+    # register their own pluggable parsers, or modify existing pluggable
+    # parsers.
+    #
+    # PluggableParser returns a WWW::Mechanize::File object for content types
+    # that it does not know how to handle.  WWW::Mechanize::File provides
+    # basic functionality for any content type, so it is a good class to
+    # extend when building your own parsers.
+    # == Example
+    # To create your own parser, just create a class that takes four
+    # parameters in the constructor.  Here is an example of registering
+    # a pluggable parser that handles CSV files:
+    #  class CSVParser < WWW::Mechanize::File
+    #    attr_reader :csv
+    #    def initialize(uri=nil, response=nil, body=nil, code=nil)
+    #      super(uri, response, body, code)
+    #      @csv = CSV.parse(body)
+    #    end
+    #  end
+    #  agent = WWW::Mechanize.new
+    #  agent.pluggable_parser.csv = CSVParser
+    #  agent.get('http://example.com/test.csv')  # => CSVParser
+    # Now any page that returns the content type of 'text/csv' will initialize
+    # a CSVParser and return that object to the caller.
+    #
+    # To register a pluggable parser for a content type that pluggable parser
+    # does not know about, just use the hash syntax:
+    #  agent.pluggable_parser['text/something'] = SomeClass
+    # 
+    # To set the default parser, just use the 'defaut' method:
+    #  agent.pluggable_parser.default = SomeClass
+    # Now all unknown content types will be instances of SomeClass.
+    class PluggableParser
+      CONTENT_TYPES = {
+        :html => 'text/html',
+        :wap  => 'application/vnd.wap.xhtml+xml',
+        :xhtml => 'application/xhtml+xml',
+        :pdf  => 'application/pdf',
+        :csv  => 'text/csv',
+        :xml  => 'text/xml',
+      }
+
+      attr_accessor :default
+
+      def initialize
+        @parsers = { CONTENT_TYPES[:html]   => Page,
+                     CONTENT_TYPES[:xhtml]  => Page,
+                     CONTENT_TYPES[:wap]    => Page,
+        }
+        @default = File
+      end
+
+      def parser(content_type)
+        content_type.nil? ? default : @parsers[content_type] || default
+      end
+
+      def register_parser(content_type, klass)
+        @parsers[content_type] = klass
+      end
+
+      def html=(klass)
+        register_parser(CONTENT_TYPES[:html], klass)
+        register_parser(CONTENT_TYPES[:xhtml], klass)
+      end
+
+      def xhtml=(klass)
+        register_parser(CONTENT_TYPES[:xhtml], klass)
+      end
+
+      def pdf=(klass)
+        register_parser(CONTENT_TYPES[:pdf], klass)
+      end
+
+      def csv=(klass)
+        register_parser(CONTENT_TYPES[:csv], klass)
+      end
+
+      def xml=(klass)
+        register_parser(CONTENT_TYPES[:xml], klass)
+      end
+
+      def [](content_type)
+        @parsers[content_type]
+      end
+
+      def []=(content_type, klass)
+        @parsers[content_type] = klass
+      end
+    end
+  end
+end