tenderlove-mechanize 0.9.3.20090617085936

data/README.rdoc

@@ -0,0 +1,60 @@
+= WWW::Mechanize
+
+* http://mechanize.rubyforge.org/
+* http://github.com/tenderlove/mechanize/tree/master
+
+== DESCRIPTION
+
+The Mechanize library is used for automating interaction with websites. 
+Mechanize automatically stores and sends cookies, follows redirects,
+can follow links, and submit forms.  Form fields can be populated and
+submitted.  Mechanize also keeps track of the sites that you have visited as
+a history.
+
+== Dependencies
+
+* ruby 1.8.6
+* nokogiri[http://nokogiri.rubyforge.org]
+
+== SUPPORT:
+
+The mechanize mailing list is available here:
+
+  * http://rubyforge.org/mailman/listinfo/mechanize-users
+
+The bug tracker is available here:
+
+  * http://rubyforge.org/tracker/?atid=5709&group_id=1453
+
+== Examples
+
+If you are just starting, check out the GUIDE.  
+Also, check out the EXAMPLES file.
+
+== Authors
+
+Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de) 
+
+Copyright (c) 2006-2009:
+
+* {Aaron Patterson}[http://tenderlovemaking.com] (aaronp@rubyforge.org)
+* {Mike Dalessio}[http://mike.daless.io] (mike@csa.net)
+
+This library comes with a shameless plug for employing me
+(Aaron[http://tenderlovemaking.com/]) programming
+Ruby, my favorite language!
+
+== Acknowledgments
+
+This library was heavily influenced by its namesake in the perl world.  A big
+thanks goes to Andy Lester (andy@petdance.com), the author of the original
+perl Mechanize which is available here[http://search.cpan.org/~petdance/WWW-Mechanize-1.20/].  Ruby Mechanize would not be around without you!
+
+Thank you to Michael Neumann for starting the Ruby version.  Thanks to everyone
+who's helped out in various ways.  Finally, thank you to the people using this
+library!
+  
+== License
+
+This library is distributed under the GPL.  Please see the LICENSE file.
+

data/Rakefile

@@ -0,0 +1,43 @@
+require 'rubygems'
+require 'hoe'
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
+require 'mechanize'
+
+HOE = Hoe.new('mechanize', WWW::Mechanize::VERSION) do |p|
+  p.developer('Aaron Patterson','aaronp@rubyforge.org')
+  p.developer('Mike Dalessio','mike.dalessio@gmail.com')
+  p.readme_file     = 'README.rdoc'
+  p.history_file    = 'CHANGELOG.rdoc'
+  p.extra_rdoc_files  = FileList['*.rdoc']
+  p.summary         = "Mechanize provides automated web-browsing"
+  p.extra_deps      = [['nokogiri', '>= 1.2.1']]
+end
+
+desc "Update SSL Certificate"
+task('ssl_cert') do |p|
+  sh "openssl genrsa -des3 -out server.key 1024"
+  sh "openssl req -new -key server.key -out server.csr"
+  sh "cp server.key server.key.org"
+  sh "openssl rsa -in server.key.org -out server.key"
+  sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt"
+  sh "cp server.key server.pem"
+  sh "mv server.key server.csr server.crt server.pem test/data/"
+  sh "rm server.key.org"
+end
+
+namespace :gem do
+  desc 'Generate a gem spec'
+  task :spec do
+    File.open("#{HOE.name}.gemspec", 'w') do |f|
+      HOE.spec.version = "#{HOE.version}.#{Time.now.strftime("%Y%m%d%H%M%S")}"
+      f.write(HOE.spec.to_ruby)
+    end
+  end
+end
+
+desc "Run code-coverage analysis"
+task :coverage do
+  rm_rf "coverage"
+  sh "rcov -x Library -I lib:test #{Dir[*HOE.test_globs].join(' ')}"
+end

data/examples/flickr_upload.rb

@@ -0,0 +1,23 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+
+# Get the flickr sign in page
+page  = agent.get('http://flickr.com/signin/flickr/')
+
+# Fill out the login form
+form  = page.forms.name('flickrloginform').first
+form.email = ARGV[0]
+form.password = ARGV[1]
+page  = agent.submit(form)
+
+# Go to the upload page
+page  = agent.click page.links.text('Upload')
+
+# Fill out the form
+form  = page.forms.action('/photos_upload_process.gne').first
+form.file_uploads.name('file1').first.file_name = ARGV[2]
+agent.submit(form)

data/examples/mech-dump.rb

@@ -0,0 +1,7 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+puts agent.get(ARGV[0]).inspect

data/examples/proxy_req.rb

@@ -0,0 +1,9 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+agent.set_proxy('localhost', '8000')
+page = agent.get(ARGV[0])
+puts page.body

data/examples/rubyforge.rb

@@ -0,0 +1,21 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+# This example logs a user in to rubyforge and prints out the body of the
+# page after logging the user in.
+require 'rubygems'
+require 'mechanize'
+
+# Create a new mechanize object
+agent = WWW::Mechanize.new { |a| a.log = Logger.new(STDERR) }
+
+# Load the rubyforge website
+page = agent.get('http://rubyforge.org/')
+page = agent.click page.links.text(/Log In/) # Click the login link
+form = page.forms[1] # Select the first form
+form.form_loginname = ARGV[0]
+form.form_pw        = ARGV[1]
+
+# Submit the form
+page = agent.submit(form, form.buttons.first)
+
+puts page.body # Print out the body

data/examples/spider.rb

@@ -0,0 +1,11 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+stack = agent.get(ARGV[0]).links
+while l = stack.pop
+  next unless l.uri.host == agent.history.first.uri.host
+  stack.push(*(agent.click(l).links)) unless agent.visited? l.href
+end

data/lib/mechanize.rb

@@ -0,0 +1,7 @@
+# Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de) 
+# Copyright (c) 2007 by Aaron Patterson (aaronp@rubyforge.org) 
+#
+# Please see the LICENSE file for licensing.
+
+
+require 'www/mechanize'

data/lib/www/mechanize/chain/auth_headers.rb

@@ -0,0 +1,80 @@
+module WWW
+  class Mechanize
+    class Chain
+      class AuthHeaders
+        include WWW::Handler
+
+        @@nonce_count = Hash.new(0)
+        CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535)))
+
+        def initialize(auth_hash, user, password, digest)
+          @auth_hash = auth_hash
+          @user      = user
+          @password  = password
+          @digest    = digest
+        end
+
+        def handle(ctx, params)
+          uri     = params[:uri]
+          request = params[:request]
+
+          if( @auth_hash[uri.host] )
+            case @auth_hash[uri.host]
+            when :basic
+              request.basic_auth(@user, @password)
+            when :iis_digest
+                digest_response = self.gen_auth_header(uri,request, @digest, true)
+                request['Authorization'] = digest_response
+            when :digest
+              if @digest
+                digest_response = self.gen_auth_header(uri,request, @digest)
+                request['Authorization'] = digest_response
+              end
+            end
+          end
+          super
+        end
+
+        def gen_auth_header(uri, request, auth_header, is_IIS = false)
+          auth_header =~ /^(\w+) (.*)/
+
+          params = {}
+          $2.gsub(/(\w+)=("[^"]*"|[^,]*)/) {
+            params[$1] = $2.gsub(/^"/, '').gsub(/"$/, '')
+          }
+  
+          @@nonce_count[params['nonce']] += 1
+
+          a_1 = "#{@user}:#{params['realm']}:#{@password}"
+          a_2 = "#{request.method}:#{uri.path}"
+          request_digest = ''
+          request_digest << Digest::MD5.hexdigest(a_1)
+          request_digest << ':' << params['nonce']
+          request_digest << ':' << ('%08x' % @@nonce_count[params['nonce']])
+          request_digest << ':' << CNONCE
+          request_digest << ':' << params['qop']
+          request_digest << ':' << Digest::MD5.hexdigest(a_2)
+  
+          header = ''
+          header << "Digest username=\"#{@user}\", "
+          if is_IIS then
+            header << "qop=\"#{params['qop']}\", "
+          else
+            header << "qop=#{params['qop']}, "
+          end
+          header << "uri=\"#{uri.path}\", "
+          header << %w{ algorithm opaque nonce realm }.map { |field|
+            next unless params[field]
+            "#{field}=\"#{params[field]}\""
+          }.compact.join(', ')
+
+          header << ", nc=#{'%08x' % @@nonce_count[params['nonce']]}, "
+          header << "cnonce=\"#{CNONCE}\", "
+          header << "response=\"#{Digest::MD5.hexdigest(request_digest)}\""
+  
+          return header
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/body_decoding_handler.rb

@@ -0,0 +1,48 @@
+module WWW
+  class Mechanize
+    class Chain
+      class BodyDecodingHandler
+        include WWW::Handler
+
+        def handle(ctx, options)
+          body = options[:response_body]
+          response = options[:response]
+
+          options[:response_body] = 
+            if encoding = response['Content-Encoding']
+              case encoding.downcase
+              when 'gzip'
+                Mechanize.log.debug('gunzip body') if Mechanize.log
+                if response['Content-Length'].to_i > 0 || body.length > 0
+                  begin
+                    Zlib::GzipReader.new(body).read
+                  rescue Zlib::BufError, Zlib::GzipFile::Error
+                    if Mechanize.log
+                      Mechanize.log.error('Caught a Zlib::BufError')
+                    end
+                    body.rewind
+                    body.read(10)
+                    Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.read)
+                  rescue Zlib::DataError
+                    if Mechanize.log
+                      Mechanize.log.error("Caught a Zlib::DataError, unable to decode page: #{$!.to_s}")
+                    end
+                    ''
+                  end
+                else
+                  ''
+                end
+              when 'x-gzip'
+                body.read
+              else
+                raise 'Unsupported content encoding'
+              end
+            else
+              body.read
+            end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/connection_resolver.rb

@@ -0,0 +1,78 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ConnectionResolver
+        include WWW::Handler
+
+        def initialize( connection_cache,
+                        keep_alive,
+                        proxy_addr,
+                        proxy_port,
+                        proxy_user,
+                        proxy_pass )
+
+          @connection_cache = connection_cache
+          @keep_alive = keep_alive
+          @proxy_addr = proxy_addr
+          @proxy_port = proxy_port
+          @proxy_user = proxy_user
+          @proxy_pass = proxy_pass
+        end
+
+        def handle(ctx, params)
+          uri = params[:uri]
+          http_obj = nil
+
+          case uri.scheme.downcase
+          when 'http', 'https'
+            cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
+              :connection         => nil,
+              :keep_alive_options => {},
+            })
+            http_obj = cache_obj[:connection]
+            if http_obj.nil? || ! http_obj.started?
+              http_obj = cache_obj[:connection] =
+                  Net::HTTP.new( uri.host,
+                          uri.port,
+                          @proxy_addr,
+                          @proxy_port,
+                          @proxy_user,
+                          @proxy_pass
+                        )
+              cache_obj[:keep_alive_options] = {}
+            end
+
+            # If we're keeping connections alive and the last request time is too
+            # long ago, stop the connection.  Or, if the max requests left is 1,
+            # reset the connection.
+            if @keep_alive && http_obj.started?
+              opts = cache_obj[:keep_alive_options]
+              if((opts[:timeout] &&
+                 Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) ||
+                  opts[:max] && opts[:max].to_i == 1)
+  
+                Mechanize.log.debug('Finishing stale connection') if Mechanize.log
+                http_obj.finish
+  
+              end
+            end
+
+            cache_obj[:last_request_time] = Time.now.to_i
+          when 'file'
+            http_obj = Object.new
+            class << http_obj
+              def started?; true; end
+              def request(request, *args, &block)
+                response = FileResponse.new(request.uri.path)
+                yield response
+              end
+            end
+          end
+
+          params[:connection] = http_obj
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/custom_headers.rb

@@ -0,0 +1,23 @@
+module WWW
+  class Mechanize
+    class Chain
+      class CustomHeaders
+        include WWW::Handler
+
+        def handle(ctx, params)
+          request = params[:request]
+          params[:headers].each do |k,v|
+            case k
+            when :etag then request["ETag"] = v
+            when :if_modified_since then request["If-Modified-Since"] = v
+            else
+              raise ArgumentError.new("unknown header symbol #{k}") if k.is_a? Symbol
+              request[k] = v
+            end
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/handler.rb

@@ -0,0 +1,9 @@
+module WWW
+  module Handler
+    attr_accessor :chain
+
+    def handle(ctx, request)
+      chain.pass(self, request)
+    end
+  end
+end

data/lib/www/mechanize/chain/header_resolver.rb

@@ -0,0 +1,53 @@
+module WWW
+  class Mechanize
+    class Chain
+      class HeaderResolver
+        include WWW::Handler
+        def initialize(keep_alive, keep_alive_time, cookie_jar, user_agent, headers)
+          @keep_alive = keep_alive
+          @keep_alive_time = keep_alive_time
+          @cookie_jar = cookie_jar
+          @user_agent = user_agent
+          @headers = headers
+        end
+
+        def handle(ctx, params)
+          uri = params[:uri]
+          referer = params[:referer]
+          request = params[:request]
+
+          if @keep_alive
+            request['Connection'] = 'keep-alive'
+            request['Keep-Alive'] = @keep_alive_time.to_s
+          else
+            request['Connection'] = 'close'
+          end
+          request['Accept-Encoding'] = 'gzip,identity'
+          request['Accept-Language'] = 'en-us,en;q=0.5'
+          host = "#{uri.host}#{[80, 443].include?(uri.port.to_i) ? '' : ':' + uri.port.to_s}"
+          request['Host'] = host
+          request['Accept-Charset'] = 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'
+  
+          unless @cookie_jar.empty?(uri)
+            cookies = @cookie_jar.cookies(uri)
+            cookie = cookies.length > 0 ? cookies.join("; ") : nil
+            request.add_field('Cookie', cookie)
+          end
+  
+          # Add Referer header to request
+          if referer && referer.uri
+            request['Referer'] = referer.uri.to_s
+          end
+  
+          # Add User-Agent header to request
+          request['User-Agent'] = @user_agent if @user_agent 
+
+          @headers.each do |k,v|
+            request[k] = v
+          end if request
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/parameter_resolver.rb

@@ -0,0 +1,24 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ParameterResolver
+        include WWW::Handler
+
+        def handle(ctx, params)
+          parameters  = params[:params]
+          uri         = params[:uri]
+          case params[:verb]
+          when :head, :get, :delete, :trace
+            if parameters.length > 0
+              uri.query ||= ''
+              uri.query << '&' if uri.query.length > 0
+              uri.query << Util.build_query_string(parameters)
+            end
+            params[:params] = []
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/pre_connect_hook.rb

@@ -0,0 +1,22 @@
+module WWW
+  class Mechanize
+    class Chain
+      class PreConnectHook
+        include WWW::Handler
+
+        attr_accessor :hooks
+        def initialize
+          @hooks = []
+        end
+
+        def handle(ctx, params)
+          @hooks.each { |hook| hook.call(params) }
+          super
+        end
+      end
+
+      class PostConnectHook < PreConnectHook
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/request_resolver.rb

@@ -0,0 +1,32 @@
+module WWW
+  class Mechanize
+    class Chain
+      class RequestResolver
+        include WWW::Handler
+
+        def handle(ctx, params)
+          uri = params[:uri]
+          if %w{ http https }.include?(uri.scheme.downcase)
+            klass = Net::HTTP.const_get(params[:verb].to_s.capitalize)
+            params[:request] ||= klass.new(uri.request_uri)
+          end
+
+          if %w{ file }.include?(uri.scheme.downcase)
+            o = Struct.new(:uri).new(uri)
+            class << o
+              def add_field(*args); end
+              alias :[]= :add_field
+              def path
+                uri.path
+              end
+              def each_header; end
+            end
+            params[:request] ||= o
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/response_body_parser.rb

@@ -0,0 +1,40 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ResponseBodyParser
+        include WWW::Handler
+
+        def initialize(pluggable_parser, watch_for_set)
+          @pluggable_parser = pluggable_parser
+          @watch_for_set = watch_for_set
+        end
+
+        def handle(ctx, params)
+          response = params[:response]
+          response_body = params[:response_body]
+          uri = params[:uri]
+
+          content_type = nil
+          unless response['Content-Type'].nil?
+            data = response['Content-Type'].match(/^([^;]*)/)
+            content_type = data[1].downcase.split(',')[0] unless data.nil?
+          end
+
+          # Find our pluggable parser
+          params[:page] = @pluggable_parser.parser(content_type).new(
+            uri,
+            response,
+            response_body,
+            response.code
+          ) { |parser|
+            parser.mech = params[:agent] if parser.respond_to? :mech=
+            if parser.respond_to?(:watch_for_set=) && @watch_for_set
+              parser.watch_for_set = @watch_for_set
+            end
+          }
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/response_header_handler.rb

@@ -0,0 +1,50 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ResponseHeaderHandler
+        include WWW::Handler
+
+        def initialize(cookie_jar, connection_cache)
+          @cookie_jar = cookie_jar
+          @connection_cache = connection_cache
+        end
+
+        def handle(ctx, params)
+          response = params[:response]
+          uri = params[:uri]
+          page = params[:page]
+          cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
+            :connection         => nil,
+            :keep_alive_options => {},
+          })
+
+          # If the server sends back keep alive options, save them
+          if keep_alive_info = response['keep-alive']
+            keep_alive_info.split(/,\s*/).each do |option|
+              k, v = option.split(/=/)
+              cache_obj[:keep_alive_options] ||= {}
+              cache_obj[:keep_alive_options][k.intern] = v
+            end
+          end
+
+          if page.is_a?(Page) && page.body =~ /Set-Cookie/n
+            page.search('//meta[@http-equiv="Set-Cookie"]').each do |meta|
+              Cookie::parse(uri, meta['content']) { |c|
+                Mechanize.log.debug("saved cookie: #{c}") if Mechanize.log
+                @cookie_jar.add(uri, c)
+              }
+            end
+          end
+
+          (response.get_fields('Set-Cookie')||[]).each do |cookie|
+            Cookie::parse(uri, cookie) { |c|
+              Mechanize.log.debug("saved cookie: #{c}") if Mechanize.log
+              @cookie_jar.add(uri, c)
+            }
+          end
+          super
+        end
+      end
+    end
+  end
+end