mechanize 0.6.3 → 0.6.4

data/{CHANGELOG → CHANGELOG.txt}

@@ -1,5 +1,22 @@
 = Mechanize CHANGELOG
 
+== 0.6.4
+
+* Adding the "redirect_ok" method to Mechanize to stop mechanize from
+  following redirects.
+	http://rubyforge.org/tracker/index.php?func=detail&aid=6571&group_id=1453&atid=5712
+* Added protected method Mechanize#set_headers so that subclasses can set
+  custom headers.
+  http://rubyforge.org/tracker/?func=detail&aid=7208&group_id=1453&atid=5712
+* Aliased Page#referer to Page#page
+* Fixed a bug when clicking relative urls
+  http://rubyforge.org/pipermail/mechanize-users/2006-November/000035.html
+* Fixing a bug when bad version or max age is passed to Cookie::parse
+  http://rubyforge.org/pipermail/mechanize-users/2006-November/000033.html
+* Fixing a bug with response codes. [#6526]
+* Fixed bug [#6548].  Input type of 'button' was not being added as a button.
+* Fixed bug [#7139]. REXML parser calls hpricot parser by accident
+
 == 0.6.3
 
 * Added keys and values methods to Form

data/Manifest.txt

@@ -0,0 +1,31 @@
+CHANGELOG.txt
+EXAMPLES.txt
+GUIDE.txt
+LICENSE.txt
+Manifest.txt
+NOTES.txt
+README.txt
+Rakefile
+eg/flickr_upload.rb
+eg/mech-dump.rb
+eg/proxy_req.rb
+eg/rubyforge.rb
+eg/spider.rb
+lib/mechanize.rb
+lib/mechanize/cookie.rb
+lib/mechanize/errors.rb
+lib/mechanize/form.rb
+lib/mechanize/form_elements.rb
+lib/mechanize/hpricot.rb
+lib/mechanize/inspect.rb
+lib/mechanize/list.rb
+lib/mechanize/mech_version.rb
+lib/mechanize/net-overrides/net/http.rb
+lib/mechanize/net-overrides/net/https.rb
+lib/mechanize/net-overrides/net/protocol.rb
+lib/mechanize/page.rb
+lib/mechanize/page_elements.rb
+lib/mechanize/parsers/rexml_page.rb
+lib/mechanize/pluggable_parsers.rb
+lib/mechanize/rexml.rb
+setup.rb

data/{NOTES → NOTES.txt}

@@ -1,5 +1,19 @@
 = Mechanize Release Notes
 
+== 0.6.4 (Gwendolyn)
+
+Custom request headers can now be added to Mechanize by subclassing mechanize
+and defining the Mechanize#set_headers method.  For example:
+  class A < WWW::Mechanize
+    def set_headers(u, r, c)
+      super(uri, request, cur_page)
+      request.add_field('Cookie', 'name=Aaron')
+      request
+    end
+  end
+The Mechanize#redirect_ok method has been added to that you can keep mechanize
+from following redirects.
+
 == 0.6.3 (Big Man)
 
 Mechanize 0.6.3 (Big Man) has a few bug fixes and some new features added to

data/{README → README.txt}

@@ -1,5 +1,9 @@
 = WWW::Mechanize
 
+  http://mechanize.rubyforge.org/
+
+== DESCRIPTION
+
 The Mechanize library is used for automating interaction with websites. 
 Mechanize automatically stores and sends cookies, follows redirects,
 can follow links, and submit forms.  Form fields can be populated and
@@ -16,8 +20,8 @@ Note that the files in the net-overrides/ directory are taken from Ruby 1.9.0.
 
 == Examples
 
-If you are just starting, check out the GUIDE[link://files/GUIDE.html].  
-Also, check out the EXAMPLES[link://files/EXAMPLES.html] file.
+If you are just starting, check out the GUIDE[link://files/GUIDE_txt.html].  
+Also, check out the EXAMPLES[link://files/EXAMPLES_txt.html] file.
 
 == Authors
 
@@ -33,5 +37,5 @@ Ruby, my favorite language!
 
 == License
 
-This library is distributed under the GPL.  Please see the LICENSE file.
+This library is distributed under the GPL.  Please see the LICENSE[link://files/LICENSE_txt.html] file.
 

data/Rakefile

@@ -0,0 +1,64 @@
+require 'rubygems'
+require 'hoe'
+
+def announce(msg='')
+  STDERR.puts msg
+end
+
+PKG_BUILD = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
+PKG_NAME = 'mechanize'
+PKG_VERSION = '0.6.4' + PKG_BUILD
+
+Hoe.new(PKG_NAME, PKG_VERSION) do |p|
+  p.rubyforge_name  = PKG_NAME
+  p.author          = 'Aaron Patterson'
+  p.email           = 'aaronp@rubyforge.org'
+  p.summary         = "Mechanize provides automated web-browsing"
+  p.description     = p.paragraphs_of('README.txt', 3).join("\n\n")
+  p.url             = p.paragraphs_of('README.txt', 1).first.strip
+  p.changes         = p.paragraphs_of('CHANGELOG.txt', 0..2).join("\n\n")
+  files = 
+    (p.test_globs + ['test/**/tc_*.rb',
+     "test/htdocs/**/*.{html,jpg}",
+     'test/data/server.*']).map { |x|
+    Dir.glob(x)
+  }.flatten + ['test/data/htpasswd']
+  p.extra_deps      = ['hpricot']
+  p.spec_extras     = { :test_files => files }
+end
+
+task :update_version do
+  announce "Updating Mechanize Version to #{PKG_VERSION}"
+  File.open("lib/mechanize/mech_version.rb", "w") do |f|
+    f.puts "module WWW"
+    f.puts "  class Mechanize"
+    f.puts "    Version = '#{PKG_VERSION}'"
+    f.puts "  end"
+    f.puts "end"
+  end
+  sh 'svn commit -m"updating version" lib/mechanize/mech_version.rb'
+end
+
+desc "Tag code"
+Rake::Task.define_task("tag") do |p|
+  baseurl = "svn+ssh://#{ENV['USER']}@rubyforge.org//var/svn/#{PKG_NAME}"
+  sh "svn cp -m 'tagged #{ PKG_VERSION }' . #{ baseurl }/tags/REL-#{ PKG_VERSION }"
+end
+
+desc "Branch code"
+Rake::Task.define_task("branch") do |p|
+  baseurl = "svn+ssh://#{ENV['USER']}@rubyforge.org/var/svn/#{PKG_NAME}"
+  sh "svn cp -m 'branched #{ PKG_VERSION }' #{baseurl}/trunk #{ baseurl }/branches/RB-#{ PKG_VERSION }"
+end
+
+desc "Update SSL Certificate"
+Rake::Task.define_task('ssl_cert') do |p|
+  sh "openssl genrsa -des3 -out server.key 1024"
+  sh "openssl req -new -key server.key -out server.csr"
+  sh "cp server.key server.key.org"
+  sh "openssl rsa -in server.key.org -out server.key"
+  sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt"
+  sh "cp server.key server.pem"
+  sh "mv server.key server.csr server.crt server.pem test/data/"
+  sh "rm server.key.org"
+end

data/eg/flickr_upload.rb

@@ -0,0 +1,23 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+
+# Get the flickr sign in page
+page  = agent.get('http://flickr.com/signin/flickr/')
+
+# Fill out the login form
+form  = page.forms.name('flickrloginform').first
+form.email = ARGV[0]
+form.password = ARGV[1]
+page  = agent.submit(form)
+
+# Go to the upload page
+page  = agent.click page.links.text('Upload')
+
+# Fill out the form
+form  = page.forms.action('/photos_upload_process.gne').first
+form.file_uploads.name('file1').first.file_name = ARGV[2]
+agent.submit(form)

data/eg/mech-dump.rb

@@ -0,0 +1,7 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+puts agent.get(ARGV[0]).inspect

data/eg/proxy_req.rb

@@ -0,0 +1,9 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+agent.set_proxy('localhost', '8000')
+page = agent.get(ARGV[0])
+puts page.body

data/eg/rubyforge.rb

@@ -0,0 +1,21 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+# This example logs a user in to rubyforge and prints out the body of the
+# page after logging the user in.
+require 'rubygems'
+require 'mechanize'
+
+# Create a new mechanize object
+agent = WWW::Mechanize.new { |a| a.log = Logger.new(STDERR) }
+
+# Load the rubyforge website
+page = agent.get('http://rubyforge.org/')
+page = agent.click page.links.text(/Log In/) # Click the login link
+form = page.forms[1] # Select the first form
+form.form_loginname = ARGV[0]
+form.form_pw        = ARGV[1]
+
+# Submit the form
+page = agent.submit(form, form.buttons.first)
+
+puts page.body # Print out the body

data/eg/spider.rb

@@ -0,0 +1,11 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'rubygems'
+require 'mechanize'
+
+agent = WWW::Mechanize.new
+stack = agent.get(ARGV[0]).links
+while l = stack.pop
+  next unless l.uri.host == agent.history.first.uri.host
+  stack.push(*(agent.click(l).links)) unless agent.visited? l.href
+end

data/lib/mechanize/cookie.rb

@@ -6,7 +6,7 @@ module WWW
   class Mechanize
   # This class is used to represent an HTTP Cookie.
     class Cookie < WEBrick::Cookie
-      def self.parse(uri, str)
+      def self.parse(uri, str, log = nil)
         return str.split(/,(?=[^;,]*=)|,$/).collect { |c|
           cookie_elem = c.split(/;/)
           first_elem = cookie_elem.shift
@@ -28,9 +28,21 @@ module WWW
               rescue
                 Time.now
               end
-            when "max-age" then cookie.max_age = Integer(value)
+            when "max-age" then
+              begin
+                cookie.max_age = Integer(value)
+              rescue
+                log.warn("Couldn't parse max age '#{value}'") if log
+                cookie.max_age = nil
+              end
             when "comment" then cookie.comment = value
-            when "version" then cookie.version = Integer(value)
+            when "version" then
+              begin
+                cookie.version = Integer(value)
+              rescue
+                log.warn("Couldn't parse version '#{value}'") if log
+                cookie.version = nil
+              end
             when "secure"  then cookie.secure = true
             end
           }

data/lib/mechanize/errors.rb

@@ -27,9 +27,7 @@ module WWW
         @response_code = page.code
       end
 
-      def inspect
-        response_code
-      end
+      def inspect; response_code; end
     end
   end
 end

data/lib/mechanize/form.rb

@@ -118,7 +118,7 @@ module WWW
           node.attributes ||= {}
           type = (node.attributes['type'] || 'text').downcase
           name = node.attributes['name']
-          next if type != 'submit' && name.nil?
+          next if name.nil? && !(type == 'submit' || type =='button')
           case type
           when 'text', 'password', 'hidden', 'int'
             @fields << Field.new(node.attributes['name'], node.attributes['value'] || '') 
@@ -130,6 +130,8 @@ module WWW
             @file_uploads << FileUpload.new(node.attributes['name'], nil) 
           when 'submit'
             @buttons << Button.new(node.attributes['name'], node.attributes['value'])
+          when 'button'
+            @buttons << Button.new(node.attributes['name'], node.attributes['value'])
           when 'image'
             @buttons << ImageButton.new(node.attributes['name'], node.attributes['value'])
           end

data/lib/mechanize/form_elements.rb

@@ -22,7 +22,7 @@ module WWW
   # class, set WWW::FileUpload#file_data= to the data of the file you want
   # to upload and WWW::FileUpload#mime_type= to the appropriate mime type
   # of the file.
-  # See the example in EXAMPLES[link://files/EXAMPLES.html]
+  # See the example in EXAMPLES[link://files/EXAMPLES_txt.html]
   class FileUpload < Field
     attr_accessor :name # Field name
     attr_accessor :file_name # File name

data/lib/mechanize/mech_version.rb

@@ -1,5 +1,5 @@
 module WWW
   class Mechanize
-    Version = '0.6.3'
+    Version = '0.6.4'
   end
 end

data/lib/mechanize/net-overrides/net/http.rb

@@ -1826,7 +1826,7 @@ module Net # :nodoc:
       '416' => HTTPRequestedRangeNotSatisfiable,
       '417' => HTTPExpectationFailed,
 
-      '501' => HTTPInternalServerError,
+      '500' => HTTPInternalServerError,
       '501' => HTTPNotImplemented,
       '502' => HTTPBadGateway,
       '503' => HTTPServiceUnavailable,

data/lib/mechanize/page.rb

@@ -1,5 +1,6 @@
 require 'fileutils'
 require 'hpricot'
+require 'forwardable'
 
 module WWW
   class Mechanize
@@ -15,6 +16,8 @@ module WWW
     #  agent.get('http://google.com/').class  #=> WWW::Mechanize::Page
     #
     class Page < File
+      extend Forwardable
+
       attr_reader :root, :title, :watch_for_set
       attr_reader :frames, :iframes, :links, :forms, :meta, :watches
       attr_accessor :mech
@@ -31,7 +34,7 @@ module WWW
 
         # construct parser and feed with HTML
         if body && response
-          @root = Hpricot.parse(body)
+          @root ||= Hpricot.parse(body)
           parse_html
         end
       end
@@ -47,16 +50,10 @@ module WWW
       end
 
       # Search through the page like HPricot
-      def search(*args)
-        @root.search(*args)
-      end
+      def_delegator :@root, :search, :search
+      def_delegator :@root, :/, :/
+      def_delegator :@root, :at, :at
 
-      def at(*args)
-        @root.at(*args)
-      end
-
-      alias :/ :search
-    
       def watch_for_set=(obj)
         @watch_for_set = obj
         parse_html if @body

data/lib/mechanize/page_elements.rb

@@ -15,6 +15,7 @@ module WWW
       attr_reader :attributes
       attr_reader :page
       alias :to_s :text
+      alias :referer :page
     
       def initialize(node, mech, page)
         node.attributes ||= {}

data/lib/mechanize/parsers/rexml_page.rb

@@ -3,15 +3,10 @@ require 'mechanize/rexml'
 
 class WWW::Mechanize::REXMLPage < WWW::Mechanize::Page
   def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
-    super(uri, response, body, code)
-    @watch_for_set = {}
-    @mech          = mech
-    
-    yield self if block_given?
-    
-    raise Mechanize::ContentTypeError.new(response['content-type']) unless
-        content_type() =~ /^text\/html/ 
-    
+    @body           = body
+    @watch_for_set  = {}
+    @mech           = mech
+
     # construct parser and feed with HTML
     parser = HTMLTree::XMLParser.new
     begin
@@ -32,6 +27,9 @@ class WWW::Mechanize::REXMLPage < WWW::Mechanize::Page
     end
 
     @root = parser.document
-    parse_html if body && response
+
+    yield self if block_given?
+
+    super(uri, response, body, code)
   end
 end

data/lib/mechanize.rb

@@ -71,10 +71,13 @@ class Mechanize
   attr_accessor :key
   attr_accessor :cert
   attr_accessor :pass
+  attr_accessor :redirect_ok
 
   attr_reader :history
   attr_reader :pluggable_parser
 
+  alias :follow_redirect? :redirect_ok
+
   def initialize
     # attr_accessors
     @cookie_jar = CookieJar.new
@@ -88,6 +91,7 @@ class Mechanize
     @cert           = nil # OpenSSL Certificate
     @key            = nil # OpenSSL Private Key
     @pass           = nil # OpenSSL Password
+    @redirect_ok    = true # Should we follow redirects?
     
     # attr_readers
     @history        = []
@@ -129,14 +133,14 @@ class Mechanize
   end
 
   # Fetches the URL passed in and returns a page.
-  def get(url, referer=nil)
+  def get(url, referer=nil, &block)
     cur_page = referer || current_page ||
                     Page.new( nil, {'content-type'=>'text/html'})
 
     # fetch the page
     abs_uri = to_absolute_uri(url, cur_page)
     request = fetch_request(abs_uri)
-    page = fetch_page(abs_uri, request, cur_page)
+    page = fetch_page(abs_uri, request, cur_page, &block)
     add_to_history(page)
     page
   end
@@ -150,15 +154,16 @@ class Mechanize
   # Clicks the WWW::Mechanize::Link object passed in and returns the
   # page fetched.
   def click(link)
-    uri = to_absolute_uri(
-      link.attributes['href'] || link.attributes['src'] || link.href
-    )
     referer =
       begin
         link.page
       rescue
         nil
       end
+    uri = to_absolute_uri(
+      link.attributes['href'] || link.attributes['src'] || link.href,
+      referer || current_page()
+    )
     get(uri, referer)
   end
 
@@ -232,6 +237,35 @@ class Mechanize
 
   alias :page :current_page
 
+  protected
+  def set_headers(uri, request, cur_page)
+    request.add_field('Accept-Encoding', 'gzip,identity')
+    request.add_field('Accept-Language', 'en-us,en;q0.5')
+    request.add_field('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
+
+    unless @cookie_jar.empty?(uri)
+      cookies = @cookie_jar.cookies(uri)
+      cookie = cookies.length > 0 ? cookies.join("; ") : nil
+      if log
+        cookies.each do |c|
+          log.debug("using cookie: #{c}")
+        end
+      end
+      request.add_field('Cookie', cookie)
+    end
+
+    # Add Referer header to request
+    unless cur_page.uri.nil?
+      request.add_field('Referer', cur_page.uri.to_s)
+    end
+
+    # Add User-Agent header to request
+    request.add_field('User-Agent', @user_agent) if @user_agent 
+
+    request.basic_auth(@user, @password) if @user || @password
+    request
+  end
+
   private
 
   def to_absolute_uri(url, cur_page=current_page())
@@ -306,28 +340,7 @@ class Mechanize
       end
     end
 
-    request.add_field('Accept-Encoding', 'gzip,identity')
-
-    unless @cookie_jar.empty?(uri)
-      cookies = @cookie_jar.cookies(uri)
-      cookie = cookies.length > 0 ? cookies.join("; ") : nil
-      if log
-        cookies.each do |c|
-          log.debug("using cookie: #{c}")
-        end
-      end
-      request.add_field('Cookie', cookie)
-    end
-
-    # Add Referer header to request
-    unless cur_page.uri.nil?
-      request.add_field('Referer', cur_page.uri.to_s)
-    end
-
-    # Add User-Agent header to request
-    request.add_field('User-Agent', @user_agent) if @user_agent 
-
-    request.basic_auth(@user, @password) if @user || @password
+    request = set_headers(uri, request, cur_page)
 
     # Log specified headers for the request
     if log
@@ -350,13 +363,20 @@ class Mechanize
         end
 
         (response.get_fields('Set-Cookie')||[]).each do |cookie|
-          Cookie::parse(uri, cookie) { |c|
+          Cookie::parse(uri, cookie, log) { |c|
             log.debug("saved cookie: #{c}") if log
             @cookie_jar.add(uri, c)
           }
         end
 
-        response.read_body
+        body = StringIO.new
+        total = 0
+        response.read_body { |part|
+          total += part.length
+          body.write(part)
+          log.debug("Read #{total} bytes") if log
+        }
+        body.rewind
 
         content_type = nil
         unless response['Content-Type'].nil?
@@ -369,12 +389,12 @@ class Mechanize
           case encoding.downcase
           when 'gzip'
             log.debug('gunzip body') if log
-            Zlib::GzipReader.new(StringIO.new(response.body)).read
+            Zlib::GzipReader.new(body).read
           else
             raise 'Unsupported content encoding'
           end
         else
-          response.body
+          body.read
         end
 
         # Find our pluggable parser
@@ -393,17 +413,19 @@ class Mechanize
           page.watch_for_set = @watch_for_set
         end
 
-        case page.code
-        when "200"
-          return page
-        when "301", "302"
+        res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s]
+
+        return page if res_klass <= Net::HTTPSuccess
+
+        if res_klass <= Net::HTTPRedirection
+          return page unless follow_redirect?
           log.info("follow redirect to: #{ response['Location'] }") if log
           abs_uri = to_absolute_uri(response['Location'].to_s, page)
           request = fetch_request(abs_uri)
           return fetch_page(abs_uri, request, page)
-        else
-          raise ResponseCodeError.new(page), "Unhandled response", caller
         end
+
+        raise ResponseCodeError.new(page), "Unhandled response", caller
       }
     }
   end