mechanize 0.6.4 → 0.6.5

data/CHANGELOG.txt

@@ -1,5 +1,20 @@
 = Mechanize CHANGELOG
 
+== 0.6.5
+
+* Copying headers to a hash to prevent memory leaks
+* Speeding up page parsing
+* Aliased fields to elements
+* Adding If-Modified-Since header
+* Added delete_field! to form.  Thanks to Sava Chankov
+* Updated uri escaping to support high order characters.  Thanks to Henrik Nyh.
+* Better handling relative URIs.  Thanks to Henrik Nyh
+* Now handles pipes in URLs
+  http://rubyforge.org/tracker/?func=detail&aid=7140&group_id=1453&atid=5709
+* Now escaping html entities in form fields.
+  http://rubyforge.org/tracker/?func=detail&aid=7563&group_id=1453&atid=5709
+* Added MSIE 7.0 user agent string
+
 == 0.6.4
 
 * Adding the "redirect_ok" method to Mechanize to stop mechanize from

data/Manifest.txt

@@ -19,7 +19,6 @@ lib/mechanize/form_elements.rb
 lib/mechanize/hpricot.rb
 lib/mechanize/inspect.rb
 lib/mechanize/list.rb
-lib/mechanize/mech_version.rb
 lib/mechanize/net-overrides/net/http.rb
 lib/mechanize/net-overrides/net/https.rb
 lib/mechanize/net-overrides/net/protocol.rb
@@ -29,3 +28,87 @@ lib/mechanize/parsers/rexml_page.rb
 lib/mechanize/pluggable_parsers.rb
 lib/mechanize/rexml.rb
 setup.rb
+test/data/htpasswd
+test/data/server.crt
+test/data/server.csr
+test/data/server.key
+test/data/server.pem
+test/htdocs/alt_text.html
+test/htdocs/bad_form_test.html
+test/htdocs/button.jpg
+test/htdocs/empty_form.html
+test/htdocs/file_upload.html
+test/htdocs/find_link.html
+test/htdocs/form_multi_select.html
+test/htdocs/form_multival.html
+test/htdocs/form_no_action.html
+test/htdocs/form_no_input_name.html
+test/htdocs/form_select.html
+test/htdocs/form_select_all.html
+test/htdocs/form_select_none.html
+test/htdocs/form_select_noopts.html
+test/htdocs/form_set_fields.html
+test/htdocs/form_test.html
+test/htdocs/frame_test.html
+test/htdocs/google.html
+test/htdocs/iframe_test.html
+test/htdocs/index.html
+test/htdocs/link with space.html
+test/htdocs/no_title_test.html
+test/htdocs/relative/tc_relative_links.html
+test/htdocs/tc_bad_links.html
+test/htdocs/tc_checkboxes.html
+test/htdocs/tc_encoded_links.html
+test/htdocs/tc_form_action.html
+test/htdocs/tc_links.html
+test/htdocs/tc_no_attributes.html
+test/htdocs/tc_pretty_print.html
+test/htdocs/tc_radiobuttons.html
+test/htdocs/tc_referer.html
+test/htdocs/tc_relative_links.html
+test/htdocs/tc_textarea.html
+test/ssl_server.rb
+test/tc_authenticate.rb
+test/tc_bad_links.rb
+test/tc_checkboxes.rb
+test/tc_cookie_class.rb
+test/tc_cookie_jar.rb
+test/tc_cookies.rb
+test/tc_encoded_links.rb
+test/tc_errors.rb
+test/tc_form_action.rb
+test/tc_form_as_hash.rb
+test/tc_form_button.rb
+test/tc_form_no_inputname.rb
+test/tc_forms.rb
+test/tc_frames.rb
+test/tc_gzipping.rb
+test/tc_html_unscape_forms.rb
+test/tc_if_modified_since.rb
+test/tc_links.rb
+test/tc_mech.rb
+test/tc_multi_select.rb
+test/tc_no_attributes.rb
+test/tc_page.rb
+test/tc_pluggable_parser.rb
+test/tc_post_form.rb
+test/tc_pretty_print.rb
+test/tc_proxy.rb
+test/tc_radiobutton.rb
+test/tc_referer.rb
+test/tc_relative_links.rb
+test/tc_response_code.rb
+test/tc_save_file.rb
+test/tc_select.rb
+test/tc_select_all.rb
+test/tc_select_none.rb
+test/tc_select_noopts.rb
+test/tc_set_fields.rb
+test/tc_ssl_server.rb
+test/tc_subclass.rb
+test/tc_textarea.rb
+test/tc_upload.rb
+test/tc_watches.rb
+test/test_all.rb
+test/test_includes.rb
+test/test_servlets.rb

data/README.txt

@@ -29,12 +29,22 @@ Original Code:
 Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de) 
 
 New Code:
-Copyright (c) 2006 by Aaron Patterson (aaronp@rubyforge.org) 
+Copyright (c) 2007 by Aaron Patterson (aaronp@rubyforge.org) 
 
 This library comes with a shameless plug for employing me
 (Aaron[http://tenderlovemaking.com/]) programming
 Ruby, my favorite language!
 
+== Acknowledgments
+
+This library was heavily influenced by its namesake in the perl world.  A big
+thanks goes to Andy Lester (andy@petdance.com), the author of the original
+perl Mechanize which is available here[http://search.cpan.org/~petdance/WWW-Mechanize-1.20/].  Ruby Mechanize would not be around without you!
+
+Thank you to Michael Neumann for starting the Ruby version.  Thanks to everyone
+who's helped out in various ways.  Finally, thank you to the people using this
+library!
+  
 == License
 
 This library is distributed under the GPL.  Please see the LICENSE[link://files/LICENSE_txt.html] file.

data/Rakefile

@@ -1,64 +1,60 @@
 require 'rubygems'
 require 'hoe'
 
-def announce(msg='')
-  STDERR.puts msg
-end
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
+require 'mechanize'
+
+class MechHoe < Hoe
+  def define_tasks
+    super
+
+    desc "Tag code"
+    task :tag do |p|
+      abort "Must supply VERSION=x.y.z" unless ENV['VERSION']
+      v = ENV['VERSION'].gsub(/\./, '_')
+
+      rf = RubyForge.new
+      user = rf.userconfig['username']
 
-PKG_BUILD = ENV['PKG_BUILD'] ? '.' + ENV['PKG_BUILD'] : ''
-PKG_NAME = 'mechanize'
-PKG_VERSION = '0.6.4' + PKG_BUILD
+      baseurl = "svn+ssh://#{user}@rubyforge.org//var/svn/#{name}"
+      sh "svn cp -m 'tagged REL-#{v}' . #{ baseurl }/tags/REL-#{ v }"
+    end
+
+    desc "Branch code"
+    Rake::Task.define_task("branch") do |p|
+      abort "Must supply VERSION=x.y.z" unless ENV['VERSION']
+      v = ENV['VERSION'].split(/\./)[0..1].join('_')
+
+      rf = RubyForge.new
+      user = rf.userconfig['username']
+
+      baseurl = "svn+ssh://#{user}@rubyforge.org/var/svn/#{name}"
+      sh "svn cp -m'branched #{v}' #{baseurl}/trunk #{baseurl}/branches/RB-#{v}"
+    end
+    
+    desc "Update SSL Certificate"
+    Rake::Task.define_task('ssl_cert') do |p|
+      sh "openssl genrsa -des3 -out server.key 1024"
+      sh "openssl req -new -key server.key -out server.csr"
+      sh "cp server.key server.key.org"
+      sh "openssl rsa -in server.key.org -out server.key"
+      sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt"
+      sh "cp server.key server.pem"
+      sh "mv server.key server.csr server.crt server.pem test/data/"
+      sh "rm server.key.org"
+    end
+  end
+end
 
-Hoe.new(PKG_NAME, PKG_VERSION) do |p|
-  p.rubyforge_name  = PKG_NAME
+MechHoe.new('mechanize', WWW::Mechanize::VERSION) do |p|
+  p.rubyforge_name  = 'mechanize'
   p.author          = 'Aaron Patterson'
   p.email           = 'aaronp@rubyforge.org'
   p.summary         = "Mechanize provides automated web-browsing"
   p.description     = p.paragraphs_of('README.txt', 3).join("\n\n")
   p.url             = p.paragraphs_of('README.txt', 1).first.strip
   p.changes         = p.paragraphs_of('CHANGELOG.txt', 0..2).join("\n\n")
-  files = 
-    (p.test_globs + ['test/**/tc_*.rb',
-     "test/htdocs/**/*.{html,jpg}",
-     'test/data/server.*']).map { |x|
-    Dir.glob(x)
-  }.flatten + ['test/data/htpasswd']
   p.extra_deps      = ['hpricot']
-  p.spec_extras     = { :test_files => files }
-end
-
-task :update_version do
-  announce "Updating Mechanize Version to #{PKG_VERSION}"
-  File.open("lib/mechanize/mech_version.rb", "w") do |f|
-    f.puts "module WWW"
-    f.puts "  class Mechanize"
-    f.puts "    Version = '#{PKG_VERSION}'"
-    f.puts "  end"
-    f.puts "end"
-  end
-  sh 'svn commit -m"updating version" lib/mechanize/mech_version.rb'
 end
 
-desc "Tag code"
-Rake::Task.define_task("tag") do |p|
-  baseurl = "svn+ssh://#{ENV['USER']}@rubyforge.org//var/svn/#{PKG_NAME}"
-  sh "svn cp -m 'tagged #{ PKG_VERSION }' . #{ baseurl }/tags/REL-#{ PKG_VERSION }"
-end
-
-desc "Branch code"
-Rake::Task.define_task("branch") do |p|
-  baseurl = "svn+ssh://#{ENV['USER']}@rubyforge.org/var/svn/#{PKG_NAME}"
-  sh "svn cp -m 'branched #{ PKG_VERSION }' #{baseurl}/trunk #{ baseurl }/branches/RB-#{ PKG_VERSION }"
-end
 
-desc "Update SSL Certificate"
-Rake::Task.define_task('ssl_cert') do |p|
-  sh "openssl genrsa -des3 -out server.key 1024"
-  sh "openssl req -new -key server.key -out server.csr"
-  sh "cp server.key server.key.org"
-  sh "openssl rsa -in server.key.org -out server.key"
-  sh "openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt"
-  sh "cp server.key server.pem"
-  sh "mv server.key server.csr server.crt server.pem test/data/"
-  sh "rm server.key.org"
-end

data/lib/mechanize.rb

@@ -11,15 +11,24 @@
 unless RUBY_VERSION > "1.8.2"
   $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "mechanize", "net-overrides")
 end
+
 require 'net/http'
 require 'net/https'
 
+# Monkey patch for ruby 1.8.4
+unless RUBY_VERSION > "1.8.4"
+module Net # :nodoc:
+  class HTTPResponse # :nodoc:
+    CODE_TO_OBJ['500'] = HTTPInternalServerError
+  end
+end
+end
+
 require 'uri'
 require 'webrick/httputils'
 require 'zlib'
 require 'stringio'
 require 'mechanize/hpricot'
-require 'mechanize/mech_version'
 require 'mechanize/cookie'
 require 'mechanize/errors'
 require 'mechanize/pluggable_parsers'
@@ -50,15 +59,23 @@ module WWW
 #  search_results = agent.submit(search_form)
 #  puts search_results.body
 class Mechanize
+  ##
+  # The version of Mechanize you are using.
+ 
+  VERSION = '0.6.5'
+
+  ##
+  # User Agent aliases
   AGENT_ALIASES = {
     'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+    'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
     'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
     'Mac Safari' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3',
     'Mac FireFox' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3',
     'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401',
     'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
     'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
-    'Mechanize' => "WWW-Mechanize/#{Version} (http://rubyforge.org/projects/mechanize/)"
+    'Mechanize' => "WWW-Mechanize/#{VERSION} (http://rubyforge.org/projects/mechanize/)"
   }
 
   attr_accessor :cookie_jar
@@ -219,9 +236,14 @@ class Mechanize
 
   # Returns whether or not a url has been visited
   def visited?(url)
+    ! visited_page(url).nil?
+  end
+
+  # Returns a visited page for the url passed in, otherwise nil
+  def visited_page(url)
     url = url.uri if url.respond_to? :uri
     uri = to_absolute_uri(url).to_s
-    ! @history.find { |h| h.uri.to_s == uri }.nil?
+    @history.reverse.find { |h| h.uri.to_s == uri }
   end
 
   # Runs given block, then resets the page history as it was before. self is
@@ -262,6 +284,11 @@ class Mechanize
     # Add User-Agent header to request
     request.add_field('User-Agent', @user_agent) if @user_agent 
 
+    # Add If-Modified-Since if page is in history
+    if( (page = visited_page(uri)) && page.response['Last-Modified'] )
+      request.add_field('If-Modified-Since', page.response['Last-Modified'])
+    end
+
     request.basic_auth(@user, @password) if @user || @password
     request
   end
@@ -269,14 +296,25 @@ class Mechanize
   private
 
   def to_absolute_uri(url, cur_page=current_page())
-    url =  URI.parse(
-           URI.unescape(Util.html_unescape(url.to_s.strip)).gsub(/ /, '%20')
-                     ) unless url.is_a? URI
+    unless url.is_a? URI
+      url = url.to_s.strip
+      url = URI.parse(
+              Util.html_unescape(
+                url.split(/%[0-9A-Fa-f]{2}/).zip(
+                  url.scan(/%[0-9A-Fa-f]{2}/)
+                ).map { |x,y|
+                  "#{URI.escape(x)}#{y}"
+                }.join('')
+              ).gsub(/%23/, '#')
+            )
+    end
 
     # construct an absolute uri
     if url.relative?
       raise 'no history. please specify an absolute URL' unless cur_page.uri
       url = cur_page.uri + url
+      # Strip initial "/.." bits from the path
+      url.path.sub!(/^(\/\.\.)+(?=\/)/, '')
     end
 
     return url
@@ -403,21 +441,23 @@ class Mechanize
           response,
           response_body,
           response.code
-        )
-
-        page.mech = self if page.respond_to? :mech=
+        ) { |parser|
+          parser.mech = self if parser.respond_to? :mech=
+          if parser.respond_to?(:watch_for_set=) && @watch_for_set
+            parser.watch_for_set = @watch_for_set
+          end
+        }
 
         log.info("status: #{ page.code }") if log
 
-        if page.respond_to? :watch_for_set
-          page.watch_for_set = @watch_for_set
-        end
-
         res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s]
 
         return page if res_klass <= Net::HTTPSuccess
 
-        if res_klass <= Net::HTTPRedirection
+        if res_klass == Net::HTTPNotModified
+          log.debug("Got cached page") if log
+          return visited_page(uri)
+        elsif res_klass <= Net::HTTPRedirection
           return page unless follow_redirect?
           log.info("follow redirect to: #{ response['Location'] }") if log
           abs_uri = to_absolute_uri(response['Location'].to_s, page)
@@ -444,17 +484,31 @@ class Mechanize
 
   def add_to_history(page)
     @history.push(page)
-    if @max_history and @history.size > @max_history
-      # keep only the last @max_history entries
-      @history = @history[@history.size - @max_history, @max_history] 
+    if @max_history and @history.length > @max_history
+      while @history.length > @max_history
+        @history[0] = nil
+        @history.shift
+      end
     end
   end
 
+  # :stopdoc:
   class Util
     def self.html_unescape(s)
-      s.to_s.gsub(/&amp;/, "&").gsub(/&quot;/, '"').gsub(/&gt;/, ">").gsub(/&lt;/, "<")
+      return s unless s
+      s.gsub(/&(\w+|#[0-9]+);/) { |match|
+        number = case match
+        when /&(\w+);/
+          Hpricot::NamedCharacters[$1]
+        when /&#([0-9]+);/
+          $1.to_i
+        end
+
+        number ? (number.chr rescue match) : match
+      }
     end
   end
+  # :startdoc:
 end
 
 end # module WWW

data/lib/mechanize/errors.rb

@@ -27,7 +27,11 @@ module WWW
         @response_code = page.code
       end
 
-      def inspect; response_code; end
+      def to_s
+        "#{response_code} => #{Net::HTTPResponse::CODE_TO_OBJ[response_code]}"
+      end
+
+      def inspect; to_s; end
     end
   end
 end

data/lib/mechanize/form.rb

@@ -26,6 +26,8 @@ module WWW
     
       attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes
       attr_reader :enctype
+
+      alias :elements :fields
     
       def initialize(form_node, elements_node)
         @form_node, @elements_node = form_node, elements_node
@@ -105,6 +107,11 @@ module WWW
         end
       end
     
+      # Removes all fields with name +field_name+. 
+      def delete_field!(field_name)
+        @fields.delete_if{ |f| f.name == field_name}
+      end
+          
       private
       def parse
         @fields       = WWW::Mechanize::List.new

data/lib/mechanize/form_elements.rb

@@ -10,7 +10,12 @@ module WWW
     attr_accessor :name, :value
   
     def initialize(name, value)
-      @name, @value = name, value
+      @name = Util.html_unescape(name)
+      @value = if value.is_a? String
+                 Util.html_unescape(value)
+               else
+                 value
+               end
     end
   
     def query_value
@@ -24,7 +29,6 @@ module WWW
   # of the file.
   # See the example in EXAMPLES[link://files/EXAMPLES_txt.html]
   class FileUpload < Field
-    attr_accessor :name # Field name
     attr_accessor :file_name # File name
     attr_accessor :mime_type # Mime Type (Optional)
     
@@ -32,7 +36,7 @@ module WWW
     alias :file_data= :value=
   
     def initialize(name, file_name)
-      @file_name = file_name
+      @file_name = Util.html_unescape(file_name)
       @file_data = nil
       super(name, @file_data)
     end
@@ -215,7 +219,7 @@ module WWW
     def initialize(node, select_list)
       node.attributes ||= {}
       @text     = node.all_text
-      @value    = node.attributes['value']
+      @value    = Util.html_unescape(node.attributes['value'])
       @selected = node.attributes.has_key?('selected') ? true : false
       @select_list = select_list # The select list this option belongs to
     end

data/lib/mechanize/hpricot.rb

@@ -1,3 +1,4 @@
+# :enddoc:
 require 'hpricot'
 class Hpricot::Elem
   def all_text

data/lib/mechanize/net-overrides/net/http.rb

@@ -1,4 +1,4 @@
-#
+# :enddoc:
 # = net/http.rb
 #
 # Copyright (C) 1999-2005 Yukihiro Matsumoto

data/lib/mechanize/net-overrides/net/https.rb

@@ -1,3 +1,4 @@
+# :enddoc:
 =begin
 
 = $RCSfile: https.rb,v $ -- SSL/TLS enhancement for Net::HTTP.

data/lib/mechanize/net-overrides/net/protocol.rb

@@ -1,4 +1,4 @@
-#
+# :enddoc:
 # = net/protocol.rb
 #
 #--

data/lib/mechanize/page.rb

@@ -18,45 +18,40 @@ module WWW
     class Page < File
       extend Forwardable
 
-      attr_reader :root, :title, :watch_for_set
+      attr_reader :parser, :title, :watch_for_set
       attr_reader :frames, :iframes, :links, :forms, :meta, :watches
       attr_accessor :mech
 
+      alias :root :parser
+
       def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
         super(uri, response, body, code)
-        @watch_for_set = {}
-        @mech          = mech
-
-        yield self if block_given?
+        @watch_for_set  ||= {}
+        @mech           ||= mech
 
         raise Mechanize::ContentTypeError.new(response['content-type']) unless
             content_type() =~ /^text\/html/ 
 
         # construct parser and feed with HTML
         if body && response
-          @root ||= Hpricot.parse(body)
+          @parser ||= Hpricot.parse(body)
           parse_html
         end
       end
     
-      # Get the response header
-      def header
-        @response
-      end
-    
       # Get the content type
       def content_type
         @response['content-type']
       end
 
       # Search through the page like HPricot
-      def_delegator :@root, :search, :search
-      def_delegator :@root, :/, :/
-      def_delegator :@root, :at, :at
+      def_delegator :@parser, :search, :search
+      def_delegator :@parser, :/, :/
+      def_delegator :@parser, :at, :at
 
       def watch_for_set=(obj)
         @watch_for_set = obj
-        parse_html if @body
+        parse_html if @body && @watch_for_set
       end
 
       def form(name)
@@ -76,24 +71,24 @@ module WWW
         @watches  = {}
     
         # Set the title
-        @title = if (@root/'title').text.length > 0
-          (@root/'title').text
+        @title = if (@parser/'title').text.length > 0
+          (@parser/'title').text
         end
 
         # Find all the form tags
-        (@root/'form').each do |html_form|
+        (@parser/'form').each do |html_form|
           form = Form.new(html_form, @mech, self)
           form.action ||= @uri
           @forms << form
         end
 
         # Find all the 'a' tags
-        (@root/'a').each do |node|
+        (@parser/'a').each do |node|
           @links << Link.new(node, @mech, self)
         end
 
         # Find all 'meta' tags
-        (@root/'meta').each do |node|
+        (@parser/'meta').each do |node|
           next if node.attributes.nil?
           next unless node.attributes.has_key? 'http-equiv'
           next unless node.attributes.has_key? 'content'
@@ -108,19 +103,19 @@ module WWW
         end
 
         # Find all 'frame' tags
-        (@root/'frame').each do |node|
+        (@parser/'frame').each do |node|
           @frames << Frame.new(node, @mech, self)
         end
 
         # Find all 'iframe' tags
-        (@root/'iframe').each do |node|
+        (@parser/'iframe').each do |node|
           @iframes << Frame.new(node, @mech, self)
         end
 
         # Find all watch tags
         unless @watch_for_set.nil?
           @watch_for_set.each do |key, klass|
-            (@root/key).each do |node|
+            (@parser/key).each do |node|
               @watches[key] ||= []
               @watches[key] << (klass ? klass.new(node) : node)
             end

data/lib/mechanize/pluggable_parsers.rb

@@ -18,11 +18,20 @@ module WWW
     #
     class File
       attr_accessor :uri, :response, :body, :code
+      alias :header :response
 
       alias :content :body
 
       def initialize(uri=nil, response=nil, body=nil, code=nil)
-        @uri, @response, @body, @code = uri, response, body, code
+        @uri, @body, @code = uri, body, code
+        # Copy the headers in to a hash to prevent memory leaks
+        if response
+          @response = Headers.new
+          response.each { |k,v|
+            @response[k] = v
+          }
+        end
+        yield self if block_given?
       end
 
       # Use this method to save the content of this object to filename
@@ -50,7 +59,7 @@ module WWW
       attr_reader :filename
 
       def initialize(uri=nil, response=nil, body=nil, code=nil)
-        @uri, @response, @body, @code = uri, response, body, code
+        super(uri, response, body, code)
         path = uri.path.empty? ? 'index.html' : uri.path.gsub(/^[\/]*/, '')
         path += 'index.html' if path =~ /\/$/
 
@@ -154,5 +163,14 @@ module WWW
         @parsers[content_type] = klass
       end
     end
+
+    class Headers < Hash
+      def [](key)
+        super(key.downcase)
+      end
+      def []=(key, value)
+        super(key.downcase, value)
+      end
+    end
   end
 end

data/lib/mechanize/rexml.rb

@@ -1,4 +1,4 @@
-#
+# :enddoc:
 # Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de).
 # Released under the same terms of license as Ruby.
 #

data/test/htdocs/relative/tc_relative_links.html

@@ -1,6 +1,7 @@
 <html>
   <body>
     <a href="../tc_relative_links.html">dot dot slash</a>
+    <a href="../../../../../tc_relative_links.html">too many dots</a>
 <FRAMESET cols="20%, 80%">
   <FRAMESET rows="100, 200, 200">
   <FRAME name="frame1" src="../tc_relative_links.html">

data/test/ssl_server.rb

@@ -0,0 +1,48 @@
+require 'webrick'
+require 'webrick/https'
+require 'servlets'
+require 'logger'
+
+base_dir = FileTest.exists?(Dir::pwd + '/test') ? Dir::pwd + '/test' : Dir::pwd
+
+s = WEBrick::HTTPServer.new(
+  :Port           => 2002,
+  :DocumentRoot   => base_dir + "/htdocs",
+  :SSLEnable      => true,
+  :SSLVerifyClient  => OpenSSL::SSL::VERIFY_NONE,
+  :SSLCertificate => OpenSSL::X509::Certificate.new(
+                                  File.read("data/server.crt")
+                       ),
+  :SSLPrivateKey    => OpenSSL::PKey::RSA.new(
+                                  File.read("data/server.pem")
+                       ),
+  :Logger         => Logger.new(nil),
+  :AccessLog      => Logger.new(nil)
+)
+s.mount("/one_cookie", OneCookieTest)
+s.mount("/one_cookie_no_space", OneCookieNoSpacesTest)
+s.mount("/many_cookies", ManyCookiesTest)
+s.mount("/many_cookies_as_string", ManyCookiesAsStringTest)
+s.mount("/send_cookies", SendCookiesTest)
+s.mount("/form_post", FormTest)
+s.mount("/form post", FormTest)
+s.mount("/response_code", ResponseCodeTest)
+s.mount("/file_upload", FileUploadTest)
+s.mount("/bad_content_type", BadContentTypeTest)
+s.mount("/content_type_test", ContentTypeTest)
+
+htpasswd = WEBrick::HTTPAuth::Htpasswd.new(base_dir + '/data/htpasswd')
+auth = WEBrick::HTTPAuth::BasicAuth.new(
+  :UserDB => htpasswd,
+  :Realm  => 'mechanize',
+  :Logger         => Logger.new(nil),
+  :AccessLog      => Logger.new(nil)
+)
+s.mount_proc('/htpasswd_auth') { |req, res|
+  auth.authenticate(req, res)
+  res.body = "You are authenticated"
+}
+
+trap("INT") { s.stop }
+
+s.start

data/test/tc_cookie_class.rb

@@ -23,6 +23,14 @@ module Enumerable
 end
 
 class CookieClassTest < Test::Unit::TestCase
+  def silently
+    warn_level = $VERBOSE
+    $VERBOSE = false
+    res = yield
+    $VERBOSE = warn_level
+    res
+  end
+
   def test_parse_dates
     url = URI.parse('http://localhost/')
 
@@ -47,9 +55,11 @@ class CookieClassTest < Test::Unit::TestCase
 
     dates.each do |date|
       cookie = "PREF=1; expires=#{date}"
-      WWW::Mechanize::Cookie.parse(url, cookie) { |cookie|
-        assert_equal(true, cookie.expires < yesterday)
-      }
+      silently do
+        WWW::Mechanize::Cookie.parse(url, cookie) { |cookie|
+          assert_equal(true, cookie.expires < yesterday)
+        }
+      end
     end
   end
 
@@ -76,11 +86,13 @@ class CookieClassTest < Test::Unit::TestCase
               "20/06/95 21:07",
     ]
 
-    dates.each do |date|
-      cookie = "PREF=1; expires=#{date}"
-      WWW::Mechanize::Cookie.parse(url, cookie) { |cookie|
-        assert_equal(true, cookie.expires > (Time.now - 86400))
-      }
+    silently do
+      dates.each do |date|
+        cookie = "PREF=1; expires=#{date}"
+        WWW::Mechanize::Cookie.parse(url, cookie) { |cookie|
+          assert_equal(true, cookie.expires > (Time.now - 86400))
+        }
+      end
     end
   end
 
@@ -120,6 +132,43 @@ class CookieClassTest < Test::Unit::TestCase
     end
   end
 
+  def test_parse_valid_cookie_empty_value
+    url = URI.parse('http://rubyforge.org/')
+    cookie_params = {}
+    cookie_params['expires']   = 'expires=Sun, 27-Sep-2037 00:00:00 GMT'
+    cookie_params['path']      = 'path=/'
+    cookie_params['domain']    = 'domain=.rubyforge.org'
+    cookie_params['httponly']  = 'HttpOnly'
+    cookie_value = '12345%7D='
+
+    expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT')
+    
+    cookie_params.keys.combine.each do |c|
+      cookie_text = "#{cookie_value}; "
+      c.each_with_index do |key, idx|
+        if idx == (c.length - 1)
+          cookie_text << "#{cookie_params[key]}"
+        else
+          cookie_text << "#{cookie_params[key]}; "
+        end
+      end
+      cookie = nil
+      WWW::Mechanize::Cookie.parse(url, cookie_text) { |p_cookie| cookie = p_cookie }
+      assert_not_nil(cookie)
+      assert_equal('12345%7D=', cookie.to_s)
+      assert_equal('', cookie.value)
+      assert_equal('/', cookie.path)
+      assert_equal('rubyforge.org', cookie.domain)
+
+      # if expires was set, make sure we parsed it
+      if c.find { |k| k == 'expires' }
+        assert_equal(expires, cookie.expires)
+      else
+        assert_nil(cookie.expires)
+      end
+    end
+  end
+
   # If no path was given, use the one from the URL
   def test_cookie_using_url_path
     url = URI.parse('http://rubyforge.org/login')

data/test/tc_cookie_jar.rb

@@ -43,6 +43,33 @@ class CookieJarTest < Test::Unit::TestCase
     assert_equal(2, jar.cookies(url2).length)
   end
 
+  def test_empty_value
+    values = {  :name     => 'Foo',
+                :value    => '',
+                :path     => '/',
+                :expires  => Time.now + (10 * 86400),
+                :domain   => 'rubyforge.org'
+             }
+    url = URI.parse('http://rubyforge.org/')
+
+    jar = WWW::Mechanize::CookieJar.new
+    assert_equal(0, jar.cookies(url).length)
+
+    # Add one cookie with an expiration date in the future
+    cookie = cookie_from_hash(values)
+    jar.add(url, cookie)
+    assert_equal(1, jar.cookies(url).length)
+
+    jar.add(url, cookie_from_hash( values.merge(  :domain => 'RuByForge.Org',
+                                                  :name   => 'aaron'
+                                               ) ) )
+                  
+    assert_equal(2, jar.cookies(url).length)
+
+    url2 = URI.parse('http://RuByFoRgE.oRg/')
+    assert_equal(2, jar.cookies(url2).length)
+  end
+
   def test_add_future_cookies
     values = {  :name     => 'Foo',
                 :value    => 'Bar',

data/test/tc_forms.rb

@@ -500,6 +500,18 @@ class FormsMechTest < Test::Unit::TestCase
     assert_not_nil(f)
     assert_equal(number_of_fields + 1, form.fields.length)
   end
+  
+  def test_delete_field
+    page = @agent.get("http://localhost:#{PORT}/form_multival.html")
+    form = page.forms.name('post_form').first
+
+    assert_not_nil(form)
+    number_of_fields = form.fields.length
+
+    form.delete_field!('first')
+    assert_nil(form['first'])
+    assert_equal(number_of_fields - 2, form.fields.length)    
+  end
 
   def test_has_field
     page = @agent.get("http://localhost:#{PORT}/form_multival.html")

data/test/tc_html_unscape_forms.rb

@@ -0,0 +1,46 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'rubygems'
+require 'mechanize'
+require 'test_includes'
+
+class TestCheckBoxes < Test::Unit::TestCase
+  include TestMethods
+
+  def test_field
+    f = WWW::Mechanize::Field.new('a&amp;b', 'a&amp;b')
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.value)
+
+    f = WWW::Mechanize::Field.new('a&b', 'a&b')
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.value)
+
+    f = WWW::Mechanize::Field.new('a&#38;b', 'a&#38;b')
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.value)
+  end
+
+  def test_file_upload
+    f = WWW::Mechanize::FileUpload.new('a&amp;b', 'a&amp;b')
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.file_name)
+
+    f = WWW::Mechanize::FileUpload.new('a&b', 'a&b')
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.file_name)
+  end
+
+  def test_image_button
+    f = WWW::Mechanize::ImageButton.new('a&amp;b', 'a&amp;b')
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.value)
+  end
+
+  def test_radio_button
+    f = WWW::Mechanize::RadioButton.new('a&amp;b', 'a&amp;b', nil, nil)
+    assert_equal('a&b', f.name)
+    assert_equal('a&b', f.value)
+  end
+end

data/test/tc_if_modified_since.rb

@@ -0,0 +1,25 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'rubygems'
+require 'mechanize'
+require 'test_includes'
+
+class TestIfModifiedSince < Test::Unit::TestCase
+  def setup
+    @agent = WWW::Mechanize.new
+  end
+
+  def test_get_twice
+    assert_equal(0, @agent.history.length)
+    page = @agent.get('http://localhost/if_modified_since')
+    assert_match(/You did not send/, page.body)
+
+    assert_equal(1, @agent.history.length)
+    page2 = @agent.get('http://localhost/if_modified_since')
+
+    assert_equal(2, @agent.history.length)
+    assert_equal(page.object_id, page2.object_id)
+    assert_match(/You did not send/, page.body)
+  end
+end

data/test/tc_mech.rb

@@ -13,6 +13,10 @@ class TestMechMethods < Test::Unit::TestCase
     @agent = WWW::Mechanize.new
   end
 
+  def test_weird_url
+    @agent.get('http://localhost/?action=bing&bang=boom=1|a=|b=|c=')
+  end
+
   def test_history
     0.upto(25) do |i|
       assert_equal(i, @agent.history.size)
@@ -24,6 +28,8 @@ class TestMechMethods < Test::Unit::TestCase
       @agent.history.last.uri.to_s)
     assert_equal("http://localhost:#{PORT}/",
       @agent.history[-2].uri.to_s)
+    assert_equal("http://localhost:#{PORT}/",
+      @agent.history[-2].uri.to_s)
 
     assert_equal(true, @agent.visited?("http://localhost:#{PORT}/"))
     assert_equal(true, @agent.visited?("/form_test.html"))
@@ -32,6 +38,16 @@ class TestMechMethods < Test::Unit::TestCase
 
   end
 
+  def test_visited
+    @agent.get("http://localhost/content_type_test?ct=application/pdf")
+    assert_equal(true,
+      @agent.visited?("http://localhost/content_type_test?ct=application/pdf"))
+    assert_equal(false,
+      @agent.visited?("http://localhost/content_type_test"))
+    assert_equal(false,
+      @agent.visited?("http://localhost/content_type_test?ct=text/html"))
+  end
+
   def test_max_history
     @agent.max_history = 10
     0.upto(10) do |i|
@@ -45,6 +61,23 @@ class TestMechMethods < Test::Unit::TestCase
     end
   end
 
+  def test_max_history_order
+    @agent.max_history = 2
+    assert_equal(0, @agent.history.length)
+
+    @agent.get('http://localhost/form_test.html')
+    assert_equal(1, @agent.history.length)
+
+    @agent.get('http://localhost/empty_form.html')
+    assert_equal(2, @agent.history.length)
+
+    @agent.get('http://localhost/tc_checkboxes.html')
+    assert_equal(2, @agent.history.length)
+    assert_equal('http://localhost/empty_form.html', @agent.history[0].uri.to_s)
+    assert_equal('http://localhost/tc_checkboxes.html',
+                 @agent.history[1].uri.to_s)
+  end
+
   def test_back_button
     0.upto(5) do |i|
       assert_equal(i, @agent.history.size)

data/test/tc_relative_links.rb

@@ -18,6 +18,13 @@ class TestRelativeLinks < Test::Unit::TestCase
     assert_equal('http://localhost/tc_relative_links.html', @agent.current_page.uri.to_s)
   end
 
+  def test_too_many_dots
+    @page = @agent.get("http://localhost/relative/tc_relative_links.html")
+    page = @page.links.text('too many dots').click
+    assert_not_nil(page)
+    assert_equal('http://localhost/tc_relative_links.html', page.uri.to_s)
+  end
+
   def test_go_forward
     @page = @agent.get("http://localhost/tc_relative_links.html")
     @page = @page.links.first.click

data/test/test_all.rb

@@ -13,6 +13,8 @@ require 'tc_form_button'
 require 'tc_form_no_inputname'
 require 'tc_forms'
 require 'tc_gzipping'
+require 'tc_html_unscape_forms'
+require 'tc_if_modified_since'
 require 'tc_links'
 require 'tc_mech'
 require 'tc_multi_select'

data/test/test_includes.rb

@@ -30,6 +30,7 @@ class Net::HTTP
     '/many_cookies'           => ManyCookiesTest,
     '/many_cookies_as_string' => ManyCookiesAsStringTest,
     '/send_cookies'           => SendCookiesTest,
+    '/if_modified_since'      => ModifiedSinceServlet,
   }
 
   PAGE_CACHE = {}

data/test/test_servlets.rb

@@ -16,6 +16,27 @@ class RefererServlet < WEBrick::HTTPServlet::AbstractServlet
   end
 end
 
+class ModifiedSinceServlet < WEBrick::HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    s_time = 'Fri, 04 May 2001 00:00:38 GMT'
+
+    my_time = Time.parse(s_time)
+
+    if req['If-Modified-Since']
+      your_time = Time.parse(req['If-Modified-Since'])
+      if my_time > your_time
+        res.body = 'This page was updated since you requested'
+      else
+        res.status = 304
+      end
+    else
+      res.body = 'You did not send an If-Modified-Since header'
+    end
+
+    res['Last-Modified'] = s_time
+  end
+end
+
 class GzipServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     if req['Accept-Encoding'] =~ /gzip/

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: mechanize
 version: !ruby/object:Gem::Version 
-  version: 0.6.4
-date: 2006-12-20 00:00:00 -08:00
+  version: 0.6.5
+date: 2007-02-26 00:00:00 -08:00
 summary: Mechanize provides automated web-browsing
 require_paths: 
   - lib
@@ -54,7 +54,6 @@ files:
   - lib/mechanize/hpricot.rb
   - lib/mechanize/inspect.rb
   - lib/mechanize/list.rb
-  - lib/mechanize/mech_version.rb
   - lib/mechanize/net-overrides/net/http.rb
   - lib/mechanize/net-overrides/net/https.rb
   - lib/mechanize/net-overrides/net/protocol.rb
@@ -64,10 +63,46 @@ files:
   - lib/mechanize/pluggable_parsers.rb
   - lib/mechanize/rexml.rb
   - setup.rb
-test_files: 
-  - test/test_all.rb
-  - test/test_includes.rb
-  - test/test_servlets.rb
+  - test/data/htpasswd
+  - test/data/server.crt
+  - test/data/server.csr
+  - test/data/server.key
+  - test/data/server.pem
+  - test/htdocs/alt_text.html
+  - test/htdocs/bad_form_test.html
+  - test/htdocs/button.jpg
+  - test/htdocs/empty_form.html
+  - test/htdocs/file_upload.html
+  - test/htdocs/find_link.html
+  - test/htdocs/form_multi_select.html
+  - test/htdocs/form_multival.html
+  - test/htdocs/form_no_action.html
+  - test/htdocs/form_no_input_name.html
+  - test/htdocs/form_select.html
+  - test/htdocs/form_select_all.html
+  - test/htdocs/form_select_none.html
+  - test/htdocs/form_select_noopts.html
+  - test/htdocs/form_set_fields.html
+  - test/htdocs/form_test.html
+  - test/htdocs/frame_test.html
+  - test/htdocs/google.html
+  - test/htdocs/iframe_test.html
+  - test/htdocs/index.html
+  - test/htdocs/link with space.html
+  - test/htdocs/no_title_test.html
+  - test/htdocs/relative/tc_relative_links.html
+  - test/htdocs/tc_bad_links.html
+  - test/htdocs/tc_checkboxes.html
+  - test/htdocs/tc_encoded_links.html
+  - test/htdocs/tc_form_action.html
+  - test/htdocs/tc_links.html
+  - test/htdocs/tc_no_attributes.html
+  - test/htdocs/tc_pretty_print.html
+  - test/htdocs/tc_radiobuttons.html
+  - test/htdocs/tc_referer.html
+  - test/htdocs/tc_relative_links.html
+  - test/htdocs/tc_textarea.html
+  - test/ssl_server.rb
   - test/tc_authenticate.rb
   - test/tc_bad_links.rb
   - test/tc_checkboxes.rb
@@ -83,6 +118,8 @@ test_files:
   - test/tc_forms.rb
   - test/tc_frames.rb
   - test/tc_gzipping.rb
+  - test/tc_html_unscape_forms.rb
+  - test/tc_if_modified_since.rb
   - test/tc_links.rb
   - test/tc_mech.rb
   - test/tc_multi_select.rb
@@ -107,45 +144,11 @@ test_files:
   - test/tc_textarea.rb
   - test/tc_upload.rb
   - test/tc_watches.rb
-  - test/htdocs/alt_text.html
-  - test/htdocs/bad_form_test.html
-  - test/htdocs/empty_form.html
-  - test/htdocs/file_upload.html
-  - test/htdocs/find_link.html
-  - test/htdocs/form_multi_select.html
-  - test/htdocs/form_multival.html
-  - test/htdocs/form_no_action.html
-  - test/htdocs/form_no_input_name.html
-  - test/htdocs/form_select.html
-  - test/htdocs/form_select_all.html
-  - test/htdocs/form_select_none.html
-  - test/htdocs/form_select_noopts.html
-  - test/htdocs/form_set_fields.html
-  - test/htdocs/form_test.html
-  - test/htdocs/frame_test.html
-  - test/htdocs/google.html
-  - test/htdocs/iframe_test.html
-  - test/htdocs/index.html
-  - test/htdocs/link with space.html
-  - test/htdocs/no_title_test.html
-  - test/htdocs/tc_bad_links.html
-  - test/htdocs/tc_checkboxes.html
-  - test/htdocs/tc_encoded_links.html
-  - test/htdocs/tc_form_action.html
-  - test/htdocs/tc_links.html
-  - test/htdocs/tc_no_attributes.html
-  - test/htdocs/tc_pretty_print.html
-  - test/htdocs/tc_radiobuttons.html
-  - test/htdocs/tc_referer.html
-  - test/htdocs/tc_relative_links.html
-  - test/htdocs/tc_textarea.html
-  - test/htdocs/relative/tc_relative_links.html
-  - test/htdocs/button.jpg
-  - test/data/server.crt
-  - test/data/server.csr
-  - test/data/server.key
-  - test/data/server.pem
-  - test/data/htpasswd
+  - test/test_all.rb
+  - test/test_includes.rb
+  - test/test_servlets.rb
+test_files: 
+  - test/test_all.rb
 rdoc_options: []
 extra_rdoc_files: []
 executables: []
@@ -161,4 +164,14 @@ dependencies:
           - ">"
           - !ruby/object:Gem::Version 
             version: 0.0.0
+      version: 
+  - !ruby/object:Gem::Dependency 
+    name: hoe
+    version_requirement: 
+    version_requirements: !ruby/object:Gem::Version::Requirement 
+      requirements: 
+        - 
+          - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.2.0
       version: 

data/lib/mechanize/mech_version.rb

@@ -1,5 +0,0 @@
-module WWW
-  class Mechanize
-    Version = '0.6.4'
-  end
-end