mechanize 0.6.8 → 0.6.9

data/CHANGELOG.txt

@@ -1,5 +1,17 @@
 = Mechanize CHANGELOG
 
+== 0.6.9
+
+* Updating UTF-8 support for urls
+* Adding AREA tags to the links list.
+  http://rubyforge.org/pipermail/mechanize-users/2007-May/000140.html
+* WWW::Mechanize#follow_meta_refresh will allow you to automatically follow
+  meta refresh tags. [#10032]
+* Adding x-gzip to accepted content-encoding.  Thanks Simon Strandgaard
+  http://rubyforge.org/tracker/index.php?func=detail&aid=11167&group_id=1453&atid=5711
+* Added Digest Authentication support.  Thanks to Ryan Davis and Eric Hodel,
+  you get a gold star!
+
 == 0.6.8
 
 * Keep alive can be shut off now with WWW::Mechanize#keep_alive

data/Manifest.txt

@@ -60,6 +60,7 @@ test/htdocs/relative/tc_relative_links.html
 test/htdocs/tc_bad_links.html
 test/htdocs/tc_checkboxes.html
 test/htdocs/tc_encoded_links.html
+test/htdocs/tc_follow_meta.html
 test/htdocs/tc_form_action.html
 test/htdocs/tc_links.html
 test/htdocs/tc_no_attributes.html
@@ -77,6 +78,7 @@ test/tc_cookie_jar.rb
 test/tc_cookies.rb
 test/tc_encoded_links.rb
 test/tc_errors.rb
+test/tc_follow_meta.rb
 test/tc_form_action.rb
 test/tc_form_as_hash.rb
 test/tc_form_button.rb

data/lib/mechanize.rb

@@ -28,6 +28,7 @@ require 'uri'
 require 'webrick/httputils'
 require 'zlib'
 require 'stringio'
+require 'digest/md5'
 require 'mechanize/monkey_patch'
 require 'mechanize/cookie'
 require 'mechanize/errors'
@@ -63,7 +64,7 @@ class Mechanize
   ##
   # The version of Mechanize you are using.
  
-  VERSION = '0.6.8'
+  VERSION = '0.6.9'
 
   ##
   # User Agent aliases
@@ -92,12 +93,16 @@ class Mechanize
   attr_accessor :keep_alive_time
   attr_accessor :keep_alive
   attr_accessor :conditional_requests
+  attr_accessor :follow_meta_refresh
 
   attr_reader :history
   attr_reader :pluggable_parser
 
   alias :follow_redirect? :redirect_ok
 
+  @@nonce_count = -1
+  CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535)))
+
   def initialize
     # attr_accessors
     @cookie_jar     = CookieJar.new
@@ -116,9 +121,11 @@ class Mechanize
     @history        = WWW::Mechanize::History.new
     @pluggable_parser = PluggableParser.new
 
-    # Basic Auth variables
-    @user           = nil # Basic Auth User
-    @password       = nil # Basic Auth Password
+    # Auth variables
+    @user           = nil # Auth User
+    @password       = nil # Auth Password
+    @digest         = nil # DigestAuth Digest
+    @auth_hash      = {}  # Keep track of urls for sending auth
 
     # Proxy settings
     @proxy_addr     = nil
@@ -128,6 +135,8 @@ class Mechanize
 
     @conditional_requests = true
 
+    @follow_meta_refresh  = false
+
     # Connection Cache & Keep alive
     @connection_cache = {}
     @keep_alive_time  = 300
@@ -157,8 +166,12 @@ class Mechanize
 
   # Sets the user and password to be used for basic authentication.
   def basic_auth(user, password)
-    @user = user
-    @password = password
+    auth(user, password)
+  end
+
+  def auth(user, password)
+    @user       = user
+    @password   = password
   end
 
   # Fetches the URL passed in and returns a page.
@@ -309,15 +322,67 @@ class Mechanize
       end
     end
 
-    request.basic_auth(@user, @password) if @user || @password
+    if( @auth_hash[uri.to_s] )
+      raise 'Please provide username and password' unless @user || @password
+      case @auth_hash[uri.to_s]
+      when :basic
+        request.basic_auth(@user, @password)
+      when :digest
+        @digest_response ||= nil
+        @digest_response = self.gen_auth_header(uri, @digest) if @digest
+        request.add_field('Authorization', @digest_response) if @digest_response
+      end
+    end
+
     request
   end
 
+  def gen_auth_header(uri, auth_header, is_IIS = false)
+    @@nonce_count += 1
+
+    user = @digest_user
+    password = @digest_password
+
+    auth_header =~ /^(\w+) (.*)/
+
+    params = {}
+    $2.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
+
+    a_1 = "#{@user}:#{params['realm']}:#{@password}"
+    a_2 = "GET:#{uri.path}"
+    request_digest = ''
+    request_digest << Digest::MD5.hexdigest(a_1)
+    request_digest << ':' << params['nonce']
+    request_digest << ':' << ('%08x' % @@nonce_count)
+    request_digest << ':' << CNONCE
+    request_digest << ':' << params['qop']
+    request_digest << ':' << Digest::MD5.hexdigest(a_2)
+
+    header = ''
+    header << "Digest username=\"#{@user}\", "
+    header << "realm=\"#{params['realm']}\", "
+    if is_IIS then
+      header << "qop=\"#{params['qop']}\", "
+    else
+      header << "qop=#{params['qop']}, "
+    end
+    header << "uri=\"#{uri.path}\", "
+    header << "nonce=\"#{params['nonce']}\", "
+    header << "nc=#{'%08x' % @@nonce_count}, "
+    header << "cnonce=\"#{CNONCE}\", "
+    header << "response=\"#{Digest::MD5.hexdigest(request_digest)}\""
+
+    return header
+  end
+
   private
 
   def to_absolute_uri(url, cur_page=current_page())
     unless url.is_a? URI
-      url = url.to_s.strip
+      url = url.to_s.strip.gsub(/[^#{0.chr}-#{125.chr}]/) { |match|
+        sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'c')[0])
+      }
+
       url = URI.parse(
               Util.html_unescape(
                 url.split(/%[0-9A-Fa-f]{2}/).zip(
@@ -467,6 +532,8 @@ class Mechanize
         when 'gzip'
           log.debug('gunzip body') if log
           Zlib::GzipReader.new(body).read
+        when 'x-gzip'
+          body.read
         else
           raise 'Unsupported content encoding'
         end
@@ -509,6 +576,10 @@ class Mechanize
 
     res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s]
 
+    if follow_meta_refresh && (redirect = page.meta.first)
+      return redirect.click
+    end
+
     return page if res_klass <= Net::HTTPSuccess
 
     if res_klass == Net::HTTPNotModified
@@ -522,6 +593,15 @@ class Mechanize
       page = fetch_page(abs_uri, fetch_request(abs_uri), page)
       @history.push(page, from_uri)
       return page
+    elsif res_klass <= Net::HTTPUnauthorized
+      if response['www-authenticate'] =~ /Digest/i
+        @auth_hash[uri.to_s] = :digest
+        @digest = response['www-authenticate']
+        return fetch_page(uri, fetch_request(uri), cur_page, request_data)
+      else
+        @auth_hash[uri.to_s] = :basic
+        return fetch_page(uri, fetch_request(uri), cur_page, request_data)
+      end
     end
 
     raise ResponseCodeError.new(page), "Unhandled response", caller
@@ -555,7 +635,7 @@ class Mechanize
           $1.to_i
         end
 
-        number ? (number.chr rescue match) : match
+        number ? ([number].pack('U') rescue match) : match
       }
     end
   end

data/lib/mechanize/page.rb

@@ -54,6 +54,7 @@ module WWW
         parse_html if @body && @watch_for_set
       end
 
+      # Find a form with +name+.  Form will be yeilded if a block is given.
       def form(name)
         f = forms.name(name).first
         yield f if block_given?
@@ -87,6 +88,11 @@ module WWW
           @links << Link.new(node, @mech, self)
         end
 
+        # Find all the 'area' tags
+        (@parser/'area').each do |node|
+          @links << Link.new(node, @mech, self)
+        end
+
         # Find all 'meta' tags
         (@parser/'meta').each do |node|
           next unless node['http-equiv']

data/test/htdocs/tc_follow_meta.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <meta http-equiv="refresh" content="0; url=http://localhost/index.html">
+  </head>
+  <body>
+    This page has a meta refresh.
+  </body>
+</html>

data/test/tc_follow_meta.rb

@@ -0,0 +1,25 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'rubygems'
+require 'mechanize'
+require 'test_includes'
+
+class FollowMetaTest < Test::Unit::TestCase
+  include TestMethods
+
+  def setup
+    @agent = WWW::Mechanize.new
+  end
+
+  def test_follow_meta
+    page = @agent.get('http://localhost/tc_follow_meta.html')
+    assert_equal('http://localhost/tc_follow_meta.html', page.uri.to_s)
+    assert_equal(1, page.meta.length)
+
+    @agent.follow_meta_refresh = true
+    page = @agent.get('http://localhost/tc_follow_meta.html')
+    assert_equal('http://localhost/index.html', page.uri.to_s)
+    assert_equal(3, @agent.history.length)
+  end
+end

data/test/tc_links.rb

@@ -21,7 +21,7 @@ class LinksMechTest < Test::Unit::TestCase
 
   def test_find_link
     page = @agent.get("http://localhost:#{PORT}/find_link.html")
-    assert_equal(15, page.links.length)
+    assert_equal(18, page.links.length)
   end
 
   def test_alt_text

data/test/tc_mech.rb

@@ -20,6 +20,17 @@ class TestMechMethods < Test::Unit::TestCase
     assert_nothing_raised {
       @agent.get('http://localhost/?a=b&#038;b=c&#038;c=d')
     }
+    assert_nothing_raised {
+      @agent.get("http://localhost/?a=#{[0xd6].pack('U')}")
+    }
+  end
+
+  def test_kcode_url
+    $KCODE = 'u'
+    page = @agent.get("http://localhost/?a=#{[0xd6].pack('U')}")
+    assert_not_nil(page)
+    assert_equal('http://localhost/?a=%D6', page.uri.to_s)
+    $KCODE = 'NONE'
   end
 
   def test_history

data/test/tc_pluggable_parser.rb

@@ -63,7 +63,7 @@ class PluggableParserTest < Test::Unit::TestCase
     @agent.pluggable_parser.html = Filter
     page = @agent.get("http://localhost:#{PORT}/find_link.html")
     assert_kind_of(Filter, page)
-    assert_equal(16, page.links.length)
+    assert_equal(19, page.links.length)
     assert_not_nil(page.links.text('Net::DAAP::Client').first)
     assert_equal(1, page.links.text('Net::DAAP::Client').length)
   end
@@ -74,7 +74,7 @@ class PluggableParserTest < Test::Unit::TestCase
     assert_kind_of(Class, @agent.pluggable_parser['text/html'])
     assert_equal(Filter, @agent.pluggable_parser['text/html'])
     assert_kind_of(Filter, page)
-    assert_equal(16, page.links.length)
+    assert_equal(19, page.links.length)
     assert_not_nil(page.links.text('Net::DAAP::Client').first)
     assert_equal(1, page.links.text('Net::DAAP::Client').length)
   end

data/test/test_all.rb

@@ -7,6 +7,7 @@ require 'tc_cookie_class'
 require 'tc_cookie_jar'
 require 'tc_cookies'
 require 'tc_errors'
+require 'tc_follow_meta'
 require 'tc_form_action'
 require 'tc_form_as_hash'
 require 'tc_form_button'

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: mechanize
 version: !ruby/object:Gem::Version 
-  version: 0.6.8
-date: 2007-04-15 00:00:00 -07:00
+  version: 0.6.9
+date: 2007-06-24 00:00:00 -07:00
 summary: Mechanize provides automated web-browsing
 require_paths: 
   - lib
@@ -95,6 +95,7 @@ files:
   - test/htdocs/tc_bad_links.html
   - test/htdocs/tc_checkboxes.html
   - test/htdocs/tc_encoded_links.html
+  - test/htdocs/tc_follow_meta.html
   - test/htdocs/tc_form_action.html
   - test/htdocs/tc_links.html
   - test/htdocs/tc_no_attributes.html
@@ -112,6 +113,7 @@ files:
   - test/tc_cookies.rb
   - test/tc_encoded_links.rb
   - test/tc_errors.rb
+  - test/tc_follow_meta.rb
   - test/tc_form_action.rb
   - test/tc_form_as_hash.rb
   - test/tc_form_button.rb
@@ -153,8 +155,17 @@ files:
   - test/test_servlets.rb
 test_files: 
   - test/test_all.rb
-rdoc_options: []
-extra_rdoc_files: []
+rdoc_options: 
+  - "--main"
+  - README.txt
+extra_rdoc_files: 
+  - CHANGELOG.txt
+  - EXAMPLES.txt
+  - GUIDE.txt
+  - LICENSE.txt
+  - Manifest.txt
+  - NOTES.txt
+  - README.txt
 executables: []
 extensions: []
 requirements: []
@@ -177,5 +188,5 @@ dependencies:
         - 
           - ">="
           - !ruby/object:Gem::Version 
-            version: 1.2.0
+            version: 1.2.1
       version: