mechanize 0.7.8 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of mechanize might be problematic. Click here for more details.

Files changed (96) hide show
  1. data/History.txt +14 -0
  2. data/Manifest.txt +30 -5
  3. data/README.txt +5 -5
  4. data/Rakefile +6 -0
  5. data/{eg → examples}/flickr_upload.rb +0 -0
  6. data/{eg → examples}/mech-dump.rb +0 -0
  7. data/{eg → examples}/proxy_req.rb +0 -0
  8. data/{eg → examples}/rubyforge.rb +0 -0
  9. data/{eg → examples}/spider.rb +0 -0
  10. data/lib/www/mechanize.rb +183 -404
  11. data/lib/www/mechanize/chain.rb +34 -0
  12. data/lib/www/mechanize/chain/auth_headers.rb +79 -0
  13. data/lib/www/mechanize/chain/body_decoding_handler.rb +43 -0
  14. data/lib/www/mechanize/chain/connection_resolver.rb +78 -0
  15. data/lib/www/mechanize/chain/custom_headers.rb +23 -0
  16. data/lib/www/mechanize/chain/handler.rb +9 -0
  17. data/lib/www/mechanize/chain/header_resolver.rb +47 -0
  18. data/lib/www/mechanize/chain/parameter_resolver.rb +23 -0
  19. data/lib/www/mechanize/chain/post_connect_hook.rb +0 -0
  20. data/lib/www/mechanize/chain/pre_connect_hook.rb +22 -0
  21. data/lib/www/mechanize/chain/request_resolver.rb +28 -0
  22. data/lib/www/mechanize/chain/response_body_parser.rb +40 -0
  23. data/lib/www/mechanize/chain/response_header_handler.rb +50 -0
  24. data/lib/www/mechanize/chain/response_reader.rb +41 -0
  25. data/lib/www/mechanize/chain/ssl_resolver.rb +36 -0
  26. data/lib/www/mechanize/chain/uri_resolver.rb +56 -0
  27. data/lib/www/mechanize/cookie.rb +1 -1
  28. data/lib/www/mechanize/file_response.rb +60 -0
  29. data/lib/www/mechanize/form.rb +12 -4
  30. data/lib/www/mechanize/form/field.rb +2 -2
  31. data/lib/www/mechanize/form/file_upload.rb +1 -1
  32. data/lib/www/mechanize/form/option.rb +1 -1
  33. data/lib/www/mechanize/list.rb +4 -0
  34. data/lib/www/mechanize/page.rb +20 -10
  35. data/lib/www/mechanize/util.rb +29 -0
  36. data/mechanize.gemspec +4 -4
  37. data/test/chain/test_argument_validator.rb +14 -0
  38. data/test/chain/test_custom_headers.rb +18 -0
  39. data/test/chain/test_parameter_resolver.rb +35 -0
  40. data/test/chain/test_request_resolver.rb +29 -0
  41. data/test/chain/test_response_reader.rb +24 -0
  42. data/test/helper.rb +3 -1
  43. data/test/servlets.rb +43 -0
  44. data/test/test_authenticate.rb +13 -12
  45. data/test/test_bad_links.rb +1 -1
  46. data/test/test_blank_form.rb +1 -1
  47. data/test/test_checkboxes.rb +1 -1
  48. data/test/test_content_type.rb +1 -1
  49. data/test/test_cookie_class.rb +1 -1
  50. data/test/test_cookie_jar.rb +1 -1
  51. data/test/test_cookies.rb +1 -1
  52. data/test/test_encoded_links.rb +1 -1
  53. data/test/test_errors.rb +1 -1
  54. data/test/test_follow_meta.rb +1 -1
  55. data/test/test_form_action.rb +1 -1
  56. data/test/test_form_as_hash.rb +1 -1
  57. data/test/test_form_button.rb +22 -17
  58. data/test/test_form_no_inputname.rb +1 -1
  59. data/test/test_forms.rb +2 -1
  60. data/test/test_frames.rb +1 -1
  61. data/test/test_get_headers.rb +1 -1
  62. data/test/test_gzipping.rb +1 -1
  63. data/test/test_hash_api.rb +17 -14
  64. data/test/test_history.rb +1 -1
  65. data/test/test_history_added.rb +1 -1
  66. data/test/test_html_unscape_forms.rb +1 -1
  67. data/test/test_if_modified_since.rb +1 -1
  68. data/test/test_keep_alive.rb +1 -1
  69. data/test/test_links.rb +1 -1
  70. data/test/test_mech.rb +18 -11
  71. data/test/test_mechanize_file.rb +1 -1
  72. data/test/test_multi_select.rb +1 -1
  73. data/test/test_no_attributes.rb +1 -1
  74. data/test/test_option.rb +2 -1
  75. data/test/test_page.rb +1 -1
  76. data/test/test_pluggable_parser.rb +1 -1
  77. data/test/test_post_form.rb +1 -1
  78. data/test/test_pretty_print.rb +1 -1
  79. data/test/test_radiobutton.rb +1 -1
  80. data/test/test_redirect_limit_reached.rb +1 -1
  81. data/test/test_referer.rb +1 -1
  82. data/test/test_relative_links.rb +1 -1
  83. data/test/test_response_code.rb +7 -1
  84. data/test/test_save_file.rb +1 -1
  85. data/test/test_scheme.rb +44 -0
  86. data/test/test_select.rb +1 -1
  87. data/test/test_select_all.rb +1 -1
  88. data/test/test_select_none.rb +1 -1
  89. data/test/test_select_noopts.rb +1 -1
  90. data/test/test_set_fields.rb +1 -1
  91. data/test/test_ssl_server.rb +1 -1
  92. data/test/test_subclass.rb +4 -11
  93. data/test/test_textarea.rb +1 -1
  94. data/test/test_upload.rb +1 -1
  95. data/test/test_verbs.rb +22 -0
  96. metadata +39 -7
@@ -0,0 +1,14 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
2
+
3
+ class TestURIResolver < Test::Unit::TestCase
4
+ def test_handle
5
+ v = WWW::Mechanize::Chain.new([
6
+ WWW::Mechanize::Chain::URIResolver.new(Hash.new { |h,k|
7
+ h[k] = lambda { |u,r| u }
8
+ })
9
+ ])
10
+ assert_raises(ArgumentError) { v.handle({}) }
11
+ assert_nothing_raised { v.handle({:uri => 'http://google.com/'}) }
12
+ assert_raises(RuntimeError) { v.handle({:uri => 'google'}) }
13
+ end
14
+ end
@@ -0,0 +1,18 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
2
+
3
+ class TestCustomHeaders < Test::Unit::TestCase
4
+ def test_handle
5
+ v = WWW::Mechanize::Chain.new([
6
+ WWW::Mechanize::Chain::CustomHeaders.new
7
+ ])
8
+ url = URI.parse('http://tenderlovemaking.com/')
9
+ hash = {
10
+ :request => Net::HTTP::Get.new(url.request_uri),
11
+ :headers => { 'Content-Length' => 300 }
12
+ }
13
+ v.handle(hash)
14
+ headers = hash[:request].to_hash
15
+ assert(headers.key?('content-length'))
16
+ assert_equal([300], headers['content-length'])
17
+ end
18
+ end
@@ -0,0 +1,35 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
2
+
3
+ class TestParameterResolver < Test::Unit::TestCase
4
+ def test_handle_get
5
+ v = WWW::Mechanize::Chain.new([
6
+ WWW::Mechanize::Chain::ParameterResolver.new
7
+ ])
8
+ hash = {
9
+ :uri => URI.parse('http://google.com/'),
10
+ :params => { :q => 'hello' },
11
+ :verb => :get
12
+ }
13
+ assert_nothing_raised {
14
+ v.handle(hash)
15
+ }
16
+ assert_equal('q=hello', hash[:uri].query)
17
+ assert_equal([], hash[:params])
18
+ end
19
+
20
+ def test_handle_post
21
+ v = WWW::Mechanize::Chain.new([
22
+ WWW::Mechanize::Chain::ParameterResolver.new
23
+ ])
24
+ hash = {
25
+ :uri => URI.parse('http://google.com/'),
26
+ :params => { :q => 'hello' },
27
+ :verb => :post
28
+ }
29
+ assert_nothing_raised {
30
+ v.handle(hash)
31
+ }
32
+ assert_equal('', hash[:uri].query.to_s)
33
+ assert_equal({ :q => 'hello' }, hash[:params])
34
+ end
35
+ end
@@ -0,0 +1,29 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
2
+
3
+ class TestRequestResolver < Test::Unit::TestCase
4
+ def test_handle_get
5
+ v = WWW::Mechanize::Chain.new([
6
+ WWW::Mechanize::Chain::RequestResolver.new
7
+ ])
8
+ hash = {
9
+ :uri => URI.parse('http://google.com'),
10
+ :verb => :get
11
+ }
12
+ assert_nothing_raised { v.handle(hash) }
13
+ assert_kind_of(Net::HTTP::Get, hash[:request])
14
+ assert_equal('/', hash[:request].path)
15
+ end
16
+
17
+ def test_handle_post
18
+ v = WWW::Mechanize::Chain.new([
19
+ WWW::Mechanize::Chain::RequestResolver.new
20
+ ])
21
+ hash = {
22
+ :uri => URI.parse('http://google.com'),
23
+ :verb => :post
24
+ }
25
+ assert_nothing_raised { v.handle(hash) }
26
+ assert_kind_of(Net::HTTP::Post, hash[:request])
27
+ assert_equal('/', hash[:request].path)
28
+ end
29
+ end
@@ -0,0 +1,24 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
2
+
3
+ class TestResponseReader < Test::Unit::TestCase
4
+ def test_handle_bad_response_code
5
+ response = Object.new
6
+ class << response
7
+ def read_body
8
+ end
9
+ def code; 999; end
10
+ end
11
+
12
+ v = WWW::Mechanize::Chain.new([
13
+ WWW::Mechanize::Chain::ResponseReader.new(response)
14
+ ])
15
+ assert_raises(WWW::Mechanize::ResponseCodeError) {
16
+ begin
17
+ v.handle({})
18
+ rescue Exception => x
19
+ assert_equal(999, x.response_code)
20
+ raise x
21
+ end
22
+ }
23
+ end
24
+ end
data/test/helper.rb CHANGED
@@ -2,7 +2,7 @@ require 'test/unit'
2
2
  require 'rubygems'
3
3
  require 'mechanize'
4
4
  require 'webrick/httputils'
5
- require 'servlets'
5
+ require File.join(File.dirname(__FILE__),'servlets')
6
6
 
7
7
  BASE_DIR = File.dirname(__FILE__)
8
8
 
@@ -44,6 +44,8 @@ class Net::HTTP
44
44
  '/if_modified_since' => ModifiedSinceServlet,
45
45
  '/http_headers' => HeaderServlet,
46
46
  '/infinite_redirect' => InfiniteRedirectTest,
47
+ '/digest_auth' => DigestAuthServlet,
48
+ '/verb' => VerbServlet,
47
49
  }
48
50
 
49
51
  PAGE_CACHE = {}
data/test/servlets.rb CHANGED
@@ -5,6 +5,18 @@ require 'zlib'
5
5
  require 'stringio'
6
6
  require 'base64'
7
7
 
8
+ class VerbServlet < WEBrick::HTTPServlet::AbstractServlet
9
+ def do_HEAD(req, res)
10
+ res.body = "method: HEAD"
11
+ end
12
+
13
+ def method_missing(method, *args, &block)
14
+ super unless method.to_s =~ /^do_([A-Z]*)$/
15
+ res = args[1]
16
+ res.body = "method: #{$1}"
17
+ end
18
+ end
19
+
8
20
  class BasicAuthServlet < WEBrick::HTTPServlet::AbstractServlet
9
21
  def do_GET(req,res)
10
22
  htpd = WEBrick::HTTPAuth::Htpasswd.new('dot.htpasswd')
@@ -26,9 +38,40 @@ class BasicAuthServlet < WEBrick::HTTPServlet::AbstractServlet
26
38
  alias :do_POST :do_GET
27
39
  end
28
40
 
41
+ class DigestAuthServlet < WEBrick::HTTPServlet::AbstractServlet
42
+ htpd = WEBrick::HTTPAuth::Htdigest.new('digest.htpasswd')
43
+ htpd.set_passwd('Blah', 'user', 'pass')
44
+ @@authenticator = WEBrick::HTTPAuth::DigestAuth.new({
45
+ :UserDB => htpd,
46
+ :Realm => 'Blah',
47
+ :Algorithm => 'MD5',
48
+ :Logger => Logger.new(nil)
49
+ }
50
+ )
51
+ def do_GET(req,res)
52
+ def req.request_time; Time.now; end
53
+ def req.request_uri; '/digest_auth'; end
54
+ def req.request_method; "GET"; end
55
+
56
+ begin
57
+ @@authenticator.authenticate(req,res)
58
+ res.body = 'You are authenticated'
59
+ rescue WEBrick::HTTPStatus::Unauthorized => ex
60
+ res.status = 401
61
+ end
62
+ FileUtils.rm('digest.htpasswd') if File.exists?('digest.htpasswd')
63
+ end
64
+ alias :do_POST :do_GET
65
+ end
66
+
29
67
  class HeaderServlet < WEBrick::HTTPServlet::AbstractServlet
30
68
  def do_GET(req, res)
31
69
  res['Content-Type'] = "text/html"
70
+
71
+ req.query.each do |x,y|
72
+ res[x] = y
73
+ end
74
+
32
75
  body = ''
33
76
  req.each_header do |k,v|
34
77
  body << "#{k}|#{v}\n"
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class BasicAuthTest < Test::Unit::TestCase
4
4
  def setup
@@ -11,14 +11,21 @@ class BasicAuthTest < Test::Unit::TestCase
11
11
  assert_equal('You are authenticated', page.body)
12
12
  end
13
13
 
14
+ def test_digest_auth_success
15
+ @agent.basic_auth('user', 'pass')
16
+ page = @agent.get("http://localhost/digest_auth")
17
+ assert_equal('You are authenticated', page.body)
18
+ end
19
+
14
20
  def test_post_auth_success
15
21
  class << @agent
16
- alias :old_fetch_request :fetch_request
22
+ alias :old_fetch_page :fetch_page
17
23
  attr_accessor :requests
18
- def fetch_request(*args)
24
+ def fetch_page(args)
19
25
  @requests ||= []
20
- @requests << old_fetch_request(*args)
21
- @requests.last
26
+ x = old_fetch_page(args)
27
+ @requests << args[:verb]
28
+ x
22
29
  end
23
30
  end
24
31
  @agent.basic_auth('user', 'pass')
@@ -27,13 +34,7 @@ class BasicAuthTest < Test::Unit::TestCase
27
34
  assert_equal(2, @agent.requests.length)
28
35
  r1 = @agent.requests[0]
29
36
  r2 = @agent.requests[1]
30
- assert r1['Content-Type']
31
- assert r2['Content-Type']
32
- assert_equal(r1['Content-Type'], r2['Content-Type'])
33
-
34
- assert r1['Content-Length']
35
- assert r2['Content-Length']
36
- assert_equal(r1['Content-Length'], r2['Content-Length'])
37
+ assert_equal(r1, r2)
37
38
  end
38
39
 
39
40
  def test_auth_bad_user_pass
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestBadLinks < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class BlankFormTest < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestCheckBoxes < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestContentType < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  module Enumerable
4
4
  def combine
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class CookieJarTest < Test::Unit::TestCase
4
4
  def cookie_from_hash(hash)
data/test/test_cookies.rb CHANGED
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class CookiesMechTest < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestEncodedLinks < Test::Unit::TestCase
4
4
  def setup
data/test/test_errors.rb CHANGED
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class MechErrorsTest < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class FollowMetaTest < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestFormAction < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestFormHash < Test::Unit::TestCase
4
4
  def setup
@@ -1,33 +1,38 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestFormButtons < Test::Unit::TestCase
4
4
  def setup
5
5
  @agent = WWW::Mechanize.new
6
6
  end
7
7
 
8
- def test_submit_button
9
- html = <<-END
10
- <html><body>
11
- <form><input type="submit" value="submit" /></form>
12
- </body></html>
13
- END
14
- page = WWW::Mechanize::Page.new( nil, html_response, html, 200, @agent )
15
- assert_equal(1, page.forms.length)
16
- assert_equal(1, page.forms.first.buttons.length)
8
+ def test_submit_input_tag
9
+ assert_form_contains_button('<input type="submit" value="submit" />')
10
+ end
11
+
12
+ def test_button_input_tag
13
+ assert_form_contains_button('<input type="button" value="submit" />')
14
+ end
15
+
16
+ def test_submit_button_tag
17
+ assert_form_contains_button('<button type="submit" value="submit"/>')
17
18
  end
18
19
 
19
- def test_button_button
20
- html = <<-END
21
- <html><body>
22
- <form><input type="button" value="submit" /></form>
23
- </body></html>
24
- END
25
- page = WWW::Mechanize::Page.new( nil, html_response, html, 200, @agent )
20
+ def test_button_button_tag
21
+ assert_form_contains_button('<button type="button" value="submit"/>')
22
+ end
23
+
24
+ def assert_form_contains_button(button)
25
+ page = WWW::Mechanize::Page.new(nil, html_response, html(button), 200, @agent)
26
26
  assert_equal(1, page.forms.length)
27
27
  assert_equal(1, page.forms.first.buttons.length)
28
28
  end
29
29
 
30
+ def html(input)
31
+ "<html><body><form>#{input}</form></body></html>"
32
+ end
33
+
30
34
  def html_response
31
35
  { 'content-type' => 'text/html' }
32
36
  end
33
37
  end
38
+
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class FormNoInputNameTest < Test::Unit::TestCase
4
4
  def setup
data/test/test_forms.rb CHANGED
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class FormsMechTest < Test::Unit::TestCase
4
4
  def setup
@@ -499,6 +499,7 @@ class FormsMechTest < Test::Unit::TestCase
499
499
 
500
500
  assert_not_nil(form)
501
501
  number_of_fields = form.fields.length
502
+ assert_equal 2, number_of_fields
502
503
 
503
504
  form.delete_field!('first')
504
505
  assert_nil(form['first'])
data/test/test_frames.rb CHANGED
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class FramesMechTest < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestGetHeaders < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestGzip < Test::Unit::TestCase
4
4
  def setup
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + "/helper"
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
2
2
 
3
3
  class TestHashApi < Test::Unit::TestCase
4
4
  def setup
@@ -25,18 +25,21 @@ class TestHashApi < Test::Unit::TestCase
25
25
  end
26
26
 
27
27
  def test_get_with_referer
28
- class << @agent
29
- attr_reader :request
30
- alias :old_set_headers :set_headers
31
- def set_headers(u, request, cur_page)
32
- old_set_headers(u, request, cur_page)
33
- @request = request
34
- end
35
- end
36
- @agent.get(:url => 'http://localhost/', :referer => URI.parse('http://google.com/'))
37
- assert_equal 'http://google.com/', @agent.request['Referer']
38
-
39
- @agent.get(:url => 'http://localhost/', :params => [], :referer => 'http://tenderlovemaking.com/')
40
- assert_equal 'http://tenderlovemaking.com/', @agent.request['Referer']
28
+ request = nil
29
+ @agent.pre_connect_hooks << lambda { |params|
30
+ request = params[:request]
31
+ }
32
+
33
+ @agent.get( :url => 'http://localhost/',
34
+ :referer => URI.parse('http://google.com/')
35
+ )
36
+
37
+ assert request
38
+ assert_equal 'http://google.com/', request['Referer']
39
+
40
+ @agent.get( :url => 'http://localhost/',
41
+ :params => [],
42
+ :referer => 'http://tenderlovemaking.com/')
43
+ assert_equal 'http://tenderlovemaking.com/', request['Referer']
41
44
  end
42
45
  end