diamond-mechanize 2.1

data/EXAMPLES.rdoc

@@ -0,0 +1,187 @@
+= Mechanize examples
+
+Note: Several examples show methods chained to the end of do/end blocks.
+Do...end is the same as curly braces ({...}). For example, do ... end.submit
+is the same as { ... }.submit.
+
+== Google
+
+  require 'rubygems'
+  require 'mechanize'
+
+  a = Mechanize.new { |agent|
+    agent.user_agent_alias = 'Mac Safari'
+  }
+
+  a.get('http://google.com/') do |page|
+    search_result = page.form_with(:name => 'f') do |search|
+      search.q = 'Hello world'
+    end.submit
+
+    search_result.links.each do |link|
+      puts link.text
+    end
+  end
+
+== Rubyforge
+
+  require 'rubygems'
+  require 'mechanize'
+
+  a = Mechanize.new
+  a.get('http://rubyforge.org/') do |page|
+    # Click the login link
+    login_page = a.click(page.link_with(:text => /Log In/))
+
+    # Submit the login form
+    my_page = login_page.form_with(:action => '/account/login.php') do |f|
+      f.form_loginname  = ARGV[0]
+      f.form_pw         = ARGV[1]
+    end.click_button
+
+    my_page.links.each do |link|
+      text = link.text.strip
+      next unless text.length > 0
+      puts text
+    end
+  end
+
+== File Upload
+
+Upload a file to flickr.
+
+  require 'rubygems'
+  require 'mechanize'
+
+  abort "#{$0} login passwd filename" if (ARGV.size != 3)
+
+  a = Mechanize.new { |agent|
+    # Flickr refreshes after login
+    agent.follow_meta_refresh = true
+  }
+
+  a.get('http://flickr.com/') do |home_page|
+    signin_page = a.click(home_page.link_with(:text => /Sign In/))
+
+    my_page = signin_page.form_with(:name => 'login_form') do |form|
+      form.login  = ARGV[0]
+      form.passwd = ARGV[1]
+    end.submit
+
+    # Click the upload link
+    upload_page = a.click(my_page.link_with(:text => /Upload/))
+
+    # We want the basic upload page.
+    upload_page = a.click(upload_page.link_with(:text => /basic Uploader/))
+
+    # Upload the file
+    upload_page.form_with(:method => 'POST') do |upload_form|
+      upload_form.file_uploads.first.file_name = ARGV[2]
+    end.submit
+  end
+
+== Pluggable Parsers
+Lets say you want html pages to automatically be parsed with Rubyful Soup.
+This example shows you how:
+
+  require 'rubygems'
+  require 'mechanize'
+  require 'rubyful_soup'
+
+  class SoupParser < Mechanize::Page
+    attr_reader :soup
+    def initialize(uri = nil, response = nil, body = nil, code = nil)
+      @soup = BeautifulSoup.new(body)
+      super(uri, response, body, code)
+    end
+  end
+
+  agent = Mechanize.new
+  agent.pluggable_parser.html = SoupParser
+
+Now all HTML pages will be parsed with the SoupParser class, and automatically
+give you access to a method called 'soup' where you can get access to the
+Beautiful Soup for that page.
+
+== Using a proxy
+
+  require 'rubygems'
+  require 'mechanize'
+
+  agent = Mechanize.new
+  agent.set_proxy 'localhost', 8000
+  page = agent.get(ARGV[0])
+  puts page.body
+
+== The transact method
+
+transact runs the given block and then resets the page history. I.e. after the
+block has been executed, you're back at the original page; no need count how
+many times to call the back method at the end of a loop (while accounting for
+possible exceptions).
+
+This example also demonstrates subclassing Mechanize.
+
+  require 'rubygems'
+  require 'mechanize'
+
+  class TestMech < Mechanize
+    def process
+      get 'http://rubyforge.org/'
+      search_form = page.forms.first
+      search_form.words = 'WWW'
+      submit search_form
+
+      page.links_with(:href => %r{/projects/} ).each do |link|
+        next if link.href =~ %r{/projects/support/}
+
+        puts 'Loading %-30s %s' % [link.href, link.text]
+        begin
+          transact do
+            click link
+            # Do stuff, maybe click more links.
+          end
+          # Now we're back at the original page.
+
+        rescue => e
+          $stderr.puts "#{e.class}: #{e.message}"
+        end
+      end
+    end
+  end
+
+  TestMech.new.process
+
+== Client Certificate Authentication (Mutual Auth)
+
+In most cases a client certificate is created as an additional layer of security
+for certain websites.  The specific case that this was initially tested on was
+for automating the download of archived images from a banks (Wachovia) lockbox
+system.  Once the certificate is installed into your browser you will have to
+export it and split the certificate and private key into separate files.
+Exported files are usually in .p12 format (IE 7 & Firefox 2.0) which stands for
+PKCS #12.  You can convert them from p12 to pem format by using the following
+commands:
+
+openssl.exe pkcs12 -in input_file.p12 -clcerts -out example.key -nocerts -nodes
+openssl.exe pkcs12 -in input_file.p12 -clcerts -out example.cer -nokeys
+
+  require 'rubygems'
+  require 'mechanize'
+
+  # create Mechanize instance
+  agent = Mechanize.new
+
+  # set the path of the certificate file
+  agent.cert = 'example.cer'
+
+  # set the path of the private key file
+  agent.key = 'example.key'
+
+  # get the login form & fill it out with the username/password
+  login_form = agent.get("http://example.com/login_page").form('Login')
+  login_form.Userid = 'TestUser'
+  login_form.Password = 'TestPassword'
+
+  # submit login form
+  agent.submit(login_form, login_form.buttons.first)

data/FAQ.rdoc

@@ -0,0 +1,11 @@
+Q:: Why do I keep getting an EOFError?
+
+A:: For older versions of mechanize turning off keep_alive could help with the
+    problem, but mechanize now has more robust handling of persistent
+    connections.
+
+    Older versions of mechanize would raise an EOFError when a chunked body was
+    not terminated properly, a common bug of IIS servers.  Since 2.0
+    Mechanize::ResponseReadError is raised containing the original response and
+    body read so far so if the server is broken you can still retrieve the
+    entire content.

data/GUIDE.rdoc

@@ -0,0 +1,163 @@
+= Getting Started With Mechanize
+
+This guide is meant to get you started using Mechanize.  By the end of this
+guide, you should be able to fetch pages, click links, fill out and submit
+forms, scrape data, and many other hopefully useful things.  This guide
+really just scratches the surface of what is available, but should be enough
+information to get you really going!
+
+== Let's Fetch a Page!
+
+First thing is first.  Make sure that you've required mechanize and that you
+instantiate a new mechanize object:
+
+  require 'rubygems'
+  require 'mechanize'
+
+  agent = Mechanize.new
+
+Now we'll use the agent we've created to fetch a page.  Let's fetch google
+with our mechanize agent:
+
+  page = agent.get('http://google.com/')
+
+What just happened?  We told mechanize to go pick up google's main page.
+Mechanize stored any cookies that were set, and followed any redirects that
+google may have sent.  The agent gave us back a page that we can use to
+scrape data, find links to click, or find forms to fill out.
+
+Next, lets try finding some links to click.
+
+== Finding Links
+
+Mechanize returns a page object whenever you get a page, post, or submit a
+form.  When a page is fetched, the agent will parse the page and put a list
+of links on the page object.
+
+Now that we've fetched google's homepage, lets try listing all of the links:
+
+  page.links.each do |link|
+    puts link.text
+  end
+
+We can list the links, but Mechanize gives a few shortcuts to help us find a
+link to click on.  Lets say we wanted to click the link whose text is 'News'.
+Normally, we would have to do this:
+
+  page = agent.page.links.find { |l| l.text == 'News' }.click
+
+But Mechanize gives us a shortcut.  Instead we can say this:
+
+  page = agent.page.link_with(:text => 'News').click
+
+That shortcut says "find all links with the name 'News'".  You're probably
+thinking "there could be multiple links with that text!", and you would be
+correct!  If you use the plural form, you can access the list.
+If you wanted to click on the second news link, you could do this:
+
+  agent.page.links_with(:text => 'News')[1].click
+
+We can even find a link with a certain href like so:
+
+  page.link_with(:href => '/something')
+
+Or chain them together to find a link with certain text and certain href:
+
+  page.link_with(:text => 'News', :href => '/something')
+
+These shortcuts that mechanize provides are available on any list that you
+can fetch like frames, iframes, or forms.  Now that we know how to find and
+click links, lets try something more complicated like filling out a form.
+
+== Filling Out Forms
+
+Lets continue with our google example.  Here's the code we have so far:
+  require 'rubygems'
+  require 'mechanize'
+
+  agent = Mechanize.new
+  page = agent.get('http://google.com/')
+
+If we pretty print the page, we can see that there is one form named 'f',
+that has a couple buttons and a few fields:
+
+  pp page
+
+Now that we know the name of the form, lets fetch it off the page:
+
+ google_form = page.form('f')
+
+Mechanize lets you access form input fields in a few different ways, but the
+most convenient is that you can access input fields as accessors on the
+object.  So lets set the form field named 'q' on the form to 'ruby mechanize':
+
+  google_form.q = 'ruby mechanize'
+
+To make sure that we set the value, lets pretty print the form, and you should
+see a line similar to this:
+
+  #<Mechanize::Field:0x1403488 @name="q", @value="ruby mechanize">
+
+If you saw that the value of 'q' changed, you're on the right track!  Now we
+can submit the form and 'press' the submit button and print the results:
+
+  page = agent.submit(google_form, google_form.buttons.first)
+  pp page
+
+What we just did was equivalent to putting text in the search field and
+clicking the 'Google Search' button.  If we had submitted the form without
+a button, it would be like typing in the text field and hitting the return
+button.
+
+Lets take a look at the code all together:
+
+  require 'rubygems'
+  require 'mechanize'
+
+  agent = Mechanize.new
+  page = agent.get('http://google.com/')
+  google_form = page.form('f')
+  google_form.q = 'ruby mechanize'
+  page = agent.submit(google_form)
+  pp page
+
+Before we go on to screen scraping, lets take a look at forms a little more
+in depth.  Unless you want to skip ahead!
+
+== Advanced Form Techniques
+
+In this section, I want to touch on using the different types in input fields
+possible with a form.  Password and textarea fields can be treated just like
+text input fields.  Select fields are very similar to text fields, but they
+have many options associated with them.  If you select one option, mechanize
+will deselect the other options (unless it is a multi select!).
+
+For example, lets select an option on a list:
+
+  form.field_with(:name => 'list').options[0].select
+
+Now lets take a look at checkboxes and radio buttons.  To select a checkbox,
+just check it like this:
+
+  form.checkbox_with(:name => 'box').check
+
+Radio buttons are very similar to checkboxes, but they know how to uncheck
+other radio buttons of the same name.  Just check a radio button like you
+would a checkbox:
+
+ form.radiobuttons_with(:name => 'box')[1].check
+
+Mechanize also makes file uploads easy!  Just find the file upload field, and
+tell it what file name you want to upload:
+
+  form.file_uploads.first.file_name = "somefile.jpg"
+
+== Scraping Data
+
+Mechanize uses nokogiri[http://nokogiri.org/] to parse
+html.  What does this mean for you?  You can treat a mechanize page like
+an nokogiri object.  After you have used Mechanize to navigate to the page
+that you need to scrape, then scrape it using nokogiri methods:
+
+  agent.get('http://someurl.com/').search(".//p[@class='posted']")
+

data/LICENSE.rdoc

@@ -0,0 +1,20 @@
+(The MIT License)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,159 @@
+.autotest
+CHANGELOG.rdoc
+EXAMPLES.rdoc
+FAQ.rdoc
+GUIDE.rdoc
+LICENSE.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+examples/flickr_upload.rb
+examples/mech-dump.rb
+examples/proxy_req.rb
+examples/rubyforge.rb
+examples/spider.rb
+lib/mechanize.rb
+lib/mechanize/content_type_error.rb
+lib/mechanize/cookie.rb
+lib/mechanize/cookie_jar.rb
+lib/mechanize/download.rb
+lib/mechanize/element_matcher.rb
+lib/mechanize/file.rb
+lib/mechanize/file_connection.rb
+lib/mechanize/file_request.rb
+lib/mechanize/file_response.rb
+lib/mechanize/file_saver.rb
+lib/mechanize/form.rb
+lib/mechanize/form/button.rb
+lib/mechanize/form/check_box.rb
+lib/mechanize/form/field.rb
+lib/mechanize/form/file_upload.rb
+lib/mechanize/form/hidden.rb
+lib/mechanize/form/image_button.rb
+lib/mechanize/form/keygen.rb
+lib/mechanize/form/multi_select_list.rb
+lib/mechanize/form/option.rb
+lib/mechanize/form/radio_button.rb
+lib/mechanize/form/reset.rb
+lib/mechanize/form/select_list.rb
+lib/mechanize/form/submit.rb
+lib/mechanize/form/text.rb
+lib/mechanize/form/textarea.rb
+lib/mechanize/headers.rb
+lib/mechanize/history.rb
+lib/mechanize/http.rb
+lib/mechanize/http/agent.rb
+lib/mechanize/http/auth_challenge.rb
+lib/mechanize/http/auth_realm.rb
+lib/mechanize/http/content_disposition_parser.rb
+lib/mechanize/http/www_authenticate_parser.rb
+lib/mechanize/monkey_patch.rb
+lib/mechanize/page.rb
+lib/mechanize/page/base.rb
+lib/mechanize/page/frame.rb
+lib/mechanize/page/image.rb
+lib/mechanize/page/label.rb
+lib/mechanize/page/link.rb
+lib/mechanize/page/meta_refresh.rb
+lib/mechanize/parser.rb
+lib/mechanize/pluggable_parsers.rb
+lib/mechanize/redirect_limit_reached_error.rb
+lib/mechanize/redirect_not_get_or_head_error.rb
+lib/mechanize/response_code_error.rb
+lib/mechanize/response_read_error.rb
+lib/mechanize/robots_disallowed_error.rb
+lib/mechanize/test_case.rb
+lib/mechanize/unauthorized_error.rb
+lib/mechanize/unsupported_scheme_error.rb
+lib/mechanize/util.rb
+test/data/htpasswd
+test/data/server.crt
+test/data/server.csr
+test/data/server.key
+test/data/server.pem
+test/htdocs/alt_text.html
+test/htdocs/bad_form_test.html
+test/htdocs/button.jpg
+test/htdocs/canonical_uri.html
+test/htdocs/dir with spaces/foo.html
+test/htdocs/empty_form.html
+test/htdocs/file_upload.html
+test/htdocs/find_link.html
+test/htdocs/form_multi_select.html
+test/htdocs/form_multival.html
+test/htdocs/form_no_action.html
+test/htdocs/form_no_input_name.html
+test/htdocs/form_order_test.html
+test/htdocs/form_select.html
+test/htdocs/form_set_fields.html
+test/htdocs/form_test.html
+test/htdocs/frame_referer_test.html
+test/htdocs/frame_test.html
+test/htdocs/google.html
+test/htdocs/index.html
+test/htdocs/link with space.html
+test/htdocs/meta_cookie.html
+test/htdocs/no_title_test.html
+test/htdocs/noindex.html
+test/htdocs/rails_3_encoding_hack_form_test.html
+test/htdocs/relative/tc_relative_links.html
+test/htdocs/robots.html
+test/htdocs/robots.txt
+test/htdocs/tc_bad_charset.html
+test/htdocs/tc_bad_links.html
+test/htdocs/tc_base_link.html
+test/htdocs/tc_blank_form.html
+test/htdocs/tc_charset.html
+test/htdocs/tc_checkboxes.html
+test/htdocs/tc_encoded_links.html
+test/htdocs/tc_field_precedence.html
+test/htdocs/tc_follow_meta.html
+test/htdocs/tc_form_action.html
+test/htdocs/tc_links.html
+test/htdocs/tc_meta_in_body.html
+test/htdocs/tc_pretty_print.html
+test/htdocs/tc_referer.html
+test/htdocs/tc_relative_links.html
+test/htdocs/tc_textarea.html
+test/htdocs/test_click.html
+test/htdocs/unusual______.html
+test/test_mechanize.rb
+test/test_mechanize_cookie.rb
+test/test_mechanize_cookie_jar.rb
+test/test_mechanize_download.rb
+test/test_mechanize_file.rb
+test/test_mechanize_file_connection.rb
+test/test_mechanize_file_request.rb
+test/test_mechanize_file_saver.rb
+test/test_mechanize_form.rb
+test/test_mechanize_form_check_box.rb
+test/test_mechanize_form_encoding.rb
+test/test_mechanize_form_field.rb
+test/test_mechanize_form_file_upload.rb
+test/test_mechanize_form_image_button.rb
+test/test_mechanize_form_keygen.rb
+test/test_mechanize_form_multi_select_list.rb
+test/test_mechanize_form_option.rb
+test/test_mechanize_form_radio_button.rb
+test/test_mechanize_form_select_list.rb
+test/test_mechanize_form_textarea.rb
+test/test_mechanize_headers.rb
+test/test_mechanize_history.rb
+test/test_mechanize_http_agent.rb
+test/test_mechanize_http_auth_challenge.rb
+test/test_mechanize_http_auth_realm.rb
+test/test_mechanize_http_content_disposition_parser.rb
+test/test_mechanize_http_www_authenticate_parser.rb
+test/test_mechanize_link.rb
+test/test_mechanize_page.rb
+test/test_mechanize_page_encoding.rb
+test/test_mechanize_page_frame.rb
+test/test_mechanize_page_link.rb
+test/test_mechanize_page_meta_refresh.rb
+test/test_mechanize_parser.rb
+test/test_mechanize_pluggable_parser.rb
+test/test_mechanize_redirect_limit_reached_error.rb
+test/test_mechanize_redirect_not_get_or_head_error.rb
+test/test_mechanize_subclass.rb
+test/test_mechanize_util.rb
+test/test_multi_select.rb