frameworks-capybara 0.3.0.rc5 → 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NGQ0MTAwMGMxNjgxNjc0MWZkYjdmNTBiYWJkOWM0N2MwMjRlNWY0Yw==
+  data.tar.gz: !binary |-
+    ODdmYmQyZWE1NGUxODJmN2ZmZTI3YTNhNjI0NGExYzE0NGY3NWI3ZA==
+SHA512:
+  metadata.gz: !binary |-
+    OTQxYjczYTljMThkMWVlZGRjNTMyNTJmZjQ2ZmY2NDQwMzhmM2NkNDMyNzg3
+    YWI2NjAyZmZmMmFjYzU2ZjIwYjk4YjNhNWZhYTBiNWJlNWNmZGU0ZWZlMDlh
+    MDkzZWRiOGU4NjBmMDQ3ODZkZmQxMTVkNTY5YTQ0ODcwZDUyMTk=
+  data.tar.gz: !binary |-
+    Mjc1NWU3MzIxY2MzNzI0NzFjMzc5MTkzZmMyNDU4MTg1ZTk1YWRlOTA5ZWFk
+    Y2NlMThhMmZhMTg0ODE2YWMzYzQ4NWQyNmViYTdmMzkyZmQ2N2U1NDJjMDI2
+    OWUyYjU0ZDMyOGYyNDEyNmJiMmZkYjJkNjBjZjAxN2FmZmI2NTk=

data/.travis.yml

@@ -3,10 +3,6 @@ before_install:
  - sudo apt-get install xvfb
 language: ruby
 rvm:
-- 1.8.7
-- 1.9.2
 - 1.9.3
-- jruby-18mode # JRuby in 1.8 mode
-- jruby-19mode # JRuby in 1.9 mode
-# uncomment this line if your project needs to run something other than `rake`:
-# # script: bundle exec rspec spec
+- 2.0.0
+- 2.1.0

data/CHANGES

@@ -0,0 +1,52 @@
+1.0.0
+Base required Ruby version now 1.9.3 or later
+Bumped gem dependency versions
+Updated Travis build lists to include Ruby 2.0, 2.1
+
+0.2.34
+Improved validate_online method to disregard system/OS errors.
+
+0.2.33
+Specify different runtime dependencies depending on the ruby version.
+
+0.2.32
+Fix Ruby 1.9 RSpec test failures introduced in release 0.2.31
+
+0.2.31
+Added Firefox profile certificate handling via the FIREFOX_CERT_PATH and FIREFOX_CERT_PREFIX variables.
+
+0.2.30
+Fixed rake profiles listings to work with Ruby 1.9+
+
+0.2.29
+Updated add_cookie patch to use new http_cookie API, following Mechanize changes (see http://bit.ly/12Do1l7)
+
+0.2.28
+Updated Mechanize::CookieJar add patch to work with latest capybara-mechanize.
+Added support for httpOnly cookie flag to capybara mechanize patch
+
+0.2.27
+Can now specify arguments passed to the browser via the BROWSER_CLI_ARGS environment variable. 
+
+0.2.25
+Modified cucumber html report monkey patch to work with cucumber >= 1.3.0
+
+0.2.24
+Added FW_BBC_DOMAIN param in order to set the top level domain used for requests.
+
+0.2.23
+Updated capybara-mechanize patch to send referer header.
+
+0.2.22
+Update Centos6 sandbox DNS to sandbox.bbc.co.uk.
+
+0.2.21
+Add support for Centos6 sandbox.
+Update Gemfile.lock to latest gem versions.
+
+0.2.20
+validate_online() now takes optional hash of parameters to pass to MarkupValidator.
+Updated proxy handling to pick up port and to work without protocol or port. 
+
+0.2.19
+Present production release.

data/Gemfile

@@ -1,7 +1,9 @@
-source :rubygems
+source 'https://rubygems.org'
 
 platforms :jruby do
   gem "jruby-openssl"
 end
 
+gem "rspec-mocks", "<2.14.0"
+
 gemspec

data/Gemfile.lock

@@ -1,22 +1,25 @@
 PATH
   remote: .
   specs:
-    frameworks-capybara (0.3.0.rc5)
-      capybara (>= 1.0.0)
+    frameworks-capybara (1.0.0)
+      capybara (~> 1.1.2)
       capybara-celerity
-      capybara-mechanize (>= 0.3.0)
-      capybara-webkit
+      capybara-mechanize (>= 0.3.0, < 1.0.0)
       cucumber (>= 0.10.5)
       headless
       json
+      mechanize (= 2.7.2)
+      mime-types
+      nokogiri
+      rubyzip
+      selenium-webdriver
       w3c_validators
 
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
-    addressable (2.2.8)
-    builder (3.0.0)
-    capybara (1.1.2)
+    builder (3.2.2)
+    capybara (1.1.4)
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
@@ -29,66 +32,69 @@ GEM
     capybara-mechanize (0.3.0)
       capybara (~> 1.1)
       mechanize (~> 2.3)
-    capybara-webkit (0.12.1)
-      capybara (>= 1.0.0, < 1.2)
-      json
     celerity (0.9.2)
-    childprocess (0.3.3)
-      ffi (~> 1.0.6)
-    cucumber (1.2.1)
+    childprocess (0.5.1)
+      ffi (~> 1.0, >= 1.0.11)
+    cucumber (1.3.10)
       builder (>= 2.1.2)
       diff-lcs (>= 1.1.3)
-      gherkin (~> 2.11.0)
-      json (>= 1.4.6)
-    diff-lcs (1.1.3)
-    domain_name (0.5.3)
-      unf (~> 0.0.3)
-    ffi (1.0.11)
-    gherkin (2.11.1)
-      json (>= 1.4.6)
-    headless (0.3.1)
-    json (1.7.3)
-    libwebsocket (0.1.4)
-      addressable
-    mechanize (2.5.1)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.0.2)
+    diff-lcs (1.2.5)
+    domain_name (0.5.16)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.9.3)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    headless (1.0.1)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.1)
+    mechanize (2.7.2)
       domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0.0)
       mime-types (~> 1.17, >= 1.17.2)
       net-http-digest_auth (~> 1.1, >= 1.1.1)
       net-http-persistent (~> 2.5, >= 2.5.2)
       nokogiri (~> 1.4)
       ntlm-http (~> 0.1, >= 0.1.1)
-      webrobots (~> 0.0, >= 0.0.9)
-    mime-types (1.19)
-    multi_json (1.3.6)
-    net-http-digest_auth (1.2.1)
-    net-http-persistent (2.7)
-    nokogiri (1.5.5)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (1.25.1)
+    mini_portile (0.5.2)
+    multi_json (1.8.4)
+    multi_test (0.0.3)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
     ntlm-http (0.1.1)
-    rack (1.4.1)
-    rack-test (0.6.1)
+    rack (1.5.2)
+    rack-test (0.6.2)
       rack (>= 1.0)
-    rake (0.9.2.2)
-    rspec (2.10.0)
-      rspec-core (~> 2.10.0)
-      rspec-expectations (~> 2.10.0)
-      rspec-mocks (~> 2.10.0)
-    rspec-core (2.10.1)
-    rspec-expectations (2.10.0)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.10.1)
-    rubyzip (0.9.9)
-    selenium-webdriver (2.24.0)
-      childprocess (>= 0.2.5)
-      libwebsocket (~> 0.1.3)
+    rake (10.1.1)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    rubyzip (1.1.0)
+    selenium-webdriver (2.40.0)
+      childprocess (>= 0.5.0)
       multi_json (~> 1.0)
-      rubyzip
-    unf (0.0.5)
+      rubyzip (~> 1.0)
+      websocket (~> 1.0.4)
+    unf (0.1.3)
       unf_ext
-    unf_ext (0.0.5)
+    unf_ext (0.0.6)
     w3c_validators (1.2)
       json
       nokogiri
-    webrobots (0.0.13)
+    webrobots (0.1.1)
+    websocket (1.0.7)
     xpath (0.1.4)
       nokogiri (~> 1.3)
 
@@ -100,3 +106,4 @@ DEPENDENCIES
   jruby-openssl
   rake
   rspec (>= 1.0.0)
+  rspec-mocks (< 2.14.0)

data/README.rdoc

@@ -1,4 +1,4 @@
-= Build Status {<img src="https://secure.travis-ci.org/mcrmfc/frameworks-capybara.png"/>}[https://secure.travis-ci.org/mcrmfc/frameworks-capybara]
+= Build Status {<img src="https://secure.travis-ci.org/bbc-frameworks/frameworks-capybara.png"/>}[https://secure.travis-ci.org/bbc-frameworks/frameworks-capybara]
 
 = frameworks-capybara
 
@@ -27,7 +27,7 @@ In your env.rb simply add:
 
 The following environment variables can be set to configure your tests:
 
-  ENVIRONMENT - this must be one of either 'sandbox', 'int', 'test', 'stage', 'live'
+  ENVIRONMENT - this must be one of either 'sandbox', 'sandbox6', 'int', 'test', 'stage', 'live'
   BROWSER - this must be one of either 'ie', 'firefox', 'chrome', 'headless', 'remote'
   HTTP_PROXY - url of proxy if required e.g. 'http://proxyhost:80', when running a headless browser this sets the proxy for the browser itself, when running a remote browser this sets the proxy for the client which will connect to the remote selenium server
   PLATFORM - used when specifying remote test on a grid that provides a choice of platforms, this must be one of either 'WINDOWS' or 'LINUX'
@@ -37,11 +37,16 @@ The following environment variables can be set to configure your tests:
   REMOTE_URL - URL of remote Selenium-Webdriver server e.g. http://yourremotehost:4444/wd/hub
   FIREFOX_PROFILE - specify a firefox profile to use when running in-browser tests (local or remote)
   FIREFOX_PREFS - specify a json string of additional preferences e.g. FIREFOX_PREFS='{"javascript.enabled": false}'
+  FIREFOX_CERT_PATH - specify optional directory containing Firefox profile certificate (.db) files
+  FIREFOX_CERT_PREFIX - specify optional prefix for locating Firefox profile certificate files (e.g. <prefix>-<file>.db)
   CREATE_NEW_FF_PROFILE - create a new Firefox profile with name provided by FIREFOX_PROFILE 
   CELERITY_JS_ENABLED (string - 'true', 'false') - determines whether Celerity (HTMLUnit) attempts to execute javascript 
   XVFB - (string - 'true', 'false') - determines whether XVFB is used to run browser (i.e. headless Firefox on *nix platform)
   FW_CERT_LOCATION - path to client certificate (combined pem)
   CHROME_SWITCHES - pass in any allowed switches for the Selenium chrome driver (http://peter.sh/experiments/chromium-command-line-switches/) e.g. CHROME_SWITCHES="--user-agent=Mozilla/5.0 (PLAYSTATION 3; 3.55)"
+  PROJECT_NAME - optional project name description which gets displayed in Cucumber html reports 
+  TEAM_NAME - optional team name description which gets displayed in Cucumber html reports 
+  BROWSER_CLI_ARGS - optional additional arguments to pass to local browser processes
 
 Here is a sample cucumber.yml:
 
@@ -74,15 +79,25 @@ This gem also sets up a number of 'base urls' for you to use in your tests, thes
   @base_static_url
   @base_open_url
 
-Finally this gem contains some useful monkey-patches to core libraries e.g. Capybara, Selenium-Webdriver and Cucumber.
+Finally this gem contains some useful monkey-patches to core libraries e.g. Capybara-Mechanize, Selenium-Webdriver and Cucumber.
 
 == Release instructions
 
 * Run tests - 'rake'
+* If you have not published with 'gem' before, run 'gem push' to enter your credentials (ignore failed publish error at the end)
 * Bump version - manually update file in lib/version.rb
-* Run 'bundle install' to pick up new version in Gemfile
+* Update the CHANGES[https://github.com/bbc-frameworks/frameworks-capybara/blob/master/CHANGES] log (see file for convention).
+* Run 'bundle install --binstubs --path vendor' to pick up new version in Gemfile
+* Commit 'Gemfile.lock' changes
 * Release - 'rake release'
 
+You should see something like:
+
+  frameworks-capybara 0.2.23 built to pkg/frameworks-capybara-0.2.23.gem
+  Tagged v0.2.23
+  Pushed git commits and tags
+  Pushed frameworks-capybara 0.2.23 to rubygems.org
+
 == Contributing to frameworks-capybara
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/frameworks-capybara.gemspec

@@ -20,9 +20,13 @@ Gem::Specification.new do |s|
 
   s.files         = `git ls-files`.split("\n")
 
-  s.add_runtime_dependency("capybara", [">=1.0.0"])
-  s.add_runtime_dependency("capybara-mechanize", [">=0.3.0"])
-  s.add_runtime_dependency("capybara-webkit")
+  s.add_runtime_dependency("mime-types")
+  s.add_runtime_dependency("nokogiri")
+  s.add_runtime_dependency("rubyzip")
+  s.add_runtime_dependency("selenium-webdriver")
+  s.add_runtime_dependency("capybara", ["~>1.1.2"])  
+  s.add_runtime_dependency("mechanize", ["2.7.2"])
+  s.add_runtime_dependency("capybara-mechanize", [">=0.3.0"], ["<1.0.0"])
   s.add_runtime_dependency("json")
   s.add_runtime_dependency("headless")
   s.add_runtime_dependency("capybara-celerity")

data/lib/frameworks/capybara.rb

@@ -1,7 +1,6 @@
 require 'capybara/cucumber'
 require 'monkey-patches/webdriver-patches'
 require 'monkey-patches/capybara-patches'
-require 'monkey-patches/capybara-webkit-patches'
 require 'monkey-patches/capybara-mechanize-patches'
 require 'monkey-patches/mechanize-patches'
 require 'monkey-patches/send-keys'
@@ -9,7 +8,6 @@ require 'monkey-patches/net-http-persistent-patches'
 require 'selenium-webdriver'
 require 'capybara/mechanize/cucumber' 
 require 'capybara/celerity'
-require 'capybara-webkit'
 
 class CapybaraSetup
 
@@ -27,19 +25,25 @@ class CapybaraSetup
       :proxy_on => ENV['PROXY_ON'],
       :url => ENV['REMOTE_URL'],
       :chrome_switches => ENV['CHROME_SWITCHES'],
-      :firefox_prefs => ENV['FIREFOX_PREFS']}
+      :firefox_prefs => ENV['FIREFOX_PREFS'],
+      :args => ENV['BROWSER_CLI_ARGS']
+    }
 
     selenium_remote_opts = {:platform => ENV['PLATFORM'],
       :browser_name => ENV['REMOTE_BROWSER'],
       :version => ENV['REMOTE_BROWSER_VERSION'],
-      :url => ENV['REMOTE_URL']}
+      :url => ENV['REMOTE_URL']
+    }
 
     custom_opts = {:job_name => ENV['SAUCE_JOB_NAME'],
-      :max_duration => ENV['SAUCE_MAX_DURATION']}
+      :max_duration => ENV['SAUCE_MAX_DURATION'],
+      :firefox_cert_path => ENV['FIREFOX_CERT_PATH'],
+      :firefox_cert_prefix => ENV['FIREFOX_CERT_PREFIX']
+    }
 
     validate_env_vars(capybara_opts.merge(selenium_remote_opts)) #validate environment variables set using cucumber.yml or passed via command line
 
-    @proxy_host =  capybara_opts[:proxy].gsub(/http:\/\//,'').gsub(/:80/,'') unless capybara_opts[:proxy].nil?
+    @proxy_host =  capybara_opts[:proxy].gsub(/http:\/\//,'').gsub(/:80.*/,'') unless capybara_opts[:proxy].nil?
     capybara_opts[:browser] = capybara_opts[:browser].intern #update :browser value to be a symbol, required for Selenium
     selenium_remote_opts[:browser_name] = selenium_remote_opts[:browser_name].intern if selenium_remote_opts[:browser_name]#update :browser value to be a symbol, required for Selenium
 
@@ -55,8 +59,6 @@ class CapybaraSetup
       @driver = register_celerity_driver(capybara_opts)
     when :mechanize then
       @driver = register_mechanize_driver(capybara_opts)
-    when :webkit then
-      @driver = register_webkit_driver(capybara_opts)
     else
       @driver = register_selenium_driver(capybara_opts, selenium_remote_opts, custom_opts)
     end
@@ -82,11 +84,39 @@ class CapybaraSetup
     end
   end
 
+  # WARNING: This modifies the Firefox profile passed in the parameters
+  def update_firefox_profile_with_certificates(profile, certificate_path, certificate_prefix = '')
+    profile_path = profile.layout_on_disk
+    
+    # Create links to the certificate files in the profile directory
+    ['cert8.db', 'key3.db', 'secmod.db'].each do |cert_file|
+      source_file = "#{certificate_prefix}#{cert_file}"
+      source_path = "#{certificate_path}" + File::SEPARATOR + source_file
+      dest_path = profile_path + File::SEPARATOR + cert_file
+      if(! File.exist?(source_path))
+        raise "Firefox cert db file #{source_path} does not exist."
+      end
+      FileUtils.cp(source_path, dest_path)
+    end 
+
+    # Force the certificates to get pulled into the profile
+    profile = Selenium::WebDriver::Firefox::Profile.new(profile_path)
+
+    # Avoid Firefox certificate alerts
+    profile["security.default_personal_cert"] = 'Select Automatically'
+    
+    return profile
+  end
+
   def register_selenium_driver(opts,remote_opts,custom_opts)
     Capybara.register_driver :selenium do |app|
 
       if opts[:firefox_prefs] || opts[:profile]
         opts[:profile] = create_profile(opts[:profile], opts[:firefox_prefs])
+
+        if custom_opts[:firefox_cert_path]
+          opts[:profile] = update_firefox_profile_with_certificates(opts[:profile], custom_opts[:firefox_cert_path], custom_opts[:firefox_cert_prefix])
+        end
       end
 
       opts[:switches] = [opts.delete(:chrome_switches)] if(opts[:chrome_switches])
@@ -128,7 +158,7 @@ class CapybaraSetup
     elsif(profile_name == 'BBC_INTERNAL')
       profile = Selenium::WebDriver::Firefox::Profile.new
       profile["network.proxy.type"] = 1
-      profile["network.proxy.no_proxies_on"] = "*.sandbox.dev.bbc.co.uk"
+      profile["network.proxy.no_proxies_on"] = "*.sandbox.dev.bbc.co.uk,*.sandbox.bbc.co.uk"
       profile["network.proxy.http"] = @proxy_host 
       profile["network.proxy.ssl"] = @proxy_host 
       profile["network.proxy.http_port"] = 80
@@ -166,13 +196,6 @@ class CapybaraSetup
     :mechanize
   end
 
-  def register_webkit_driver(opts)
-    Capybara.register_driver :webkit do |app|
-      Capybara::Driver::Webkit.new(app)
-    end
-    Capybara.javascript_driver = :webkit
-    :webkit
-  end
 
   def clean_opts(opts, *args)
     args.each do |arg|

data/lib/frameworks/cucumber.rb

@@ -3,7 +3,7 @@ require 'w3c_validators'
 require 'monkey-patches/cucumber-patches'
 
 #This is hackish but means we only run once in cucumber and not every scenario
-if(ENV['XVFB']=='true' || ENV['BROWSER'] == 'webkit') 
+if(ENV['XVFB']=='true')
   puts "You have chosen to use XVFB - ensure you have yum installed Xvfb Xorg and firefox"
   require 'headless'
   headless = Headless.new
@@ -27,8 +27,15 @@ module Frameworks
         @pal_base_url = @sandbox + @bbc_domain 
         @ssl_base_url = @sslsandbox + @bbc_domain
         @static_base_url = @static_sandbox + @bbc_domain
-        @mobile_base_url = @mobiledot_prefix + "sandbox.dev.bbc.co.uk"
-        @m_base_url = @mdot_prefix + "sandbox.dev.bbc.co.uk"
+        @mobile_base_url = @mobiledot_prefix + "sandbox.dev" + @bbc_domain
+        @m_base_url = @mdot_prefix + "sandbox.dev" + @bbc_domain
+      elsif(environment =='sandbox6')
+        @base_url = @sandbox6 + @bbc_domain 
+        @pal_base_url = @sandbox6 + @bbc_domain 
+        @ssl_base_url = @sslsandbox6 + @bbc_domain
+        @static_base_url = @static_sandbox6 + @bbc_domain
+        @mobile_base_url = @mobiledot_prefix + "sandbox" + @bbc_domain
+        @m_base_url = @mdot_prefix + "sandbox" + @bbc_domain
       elsif (environment =='live' && ENV['WWW_LIVE']=='false')
         @base_url = @www_prefix.chop + @bbc_domain
         @pal_base_url = @pal_prefix + environment + @bbc_domain
@@ -49,12 +56,24 @@ module Frameworks
       end
 
       proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] 
-      @proxy_host = proxy.scan(/http:\/\/(.*):80/)[0][0] if proxy
+      proxy_parts = proxy.scan(/(?:http:\/\/)?([^\:]+)(?::(\d+))?/) if proxy && !proxy.empty?
+      if proxy_parts && !proxy_parts.empty?
+          @proxy_host = proxy_parts[0][0]
+          if proxy_parts[0][1]
+              @proxy_port = proxy_parts[0][1]
+          else
+              @proxy_port = "80"
+          end
+      end
     end
 
-    def validate_online(src)
+    def validate_online(src, validator_args = nil)
 
-      @validator = MarkupValidator.new({:proxy_host => @proxy_host,:proxy_port => 80})
+      args = {:proxy_host => @proxy_host,:proxy_port => @proxy_port} 
+      if(validator_args != nil)
+         args = args.merge(validator_args)
+      end    
+      @validator = MarkupValidator.new(args)
 
       @validator.set_doctype!(:xhtml)
       begin
@@ -65,11 +84,11 @@ module Frameworks
           results.errors.each do |err|
             puts err.to_s
           end
-          raise "W3C Validation of " + current_url + " failed."
+          raise "W3C Validation failed."
         end
 
-      rescue Net::HTTPFatalError => e
-        puts "WARNING - OUTGOING NETWORK ERROR FROM FORGE TO W3C - Validation Not Performed"
+      rescue SystemCallError => e
+        puts "System error whilst performing request to W3C: #{e}"  
       end
     end
 
@@ -80,13 +99,41 @@ module Frameworks
       @ssl_prefix = "https://ssl."
       @static_prefix = "#{scheme}://static."
       @open_prefix = "#{scheme}://open."
-      @bbc_domain = '.bbc.co.uk'
+      @bbc_domain = '.' + (ENV['FW_BBC_DOMAIN'] || 'bbc.co.uk')
       @bbci_domain = '.bbci.co.uk'
       @sandbox = "#{scheme}://pal.sandbox.dev"
+      @sandbox6 = "#{scheme}://sandbox"
       @mobiledot_prefix = "#{scheme}://mobile."
       @mdot_prefix = "#{scheme}://m."
       @sslsandbox = "https://ssl.sandbox.dev"
+      @sslsandbox6 = "https://ssl.sandbox"
       @static_sandbox = "#{scheme}://static.sandbox.dev"
+      @static_sandbox6 = "#{scheme}://static.sandbox"      
+    end
+
+    def setup_mechanize(agent, http_proxy=nil)
+      http_proxy = http_proxy || ENV['HTTP_PROXY'] || ENV['http_proxy']
+
+      if ENV['FW_CERT_LOCATION']
+        agent.cert, agent.key = ENV['FW_CERT_LOCATION'], ENV['FW_CERT_LOCATION'] 
+      end
+
+      agent.ca_file = ENV['CA_CERT_LOCATION'] if ENV['CA_CERT_LOCATION']
+      agent.set_proxy(http_proxy.scan(/http:\/\/(.*):80/)[0][0].to_s,80) if http_proxy && !http_proxy.empty?
+
+      #This is necessary because Mech2 does not ship with root certs like Mech1 did and boxes may not have the OpenSSL set installed
+      agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+      # This prevents Mechanize from raising a Mechanize::ResponseCodeError
+      # when the HTTP Response Code is 404 or 503. This lets capybara continue the journey.
+      agent.agent.allowed_error_codes = ['404', '503']
+    end
+
+    def new_mechanize(http_proxy=nil)
+      require 'mechanize'
+      agent = Mechanize.new
+      setup_mechanize(agent, http_proxy)
+      agent
     end
 
   end #EnvHelper
@@ -100,29 +147,7 @@ World(Frameworks::EnvHelper)
 Before do
   #This is ugly but unavoidable since Capybara::RackTest::Driver.reset_host! does @browser = nil and wipes all brower level settings
   #it was either this or a monkey patch - need to think about pushing a softer reset change to capybara-mechanize to override this
-  http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
-  if page.driver.class == Capybara::Mechanize::Driver
-
-    if ENV['FW_CERT_LOCATION']
-      page.driver.browser.agent.cert, page.driver.browser.agent.key = 
-        ENV['FW_CERT_LOCATION'],
-        ENV['FW_CERT_LOCATION'] 
-    end
-
-    page.driver.browser.agent.ca_file = ENV['CA_CERT_LOCATION'] if ENV['CA_CERT_LOCATION']
-
-    page.driver.browser.agent.set_proxy(http_proxy.scan(/http:\/\/(.*):80/).to_s,80) if http_proxy
-
-    #This is necessary because Mech2 does not ship with root certs like Mech1 did and boxes may not have the OpenSSL set installed
-    page.driver.browser.agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
-  end
-
-  #TODO: See if this can be set at the driver level rather than in before do - and only want to start headless once
-  if page.driver.class == Capybara::Driver::Webkit
-    page.driver.browser.ignore_ssl_errors #shouldn't have to do this on test but for some reason do
-    page.driver.browser.set_proxy(:host => http_proxy.scan(/http:\/\/(.*):80/).to_s,:port => '80') if http_proxy
-  end
-  
+  setup_mechanize(page.driver.browser.agent) if page.driver.class == Capybara::Mechanize::Driver
 
   generate_base_urls
 end

data/lib/monkey-patches/capybara-mechanize-patches.rb

@@ -2,6 +2,7 @@ require 'capybara/mechanize/cucumber'
 require 'uri'
 class Capybara::Mechanize::Browser
   #patch to remove catching all Mechanize exceptions (which are nice and specific) and throwing a useless RuntimeError
+  #patch to add Referer (Mechanize@0.3.0 won't add Referer for urls starting with http(s)://.)
   def process_remote_request(method, url, attributes, headers)
     if remote?(url)
       uri = URI.parse(url)
@@ -9,11 +10,23 @@ class Capybara::Mechanize::Browser
       @last_remote_uri = uri
       url = uri.to_s
 
+      referer = nil
+      referer_candidate = current_url
+      unless referer_candidate.empty? or (referer_candidate.start_with?('https://') and url.start_with?('http://'))
+        referer = referer_candidate
+      end
+
       reset_cache!
       args = []
       args << attributes unless attributes.empty?
       args << headers unless headers.empty?
-      @agent.send(method, url, *args)
+
+      if method == :get
+        agent.send(method, url, attributes, referer, headers)
+      else
+        agent.send(method, url, *args)
+      end
+
       @last_request_remote = true
     end
   end
@@ -27,12 +40,14 @@ class Capybara::Mechanize::Driver
     browser.agent.cookie_jar.jar.each do |domain|
       domain[1].each do |path|
         path[1].each do |cookie|
+          
           cookies.push({
             :name => cookie[1].name,
             :value => cookie[1].value,
             :domain => cookie[1].domain,
             :secure => cookie[1].secure,
             :expires => cookie[1].expires,
+            :httponly => cookie[1].httponly,
             :path => cookie[1].path
           })
         end
@@ -67,14 +82,20 @@ class Capybara::Mechanize::Driver
     end
   end
 
-  FakeURI = Struct.new(:host)
   def add_cookie(attribs)
-    c = Mechanize::Cookie.new(attribs[:name],attribs[:value])
-    # remember: mechanize always removes leading '.' from domains
-    c.domain = attribs[:domain]
-    c.path = '/'
-    c.expires = attribs[:expires]
-    c.secure = attribs[:secure]
-    browser.agent.cookie_jar.add(FakeURI.new(c.domain),c)
+    
+    cookie = HTTP::Cookie.new(
+      attribs[:name],
+      attribs[:value],
+      :domain => attribs[:domain],
+      :for_domain => true,
+      :path => '/',
+      :expires => attribs[:expires],
+      :secure => attribs[:secure],
+      :httponly => attribs[:httponly]
+    )
+
+    browser.agent.cookie_jar.add(cookie)
+
   end
 end