puffing-billy 2.1.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6e9bb7a75fe76a33e705c89f01e30fa987d1df3f4cfee135bb53393f28a4398
-  data.tar.gz: 47e189ad311796a8a192479fc60034d2651f1879d8e17c42afb174f6296a491f
+  metadata.gz: 98ce893c9461f4f8c8eedd69eee46870634f789dd03a2007370bb7b03361bc26
+  data.tar.gz: 35f7939ff8eb3cbccd4bf4e91ccefbddd1b4b999fddfbc86ec0ff82885ae0dad
 SHA512:
-  metadata.gz: 21e38be3cbfe954494685117ec5828cc989b9d2997170f33bfe08e2259bfdaec52e3468e7483db6a8883f5505b06dd7b7a5c1d8ccfe81447ab866d5da94ef323
-  data.tar.gz: 8fc3ba732c1ab60daa5599c0eb77e3c02d4a5a083b43941680e2798898ae6b11addd0f0060de6aa5cabb1c7f5cd2b598ca5d485fe6019da0e5ca12549a46037c
+  metadata.gz: 73d5bc111493abd8a2bf9157f9bc3b717f0d4bc2a9276a757280d6e6ad7b3e5a1acd90a30234e00bdd9245380f6d9c00aa9bb86c3ec6bd7eed46643af1b85451
+  data.tar.gz: a32ac78dec161016266bf1ac2d2cdedcb68247c03cea681f096dc1dd4eb73dd4d166f27173a9f8b0040df39fd168d97010960a074b8b23c49183bee6a737bd50

data/.travis.yml

@@ -1,23 +1,36 @@
 language: ruby
-cache: bundler
-before_install:
-  - gem install bundler -v '< 2'
-  - export PHANTOMJS_VERSION='2.1.1'
-  - export PHANTOMJS_URL='https://github.com/Medium/phantomjs'
-  - export PHANTOMJS_URL+="/releases/download/v${PHANTOMJS_VERSION}"
-  - export PHANTOMJS_URL+="/phantomjs-${PHANTOMJS_VERSION}-linux-x86_64.tar.bz2"
-  - >
-    wget -q ${PHANTOMJS_URL} &&
-    tar xfv phantomjs-${PHANTOMJS_VERSION}-linux-x86_64.tar.bz2 \
-      --wildcards */bin/phantomjs --strip-components=2
-  - export PATH="`pwd`:${PATH}"
-before_script:
-  - phantomjs --version
-  - bundle --version
+
 rvm:
   - 2.3
   - 2.4
   - 2.5
   - 2.6
-script: bundle exec rspec spec
+
+cache:
+  bundler: true
+  directories:
+  - $HOME/.webdrivers
+
+addons:
+  chrome: stable
+  apt:
+    update: true
+
+sudo: false
+dist: xenial
+
+rvm:
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+
+matrix:
+  fast_finish: true
+
+bundler_args: --jobs 3 --retry 3
+
+before_install:
+  - gem update --system
+  - gem install bundler
 

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+v2.4.0, 2020-08-26
+-------------------
+  * Make verify_peer configurable and default it to false [#294](https://github.com/oesmith/puffing-billy/pull/294)
+  * Include pid in names of temporary files [#290](https://github.com/oesmith/puffing-billy/pull/290)
+
+v2.3.1, 2020-03-19
+-------------------
+  * Update min 'faraday' gem version required [#285](https://github.com/oesmith/puffing-billy/pull/285)
+
+v2.3.0, 2019-12-26
+-------------------
+  * Add `cache_whitelist` config option [#279](https://github.com/oesmith/puffing-billy/pull/279)
+  * Ignore certificate errors in Chrome [#280](https://github.com/oesmith/puffing-billy/pull/280)
+
+v2.2.0, 2019-10-26
+-------------------
+  * Before handle request method [#273](https://github.com/oesmith/puffing-billy/pull/273)
+  * Add Selenium headless driver [#272](https://github.com/oesmith/puffing-billy/pull/272)
+
+v2.1.1, 2019-06-15
+-------------------
+  * Add driver registration for chrome headless [#267](https://github.com/oesmith/puffing-billy/pull/267)
+
 v2.1.0, 2019-03-17
 -------------------
   * Allow stubbing all request methods [#263](https://github.com/oesmith/puffing-billy/pull/263)

data/README.md

@@ -14,14 +14,14 @@ Billy spawns an EventMachine-based proxy server, which it uses to intercept
 requests sent by your browser. It has a simple API for configuring which
 requests need stubbing and what they should return.
 
-Billy lets you test against known, repeatable data.  It also allows you to
+Billy lets you test against known, repeatable data. It also allows you to
 test for failure cases.  Does your twitter (or facebook/google/etc)
-integration degrade gracefully when the API starts returning 500s?  Well now
+integration degrade gracefully when the API starts returning 500s? Well now
 you can test it!
 
 ```ruby
 it 'should stub google' do
-  proxy.stub('http://www.google.com/').and_return(:text => "I'm not Google!")
+  proxy.stub('http://www.google.com/').and_return(text: "I'm not Google!")
   visit 'http://www.google.com/'
   expect(page).to have_content("I'm not Google!")
 end
@@ -32,21 +32,25 @@ You can also record HTTP interactions and replay them later. See
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Add this line to your application's `Gemfile`:
 
-    gem 'puffing-billy'
+```ruby
+gem 'puffing-billy', group: :test
+```
 
 And then execute:
 
-    $ bundle
+```sh
+$ bundle
+```
 
 Or install it yourself as:
 
-    $ gem install puffing-billy
-
-## RSpec Usage
+```sh
+$ gem install puffing-billy
+```
 
-### Setup for Capybara
+## Setup for Capybara
 
 In your `rails_helper.rb`:
 
@@ -55,7 +59,9 @@ require 'billy/capybara/rspec'
 
 # select a driver for your chosen browser environment
 Capybara.javascript_driver = :selenium_billy # Uses Firefox
+# Capybara.javascript_driver = :selenium_headless_billy # Uses Firefox in headless mode
 # Capybara.javascript_driver = :selenium_chrome_billy
+# Capybara.javascript_driver = :selenium_chrome_headless_billy
 # Capybara.javascript_driver = :apparition_billy
 # Capybara.javascript_driver = :webkit_billy
 # Capybara.javascript_driver = :poltergeist_billy
@@ -66,7 +72,7 @@ Capybara.javascript_driver = :selenium_billy # Uses Firefox
 headless specs when using puffing-billy for other local rack apps.
 See [this phantomjs issue](https://github.com/ariya/phantomjs/issues/11342) for any updates.
 
-### Setup for Watir
+## Setup for Watir
 
 In your `rails_helper.rb`:
 
@@ -79,87 +85,7 @@ require 'billy/watir/rspec'
 # @browser = Billy::Browsers::Watir.new = :phantomjs
 ```
 
-### In your tests (Capybara/Watir)
-
-```ruby
-# Stub and return text, json, jsonp (or anything else)
-proxy.stub('http://example.com/text/').and_return(:text => 'Foobar')
-proxy.stub('http://example.com/json/').and_return(:json => { :foo => 'bar' })
-proxy.stub('http://example.com/jsonp/').and_return(:jsonp => { :foo => 'bar' })
-proxy.stub('http://example.com/headers/').and_return({
-  :headers => { 'Access-Control-Allow-Origin' => '*' },
-  :json    => { :foo => 'bar' }
-})
-proxy.stub('http://example.com/wtf/').and_return(:body => 'WTF!?', :content_type => 'text/wtf')
-
-# Stub redirections and other return codes
-proxy.stub('http://example.com/redirect/').and_return(:redirect_to => 'http://example.com/other')
-proxy.stub('http://example.com/missing/').and_return(:code => 404, :body => 'Not found')
-
-# Even stub HTTPS!
-proxy.stub('https://example.com:443/secure/').and_return(:text => 'secrets!!1!')
-
-# Pass a Proc (or Proc-style object) to create dynamic responses.
-#
-# The proc will be called with the following arguments:
-#   params:  Query string parameters hash, CGI::escape-style
-#   headers: Headers hash
-#   body:    Request body string
-#   url:     The actual URL which was requested
-#   method:  The HTTP verb which was requested
-proxy.stub('https://example.com/proc/').and_return(Proc.new { |params, headers, body, url, method|
-  {
-    :code => 200,
-    :text => "Hello, #{params['name'][0]}"
-  }
-})
-
-# You can also use Puffing Billy to intercept requests and responses. Just pass
-# a Proc and use the pass_request method. You can manipulate the request
-# (headers, URL, HTTP method, etc) and also the response from the upstream
-# server. The scope of the delivered callable is the user scope where
-# it was defined. Setting method to 'all' will intercept requests regardless of
-# the method.
-proxy.stub('http://example.com/', method => 'all').and_return(Proc.new { |*args|
-  response = Billy.pass_request(*args)
-  response[:headers]['Content-Type'] = 'text/plain'
-  response[:body] = 'Hello World!'
-  response[:code] = 200
-  response
-})
-
-# Stub out a POST. Don't forget to allow a CORS request and set the method to 'post'
-proxy.stub('http://example.com/api', :method => 'post').and_return(
-  :headers => { 'Access-Control-Allow-Origin' => '*' },
-  :code => 201
-)
-
-# Stub out an OPTIONS request. Set the headers to the values you require.
-proxy.stub('http://example.com/api', :method => :options).and_return(
-  :headers => {
-    'Access-Control-Allow-Methods' => 'GET, PATCH, POST, PUT, OPTIONS',
-    'Access-Control-Allow-Headers' => 'X-Requested-With, X-Prototype-Version, Content-Type',
-    'Access-Control-Allow-Origin'  => '*'
-  },
-  :code => 200
-)
-```
-
-Stubs are reset between tests.  Any requests that are not stubbed will be
-proxied to the remote server.
-
-If for any reason you'd need to reset stubs manually you can do it in two ways:
-
-```ruby
-# reset a single stub
-example_stub = proxy.stub('http://example.com/text/').and_return(:text => 'Foobar')
-proxy.unstub example_stub
-
-# reset all stubs
-proxy.reset
-```
-
-## Cucumber Usage
+## Setup for Cucumber
 
 An example feature:
 
@@ -171,7 +97,7 @@ Feature: Stubbing via billy
     And a stub for google
 ```
 
-### Capybara
+### Setup for Cucumber + Capybara
 
 In your `features/support/env.rb`:
 
@@ -191,7 +117,7 @@ Before('@billy') do
 end
 
 And /^a stub for google$/ do
-  proxy.stub('http://www.google.com/').and_return(:text => "I'm not Google!")
+  proxy.stub('http://www.google.com/').and_return(text: "I'm not Google!")
   visit 'http://www.google.com/'
   expect(page).to have_content("I'm not Google!")
 end
@@ -202,7 +128,7 @@ It's good practice to reset the driver after each scenario, so having an
 stubs are reset after each step, so any usage of a stub should be in the
 same step that it was created in.
 
-### Watir
+### Setup for Cucumber + Watir
 
 In your `features/support/env.rb`:
 
@@ -222,7 +148,7 @@ Before('@billy') do
 end
 
 And /^a stub for google$/ do
-  proxy.stub('http://www.google.com/').and_return(:text => "I'm not Google!")
+  proxy.stub('http://www.google.com/').and_return(text: "I'm not Google!")
   @browser.goto 'http://www.google.com/'
   expect(@browser.text).to eq("I'm not Google!")
 end
@@ -234,6 +160,86 @@ Please see [this link](https://gist.github.com/sauy7/1b081266dd453a1b737b) for
 details and report back to [Issue #49](https://github.com/oesmith/puffing-billy/issues/49)
 if you get it fully working.
 
+## Examples
+
+```ruby
+# Stub and return text, json, jsonp (or anything else)
+proxy.stub('http://example.com/text/').and_return(text: 'Foobar')
+proxy.stub('http://example.com/json/').and_return(json: { foo: 'bar' })
+proxy.stub('http://example.com/jsonp/').and_return(jsonp: { foo: 'bar' })
+proxy.stub('http://example.com/headers/').and_return(
+  headers: { 'Access-Control-Allow-Origin' => '*' },
+  json: { foo: 'bar' }
+)
+proxy.stub('http://example.com/wtf/').and_return(body: 'WTF!?', content_type: 'text/wtf')
+
+# Stub redirections and other return codes
+proxy.stub('http://example.com/redirect/').and_return(redirect_to: 'http://example.com/other')
+proxy.stub('http://example.com/missing/').and_return(code: 404, body: 'Not found')
+
+# Even stub HTTPS!
+proxy.stub('https://example.com:443/secure/').and_return(text: 'secrets!!1!')
+
+# Pass a Proc (or Proc-style object) to create dynamic responses.
+#
+# The proc will be called with the following arguments:
+#   params:  Query string parameters hash, CGI::escape-style
+#   headers: Headers hash
+#   body:    Request body string
+#   url:     The actual URL which was requested
+#   method:  The HTTP verb which was requested
+proxy.stub('https://example.com/proc/').and_return(Proc.new { |params, headers, body, url, method|
+  {
+    code: 200,
+    text: "Hello, #{params['name'][0]}"
+  }
+})
+
+# You can also use Puffing Billy to intercept requests and responses. Just pass
+# a Proc and use the pass_request method. You can manipulate the request
+# (headers, URL, HTTP method, etc) and also the response from the upstream
+# server. The scope of the delivered callable is the user scope where
+# it was defined. Setting method to 'all' will intercept requests regardless of
+# the method.
+proxy.stub('http://example.com/', method: 'all').and_return(Proc.new { |*args|
+  response = Billy.pass_request(*args)
+  response[:headers]['Content-Type'] = 'text/plain'
+  response[:body] = 'Hello World!'
+  response[:code] = 200
+  response
+})
+
+# Stub out a POST. Don't forget to allow a CORS request and set the method to 'post'
+proxy.stub('http://example.com/api', method: 'post').and_return(
+  headers: { 'Access-Control-Allow-Origin' => '*' },
+  code: 201
+)
+
+# Stub out an OPTIONS request. Set the headers to the values you require.
+proxy.stub('http://example.com/api', method: 'options').and_return(
+  headers: {
+    'Access-Control-Allow-Methods' => 'GET, PATCH, POST, PUT, OPTIONS',
+    'Access-Control-Allow-Headers' => 'X-Requested-With, X-Prototype-Version, Content-Type',
+    'Access-Control-Allow-Origin'  => '*'
+  },
+  code: 200
+)
+```
+
+Stubs are reset between tests.  Any requests that are not stubbed will be
+proxied to the remote server.
+
+If for any reason you'd need to reset stubs manually you can do it in two ways:
+
+```ruby
+# reset a single stub
+example_stub = proxy.stub('http://example.com/text/').and_return(text: 'Foobar')
+proxy.unstub example_stub
+
+# reset all stubs
+proxy.reset
+```
+
 ## Caching
 
 Requests routed through the external proxy are cached.
@@ -246,7 +252,8 @@ In your `rails_helper.rb`:
 
 ```ruby
 Billy.configure do |c|
-  c.whitelist = ['test.host', 'localhost', '127.0.0.1']
+  c.whitelist = ['test.host', 'localhost', '127.0.0.1'] # To replace the default whitelist, OR
+  c.whitelist << 'mynewhost.local' # to append a host without overriding the defaults.
 end
 ```
 
@@ -260,6 +267,8 @@ server = Capybara.current_session.server
 Billy.config.whitelist = ["#{server.host}:#{server.port}"]
 ```
 
+If you would like to cache whitelisted URLs, you can define them in `c.cache_whitelist`. This is useful for scenarios where you may want to set `c.non_whitelisted_requests_disabled` to `true` to only allow whitelisted URLs to be accessed, but still allow specific URLs to be treated as if they were non-whitelisted.
+
 If you want to use puffing-billy like you would [VCR](https://github.com/vcr/vcr)
 you can turn on cache persistence. This way you don't have to manually mock out
 everything as requests are automatically recorded and played back. With cache
@@ -289,6 +298,7 @@ Billy.configure do |c|
   c.proxy_port = 12345 # defaults to random
   c.proxied_request_host = nil
   c.proxied_request_port = 80
+  c.cache_whitelist = []
   c.record_requests = true # defaults to false
   c.cache_request_body_methods = ['post', 'patch', 'put'] # defaults to ['post']
 end
@@ -434,9 +444,6 @@ internally on this request, or your test ended before it could complete successf
 
 `c.after_cache_handles_request` is used to configure a callback that can operate on the response after it has been retrieved from the cache but before it is returned. The callback receives the request and response as arguments, with a request object like: `{ method: method, url: url, headers: headers, body: body }`. An example usage would be manipulating the Access-Control-Allow-Origin header so that your test server doesn't always have to run on the same port in order to accept cached responses to CORS requests:
 
-`c.use_ignore_params` is used to choose whether to use the ignore_params blacklist or the allow_params whitelist. Set to `true` to use `c.ignore_params`,
-`false` to use `c.allow_params`
-
 ```
 Billy.configure do |c|
   ...
@@ -454,6 +461,18 @@ Billy.configure do |c|
 end
 ```
 
+`c.use_ignore_params` is used to choose whether to use the ignore_params blacklist or the allow_params whitelist. Set to `true` to use `c.ignore_params`,
+`false` to use `c.allow_params`
+
+`c.before_handle_request` is used to modify `method`, `url`, `headers`, `body` before handle request by `stubs`, `cache` or `proxy`. Method accept 4 argumens and must return array of this arguments:
+
+```
+c.before_handle_request = proc { |method, url, headers, body|
+  filtered_body = JSON.dump(filter_secret_data(JSON.load(body)))
+  [method, url, headers, filtered_body]
+}
+```
+
 `c.cache_simulates_network_delays` is used to add some delay before cache returns response. When set to `true`, cached requests will wait from configured delay time before responding. This allows to catch various race conditions in asynchronous front-end requests. The default is `false`.
 
 `c.cache_simulates_network_delay_time` is used to configure time (in seconds) to wait until responding from cache. The default is `0.1`.
@@ -524,20 +543,23 @@ end
 If you want the cache for each test to be independent, i.e. have it's own directory where the cache files are stored, you can do so.
 
 ### in Cucumber
-use a Before tag:
+
+use a `Before` tag:
 ```rb
 Before('@javascript') do |scenario, block|
   Billy.configure do |c|
     feature_name = scenario.feature.name.underscore
     scenario_name = scenario.name.underscore
     c.cache_path = "features/support/fixtures/req_cache/#{feature_name}/#{scenario_name}/"
-    Dir.mkdir_p(Billy.config.cache_path) unless File.exist?(Billy.config.cache_path)
+    FileUtils.mkdir_p(Billy.config.cache_path) unless File.exist?(Billy.config.cache_path)
   end
 end
 ```
 
 ### in Rspec
-use a before(:each) block:
+
+use a `before(:each)` block:
+
 ```rb
 RSpec.configure do |config|
   base_cache_path = Billy.config.cache_path
@@ -558,7 +580,6 @@ RSpec.configure do |config|
 end
 ```
 
-
 ## Stub requests recording
 
 If you want to record requests to stubbed URIs, set the following configuration option:
@@ -602,6 +623,7 @@ and tell it to ignore SSL certificate warnings. See
 to see how Billy's default drivers are configured.
 
 ## Working with VCR and Webmock
+
 If you use VCR and Webmock elsewhere in your specs, you may need to disable them
 for your specs utilizing Puffing Billy. To do so, you can configure your `rails_helper.rb`
 as shown below:
@@ -631,7 +653,7 @@ Note that this approach may cause unexpected behavior if your backend sends the
 
 ### Raising errors from stubs
 
-By default PuffingBilly suppress errors from stub-blocks.
+By default Puffing Billy suppresses errors from stub-blocks.
 To make it raise errors instead, add this test initializers:
 
 ```ruby
@@ -718,17 +740,24 @@ the system store. So after a run of your the suite only one certificate will be
 left over. If this is not enough you can handling the cleanup again with a
 custom on-after hook.
 
-## Resources
+### TLS hostname validation
 
-* [Bring Ruby VCR to Javascript testing with Capybara and puffing-billy](http://architects.dzone.com/articles/bring-ruby-vcr-javascript)
-* [Integration Testing Stripe.js With Mocked Network Requests](http://dev.contractual.ly/testing-stripe-js-with-mocked-network/)
-* [Clean-up unused cache files periodically with this config](https://github.com/oesmith/puffing-billy/pull/26#issuecomment-29905030)
+em-http-request was modified to emit a warning if being used without the TLS
+``verify_peer`` option.  Puffing Billy defaults to specifying ``verify_peer: false``
+but you can now modify configuration to do peer verification. So if you've
+gone to the trouble of setting up your own certificate authority and self-signed
+certs you can enable it like so:
 
-## FAQ
+```ruby
+Billy.configure do |c|
+  c.verify_peer = true
+end
+```
 
-1. Why name it after a train?
+## Resources
 
-   Trains are *cool*.
+* [Bring Ruby VCR to Javascript testing with Capybara and puffing-billy](https://dzone.com/articles/bring-ruby-vcr-javascript)
+* [Clean-up unused cache files periodically with this config](https://github.com/oesmith/puffing-billy/pull/26#issuecomment-29905030)
 
 ## Contributing
 
@@ -740,5 +769,5 @@ custom on-after hook.
 
 ## TODO
 
-1. Integration for test frameworks other than rspec.
+1. Integration for test frameworks other than RSpec.
 2. Show errors from the EventMachine reactor loop in the test output.

data/Rakefile

@@ -1,2 +1,11 @@
 #!/usr/bin/env rake
 require 'bundler/gem_tasks'
+
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+end
+
+desc 'Run all tests'
+task default: [:spec]

data/examples/tumblr_api_https.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<body>
+  <h1>Latest news</h1>
+  <div id="news"></div>
+
+  <script type='text/javascript' src='http://code.jquery.com/jquery-1.8.2.min.js'></script>
+  <script type='text/javascript'>
+  $(function () {
+    var url = 'https://blog.howmanyleft.co.uk/api/read/json?callback=?&type=text&num=3&filter=text';
+    $.getJSON(url, function (data) {
+      $.each(data.posts, function (idx, post) {
+        var title = post['regular-title'];
+        var href = post['url-with-slug'];
+        var body = post['regular-body'];
+        $('#news').append(
+          '<h3><a href="' + href + '">' + title + '</a></h3>' +
+          '<p>' + body + '</p>');
+      });
+    });
+  })
+  </script>
+</body>

data/lib/billy/browsers/capybara.rb

@@ -47,21 +47,50 @@ module Billy
 
       def self.register_selenium_driver
         ::Capybara.register_driver :selenium_billy do |app|
-          profile = Selenium::WebDriver::Firefox::Profile.new
-          profile.assume_untrusted_certificate_issuer = false
-          profile.proxy = Selenium::WebDriver::Proxy.new(
-            http: "#{Billy.proxy.host}:#{Billy.proxy.port}",
-            ssl: "#{Billy.proxy.host}:#{Billy.proxy.port}")
-          ::Capybara::Selenium::Driver.new(app, profile: profile)
+          options = build_selenium_options_for_firefox
+          capabilities = Selenium::WebDriver::Remote::Capabilities.firefox(accept_insecure_certs: true)
+
+          ::Capybara::Selenium::Driver.new(app, options: options, desired_capabilities: capabilities)
+        end
+
+        ::Capybara.register_driver :selenium_headless_billy do |app|
+          options = build_selenium_options_for_firefox.tap do |opts|
+            opts.add_argument '-headless'
+          end
+          capabilities = Selenium::WebDriver::Remote::Capabilities.firefox(accept_insecure_certs: true)
+          
+          ::Capybara::Selenium::Driver.new(app, options: options, desired_capabilities: capabilities)
         end
 
         ::Capybara.register_driver :selenium_chrome_billy do |app|
           options = Selenium::WebDriver::Chrome::Options.new
+          options.add_argument('--ignore-certificate-errors')
           options.add_argument("--proxy-server=#{Billy.proxy.host}:#{Billy.proxy.port}")
 
           ::Capybara::Selenium::Driver.new(
-            app, browser: :chrome,
-            options: options
+            app,
+            browser: :chrome,
+            options: options,
+            clear_local_storage: true,
+            clear_session_storage: true
+          )
+        end
+
+        ::Capybara.register_driver :selenium_chrome_headless_billy do |app|
+            options = Selenium::WebDriver::Chrome::Options.new
+            options.headless!
+            options.add_argument('--enable-features=NetworkService,NetworkServiceInProcess')
+            options.add_argument('--ignore-certificate-errors')
+            options.add_argument("--proxy-server=#{Billy.proxy.host}:#{Billy.proxy.port}")
+            options.add_argument('--disable-gpu') if Gem.win_platform?
+            options.add_argument('--no-sandbox') if ENV['CI']
+
+          ::Capybara::Selenium::Driver.new(
+            app,
+            browser: :chrome,
+            options: options,
+            clear_local_storage: true,
+            clear_session_storage: true
           )
         end
       end
@@ -73,6 +102,17 @@ module Billy
           end
         end
       end
+
+      def self.build_selenium_options_for_firefox
+        profile = Selenium::WebDriver::Firefox::Profile.new.tap do |prof|
+          prof.assume_untrusted_certificate_issuer = false
+          prof.proxy = Selenium::WebDriver::Proxy.new(
+            http: "#{Billy.proxy.host}:#{Billy.proxy.port}",
+            ssl: "#{Billy.proxy.host}:#{Billy.proxy.port}")
+        end
+
+        Selenium::WebDriver::Firefox::Options.new(profile: profile)
+      end
     end
   end
 end

data/lib/billy/browsers/watir.rb

@@ -19,6 +19,7 @@ module Billy
       private
 
       def configure_chrome(args)
+        args[:headless] = true
         args[:switches] ||= []
         args[:switches] += %W[--proxy-server=#{Billy.proxy.host}:#{Billy.proxy.port}]
         args

data/lib/billy/config.rb

@@ -6,12 +6,12 @@ module Billy
     DEFAULT_WHITELIST = ['127.0.0.1', 'localhost']
     RANDOM_AVAILABLE_PORT = 0 # https://github.com/eventmachine/eventmachine/wiki/FAQ#wiki-can-i-start-a-server-on-a-random-available-port
 
-    attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params, :allow_params,
+    attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :cache_whitelist, :path_blacklist, :ignore_params, :allow_params,
                   :persist_cache, :ignore_cache_port, :non_successful_cache_disabled, :non_successful_error_level,
-                  :non_whitelisted_requests_disabled, :cache_path, :certs_path, :proxy_host, :proxy_port, :proxied_request_inactivity_timeout,
+                  :non_whitelisted_requests_disabled, :cache_path, :certs_path, :verify_peer, :proxy_host, :proxy_port, :proxied_request_inactivity_timeout,
                   :proxied_request_connect_timeout, :dynamic_jsonp, :dynamic_jsonp_keys, :dynamic_jsonp_callback_name, :merge_cached_responses_whitelist,
                   :strip_query_params, :proxied_request_host, :proxied_request_port, :cache_request_body_methods, :after_cache_handles_request,
-                  :cache_simulates_network_delays, :cache_simulates_network_delay_time, :record_requests, :record_stub_requests, :use_ignore_params
+                  :cache_simulates_network_delays, :cache_simulates_network_delay_time, :record_requests, :record_stub_requests, :use_ignore_params, :before_handle_request
 
     def initialize
       @logger = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
@@ -21,6 +21,7 @@ module Billy
     def reset
       @cache = true
       @cache_request_headers = false
+      @cache_whitelist = []
       @whitelist = DEFAULT_WHITELIST
       @path_blacklist = []
       @merge_cached_responses_whitelist = []
@@ -36,6 +37,7 @@ module Billy
       @non_whitelisted_requests_disabled = false
       @cache_path = File.join(Dir.tmpdir, 'puffing-billy')
       @certs_path = File.join(Dir.tmpdir, 'puffing-billy', 'certs')
+      @verify_peer = false
       @proxy_host = 'localhost'
       @proxy_port = RANDOM_AVAILABLE_PORT
       @proxied_request_inactivity_timeout = 10 # defaults from https://github.com/igrigorik/em-http-request/wiki/Redirects-and-Timeouts
@@ -50,6 +52,7 @@ module Billy
       @record_requests = false
       @record_stub_requests = false
       @use_ignore_params = true
+      @before_handle_request = nil
     end
   end
 

data/lib/billy/handlers/proxy_handler.rb

@@ -16,6 +16,10 @@ module Billy
         opts = { inactivity_timeout: Billy.config.proxied_request_inactivity_timeout,
                  connect_timeout:    Billy.config.proxied_request_connect_timeout }
 
+        if url =~ /^https/
+          opts.merge!({tls: {verify_peer: Billy.config.verify_peer}})
+        end
+
         if Billy.config.proxied_request_host && !bypass_internal_proxy?(url)
           opts.merge!({ proxy: { host: Billy.config.proxied_request_host,
                                  port: Billy.config.proxied_request_port }} )
@@ -110,7 +114,7 @@ module Billy
 
       url = Addressable::URI.parse(url)
       # Cache the responses if they aren't whitelisted host[:port]s but always cache blacklisted paths on any hosts
-      cacheable_status?(status) && (!whitelisted_url?(url) || blacklisted_path?(url.path))
+      cacheable_status?(status) && (!whitelisted_url?(url) || blacklisted_path?(url.path) || cache_whitelisted_url?(url))
     end
 
     def whitelisted_url?(url)
@@ -123,6 +127,16 @@ module Billy
       end
     end
 
+    def cache_whitelisted_url?(url)
+      Billy.config.cache_whitelist.any? do |value|
+        if value.is_a?(Regexp)
+          url.to_s =~ value || url.omit(:port).to_s =~ value
+        else
+          value =~ /^#{url.host}(?::#{url.port})?$/
+        end
+      end
+    end
+
     def blacklisted_path?(path)
       !Billy.config.path_blacklist.index { |bl| bl.is_a?(Regexp) ? path =~ bl : path.include?(bl) }.nil?
     end

data/lib/billy/handlers/request_handler.rb

@@ -17,6 +17,10 @@ module Billy
     def handle_request(method, url, headers, body)
       request = request_log.record(method, url, headers, body)
 
+      if Billy.config.before_handle_request
+        method, url, headers, body = Billy.config.before_handle_request.call(method, url, headers, body)
+      end
+
       # Process the handlers by order of importance
       [:stubs, :cache, :proxy].each do |key|
         if (response = handlers[key].handle_request(method, url, headers, body))

data/lib/billy/ssl/certificate_helpers.rb

@@ -27,7 +27,7 @@ module Billy
     # and ensure the location is safely created. Pass
     # back the resulting path.
     def write_file(name, contents)
-      path = File.join(Billy.config.certs_path, name)
+      path = File.join(Billy.config.certs_path, "#{Process.pid}-#{name}")
       FileUtils.mkdir_p(File.dirname(path))
       File.write(path, contents)
       path

data/lib/billy/version.rb

@@ -1,3 +1,3 @@
 module Billy
-  VERSION = '2.1.0'
+  VERSION = '2.4.0'
 end

data/puffing-billy.gemspec

@@ -17,19 +17,18 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'thin'
-  gem.add_development_dependency 'faraday'
-  gem.add_development_dependency 'poltergeist'
+  gem.add_development_dependency 'faraday', '>= 0.9.0'
   gem.add_development_dependency 'apparition'
-  # selenium-webdriver 3.8 drops support for PhantomJS
-  gem.add_development_dependency 'selenium-webdriver', '<= 3.7.0'
   gem.add_development_dependency 'capybara'
-  gem.add_development_dependency 'capybara-webkit', '~> 1.0'
+  gem.add_development_dependency 'selenium-webdriver'
   gem.add_development_dependency 'rack'
+  gem.add_development_dependency 'rake'
   gem.add_development_dependency 'guard'
   gem.add_development_dependency 'rb-inotify'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'cucumber'
   gem.add_development_dependency 'watir', '~> 6.10.0'
+  gem.add_development_dependency 'webdrivers'
   gem.add_runtime_dependency 'addressable', '~> 2.5'
   gem.add_runtime_dependency 'eventmachine', '~> 1.2'
   gem.add_runtime_dependency 'em-synchrony'

data/spec/lib/billy/browsers/capybara_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe 'Capybara drivers', type: :feature, js: true do
+  it 'allows HTTPS calls' do
+    proxy.stub('https://blog.howmanyleft.co.uk:443/api/read/json').and_return(
+      jsonp: {
+        posts: [
+          {
+            'regular-title' => 'News Item 1',
+            'url-with-slug' => 'http://example.com/news/1',
+            'regular-body' => 'News item 1 content here'
+          },
+          {
+            'regular-title' => 'News Item 2',
+            'url-with-slug' => 'http://example.com/news/2',
+            'regular-body' => 'News item 2 content here'
+          }
+        ]
+      })
+
+    visit '/tumblr_api_https.html'
+
+    expect(page).to have_link('News Item 1', href: 'http://example.com/news/1')
+    expect(page).to have_content('News item 1 content here')
+    expect(page).to have_link('News Item 2', href: 'http://example.com/news/2')
+    expect(page).to have_content('News item 2 content here')
+  end
+end

data/spec/lib/billy/handlers/proxy_handler_spec.rb

@@ -175,6 +175,34 @@ describe Billy::ProxyHandler do
                                request[:body])
       end
 
+      it 'does NOT cache the response if the host is whitelisted but not cache_whitelisted' do
+        uri = Addressable::URI.parse(request[:url])
+
+        expect(subject).to receive(:allowed_response_code?).and_return(true)
+        expect(Billy.config).to receive(:whitelist).and_return([uri.host])
+        expect(Billy.config).to receive(:cache_whitelist).and_return([])
+
+        expect(Billy::Cache.instance).not_to receive(:store)
+        subject.handle_request(request[:method],
+                               request[:url],
+                               request[:headers],
+                               request[:body])
+      end
+
+      it 'caches the response if the host is whitelisted AND cache_whitelisted' do
+        uri = Addressable::URI.parse(request[:url])
+
+        expect(subject).to receive(:allowed_response_code?).and_return(true)
+        expect(Billy.config).to receive(:whitelist).and_return([uri.host])
+        expect(Billy.config).to receive(:cache_whitelist).and_return([uri.host])
+
+        expect(Billy::Cache.instance).to receive(:store)
+        subject.handle_request(request[:method],
+                               request[:url],
+                               request[:headers],
+                               request[:body])
+      end
+
       it 'uses the timeouts defined in configuration' do
         allow(Billy.config).to receive(:proxied_request_inactivity_timeout).and_return(42)
         allow(Billy.config).to receive(:proxied_request_connect_timeout).and_return(24)

data/spec/lib/billy/handlers/request_handler_spec.rb

@@ -128,6 +128,28 @@ describe Billy::RequestHandler do
         allow(stub_handler).to receive(:handle_request).and_raise("Any Stub Error")
         expect(subject.handle_request(*args)).to eql(error: "Any Stub Error")
       end
+
+      context 'before_handle_request activated' do
+        before do
+          handle_request = proc { |method, url, headers, body|
+            [method, url, headers, "#{body}_modified"]
+          }
+          allow(Billy::config).to receive(:before_handle_request).and_return(handle_request)
+        end
+
+        after do
+          allow(Billy::config).to receive(:before_handle_request).and_call_original
+        end
+
+        it 'modify request before handling' do
+          new_args = %w(get url headers body_modified)
+          expect(stub_handler).to receive(:handle_request).with(*new_args)
+          expect(cache_handler).to receive(:handle_request).with(*new_args).and_return('bar')
+          expect(proxy_handler).to_not receive(:handle_request)
+          expect(subject.handle_request(*args)).to eql 'bar'
+          expect(subject.requests).to eql([{status: :complete, handler: :cache, method: 'get', url: 'url', headers: 'headers', body: 'body'}])
+        end
+      end
     end
 
     describe '#stubs' do

data/spec/lib/proxy_spec.rb

@@ -118,6 +118,29 @@ shared_examples_for 'a cache' do
     end
   end
 
+  context 'cache_whitelist GET requests' do
+    before do
+      Billy.config.whitelist = [http.host]
+      Billy.config.cache_whitelist = [http.host]
+    end
+
+    it 'should be cached' do
+      assert_cached_url
+    end
+
+    context 'with ports' do
+      before do
+        rack_app_url = URI(http.url_prefix)
+        Billy.config.whitelist = ["#{rack_app_url.host}:#{rack_app_url.port + 1}"]
+        Billy.config.cache_whitelist = Billy.config.whitelist
+      end
+
+      it 'should be cached' do
+        assert_cached_url
+      end
+    end
+  end
+
   context 'ignore_params GET requests' do
     before do
       Billy.config.ignore_params = ['/analytics']
@@ -232,7 +255,7 @@ shared_examples_for 'a cache' do
 
         it 'should raise error when disabled' do
           # TODO: Suppress stderr output: https://gist.github.com/adamstegman/926858
-          expect { http.get('/foo') }.to raise_error(Faraday::Error::ConnectionFailed, 'end of file reached')
+          expect { http.get('/foo') }.to raise_error(Faraday::ConnectionFailed, 'end of file reached')
         end
       end
 
@@ -261,7 +284,7 @@ shared_examples_for 'a cache' do
         end
 
         it 'should raise error for non-successful responses when :error' do
-          expect { http_error.get('/foo') }.to raise_error(Faraday::Error::ConnectionFailed)
+          expect { http_error.get('/foo') }.to raise_error(Faraday::ConnectionFailed)
         end
       end
     end

data/spec/spec_helper.rb

@@ -6,11 +6,17 @@ require 'billy/watir/rspec'
 require 'rack'
 require 'logger'
 require 'fileutils'
+require 'webdrivers'
 
-browser = Billy::Browsers::Watir.new :phantomjs
-Capybara.app = Rack::Directory.new(File.expand_path('../../examples', __FILE__))
-Capybara.server = :webrick
-Capybara.javascript_driver = :poltergeist_billy
+$stdout.puts `#{::Selenium::WebDriver::Chrome::Service.driver_path.call} --version` if ENV['CI']
+
+browser = Billy::Browsers::Watir.new :chrome
+
+Capybara.configure do |config|
+  config.app = Rack::Directory.new(File.expand_path('../../examples', __FILE__))
+  config.server = :webrick
+  config.javascript_driver = :selenium_chrome_headless_billy
+end
 
 Billy.configure do |config|
   config.logger = Logger.new(File.expand_path('../../log/test.log', __FILE__))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puffing-billy
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Olly Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-17 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -44,16 +44,16 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
-  name: poltergeist
+  name: apparition
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: apparition
+  name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -82,20 +82,6 @@ dependencies:
         version: '0'
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 3.7.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 3.7.0
-- !ruby/object:Gem::Dependency
-  name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,21 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: capybara-webkit
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -206,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 6.10.0
+- !ruby/object:Gem::Dependency
+  name: webdrivers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -336,6 +336,7 @@ files:
 - examples/post_api.html
 - examples/preflight_request.html
 - examples/tumblr_api.html
+- examples/tumblr_api_https.html
 - lib/billy.rb
 - lib/billy/browsers/capybara.rb
 - lib/billy/browsers/watir.rb
@@ -375,6 +376,7 @@ files:
 - spec/features/examples/post_api_spec.rb
 - spec/features/examples/preflight_request_spec.rb
 - spec/features/examples/tumblr_api_spec.rb
+- spec/lib/billy/browsers/capybara_spec.rb
 - spec/lib/billy/cache_spec.rb
 - spec/lib/billy/handlers/cache_handler_spec.rb
 - spec/lib/billy/handlers/handler_spec.rb
@@ -421,6 +423,7 @@ test_files:
 - spec/features/examples/post_api_spec.rb
 - spec/features/examples/preflight_request_spec.rb
 - spec/features/examples/tumblr_api_spec.rb
+- spec/lib/billy/browsers/capybara_spec.rb
 - spec/lib/billy/cache_spec.rb
 - spec/lib/billy/handlers/cache_handler_spec.rb
 - spec/lib/billy/handlers/handler_spec.rb