puffing-billy 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49c2a689f350d56b8719c2d63a9ee65a8b9e08a6
-  data.tar.gz: c7f1d50a3a17009c4e394ed87a4c07cae64dc0de
+  metadata.gz: 9e495bcc795342695eb34d2fe60ae1d43b44e99e
+  data.tar.gz: 1765e2fc100394d5244f369ca130351cabacb37a
 SHA512:
-  metadata.gz: 85873ec39d4108e7914ed711f777b9ac96a50250ebff960840132410d9dc221837982d4c90c086b4257681a073850340c96ffb08f1ba3d52d007271ff8b50511
-  data.tar.gz: ba586d5487c7bef9c682fc834c06c91f2e93364cc450ed4c74ab868a2399ec61a907f5d44d8cf728e7fd66d83c0cf41920df0aae4ad05e9b973421632d25971e
+  metadata.gz: 177544dfed171ecbc43f81e8151d800d38c7f199bd627aea62935da083994fce5cafaf5e936afc1596fdc04047d1b475b3cb0b14f4ebc7514ee41f1bbaa399da
+  data.tar.gz: 370c320802c05f34e53660f006e5d59b66e5a0ffdc5c880be5d04c803fe653d9d365b8daaed9f16f2454806302cb1cb9d35b204914bbbb8c19843d0fa119048d

data/.gitignore

@@ -1,4 +1,5 @@
 node_modules/
+/Gemfile.lock
 log/test.log
 .idea/
 .ruby-version

data/.travis.yml

@@ -1,7 +1,19 @@
 language: ruby
+cache: bundler
 before_install:
   - gem install bundler
+  - export PHANTOMJS_VERSION='2.1.1'
+  - export PHANTOMJS_URL='https://github.com/Medium/phantomjs'
+  - export PHANTOMJS_URL+="/releases/download/v${PHANTOMJS_VERSION}"
+  - export PHANTOMJS_URL+="/phantomjs-${PHANTOMJS_VERSION}-linux-x86_64.tar.bz2"
+  - >
+    wget -q ${PHANTOMJS_URL} &&
+    tar xfv phantomjs-${PHANTOMJS_VERSION}-linux-x86_64.tar.bz2 \
+      --wildcards */bin/phantomjs --strip-components=2
+  - export PATH="`pwd`:${PATH}"
+before_script:
   - phantomjs --version
+  - bundle --version
 rvm:
   - 1.9.3
   - 2.0.0

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+v0.11.0, 2017-11-09
+-------------------
+  * Improved semantic versioning of dependencies [#197](https://github.com/oesmith/puffing-billy/pull/197)
+  * Implemented a dynamic generation of SSL request certificates [#198](https://github.com/oesmith/puffing-billy/pull/198)
+  * Added Billy.config.allow_params whitelist feature [#200](https://github.com/oesmith/puffing-billy/pull/200)
+
 v0.10.1, 2017-10-12
 -------------------
   * Fix selenium webdriver deprecation warning [#194](https://github.com/oesmith/puffing-billy/pull/194)

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM ruby:1.9.3
+
+RUN apt-get update -y
+RUN apt-get install -y qt5-default libqt5webkit5-dev gstreamer1.0-plugins-base gstreamer1.0-tools gstreamer1.0-x
+RUN gem install bundler
+RUN \
+    export PHANTOMJS_VERSION='2.1.1'                                             && \
+    export PHANTOMJS_URL='https://github.com/Medium/phantomjs/releases/download/v2.1.1/phantomjs-2.1.1-linux-x86_64.tar.bz2' && \
+    wget -q ${PHANTOMJS_URL}                                                     && \
+    tar xfv phantomjs-${PHANTOMJS_VERSION}-linux-x86_64.tar.bz2                  \
+      -C /usr/bin --wildcards */bin/phantomjs --strip-components=2
+RUN mkdir -p /app
+COPY . /app
+RUN cd /app && bundle install

data/README.md

@@ -253,6 +253,7 @@ Billy.configure do |c|
   c.non_successful_error_level = :warn
   c.non_whitelisted_requests_disabled = false
   c.cache_path = 'spec/req_cache/'
+  c.certs_path = 'spec/req_certs/'
   c.proxy_host = 'example.com' # defaults to localhost
   c.proxy_port = 12345 # defaults to random
   c.proxied_request_host = nil
@@ -273,6 +274,10 @@ caching. You should mostly use this for analytics and various social buttons as
 they use cache avoidance techniques, but return practically the same response
 that most often does not affect your test results.
 
+`c.allow_params` is used to allow parameters of certain requests when caching. This is best used when a site
+has a large number of analytics and social buttons. `c.allow_params` is the opposite of `c.ignore_params`,
+a whitelist to a blacklist. In order to toggle between using one or the other, use `c.use_ignore_params`.
+
 `c.strip_query_params` is used to strip query parameters when you stub some requests
 with query parameters. Default value is true. For example, `proxy.stub('http://myapi.com/user/?country=FOO')`
 is considered the same as: `proxy.stub('http://myapi.com/user/?anything=FOO')` and
@@ -291,7 +296,7 @@ using `c.dynamic_jsonp`. This is helpful when JSONP APIs use cache-busting
 parameters. For example, if you want `http://example.com/foo?callback=bar&id=1&cache_bust=12345` and `http://example.com/foo?callback=baz&id=1&cache_bust=98765` to be cache hits for each other, you would set `c.dynamic_jsonp_keys = ['callback', 'cache_bust']` to ignore both params. Note
 that in this example the `id` param would still be considered important.
 
-`c.dynamic_jsonp_callback_name` is used to configure the name of the JSONP callback 
+`c.dynamic_jsonp_callback_name` is used to configure the name of the JSONP callback
 parameter. The default is `callback`.
 
 `c.path_blacklist = []` is used to always cache specific paths on any hostnames,
@@ -323,6 +328,13 @@ allowed, all others will throw an error with the URL attempted to be accessed.
 This is useful for debugging issues in isolated environments (ie.
 continuous integration).
 
+`c.cache_path` can be used to locate the cache directory to a different place
+other than `system temp directory/puffing-billy`.
+
+`c.certs_path` can be used to locate the directory for dynamically generated
+SSL certificates to a different place other than `system temp
+directory/puffing-billy/certs`.
+
 `c.proxy_host` and `c.proxy_port` are used for the Billy proxy itself which runs locally.
 
 `c.proxied_request_host` and `c.proxied_request_port` are used if an internal proxy
@@ -332,6 +344,9 @@ server is required to access the internet.  Most common in larger companies.
 
 `c.after_cache_handles_request` is used to configure a callback that can operate on the response after it has been retrieved from the cache but before it is returned. The callback receives the request and response as arguments, with a request object like: `{ method: method, url: url, headers: headers, body: body }`. An example usage would be manipulating the Access-Control-Allow-Origin header so that your test server doesn't always have to run on the same port in order to accept cached responses to CORS requests:
 
+`c.use_ignore_params` is used to choose whether to use the ignore_params blacklist or the allow_params whitelist. Set to `true` to use `c.ignore_params`,
+`false` to use `c.allow_params`
+
 ```
 Billy.configure do |c|
   ...
@@ -500,6 +515,54 @@ end
 
 Note that this approach may cause unexpected behavior if your backend sends the Referer HTTP header (which is unlikely).
 
+## SSL usage
+
+Unfortunately we cannot setup the runtime certificate authority on your browser
+at time of configuring the Capybara driver.  So you need to take care of this
+step yourself as a prepartion. A good point would be directly after configuring
+this gem.
+
+### Google Chrome Headless example
+
+Google Chrome/Chromium is capable to run as a test browser with the new
+headless mode which is not able to handle the deprecated
+`--ignore-certificate-errors` flag. But the headless mode is capable of
+handling the user PKI certificate store.  So you just need to import the
+runtime Puffing Billy certificate authority on your system store, or generate a
+new store for your current session. The following examples demonstrates the
+former variant:
+
+```ruby
+# Overwrite the local home directory for chrome. We use this
+# to setup a custom SSL certificate store.
+ENV['HOME'] = "#{Dir.tmpdir}/chrome-home-#{Time.now.to_i}"
+
+# Clear and recreate the Chrome home directory.
+FileUtils.rm_rf(ENV['HOME'])
+FileUtils.mkdir_p(ENV['HOME'])
+
+# Setup a new pki certificate database for Chrome
+system <<~SCRIPT
+cd "#{ENV['HOME']}"
+curl -s -k -o "cacert-root.crt" "http://www.cacert.org/certs/root.crt"
+curl -s -k -o "cacert-class3.crt" "http://www.cacert.org/certs/class3.crt"
+echo > .password
+mkdir -p .pki/nssdb
+CERT_DIR=sql:$HOME/.pki/nssdb
+certutil -N -d .pki/nssdb -f .password
+certutil -d ${CERT_DIR}  -A -t TC \
+  -n "CAcert.org" -i cacert-root.crt
+certutil -d ${CERT_DIR} -A -t TC \
+  -n "CAcert.org Class 3" -i cacert-class3.crt
+certutil -d sql:$HOME/.pki/nssdb -A \
+  -n puffing-billy -t "CT,C,C" -i #{Billy.certificate_authority.cert_file}
+SCRIPT
+```
+
+Mind the reset of the `HOME` environment variable. Fortunately Chrome takes
+care of the users home, so we can setup a new temporary directory for the test
+run, without messing with potential user configurations.
+
 ## Resources
 
 * [Bring Ruby VCR to Javascript testing with Capybara and puffing-billy](http://architects.dzone.com/articles/bring-ruby-vcr-javascript)

data/lib/billy.rb

@@ -7,6 +7,10 @@ require 'billy/handlers/proxy_handler'
 require 'billy/handlers/cache_handler'
 require 'billy/proxy_request_stub'
 require 'billy/cache'
+require 'billy/ssl/certificate_helpers'
+require 'billy/ssl/authority'
+require 'billy/ssl/certificate'
+require 'billy/ssl/certificate_chain'
 require 'billy/proxy'
 require 'billy/proxy_connection'
 require 'billy/railtie' if defined?(Rails)
@@ -19,4 +23,8 @@ module Billy
       proxy
     )
   end
+
+  def self.certificate_authority
+    @certificate_authority ||= Billy::Authority.new
+  end
 end

data/lib/billy/cache.rb

@@ -69,7 +69,11 @@ module Billy
     end
 
     def key(method, orig_url, body, log_key = false)
-      ignore_params = Billy.config.ignore_params.include?(format_url(orig_url, true))
+      if Billy.config.use_ignore_params
+        ignore_params = Billy.config.ignore_params.include?(format_url(orig_url, true))
+      else
+        ignore_params = !Billy.config.allow_params.include?(format_url(orig_url, true))
+      end
       merge_cached_response_key = _merge_cached_response_key(orig_url)
       url = Addressable::URI.parse(format_url(orig_url, ignore_params))
       key = if merge_cached_response_key

data/lib/billy/config.rb

@@ -6,12 +6,12 @@ module Billy
     DEFAULT_WHITELIST = ['127.0.0.1', 'localhost']
     RANDOM_AVAILABLE_PORT = 0 # https://github.com/eventmachine/eventmachine/wiki/FAQ#wiki-can-i-start-a-server-on-a-random-available-port
 
-    attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params,
+    attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params, :allow_params,
                   :persist_cache, :ignore_cache_port, :non_successful_cache_disabled, :non_successful_error_level,
-                  :non_whitelisted_requests_disabled, :cache_path, :proxy_host, :proxy_port, :proxied_request_inactivity_timeout,
+                  :non_whitelisted_requests_disabled, :cache_path, :certs_path, :proxy_host, :proxy_port, :proxied_request_inactivity_timeout,
                   :proxied_request_connect_timeout, :dynamic_jsonp, :dynamic_jsonp_keys, :dynamic_jsonp_callback_name, :merge_cached_responses_whitelist,
                   :strip_query_params, :proxied_request_host, :proxied_request_port, :cache_request_body_methods, :after_cache_handles_request,
-                  :cache_simulates_network_delays, :cache_simulates_network_delay_time, :record_stub_requests
+                  :cache_simulates_network_delays, :cache_simulates_network_delay_time, :record_stub_requests, :use_ignore_params
 
     def initialize
       @logger = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
@@ -25,6 +25,7 @@ module Billy
       @path_blacklist = []
       @merge_cached_responses_whitelist = []
       @ignore_params = []
+      @allow_params = []
       @persist_cache = false
       @dynamic_jsonp = false
       @dynamic_jsonp_keys = ['callback']
@@ -34,6 +35,7 @@ module Billy
       @non_successful_error_level = :warn
       @non_whitelisted_requests_disabled = false
       @cache_path = File.join(Dir.tmpdir, 'puffing-billy')
+      @certs_path = File.join(Dir.tmpdir, 'puffing-billy', 'certs')
       @proxy_host = 'localhost'
       @proxy_port = RANDOM_AVAILABLE_PORT
       @proxied_request_inactivity_timeout = 10 # defaults from https://github.com/igrigorik/em-http-request/wiki/Redirects-and-Timeouts
@@ -46,6 +48,7 @@ module Billy
       @cache_simulates_network_delays = false
       @cache_simulates_network_delay_time = 0.1
       @record_stub_requests = false
+      @use_ignore_params = true
     end
   end
 

data/lib/billy/proxy_connection.rb

@@ -53,10 +53,7 @@ module Billy
       @ssl = url
       @parser = Http::Parser.new(self)
       send_data("HTTP/1.0 200 Connection established\r\nProxy-agent: Puffing-Billy/0.0.0\r\n\r\n")
-      start_tls(
-        private_key_file: File.expand_path('../mitm.key', __FILE__),
-        cert_chain_file: File.expand_path('../mitm.crt', __FILE__)
-      )
+      start_tls(certificate_chain(url))
     end
 
     def handle_request
@@ -93,6 +90,15 @@ module Billy
       res.content = response[:content]
       res.send_response
     end
-    
+
+    def certificate_chain(url)
+      domain = url.split(':').first
+      ca = Billy.certificate_authority.cert
+      cert = Billy::Certificate.new(domain)
+      chain = Billy::CertificateChain.new(domain, cert.cert, ca)
+
+      { private_key_file: cert.key_file,
+        cert_chain_file: chain.file }
+    end
   end
 end

data/lib/billy/ssl/authority.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'openssl'
+require 'fileutils'
+
+module Billy
+  # This class is dedicated to the generation of a brand new certificate
+  # authority which can be picked up by a browser to verify and secure any
+  # communication with puffing billy. This authority certificate will be
+  # generated once on runtime and will sign each request certificate. So
+  # we do not have to deal with outdated certificates or stuff like that.
+  #
+  # The resulting certificate authority is at its bare minimum to keep
+  # things simple and snappy. We do not handle a certificate revoke list
+  # (CRL) nor any other special key handling, even if we enable these
+  # extensions. It's just a mimic of the mighty mitmproxy certificate
+  # authority file.
+  class Authority
+    include Billy::CertificateHelpers
+
+    attr_reader :key, :cert
+
+    # The authority generation does not require any arguments from outside
+    # of this class definition. We just generate the certificate and thats
+    # it.
+    #
+    # Example:
+    #
+    #   ca = Billy::Authority.new
+    #   [ca.cert_file, ca.key_file]
+    def initialize
+      @key = OpenSSL::PKey::RSA.new(2048)
+      @cert = generate
+    end
+
+    # Write out the private key to file (PEM format) and give back the
+    # file path.
+    def key_file
+      write_file('ca.key', key.to_pem)
+    end
+
+    # Write out the certifcate to file (PEM format) and give back the
+    # file path.
+    def cert_file
+      write_file('ca.crt', cert.to_pem)
+    end
+
+    private
+
+    # Defines a static list of available extensions on the certificate.
+    def extensions
+      [
+        # ln_sn, value, critical
+        ['basicConstraints', 'CA:TRUE', true],
+        ['keyUsage', 'keyCertSign, cRLSign', true],
+        ['subjectKeyIdentifier', 'hash', false],
+        ['authorityKeyIdentifier', 'keyid:always', false]
+      ]
+    end
+
+    # Give back the static subject name of the certificate.
+    def name
+      '/CN=Puffing Billy/O=Puffing Billy/'
+    end
+
+    # Generate a fresh new certificate for the configured domain.
+    def generate
+      cert = OpenSSL::X509::Certificate.new
+      configure(cert)
+      add_extensions(cert)
+      cert.sign(key, OpenSSL::Digest::SHA256.new)
+    end
+
+    # Setup all relevant properties of the given certificate to produce
+    # a valid and useable certificate.
+    def configure(cert)
+      cert.version = 2
+      cert.serial = serial
+      cert.subject = OpenSSL::X509::Name.parse(name)
+      cert.issuer = cert.subject
+      cert.public_key = key.public_key
+      cert.not_before = days_ago(2)
+      cert.not_after = days_from_now(2)
+    end
+
+    # Add all extensions (defined by the +extensions+ method) to the given
+    # certificate.
+    def add_extensions(cert)
+      factory = OpenSSL::X509::ExtensionFactory.new
+      factory.subject_certificate = cert
+      factory.issuer_certificate = cert
+      extensions.each do |ln_sn, value, critical|
+        cert.add_extension(factory.create_extension(ln_sn, value, critical))
+      end
+    end
+  end
+end

data/lib/billy/ssl/certificate.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'openssl'
+require 'fileutils'
+
+module Billy
+  # This class is dedicated to the generation of a request certifcate for a
+  # given domain name. We have to generate for each handled connection a new
+  # request certifcate, due to the fact that each request has probably a
+  # different domain name which will be proxied. So we can't know of future
+  # domain name we could include in the list of subject alternative names
+  # which is required by modern browsers. (Chrome 58+)
+  #
+  # We use our generated certifcate authority to sign any request certifcate,
+  # so a client can be prepared to trust us before a possible test scenario
+  # starts.
+  #
+  # This behaviour and functionality mimics the mighty mitmproxy and it will
+  # enable the usage of Chrome Headless at a time where no ssl issue ignoring
+  # works. And its even secure at testing level.
+  class Certificate
+    include Billy::CertificateHelpers
+
+    attr_reader :key, :cert, :domain
+
+    # To generate a new request certifcate just pass the domain in and you
+    # are ready to go.
+    #
+    # Example:
+    #
+    #   cert = Billy::Certificate.new('localhost')
+    #   [cert.cert_file, cert.key_file]
+    def initialize(domain)
+      @domain = domain
+      @key = OpenSSL::PKey::RSA.new(2048)
+      @cert = generate
+    end
+
+    # Write out the private key to file (PEM format) and give back the
+    # file path.
+    def key_file
+      write_file("request-#{domain}.key", key.to_pem)
+    end
+
+    # Write out the certifcate to file (PEM format) and give back the
+    # file path.
+    def cert_file
+      write_file("request-#{domain}.crt", cert.to_pem)
+    end
+
+    private
+
+    # Defines a static list of available extensions on the certificate.
+    def extensions
+      # ln_sn, value, critical
+      [['subjectAltName', "DNS:#{domain}", false]]
+    end
+
+    # Generate a fresh new certificate for the configured domain.
+    def generate
+      cert = OpenSSL::X509::Certificate.new
+      configure(cert)
+      add_extensions(cert)
+      cert.sign(Billy.certificate_authority.key, OpenSSL::Digest::SHA256.new)
+    end
+
+    # Generate a new certificate signing request (CSR) which will be picked
+    # up by the certificate subject and public key.
+    def signing_request
+      req = OpenSSL::X509::Request.new
+      req.public_key = key.public_key
+      req.subject = OpenSSL::X509::Name.new([['CN', domain]])
+      req.sign(key, OpenSSL::Digest::SHA256.new)
+    end
+
+    # Setup all relevant properties of the given certificate to produce
+    # a valid and useable certificate.
+    def configure(cert)
+      req = signing_request
+      cert.issuer = Billy.certificate_authority.cert.subject
+      cert.not_before = days_ago(2)
+      cert.not_after = days_from_now(2)
+      cert.public_key = req.public_key
+      cert.serial = serial
+      cert.subject = req.subject
+      cert.version = 2
+    end
+
+    # Add all extensions (defined by the +extensions+ method) to the given
+    # certificate.
+    def add_extensions(cert)
+      factory = OpenSSL::X509::ExtensionFactory.new
+      factory.issuer_certificate = Billy.certificate_authority.cert
+      factory.subject_certificate = cert
+      extensions.each do |ln_sn, value, critical|
+        cert.add_extension(factory.create_extension(ln_sn, value, critical))
+      end
+    end
+  end
+end