puffing-billy 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77894c8ea71e9f2b91cb3aae48c6bc1311ab45ba
-  data.tar.gz: 2af9cc1a8fafdfafa15ac9bf1d088e2cb648ae59
+  metadata.gz: 30ec3de1e00ab91373eb88106e301a8fe9963701
+  data.tar.gz: 7300cdd6f73ae4e5a5a6e6a1dd6108cb9074d272
 SHA512:
-  metadata.gz: 00faa0d3c37550847f947b5440770e628978593476771c27f26d31516b8ff176540340a00cba098b2fca5ebcd86bc53b29a94d53af57c4b92547be3a3b07f603
-  data.tar.gz: 642297885633e8f29efab30ad78faef48d173dd4b4a902cb92cbef118c616b349a01671d27f5b3b38af8b8ad4b341f251d0d23fdff9e1b7b3096b118efd246a6
+  metadata.gz: 1e00330461d05f7fa3e3f01cad76e35e152f2ffc44c7b3633d0b780c60e714429185b7200897eaef6c0d96e7067a78cb1cd63e680013601f9f873f3e0f58eeff
+  data.tar.gz: 6d1099b167707b5c41f38444f81ff39c6d6a1753e8072fdbbca85c2e2a605c391591ebf095bff3b6173e4a7060304b0c30c52e1a92723f0c502bb7a4a78bf036

data/.gitignore

@@ -2,3 +2,4 @@ node_modules/
 log/test.log
 .idea/
 .ruby-version
+*.swp

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+script: bundle exec rspec spec

data/CHANGELOG.md

@@ -0,0 +1,65 @@
+v0.3.0, 2014-12-29
+------------------
+
+  * Fixing a bug where proxy to SSL can duplicate https in the request_url (#36)
+  * Refactor proxy responses (#37)
+  * Update http_parser to 0.6.0 and remove CONNECT request hack (#38)
+  * Allow configurable Billy proxy port (#40)
+  * Refactor handlers (#41)
+  * Mark step definitions as ruby code (#52)
+  * Adds EventMachine timeout configuration (#57)
+  * Support dynamic jsonp with params (#58)
+  * Writing error messages to the logger rather than stdout (#69)
+  * Don't recommend changing javascript_driver config (#70)
+  * README link pointing at wrong target (#73)
+  * Adding example config to README for playing nicely with Webmock, VCR (#74)
+  * Make dynamic_jsonp regex less brittle (#81)
+
+v0.2.3, 2014-02-07
+------------------
+
+  * Fixed facebook spec (#24)
+  * Check for existing persistent cache files on demand (#26)
+  * Lazy-loading proxy and other minor fixes (#28)
+  * Support service-oriented architectures, scope cache to scenarios, and sort JSON in POSTs to avoid duplicate cache files (#30)
+  * Set the minimum version of capybara-webkit to 1.0.0 (#31)
+  * Updated gems, cache request headers, handle non-successful responses, ability to stop new connections (#33)
+  * Add requires matching gem name (#34)
+  * Remove duplicated rspec devel dependency (#35)
+
+v0.2.1, 2013-05-12
+------------------
+
+  * Add cucumber documentation to readme. (#12)
+  * Use multi_json (#13)
+  * Remove require from Gemfile example (#14)
+  * Add a README example of returning headers (#16)
+
+v0.2.0, 2013-03-17
+------------------
+
+  * Update README with HTTPS quirk and trailing slash behaviour. (#3)
+  * Fixes to work with Capybara-Webkit (#6)
+  * VCR-like cache (#7)
+
+v0.1.3, 2012-11-05
+------------------
+
+  * Implemented caching
+
+v0.1.2, 2012-10-12
+------------------
+
+  * Slightly saner driver setup
+  * Updated README
+
+v0.1.1, 2012-10-12
+------------------
+
+  * Content encoding fixes
+  * Updated README
+
+v0.1.0, 2012-10-11
+------------------
+
+  * Initial release

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    puffing-billy (0.2.3)
+    puffing-billy (0.3.0)
       capybara
       em-http-request
+      em-synchrony
       eventmachine
       eventmachine_httpserver
-      http_parser.rb
+      http_parser.rb (~> 0.6.0)
       multi_json
 
 GEM
@@ -29,7 +30,7 @@ GEM
       ffi (~> 1.0, >= 1.0.11)
     cliver (0.3.2)
     coderay (1.1.0)
-    cookiejar (0.3.0)
+    cookiejar (0.3.1)
     cucumber (1.3.10)
       builder (>= 2.1.2)
       diff-lcs (>= 1.1.3)
@@ -46,6 +47,8 @@ GEM
       http_parser.rb (>= 0.6.0)
     em-socksify (0.3.0)
       eventmachine (>= 1.0.0.beta.4)
+    em-synchrony (1.0.3)
+      eventmachine (>= 1.0.0.beta.1)
     eventmachine (1.0.3)
     eventmachine_httpserver (0.2.1)
     faraday (0.9.0)

data/README.md

@@ -120,10 +120,9 @@ Feature: Stubbing via billy
 
 And in steps:
 
-```
+```ruby
 Before('@billy') do
   Capybara.current_driver = :poltergeist_billy
-  Capybara.javascript_driver = :poltergeist_billy
 end
 
 And /^a stub for google$/ do
@@ -202,6 +201,18 @@ caching. You should mostly use this for analytics and various social buttons as
 they use cache avoidance techniques, but return practically the same response
 that most often does not affect your test results.
 
+`c.dynamic_jsonp` is used to rewrite the body of JSONP responses based on the
+callback parameter. For example, if a request to `http://example.com/foo?callback=bar` 
+returns `bar({"some": "json"});` and is recorded, then a later request to
+`http://example.com/foo?callback=baz` will be a cache hit and respond with
+`baz({"some": "json"});` This is useful because most JSONP implementations 
+base the callback name off of a timestamp or something else dynamic.
+
+`c.dynamic_jsonp_keys` is used to configure which parameters to ignore when
+using `c.dynamic_jsonp`. This is helpful when JSONP APIs use cache-busting
+parameters. For example, if you want `http://example.com/foo?callback=bar&id=1&cache_bust=12345` and `http://example.com/foo?callback=baz&id=1&cache_bust=98765` to be cache hits for each other, you would set `c.dynamic_jsonp_keys = ['callback', 'cache_bust']` to ignore both params. Note
+that in this example the `id` param would still be considered important.
+
 `c.path_blacklist = []` is used to always cache specific paths on any hostnames,
 including whitelisted ones.  This is useful if your AUT has routes that get data
 from external services, such as `/api` where the ajax request is a local URL but
@@ -287,13 +298,42 @@ RSpec.configure do |config|
 end
 ```
 
+## Proxy timeouts
+
+By default, the Puffing Billy proxy will use the EventMachine:HttpRequest timeouts of 5 seconds
+for connect and 10 seconds for inactivity when talking to downstream servers.
+
+These can be configured as follows:
+
+```ruby
+Billy.configure do |c|
+  c.proxied_request_connect_timeout = 20
+  c.proxied_request_inactivity_timeout = 20
+end
+```
+
 ## Customising the javascript driver
 
 If you use a customised Capybara driver, remember to set the proxy address
 and tell it to ignore SSL certificate warnings. See
-[lib/billy/rspec.rb](https://github.com/oesmith/puffing-billy/blob/master/lib/billy/rspec.rb)
+[lib/billy.rb](https://github.com/oesmith/puffing-billy/blob/master/lib/billy.rb)
 to see how Billy's default drivers are configured.
 
+## Working with VCR and Webmock
+If you use VCR and Webmock elsewhere in your specs, you will need to disable them
+for your specs utilizing Puffing Billy. To do so, you can configure your `spec_helper.rb`
+as shown below:
+
+```ruby
+RSpec.configure do |config|
+  config.around(:each, type: :feature) do |example|
+    WebMock.allow_net_connect!
+    VCR.turned_off { example.run }
+    WebMock.disable_net_connect!
+  end
+end
+```
+
 ## FAQ
 
 1. Why name it after a train?

data/lib/billy.rb

@@ -1,5 +1,10 @@
 require "billy/version"
 require "billy/config"
+require "billy/handlers/handler"
+require "billy/handlers/request_handler"
+require "billy/handlers/stub_handler"
+require "billy/handlers/proxy_handler"
+require "billy/handlers/cache_handler"
 require "billy/proxy_request_stub"
 require "billy/cache"
 require "billy/proxy"

data/lib/billy/cache.rb

@@ -2,9 +2,12 @@ require 'resolv'
 require 'uri'
 require 'yaml'
 require 'billy/json_utils'
+require 'singleton'
 
 module Billy
   class Cache
+    include Singleton
+
     attr_reader :scope
 
     def initialize
@@ -30,7 +33,7 @@ module Billy
       begin
         @cache[key] = YAML.load(File.open(cache_file(key))) if persisted?(key)
       rescue ArgumentError => e
-        puts "Could not parse YAML: #{e.message}"
+        Billy.log :error, "Could not parse YAML: #{e.message}"
         nil
       end
     end
@@ -83,14 +86,27 @@ module Billy
       key
     end
 
-    def format_url(url, ignore_params=false)
+    def format_url(url, ignore_params=false, dynamic_jsonp=Billy.config.dynamic_jsonp)
       url = URI(url)
       port_to_include = Billy.config.ignore_cache_port ? '' : ":#{url.port}"
       formatted_url = url.scheme+'://'+url.host+port_to_include+url.path
-      unless ignore_params
-        formatted_url += '?'+url.query if url.query
-        formatted_url += '#'+url.fragment if url.fragment
+
+      return formatted_url if ignore_params
+
+      if url.query
+        query_string = if dynamic_jsonp
+                         query_hash = Rack::Utils.parse_query(url.query)
+                         Billy.config.dynamic_jsonp_keys.each{|k| query_hash.delete(k) }
+                         Rack::Utils.build_query(query_hash)
+                       else
+                         url.query
+                       end
+
+        formatted_url += "?#{query_string}"
       end
+
+      formatted_url += '#'+url.fragment if url.fragment
+
       formatted_url
     end
 

data/lib/billy/config.rb

@@ -4,10 +4,12 @@ require 'tmpdir'
 module Billy
   class Config
     DEFAULT_WHITELIST = ['127.0.0.1', 'localhost']
+    RANDOM_AVAILABLE_PORT = 0 # https://github.com/eventmachine/eventmachine/wiki/FAQ#wiki-can-i-start-a-server-on-a-random-available-port
 
     attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params,
                   :persist_cache, :ignore_cache_port, :non_successful_cache_disabled, :non_successful_error_level,
-                  :non_whitelisted_requests_disabled, :cache_path
+                  :non_whitelisted_requests_disabled, :cache_path, :proxy_port, :proxied_request_inactivity_timeout,
+                  :proxied_request_connect_timeout, :dynamic_jsonp, :dynamic_jsonp_keys
 
     def initialize
       @logger = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
@@ -21,11 +23,16 @@ module Billy
       @path_blacklist = []
       @ignore_params = []
       @persist_cache = false
+      @dynamic_jsonp = false
+      @dynamic_jsonp_keys = ["callback"]
       @ignore_cache_port = true
       @non_successful_cache_disabled = false
       @non_successful_error_level = :warn
       @non_whitelisted_requests_disabled = false
       @cache_path = File.join(Dir.tmpdir, 'puffing-billy')
+      @proxy_port = RANDOM_AVAILABLE_PORT
+      @proxied_request_inactivity_timeout = 10 # defaults from https://github.com/igrigorik/em-http-request/wiki/Redirects-and-Timeouts
+      @proxied_request_connect_timeout = 5
     end
   end
 

data/lib/billy/handlers/cache_handler.rb

@@ -0,0 +1,51 @@
+require 'billy/handlers/handler'
+require 'cgi'
+
+module Billy
+  class CacheHandler
+    extend Forwardable
+    include Handler
+
+    def_delegators :cache, :reset, :cached?
+
+    def initialize
+      @cache = Billy::Cache.instance
+    end
+
+    def handle_request(method, url, headers, body)
+      method = method.downcase
+      if handles_request?(method, url, headers, body)
+        if (response = cache.fetch(method, url, body))
+          Billy.log(:info, "puffing-billy: CACHE #{method} for '#{url}'")
+
+          if Billy.config.dynamic_jsonp
+            replace_response_callback(response, url)
+          end
+
+          return response
+        end
+      end
+      nil
+    end
+
+    def handles_request?(method, url, headers, body)
+      cached?(method, url, body)
+    end
+
+    private
+
+    def replace_response_callback(response, url)
+      request_uri = URI::parse(url)
+      if request_uri.query
+        params = CGI::parse(request_uri.query)
+        if params['callback'].any? and response[:content].match(/\w+\(/)
+          response[:content].sub!(/\w+\(/, params['callback'].first + '(')
+        end
+      end
+    end
+
+    def cache
+      @cache
+    end
+  end
+end

data/lib/billy/handlers/handler.rb

@@ -0,0 +1,45 @@
+module Billy
+  module Handler
+    ##
+    #
+    # Handles an incoming HTTP request and returns a response.
+    #
+    # This method accepts HTTP request parameters and must return
+    # a response hash containing the keys :status, :headers,
+    # and :content, or nil if the request cannot be fulfilled.
+    #
+    # @param  [String] http_method  The HTTP method used, e.g. 'http' or 'https'
+    # @param  [String] url          The URL requested.
+    # @param  [String] headers      The headers of the HTTP request.
+    # @param  [String] body         The body of the HTTP request.
+    # @return [Hash]                A hash with the keys [:status, :headers, :content]
+    #                               Returns {:error => "Some error message"} if a failure occurs.
+    #                               Returns nil if the request cannot be fulfilled.
+    def handle_request(http_method, url, headers, body)
+      { error: 'The handler has not overridden the handle_request method!' }
+    end
+
+    ##
+    #
+    # Checks if the Handler can respond to the given request.
+    #
+    # @param  [String] http_method  The HTTP method used, e.g. 'http' or 'https'
+    # @param  [String] url          The URL requested.
+    # @param  [String] headers      The headers of the HTTP request.
+    # @param  [String] body         The body of the HTTP request.
+    # @return [Boolean]             True if the Handler can respond to the request, else false.
+    #
+    def handles_request?(http_method, url, headers, body)
+      false
+    end
+
+    ##
+    #
+    # Resets the Handler to the default/new state
+    #
+    # This allows the handler to be set back to its default state
+    # at the end of tests or whenever else necessary.
+    #
+    def reset; end
+  end
+end

data/lib/billy/handlers/proxy_handler.rb

@@ -0,0 +1,112 @@
+require 'billy/handlers/handler'
+require 'eventmachine'
+require 'em-synchrony/em-http'
+
+module Billy
+  class ProxyHandler
+    include Handler
+
+    def handles_request?(method, url, headers, body)
+      !disabled_request?(url)
+    end
+
+    def handle_request(method, url, headers, body)
+      if handles_request?(method, url, headers, body)
+        req = EventMachine::HttpRequest.new(url, {
+            :inactivity_timeout => Billy.config.proxied_request_inactivity_timeout,
+            :connect_timeout => Billy.config.proxied_request_connect_timeout})
+
+        req = req.send(method.downcase, build_request_options(headers, body))
+
+        if req.error
+          return { :error => "Request to #{url} failed with error: #{req.error}" }
+        end
+
+        if req.response
+          response = process_response(req)
+
+          unless allowed_response_code?(response[:status])
+            if Billy.config.non_successful_error_level == :error
+              return { :error => "Request failed due to response status #{response[:status]} for '#{url}' which was not allowed." }
+            else
+              Billy.log(:warn, "puffing-billy: Received response status code #{response[:status]} for '#{url}'")
+            end
+          end
+
+          if cacheable?(url, response[:headers], response[:status])
+            Billy::Cache.instance.store(method.downcase, url, headers, body, response[:headers], response[:status], response[:content])
+          end
+
+          Billy.log(:info, "puffing-billy: PROXY #{method} succeeded for '#{url}'")
+          return response
+        end
+      end
+      nil
+    end
+
+    private
+
+    def build_request_options(headers, body)
+      headers = Hash[headers.map { |k,v| [k.downcase, v] }]
+      headers.delete('accept-encoding')
+
+      req_opts = {
+          :redirects => 0,
+          :keepalive => false,
+          :head => headers,
+          :ssl => { :verify => false }
+      }
+      req_opts[:body] = body if body
+      req_opts
+    end
+
+    def process_response(req)
+      response = {
+          :status  => req.response_header.status,
+          :headers => req.response_header.raw,
+          :content => req.response.force_encoding('BINARY') }
+      response[:headers].merge!('Connection' => 'close')
+      response[:headers].delete('Transfer-Encoding')
+      response
+    end
+
+    def disabled_request?(url)
+      return false unless Billy.config.non_whitelisted_requests_disabled
+
+      uri = URI(url)
+      # In isolated environments, you may want to stop the request from happening
+      # or else you get "getaddrinfo: Name or service not known" errors
+      blacklisted_path?(uri.path) || !whitelisted_url?(uri)
+    end
+
+    def allowed_response_code?(status)
+      successful_status?(status)
+    end
+
+    def cacheable?(url, headers, status)
+      return false unless Billy.config.cache
+
+      url = URI(url)
+      # Cache the responses if they aren't whitelisted host[:port]s but always cache blacklisted paths on any hosts
+      cacheable_status?(status) && (!whitelisted_url?(url) || blacklisted_path?(url.path))
+    end
+
+    def whitelisted_url?(url)
+      !Billy.config.whitelist.index do |v|
+        v =~ /^#{url.host}(?::#{url.port})?$/
+      end.nil?
+    end
+
+    def blacklisted_path?(path)
+      !Billy.config.path_blacklist.index{|bl| path.include?(bl)}.nil?
+    end
+
+    def successful_status?(status)
+      (200..299).include?(status) || status == 304
+    end
+
+    def cacheable_status?(status)
+      Billy.config.non_successful_cache_disabled ? successful_status?(status) : true
+    end
+  end
+end