puffing-billy 0.2.1 → 0.2.3

data/lib/billy/config.rb

@@ -5,7 +5,9 @@ module Billy
   class Config
     DEFAULT_WHITELIST = ['127.0.0.1', 'localhost']
 
-    attr_accessor :logger, :cache, :whitelist, :ignore_params, :persist_cache, :cache_path
+    attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params,
+                  :persist_cache, :ignore_cache_port, :non_successful_cache_disabled, :non_successful_error_level,
+                  :non_whitelisted_requests_disabled, :cache_path
 
     def initialize
       @logger = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
@@ -14,10 +16,16 @@ module Billy
 
     def reset
       @cache = true
+      @cache_request_headers = false
       @whitelist = DEFAULT_WHITELIST
+      @path_blacklist = []
       @ignore_params = []
       @persist_cache = false
-      @cache_path = Dir.tmpdir
+      @ignore_cache_port = true
+      @non_successful_cache_disabled = false
+      @non_successful_error_level = :warn
+      @non_whitelisted_requests_disabled = false
+      @cache_path = File.join(Dir.tmpdir, 'puffing-billy')
     end
   end
 

data/lib/billy/json_utils.rb

@@ -0,0 +1,40 @@
+require 'uri'
+require 'json'
+
+module Billy
+  module JSONUtils
+
+    def self.json?(value)
+      !!JSON.parse(value)
+    rescue JSON::ParserError, TypeError
+      false
+    end
+
+    # Recursively sorts the key/value pairs of all hashes within the given
+    #   data structure while preserving the order of arrays.
+    def self.sort_hash_keys(data)
+      return data unless data.is_a?(Hash) || data.is_a?(Array)
+      if data.is_a? Hash
+        data.keys.sort.reduce({}) do |seed, key|
+          seed[key] = sort_hash_keys(data[key])
+          seed
+        end
+      else
+        data.map do |element|
+          sort_hash_keys(element)
+        end
+      end
+    end
+
+    # Recursively sorts the name/value pairs of JSON objects. For instance,
+    #   sort_json('{"b" : "2", "a" : "1"}') == sort_json('{"a" : "1", "b" : "2"}')
+    #   Per http://json.org/, "An object is an unordered set of name/value pairs"
+    #   and "An array is an ordered collection of values". So, we can sort the
+    #   name/value pairs by name (key), but we must preserve the order of an array.
+    #   Processing JSON in this way enables a consistent SHA to be derived from
+    #   JSON payloads which have the same name/value pairs, but different orders.
+    def self.sort_json(json_str)
+      JSONUtils::sort_hash_keys(JSON.parse(json_str, symbolize_names: true)).to_json
+    end
+  end
+end

data/lib/billy/proxy.rb

@@ -4,6 +4,8 @@ require 'eventmachine'
 
 module Billy
   class Proxy
+    attr_reader :cache
+
     def initialize
       reset
       @cache = Billy::Cache.new
@@ -54,8 +56,8 @@ module Billy
     end
 
     def restore_cache
+      warn "[DEPRECATION] `restore_cache` is deprecated as cache files are dynamincally checked. Use `reset_cache` if you just want to clear the cache."
       @cache.reset
-      @cache.load_dir
     end
 
     protected
@@ -76,7 +78,7 @@ module Billy
           p.cache = @cache
         end
 
-        Billy.log(:info, "Proxy listening on #{url}")
+        Billy.log(:info, "puffing-billy: Proxy listening on #{url}")
       end
     end
   end

data/lib/billy/proxy_connection.rb

@@ -74,21 +74,38 @@ module Billy
       end
 
       if result
-        Billy.log(:info, "STUB #{@parser.http_method} #{@url}")
-        response = EM::DelegatedHttpResponse.new(self)
-        response.status = result[0]
-        response.headers = result[1].merge('Connection' => 'close')
-        response.content = result[2]
-        response.send_response
+        Billy.log(:info, "puffing-billy: STUB #{@parser.http_method} for '#{@url}'")
+        stub_request(result)
       elsif cache.cached?(@parser.http_method.downcase, @url, @body)
-        Billy.log(:info, "CACHE #{@parser.http_method} #{@url}")
+        Billy.log(:info, "puffing-billy: CACHE #{@parser.http_method} for '#{@url}'")
         respond_from_cache
-      else
-        Billy.log(:info, "PROXY #{@parser.http_method} #{@url}")
+      elsif !disabled_request?
+        Billy.log(:info, "puffing-billy: PROXY #{@parser.http_method} for '#{@url}'")
         proxy_request
+      else
+        close_connection
+        body_msg = @parser.http_method == 'post' ? " with body '#{@body}'" : ''
+        raise "puffing-billy: Connection to #{@url}#{body_msg} not cached and new http connections are disabled"
       end
     end
 
+    def stub_request(result)
+      response = EM::DelegatedHttpResponse.new(self)
+      response.status  = result[0]
+      response.headers = result[1].merge('Connection' => 'close')
+      response.content = result[2]
+      response.send_response
+    end
+
+    def respond_from_cache
+      cached_res = cache.fetch(@parser.http_method.downcase, @url, @body)
+      res = EM::DelegatedHttpResponse.new(self)
+      res.status = cached_res[:status]
+      res.headers = cached_res[:headers]
+      res.content = cached_res[:content]
+      res.send_response
+    end
+
     def proxy_request
       headers = Hash[@headers.map { |k,v| [k.downcase, v] }]
       headers.delete('accept-encoding')
@@ -105,7 +122,7 @@ module Billy
       req = req.send(@parser.http_method.downcase, req_opts)
 
       req.errback do
-        Billy.log(:error, "Request failed: #{@url}")
+        Billy.log(:error, "puffing-billy: Request failed: #{@url}")
         close_connection
       end
 
@@ -115,8 +132,11 @@ module Billy
         res_headers = res_headers.merge('Connection' => 'close')
         res_headers.delete('Transfer-Encoding')
         res_content = req.response.force_encoding('BINARY')
-        if cache.cacheable?(@url, res_headers)
-          cache.store(@parser.http_method.downcase, @url, @body, res_status, res_headers, res_content)
+
+        handle_response_code(res_status)
+
+        if cacheable?(res_headers, res_status)
+          cache.store(@parser.http_method.downcase, @url, headers, @body, res_headers, res_status, res_content)
         end
 
         res = EM::DelegatedHttpResponse.new(self)
@@ -127,13 +147,59 @@ module Billy
       end
     end
 
-    def respond_from_cache
-      cached_res = cache.fetch(@parser.http_method.downcase, @url, @body)
-      res = EM::DelegatedHttpResponse.new(self)
-      res.status = cached_res[:status]
-      res.headers = cached_res[:headers]
-      res.content = cached_res[:content]
-      res.send_response
+    def disabled_request?
+      url = URI(@url)
+      # In isolated environments, you may want to stop the request from happening
+      # or else you get "getaddrinfo: Name or service not known" errors
+      if Billy.config.non_whitelisted_requests_disabled
+        blacklisted_path?(url.path) || !whitelisted_url?(url)
+      end
+    end
+
+    def handle_response_code(status)
+      log_level = successful_status?(status) ? :info : Billy.config.non_successful_error_level
+      log_message = "puffing-billy: Received response status code #{status} for #{@url}"
+      Billy.log(log_level, log_message)
+      if log_level == :error
+        close_connection
+        raise log_message
+      end
+    end
+
+    def cacheable?(headers, status)
+      if Billy.config.cache
+        url = URI(@url)
+        # Cache the responses if they aren't whitelisted host[:port]s but always cache blacklisted paths on any hosts
+        cacheable_headers?(headers) && cacheable_status?(status) && (!whitelisted_url?(url) || blacklisted_path?(url.path))
+      end
+    end
+
+ private
+
+    def whitelisted_host?(host)
+      Billy.config.whitelist.include?(host)
     end
+
+    def whitelisted_url?(url)
+      whitelisted_host?(url.host) || whitelisted_host?("#{url.host}:#{url.port}")
+    end
+
+    def blacklisted_path?(path)
+      Billy.config.path_blacklist.index{|bl| path.include?(bl)}
+    end
+
+    def successful_status?(status)
+      (200..299).include?(status) || status == 304
+    end
+
+    def cacheable_headers?(headers)
+      #TODO: test headers for cacheability (ie. Cache-Control: no-cache)
+      true
+    end
+
+    def cacheable_status?(status)
+      Billy.config.non_successful_cache_disabled ? successful_status?(status) : true
+    end
+
   end
 end

data/lib/billy/proxy_request_stub.rb

@@ -6,7 +6,7 @@ module Billy
       @options = {:method => :get}.merge(options)
       @method = @options[:method].to_s.upcase
       @url = url
-      @response = [204, {}, ""]
+      @response = {code: 204, headers: {}, text: ""}
     end
 
     def and_return(response)

data/lib/billy/railtie.rb

@@ -0,0 +1,9 @@
+module Billy
+  class Railtie < Rails::Railtie
+    railtie_name 'billy'
+
+    rake_tasks do
+      load 'tasks/billy.rake'
+    end
+  end
+end

data/lib/billy/rspec.rb

@@ -1,4 +1,3 @@
-require 'rspec'
 require 'capybara/rspec'
 require 'billy'
 

data/lib/billy/version.rb

@@ -1,3 +1,3 @@
 module Billy
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

data/lib/puffing-billy.rb

@@ -0,0 +1,2 @@
+require 'billy'
+PuffingBilly ||= Billy

data/lib/puffing-billy/rspec.rb

@@ -0,0 +1,2 @@
+require 'billy/rspec'
+PuffingBilly ||= Billy

data/lib/tasks/billy.rake

@@ -0,0 +1,87 @@
+namespace :cache do
+
+  desc 'Print out all cache file information'
+  task :print_all do
+    cache_array = load_cache
+
+    sort_cache(cache_array).each do |cache|
+      print_cache_details(cache)
+    end
+  end
+
+  desc 'Print out specific cache file information'
+  task :print_details, :sha do |t, args|
+    raise "Missing sha; usage: rake cache:print_details['<sha>']" unless args[:sha]
+    cache_array = load_cache(Billy.config.cache_path, '*'+args[:sha]+'*.yml')
+
+    sort_cache(cache_array).each do |cache|
+      print_cache_details(cache)
+    end
+  end
+
+  desc 'Find specific cache files by URL'
+  task :find_by_url, :api_path do |t, args|
+    raise "Missing api path; usage: rake cache:find_by_url['<api_path>']" unless args[:api_path]
+    cache_array = load_cache
+    filtered_cache_array = cache_array.select {|f| f[:url_path].include?(args[:api_path]) }
+
+    sort_cache(filtered_cache_array).each do |cache|
+      print_cache_details(cache)
+    end
+  end
+
+  desc 'Find specific cache files by scope'
+  task :find_by_scope, :scope do |t, args|
+    raise "Missing scope; usage: rake cache:find_by_scope['<scope>']" unless args[:scope]
+    cache_array = load_cache
+    filtered_cache_array = cache_array.select {|f| f[:scope] && f[:scope].include?(args[:scope]) }
+
+    sort_cache(filtered_cache_array).each do |cache|
+      print_cache_details(cache)
+    end
+  end
+
+  desc 'Find cache files with non-successful status codes'
+  task :find_non_successful do
+    cache_array = load_cache
+    filtered_cache_array = cache_array.select {|f| !(200..299).include?(f[:status]) }
+
+    sort_cache(filtered_cache_array).each do |cache|
+      print_cache_details(cache)
+    end
+  end
+
+  def load_cache(cache_directory = Billy.config.cache_path, file_pattern = '*.yml')
+    cache_path  = Rails.root.join(cache_directory)
+    cache_array = []
+
+    Dir.glob(cache_path+file_pattern) do |filename|
+      data = load_cache_file(filename)
+      url = URI(data[:url])
+      data[:url_path] = "#{url.path}#{url.query ? '?'+url.query : ''}#{url.fragment ? '#'+url.fragment : ''}"
+      data[:filename] = filename.gsub(Rails.root.to_s+'/','')
+      cache_array << data
+    end
+    cache_array
+  end
+
+  def load_cache_file(filename)
+    YAML.load(File.open(filename))
+  rescue ArgumentError => e
+    puts "Could not parse YAML: #{e.message}"
+  end
+
+  def print_cache_details(cache)
+    puts "   Scope: #{cache[:scope]}" if cache[:scope]
+    puts "     URL: #{cache[:url]}"
+    puts "    Body: #{cache[:body]}" if cache[:method] == 'post'
+    puts " Details: Request method '#{cache[:method]}' returned response status code: '#{cache[:status]}'"
+    puts "Filename: #{cache[:filename]}"
+    puts "\n\n"
+  end
+
+  def sort_cache(cache, key = :url_path)
+    cache.sort_by { |hsh| hsh[key] }
+  end
+
+end

data/puffing-billy.gemspec

@@ -20,11 +20,11 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "faraday"
   gem.add_development_dependency "poltergeist"
   gem.add_development_dependency "selenium-webdriver"
-  gem.add_development_dependency "capybara-webkit"
+  gem.add_development_dependency "capybara-webkit", "~> 1.0"
   gem.add_development_dependency "rack"
   gem.add_development_dependency "guard"
   gem.add_development_dependency "rb-inotify"
-  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
   gem.add_development_dependency "cucumber"
   gem.add_runtime_dependency "eventmachine"
   gem.add_runtime_dependency "em-http-request"

data/spec/features/examples/facebook_api_spec.rb

@@ -17,6 +17,7 @@ describe 'Facebook API example', :type => :feature, :js => true do
 
   it 'should show me as logged-in', :js => true do
     visit '/facebook_api.html'
-    page.should have_content "Hi, Tester 1"
+    click_on "Login"
+    expect(page).to have_content "Hi, Tester 1"
   end
 end

data/spec/features/examples/tumblr_api_spec.rb

@@ -21,9 +21,9 @@ describe 'Tumblr API example', :type => :feature, :js => true do
 
   it 'should show news stories', :js => true do
     visit '/tumblr_api.html'
-    page.should have_link('News Item 1', :href => 'http://example.com/news/1')
-    page.should have_content('News item 1 content here')
-    page.should have_link('News Item 2', :href => 'http://example.com/news/2')
-    page.should have_content('News item 2 content here')
+    expect(page).to have_link('News Item 1', :href => 'http://example.com/news/1')
+    expect(page).to have_content('News item 1 content here')
+    expect(page).to have_link('News Item 2', :href => 'http://example.com/news/2')
+    expect(page).to have_content('News item 2 content here')
   end
 end

data/spec/lib/billy/cache_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe Billy::Cache do
+  describe 'format_url' do
+    let(:cache) { Billy::Cache.new }
+    let(:params) { '?foo=bar' }
+    let(:fragment) { '#baz' }
+    let(:base_url) { 'http://example.com' }
+    let(:fragment_url) { "#{base_url}/#{fragment}" }
+    let(:params_url) { "#{base_url}#{params}" }
+    let(:params_fragment_url) { "#{base_url}#{params}#{fragment}" }
+
+    context 'with ignore_params set to false' do
+      it 'is a no-op if there are no params' do
+        expect(cache.format_url(base_url)).to eq base_url
+      end
+      it 'appends params if there are params' do
+        expect(cache.format_url(params_url)).to eq params_url
+      end
+      it 'appends params and fragment if both are present' do
+        expect(cache.format_url(params_fragment_url)).to eq params_fragment_url
+      end
+    end
+
+    context 'with ignore_params set to true' do
+      it 'is a no-op if there are no params' do
+        expect(cache.format_url(base_url, true)).to eq base_url
+      end
+      it 'omits params if there are params' do
+        expect(cache.format_url(params_url, true)).to eq base_url
+      end
+      it 'omits params and fragment if both are present' do
+        expect(cache.format_url(params_fragment_url, true)).to eq base_url
+      end
+    end
+  end
+end

data/spec/lib/billy/proxy_request_stub_spec.rb

@@ -3,32 +3,42 @@ require 'spec_helper'
 describe Billy::ProxyRequestStub do
   context '#matches?' do
     it 'should match urls and methods' do
-      Billy::ProxyRequestStub.new('http://example.com').
-        matches?('GET', 'http://example.com').should be
-      Billy::ProxyRequestStub.new('http://example.com').
-        matches?('POST', 'http://example.com').should_not be
-      Billy::ProxyRequestStub.new('http://example.com', :method => :get).
-        matches?('GET', 'http://example.com').should be
-      Billy::ProxyRequestStub.new('http://example.com', :method => :post).
-        matches?('GET', 'http://example.com').should_not be
-      Billy::ProxyRequestStub.new('http://example.com', :method => :post).
-        matches?('POST', 'http://example.com').should be
-      Billy::ProxyRequestStub.new('http://fooxample.com', :method => :post).
-        matches?('POST', 'http://example.com').should_not be
+      expect(Billy::ProxyRequestStub.new('http://example.com').
+        matches?('GET', 'http://example.com')).to be
+      expect(Billy::ProxyRequestStub.new('http://example.com').
+        matches?('POST', 'http://example.com')).to_not be
+
+      expect(Billy::ProxyRequestStub.new('http://example.com', :method => :get).
+        matches?('GET', 'http://example.com')).to be
+      expect(Billy::ProxyRequestStub.new('http://example.com', :method => :post).
+        matches?('GET', 'http://example.com')).to_not be
+
+      expect(Billy::ProxyRequestStub.new('http://example.com', :method => :post).
+        matches?('POST', 'http://example.com')).to be
+      expect(Billy::ProxyRequestStub.new('http://fooxample.com', :method => :post).
+        matches?('POST', 'http://example.com')).to_not be
     end
 
     it 'should match regexps' do
-      Billy::ProxyRequestStub.new(/http:\/\/.+\.com/, :method => :post).
-        matches?('POST', 'http://example.com').should be
-      Billy::ProxyRequestStub.new(/http:\/\/.+\.co\.uk/, :method => :get).
-        matches?('GET', 'http://example.com').should_not be
+      expect(Billy::ProxyRequestStub.new(/http:\/\/.+\.com/, :method => :post).
+        matches?('POST', 'http://example.com')).to be
+      expect(Billy::ProxyRequestStub.new(/http:\/\/.+\.co\.uk/, :method => :get).
+        matches?('GET', 'http://example.com')).to_not be
     end
 
     it 'should match up to but not including query strings' do
       stub = Billy::ProxyRequestStub.new('http://example.com/foo/bar/')
-      stub.matches?('GET', 'http://example.com/foo/').should_not be
-      stub.matches?('GET', 'http://example.com/foo/bar/').should be
-      stub.matches?('GET', 'http://example.com/foo/bar/?baz=bap').should be
+      expect(stub.matches?('GET', 'http://example.com/foo/')).to_not be
+      expect(stub.matches?('GET', 'http://example.com/foo/bar/')).to be
+      expect(stub.matches?('GET', 'http://example.com/foo/bar/?baz=bap')).to be
+    end
+  end
+
+  context "#call (without #and_return)" do
+    let(:subject) { Billy::ProxyRequestStub.new('url') }
+
+    it "returns a 204 empty response" do
+      expect(subject.call({}, {}, nil)).to eql [204, {"Content-Type" => "text/plain"}, ""]
     end
   end
 
@@ -37,7 +47,7 @@ describe Billy::ProxyRequestStub do
 
     it 'should generate bare responses' do
       subject.and_return :body => 'baz foo bar'
-      subject.call({}, {}, nil).should == [
+      expect(subject.call({}, {}, nil)).to eql [
         200,
         {},
         'baz foo bar'
@@ -46,7 +56,7 @@ describe Billy::ProxyRequestStub do
 
     it 'should generate text responses' do
       subject.and_return :text => 'foo bar baz'
-      subject.call({}, {}, nil).should == [
+      expect(subject.call({}, {}, nil)).to eql [
         200,
         {'Content-Type' => 'text/plain'},
         'foo bar baz'
@@ -55,7 +65,7 @@ describe Billy::ProxyRequestStub do
 
     it 'should generate JSON responses' do
       subject.and_return :json => { :foo => 'bar' }
-      subject.call({}, {}, nil).should == [
+      expect(subject.call({}, {}, nil)).to eql [
         200,
         {'Content-Type' => 'application/json'},
         '{"foo":"bar"}'
@@ -65,7 +75,7 @@ describe Billy::ProxyRequestStub do
     context 'JSONP' do
       it 'should generate JSONP responses' do
         subject.and_return :jsonp => { :foo => 'bar' }
-        subject.call({ 'callback' => ['baz'] }, {}, nil).should == [
+        expect(subject.call({ 'callback' => ['baz'] }, {}, nil)).to eql [
           200,
           {'Content-Type' => 'application/javascript'},
           'baz({"foo":"bar"})'
@@ -74,7 +84,7 @@ describe Billy::ProxyRequestStub do
 
       it 'should generate JSONP responses with custom callback parameter' do
         subject.and_return :jsonp => { :foo => 'bar' }, :callback_param => 'cb'
-        subject.call({ 'cb' => ['bap'] }, {}, nil).should == [
+        expect(subject.call({ 'cb' => ['bap'] }, {}, nil)).to eql [
           200,
           {'Content-Type' => 'application/javascript'},
           'bap({"foo":"bar"})'
@@ -83,7 +93,7 @@ describe Billy::ProxyRequestStub do
 
       it 'should generate JSONP responses with custom callback name' do
         subject.and_return :jsonp => { :foo => 'bar' }, :callback => 'cb'
-        subject.call({}, {}, nil).should == [
+        expect(subject.call({}, {}, nil)).to eql [
           200,
           {'Content-Type' => 'application/javascript'},
           'cb({"foo":"bar"})'
@@ -93,7 +103,7 @@ describe Billy::ProxyRequestStub do
 
     it 'should generate redirection responses' do
       subject.and_return :redirect_to => 'http://example.com'
-      subject.call({}, {}, nil).should == [
+      expect(subject.call({}, {}, nil)).to eql [
         302,
         {'Location' => 'http://example.com'},
         nil
@@ -102,7 +112,7 @@ describe Billy::ProxyRequestStub do
 
     it 'should set headers' do
       subject.and_return :text => 'foo', :headers => {'HTTP-X-Foo' => 'bar'}
-      subject.call({}, {}, nil).should == [
+      expect(subject.call({}, {}, nil)).to eql [
         200,
         {'Content-Type' => 'text/plain', 'HTTP-X-Foo' => 'bar'},
         'foo'
@@ -111,7 +121,7 @@ describe Billy::ProxyRequestStub do
 
     it 'should set status codes' do
       subject.and_return :text => 'baz', :code => 410
-      subject.call({}, {}, nil).should == [
+      expect(subject.call({}, {}, nil)).to eql [
         410,
         {'Content-Type' => 'text/plain'},
         'baz'
@@ -124,12 +134,12 @@ describe Billy::ProxyRequestStub do
       expected_body = 'body text'
 
       subject.and_return(Proc.new { |params, headers, body|
-        params.should == expected_params
-        headers.should == expected_headers
-        body.should == 'body text'
+        expect(params).to eql expected_params
+        expect(headers).to eql expected_headers
+        expect(body).to eql 'body text'
         {:code => 418, :text => 'success'}
       })
-      subject.call(expected_params, expected_headers, expected_body).should == [
+      expect(subject.call(expected_params, expected_headers, expected_body)).to eql [
         418,
         {'Content-Type' => 'text/plain'},
         'success'