puffing-billy 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30ec3de1e00ab91373eb88106e301a8fe9963701
-  data.tar.gz: 7300cdd6f73ae4e5a5a6e6a1dd6108cb9074d272
+  metadata.gz: 17920d6438879d1d259e39e392c06bd47b42fce5
+  data.tar.gz: c9637100ed8fe09ac260dc2c29fe3047975e9a69
 SHA512:
-  metadata.gz: 1e00330461d05f7fa3e3f01cad76e35e152f2ffc44c7b3633d0b780c60e714429185b7200897eaef6c0d96e7067a78cb1cd63e680013601f9f873f3e0f58eeff
-  data.tar.gz: 6d1099b167707b5c41f38444f81ff39c6d6a1753e8072fdbbca85c2e2a605c391591ebf095bff3b6173e4a7060304b0c30c52e1a92723f0c502bb7a4a78bf036
+  metadata.gz: 4eb515c5510665d687778a9672ab955fade8c6fb2c4970056e6122c8688b01755c910e675929588c519b77a9f41ca4f799e0de33e6c5eb0656a286f52aa255ae
+  data.tar.gz: b33a22eb955333f85094a73ffeaf7a59416bdef8b87686815d50329e15619ba70c9dd3bbfe913f47a7805c830bab4f85d266a5d66eaa7845763683ca7d223a55

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+v0.4.0, 2015-01-02
+
+  * Add new configuration to merge cache hits by URL regex (#76)
+  * Allow pipes in URLs by switching to Addressable (#84)
+  * Consistently ignore SSL errors (added to selenium driver) (#85)
+
 v0.3.0, 2014-12-29
 ------------------
 

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    puffing-billy (0.3.0)
-      capybara
+    puffing-billy (0.4.0)
+      addressable
       em-http-request
       em-synchrony
       eventmachine
@@ -13,7 +13,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.3.5)
+    addressable (2.3.6)
     builder (3.2.2)
     capybara (2.1.0)
       mime-types (>= 1.16)
@@ -30,7 +30,7 @@ GEM
       ffi (~> 1.0, >= 1.0.11)
     cliver (0.3.2)
     coderay (1.1.0)
-    cookiejar (0.3.1)
+    cookiejar (0.3.2)
     cucumber (1.3.10)
       builder (>= 2.1.2)
       diff-lcs (>= 1.1.3)
@@ -123,6 +123,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  capybara
   capybara-webkit (~> 1.0)
   cucumber
   faraday

data/README.md

@@ -180,6 +180,7 @@ Billy.configure do |c|
                      "https://www.facebook.com/dialog/oauth",
                      "http://cdn.api.twitter.com/1/urls/count.json"]
   c.path_blacklist = []
+  c.merge_cached_responses_whitelist = []
   c.persist_cache = true
   c.ignore_cache_port = true # defaults to true
   c.non_successful_cache_disabled = false
@@ -218,6 +219,12 @@ including whitelisted ones.  This is useful if your AUT has routes that get data
 from external services, such as `/api` where the ajax request is a local URL but
 the actual data is coming from a different application that you want to cache.
 
+`c.merge_cached_responses_whitelist = []` is used to group together the cached
+responses for specific uri regexes that match any part of the url. This is useful
+for ensuring that any kind of analytics and various social buttons that have
+slightly different urls each time can be recorded once and reused nicely. Note
+that the request body is ignored for requests that contain a body.
+
 `c.ignore_cache_port` is used to strip the port from the URL if it exists.  This
 is useful when caching local paths (via `path_blacklist`) or other local rack apps
 that are running on random ports.

data/lib/billy.rb

@@ -43,9 +43,9 @@ module Billy
     if defined?(Capybara::Webkit::Driver)
       Capybara.register_driver :webkit_billy do |app|
         driver = Capybara::Webkit::Driver.new(app)
+        driver.browser.ignore_ssl_errors
         driver.browser.set_proxy(:host => Billy.proxy.host,
                                  :port => Billy.proxy.port)
-        driver.browser.ignore_ssl_errors
         driver
       end
     end
@@ -53,6 +53,7 @@ module Billy
     if defined?(Selenium::WebDriver)
       Capybara.register_driver :selenium_billy do |app|
         profile = Selenium::WebDriver::Firefox::Profile.new
+        profile.assume_untrusted_certificate_issuer = false
         profile.proxy = Selenium::WebDriver::Proxy.new(
           :http => "#{Billy.proxy.host}:#{Billy.proxy.port}",
           :ssl => "#{Billy.proxy.host}:#{Billy.proxy.port}")

data/lib/billy/cache.rb

@@ -1,5 +1,5 @@
 require 'resolv'
-require 'uri'
+require 'addressable/uri'
 require 'yaml'
 require 'billy/json_utils'
 require 'singleton'
@@ -72,11 +72,16 @@ module Billy
 
     def key(method, orig_url, body, log_key = false)
       ignore_params = Billy.config.ignore_params.include?(format_url(orig_url, true))
-      url = URI(format_url(orig_url, ignore_params))
-      key = method+'_'+url.host+'_'+Digest::SHA1.hexdigest(scope.to_s + url.to_s)
+      merge_cached_response_key = _merge_cached_response_key(orig_url)
+      url = Addressable::URI.parse(format_url(orig_url, ignore_params))
+      key = if merge_cached_response_key
+          method+'_'+Digest::SHA1.hexdigest(scope.to_s + merge_cached_response_key)
+        else
+          method+'_'+url.host+'_'+Digest::SHA1.hexdigest(scope.to_s + url.to_s)
+        end
       body_msg = ''
 
-      if method == 'post' and !ignore_params
+      if method == 'post' && !ignore_params && !merge_cached_response_key
         body_formatted = JSONUtils::json?(body.to_s) ? JSONUtils::sort_json(body.to_s) : body.to_s
         body_msg = " with body '#{body_formatted}'"
         key += '_'+Digest::SHA1.hexdigest(body_formatted)
@@ -87,7 +92,7 @@ module Billy
     end
 
     def format_url(url, ignore_params=false, dynamic_jsonp=Billy.config.dynamic_jsonp)
-      url = URI(url)
+      url = Addressable::URI.parse(url)
       port_to_include = Billy.config.ignore_cache_port ? '' : ":#{url.port}"
       formatted_url = url.scheme+'://'+url.host+port_to_include+url.path
 
@@ -133,6 +138,15 @@ module Billy
 
     private
 
+    def _merge_cached_response_key(url)
+      Billy.config.merge_cached_responses_whitelist.each do |disable_regex|
+        if url =~ disable_regex
+          return disable_regex.to_s # Use the stringified regex as the cache key if it matches
+        end
+      end
+      nil
+    end
+
     attr_writer :scope
   end
 end

data/lib/billy/config.rb

@@ -9,7 +9,7 @@ module Billy
     attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params,
                   :persist_cache, :ignore_cache_port, :non_successful_cache_disabled, :non_successful_error_level,
                   :non_whitelisted_requests_disabled, :cache_path, :proxy_port, :proxied_request_inactivity_timeout,
-                  :proxied_request_connect_timeout, :dynamic_jsonp, :dynamic_jsonp_keys
+                  :proxied_request_connect_timeout, :dynamic_jsonp, :dynamic_jsonp_keys, :merge_cached_responses_whitelist
 
     def initialize
       @logger = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
@@ -21,6 +21,7 @@ module Billy
       @cache_request_headers = false
       @whitelist = DEFAULT_WHITELIST
       @path_blacklist = []
+      @merge_cached_responses_whitelist = []
       @ignore_params = []
       @persist_cache = false
       @dynamic_jsonp = false

data/lib/billy/version.rb

@@ -1,3 +1,3 @@
 module Billy
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/puffing-billy.gemspec

@@ -20,17 +20,18 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "faraday"
   gem.add_development_dependency "poltergeist"
   gem.add_development_dependency "selenium-webdriver"
+  gem.add_development_dependency "capybara"
   gem.add_development_dependency "capybara-webkit", "~> 1.0"
   gem.add_development_dependency "rack"
   gem.add_development_dependency "guard"
   gem.add_development_dependency "rb-inotify"
   gem.add_development_dependency "pry"
   gem.add_development_dependency "cucumber"
+  gem.add_runtime_dependency "addressable"
   gem.add_runtime_dependency "eventmachine"
   gem.add_runtime_dependency "em-synchrony"
   gem.add_runtime_dependency "em-http-request"
   gem.add_runtime_dependency "eventmachine_httpserver"
   gem.add_runtime_dependency "http_parser.rb", "~> 0.6.0"
   gem.add_runtime_dependency "multi_json"
-  gem.add_runtime_dependency "capybara"
 end

data/spec/lib/billy/cache_spec.rb

@@ -7,6 +7,7 @@ describe Billy::Cache do
     let(:callback) { '&callback=quux' }
     let(:fragment) { '#baz' }
     let(:base_url) { 'http://example.com' }
+    let(:pipe_url) { 'https://fonts.googleapis.com:443/css?family=Cabin+Sketch:400,700|Love+Ya+Like+A+Sister' }
     let(:fragment_url) { "#{base_url}/#{fragment}" }
     let(:params_url) { "#{base_url}#{params}" }
     let(:params_url_with_callback) { "#{base_url}#{params}#{callback}" }
@@ -22,6 +23,10 @@ describe Billy::Cache do
       it 'appends params and fragment if both are present' do
         expect(cache.format_url(params_fragment_url)).to eq params_fragment_url
       end
+      it 'does not raise error for URLs with pipes' do
+        expect { cache.format_url(pipe_url) }.not_to raise_error
+      end
+
       context "when dynamic_jsonp is true" do
         it 'omits the callback param by default' do
           expect(cache.format_url(params_url_with_callback, false, true)).to eq params_url
@@ -50,5 +55,38 @@ describe Billy::Cache do
         expect(cache.format_url(params_fragment_url, true)).to eq base_url
       end
     end
+
+    context "with merge_cached_responses_whitelist set" do
+
+      let(:analytics_url1) { "http://www.example-analytics.com/user/SDF879932/" }
+      let(:analytics_url2) { "http://www.example-analytics.com/user/OIWEMLW39/" }
+      let(:regular_url) { "http://www.example-analytics.com/user.js" }
+
+      let(:regex_to_match_analytics_urls_only) do
+        # Note that it matches the forward slash at the end of the URL, which doesn't match regular_url:
+        /www\.example\-analytics\.com\/user\//
+      end
+
+      before do
+        allow(Billy.config).to receive(:merge_cached_responses_whitelist) {
+          [regex_to_match_analytics_urls_only]
+        }
+      end
+
+      it "has one cache key for the two analytics urls that match, and a separate one for the other that doesn't" do
+        expect(cache.key("post", analytics_url1, "body")).to eq cache.key("post", analytics_url2, "body")
+        expect(cache.key("post", analytics_url1, "body")).not_to eq cache.key("post", regular_url, "body")
+      end
+
+      it "More specifically, the cache keys should be identical for the 2 analytics urls" do
+        identical_cache_key = "post_5fcb7a450e4cd54dcffcb526212757ee0ca9dc17"
+        distinct_cache_key = "post_www.example-analytics.com_81f097654a523bd7ddb10fd4aee781723e076a1a_02083f4579e08a612425c0c1a17ee47add783b94"
+
+        expect(cache.key("post", analytics_url1, "body")).to eq identical_cache_key
+        expect(cache.key("post", regular_url, "body")).to eq distinct_cache_key
+        expect(cache.key("post", analytics_url2, "body")).to eq identical_cache_key
+      end
+
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puffing-billy
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Olly Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-29 00:00:00.000000000 Z
+date: 2015-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: capybara-webkit
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -248,20 +276,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: capybara
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: A stubbing proxy server for ruby. Connect it to your browser in integration
   tests to fake interactions with remote HTTP(S) servers.
 email: