kontrast 0.2.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ccae0e8fea2906d0f534b11feee35bd3690fc4d2
-  data.tar.gz: 0d94b6a934294b8ffb3e480abceb09f5263e637d
+  metadata.gz: 4ac6daf0d690d86b5ae6f30a956cff2fd788d0e5
+  data.tar.gz: 67f5e90bf221e138da38b5adc1b9a754da096c57
 SHA512:
-  metadata.gz: b36e5e0f5d4edf9945f7fe6183b3a8accb2ddcbf1bbe29ded50ce0f236c7a9735e4e34e72ba0addfeaec46e3ec807f38753920401bcfaaa3be53f660133e8d35
-  data.tar.gz: a011155d29e1817de408fb029941856721be1fc05f2e02e1f0afdffe7b25def4b7bff43d92687f5c8952044d18a7fa4f67272037c399540f9b6def91d99c892a
+  metadata.gz: afa4cee7b23ff3689234a2b5c239f1a56e9a2629274db07ebc8a2292f4b7ba7198922d15d3219c2af89d49c27b2e87fd60aa4b46f845f8e072deb539f1940ca3
+  data.tar.gz: d3bf6cad58425d84ec0400259cd202bac4dfdb26f1bf0fa03b4340997d5a7fdf9482ef9c1159476cd7c4dc990a2999122bbdcb59fb256f8acd3c3e0ac1c68838

data/README.md

@@ -6,11 +6,13 @@ Kontrast lets you build a test suite to run against your test and production web
 
 ## Prerequisites
 
-1. Install ImageMagick. You can do this on OS X via brew with:
+1. Ruby 2.0+
 
-		$ brew install imagemagick
+2. Install ImageMagick. You can do this on OS X via brew with:
 
-2. Make sure you have Firefox or a different Selenium-compatible browser installed. By default, Firefox is used.
+        $ brew install imagemagick
+
+3. Make sure you have Firefox or a different Selenium-compatible browser installed. By default, Firefox is used.
 
 ## Installation
 
@@ -28,7 +30,7 @@ Or install it yourself as:
 
 Lastly, generate the config file:
 
-	$ kontrast generate_config
+    $ kontrast generate_config
 
 If you're in Rails, the config file will be generated in `config/initializers/kontrast.rb`.  
 Otherwise, the config file will be generated in your current directory.
@@ -37,39 +39,39 @@ Otherwise, the config file will be generated in your current directory.
 
 Here's all the config you need to get started:
 
-	Kontrast.configure do |config|
-		# Set your test and production domains
-		config.test_domain = "http://localhost:3000"
-		config.production_domain = "http://www.example.com"
-
-		# Build your test suite
-		# These pages will open in a 1280px-wide browser
-		config.pages(1280) do |page|
-			page.home "/"
-			page.about "/about"
-		end
-
-		# These pages will open in a 320px-wide browser
-		config.pages(320) do |page|
-			page.home "/"
-			page.about "/about"
-		end
-	end
+    Kontrast.configure do |config|
+        # Set your test and production domains
+        config.test_domain = "http://localhost:3000"
+        config.production_domain = "http://www.example.com"
+
+        # Build your test suite
+        # These pages will open in a 1280px-wide browser
+        config.pages(1280) do |page|
+            page.home "/"
+            page.about "/about"
+        end
+
+        # These pages will open in a 320px-wide browser
+        config.pages(320) do |page|
+            page.home "/"
+            page.about "/about"
+        end
+    end
 
 ## Basic Usage
 Run Kontrast (use `bundle exec` and omit the --config flag if you're within a Rails app):
 
-	$ kontrast local_run --config ./kontrast_config.rb
-	...
-	...
-	...
-	Kontrast is all done!
-	You can find the gallery at: /tmp/shots/1410309651/gallery/gallery.html
+    $ kontrast local_run --config ./kontrast_config.rb
+    ...
+    ...
+    ...
+    Kontrast is all done!
+    You can find the gallery at: /tmp/shots/1410309651/gallery/gallery.html
 
 Review the gallery in your Favorite Browser:
 
-	$ open /tmp/shots/1410309651/gallery/gallery.html
-	
+    $ open /tmp/shots/1410309651/gallery/gallery.html
+
 ## Parallelized Usage
 We designed Kontrast from the very beginning to work with multiple nodes. At Harry's, we use CircleCI for testing and Kontrast works perfectly with CircleCI's multi-container features.
 
@@ -81,73 +83,73 @@ Here's how to get set up:
 
 ### 1. Enable Parallelization
 
-	config.run_parallel = true
+    config.run_parallel = true
 
 ### 2. Configure Nodes
 Set how many nodes you have in total and the zero-based index of the current node. Kontrast will automatically split up tests among these nodes.
 
-	config.total_nodes = 6
+    config.total_nodes = 6
     config.current_node = 2
-    
+
 ### 3. Configure Remote Options
 Set your S3 details:
 
     config.aws_bucket = "kontrast-test-results"
     config.aws_key = ENV['AWS_KEY']
     config.aws_secret = ENV['AWS_SECRET']
-    
+
 Set the **local** path where output images will be stored before they are uploaded to S3. This is also where the gallery will be saved on the node that runs the `make_gallery` command. This path will be created if it doesn't already exist.
 
     config.local_path = "tmp/kontrast"
-    
+
 Set the **remote** path relative to your S3 bucket's root where Kontrast's output files will be uploaded to. It should be unique to every test.
 
     config.remote_path = "artifacts.#{ENV['BUILD_NUMBER']}"
-    
+
 ### 4. Run the Tests
 This command should run in parallel on every node. Use `bundle exec` and omit the --config flag if your app is `bundle`'d along with Rails.
 
-	$ kontrast run_tests --config /path/to/config.rb
+    $ kontrast run_tests --config /path/to/config.rb
 
 ### 5. Create the Gallery
 This command should only run on one node after all the other nodes have completed the previous command. Use `bundle exec` and omit the --config flag if your app is `bundle`'d along with Rails.
 
-	$ kontrast make_gallery --config /path/to/config.rb
-	
+    $ kontrast make_gallery --config /path/to/config.rb
+
 ### 6. Review Your Results
 At this point, the gallery should be saved to `config.local_path` and uploaded to `config.remote_path`. Check it out in your Favorite Browser.
 
 ### Sample circle.yml
 Here's an example of how to run Kontrast within a Rails app using CircleCI:
 
-	test:
-  		post:
-      		- bundle exec rails server:
-          		background: true
-          		parallel: true
-      		- bundle exec kontrast run_tests:
-          		parallel: true
-          	- bundle exec kontrast make_gallery
+    test:
+        post:
+            - bundle exec rails server:
+                background: true
+                parallel: true
+            - bundle exec kontrast run_tests:
+                parallel: true
+            - bundle exec kontrast make_gallery
 
 ## Advanced Configuration
 
 ### Test Suite
 
 #### fail_build
-If you want Kontrast to exit with an error code if it finds any diffs, use this option:
+If you want Kontrast to exit with an error code (and fail your build) if an exception is raised while running, use this option:
 
-	config.fail_build = true
+    config.fail_build = true
 
 ### Selenium Driver
 #### browser_driver
 Choose which Selenium driver you'd like to use. Kontrast has only been tested on the default Firefox driver but we would love feedback and/or pull requests for other drivers.
 
-	config.browser_driver = "firefox"
-	
+    config.browser_driver = "firefox"
+
 #### browser_profile
 You may set a driver's profile options in this hash.
 
-	config.browser_profile = {
+    config.browser_profile = {
         "general.useragent.override" => "Some Cool Kontrast User Agent",
         "image.animation_mode" => "none"
     }
@@ -156,19 +158,19 @@ You may set a driver's profile options in this hash.
 #### distortion_metric
 See [http://www.imagemagick.org/RMagick/doc/constants.html#MetricType]() for available values.
 
-	config.distortion_metric = "MeanAbsoluteErrorMetric"
-	
+    config.distortion_metric = "MeanAbsoluteErrorMetric"
+
 #### highlight_color
 The ImageMagick comparison tool emphasizes differences with this color.
 Valid options are an RMagick color name or pixel.
 
-	config.highlight_color = "blue"
-	
+    config.highlight_color = "blue"
+
 #### lowlight_color
 The ImageMagick comparison tool deemphasizes differences with this color.
 Valid options are an RMagick color name or pixel.
 
-	config.lowlight_color = "rgba(255, 255, 255, 0.3)"
+    config.lowlight_color = "rgba(255, 255, 255, 0.3)"
 
 ### Hooks
 To make Kontrast even more powerful, we provided a set of hooks that you can use in your configuration.
@@ -176,55 +178,55 @@ To make Kontrast even more powerful, we provided a set of hooks that you can use
 #### before_run
 Runs before the entire suite.
 
-	config.before_run do
-		WebMock.disable!
-	end
+    config.before_run do
+        WebMock.disable!
+    end
 
 #### after_run
 Runs after the entire suite.
 
-	config.after_run do
-		WebMock.enable!
-	end
+    config.after_run do
+        WebMock.enable!
+    end
 
 #### before_gallery
 Runs before the gallery creation step.
 
-	config.before_gallery do
-		WebMock.disable!
-	end
+    config.before_gallery do
+        WebMock.disable!
+    end
 
 #### after_gallery
 Runs after the gallery creation step.
 
-	config.after_gallery do |diffs, gallery_path|
-		# diffs is a hash containing all the differences that Kontrast found in your test suite
-		# gallery_path is where Kontrast saved the gallery
-	end
+    config.after_gallery do |diffs, gallery_path|
+        # diffs is a hash containing all the differences that Kontrast found in your test suite
+        # gallery_path is where Kontrast saved the gallery
+    end
 
 #### before_screenshot
 Runs on every test before Selenium takes a screenshot.
 
-	config.before_screenshot do |test_driver, production_driver, test_info|
-		# test_driver and production_driver are instances of Selenium::WebDriver that you can control
-		# test_info is a hash with the current test's name and width
-	end
+    config.before_screenshot do |test_driver, production_driver, test_info|
+        # test_driver and production_driver are instances of Selenium::WebDriver that you can control
+        # test_info is a hash with the current test's name and width
+    end
 
 #### after_screenshot
 Runs on every test after Selenium takes a screenshot.
 
-	config.after_screenshot do |test_driver, production_driver, test_info|
-		# same variables are available as with before_screenshot
-	end
-	
+    config.after_screenshot do |test_driver, production_driver, test_info|
+        # same variables are available as with before_screenshot
+    end
+
 ## Customizing Kontrast
 Kontrast's hooks allow you to insert custom functionality into many parts of the test suite. Here are some examples of how we use hooks at Harry's:
 
 ### Integrating with HipChat
 Once a build finishes, we let HipChat know if Kontrast found any diffs using the `hipchat` gem:
 
-	config.after_gallery do |diffs, gallery_path|
-		hipchat_room = "Kontrast Results"
+    config.after_gallery do |diffs, gallery_path|
+        hipchat_room = "Kontrast Results"
         hipchat_user = "KontrastBot"
 
         if !diffs.empty?
@@ -232,27 +234,64 @@ Once a build finishes, we let HipChat know if Kontrast found any diffs using the
             client = HipChat::Client.new(ENV["HIPCHAT_TOKEN"])
             client[hipchat_room].send(hipchat_user, msg, :color => "red")
         end
-	end
-	
+    end
+
 ### Setting Cookies
 Testing our cart page required a bit more setup before we could take a screenshot of it:
 
-	config.before_screenshot do |test_driver, production_driver, test|
-		if test[:name] == "cart"
-			# prepare our cookie value
-       		cookie_value = super_secret_magic_cart_cookie
-       	
-       		# write cookies using Mootools
-       		# http://mootools.net/docs/core/Utilities/Cookie
-        	test_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
-        	production_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
-        
-        	# refresh the page
-        	test_driver.navigate.refresh
-        	production_driver.navigate.refresh
-    	end
+    config.before_screenshot do |test_driver, production_driver, test|
+        if test[:name] == "cart"
+            # prepare our cookie value
+            cookie_value = super_secret_magic_cart_cookie
+
+            # write cookies using Mootools
+            # http://mootools.net/docs/core/Utilities/Cookie
+            test_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
+            production_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
+
+            # refresh the page
+            test_driver.navigate.refresh
+            production_driver.navigate.refresh
+        end
     end
 
+### Adding URL Parameters To All Pages
+You may want to append a URL param to the end of every test path. To avoid doing something like this:
+
+    config.pages(1280) do |page|
+        page.home "/?mobile=1"
+        page.about "/about?mobile=1"
+    end
+
+you can do this instead:
+
+    config.pages(1280, { mobile: 1 }) do |page|
+        page.home "/"
+        page.about "/about"
+    end
+
+## Specs
+To avoid cluttering up the Kontrast config with lots of per-test hook logic, we made an easy way for you to specify per-test hooks. We do this with specs, which are RSpec-inspired files that contain hooks which only run with their respective tests.
+
+### How to Name Specs
+The name of a spec is passed into the `Kontrast#describe` method. This spec is automatically bound to any test whose name includes the name of the spec. In the example below, this spec would only run on the `1280_home` test. But if you name your spec `home`, it will run on both the `320_home` and `1280_home` tests.
+
+### How to Write Specs
+
+    # 1280_home_spec.rb
+    Kontrast.describe("1280_home") do |spec|
+        spec.before_screenshot do |test_driver, production_driver, test|
+            # Do some stuff before screenshotting the 1280_home page
+        end
+
+        spec.after_screenshot do |test_driver, production_driver, test|
+            # Do some stuff after screenshotting the 1280_home page
+        end
+    end
+
+### Where to Put Specs
+Spec files should go into the `./kontrast_specs` folder by default and must end with `_spec.rb`. You can tell Kontrast to look for specs in a different path using the `--specs-path` flag. If you're using Rails and don't include the `--specs-path` flag, the specs should go in `"#{Rails.root}/kontrast_specs"`.
+
 ## Contributing
 
 1. Fork it

data/bin/kontrast

@@ -16,17 +16,18 @@ require "thor"
 module Kontrast
     class CLI < Thor
         class_option :config
+        class_option :specs_path
 
         desc "run_tests", "Run Kontrast test suite"
         def run_tests
-            load_config(options[:config])
+            load_config(options[:config], specs_path: options[:specs_path])
             Kontrast.run
         end
 
         desc "make_gallery --result-path PATH", "Create gallery given an optional local path of test results"
         option :result_path
         def make_gallery
-            load_config(options[:config])
+            load_config(options[:config], specs_path: options[:specs_path])
 
             # We're only allowed to give no path in the remote case
             if options[:result_path].nil? && !Kontrast.configuration.run_parallel
@@ -40,7 +41,7 @@ module Kontrast
         # todo: option to specify an output path
         desc "local_run", "Run Kontrast locally"
         def local_run
-            load_config(options[:config])
+            load_config(options[:config], specs_path: options[:specs_path])
 
             # Make sure config run_parallel is set to false
             if Kontrast.configuration.run_parallel
@@ -77,7 +78,7 @@ module Kontrast
         end
 
         private
-            def load_config(config)
+            def load_config(config, specs_path: nil)
                 # Let's check if we're within Rails.
                 if !Kontrast.in_rails?
                     begin
@@ -86,7 +87,7 @@ module Kontrast
                         raise ConfigurationException.new("Error parsing the config flag '#{config}'")
                     rescue LoadError => e
                         raise ConfigurationException.new("Could not load '#{config}'")
-                    rescue Exception => e
+                    rescue StandardError => e
                         raise ConfigurationException.new("An unexpected error occurred while trying to load the given config file: #{e.inspect}")
                     end
                 else
@@ -103,6 +104,9 @@ module Kontrast
                 # Make sure we have the bare minimum configuration to continue
                 Kontrast.configuration.validate
 
+                # Load specs
+                SpecBuilder.load_specs(specs_path)
+
                 return true
             end
     end

data/lib/kontrast.rb

@@ -1,15 +1,27 @@
 # Dependencies
-require "fog"
+require "fog/aws"
 require "bundler"
 
 # Load classes
 require "kontrast/exceptions"
 require "kontrast/configuration"
+require "kontrast/test"
+require "kontrast/page_test"
+require "kontrast/api_endpoint_test"
 require "kontrast/test_builder"
+require "kontrast/test_suite"
+require "kontrast/spec"
+require "kontrast/spec_builder"
 require "kontrast/selenium_handler"
-require "kontrast/image_handler"
+require "kontrast/image_helper"
 require "kontrast/gallery_creator"
-require "kontrast/runner"
+require "kontrast/global_runner"
+require "kontrast/image_uploader"
+require "kontrast/thumbnail_creator"
+require "kontrast/page_runner"
+require "kontrast/api_endpoint_runner"
+require "kontrast/page_comparator"
+require "kontrast/api_endpoint_comparator"
 
 module Kontrast
     class << self
@@ -26,7 +38,7 @@ module Kontrast
                 Bundler.environment.current_dependencies.each do |dep|
                     return true if dep.name == "rails"
                 end
-            rescue Exception => e
+            rescue StandardError => e
                 # Quietly ignore any exceptions
             end
             return false
@@ -61,7 +73,7 @@ module Kontrast
                 # Call "before" hook
                 Kontrast.configuration.before_run
 
-                runner = Runner.new
+                runner = GlobalRunner.new
                 runner.run
             ensure
                 # Call "after" hook