kontrast 0.2.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +135 -96
  3. data/bin/kontrast +9 -5
  4. data/lib/kontrast.rb +17 -5
  5. data/lib/kontrast/api_client.rb +56 -0
  6. data/lib/kontrast/api_endpoint_comparator.rb +126 -0
  7. data/lib/kontrast/api_endpoint_runner.rb +99 -0
  8. data/lib/kontrast/api_endpoint_test.rb +8 -0
  9. data/lib/kontrast/configuration.rb +36 -7
  10. data/lib/kontrast/exceptions.rb +2 -1
  11. data/lib/kontrast/gallery/template.erb +152 -21
  12. data/lib/kontrast/gallery_creator.rb +105 -47
  13. data/lib/kontrast/global_runner.rb +121 -0
  14. data/lib/kontrast/image_helper.rb +63 -0
  15. data/lib/kontrast/image_uploader.rb +18 -0
  16. data/lib/kontrast/page_comparator.rb +46 -0
  17. data/lib/kontrast/page_runner.rb +95 -0
  18. data/lib/kontrast/page_test.rb +32 -0
  19. data/lib/kontrast/selenium_handler.rb +18 -7
  20. data/lib/kontrast/spec.rb +21 -0
  21. data/lib/kontrast/spec_builder.rb +54 -0
  22. data/lib/kontrast/test.rb +27 -0
  23. data/lib/kontrast/test_builder.rb +25 -9
  24. data/lib/kontrast/test_suite.rb +42 -0
  25. data/lib/kontrast/thumbnail_creator.rb +18 -0
  26. data/lib/kontrast/version.rb +1 -1
  27. data/spec/api_endpoint_comparator_spec.rb +125 -0
  28. data/spec/configuration_spec.rb +19 -0
  29. data/spec/gallery_creator_spec.rb +26 -20
  30. data/spec/{image_handler_spec.rb → global_runner_spec.rb} +17 -12
  31. data/spec/page_comparator_spec.rb +31 -0
  32. data/spec/page_runner_spec.rb +45 -0
  33. data/spec/spec_builder_spec.rb +32 -0
  34. data/spec/support/fixtures/image.jpg +0 -0
  35. data/spec/support/fixtures/image_clone.jpg +0 -0
  36. data/spec/support/fixtures/img1.jpg +0 -0
  37. data/spec/support/fixtures/img2.jpg +0 -0
  38. data/spec/support/fixtures/other_image.jpg +0 -0
  39. data/spec/test_builder_spec.rb +6 -3
  40. data/spec/test_spec.rb +53 -0
  41. data/spec/test_suite_spec.rb +56 -0
  42. metadata +91 -30
  43. data/lib/kontrast/image_handler.rb +0 -119
  44. data/lib/kontrast/runner.rb +0 -141
  45. data/spec/runner_spec.rb +0 -37
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ccae0e8fea2906d0f534b11feee35bd3690fc4d2
4
- data.tar.gz: 0d94b6a934294b8ffb3e480abceb09f5263e637d
3
+ metadata.gz: 4ac6daf0d690d86b5ae6f30a956cff2fd788d0e5
4
+ data.tar.gz: 67f5e90bf221e138da38b5adc1b9a754da096c57
5
5
  SHA512:
6
- metadata.gz: b36e5e0f5d4edf9945f7fe6183b3a8accb2ddcbf1bbe29ded50ce0f236c7a9735e4e34e72ba0addfeaec46e3ec807f38753920401bcfaaa3be53f660133e8d35
7
- data.tar.gz: a011155d29e1817de408fb029941856721be1fc05f2e02e1f0afdffe7b25def4b7bff43d92687f5c8952044d18a7fa4f67272037c399540f9b6def91d99c892a
6
+ metadata.gz: afa4cee7b23ff3689234a2b5c239f1a56e9a2629274db07ebc8a2292f4b7ba7198922d15d3219c2af89d49c27b2e87fd60aa4b46f845f8e072deb539f1940ca3
7
+ data.tar.gz: d3bf6cad58425d84ec0400259cd202bac4dfdb26f1bf0fa03b4340997d5a7fdf9482ef9c1159476cd7c4dc990a2999122bbdcb59fb256f8acd3c3e0ac1c68838
data/README.md CHANGED
@@ -6,11 +6,13 @@ Kontrast lets you build a test suite to run against your test and production web
6
6
 
7
7
  ## Prerequisites
8
8
 
9
- 1. Install ImageMagick. You can do this on OS X via brew with:
9
+ 1. Ruby 2.0+
10
10
 
11
- $ brew install imagemagick
11
+ 2. Install ImageMagick. You can do this on OS X via brew with:
12
12
 
13
- 2. Make sure you have Firefox or a different Selenium-compatible browser installed. By default, Firefox is used.
13
+ $ brew install imagemagick
14
+
15
+ 3. Make sure you have Firefox or a different Selenium-compatible browser installed. By default, Firefox is used.
14
16
 
15
17
  ## Installation
16
18
 
@@ -28,7 +30,7 @@ Or install it yourself as:
28
30
 
29
31
  Lastly, generate the config file:
30
32
 
31
- $ kontrast generate_config
33
+ $ kontrast generate_config
32
34
 
33
35
  If you're in Rails, the config file will be generated in `config/initializers/kontrast.rb`.
34
36
  Otherwise, the config file will be generated in your current directory.
@@ -37,39 +39,39 @@ Otherwise, the config file will be generated in your current directory.
37
39
 
38
40
  Here's all the config you need to get started:
39
41
 
40
- Kontrast.configure do |config|
41
- # Set your test and production domains
42
- config.test_domain = "http://localhost:3000"
43
- config.production_domain = "http://www.example.com"
44
-
45
- # Build your test suite
46
- # These pages will open in a 1280px-wide browser
47
- config.pages(1280) do |page|
48
- page.home "/"
49
- page.about "/about"
50
- end
51
-
52
- # These pages will open in a 320px-wide browser
53
- config.pages(320) do |page|
54
- page.home "/"
55
- page.about "/about"
56
- end
57
- end
42
+ Kontrast.configure do |config|
43
+ # Set your test and production domains
44
+ config.test_domain = "http://localhost:3000"
45
+ config.production_domain = "http://www.example.com"
46
+
47
+ # Build your test suite
48
+ # These pages will open in a 1280px-wide browser
49
+ config.pages(1280) do |page|
50
+ page.home "/"
51
+ page.about "/about"
52
+ end
53
+
54
+ # These pages will open in a 320px-wide browser
55
+ config.pages(320) do |page|
56
+ page.home "/"
57
+ page.about "/about"
58
+ end
59
+ end
58
60
 
59
61
  ## Basic Usage
60
62
  Run Kontrast (use `bundle exec` and omit the --config flag if you're within a Rails app):
61
63
 
62
- $ kontrast local_run --config ./kontrast_config.rb
63
- ...
64
- ...
65
- ...
66
- Kontrast is all done!
67
- You can find the gallery at: /tmp/shots/1410309651/gallery/gallery.html
64
+ $ kontrast local_run --config ./kontrast_config.rb
65
+ ...
66
+ ...
67
+ ...
68
+ Kontrast is all done!
69
+ You can find the gallery at: /tmp/shots/1410309651/gallery/gallery.html
68
70
 
69
71
  Review the gallery in your Favorite Browser:
70
72
 
71
- $ open /tmp/shots/1410309651/gallery/gallery.html
72
-
73
+ $ open /tmp/shots/1410309651/gallery/gallery.html
74
+
73
75
  ## Parallelized Usage
74
76
  We designed Kontrast from the very beginning to work with multiple nodes. At Harry's, we use CircleCI for testing and Kontrast works perfectly with CircleCI's multi-container features.
75
77
 
@@ -81,73 +83,73 @@ Here's how to get set up:
81
83
 
82
84
  ### 1. Enable Parallelization
83
85
 
84
- config.run_parallel = true
86
+ config.run_parallel = true
85
87
 
86
88
  ### 2. Configure Nodes
87
89
  Set how many nodes you have in total and the zero-based index of the current node. Kontrast will automatically split up tests among these nodes.
88
90
 
89
- config.total_nodes = 6
91
+ config.total_nodes = 6
90
92
  config.current_node = 2
91
-
93
+
92
94
  ### 3. Configure Remote Options
93
95
  Set your S3 details:
94
96
 
95
97
  config.aws_bucket = "kontrast-test-results"
96
98
  config.aws_key = ENV['AWS_KEY']
97
99
  config.aws_secret = ENV['AWS_SECRET']
98
-
100
+
99
101
  Set the **local** path where output images will be stored before they are uploaded to S3. This is also where the gallery will be saved on the node that runs the `make_gallery` command. This path will be created if it doesn't already exist.
100
102
 
101
103
  config.local_path = "tmp/kontrast"
102
-
104
+
103
105
  Set the **remote** path relative to your S3 bucket's root where Kontrast's output files will be uploaded to. It should be unique to every test.
104
106
 
105
107
  config.remote_path = "artifacts.#{ENV['BUILD_NUMBER']}"
106
-
108
+
107
109
  ### 4. Run the Tests
108
110
  This command should run in parallel on every node. Use `bundle exec` and omit the --config flag if your app is `bundle`'d along with Rails.
109
111
 
110
- $ kontrast run_tests --config /path/to/config.rb
112
+ $ kontrast run_tests --config /path/to/config.rb
111
113
 
112
114
  ### 5. Create the Gallery
113
115
  This command should only run on one node after all the other nodes have completed the previous command. Use `bundle exec` and omit the --config flag if your app is `bundle`'d along with Rails.
114
116
 
115
- $ kontrast make_gallery --config /path/to/config.rb
116
-
117
+ $ kontrast make_gallery --config /path/to/config.rb
118
+
117
119
  ### 6. Review Your Results
118
120
  At this point, the gallery should be saved to `config.local_path` and uploaded to `config.remote_path`. Check it out in your Favorite Browser.
119
121
 
120
122
  ### Sample circle.yml
121
123
  Here's an example of how to run Kontrast within a Rails app using CircleCI:
122
124
 
123
- test:
124
- post:
125
- - bundle exec rails server:
126
- background: true
127
- parallel: true
128
- - bundle exec kontrast run_tests:
129
- parallel: true
130
- - bundle exec kontrast make_gallery
125
+ test:
126
+ post:
127
+ - bundle exec rails server:
128
+ background: true
129
+ parallel: true
130
+ - bundle exec kontrast run_tests:
131
+ parallel: true
132
+ - bundle exec kontrast make_gallery
131
133
 
132
134
  ## Advanced Configuration
133
135
 
134
136
  ### Test Suite
135
137
 
136
138
  #### fail_build
137
- If you want Kontrast to exit with an error code if it finds any diffs, use this option:
139
+ If you want Kontrast to exit with an error code (and fail your build) if an exception is raised while running, use this option:
138
140
 
139
- config.fail_build = true
141
+ config.fail_build = true
140
142
 
141
143
  ### Selenium Driver
142
144
  #### browser_driver
143
145
  Choose which Selenium driver you'd like to use. Kontrast has only been tested on the default Firefox driver but we would love feedback and/or pull requests for other drivers.
144
146
 
145
- config.browser_driver = "firefox"
146
-
147
+ config.browser_driver = "firefox"
148
+
147
149
  #### browser_profile
148
150
  You may set a driver's profile options in this hash.
149
151
 
150
- config.browser_profile = {
152
+ config.browser_profile = {
151
153
  "general.useragent.override" => "Some Cool Kontrast User Agent",
152
154
  "image.animation_mode" => "none"
153
155
  }
@@ -156,19 +158,19 @@ You may set a driver's profile options in this hash.
156
158
  #### distortion_metric
157
159
  See [http://www.imagemagick.org/RMagick/doc/constants.html#MetricType]() for available values.
158
160
 
159
- config.distortion_metric = "MeanAbsoluteErrorMetric"
160
-
161
+ config.distortion_metric = "MeanAbsoluteErrorMetric"
162
+
161
163
  #### highlight_color
162
164
  The ImageMagick comparison tool emphasizes differences with this color.
163
165
  Valid options are an RMagick color name or pixel.
164
166
 
165
- config.highlight_color = "blue"
166
-
167
+ config.highlight_color = "blue"
168
+
167
169
  #### lowlight_color
168
170
  The ImageMagick comparison tool deemphasizes differences with this color.
169
171
  Valid options are an RMagick color name or pixel.
170
172
 
171
- config.lowlight_color = "rgba(255, 255, 255, 0.3)"
173
+ config.lowlight_color = "rgba(255, 255, 255, 0.3)"
172
174
 
173
175
  ### Hooks
174
176
  To make Kontrast even more powerful, we provided a set of hooks that you can use in your configuration.
@@ -176,55 +178,55 @@ To make Kontrast even more powerful, we provided a set of hooks that you can use
176
178
  #### before_run
177
179
  Runs before the entire suite.
178
180
 
179
- config.before_run do
180
- WebMock.disable!
181
- end
181
+ config.before_run do
182
+ WebMock.disable!
183
+ end
182
184
 
183
185
  #### after_run
184
186
  Runs after the entire suite.
185
187
 
186
- config.after_run do
187
- WebMock.enable!
188
- end
188
+ config.after_run do
189
+ WebMock.enable!
190
+ end
189
191
 
190
192
  #### before_gallery
191
193
  Runs before the gallery creation step.
192
194
 
193
- config.before_gallery do
194
- WebMock.disable!
195
- end
195
+ config.before_gallery do
196
+ WebMock.disable!
197
+ end
196
198
 
197
199
  #### after_gallery
198
200
  Runs after the gallery creation step.
199
201
 
200
- config.after_gallery do |diffs, gallery_path|
201
- # diffs is a hash containing all the differences that Kontrast found in your test suite
202
- # gallery_path is where Kontrast saved the gallery
203
- end
202
+ config.after_gallery do |diffs, gallery_path|
203
+ # diffs is a hash containing all the differences that Kontrast found in your test suite
204
+ # gallery_path is where Kontrast saved the gallery
205
+ end
204
206
 
205
207
  #### before_screenshot
206
208
  Runs on every test before Selenium takes a screenshot.
207
209
 
208
- config.before_screenshot do |test_driver, production_driver, test_info|
209
- # test_driver and production_driver are instances of Selenium::WebDriver that you can control
210
- # test_info is a hash with the current test's name and width
211
- end
210
+ config.before_screenshot do |test_driver, production_driver, test_info|
211
+ # test_driver and production_driver are instances of Selenium::WebDriver that you can control
212
+ # test_info is a hash with the current test's name and width
213
+ end
212
214
 
213
215
  #### after_screenshot
214
216
  Runs on every test after Selenium takes a screenshot.
215
217
 
216
- config.after_screenshot do |test_driver, production_driver, test_info|
217
- # same variables are available as with before_screenshot
218
- end
219
-
218
+ config.after_screenshot do |test_driver, production_driver, test_info|
219
+ # same variables are available as with before_screenshot
220
+ end
221
+
220
222
  ## Customizing Kontrast
221
223
  Kontrast's hooks allow you to insert custom functionality into many parts of the test suite. Here are some examples of how we use hooks at Harry's:
222
224
 
223
225
  ### Integrating with HipChat
224
226
  Once a build finishes, we let HipChat know if Kontrast found any diffs using the `hipchat` gem:
225
227
 
226
- config.after_gallery do |diffs, gallery_path|
227
- hipchat_room = "Kontrast Results"
228
+ config.after_gallery do |diffs, gallery_path|
229
+ hipchat_room = "Kontrast Results"
228
230
  hipchat_user = "KontrastBot"
229
231
 
230
232
  if !diffs.empty?
@@ -232,27 +234,64 @@ Once a build finishes, we let HipChat know if Kontrast found any diffs using the
232
234
  client = HipChat::Client.new(ENV["HIPCHAT_TOKEN"])
233
235
  client[hipchat_room].send(hipchat_user, msg, :color => "red")
234
236
  end
235
- end
236
-
237
+ end
238
+
237
239
  ### Setting Cookies
238
240
  Testing our cart page required a bit more setup before we could take a screenshot of it:
239
241
 
240
- config.before_screenshot do |test_driver, production_driver, test|
241
- if test[:name] == "cart"
242
- # prepare our cookie value
243
- cookie_value = super_secret_magic_cart_cookie
244
-
245
- # write cookies using Mootools
246
- # http://mootools.net/docs/core/Utilities/Cookie
247
- test_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
248
- production_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
249
-
250
- # refresh the page
251
- test_driver.navigate.refresh
252
- production_driver.navigate.refresh
253
- end
242
+ config.before_screenshot do |test_driver, production_driver, test|
243
+ if test[:name] == "cart"
244
+ # prepare our cookie value
245
+ cookie_value = super_secret_magic_cart_cookie
246
+
247
+ # write cookies using Mootools
248
+ # http://mootools.net/docs/core/Utilities/Cookie
249
+ test_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
250
+ production_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
251
+
252
+ # refresh the page
253
+ test_driver.navigate.refresh
254
+ production_driver.navigate.refresh
255
+ end
254
256
  end
255
257
 
258
+ ### Adding URL Parameters To All Pages
259
+ You may want to append a URL param to the end of every test path. To avoid doing something like this:
260
+
261
+ config.pages(1280) do |page|
262
+ page.home "/?mobile=1"
263
+ page.about "/about?mobile=1"
264
+ end
265
+
266
+ you can do this instead:
267
+
268
+ config.pages(1280, { mobile: 1 }) do |page|
269
+ page.home "/"
270
+ page.about "/about"
271
+ end
272
+
273
+ ## Specs
274
+ To avoid cluttering up the Kontrast config with lots of per-test hook logic, we made an easy way for you to specify per-test hooks. We do this with specs, which are RSpec-inspired files that contain hooks which only run with their respective tests.
275
+
276
+ ### How to Name Specs
277
+ The name of a spec is passed into the `Kontrast#describe` method. This spec is automatically bound to any test whose name includes the name of the spec. In the example below, this spec would only run on the `1280_home` test. But if you name your spec `home`, it will run on both the `320_home` and `1280_home` tests.
278
+
279
+ ### How to Write Specs
280
+
281
+ # 1280_home_spec.rb
282
+ Kontrast.describe("1280_home") do |spec|
283
+ spec.before_screenshot do |test_driver, production_driver, test|
284
+ # Do some stuff before screenshotting the 1280_home page
285
+ end
286
+
287
+ spec.after_screenshot do |test_driver, production_driver, test|
288
+ # Do some stuff after screenshotting the 1280_home page
289
+ end
290
+ end
291
+
292
+ ### Where to Put Specs
293
+ Spec files should go into the `./kontrast_specs` folder by default and must end with `_spec.rb`. You can tell Kontrast to look for specs in a different path using the `--specs-path` flag. If you're using Rails and don't include the `--specs-path` flag, the specs should go in `"#{Rails.root}/kontrast_specs"`.
294
+
256
295
  ## Contributing
257
296
 
258
297
  1. Fork it
@@ -16,17 +16,18 @@ require "thor"
16
16
  module Kontrast
17
17
  class CLI < Thor
18
18
  class_option :config
19
+ class_option :specs_path
19
20
 
20
21
  desc "run_tests", "Run Kontrast test suite"
21
22
  def run_tests
22
- load_config(options[:config])
23
+ load_config(options[:config], specs_path: options[:specs_path])
23
24
  Kontrast.run
24
25
  end
25
26
 
26
27
  desc "make_gallery --result-path PATH", "Create gallery given an optional local path of test results"
27
28
  option :result_path
28
29
  def make_gallery
29
- load_config(options[:config])
30
+ load_config(options[:config], specs_path: options[:specs_path])
30
31
 
31
32
  # We're only allowed to give no path in the remote case
32
33
  if options[:result_path].nil? && !Kontrast.configuration.run_parallel
@@ -40,7 +41,7 @@ module Kontrast
40
41
  # todo: option to specify an output path
41
42
  desc "local_run", "Run Kontrast locally"
42
43
  def local_run
43
- load_config(options[:config])
44
+ load_config(options[:config], specs_path: options[:specs_path])
44
45
 
45
46
  # Make sure config run_parallel is set to false
46
47
  if Kontrast.configuration.run_parallel
@@ -77,7 +78,7 @@ module Kontrast
77
78
  end
78
79
 
79
80
  private
80
- def load_config(config)
81
+ def load_config(config, specs_path: nil)
81
82
  # Let's check if we're within Rails.
82
83
  if !Kontrast.in_rails?
83
84
  begin
@@ -86,7 +87,7 @@ module Kontrast
86
87
  raise ConfigurationException.new("Error parsing the config flag '#{config}'")
87
88
  rescue LoadError => e
88
89
  raise ConfigurationException.new("Could not load '#{config}'")
89
- rescue Exception => e
90
+ rescue StandardError => e
90
91
  raise ConfigurationException.new("An unexpected error occurred while trying to load the given config file: #{e.inspect}")
91
92
  end
92
93
  else
@@ -103,6 +104,9 @@ module Kontrast
103
104
  # Make sure we have the bare minimum configuration to continue
104
105
  Kontrast.configuration.validate
105
106
 
107
+ # Load specs
108
+ SpecBuilder.load_specs(specs_path)
109
+
106
110
  return true
107
111
  end
108
112
  end
@@ -1,15 +1,27 @@
1
1
  # Dependencies
2
- require "fog"
2
+ require "fog/aws"
3
3
  require "bundler"
4
4
 
5
5
  # Load classes
6
6
  require "kontrast/exceptions"
7
7
  require "kontrast/configuration"
8
+ require "kontrast/test"
9
+ require "kontrast/page_test"
10
+ require "kontrast/api_endpoint_test"
8
11
  require "kontrast/test_builder"
12
+ require "kontrast/test_suite"
13
+ require "kontrast/spec"
14
+ require "kontrast/spec_builder"
9
15
  require "kontrast/selenium_handler"
10
- require "kontrast/image_handler"
16
+ require "kontrast/image_helper"
11
17
  require "kontrast/gallery_creator"
12
- require "kontrast/runner"
18
+ require "kontrast/global_runner"
19
+ require "kontrast/image_uploader"
20
+ require "kontrast/thumbnail_creator"
21
+ require "kontrast/page_runner"
22
+ require "kontrast/api_endpoint_runner"
23
+ require "kontrast/page_comparator"
24
+ require "kontrast/api_endpoint_comparator"
13
25
 
14
26
  module Kontrast
15
27
  class << self
@@ -26,7 +38,7 @@ module Kontrast
26
38
  Bundler.environment.current_dependencies.each do |dep|
27
39
  return true if dep.name == "rails"
28
40
  end
29
- rescue Exception => e
41
+ rescue StandardError => e
30
42
  # Quietly ignore any exceptions
31
43
  end
32
44
  return false
@@ -61,7 +73,7 @@ module Kontrast
61
73
  # Call "before" hook
62
74
  Kontrast.configuration.before_run
63
75
 
64
- runner = Runner.new
76
+ runner = GlobalRunner.new
65
77
  runner.run
66
78
  ensure
67
79
  # Call "after" hook