kontrast 0.2.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +135 -96
- data/bin/kontrast +9 -5
- data/lib/kontrast.rb +17 -5
- data/lib/kontrast/api_client.rb +56 -0
- data/lib/kontrast/api_endpoint_comparator.rb +126 -0
- data/lib/kontrast/api_endpoint_runner.rb +99 -0
- data/lib/kontrast/api_endpoint_test.rb +8 -0
- data/lib/kontrast/configuration.rb +36 -7
- data/lib/kontrast/exceptions.rb +2 -1
- data/lib/kontrast/gallery/template.erb +152 -21
- data/lib/kontrast/gallery_creator.rb +105 -47
- data/lib/kontrast/global_runner.rb +121 -0
- data/lib/kontrast/image_helper.rb +63 -0
- data/lib/kontrast/image_uploader.rb +18 -0
- data/lib/kontrast/page_comparator.rb +46 -0
- data/lib/kontrast/page_runner.rb +95 -0
- data/lib/kontrast/page_test.rb +32 -0
- data/lib/kontrast/selenium_handler.rb +18 -7
- data/lib/kontrast/spec.rb +21 -0
- data/lib/kontrast/spec_builder.rb +54 -0
- data/lib/kontrast/test.rb +27 -0
- data/lib/kontrast/test_builder.rb +25 -9
- data/lib/kontrast/test_suite.rb +42 -0
- data/lib/kontrast/thumbnail_creator.rb +18 -0
- data/lib/kontrast/version.rb +1 -1
- data/spec/api_endpoint_comparator_spec.rb +125 -0
- data/spec/configuration_spec.rb +19 -0
- data/spec/gallery_creator_spec.rb +26 -20
- data/spec/{image_handler_spec.rb → global_runner_spec.rb} +17 -12
- data/spec/page_comparator_spec.rb +31 -0
- data/spec/page_runner_spec.rb +45 -0
- data/spec/spec_builder_spec.rb +32 -0
- data/spec/support/fixtures/image.jpg +0 -0
- data/spec/support/fixtures/image_clone.jpg +0 -0
- data/spec/support/fixtures/img1.jpg +0 -0
- data/spec/support/fixtures/img2.jpg +0 -0
- data/spec/support/fixtures/other_image.jpg +0 -0
- data/spec/test_builder_spec.rb +6 -3
- data/spec/test_spec.rb +53 -0
- data/spec/test_suite_spec.rb +56 -0
- metadata +91 -30
- data/lib/kontrast/image_handler.rb +0 -119
- data/lib/kontrast/runner.rb +0 -141
- data/spec/runner_spec.rb +0 -37
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ac6daf0d690d86b5ae6f30a956cff2fd788d0e5
|
4
|
+
data.tar.gz: 67f5e90bf221e138da38b5adc1b9a754da096c57
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: afa4cee7b23ff3689234a2b5c239f1a56e9a2629274db07ebc8a2292f4b7ba7198922d15d3219c2af89d49c27b2e87fd60aa4b46f845f8e072deb539f1940ca3
|
7
|
+
data.tar.gz: d3bf6cad58425d84ec0400259cd202bac4dfdb26f1bf0fa03b4340997d5a7fdf9482ef9c1159476cd7c4dc990a2999122bbdcb59fb256f8acd3c3e0ac1c68838
|
data/README.md
CHANGED
@@ -6,11 +6,13 @@ Kontrast lets you build a test suite to run against your test and production web
|
|
6
6
|
|
7
7
|
## Prerequisites
|
8
8
|
|
9
|
-
1.
|
9
|
+
1. Ruby 2.0+
|
10
10
|
|
11
|
-
|
11
|
+
2. Install ImageMagick. You can do this on OS X via brew with:
|
12
12
|
|
13
|
-
|
13
|
+
$ brew install imagemagick
|
14
|
+
|
15
|
+
3. Make sure you have Firefox or a different Selenium-compatible browser installed. By default, Firefox is used.
|
14
16
|
|
15
17
|
## Installation
|
16
18
|
|
@@ -28,7 +30,7 @@ Or install it yourself as:
|
|
28
30
|
|
29
31
|
Lastly, generate the config file:
|
30
32
|
|
31
|
-
|
33
|
+
$ kontrast generate_config
|
32
34
|
|
33
35
|
If you're in Rails, the config file will be generated in `config/initializers/kontrast.rb`.
|
34
36
|
Otherwise, the config file will be generated in your current directory.
|
@@ -37,39 +39,39 @@ Otherwise, the config file will be generated in your current directory.
|
|
37
39
|
|
38
40
|
Here's all the config you need to get started:
|
39
41
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
42
|
+
Kontrast.configure do |config|
|
43
|
+
# Set your test and production domains
|
44
|
+
config.test_domain = "http://localhost:3000"
|
45
|
+
config.production_domain = "http://www.example.com"
|
46
|
+
|
47
|
+
# Build your test suite
|
48
|
+
# These pages will open in a 1280px-wide browser
|
49
|
+
config.pages(1280) do |page|
|
50
|
+
page.home "/"
|
51
|
+
page.about "/about"
|
52
|
+
end
|
53
|
+
|
54
|
+
# These pages will open in a 320px-wide browser
|
55
|
+
config.pages(320) do |page|
|
56
|
+
page.home "/"
|
57
|
+
page.about "/about"
|
58
|
+
end
|
59
|
+
end
|
58
60
|
|
59
61
|
## Basic Usage
|
60
62
|
Run Kontrast (use `bundle exec` and omit the --config flag if you're within a Rails app):
|
61
63
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
64
|
+
$ kontrast local_run --config ./kontrast_config.rb
|
65
|
+
...
|
66
|
+
...
|
67
|
+
...
|
68
|
+
Kontrast is all done!
|
69
|
+
You can find the gallery at: /tmp/shots/1410309651/gallery/gallery.html
|
68
70
|
|
69
71
|
Review the gallery in your Favorite Browser:
|
70
72
|
|
71
|
-
|
72
|
-
|
73
|
+
$ open /tmp/shots/1410309651/gallery/gallery.html
|
74
|
+
|
73
75
|
## Parallelized Usage
|
74
76
|
We designed Kontrast from the very beginning to work with multiple nodes. At Harry's, we use CircleCI for testing and Kontrast works perfectly with CircleCI's multi-container features.
|
75
77
|
|
@@ -81,73 +83,73 @@ Here's how to get set up:
|
|
81
83
|
|
82
84
|
### 1. Enable Parallelization
|
83
85
|
|
84
|
-
|
86
|
+
config.run_parallel = true
|
85
87
|
|
86
88
|
### 2. Configure Nodes
|
87
89
|
Set how many nodes you have in total and the zero-based index of the current node. Kontrast will automatically split up tests among these nodes.
|
88
90
|
|
89
|
-
|
91
|
+
config.total_nodes = 6
|
90
92
|
config.current_node = 2
|
91
|
-
|
93
|
+
|
92
94
|
### 3. Configure Remote Options
|
93
95
|
Set your S3 details:
|
94
96
|
|
95
97
|
config.aws_bucket = "kontrast-test-results"
|
96
98
|
config.aws_key = ENV['AWS_KEY']
|
97
99
|
config.aws_secret = ENV['AWS_SECRET']
|
98
|
-
|
100
|
+
|
99
101
|
Set the **local** path where output images will be stored before they are uploaded to S3. This is also where the gallery will be saved on the node that runs the `make_gallery` command. This path will be created if it doesn't already exist.
|
100
102
|
|
101
103
|
config.local_path = "tmp/kontrast"
|
102
|
-
|
104
|
+
|
103
105
|
Set the **remote** path relative to your S3 bucket's root where Kontrast's output files will be uploaded to. It should be unique to every test.
|
104
106
|
|
105
107
|
config.remote_path = "artifacts.#{ENV['BUILD_NUMBER']}"
|
106
|
-
|
108
|
+
|
107
109
|
### 4. Run the Tests
|
108
110
|
This command should run in parallel on every node. Use `bundle exec` and omit the --config flag if your app is `bundle`'d along with Rails.
|
109
111
|
|
110
|
-
|
112
|
+
$ kontrast run_tests --config /path/to/config.rb
|
111
113
|
|
112
114
|
### 5. Create the Gallery
|
113
115
|
This command should only run on one node after all the other nodes have completed the previous command. Use `bundle exec` and omit the --config flag if your app is `bundle`'d along with Rails.
|
114
116
|
|
115
|
-
|
116
|
-
|
117
|
+
$ kontrast make_gallery --config /path/to/config.rb
|
118
|
+
|
117
119
|
### 6. Review Your Results
|
118
120
|
At this point, the gallery should be saved to `config.local_path` and uploaded to `config.remote_path`. Check it out in your Favorite Browser.
|
119
121
|
|
120
122
|
### Sample circle.yml
|
121
123
|
Here's an example of how to run Kontrast within a Rails app using CircleCI:
|
122
124
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
125
|
+
test:
|
126
|
+
post:
|
127
|
+
- bundle exec rails server:
|
128
|
+
background: true
|
129
|
+
parallel: true
|
130
|
+
- bundle exec kontrast run_tests:
|
131
|
+
parallel: true
|
132
|
+
- bundle exec kontrast make_gallery
|
131
133
|
|
132
134
|
## Advanced Configuration
|
133
135
|
|
134
136
|
### Test Suite
|
135
137
|
|
136
138
|
#### fail_build
|
137
|
-
If you want Kontrast to exit with an error code if
|
139
|
+
If you want Kontrast to exit with an error code (and fail your build) if an exception is raised while running, use this option:
|
138
140
|
|
139
|
-
|
141
|
+
config.fail_build = true
|
140
142
|
|
141
143
|
### Selenium Driver
|
142
144
|
#### browser_driver
|
143
145
|
Choose which Selenium driver you'd like to use. Kontrast has only been tested on the default Firefox driver but we would love feedback and/or pull requests for other drivers.
|
144
146
|
|
145
|
-
|
146
|
-
|
147
|
+
config.browser_driver = "firefox"
|
148
|
+
|
147
149
|
#### browser_profile
|
148
150
|
You may set a driver's profile options in this hash.
|
149
151
|
|
150
|
-
|
152
|
+
config.browser_profile = {
|
151
153
|
"general.useragent.override" => "Some Cool Kontrast User Agent",
|
152
154
|
"image.animation_mode" => "none"
|
153
155
|
}
|
@@ -156,19 +158,19 @@ You may set a driver's profile options in this hash.
|
|
156
158
|
#### distortion_metric
|
157
159
|
See [http://www.imagemagick.org/RMagick/doc/constants.html#MetricType]() for available values.
|
158
160
|
|
159
|
-
|
160
|
-
|
161
|
+
config.distortion_metric = "MeanAbsoluteErrorMetric"
|
162
|
+
|
161
163
|
#### highlight_color
|
162
164
|
The ImageMagick comparison tool emphasizes differences with this color.
|
163
165
|
Valid options are an RMagick color name or pixel.
|
164
166
|
|
165
|
-
|
166
|
-
|
167
|
+
config.highlight_color = "blue"
|
168
|
+
|
167
169
|
#### lowlight_color
|
168
170
|
The ImageMagick comparison tool deemphasizes differences with this color.
|
169
171
|
Valid options are an RMagick color name or pixel.
|
170
172
|
|
171
|
-
|
173
|
+
config.lowlight_color = "rgba(255, 255, 255, 0.3)"
|
172
174
|
|
173
175
|
### Hooks
|
174
176
|
To make Kontrast even more powerful, we provided a set of hooks that you can use in your configuration.
|
@@ -176,55 +178,55 @@ To make Kontrast even more powerful, we provided a set of hooks that you can use
|
|
176
178
|
#### before_run
|
177
179
|
Runs before the entire suite.
|
178
180
|
|
179
|
-
|
180
|
-
|
181
|
-
|
181
|
+
config.before_run do
|
182
|
+
WebMock.disable!
|
183
|
+
end
|
182
184
|
|
183
185
|
#### after_run
|
184
186
|
Runs after the entire suite.
|
185
187
|
|
186
|
-
|
187
|
-
|
188
|
-
|
188
|
+
config.after_run do
|
189
|
+
WebMock.enable!
|
190
|
+
end
|
189
191
|
|
190
192
|
#### before_gallery
|
191
193
|
Runs before the gallery creation step.
|
192
194
|
|
193
|
-
|
194
|
-
|
195
|
-
|
195
|
+
config.before_gallery do
|
196
|
+
WebMock.disable!
|
197
|
+
end
|
196
198
|
|
197
199
|
#### after_gallery
|
198
200
|
Runs after the gallery creation step.
|
199
201
|
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
202
|
+
config.after_gallery do |diffs, gallery_path|
|
203
|
+
# diffs is a hash containing all the differences that Kontrast found in your test suite
|
204
|
+
# gallery_path is where Kontrast saved the gallery
|
205
|
+
end
|
204
206
|
|
205
207
|
#### before_screenshot
|
206
208
|
Runs on every test before Selenium takes a screenshot.
|
207
209
|
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
210
|
+
config.before_screenshot do |test_driver, production_driver, test_info|
|
211
|
+
# test_driver and production_driver are instances of Selenium::WebDriver that you can control
|
212
|
+
# test_info is a hash with the current test's name and width
|
213
|
+
end
|
212
214
|
|
213
215
|
#### after_screenshot
|
214
216
|
Runs on every test after Selenium takes a screenshot.
|
215
217
|
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
218
|
+
config.after_screenshot do |test_driver, production_driver, test_info|
|
219
|
+
# same variables are available as with before_screenshot
|
220
|
+
end
|
221
|
+
|
220
222
|
## Customizing Kontrast
|
221
223
|
Kontrast's hooks allow you to insert custom functionality into many parts of the test suite. Here are some examples of how we use hooks at Harry's:
|
222
224
|
|
223
225
|
### Integrating with HipChat
|
224
226
|
Once a build finishes, we let HipChat know if Kontrast found any diffs using the `hipchat` gem:
|
225
227
|
|
226
|
-
|
227
|
-
|
228
|
+
config.after_gallery do |diffs, gallery_path|
|
229
|
+
hipchat_room = "Kontrast Results"
|
228
230
|
hipchat_user = "KontrastBot"
|
229
231
|
|
230
232
|
if !diffs.empty?
|
@@ -232,27 +234,64 @@ Once a build finishes, we let HipChat know if Kontrast found any diffs using the
|
|
232
234
|
client = HipChat::Client.new(ENV["HIPCHAT_TOKEN"])
|
233
235
|
client[hipchat_room].send(hipchat_user, msg, :color => "red")
|
234
236
|
end
|
235
|
-
|
236
|
-
|
237
|
+
end
|
238
|
+
|
237
239
|
### Setting Cookies
|
238
240
|
Testing our cart page required a bit more setup before we could take a screenshot of it:
|
239
241
|
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
242
|
+
config.before_screenshot do |test_driver, production_driver, test|
|
243
|
+
if test[:name] == "cart"
|
244
|
+
# prepare our cookie value
|
245
|
+
cookie_value = super_secret_magic_cart_cookie
|
246
|
+
|
247
|
+
# write cookies using Mootools
|
248
|
+
# http://mootools.net/docs/core/Utilities/Cookie
|
249
|
+
test_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
|
250
|
+
production_driver.execute_script("Cookie.write('cart', '#{cookie_value}');")
|
251
|
+
|
252
|
+
# refresh the page
|
253
|
+
test_driver.navigate.refresh
|
254
|
+
production_driver.navigate.refresh
|
255
|
+
end
|
254
256
|
end
|
255
257
|
|
258
|
+
### Adding URL Parameters To All Pages
|
259
|
+
You may want to append a URL param to the end of every test path. To avoid doing something like this:
|
260
|
+
|
261
|
+
config.pages(1280) do |page|
|
262
|
+
page.home "/?mobile=1"
|
263
|
+
page.about "/about?mobile=1"
|
264
|
+
end
|
265
|
+
|
266
|
+
you can do this instead:
|
267
|
+
|
268
|
+
config.pages(1280, { mobile: 1 }) do |page|
|
269
|
+
page.home "/"
|
270
|
+
page.about "/about"
|
271
|
+
end
|
272
|
+
|
273
|
+
## Specs
|
274
|
+
To avoid cluttering up the Kontrast config with lots of per-test hook logic, we made an easy way for you to specify per-test hooks. We do this with specs, which are RSpec-inspired files that contain hooks which only run with their respective tests.
|
275
|
+
|
276
|
+
### How to Name Specs
|
277
|
+
The name of a spec is passed into the `Kontrast#describe` method. This spec is automatically bound to any test whose name includes the name of the spec. In the example below, this spec would only run on the `1280_home` test. But if you name your spec `home`, it will run on both the `320_home` and `1280_home` tests.
|
278
|
+
|
279
|
+
### How to Write Specs
|
280
|
+
|
281
|
+
# 1280_home_spec.rb
|
282
|
+
Kontrast.describe("1280_home") do |spec|
|
283
|
+
spec.before_screenshot do |test_driver, production_driver, test|
|
284
|
+
# Do some stuff before screenshotting the 1280_home page
|
285
|
+
end
|
286
|
+
|
287
|
+
spec.after_screenshot do |test_driver, production_driver, test|
|
288
|
+
# Do some stuff after screenshotting the 1280_home page
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
### Where to Put Specs
|
293
|
+
Spec files should go into the `./kontrast_specs` folder by default and must end with `_spec.rb`. You can tell Kontrast to look for specs in a different path using the `--specs-path` flag. If you're using Rails and don't include the `--specs-path` flag, the specs should go in `"#{Rails.root}/kontrast_specs"`.
|
294
|
+
|
256
295
|
## Contributing
|
257
296
|
|
258
297
|
1. Fork it
|
data/bin/kontrast
CHANGED
@@ -16,17 +16,18 @@ require "thor"
|
|
16
16
|
module Kontrast
|
17
17
|
class CLI < Thor
|
18
18
|
class_option :config
|
19
|
+
class_option :specs_path
|
19
20
|
|
20
21
|
desc "run_tests", "Run Kontrast test suite"
|
21
22
|
def run_tests
|
22
|
-
load_config(options[:config])
|
23
|
+
load_config(options[:config], specs_path: options[:specs_path])
|
23
24
|
Kontrast.run
|
24
25
|
end
|
25
26
|
|
26
27
|
desc "make_gallery --result-path PATH", "Create gallery given an optional local path of test results"
|
27
28
|
option :result_path
|
28
29
|
def make_gallery
|
29
|
-
load_config(options[:config])
|
30
|
+
load_config(options[:config], specs_path: options[:specs_path])
|
30
31
|
|
31
32
|
# We're only allowed to give no path in the remote case
|
32
33
|
if options[:result_path].nil? && !Kontrast.configuration.run_parallel
|
@@ -40,7 +41,7 @@ module Kontrast
|
|
40
41
|
# todo: option to specify an output path
|
41
42
|
desc "local_run", "Run Kontrast locally"
|
42
43
|
def local_run
|
43
|
-
load_config(options[:config])
|
44
|
+
load_config(options[:config], specs_path: options[:specs_path])
|
44
45
|
|
45
46
|
# Make sure config run_parallel is set to false
|
46
47
|
if Kontrast.configuration.run_parallel
|
@@ -77,7 +78,7 @@ module Kontrast
|
|
77
78
|
end
|
78
79
|
|
79
80
|
private
|
80
|
-
def load_config(config)
|
81
|
+
def load_config(config, specs_path: nil)
|
81
82
|
# Let's check if we're within Rails.
|
82
83
|
if !Kontrast.in_rails?
|
83
84
|
begin
|
@@ -86,7 +87,7 @@ module Kontrast
|
|
86
87
|
raise ConfigurationException.new("Error parsing the config flag '#{config}'")
|
87
88
|
rescue LoadError => e
|
88
89
|
raise ConfigurationException.new("Could not load '#{config}'")
|
89
|
-
rescue
|
90
|
+
rescue StandardError => e
|
90
91
|
raise ConfigurationException.new("An unexpected error occurred while trying to load the given config file: #{e.inspect}")
|
91
92
|
end
|
92
93
|
else
|
@@ -103,6 +104,9 @@ module Kontrast
|
|
103
104
|
# Make sure we have the bare minimum configuration to continue
|
104
105
|
Kontrast.configuration.validate
|
105
106
|
|
107
|
+
# Load specs
|
108
|
+
SpecBuilder.load_specs(specs_path)
|
109
|
+
|
106
110
|
return true
|
107
111
|
end
|
108
112
|
end
|
data/lib/kontrast.rb
CHANGED
@@ -1,15 +1,27 @@
|
|
1
1
|
# Dependencies
|
2
|
-
require "fog"
|
2
|
+
require "fog/aws"
|
3
3
|
require "bundler"
|
4
4
|
|
5
5
|
# Load classes
|
6
6
|
require "kontrast/exceptions"
|
7
7
|
require "kontrast/configuration"
|
8
|
+
require "kontrast/test"
|
9
|
+
require "kontrast/page_test"
|
10
|
+
require "kontrast/api_endpoint_test"
|
8
11
|
require "kontrast/test_builder"
|
12
|
+
require "kontrast/test_suite"
|
13
|
+
require "kontrast/spec"
|
14
|
+
require "kontrast/spec_builder"
|
9
15
|
require "kontrast/selenium_handler"
|
10
|
-
require "kontrast/
|
16
|
+
require "kontrast/image_helper"
|
11
17
|
require "kontrast/gallery_creator"
|
12
|
-
require "kontrast/
|
18
|
+
require "kontrast/global_runner"
|
19
|
+
require "kontrast/image_uploader"
|
20
|
+
require "kontrast/thumbnail_creator"
|
21
|
+
require "kontrast/page_runner"
|
22
|
+
require "kontrast/api_endpoint_runner"
|
23
|
+
require "kontrast/page_comparator"
|
24
|
+
require "kontrast/api_endpoint_comparator"
|
13
25
|
|
14
26
|
module Kontrast
|
15
27
|
class << self
|
@@ -26,7 +38,7 @@ module Kontrast
|
|
26
38
|
Bundler.environment.current_dependencies.each do |dep|
|
27
39
|
return true if dep.name == "rails"
|
28
40
|
end
|
29
|
-
rescue
|
41
|
+
rescue StandardError => e
|
30
42
|
# Quietly ignore any exceptions
|
31
43
|
end
|
32
44
|
return false
|
@@ -61,7 +73,7 @@ module Kontrast
|
|
61
73
|
# Call "before" hook
|
62
74
|
Kontrast.configuration.before_run
|
63
75
|
|
64
|
-
runner =
|
76
|
+
runner = GlobalRunner.new
|
65
77
|
runner.run
|
66
78
|
ensure
|
67
79
|
# Call "after" hook
|