wraith 3.0.4 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -0
  3. data/Gemfile +1 -1
  4. data/lib/wraith/cli.rb +108 -73
  5. data/lib/wraith/compare_images.rb +4 -2
  6. data/lib/wraith/crop.rb +3 -1
  7. data/lib/wraith/folder.rb +6 -4
  8. data/lib/wraith/gallery.rb +21 -22
  9. data/lib/wraith/helpers/capture_options.rb +52 -0
  10. data/lib/wraith/helpers/custom_exceptions.rb +8 -0
  11. data/lib/wraith/helpers/logger.rb +20 -0
  12. data/lib/wraith/helpers/save_metadata.rb +31 -0
  13. data/lib/wraith/{utilities.rb → helpers/utilities.rb} +5 -4
  14. data/lib/wraith/javascript/_helper.js +0 -2
  15. data/lib/wraith/javascript/casper.js +26 -16
  16. data/lib/wraith/javascript/phantom.js +148 -5
  17. data/lib/wraith/save_images.rb +28 -104
  18. data/lib/wraith/spider.rb +12 -5
  19. data/lib/wraith/thumbnails.rb +0 -2
  20. data/lib/wraith/validate.rb +98 -0
  21. data/lib/wraith/version.rb +1 -1
  22. data/lib/wraith/wraith.rb +19 -12
  23. data/spec/_helpers.rb +4 -10
  24. data/spec/before_capture_spec.rb +19 -10
  25. data/spec/config_spec.rb +2 -9
  26. data/spec/configs/test_config--casper.yaml +1 -5
  27. data/spec/construct_command_spec.rb +43 -0
  28. data/spec/gallery_spec.rb +1 -2
  29. data/spec/js/custom_snap_file.js +26 -16
  30. data/spec/js/global.js +2 -1
  31. data/spec/js/path.js +2 -1
  32. data/spec/resize_reload_spec.rb +4 -8
  33. data/spec/save_images_spec.rb +3 -2
  34. data/spec/validate_spec.rb +76 -0
  35. data/templates/configs/capture.yaml +61 -0
  36. data/templates/configs/history.yaml +81 -0
  37. data/templates/configs/spider.yaml +13 -2
  38. data/templates/javascript/cookies_and_headers--casper.js +16 -0
  39. data/templates/javascript/cookies_and_headers--phantom.js +26 -0
  40. data/templates/javascript/disable_javascript--casper.js +11 -0
  41. data/templates/javascript/disable_javascript--phantom.js +13 -0
  42. data/templates/javascript/interact--casper.js +11 -0
  43. data/templates/javascript/interact--phantom.js +17 -0
  44. data/templates/javascript/wait--casper.js +8 -0
  45. data/templates/javascript/wait--phantom.js +8 -0
  46. data/wraith.gemspec +1 -1
  47. metadata +27 -14
  48. data/lib/wraith/javascript/_phantom__common.js +0 -120
  49. data/lib/wraith/javascript/phantom--nojs.js +0 -6
  50. data/templates/configs/component.yaml +0 -60
  51. data/templates/configs/multiple_domains.yaml +0 -53
  52. data/templates/javascript/beforeCapture--casper_example.js +0 -12
  53. data/templates/javascript/beforeCapture--phantom_example.js +0 -36
@@ -0,0 +1,52 @@
1
+ require "wraith"
2
+ require "wraith/helpers/utilities"
3
+
4
+ class CaptureOptions
5
+ attr_reader :options, :wraith
6
+
7
+ def initialize(options, wraith)
8
+ @options = options
9
+ @wraith = wraith
10
+ end
11
+
12
+ def path
13
+ has_casper(options)
14
+ end
15
+
16
+ def selector
17
+ options["selector"] || "body"
18
+ end
19
+
20
+ def resize
21
+ # path level, or YAML-file level `resize_or_reload` property value
22
+ if @options["resize_or_reload"]
23
+ (@options["resize_or_reload"] == 'resize')
24
+ else
25
+ @wraith.resize
26
+ end
27
+ end
28
+
29
+ def before_capture
30
+ @options["before_capture"] ? convert_to_absolute(@options["before_capture"]) : false
31
+ end
32
+
33
+ def base_url
34
+ base_urls(path)
35
+ end
36
+
37
+ def compare_url
38
+ compare_urls(path)
39
+ end
40
+
41
+ def base_urls(path)
42
+ wraith.base_domain + path unless wraith.base_domain.nil?
43
+ end
44
+
45
+ def compare_urls(path)
46
+ wraith.comp_domain + path unless wraith.comp_domain.nil?
47
+ end
48
+
49
+ def has_casper(options)
50
+ options["path"] ? options["path"] : options
51
+ end
52
+ end
@@ -0,0 +1,8 @@
1
+ class CustomError < StandardError
2
+ end
3
+
4
+ class InvalidDomainsError < CustomError
5
+ end
6
+
7
+ class MissingRequiredPropertyError < CustomError
8
+ end
@@ -0,0 +1,20 @@
1
+ # Logging Module, credit: http://stackoverflow.com/a/6768164
2
+ require 'logger'
3
+
4
+ module Logging
5
+ # This is the magical bit that gets mixed into your classes
6
+ def logger
7
+ Logging.logger
8
+ end
9
+
10
+ # Global, memoized, lazy initialized instance of a logger
11
+ def self.logger
12
+ unless @logger
13
+ @logger = Logger.new(STDOUT)
14
+ @logger.formatter = proc do |severity, datetime, progname, msg|
15
+ (severity == 'INFO') ? "#{msg}\n" : "#{severity}: #{msg}\n"
16
+ end
17
+ end
18
+ @logger
19
+ end
20
+ end
@@ -0,0 +1,31 @@
1
+ require "wraith"
2
+
3
+ class SaveMetadata
4
+ attr_reader :wraith, :history
5
+
6
+ def initialize(config, history)
7
+ @wraith = config
8
+ @history = history
9
+ end
10
+
11
+ def history_label
12
+ history ? "_latest" : ""
13
+ end
14
+
15
+ def file_names(width, label, domain_label)
16
+ width = "MULTI" if width.is_a? Array
17
+ "#{wraith.directory}/#{label}/#{width}_#{engine}_#{domain_label}.png"
18
+ end
19
+
20
+ def base_label
21
+ "#{wraith.base_domain_label}#{history_label}"
22
+ end
23
+
24
+ def compare_label
25
+ "#{wraith.comp_domain_label}#{history_label}"
26
+ end
27
+
28
+ def engine
29
+ wraith.engine
30
+ end
31
+ end
@@ -1,12 +1,13 @@
1
- # @TODO - extract more shared functions to this file
1
+ require "wraith/helpers/custom_exceptions"
2
2
 
3
3
  def convert_to_absolute(filepath)
4
- if filepath[0] == '/'
4
+ if !filepath
5
+ 'false'
6
+ elsif filepath[0] == '/'
5
7
  # filepath is already absolute. return unchanged
6
8
  filepath
7
9
  else
8
10
  # filepath is relative. it must be converted to absolute
9
- working_dir = `pwd`.chomp
10
- "#{working_dir}/#{filepath}"
11
+ "#{Dir.pwd}/#{filepath}"
11
12
  end
12
13
  end
@@ -1,8 +1,6 @@
1
1
  module.exports = function (commandLineDimensions) {
2
2
 
3
3
  commandLineDimensions = '' + commandLineDimensions; // cast to string
4
- // remove quotes from dimensions string
5
- commandLineDimensions = commandLineDimensions.replace(/'/gi, '');
6
4
 
7
5
  function getWidthAndHeight(dimensions) {
8
6
  dimensions = /(\d*)x?((\d*))?/i.exec(dimensions);
@@ -57,25 +57,35 @@ casper.start();
57
57
  casper.open(url);
58
58
  casper.viewport(currentDimensions.viewportWidth, currentDimensions.viewportHeight);
59
59
  casper.then(function() {
60
- if (globalBeforeCaptureJS) {
61
- require(globalBeforeCaptureJS)(this);
60
+ var self = this;
61
+ if (globalBeforeCaptureJS && pathBeforeCaptureJS) {
62
+ require(globalBeforeCaptureJS)(self, function thenExecuteOtherBeforeCaptureFile() {
63
+ require(pathBeforeCaptureJS)(self, captureImage);
64
+ });
62
65
  }
63
- });
64
- casper.then(function() {
65
- if (pathBeforeCaptureJS) {
66
- require(pathBeforeCaptureJS)(this);
66
+ else if (globalBeforeCaptureJS) {
67
+ require(globalBeforeCaptureJS)(self, captureImage);
68
+ }
69
+ else if (pathBeforeCaptureJS) {
70
+ require(pathBeforeCaptureJS)(self, captureImage);
71
+ }
72
+ else {
73
+ captureImage();
67
74
  }
68
75
  });
69
- // waits for all images to download before taking screenshots
70
- // (broken images are a big cause of Wraith failures!)
71
- // Credit: http://reff.it/8m3HYP
72
- casper.waitFor(function() {
73
- return this.evaluate(function() {
74
- var images = document.getElementsByTagName('img');
75
- return Array.prototype.every.call(images, function(i) { return i.complete; });
76
+
77
+ function captureImage() {
78
+ // waits for all images to download before taking screenshots
79
+ // (broken images are a big cause of Wraith failures!)
80
+ // Credit: http://reff.it/8m3HYP
81
+ casper.waitFor(function() {
82
+ return this.evaluate(function() {
83
+ var images = document.getElementsByTagName('img');
84
+ return Array.prototype.every.call(images, function(i) { return i.complete; });
85
+ });
86
+ }, function then () {
87
+ snap.bind(this)();
76
88
  });
77
- }, function then () {
78
- snap.bind(this)();
79
- });
89
+ }
80
90
 
81
91
  casper.run();
@@ -1,6 +1,149 @@
1
- var system = require('system');
1
+ // modules
2
+ var system = require('system'),
3
+ page = require('webpage').create(),
4
+ helper = require('./_helper.js')(system.args[2]);
2
5
 
3
- require('./_phantom__common.js')({
4
- systemArgs: system.args,
5
- javascriptEnabled: true
6
- });
6
+ // command line arguments
7
+ var url = system.args[1],
8
+ dimensions = helper.dimensions,
9
+ image_name = system.args[3],
10
+ selector = system.args[4],
11
+ globalBeforeCaptureJS = system.args[5],
12
+ pathBeforeCaptureJS = system.args[6],
13
+ dimensionsProcessed = 0,
14
+ currentDimensions;
15
+
16
+ globalBeforeCaptureJS = globalBeforeCaptureJS === 'false' ? false : globalBeforeCaptureJS;
17
+ pathBeforeCaptureJS = pathBeforeCaptureJS === 'false' ? false : pathBeforeCaptureJS;
18
+
19
+ var current_requests = 0;
20
+ var last_request_timeout;
21
+ var final_timeout;
22
+
23
+ var setupJavaScriptRan = false;
24
+
25
+ var waitTime = 300,
26
+ maxWait = 5000,
27
+ beenLoadingFor = 0;
28
+
29
+ if (helper.takingMultipleScreenshots(dimensions)) {
30
+ currentDimensions = dimensions[0];
31
+ image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
32
+ }
33
+ else {
34
+ currentDimensions = dimensions;
35
+ }
36
+
37
+ page.settings = { loadImages: true, javascriptEnabled: true};
38
+ page.settings.userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.17';
39
+
40
+ page.onError = function(msg, trace) {
41
+ // suppress JS errors from Wraith output
42
+ // http://stackoverflow.com/a/19538646
43
+ };
44
+
45
+ page.onResourceRequested = function(req) {
46
+ current_requests += 1;
47
+ };
48
+
49
+ page.onResourceReceived = function(res) {
50
+ if (res.stage === 'end') {
51
+ current_requests -= 1;
52
+ }
53
+ };
54
+
55
+ console.log('Loading ' + url + ' at dimensions: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
56
+ page.viewportSize = { width: currentDimensions.viewportWidth, height: currentDimensions.viewportHeight};
57
+
58
+ page.open(url, function(status) {
59
+ if (status !== 'success') {
60
+ console.log('Error with page ' + url);
61
+ phantom.exit();
62
+ }
63
+
64
+ setTimeout(checkStatusOfAssets, waitTime);
65
+ });
66
+
67
+
68
+ function checkStatusOfAssets() {
69
+ if (current_requests >= 1) {
70
+
71
+ if (beenLoadingFor > maxWait) {
72
+ // sometimes not all assets will download in an acceptable time - continue anyway.
73
+ markPageAsLoaded();
74
+ }
75
+ else {
76
+ beenLoadingFor += waitTime;
77
+ setTimeout(checkStatusOfAssets, waitTime);
78
+ }
79
+ }
80
+ else {
81
+ markPageAsLoaded();
82
+ }
83
+ }
84
+
85
+ function markPageAsLoaded() {
86
+ if (!setupJavaScriptRan) {
87
+ runSetupJavaScriptThen(captureImage);
88
+ }
89
+ else {
90
+ captureImage();
91
+ }
92
+ }
93
+
94
+
95
+ function runSetupJavaScriptThen(callback) {
96
+ setupJavaScriptRan = true;
97
+ if (globalBeforeCaptureJS && pathBeforeCaptureJS) {
98
+ require(globalBeforeCaptureJS)(page, function thenExecuteOtherBeforeCaptureFile() {
99
+ require(pathBeforeCaptureJS)(page, callback);
100
+ });
101
+ }
102
+ else if (globalBeforeCaptureJS) {
103
+ require(globalBeforeCaptureJS)(page, callback);
104
+ }
105
+ else if (pathBeforeCaptureJS) {
106
+ require(pathBeforeCaptureJS)(page, callback);
107
+ }
108
+ else {
109
+ callback();
110
+ }
111
+ }
112
+
113
+ function captureImage() {
114
+ takeScreenshot();
115
+
116
+ dimensionsProcessed++;
117
+ if (helper.takingMultipleScreenshots(dimensions) && dimensionsProcessed < dimensions.length) {
118
+ currentDimensions = dimensions[dimensionsProcessed];
119
+ image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
120
+ setTimeout(resizeAndCaptureImage, waitTime);
121
+ }
122
+ else {
123
+ exit_phantom();
124
+ }
125
+ }
126
+
127
+ function resizeAndCaptureImage() {
128
+ console.log('Resizing ' + url + ' to: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
129
+ page.viewportSize = { width: currentDimensions.viewportWidth, height: currentDimensions.viewportHeight};
130
+ setTimeout(captureImage, 5000); // give page time to re-render properly
131
+ }
132
+
133
+ function takeScreenshot() {
134
+ console.log('Snapping ' + url + ' at: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
135
+ page.clipRect = {
136
+ top: 0,
137
+ left: 0,
138
+ height: currentDimensions.viewportHeight,
139
+ width: currentDimensions.viewportWidth
140
+ };
141
+ page.render(image_name);
142
+ }
143
+
144
+ function exit_phantom() {
145
+ // prevent CI from failing from 'Unsafe JavaScript attempt to access frame with URL about:blank from frame with URL' errors. See https://github.com/n1k0/casperjs/issues/1068
146
+ setTimeout(function(){
147
+ phantom.exit();
148
+ }, 30);
149
+ }
@@ -1,9 +1,13 @@
1
- require "wraith"
2
1
  require "parallel"
3
2
  require "shellwords"
4
- require "wraith/utilities"
3
+ require "wraith"
4
+ require "wraith/helpers/capture_options"
5
+ require "wraith/helpers/logger"
6
+ require "wraith/helpers/save_metadata"
7
+ require "wraith/helpers/utilities"
5
8
 
6
9
  class Wraith::SaveImages
10
+ include Logging
7
11
  attr_reader :wraith, :history, :meta
8
12
 
9
13
  def initialize(config, history = false, yaml_passed = false)
@@ -54,26 +58,15 @@ class Wraith::SaveImages
54
58
  end
55
59
 
56
60
  def prepare_widths_for_cli(width)
57
- if width.kind_of? Array
58
- # prepare for the command line. [30,40,50] => "'30','40','50'"
59
- width = width.map{ |i| "'#{i}'" }.join(',')
60
- end
61
+ # prepare for the command line. [30,40,50] => "30,40,50"
62
+ width = width.join(',') if width.is_a? Array
61
63
  width
62
64
  end
63
65
 
64
- def capture_page_image(browser, url, width, file_name, selector, global_before_capture, path_before_capture)
65
-
66
- command = "#{browser} #{wraith.phantomjs_options} '#{wraith.snap_file}' '#{url}' \"#{width}\" '#{file_name}' '#{selector}' '#{global_before_capture}' '#{path_before_capture}'"
67
-
68
- # @TODO - uncomment the following line when we add a verbose mode
69
- # puts command
70
- run_command command
71
- end
72
-
73
66
  def run_command(command)
74
67
  output = []
75
68
  IO.popen(command).each do |line|
76
- puts line
69
+ logger.info line
77
70
  output << line.chomp!
78
71
  end.close
79
72
  output
@@ -82,17 +75,30 @@ class Wraith::SaveImages
82
75
  def parallel_task(jobs)
83
76
  Parallel.each(jobs, :in_threads => 8) do |_label, _path, width, url, filename, selector, global_before_capture, path_before_capture|
84
77
  begin
85
- attempt_image_capture(width, url, filename, selector, global_before_capture, path_before_capture, 5)
78
+ command = construct_command(width, url, filename, selector, global_before_capture, path_before_capture)
79
+ attempt_image_capture(command, filename)
86
80
  rescue => e
87
- puts e
81
+ logger.error e
88
82
  create_invalid_image(filename, width)
89
83
  end
90
84
  end
91
85
  end
92
86
 
93
- def attempt_image_capture(width, url, filename, selector, global_before_capture, path_before_capture, max_attempts)
87
+ def construct_command(width, url, file_name, selector, global_before_capture, path_before_capture)
88
+ width = prepare_widths_for_cli(width)
89
+ selector = selector.gsub '#', '\#' # make sure id selectors aren't escaped in the CLI
90
+ global_before_capture = convert_to_absolute global_before_capture
91
+ path_before_capture = convert_to_absolute path_before_capture
92
+
93
+ command_to_run = "#{meta.engine} #{wraith.phantomjs_options} '#{wraith.snap_file}' '#{url}' '#{width}' '#{file_name}' '#{selector}' '#{global_before_capture}' '#{path_before_capture}'"
94
+ logger.debug command_to_run
95
+ command_to_run
96
+ end
97
+
98
+ def attempt_image_capture(capture_page_image, filename)
99
+ max_attempts = 5
94
100
  max_attempts.times do |i|
95
- capture_page_image meta.engine, url, width, filename, selector, global_before_capture, path_before_capture
101
+ run_command capture_page_image
96
102
 
97
103
  if wraith.resize
98
104
  return # @TODO - need to check if the image was generated, as per the reload example below
@@ -100,14 +106,14 @@ class Wraith::SaveImages
100
106
 
101
107
  return if File.exist? filename
102
108
 
103
- puts "Failed to capture image #{filename} on attempt number #{i + 1} of #{max_attempts}"
109
+ logger.warn "Failed to capture image #{filename} on attempt number #{i + 1} of #{max_attempts}"
104
110
  end
105
111
 
106
112
  fail "Unable to capture image #{filename} after #{max_attempts} attempt(s)"
107
113
  end
108
114
 
109
115
  def create_invalid_image(filename, width)
110
- puts "Using fallback image instead"
116
+ logger.warn "Using fallback image instead"
111
117
  invalid = File.expand_path("../../assets/invalid.jpg", File.dirname(__FILE__))
112
118
  FileUtils.cp invalid, filename
113
119
 
@@ -118,85 +124,3 @@ class Wraith::SaveImages
118
124
  `convert #{image.shellescape} -background none -extent #{width}x0 #{image.shellescape}`
119
125
  end
120
126
  end
121
-
122
- class CaptureOptions
123
- attr_reader :options, :wraith
124
-
125
- def initialize(options, wraith)
126
- @options = options
127
- @wraith = wraith
128
- end
129
-
130
- def path
131
- has_casper(options)
132
- end
133
-
134
- def selector
135
- options["selector"] || "body"
136
- end
137
-
138
- def resize
139
- # path level, or YAML-file level `resize_or_reload` property value
140
- if @options["resize_or_reload"]
141
- (@options["resize_or_reload"] == 'resize')
142
- else
143
- @wraith.resize
144
- end
145
- end
146
-
147
- def before_capture
148
- @options["before_capture"] ? convert_to_absolute(@options["before_capture"]) : "false"
149
- end
150
-
151
- def base_url
152
- base_urls(path)
153
- end
154
-
155
- def compare_url
156
- compare_urls(path)
157
- end
158
-
159
- def base_urls(path)
160
- wraith.base_domain + path unless wraith.base_domain.nil?
161
- end
162
-
163
- def compare_urls(path)
164
- wraith.comp_domain + path unless wraith.comp_domain.nil?
165
- end
166
-
167
- def has_casper(options)
168
- options["path"] ? options["path"] : options
169
- end
170
- end
171
-
172
- class SaveMetadata
173
- attr_reader :wraith, :history
174
-
175
- def initialize(config, history)
176
- @wraith = config
177
- @history = history
178
- end
179
-
180
- def history_label
181
- history ? "_latest" : ""
182
- end
183
-
184
- def file_names(width, label, domain_label)
185
- if width.kind_of? Array
186
- width = 'MULTI'
187
- end
188
- "#{wraith.directory}/#{label}/#{width}_#{engine}_#{domain_label}.png"
189
- end
190
-
191
- def base_label
192
- "#{wraith.base_domain_label}#{history_label}"
193
- end
194
-
195
- def compare_label
196
- "#{wraith.comp_domain_label}#{history_label}"
197
- end
198
-
199
- def engine
200
- wraith.engine
201
- end
202
- end