wraith 3.0.4 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -0
  3. data/Gemfile +1 -1
  4. data/lib/wraith/cli.rb +108 -73
  5. data/lib/wraith/compare_images.rb +4 -2
  6. data/lib/wraith/crop.rb +3 -1
  7. data/lib/wraith/folder.rb +6 -4
  8. data/lib/wraith/gallery.rb +21 -22
  9. data/lib/wraith/helpers/capture_options.rb +52 -0
  10. data/lib/wraith/helpers/custom_exceptions.rb +8 -0
  11. data/lib/wraith/helpers/logger.rb +20 -0
  12. data/lib/wraith/helpers/save_metadata.rb +31 -0
  13. data/lib/wraith/{utilities.rb → helpers/utilities.rb} +5 -4
  14. data/lib/wraith/javascript/_helper.js +0 -2
  15. data/lib/wraith/javascript/casper.js +26 -16
  16. data/lib/wraith/javascript/phantom.js +148 -5
  17. data/lib/wraith/save_images.rb +28 -104
  18. data/lib/wraith/spider.rb +12 -5
  19. data/lib/wraith/thumbnails.rb +0 -2
  20. data/lib/wraith/validate.rb +98 -0
  21. data/lib/wraith/version.rb +1 -1
  22. data/lib/wraith/wraith.rb +19 -12
  23. data/spec/_helpers.rb +4 -10
  24. data/spec/before_capture_spec.rb +19 -10
  25. data/spec/config_spec.rb +2 -9
  26. data/spec/configs/test_config--casper.yaml +1 -5
  27. data/spec/construct_command_spec.rb +43 -0
  28. data/spec/gallery_spec.rb +1 -2
  29. data/spec/js/custom_snap_file.js +26 -16
  30. data/spec/js/global.js +2 -1
  31. data/spec/js/path.js +2 -1
  32. data/spec/resize_reload_spec.rb +4 -8
  33. data/spec/save_images_spec.rb +3 -2
  34. data/spec/validate_spec.rb +76 -0
  35. data/templates/configs/capture.yaml +61 -0
  36. data/templates/configs/history.yaml +81 -0
  37. data/templates/configs/spider.yaml +13 -2
  38. data/templates/javascript/cookies_and_headers--casper.js +16 -0
  39. data/templates/javascript/cookies_and_headers--phantom.js +26 -0
  40. data/templates/javascript/disable_javascript--casper.js +11 -0
  41. data/templates/javascript/disable_javascript--phantom.js +13 -0
  42. data/templates/javascript/interact--casper.js +11 -0
  43. data/templates/javascript/interact--phantom.js +17 -0
  44. data/templates/javascript/wait--casper.js +8 -0
  45. data/templates/javascript/wait--phantom.js +8 -0
  46. data/wraith.gemspec +1 -1
  47. metadata +27 -14
  48. data/lib/wraith/javascript/_phantom__common.js +0 -120
  49. data/lib/wraith/javascript/phantom--nojs.js +0 -6
  50. data/templates/configs/component.yaml +0 -60
  51. data/templates/configs/multiple_domains.yaml +0 -53
  52. data/templates/javascript/beforeCapture--casper_example.js +0 -12
  53. data/templates/javascript/beforeCapture--phantom_example.js +0 -36
@@ -0,0 +1,52 @@
1
+ require "wraith"
2
+ require "wraith/helpers/utilities"
3
+
4
+ class CaptureOptions
5
+ attr_reader :options, :wraith
6
+
7
+ def initialize(options, wraith)
8
+ @options = options
9
+ @wraith = wraith
10
+ end
11
+
12
+ def path
13
+ has_casper(options)
14
+ end
15
+
16
+ def selector
17
+ options["selector"] || "body"
18
+ end
19
+
20
+ def resize
21
+ # path level, or YAML-file level `resize_or_reload` property value
22
+ if @options["resize_or_reload"]
23
+ (@options["resize_or_reload"] == 'resize')
24
+ else
25
+ @wraith.resize
26
+ end
27
+ end
28
+
29
+ def before_capture
30
+ @options["before_capture"] ? convert_to_absolute(@options["before_capture"]) : false
31
+ end
32
+
33
+ def base_url
34
+ base_urls(path)
35
+ end
36
+
37
+ def compare_url
38
+ compare_urls(path)
39
+ end
40
+
41
+ def base_urls(path)
42
+ wraith.base_domain + path unless wraith.base_domain.nil?
43
+ end
44
+
45
+ def compare_urls(path)
46
+ wraith.comp_domain + path unless wraith.comp_domain.nil?
47
+ end
48
+
49
+ def has_casper(options)
50
+ options["path"] ? options["path"] : options
51
+ end
52
+ end
@@ -0,0 +1,8 @@
1
+ class CustomError < StandardError
2
+ end
3
+
4
+ class InvalidDomainsError < CustomError
5
+ end
6
+
7
+ class MissingRequiredPropertyError < CustomError
8
+ end
@@ -0,0 +1,20 @@
1
+ # Logging Module, credit: http://stackoverflow.com/a/6768164
2
+ require 'logger'
3
+
4
+ module Logging
5
+ # This is the magical bit that gets mixed into your classes
6
+ def logger
7
+ Logging.logger
8
+ end
9
+
10
+ # Global, memoized, lazy initialized instance of a logger
11
+ def self.logger
12
+ unless @logger
13
+ @logger = Logger.new(STDOUT)
14
+ @logger.formatter = proc do |severity, datetime, progname, msg|
15
+ (severity == 'INFO') ? "#{msg}\n" : "#{severity}: #{msg}\n"
16
+ end
17
+ end
18
+ @logger
19
+ end
20
+ end
@@ -0,0 +1,31 @@
1
+ require "wraith"
2
+
3
+ class SaveMetadata
4
+ attr_reader :wraith, :history
5
+
6
+ def initialize(config, history)
7
+ @wraith = config
8
+ @history = history
9
+ end
10
+
11
+ def history_label
12
+ history ? "_latest" : ""
13
+ end
14
+
15
+ def file_names(width, label, domain_label)
16
+ width = "MULTI" if width.is_a? Array
17
+ "#{wraith.directory}/#{label}/#{width}_#{engine}_#{domain_label}.png"
18
+ end
19
+
20
+ def base_label
21
+ "#{wraith.base_domain_label}#{history_label}"
22
+ end
23
+
24
+ def compare_label
25
+ "#{wraith.comp_domain_label}#{history_label}"
26
+ end
27
+
28
+ def engine
29
+ wraith.engine
30
+ end
31
+ end
@@ -1,12 +1,13 @@
1
- # @TODO - extract more shared functions to this file
1
+ require "wraith/helpers/custom_exceptions"
2
2
 
3
3
  def convert_to_absolute(filepath)
4
- if filepath[0] == '/'
4
+ if !filepath
5
+ 'false'
6
+ elsif filepath[0] == '/'
5
7
  # filepath is already absolute. return unchanged
6
8
  filepath
7
9
  else
8
10
  # filepath is relative. it must be converted to absolute
9
- working_dir = `pwd`.chomp
10
- "#{working_dir}/#{filepath}"
11
+ "#{Dir.pwd}/#{filepath}"
11
12
  end
12
13
  end
@@ -1,8 +1,6 @@
1
1
  module.exports = function (commandLineDimensions) {
2
2
 
3
3
  commandLineDimensions = '' + commandLineDimensions; // cast to string
4
- // remove quotes from dimensions string
5
- commandLineDimensions = commandLineDimensions.replace(/'/gi, '');
6
4
 
7
5
  function getWidthAndHeight(dimensions) {
8
6
  dimensions = /(\d*)x?((\d*))?/i.exec(dimensions);
@@ -57,25 +57,35 @@ casper.start();
57
57
  casper.open(url);
58
58
  casper.viewport(currentDimensions.viewportWidth, currentDimensions.viewportHeight);
59
59
  casper.then(function() {
60
- if (globalBeforeCaptureJS) {
61
- require(globalBeforeCaptureJS)(this);
60
+ var self = this;
61
+ if (globalBeforeCaptureJS && pathBeforeCaptureJS) {
62
+ require(globalBeforeCaptureJS)(self, function thenExecuteOtherBeforeCaptureFile() {
63
+ require(pathBeforeCaptureJS)(self, captureImage);
64
+ });
62
65
  }
63
- });
64
- casper.then(function() {
65
- if (pathBeforeCaptureJS) {
66
- require(pathBeforeCaptureJS)(this);
66
+ else if (globalBeforeCaptureJS) {
67
+ require(globalBeforeCaptureJS)(self, captureImage);
68
+ }
69
+ else if (pathBeforeCaptureJS) {
70
+ require(pathBeforeCaptureJS)(self, captureImage);
71
+ }
72
+ else {
73
+ captureImage();
67
74
  }
68
75
  });
69
- // waits for all images to download before taking screenshots
70
- // (broken images are a big cause of Wraith failures!)
71
- // Credit: http://reff.it/8m3HYP
72
- casper.waitFor(function() {
73
- return this.evaluate(function() {
74
- var images = document.getElementsByTagName('img');
75
- return Array.prototype.every.call(images, function(i) { return i.complete; });
76
+
77
+ function captureImage() {
78
+ // waits for all images to download before taking screenshots
79
+ // (broken images are a big cause of Wraith failures!)
80
+ // Credit: http://reff.it/8m3HYP
81
+ casper.waitFor(function() {
82
+ return this.evaluate(function() {
83
+ var images = document.getElementsByTagName('img');
84
+ return Array.prototype.every.call(images, function(i) { return i.complete; });
85
+ });
86
+ }, function then () {
87
+ snap.bind(this)();
76
88
  });
77
- }, function then () {
78
- snap.bind(this)();
79
- });
89
+ }
80
90
 
81
91
  casper.run();
@@ -1,6 +1,149 @@
1
- var system = require('system');
1
+ // modules
2
+ var system = require('system'),
3
+ page = require('webpage').create(),
4
+ helper = require('./_helper.js')(system.args[2]);
2
5
 
3
- require('./_phantom__common.js')({
4
- systemArgs: system.args,
5
- javascriptEnabled: true
6
- });
6
+ // command line arguments
7
+ var url = system.args[1],
8
+ dimensions = helper.dimensions,
9
+ image_name = system.args[3],
10
+ selector = system.args[4],
11
+ globalBeforeCaptureJS = system.args[5],
12
+ pathBeforeCaptureJS = system.args[6],
13
+ dimensionsProcessed = 0,
14
+ currentDimensions;
15
+
16
+ globalBeforeCaptureJS = globalBeforeCaptureJS === 'false' ? false : globalBeforeCaptureJS;
17
+ pathBeforeCaptureJS = pathBeforeCaptureJS === 'false' ? false : pathBeforeCaptureJS;
18
+
19
+ var current_requests = 0;
20
+ var last_request_timeout;
21
+ var final_timeout;
22
+
23
+ var setupJavaScriptRan = false;
24
+
25
+ var waitTime = 300,
26
+ maxWait = 5000,
27
+ beenLoadingFor = 0;
28
+
29
+ if (helper.takingMultipleScreenshots(dimensions)) {
30
+ currentDimensions = dimensions[0];
31
+ image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
32
+ }
33
+ else {
34
+ currentDimensions = dimensions;
35
+ }
36
+
37
+ page.settings = { loadImages: true, javascriptEnabled: true};
38
+ page.settings.userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.17';
39
+
40
+ page.onError = function(msg, trace) {
41
+ // suppress JS errors from Wraith output
42
+ // http://stackoverflow.com/a/19538646
43
+ };
44
+
45
+ page.onResourceRequested = function(req) {
46
+ current_requests += 1;
47
+ };
48
+
49
+ page.onResourceReceived = function(res) {
50
+ if (res.stage === 'end') {
51
+ current_requests -= 1;
52
+ }
53
+ };
54
+
55
+ console.log('Loading ' + url + ' at dimensions: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
56
+ page.viewportSize = { width: currentDimensions.viewportWidth, height: currentDimensions.viewportHeight};
57
+
58
+ page.open(url, function(status) {
59
+ if (status !== 'success') {
60
+ console.log('Error with page ' + url);
61
+ phantom.exit();
62
+ }
63
+
64
+ setTimeout(checkStatusOfAssets, waitTime);
65
+ });
66
+
67
+
68
+ function checkStatusOfAssets() {
69
+ if (current_requests >= 1) {
70
+
71
+ if (beenLoadingFor > maxWait) {
72
+ // sometimes not all assets will download in an acceptable time - continue anyway.
73
+ markPageAsLoaded();
74
+ }
75
+ else {
76
+ beenLoadingFor += waitTime;
77
+ setTimeout(checkStatusOfAssets, waitTime);
78
+ }
79
+ }
80
+ else {
81
+ markPageAsLoaded();
82
+ }
83
+ }
84
+
85
+ function markPageAsLoaded() {
86
+ if (!setupJavaScriptRan) {
87
+ runSetupJavaScriptThen(captureImage);
88
+ }
89
+ else {
90
+ captureImage();
91
+ }
92
+ }
93
+
94
+
95
+ function runSetupJavaScriptThen(callback) {
96
+ setupJavaScriptRan = true;
97
+ if (globalBeforeCaptureJS && pathBeforeCaptureJS) {
98
+ require(globalBeforeCaptureJS)(page, function thenExecuteOtherBeforeCaptureFile() {
99
+ require(pathBeforeCaptureJS)(page, callback);
100
+ });
101
+ }
102
+ else if (globalBeforeCaptureJS) {
103
+ require(globalBeforeCaptureJS)(page, callback);
104
+ }
105
+ else if (pathBeforeCaptureJS) {
106
+ require(pathBeforeCaptureJS)(page, callback);
107
+ }
108
+ else {
109
+ callback();
110
+ }
111
+ }
112
+
113
+ function captureImage() {
114
+ takeScreenshot();
115
+
116
+ dimensionsProcessed++;
117
+ if (helper.takingMultipleScreenshots(dimensions) && dimensionsProcessed < dimensions.length) {
118
+ currentDimensions = dimensions[dimensionsProcessed];
119
+ image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
120
+ setTimeout(resizeAndCaptureImage, waitTime);
121
+ }
122
+ else {
123
+ exit_phantom();
124
+ }
125
+ }
126
+
127
+ function resizeAndCaptureImage() {
128
+ console.log('Resizing ' + url + ' to: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
129
+ page.viewportSize = { width: currentDimensions.viewportWidth, height: currentDimensions.viewportHeight};
130
+ setTimeout(captureImage, 5000); // give page time to re-render properly
131
+ }
132
+
133
+ function takeScreenshot() {
134
+ console.log('Snapping ' + url + ' at: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
135
+ page.clipRect = {
136
+ top: 0,
137
+ left: 0,
138
+ height: currentDimensions.viewportHeight,
139
+ width: currentDimensions.viewportWidth
140
+ };
141
+ page.render(image_name);
142
+ }
143
+
144
+ function exit_phantom() {
145
+ // prevent CI from failing from 'Unsafe JavaScript attempt to access frame with URL about:blank from frame with URL' errors. See https://github.com/n1k0/casperjs/issues/1068
146
+ setTimeout(function(){
147
+ phantom.exit();
148
+ }, 30);
149
+ }
@@ -1,9 +1,13 @@
1
- require "wraith"
2
1
  require "parallel"
3
2
  require "shellwords"
4
- require "wraith/utilities"
3
+ require "wraith"
4
+ require "wraith/helpers/capture_options"
5
+ require "wraith/helpers/logger"
6
+ require "wraith/helpers/save_metadata"
7
+ require "wraith/helpers/utilities"
5
8
 
6
9
  class Wraith::SaveImages
10
+ include Logging
7
11
  attr_reader :wraith, :history, :meta
8
12
 
9
13
  def initialize(config, history = false, yaml_passed = false)
@@ -54,26 +58,15 @@ class Wraith::SaveImages
54
58
  end
55
59
 
56
60
  def prepare_widths_for_cli(width)
57
- if width.kind_of? Array
58
- # prepare for the command line. [30,40,50] => "'30','40','50'"
59
- width = width.map{ |i| "'#{i}'" }.join(',')
60
- end
61
+ # prepare for the command line. [30,40,50] => "30,40,50"
62
+ width = width.join(',') if width.is_a? Array
61
63
  width
62
64
  end
63
65
 
64
- def capture_page_image(browser, url, width, file_name, selector, global_before_capture, path_before_capture)
65
-
66
- command = "#{browser} #{wraith.phantomjs_options} '#{wraith.snap_file}' '#{url}' \"#{width}\" '#{file_name}' '#{selector}' '#{global_before_capture}' '#{path_before_capture}'"
67
-
68
- # @TODO - uncomment the following line when we add a verbose mode
69
- # puts command
70
- run_command command
71
- end
72
-
73
66
  def run_command(command)
74
67
  output = []
75
68
  IO.popen(command).each do |line|
76
- puts line
69
+ logger.info line
77
70
  output << line.chomp!
78
71
  end.close
79
72
  output
@@ -82,17 +75,30 @@ class Wraith::SaveImages
82
75
  def parallel_task(jobs)
83
76
  Parallel.each(jobs, :in_threads => 8) do |_label, _path, width, url, filename, selector, global_before_capture, path_before_capture|
84
77
  begin
85
- attempt_image_capture(width, url, filename, selector, global_before_capture, path_before_capture, 5)
78
+ command = construct_command(width, url, filename, selector, global_before_capture, path_before_capture)
79
+ attempt_image_capture(command, filename)
86
80
  rescue => e
87
- puts e
81
+ logger.error e
88
82
  create_invalid_image(filename, width)
89
83
  end
90
84
  end
91
85
  end
92
86
 
93
- def attempt_image_capture(width, url, filename, selector, global_before_capture, path_before_capture, max_attempts)
87
+ def construct_command(width, url, file_name, selector, global_before_capture, path_before_capture)
88
+ width = prepare_widths_for_cli(width)
89
+ selector = selector.gsub '#', '\#' # make sure id selectors aren't escaped in the CLI
90
+ global_before_capture = convert_to_absolute global_before_capture
91
+ path_before_capture = convert_to_absolute path_before_capture
92
+
93
+ command_to_run = "#{meta.engine} #{wraith.phantomjs_options} '#{wraith.snap_file}' '#{url}' '#{width}' '#{file_name}' '#{selector}' '#{global_before_capture}' '#{path_before_capture}'"
94
+ logger.debug command_to_run
95
+ command_to_run
96
+ end
97
+
98
+ def attempt_image_capture(capture_page_image, filename)
99
+ max_attempts = 5
94
100
  max_attempts.times do |i|
95
- capture_page_image meta.engine, url, width, filename, selector, global_before_capture, path_before_capture
101
+ run_command capture_page_image
96
102
 
97
103
  if wraith.resize
98
104
  return # @TODO - need to check if the image was generated, as per the reload example below
@@ -100,14 +106,14 @@ class Wraith::SaveImages
100
106
 
101
107
  return if File.exist? filename
102
108
 
103
- puts "Failed to capture image #{filename} on attempt number #{i + 1} of #{max_attempts}"
109
+ logger.warn "Failed to capture image #{filename} on attempt number #{i + 1} of #{max_attempts}"
104
110
  end
105
111
 
106
112
  fail "Unable to capture image #{filename} after #{max_attempts} attempt(s)"
107
113
  end
108
114
 
109
115
  def create_invalid_image(filename, width)
110
- puts "Using fallback image instead"
116
+ logger.warn "Using fallback image instead"
111
117
  invalid = File.expand_path("../../assets/invalid.jpg", File.dirname(__FILE__))
112
118
  FileUtils.cp invalid, filename
113
119
 
@@ -118,85 +124,3 @@ class Wraith::SaveImages
118
124
  `convert #{image.shellescape} -background none -extent #{width}x0 #{image.shellescape}`
119
125
  end
120
126
  end
121
-
122
- class CaptureOptions
123
- attr_reader :options, :wraith
124
-
125
- def initialize(options, wraith)
126
- @options = options
127
- @wraith = wraith
128
- end
129
-
130
- def path
131
- has_casper(options)
132
- end
133
-
134
- def selector
135
- options["selector"] || "body"
136
- end
137
-
138
- def resize
139
- # path level, or YAML-file level `resize_or_reload` property value
140
- if @options["resize_or_reload"]
141
- (@options["resize_or_reload"] == 'resize')
142
- else
143
- @wraith.resize
144
- end
145
- end
146
-
147
- def before_capture
148
- @options["before_capture"] ? convert_to_absolute(@options["before_capture"]) : "false"
149
- end
150
-
151
- def base_url
152
- base_urls(path)
153
- end
154
-
155
- def compare_url
156
- compare_urls(path)
157
- end
158
-
159
- def base_urls(path)
160
- wraith.base_domain + path unless wraith.base_domain.nil?
161
- end
162
-
163
- def compare_urls(path)
164
- wraith.comp_domain + path unless wraith.comp_domain.nil?
165
- end
166
-
167
- def has_casper(options)
168
- options["path"] ? options["path"] : options
169
- end
170
- end
171
-
172
- class SaveMetadata
173
- attr_reader :wraith, :history
174
-
175
- def initialize(config, history)
176
- @wraith = config
177
- @history = history
178
- end
179
-
180
- def history_label
181
- history ? "_latest" : ""
182
- end
183
-
184
- def file_names(width, label, domain_label)
185
- if width.kind_of? Array
186
- width = 'MULTI'
187
- end
188
- "#{wraith.directory}/#{label}/#{width}_#{engine}_#{domain_label}.png"
189
- end
190
-
191
- def base_label
192
- "#{wraith.base_domain_label}#{history_label}"
193
- end
194
-
195
- def compare_label
196
- "#{wraith.comp_domain_label}#{history_label}"
197
- end
198
-
199
- def engine
200
- wraith.engine
201
- end
202
- end