wraith 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6737fd591c334bd813dbc07970efe9bea443d762
-  data.tar.gz: c1191e025c5fb12056dd8fa15b45a4be5c8027a9
+  metadata.gz: 0ad5c9e4eecd6cbe1a694bc29c62410ff45b7c86
+  data.tar.gz: c84b6e122617e02962a281ad885717342c919d35
 SHA512:
-  metadata.gz: a633654b6891f6803e074dffdd3ce37be5b55be336dbe33dfc0872dd16ec07d64878f779a1667e7b67eb864f5447d6eb3ca0e29f44f9b492a74bd2091653e8a2
-  data.tar.gz: 80cd0a7d9f9ed8216ab2e9b368f8c44c1e562a7a06dead497096a4f4a8e7d3e2ea27e007c64f02e2585c4262cf4e8e243c799277720637577fcd44f7de967e5f
+  metadata.gz: 5d415de27039b977ba067eac6a0f17edacdd301ea5a43e5058454b8b8b35151f1a1bc9cc0b0216695374ff9b2d11bce6afc90ffe11299c5ca14af8818ade9917
+  data.tar.gz: 68d6c30abf6631ba704fe705290b9ca909d95859b8268e0886ef19773c991fdb90aa36972fba343ad3a7089729ca41f3e45365005b4a3c90ad8110f81d642419

data/configs/config.yaml

@@ -44,3 +44,17 @@ fuzz: '20%'
 #Set the number of days to keep the site spider file
 spider_days:
   - 10
+
+#A list of URLs to skip when spidering. Ruby regular expressions can be 
+#used, if prefixed with !ruby/regexp as defined in the YAML Cookbook
+#http://www.yaml.org/YAML_for_ruby.html#regexps
+#
+# spider_skips:
+#   - /foo/bar.html # Matches /foo/bar.html explcitly
+#   - !ruby/regexp /^\/baz\// # Matches any URLs that start with /baz
+
+#Choose how results are displayed, by default alphanumeric. Different screen widths are always grouped.
+#alphanumeric - all paths (with, and without, a difference) are shown, sorted by path
+#diffs_first - all paths (with, and without, a difference) are shown, sorted by difference size (largest first)
+#diffs_only - only paths with a difference are shown, sorted by difference size (largest first)
+#mode: diffs_first

data/configs/config_nojs.yaml

@@ -41,3 +41,11 @@ fuzz: '20%'
 #Set the number of days to keep the site spider file
 spider_days:
   - 10
+
+    #A list of URLs to skip when spidering. Ruby regular expressions can be 
+#used, if prefixed with !ruby/regexp as defined in the YAML Cookbook
+#http://www.yaml.org/YAML_for_ruby.html#regexps
+#
+# spider_skips:
+#   - /foo/bar.html # Matches /foo/bar.html explcitly
+#   - !ruby/regexp /^\/baz\// # Matches any URLs that start with /baz

data/configs/test_config.yaml

@@ -44,3 +44,10 @@ fuzz: '20%'
 #Set the number of days to keep the site spider file
 spider_days:
   - 10
+#A list of URLs to skip when spidering. Ruby regular expressions can be 
+#used, if prefixed with !ruby/regexp as defined in the YAML Cookbook
+#http://www.yaml.org/YAML_for_ruby.html#regexps
+#
+# spider_skips:
+#   - /foo/bar.html # Matches /foo/bar.html explcitly
+#   - !ruby/regexp /^\/baz\// # Matches any URLs that start with /baz

data/lib/wraith/compare_images.rb

@@ -16,7 +16,6 @@ class Wraith::CompareImages
       diff = base.gsub(/([a-z0-9]+).png$/, 'diff.png')
       info = base.gsub(/([a-z0-9]+).png$/, 'data.txt')
       compare_task(base, compare, diff, info)
-      Dir.glob("#{wraith.directory}/*/*.txt").map { |f| "\n#{f}\n#{File.read(f)}" }
       puts 'Saved diff'
     end
   end

data/lib/wraith/folder.rb

@@ -33,7 +33,7 @@ class Wraith::FolderManager
     spider_paths.each do |folder_label, path|
       unless path
         path = folder_label
-        folder_label = path.gsub('/', '_')
+        folder_label = path.gsub('/', '__')
       end
 
       FileUtils.mkdir_p("#{dir}/thumbnails/#{folder_label}")
@@ -41,4 +41,20 @@ class Wraith::FolderManager
     end
     puts 'Creating Folders'
   end
+
+  # Tidy up the shots folder, removing uncessary files
+  #
+  def tidy_shots_folder dirs
+    if wraith.mode == 'diffs_only'
+      dirs.each do |a, b|
+        # If we are running in "diffs_only mode, and none of the variants show a difference
+        # we remove the file from the shots folder
+        if b.none? {|k, v| v[:data] > 0}
+          FileUtils.rm_rf("#{wraith.directory}/#{a}")
+          dirs.delete(a)
+        end
+      end
+    end
+  end
+
 end

data/lib/wraith/gallery.rb

@@ -15,6 +15,7 @@ class Wraith::GalleryGenerator
   def initialize(config)
     @wraith = Wraith::Wraith.new(config)
     @location = wraith.directory
+    @folder_manager = Wraith::FolderManager.new(config)
   end
 
   def parse_directories(dirname)
@@ -55,7 +56,7 @@ class Wraith::GalleryGenerator
               filename: filepath, thumb: thumbnail
             }
           when 'data'
-            size_dict[:data] = File.read("#{dirname}/#{filepath}")
+            size_dict[:data] = File.read("#{dirname}/#{filepath}").to_f
           else
             size_dict[:variants] << {
               name: group,
@@ -68,7 +69,14 @@ class Wraith::GalleryGenerator
         end
       end
     end
-    @dirs
+    @folder_manager.tidy_shots_folder(@dirs)
+    if [ 'diffs_only', 'diffs_first' ].include?(wraith.mode)
+      @sorted = @dirs.sort_by { |category, sizes| -1 * sizes.max_by { |size, dict| dict[:data]}[1][:data] }
+    else
+      @sorted = @dirs.sort_by { |category, sizes| category }
+    end
+    # The sort has made this into an enumerable, convert it back to a Hash
+    Hash[@sorted]
   end
 
   def generate_html(location, directories, template, destination, path)

data/lib/wraith/gallery_template/gallery_template.erb

@@ -20,17 +20,17 @@
                     <div class="panel">
                         <div class="panel-heading">Screenshots:</div>
                         <ul class="list-group list-group-flush">
-                            <% directories.keys.sort.each do |dir| %>
-                                <li class="list-group-item"><a href="#<%=path%><%=dir%>"><%=dir%></a></li>
+                            <% directories.keys.each do |dir| %>
+                                <li class="list-group-item"><a href="#<%=path%><%=dir%>"><%=dir.gsub('__', '/')%></a></li>
                             <% end %>
                         </ul>
                     </div>
                 </div>
                 <div class="col-lg-10">
-                    <% directories.to_a.sort.each do |dir, sizes| %>
+                    <% directories.each do |dir, sizes| %>
                         <div class="row">
                             <a name="<%= dir %>"></a>
-                            <h2><%= dir %></h2>
+                            <h2><%= dir.gsub('__', '/') %></h2>
                         </div>
                         <% sizes.to_a.sort.each do |size, files| %>
                             <div class="row">

data/lib/wraith/save_images.rb

@@ -54,7 +54,7 @@ class Wraith::SaveImages
     check_paths.each do |label, path|
       unless path
         path = label
-        label = path.gsub('/', '_')
+        label = path.gsub('/', '__')
       end
 
       base_url = base_urls(path)

data/lib/wraith/spider.rb

@@ -1,64 +1,108 @@
 require 'wraith'
 require 'anemone'
+require 'nokogiri'
 require 'uri'
 
 class Wraith::Spidering
-  attr_reader :wraith
 
   def initialize(config)
     @wraith = Wraith::Wraith.new(config)
   end
 
   def check_for_paths
-    unless wraith.paths
-      puts 'no paths defined'
-      spider
+    if @wraith.paths.nil?
+      unless @wraith.sitemap.nil?
+        puts 'no paths defined in config, loading paths from sitemap'
+        spider = Wraith::Sitemap.new(@wraith)
+      else
+        puts 'no paths defined in config, crawling from site root'
+        spider = Wraith::Crawler.new(@wraith)
+      end
+      spider.determine_paths
     end
   end
+end
+
+class Wraith::Spider
+
+  def initialize(wraith)
+    @wraith = wraith
+    @paths = {}
+  end
+
+  def determine_paths
+    spider
+    write_file
+  end
+
+  private
+
+  def write_file
+    File.open(@wraith.spider_file, 'w+') { |file| file.write(@paths) }
+  end
+  
+  def add_path(path)
+    @paths[path == '/' ? 'home' : path.gsub('/', '__').chomp('__').downcase] = path.downcase
+  end
 
   def spider
-    if File.exist?(wraith.spider_file)
-      check_file
-    else
-      puts 'creating new spider file'
-      spider_base_domain
-    end
+
   end
 
-  def check_file
-    if (Time.now - File.ctime(wraith.spider_file)) / (24 * 3600) < wraith.spider_days[0]
+end
+
+class Wraith::Crawler < Wraith::Spider
+
+  EXT = %w(flv swf png jpg gif asx zip rar tar 7z \
+           gz jar js css dtd xsd ico raw mp3 mp4 \
+           wav wmv ape aac ac3 wma aiff mpg mpeg \
+           avi mov ogg mkv mka asx asf mp2 m1v \
+           m3u f4v pdf doc xls ppt pps bin exe rss xml)
+
+  def spider
+    if File.exist?(@wraith.spider_file) && modified_since(@wraith.spider_file, @wraith.spider_days[0])
       puts 'using existing spider file'
     else
       puts 'creating new spider file'
-      spider_base_domain
+      spider_list = []
+      Anemone.crawl(@wraith.base_domain) do |anemone|
+        anemone.skip_links_like(/\.#{EXT.join('|')}$/)
+        # Add user specified skips
+        anemone.skip_links_like(@wraith.spider_skips)
+        anemone.on_every_page { |page| add_path(page.url.path) }
+      end
     end
   end
 
-  def spider_base_domain
-    @spider_list = []
-    crawl_url = wraith.base_domain
-    ext = %w(flv swf png jpg gif asx zip rar tar 7z \
-             gz jar js css dtd xsd ico raw mp3 mp4 \
-             wav wmv ape aac ac3 wma aiff mpg mpeg \
-             avi mov ogg mkv mka asx asf mp2 m1v \
-             m3u f4v pdf doc xls ppt pps bin exe rss xml)
-
-    Anemone.crawl(crawl_url) do |anemone|
-      anemone.skip_links_like(/\.#{ext.join('|')}$/)
-      anemone.on_every_page { |page| @spider_list << page.url.path }
-    end
-    create_spider_file
+  def modified_since(file, since)
+    (Time.now - File.ctime(file)) / (24 * 3600) < since
   end
 
-  def create_spider_file
-    @i = 0
-    spider = Hash.new { |h, k| h[k] = [] }
-    while @i < @spider_list.length
-      lab = @spider_list[@i].to_s.split('/').last
-      lab = 'home' if @spider_list[@i] == '/'
-      spider[lab] = @spider_list[@i]
-      @i += 1
+end
+
+class Wraith::Sitemap < Wraith::Spider
+
+  def spider
+    unless @wraith.sitemap.nil?
+      puts "reading sitemap.xml from #{@wraith.sitemap}"
+      if @wraith.sitemap =~ URI::regexp
+        sitemap = Nokogiri::XML(open(@wraith.sitemap))
+      else
+        sitemap = Nokogiri::XML(File.open(@wraith.sitemap))
+      end
+      urls = {}
+      sitemap.css('loc').each do |loc|
+        path = loc.content
+        # Allow use of either domain in the sitemap.xml
+        @wraith.domains.each do |k, v|
+          path.sub!(v, '')
+        end
+        if @wraith.spider_skips.nil? || @wraith.spider_skips.none? { |regex| regex.match(path) }
+          add_path(path)
+        end
+      end
     end
-    File.open(wraith.spider_file, 'w+') { |file| file.write(spider) }
   end
+
 end
+

data/lib/wraith/version.rb

@@ -1,3 +1,3 @@
 module Wraith
-  VERSION = '1.2.2'
+  VERSION = '1.3.0'
 end

data/lib/wraith/wraith.rb

@@ -47,6 +47,14 @@ class Wraith::Wraith
     @config['spider_days']
   end
 
+  def sitemap
+    @config['sitemap']
+  end
+
+  def spider_skips
+    @config['spider_skips']
+  end
+
   def paths
     @config['paths']
   end
@@ -59,13 +67,21 @@ class Wraith::Wraith
     @config['fuzz']
   end
 
+  def mode
+    if ['diffs_only', 'diffs_first', 'alphanumeric'].include?(@config['mode'])
+      @config['mode']
+    else
+      'alphanumeric'
+    end
+  end
+
   def capture_page_image(browser, url, width, file_name)
     puts `"#{browser}" #{@config['phantomjs_options']} "#{snap_file}" "#{url}" "#{width}" "#{file_name}"`
   end
 
   def self.crop_images(crop, height)
     # For compatibility with windows file structures switch commenting on the following 2 lines
-    puts `convert #{crop} -background none -extent 0x#{height} #{crop}`
+    `convert #{crop} -background none -extent 0x#{height} #{crop}`
     # puts `convert #{crop.gsub('/', '\\')} -background none -extent 0x#{height} #{crop.gsub('/', '\\')}`
   end
 
@@ -75,7 +91,7 @@ class Wraith::Wraith
 
   def set_image_width(image, width)
     # For compatibility with windows file structures switch commenting on the following 2 lines
-    puts `convert #{image} -background none -extent #{width}x0 #{image}`
+    `convert #{image} -background none -extent #{width}x0 #{image}`
     # puts `convert #{image.gsub('/', '\\')} -background none -extent #{width}x0 #{image.gsub('/', '\\')}`
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wraith
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Dave Blooman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-26 00:00:00.000000000 Z
+date: 2014-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry