blinkr 0.2.9 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39d18a782d87077bd68ed1c41307c385f082b4fc
-  data.tar.gz: 87a369d2d62eb1ac451b1500222469e20a454357
+  metadata.gz: 2b54b1fe0983042827e5f00868dcf9e65ef5c2f2
+  data.tar.gz: f90a1abe798a9e3cfaebe7544f0e618b4379116d
 SHA512:
-  metadata.gz: ebe1ff145b684dc28e397c5f2cf87ca392d52535c807d8be53452746c564e1e323eaa71bb23a67a1c1c07ddac2cc22df0080b124d74828183defb1af6ae7d7d2
-  data.tar.gz: 0a20c4ee727298906e7f46b82a622cec2a83f2ce67e529cc29ed9f07df6c35c1954aee150554aba4e60435392dccf2873774eb469c0589d11d776b54b4b1fd65
+  metadata.gz: bdbee3d503b16a6354a29171b775ff5bfb1667203529b158cc2e26ce3f58749402fea5098dd1a349387050d76e8454861dbe8919d8473cdf57e7d6205b6503c7
+  data.tar.gz: bfc4cc60420fed8c53c20bcec5b224fb341a3e3a3d53a054aaa2e4cdff2575d5f5b0f62b391c459b4df89cae5875e7b1353ab23723ac99138cd70d13aba7d460

data/Gemfile

@@ -1,4 +1,7 @@
 source 'https://rubygems.org'
 
+platform :jruby do
+  gem 'manticore'
+end
 # Specify your gem's dependencies in blinkr.gemspec
 gemspec

data/bin/blinkr

@@ -22,6 +22,7 @@ OptionParser.new do |opts|
   end
   opts.on("-s", "--single-url URL", "test a single URL, outputting the response to the console") do |opt|
     options[:single_url] = opt
+    options[:base_url] = opt
   end
 end.parse!
 

data/blinkr.gemspec

@@ -4,26 +4,34 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'blinkr/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "blinkr"
+  spec.name = 'blinkr'
   spec.version       = Blinkr::VERSION
-  spec.authors       = ["Pete Muir"]
-  spec.email         = ["pmuir@bleepbleep.org.uk"]
+  spec.authors = ['Pete Muir', 'Jason Porter']
+  spec.email = %w(pmuir@bleepbleep.org.uk lightguard.jp@gmail.com)
   spec.summary       = %q{A simple broken link checker}
   spec.description   = %q{A broken page and link checker for websites. Optionally uses phantomjs to render pages to check resource loading, links created by JS, and report any JS page load errors.}
-  spec.homepage      = "https://github.com/pmuir/blinkr"
-  spec.license       = "Apache-2.0"
+  spec.homepage = 'https://github.com/pmuir/blinkr'
+  spec.license = 'Apache-2.0'
 
   spec.files         = `git ls-files`.split($/)
+  spec.files         -= ['.gitignore', '.ruby-version', '.ruby-gemset']
+
+
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '~> 2.0'
+  spec.required_ruby_version = '~> 1.9'
 
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency 'bundler', '~> 1.5'
   spec.add_development_dependency 'rake', '~> 10.3'
   spec.add_dependency 'nokogiri', '~> 1.5'
   spec.add_dependency 'typhoeus', '~> 0.7'
   spec.add_dependency 'slim', '~> 3.0'
   spec.add_dependency 'parallel', '~> 1.3'
+
+  if defined? JRUBY_VERSION
+    spec.platform = 'java'
+    spec.add_dependency 'manticore', '~> 0.4'
+  end
 end

data/lib/blinkr.rb

@@ -2,12 +2,13 @@ require 'blinkr/version'
 require 'blinkr/engine'
 require 'blinkr/report'
 require 'blinkr/config'
+require 'blinkr/error'
 require 'blinkr/typhoeus_wrapper'
 require 'yaml'
 
 module Blinkr
   def self.run(base_url, config = 'blinkr.yaml', single, verbose, vverbose)
-    args = { :base_url => base_url, :verbose => verbose, :vverbose => vverbose }
+    args = {:base_url => base_url, :verbose => verbose, :vverbose => vverbose}
     if !config.nil? && File.exists?(config)
       config = Blinkr::Config.read config, args
     else

data/lib/blinkr/cache.rb

@@ -9,7 +9,11 @@ module Blinkr
     end
 
     def set(request, response)
-      @memory[request] = response unless response.timed_out?
+      if request.is_a? String # HACK for caching resource and js errors
+        @memory[request] = response
+      else
+        @memory[request] = response unless response.timed_out?
+      end
     end
 
     def size

data/lib/blinkr/config.rb

@@ -8,7 +8,8 @@ module Blinkr
       Config.new(YAML.load_file(file).merge(args).merge({ :config_file => file }))
     end
 
-    DEFAULTS = {:skips => [], :ignores => [], :max_retrys => 3, :browser => 'typhoeus', :viewport => 1200, :phantomjs_threads => 8, :report => 'blinkr.html'}
+    DEFAULTS = {:skips => [], :ignores => [], :max_retrys => 3, :browser => 'typhoeus',
+                :viewport => 1200, :phantomjs_threads => 8, :report => 'blinkr.html'}
 
     def initialize(hash={})
       super(DEFAULTS.merge(hash))

data/lib/blinkr/engine.rb

@@ -9,6 +9,8 @@ require 'blinkr/extensions/javascript'
 require 'blinkr/extensions/resources'
 require 'blinkr/extensions/pipeline'
 require 'json'
+require 'pathname'
+require 'fileutils'
 require 'ostruct'
 
 # Monkeypatch OpenStruct
@@ -27,7 +29,7 @@ module Blinkr
     include HttpUtils
     include Sitemap
 
-    def initialize config
+    def initialize(config)
       @config = config.validate
       @extensions = []
       load_pipeline
@@ -35,15 +37,25 @@ module Blinkr
 
     def run
       context = OpenStruct.new({:pages => {}})
-      browser = TyphoeusWrapper.new(@config, context)
+      if defined?(JRUBY_VERSION) && @config.browser == 'manticore'
+        require 'blinkr/manticore_wrapper'
+        bulk_browser = browser = ManticoreWrapper.new(@config, context)
+      else
+        bulk_browser = browser = TyphoeusWrapper.new(@config, context)
+      end
       browser = PhantomJSWrapper.new(@config, context) if @config.browser == 'phantomjs'
       page_count = 0
-      browser.process_all(sitemap_locations, @config.max_page_retrys) do |response, resource_errors, javascript_errors|
+      urls = sitemap_locations.uniq
+      puts "Fetching #{urls.size} pages from sitemap"
+      browser.process_all(urls, @config.max_page_retrys) do |response, resource_errors, javascript_errors|
+        url = response.request.base_url
         if response.success?
-          url = response.request.base_url
           puts "Loaded page #{url}" if @config.verbose
           body = Nokogiri::HTML(response.body)
-          page = OpenStruct.new({ :response => response, :body => body, :errors => ErrorArray.new(@config), :resource_errors => resource_errors || [], :javascript_errors => javascript_errors || [] })
+          page = OpenStruct.new({:response => response, :body => body.freeze,
+                                 :errors => ErrorArray.new(@config),
+                                 :resource_errors => resource_errors || [],
+                                 :javascript_errors => javascript_errors || []})
           context.pages[url] = page
           collect page
           page_count += 1
@@ -51,25 +63,26 @@ module Blinkr
           puts "#{response.code} #{response.status_message} Unable to load page #{url} #{'(' + response.return_message + ')' unless response.return_message.nil?}"
         end
       end
-      typhoeus.hydra.run if @config.browser == 'typhoeus'
-      analyze context, typhoeus
-      puts "Loaded #{page_count} pages using #{browser.name}. Performed #{typhoeus.count} requests using typhoeus."
-      context.pages.reject! { |url, page| page.errors.empty? }
+      puts 'Executing Typhoeus::Hydra.run, this could take awhile' if @config.browser == 'typhoeus'
+      # browser.hydra.run if @config.browser == 'typhoeus'
+      puts "Loaded #{page_count} pages using #{browser.name}."
+      puts 'Analyzing pages'
+      analyze context, bulk_browser
+      context.pages.reject! { |_, page| page.errors.empty? }
+
       unless @config.export.nil?
-        File.open(@config.export, 'w') do |file| 
-          file.write(context.to_json)
-        end
+        FileUtils.mkdir_p Pathname.new(@config.report).parent
       end
       Blinkr::Report.new(context, self, @config).render
     end
 
-    def append context
-      exec :append, context
+    def append(context)
+      execute :append, context
     end
-      
-    def transform page, error, &block
+
+    def transform(page, error, &block)
       default = yield
-      result = exec(:transform, page, error, default)
+      result = execute(:transform, page, error, default)
       if result.empty?
         default
       else
@@ -77,33 +90,33 @@ module Blinkr
       end
     end
 
-    def analyze context, typhoeus
-      exec :analyze, context, typhoeus
+    def analyze(context, typhoeus)
+      execute :analyze, context, typhoeus
     end
 
-    def collect page
-      exec :collect, page
+    def collect(page)
+      execute :collect, page
     end
 
     private
 
     class ErrorArray < Array
 
-      def initialize config
+      def initialize(config)
         @config = config
       end
 
-      def << error
-        unless @config.ignored?(error.url, error.code, error.message)
-          super
-        else
+      def <<(error)
+        if @config.ignored?(error.url, error.code, error.message)
           self
+        else
+          super
         end
       end
 
     end
 
-    def extension ext
+    def extension(ext)
       @extensions << ext
     end
 
@@ -113,7 +126,7 @@ module Blinkr
       extension Blinkr::Extensions::Resources.new @config
     end
 
-    def exec method, *args
+    def execute(method, *args)
       result = []
       @extensions.each do |e|
         result << e.send(method, *args) if e.respond_to? method
@@ -122,24 +135,24 @@ module Blinkr
     end
 
     def load_pipeline
-      unless @config.pipeline.nil?
+      if @config.pipeline.nil?
+        puts 'Loaded default pipeline' if @config.verbose
+        default_pipeline
+      else
         pipeline_file = File.join(File.dirname(@config.config_file), @config.pipeline)
-        if  File.exists?( pipeline_file )
-          p = eval(File.read( pipeline_file ), nil, pipeline_file, 1).load @config
+        if File.exists?(pipeline_file)
+          p = eval(File.read(pipeline_file), nil, pipeline_file, 1).load @config
           p.extensions.each do |e|
-            extension( e )
+            extension(e)
           end
           if @config.verbose
             puts "Loaded custom pipeline from #{@config.pipeline}"
-            pipeline = @extensions.inject{|memo, v| "#{memo}, #{v}" }
+            pipeline = @extensions.inject { |memo, v| "#{memo}, #{v}" }
             puts "Pipeline: #{pipeline}"
           end
         else
           raise "Cannot find pipeline file #{pipeline_file}"
         end
-      else
-        puts "Loaded default pipeline" if @config.verbose
-        default_pipeline
       end
     end
 

data/lib/blinkr/error.rb

@@ -0,0 +1,30 @@
+module Blinkr
+  SEVERITY = [:success, :info, :warning, :danger]
+  class Error
+    attr_reader :severity, :category, :type, :title, :message, :snippet, :icon, :code, :detail, :url
+
+    def initialize (opts = {})
+      raise TypeError 'severity must be a string or symbol' unless opts[:severity].is_a?(String) || opts[:severity].is_a?(Symbol)
+      raise 'severity not a recognized value' unless SEVERITY.include? opts[:severity].to_sym
+
+      @severity = opts[:severity]
+      @category = opts[:category]
+      @type = opts[:type]
+      @title = opts[:title]
+      @message = opts[:message]
+      @snippet = opts[:snippet]
+      @icon = opts[:icon]
+      @code = opts[:code]
+      @url = opts[:url]
+      @detail = opts[:detail]
+    end
+
+    def to_json(*args)
+      content = {}
+      instance_variables.each do |v|
+        content[v.to_s[1..-1]] = instance_variable_get v
+      end
+      content.to_json
+    end
+  end
+end

data/lib/blinkr/extensions/a_title.rb

@@ -1,3 +1,5 @@
+require 'blinkr/error'
+
 module Blinkr
   module Extensions
     class ATitle
@@ -8,7 +10,11 @@ module Blinkr
 
       def collect page
         page.body.css('a:not([title])').each do |a|
-          page.errors << OpenStruct.new({ :severity => 'info', :category => 'SEO', :type => '<a title=""> missing',  :title => "#{a['href']} (line #{a.line})", :message => '<a title=""> missing', :snippet => a.to_s, :icon => 'fa-info' })
+          page.errors << Blinkr::Error.new({:severity => 'info', :category => 'SEO',
+                                            :type => '<a title=""> missing',
+                                            :title => "#{a['href']} (line #{a.line})",
+                                            :message => '<a title=""> missing',
+                                            :snippet => a.to_s, :icon => 'fa-info'})
         end
       end
 

data/lib/blinkr/extensions/empty_a_href.rb

@@ -1,3 +1,5 @@
+require 'blinkr/error'
+
 module Blinkr
   module Extensions
     class EmptyAHref
@@ -9,7 +11,11 @@ module Blinkr
       def collect page
         page.body.css('a[href]').each do |a|
           if a['href'].empty?
-            page.errors << OpenStruct.new({ :severity => 'info', :category => 'HTML Compatibility/Correctness', :type => '<a href=""> empty',  :title => %Q{<a href=""> empty (line #{a.line})}, :message => %Q{<a href=""> empty}, :snippet => a.to_s, :icon => 'fa-info' })
+            page.errors << Blinkr::Error.new({:severity => 'info', :category => 'HTML Compatibility/Correctness',
+                                              :type => '<a href=""> empty',
+                                              :title => %Q{<a href=""> empty (line #{a.line})},
+                                              :message => %Q{<a href=""> empty}, :snippet => a.to_s,
+                                              :icon => 'fa-info'})
           end
         end
       end

data/lib/blinkr/extensions/img_alt.rb

@@ -1,3 +1,5 @@
+require 'blinkr/error'
+
 module Blinkr
   module Extensions
     class ImgAlt
@@ -8,7 +10,11 @@ module Blinkr
 
       def collect page
         page.body.css('img:not([alt])').each do |img|
-          page.errors << OpenStruct.new({ :severity => 'warning', :category => 'SEO', :type => '<img alt=""> missing',  :title => "#{img['src']} (line #{img.line})", :message => '<img alt=""> missing', :snippet => img.to_s, :icon => 'fa-info' })
+          page.errors << OpenStruct.new({:severity => 'warning', :category => 'SEO',
+                                         :type => '<img alt=""> missing',
+                                         :title => "#{img['src']} (line #{img.line})",
+                                         :message => '<img alt=""> missing', :snippet => img.to_s,
+                                         :icon => 'fa-info'})
         end
       end
 

data/lib/blinkr/extensions/inline_css.rb

@@ -1,3 +1,5 @@
+require 'blinkr/error'
+
 module Blinkr
   module Extensions
     class InlineCss
@@ -9,9 +11,18 @@ module Blinkr
       def collect page
         page.body.css('[style]').each do |elm|
           if elm['style'] == ""
-            page.errors << OpenStruct.new({ :severity => 'info', :category => 'HTML Compatibility/Correctness', :type => 'style attribute is empty',  :title => %Q{"#{elm['style']}" (line #{elm.line})}, :message => 'style attribute is empty', :snippet => elm.to_s, :icon => 'fa-info' })
+            page.errors << Blinkr::Error.new({:severity => 'info', :category => 'HTML Compatibility/Correctness',
+                                              :type => 'style attribute is empty',
+                                              :title => %Q{"#{elm['style']}" (line #{elm.line})},
+                                              :message => 'style attribute is empty', :snippet => elm.to_s,
+                                              :icon => 'fa-info'})
           else
-            page.errors << OpenStruct.new({ :severity => 'info', :category => 'HTML Compatibility/Correctness', :type => 'Inline CSS detected',  :title => %Q{"#{elm['style']}" (line #{elm.line})}, :message => 'inline style', :snippet => elm.to_s, :icon => 'fa-info' })
+            page.errors << Blinkr::Error.new({:severity => 'info',
+                                              :category => 'HTML Compatibility/Correctness',
+                                              :type => 'Inline CSS detected',
+                                              :title => %Q{"#{elm['style']}" (line #{elm.line})},
+                                              :message => 'inline style', :snippet => elm.to_s,
+                                              :icon => 'fa-info'})
           end
         end
       end

data/lib/blinkr/extensions/javascript.rb

@@ -1,3 +1,5 @@
+require 'blinkr/error'
+
 module Blinkr
   module Extensions
     class JavaScript
@@ -8,7 +10,9 @@ module Blinkr
 
       def collect page
         page.javascript_errors.each do |error|
-          page.errors << OpenStruct.new({ :severity => 'danger', :category => 'JavaScript', :type => 'JavaScript error', :title => error['msg'], :snippet => error['trace'], :icon => 'fa-gears' })
+          page.errors << Blinkr::Error.new(:severity => 'danger', :category => 'JavaScript',
+                                           :type => 'JavaScript error', :title => error['msg'],
+                                           :snippet => error['trace'], :icon => 'fa-gears')
         end
       end
 

data/lib/blinkr/extensions/links.rb

@@ -1,50 +1,94 @@
+require 'uri'
+require 'blinkr/error'
 require 'blinkr/http_utils'
 
 module Blinkr
   module Extensions
     class Links
-      include HttpUtils
+      include Blinkr::HttpUtils
 
-      def initialize config
+      def initialize(config)
         @config = config
         @links = {}
       end
 
-      def collect page
+      def collect(page)
         page.body.css('a[href]').each do |a|
           attr = a.attribute('href')
           src = page.response.effective_url
           url = attr.value
-          url = sanitize url, src
-          unless url.nil? || @config.skipped?(url)
-            @links[url] ||= []
-            @links[url] << {:page => page, :line => attr.line, :snippet => attr.parent.to_s}
+          unless @config.skipped?(url)
+            url = sanitize url, src
+            unless url.nil?
+              @links[url] ||= []
+              @links[url] << {:page => page, :line => attr.line, :snippet => attr.parent.to_s}
+            end
           end
         end
       end
 
-      def analyze context, typhoeus
-        puts "----------------------" if @config.verbose
-        puts " #{@links.length} links to check " if @config.verbose
-        puts "----------------------" if @config.verbose
-        @links.each do |url, metadata|
-          typhoeus.process(url, @config.max_retrys) do |resp|
-            puts "Loaded #{url} via typhoeus #{'(cached)' if resp.cached?}" if @config.verbose
-            unless resp.success? || resp.code == 200
+      def analyze(context, browser)
+        puts '----------------------'
+        puts " #{@links.length} links to check "
+        puts '----------------------'
+        external_links = @links.reject { |k| k.start_with? @config.base_url }
+        processed = 0
+        # Find the internal links
+        @links.select{|k| k.start_with? @config.base_url}.each do |url, locations|
+          link = URI.parse(url)
+          link.fragment = nil
+          link.query = nil
+          unless context.pages.keys.include?(link.to_s) || context.pages.keys.include?((link.to_s + '/'))
+            locations.each do |location|
+              location[:page].errors << Blinkr::Error.new({:severity => :warning,
+                                                           :category => 'Resource missing from sitemap',
+                                                           :type => '<a href=""> target missing from sitemap',
+                                                           :url => url, :title => "#{url} (line #{location[:line]})",
+                                                           :code => nil,
+                                                           :message => 'Missing from sitemap',
+                                                           :detail => 'Checked with Typheous',
+                                                           :snippet => location[:snippet],
+                                                           :icon => 'fa-bookmark-o'
+                                                          })
+              # It wasn't in the sitemap, so we'll add it to the "external_links" to still be checked
+              external_links[url] = locations
+            end
+          end
+        end
+        external_links.each do |url, metadata|
+          # if link start_with? @config.base_url check to see if it's in the sitemap.xml
+          browser.process(url, @config.max_retrys, :method => :get, :followlocation => true) do |resp|
+            puts "Loaded #{url} via #{browser.name} #{'(cached)' if resp.cached?}" if @config.verbose
+            if resp.code.to_i < 200 || resp.code.to_i > 300
+              response = resp
+
               metadata.each do |src|
-                code = resp.code.to_i unless resp.code.nil? || resp.code == 0
-                if resp.status_message.nil?
-                  message = resp.return_message
+                detail = nil
+                if response.status_message.nil?
+                  message = response.return_message
                 else
-                  message = resp.status_message
-                  detail = resp.return_message unless resp.return_message == "No error"
+                  message = response.status_message
+                  detail = response.return_message unless resp.return_message == 'No error'
+                end
+
+                severity = :danger
+                if response.code.to_i >= 300 && response.code.to_i < 400
+                  severity = :warning
                 end
-                src[:page].errors << OpenStruct.new({ :severity => 'danger', :category => 'Resources missing', :type => '<a href=""> target cannot be loaded', :url => url, :title => "#{url} (line #{src[:line]})", :code => code, :message => message, :detail => detail, :snippet => src[:snippet], :icon => 'fa-bookmark-o' })
+                src[:page].errors << Blinkr::Error.new({:severity => severity,
+                :category => 'Resources missing',
+                :type => '<a href=""> target cannot be loaded',
+                :url => url, :title => "#{url} (line #{src[:line]})",
+                :code => response.code.to_i, :message => message,
+                :detail => detail, :snippet => src[:snippet],
+                :icon => 'fa-bookmark-o'}) unless response.success?
               end
             end
+            processed += 1
+            puts "Processed #{processed} of #{external_links.size}" if @config.verbose
           end
         end
-        typhoeus.hydra.run
+        browser.hydra.run if browser.is_a? Blinkr::TyphoeusWrapper
       end
 
     end