blinkr 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8e918082f460b113ad7517dc20f4838c1b3052d
-  data.tar.gz: 523df75b0eb689beb7ce691371c4e0fc8eb422c0
+  metadata.gz: 8f345eab68efa8b4884096359a7bf64c0286687f
+  data.tar.gz: 2366eee2265b2033197e68341ded32dad7c68195
 SHA512:
-  metadata.gz: aac832c6fb06a704a95d91249b64d278c694c6b990b30954ac1daacb8f93f721595f2fa9ae2daecc474eb5bd72415b315fc509a528dbb019b7a339326e9aa679
-  data.tar.gz: ad1d2fc269e08db56f02eee1b0f4d101435d026c4de6357e750ea96f3e2dd8888aa332f686fa8ca453f7635a3e7bfb7acb9e82fbb4fba502ce88d3e61ea2ab7b
+  metadata.gz: 455e68a0cf36690b202beced966708b49aeca94d77ccbd46d01cff174731630175eb6e1e17b7437df9fedb5266ad9f4ba2fe1b20aa7ac7f73f98c24318fc40c0
+  data.tar.gz: 538f12426478bf7cc97208d3c244016ec6176c7d932a399c87555f13e5d69dbe56a065a620b929e0b47d37d050a51cd597724ab49297ca8c4a69ab81fe79ef17

data/README.md

@@ -87,6 +87,8 @@ ignore_fragments: true
 # Control the number of threads used to run phantomjs. By default 8.
 phantomjs_threads: 8
 
+# Export the report to phantomjs
+
 ````
 
 You can specify a custom config file on the command link:
@@ -114,6 +116,70 @@ mode (this is very verbose, so normally used with `-s`):
 blinkr -c my_blinkr.yaml -s http://www.acme.com/corp -v
 ````
 
+## Extending Blinkr
+
+Blinkr is based around a pipeline. Issues with the pages are *collected*, 
+*analysed*, and then passed to the report for *transformation* and rendering.
+Additional sections may *appended* to the report.
+
+To add extensions to blinkr, you need to define a custom pipeline. The pipeline
+is defined in a ruby file (e.g. `blinkr.rb`)
+
+````
+require 'acme/spellcheck'
+
+Blinkr::Extensions::Pipeline.new do |config|
+  # define the default extensions
+  extension Blinkr::Extensions::Links.new config
+  extension Blinkr::Extensions::JavaScript.new config
+  extension Blinkr::Extensions::Resources.new config
+
+  # define custom extensions
+  extension ACME::Extensions::SpellCheck.new config
+end
+````
+
+NOTE: You must add the default extensions to a custom pipeline, for them to be
+executed.
+
+The pipeline is defined in `blinkr.yaml`:
+
+````
+# Use a custom pipeline
+pipeline: blinkr.rb
+````
+
+An extension is just a standard Ruby class. It should declare an 
+`initialize(config)` method, and may declare one or more of:
+
+* `collect(page)`
+* `analyze(context, typhoeus)`
+* `transform(page, error, default_html)`
+* `append(context)`
+
+Each method is called as the pipeline progresses. Arguments passed are:
+
+* `page` - a object containing the tyhpoeus `response`, the page `body` (as a
+  Nokogiri HTML document), an array of `errors` for the page, any 
+  `resource_errors` which ocurred when the page was loaded, and any 
+  `javascript_errors` which ocurred when the page was loaded
+* `context` - a map of `url` => `page`s which are being analysed. After the
+  analyze phase, and before the transform phase, any pages with no errors
+  are removed from the context
+* `typhoeus` - a wrapper around typhoeus, defining a `process` method and 
+  a `process_all` method, both of which take a `url` and a `retry` limit, and
+  accept a block to execute when a response is returned.
+* `error` - an individual error, consisting of a `type`, a `url`, a `title`, a
+  `code`, a `message`, a `detail`, a `snippet` and an fontawesome `icon` class
+* `default_html` - the default HTML used to display the error
+
+`transform` should return the HTML used to display the error. `append` should 
+return any HTML to be appended to the report. A templating language, such as
+slim or haml may be used to generate the HTML.
+
+The build extensions, in lib/blinkr/extensions are good examples of how 
+extensions can perform broken link analysis, or collect and format resource
+loading and javascript execution errors.
 
 ## Contributing
 

data/lib/blinkr.rb

@@ -1,21 +1,25 @@
 require 'blinkr/version'
-require 'blinkr/check'
+require 'blinkr/engine'
 require 'blinkr/report'
+require 'blinkr/config'
+require 'blinkr/typhoeus_wrapper'
 require 'yaml'
 
 module Blinkr
   def self.run(base_url, config = 'blinkr.yaml', single, verbose, vverbose)
+    args = { :base_url => base_url, :verbose => verbose, :vverbose => vverbose }
     if !config.nil? && File.exists?(config)
-      config = YAML.load_file(config) 
+      config = Blinkr::Config.read config, args
     else
-      config = {}
+      config = Blinkr::Config.new args
     end
-    blinkr = Blinkr::Check.new(base_url || config['base_url'], sitemap: config['sitemap'], skips: config['skips'], max_retrys: config['max_retrys'], max_page_retrys: config['max_page_retrys'], verbose: verbose, vverbose: vverbose, browser: config['browser'], viewport: config['viewport'], ignore_fragments: config['ignore_fragments'], ignores: config['ignores'], phantomjs_threads: config['phantomjs_threads'])
+    
     if single.nil?
-      Blinkr::Report.render(blinkr.check, config['report'])
+      Blinkr::Engine.new(config).run
     else
-      blinkr.single single
+      Blinkr::TyphoeusWrapper.new(config).debug(single)
     end
   end
 
 end
+

data/lib/blinkr/config.rb

@@ -0,0 +1,46 @@
+require 'ostruct'
+
+module Blinkr
+  class Config < OpenStruct
+
+    def self.read file, args
+      raise "Cannot read #{file}" unless File.exists? file
+      Config.new(YAML.load_file(file).merge(args).merge({ :config_file => file }))
+    end
+
+    DEFAULTS = {:skips => [], :ignores => [], :max_retrys => 3, :browser => 'typhoeus', :viewport => 1200, :phantomjs_threads => 8, :report => 'blinkr.html'}
+
+    def initialize(hash={})
+      super(DEFAULTS.merge(hash))
+    end
+
+    def validate
+      ignores.each {|ignore| raise "An ignore must be a hash" unless ignore.is_a? Hash}
+      raise "Must specify base_url" if base_url.nil?
+      raise "Must specify sitemap" if sitemap.nil?
+      self
+    end
+
+    def sitemap
+      if super.nil?
+        URI.join(base_url, 'sitemap.xml').to_s 
+      else
+        super
+      end
+    end
+
+    def max_page_retrys
+      @max_page_retrys || @max_retrys
+    end
+
+    def ignored? url, code, message
+      ignores.any? { |ignore| ( !url.nil? && ignore.has_key?('url') ? !ignore['url'].match(url).nil? : true )  && ( !code.nil? && ignore.has_key?('code') ? ignore['code'] == code : true ) && ( !message.nil? && ignore.has_key?('message') ? !ignore['message'].match(message).nil? : true ) }
+    end
+
+    def skipped? url
+      skips.any? { |regex| regex.match(url) }
+    end
+
+  end
+end
+

data/lib/blinkr/engine.rb

@@ -0,0 +1,142 @@
+require 'nokogiri'
+require 'blinkr/phantomjs_wrapper'
+require 'blinkr/typhoeus_wrapper'
+require 'blinkr/http_utils'
+require 'blinkr/sitemap'
+require 'blinkr/report'
+require 'blinkr/extensions/links'
+require 'blinkr/extensions/javascript'
+require 'blinkr/extensions/resources'
+require 'blinkr/extensions/pipeline'
+require 'json'
+require 'ostruct'
+
+# Monkeypatch OpenStruct
+class OpenStruct
+
+  EXCEPT = [:response, :body, :resource_errors, :javascript_errors]
+  
+  def to_json(*args)
+    to_h.delete_if{ |k, v| EXCEPT.include?(k) }.to_json(*args)
+  end
+
+end
+
+module Blinkr
+  class Engine 
+    include HttpUtils
+    include Sitemap
+
+    def initialize config
+      @config = config.validate
+      @extensions = []
+      load_pipeline
+    end
+
+    def run
+      context = OpenStruct.new({:pages => {}})
+      typhoeus, browser = TyphoeusWrapper.new(@config, context)
+      browser = PhantomJSWrapper.new(@config, context) if @config.browser == 'phantomjs'
+      page_count = 0
+      browser.process_all(sitemap_locations, @config.max_page_retrys) do |response, resource_errors, javascript_errors|
+        if response.success?
+          url = response.request.base_url
+          puts "Loaded page #{url}" if @config.verbose
+          body = Nokogiri::HTML(response.body)
+          page = OpenStruct.new({ :response => response, :body => body, :errors => ErrorArray.new(@config), :resource_errors => resource_errors || [], :javascript_errors => javascript_errors || [] })
+          context.pages[url] = page
+          collect page
+          page_count += 1
+        else
+          puts "#{respones.code} #{response.status_message} Unable to load page #{url} #{'(' + response.return_message + ')' unless response.return_message.nil?}"
+        end
+      end
+      typhoeus.hydra.run if @config.browser == 'typhoeus'
+      analyze context, typhoeus
+      puts "Loaded #{page_count} pages using #{browser.name}. Performed #{typhoeus.count} requests using typhoeus."
+      context.pages.reject! { |url, page| page.errors.empty? }
+      unless @config.export.nil?
+        File.open(@config.export, 'w') do |file| 
+          file.write(context.to_json)
+        end
+      end
+      Blinkr::Report.new(context, self, @config).render
+    end
+
+    def append context
+      exec :append, context
+    end
+      
+    def transform page, error, &block
+      default = yield
+      result = exec(:transform, page, error, default)
+      if result.empty?
+        default
+      else
+        result.join
+      end
+    end
+
+    def analyze context, typhoeus
+      exec :analyze, context, typhoeus
+    end
+
+    def collect page
+      exec :collect, page
+    end
+
+    private
+
+    class ErrorArray < Array
+
+      def initialize config
+        @config = config
+      end
+
+      def << error
+        unless @config.ignored?(error.url, error.code, error.message)
+          super
+        else
+          self
+        end
+      end
+
+    end
+
+    def extension ext
+      @extensions << ext
+    end
+
+    def default_pipeline
+      extension Blinkr::Extensions::Links.new @config
+      extension Blinkr::Extensions::JavaScript.new @config
+      extension Blinkr::Extensions::Resources.new @config
+    end
+
+    def exec method, *args
+      result = []
+      @extensions.each do |e|
+        result << e.send(method, *args) if e.respond_to? method
+      end
+      result
+    end
+
+    def load_pipeline
+      unless @config.pipeline.nil?
+        pipeline_file = File.join(File.dirname(@config.config_file), @config.pipeline)
+        if  File.exists?( pipeline_file )
+          p = eval(File.read( pipeline_file ), nil, pipeline_file, 1).load @config
+          p.extensions.each do |e|
+            extension( e )
+          end
+        else
+          raise "Cannot find pipeline file #{pipeline_file}"
+        end
+      else
+        default_pipeline
+      end
+    end
+
+  end
+end
+

data/lib/blinkr/extensions/a_title.rb

@@ -0,0 +1,17 @@
+module Blinkr
+  module Extensions
+    class ATitle
+
+      def initialize config
+        @config = config
+      end
+
+      def collect page
+        page.body.css('a:not([title])').each do |a|
+          page.errors << OpenStruct.new({ :severity => 'info', :category => 'SEO', :type => '<a title=""> missing',  :title => "#{a['href']} (line #{a.line})", :message => '<a title=""> missing', :snippet => a.to_s, :icon => 'fa-info' })
+        end
+      end
+
+    end
+  end
+end

data/lib/blinkr/extensions/empty_a_href.rb

@@ -0,0 +1,19 @@
+module Blinkr
+  module Extensions
+    class EmptyAHref
+
+      def initialize config
+        @config = config
+      end
+
+      def collect page
+        page.body.css('a[href]').each do |a|
+          if a['href'].empty?
+            page.errors << OpenStruct.new({ :severity => 'info', :category => 'HTML Compatibility/Correctness', :type => '<a href=""> empty',  :title => %Q{<a href=""> empty (line #{a.line})}, :message => %Q{<a href=""> empty}, :snippet => a.to_s, :icon => 'fa-info' })
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/blinkr/extensions/img_alt.rb

@@ -0,0 +1,17 @@
+module Blinkr
+  module Extensions
+    class ImgAlt
+
+      def initialize config
+        @config = config
+      end
+
+      def collect page
+        page.body.css('img:not([alt])').each do |img|
+          page.errors << OpenStruct.new({ :severity => 'warning', :category => 'SEO', :type => '<img alt=""> missing',  :title => "#{img['src']} (line #{img.line})", :message => '<img alt=""> missing', :snippet => img.to_s, :icon => 'fa-info' })
+        end
+      end
+
+    end
+  end
+end

data/lib/blinkr/extensions/inline_css.rb

@@ -0,0 +1,21 @@
+module Blinkr
+  module Extensions
+    class InlineCss
+
+      def initialize config
+        @config = config
+      end
+
+      def collect page
+        page.body.css('[style]').each do |elm|
+          if elm['style'] == ""
+            page.errors << OpenStruct.new({ :severity => 'info', :category => 'HTML Compatibility/Correctness', :type => 'style attribute is empty',  :title => %Q{"#{elm['style']}" (line #{elm.line})}, :message => 'style attribute is empty', :snippet => elm.to_s, :icon => 'fa-info' })
+          else
+            page.errors << OpenStruct.new({ :severity => 'info', :category => 'HTML Compatibility/Correctness', :type => 'Inline CSS detected',  :title => %Q{"#{elm['style']}" (line #{elm.line})}, :message => 'inline style', :snippet => elm.to_s, :icon => 'fa-info' })
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/blinkr/extensions/javascript.rb

@@ -0,0 +1,17 @@
+module Blinkr
+  module Extensions
+    class JavaScript
+
+      def initialize config
+        @config = config
+      end
+
+      def collect page
+        page.javascript_errors.each do |error|
+          page.errors << OpenStruct.new({ :severity => 'danger', :category => 'JavaScript', :type => 'JavaScript error', :title => error.msg, :snippet => error.trace, :icon => 'fa-gears' })
+        end
+      end
+
+    end
+  end
+end

data/lib/blinkr/extensions/links.rb

@@ -0,0 +1,53 @@
+require 'blinkr/http_utils'
+
+module Blinkr
+  module Extensions
+    class Links
+      include HttpUtils
+
+      def initialize config
+        @config = config
+        @links = {}
+      end
+
+      def collect page
+        page.body.css('a[href]').each do |a|
+          attr = a.attribute('href')
+          src = page.response.effective_url
+          url = attr.value
+          url = sanitize url, src
+          unless url.nil? || @config.skipped?(url)
+            @links[url] ||= []
+            @links[url] << {:page => page, :line => attr.line, :snippet => attr.parent.to_s}
+          end
+        end
+      end
+
+      def analyze context, typhoeus
+        puts "----------------------" if @config.verbose
+        puts " #{@links.length} links to check " if @config.verbose
+        puts "----------------------" if @config.verbose
+        @links.each do |url, metadata|
+          typhoeus.process(url, @config.max_retrys) do |resp|
+            puts "Loaded #{url} via typhoeus #{'(cached)' if resp.cached?}" if @config.verbose
+            unless resp.success? || resp.code == 200
+              metadata.each do |src|
+                code = resp.code.to_i unless resp.code.nil? || resp.code == 0
+                if resp.status_message.nil?
+                  message = resp.return_message
+                else
+                  message = resp.status_message
+                  detail = resp.return_message unless resp.return_message == "No error"
+                end
+                src[:page].errors << OpenStruct.new({ :severity => 'danger', :category => 'Resources missing', :type => '<a href=""> target cannot be loaded', :url => url, :title => "#{url} (line #{src[:line]})", :code => code, :message => message, :detail => detail, :snippet => src[:snippet], :icon => 'fa-bookmark-o' })
+              end
+            end
+          end
+        end
+        typhoeus.hydra.run
+      end
+
+    end
+  end
+end
+