headhunter 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df58d59328c6942d9153f49b22765fbef1aa7c62
-  data.tar.gz: 4e7c82703bb398fbb1a08f3d8b33e80f4793621e
+  metadata.gz: 14ca57c35b2c4f399b3f3d3a475489f9ae3b50fc
+  data.tar.gz: 8d3fe32d258b3952fe649ff35d375635ad162afc
 SHA512:
-  metadata.gz: 68c02e914808c4fababacc1edea8d50d5c7576522154d023434edf6b54fe46fe4026998a57e7991b24848de0eac634b2a85da8105c844a1389fdf6f790dd0922
-  data.tar.gz: 5b93567d530e2e222476b825f5e0f53f33d3c1f8bb05f90571c06ead976462f3a61741ed9ed32d93135023a2a87aa2a1f9d806dcaad7d56d51562ff062ce12d0
+  metadata.gz: 3478cf3aac060a39857d989640622f6b0eeaea7ead4fb1add028c4513336ce38e972a4a6ce757864b2b6ae7a7c9c91a821c61a468fea6917cbb7a5d825dc384e
+  data.tar.gz: f9861651aeed1cc736d042ab0f86d567b6c5b0ea3c8522fb05825e641c8907c34972b9fde3c82845cf311614468c400a79c7aa4e9e8958b7de62398980b9ede6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    headhunter (0.1.3)
+    headhunter (0.1.4)
       colorize
       css_parser (>= 1.2.6)
       html_validation

data/README.md

@@ -6,6 +6,8 @@
 [![Code Climate](https://codeclimate.com/github/jmuheim/headhunter.png)](https://codeclimate.com/github/jmuheim/headhunter)
 [![Travis CI](https://api.travis-ci.org/jmuheim/headhunter.png)](https://travis-ci.org/jmuheim/headhunter)
 [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/jmuheim/headhunter/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
+[![Dependency Status](https://gemnasium.com/jmuheim/headhunter.png)](https://gemnasium.com/jmuheim/headhunter)
+[![Coverage Status](https://coveralls.io/repos/jmuheim/headhunter/badge.png)](https://coveralls.io/r/jmuheim/headhunter)
 
 Headhunter is an HTML and CSS validation tool that injects itself into your Rails feature tests and auto<strong>magic</strong>ally checks all your generated HTML and CSS for validity.
 
@@ -59,8 +61,11 @@ You need a **Java Runtime Environment** to run CSS validation. This should norma
 - In feature tests, the same views are getting rendered again and again. These same sources shouldn't be validated over and over again!
 - More configuration options needed! CSS1, CSS2, CSS2.1, CSS3! XHTML, HTML5, etc.
 - Better output needed! With context lines, etc.
+  - HTML output: info about GET/POST/PUT/DELETE!
 - Look out for multiple used IDs on the same page and raise error!
 - Look out for invalid tags and raise error (tidy doesn't seem to do this?!)!
+- Cache HTML validations: use MD5 to prevent validating the same source again and again! (Maybe even keep the cache between running specs!)
+- Add option to CssHunter that ignores "bare" rules like applet, blockquote, etc. (which usually stem from libraries like Compass reset or Normalize)
 
 ## Disclaimer
 

data/lib/headhunter/css_hunter.rb

@@ -1,104 +1,76 @@
 require 'css_parser'
 require 'nokogiri'
-require 'open-uri'
 
 module Headhunter
   class CssHunter
-    def initialize(stylesheets)
+    attr_reader :unused_selectors, :used_selectors, :error_selectors
+
+    def initialize(stylesheets = [])
       @stylesheets      = stylesheets
-      @parsed_rules     = {}
       @unused_selectors = []
       @used_selectors   = []
       @error_selectors  = []
 
-      load_css!
+      @stylesheets.each do |stylesheet|
+        add_css_selectors_from(IO.read(stylesheet))
+      end
     end
 
-    def process!(url, html)
-      analyze(html).each do |selector|
+    def process(html)
+      detect_used_selectors_in(html).each do |selector|
+        @used_selectors << selector
         @unused_selectors.delete(selector)
       end
     end
 
-    def report
-      puts "Found #{@used_selectors.size + @unused_selectors.size + @error_selectors.size} CSS selectors.".yellow
-      puts "#{@used_selectors.size} selectors are in use.".green if @used_selectors.size > 0
-      puts "#{@unused_selectors.size} selectors are not in use: #{@unused_selectors.sort.join(', ').red}".red if @unused_selectors.size > 0
-      puts "#{@error_selectors.size} selectors could not be parsed: #{@error_selectors.sort.join(', ').red}".red if @unused_selectors.size > 0
-      puts
-    end
+    def statistics
+      lines = []
 
-    private
+      lines << "Found #{used_selectors.size + unused_selectors.size + error_selectors.size} CSS selectors.".yellow
+      lines << 'All selectors are in use.'.green if unused_selectors.size + error_selectors.size == 0
+      lines << "#{unused_selectors.size} selectors are not in use: #{unused_selectors.sort.join(', ').red}".red if unused_selectors.size > 0
+      lines << "#{error_selectors.size} selectors could not be parsed: #{error_selectors.sort.join(', ').red}".red if error_selectors.size > 0
 
-    def analyze(html)
-      doc = Nokogiri::HTML(html)
+      lines.join("\n")
+    end
 
-      @unused_selectors.collect do |selector, declarations|
-        # We test against the selector stripped of any pseudo classes,
-        # but we report on the selector with its pseudo classes.
-        stripped_selector = strip(selector)
+    def detect_used_selectors_in(html)
+      document = Nokogiri::HTML(html)
 
-        next if stripped_selector.empty?
+      @unused_selectors.collect do |selector, declarations|
+        bare_selector = bare_selector_from(selector)
 
         begin
-          if doc.search(stripped_selector).any?
-            @used_selectors << selector
-            selector
-          end
+          selector if document.search(bare_selector).any?
         rescue Nokogiri::CSS::SyntaxError => e
           @error_selectors << selector
+          @unused_selectors.delete(selector)
         end
-      end
-    end
-
-    def load_css!
-      @stylesheets.each do |stylesheet|
-        new_selector_count = add_css!(fetch(stylesheet))
-      end
-    end
-
-    def fetch(path)
-      loc = path
-
-      begin
-        open(loc).read
-      rescue Errno::ENOENT
-        raise FetchError.new("#{loc} was not found")
-      rescue OpenURI::HTTPError => e
-        raise FetchError.new("retrieving #{loc} raised an HTTP error: #{e.message}")
-      end
+      end.compact # FIXME: Why is compact needed?
     end
 
-    def add_css!(css)
+    def add_css_selectors_from(css)
       parser = CssParser::Parser.new
       parser.add_block!(css)
 
-      selector_count = 0
-
       parser.each_selector do |selector, declarations, specificity|
-        next if @unused_selectors.include?(selector)
-        next if selector =~ @ignore_selectors
-        next if has_pseudo_classes(selector) and @unused_selectors.include?(strip(selector))
-
+        # next if @unused_selectors.include?(selector)
+        # next if has_pseudo_classes?(selector) and @unused_selectors.include?(bare_selector_from(selector))
         @unused_selectors << selector
-        @parsed_rules[selector] = declarations
-
-        selector_count += 1
       end
-
-      selector_count
     end
 
-    def has_pseudo_classes(selector)
-      selector =~ /::?[\w\-]+/
+    # def has_pseudo_classes?(selector)
+    #   selector =~ /::?[\w\-]+/
+    # end
+
+    def bare_selector_from(selector)
+      # Add more clean up stuff here, e.g. stuff like @keyframe (Deadweight implemented this)?
+      remove_pseudo_classes_from(selector)
     end
 
-    def strip(selector)
-      selector = selector.gsub(/^@.*/, '') # @-webkit-keyframes ...
-      selector = selector.gsub(/:.*/, '')  # input#x:nth-child(2):not(#z.o[type='file'])
-      selector
+    def remove_pseudo_classes_from(selector)
+      selector.gsub(/:.*/, '')  # input#x:nth-child(2):not(#z.o[type='file'])
     end
   end
-
-  class FetchError < StandardError; end
 end

data/lib/headhunter/css_validator.rb

@@ -1,73 +1,65 @@
 require 'net/http'
-require 'rexml/document'
+require 'nokogiri/xml'
 
 module Headhunter
-  class LocalResponse
-    attr_reader :body
-
-    def initialize(body)
-      @body = body
-      @headers = {'x-w3c-validator-status' => valid?(body)}
-    end
+  class CssValidator
+    VALIDATOR_PATH = Gem.loaded_specs['headhunter'].full_gem_path + '/lib/css-validator/'
 
-    def [](key)
-      @headers[key]
-    end
+    attr_reader :stylesheets, :responses
 
-    private
+    def initialize(stylesheets = [], profile = 'css3', vextwarning = true)
+      @stylesheets = stylesheets
+      @profile     = profile     # TODO!
+      @vextwarning = vextwarning # TODO!
 
-    def valid?(body)
-      REXML::Document.new(body).root.each_element('//m:validity') { |e| return e.text == 'true' }
+      @responses = @stylesheets.map do |stylesheet|
+                     validate(stylesheet)
+                   end
     end
-  end
 
-  class CssValidator
-    USE_LOCAL_VALIDATOR = true
+    def validate(uri)
+      # See http://stackoverflow.com/questions/1137884/is-there-an-open-source-css-validator-that-can-be-run-locally
+      # More config options see http://jigsaw.w3.org/css-validator/manual.html
+      results = if File.exists?(uri)
+                  Dir.chdir(VALIDATOR_PATH) { `java -jar css-validator.jar --output=soap12 file:#{uri}` }
+                else
+                  raise "Couldn't locate uri #{uri}"
+                end
 
-    def initialize(stylesheets = [])
-      @profile = 'css3' # TODO: Option for profile css1 and css21
-      @stylesheets = stylesheets
-      @messages_per_stylesheet = {}
+      Response.new(results)
     end
 
-    def add_stylesheet(stylesheet)
-      @stylesheets << stylesheet
+    def valid_responses
+      @responses.select(&:valid?)
     end
 
-    def process!
-      @stylesheets.each do |stylesheet|
-        css = fetch(stylesheet)
-        css = ' ' if css.empty? # The validator returns a 500 error if it receives an empty string
-
-        response = get_validation_response({text: css, profile: @profile, vextwarning: 'true'})
-        unless response_indicates_valid?(response)
-          process_errors(stylesheet, css, response)
-        end
-      end
+    def invalid_responses
+      @responses.reject(&:valid?)
     end
 
-    def report
-      puts "Validated #{@stylesheets.size} stylesheets.".yellow
-      puts "#{x_stylesheets_be(@stylesheets.size - @messages_per_stylesheet.size)} valid.".green if @messages_per_stylesheet.size < @stylesheets.size
-      puts "#{x_stylesheets_be(@messages_per_stylesheet.size)} invalid.".red if @messages_per_stylesheet.size > 0
+    def statistics
+      lines = []
+
+      lines << "Validated #{responses.size} stylesheets.".yellow
+      lines << "All stylesheets are valid.".green if invalid_responses.size == 0
+      lines << "#{x_stylesheets_be(invalid_responses.size)} invalid.".red if invalid_responses.size > 0
 
-      @messages_per_stylesheet.each_pair do |stylesheet, messages|
-        puts "  #{extract_filename(stylesheet)}:".red
+      invalid_responses.each do |response|
+        lines << "  #{extract_filename(response.uri)}:".red
 
-        messages.each { |message| puts "  - #{message}".red }
+        response.errors.each do |error|
+          lines << "    - #{error.to_s}".red
+        end
       end
 
-      puts
+      lines.join("\n")
     end
 
-    private
-
-    # Converts a path like public/assets/application-d205d6f344d8623ca0323cb6f6bd7ca1.css to application.css
     def extract_filename(path)
-      if matches = path.match(/^public\/assets\/(.*)-?([a-z0-9]*)(\.css)/)
-        matches[1] + matches[3]
+      if matches = path.match(/public\/assets\/([a-z\-_]*)-([a-z0-9]{32})(\.css)$/)
+        matches[1] + matches[3] # application-d205d6f344d8623ca0323cb6f6bd7ca1.css becomes application.css
       else
-        raise "Unexpected path: #{path}"
+        File.basename(path)
       end
     end
 
@@ -79,115 +71,72 @@ module Headhunter
       end
     end
 
-    def process_errors(file, css, response)
-      @messages_per_stylesheet[file] = []
-
-      REXML::Document.new(response.body).root.each_element('//m:error') do |e|
-        @messages_per_stylesheet[file] << "Line #{extract_line_from_error(e)}: #{extract_message_from_error(e)}"
+    class Response
+      def initialize(response = nil)
+        @document = Nokogiri::XML(convert_soap_to_xml(response)) if response
       end
-    end
-
-    def extract_line_from_error(e)
-      e.elements['m:line'].text
-    end
 
-    def extract_message_from_error(e)
-      e.elements['m:message'].get_text.value.strip[0..-2]
-    end
-
-    def fetch(path) # TODO: Move to Headhunter!
-      loc = path
-
-      begin
-        open(loc).read
-      rescue Errno::ENOENT
-        raise FetchError.new("#{loc} was not found")
-      rescue OpenURI::HTTPError => e
-        raise FetchError.new("retrieving #{loc} raised an HTTP error: #{e.message}")
+      def [](key)
+        @headers[key]
       end
-    end
-
-    def get_validation_response(query_params)
-      query_params.merge!({:output => 'soap12'})
 
-      if USE_LOCAL_VALIDATOR
-        call_local_validator(query_params)
-      else
-        call_remote_validator(query_params)
+      def valid?
+        @document.css('validity').text == 'true'
       end
-    end
-
-    def response_indicates_valid?(response)
-      response['x-w3c-validator-status'] == 'Valid'
-    end
-
-    def call_remote_validator(query_params = {})
-      boundary = Digest::MD5.hexdigest(Time.now.to_s)
-      data = encode_multipart_params(boundary, query_params)
-      response = http_start(validator_host).post2(validator_path,
-                                                  data,
-                                                  'Content-type' => "multipart/form-data; boundary=#{boundary}")
 
-      raise "HTTP error: #{response.code}" unless response.is_a? Net::HTTPSuccess
-      response
-    end
-
-    def call_local_validator(query_params)
-      path         = Gem.loaded_specs['headhunter'].full_gem_path + '/lib/css-validator/'
-      css_file     = 'tmp.css'
-      results_file = 'results'
-      results      = nil
-
-      Dir.chdir(path) do
-        File.open(css_file, 'a') { |f| f.write query_params[:text] }
-
-        # See http://stackoverflow.com/questions/1137884/is-there-an-open-source-css-validator-that-can-be-run-locally
-        if system "java -jar css-validator.jar --output=soap12 file:#{css_file} > #{results_file}"
-          results = IO.read results_file
-        else
-          raise 'Could not execute local validation!'
+      def errors
+        @document.css('errors error').map do |error|
+          Error.new( error.css('line').text.strip.to_i,
+                     error.css('message').text.strip[0..-3],
+                     errortype: error.css('errortype').text.strip,
+                     context: error.css('context').text.strip,
+                     errorsubtype: error.css('errorsubtype').text.strip,
+                     skippedstring: error.css('skippedstring').text.strip
+                   )
         end
+      end
 
-        File.delete css_file
-        File.delete results_file
+      def uri
+        @document.css('cssvalidationresponse > uri').text
       end
 
-      LocalResponse.new(results)
-    end
+      private
 
-    def encode_multipart_params(boundary, params = {})
-      ret = ''
-      params.each do |k,v|
-        unless v.empty?
-          ret << "\r\n--#{boundary}\r\n"
-          ret << "Content-Disposition: form-data; name=\"#{k.to_s}\"\r\n\r\n"
-          ret << v
-        end
+      def convert_soap_to_xml(soap)
+        sanitize_prefixed_tags_from(
+          remove_first_line_from(soap)
+        )
       end
-      ret << "\r\n--#{boundary}--\r\n"
-      ret
-    end
 
-    def http_start(host)
-      if ENV['http_proxy']
-        uri = URI.parse(ENV['http_proxy'])
-        proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
-        Net::HTTP.start(host, nil, uri.host, uri.port, proxy_user, proxy_pass)
-      else
-        Net::HTTP.start(host)
+      # The first line of the validator's response contains parameter options like this:
+      #
+      #     {vextwarning=false, output=soap, lang=en, warning=2, medium=all, profile=css3}
+      #
+      # We remove this so Nokogiri can parse the document as XML.
+      def remove_first_line_from(soap)
+        soap.split("\n")[1..-1].join("\n")
       end
-    end
 
-    def validator_host
-      'jigsaw.w3.org'
-    end
+      # The validator's response contains strange SOAP tags like `m:error` or `env:body` which need to be sanitized for Nokogiri.
+      #
+      # We simply remove the `m:` and `env:` prefixes from the source, so e.g. `<env:body>` becomes `<body>`.
+      def sanitize_prefixed_tags_from(soap)
+        soap.gsub /(m|env):/, ''
+      end
 
-    def validator_path
-      '/css-validator/validator'
-    end
+      class Error
+        attr_reader :line, :message, :details
+
+        def initialize(line, message, details = {})
+          @line    = line
+          @message = message
+          @details = details
+        end
 
-    def error_line_prefix
-      'Invalid css'
+        def to_s
+          "Line #{@line}: #{@message}."
+        end
+      end
     end
   end
 end

data/lib/headhunter/html_validator.rb

@@ -2,46 +2,43 @@ require 'html_validation'
 
 module Headhunter
   class HtmlValidator
+    attr_reader :responses
+
     def initialize
-      @valid_results   = []
-      @invalid_results = []
+      @responses = []
     end
 
-    def process!(url, html)
-      html_validation = PageValidations::HTMLValidation.new.validation(html, random_name)
-      (html_validation.valid? ? @valid_results : @invalid_results) << html_validation
+    def validate(url, html)
+      # Docs for Tidy: http://tidy.sourceforge.net/docs/quickref.html
+      @responses << PageValidations::HTMLValidation.new.validation(html, url)
+      @responses.last
     end
 
-    def prepare_results_html
-      html = File.read File.dirname(File.expand_path(__FILE__)) + '/templates/results.html'
-      html.gsub! '{{VALID_RESULTS}}', prepare_results_for(@valid_results)
-      html.gsub! '{{INVALID_RESULTS}}', prepare_results_for(@invalid_results)
-      File.open('.validation/results.html', 'w') { |file| file.write(html) }
+    def valid_responses
+      @responses.select(&:valid?)
     end
 
-    def prepare_results_for(results)
-      results.map do |result|
-        exceptions_html = ::Rack::Utils.escape_html(File.read(".validation/#{result.resource}.exceptions.txt"))
+    def invalid_responses
+      @responses.reject(&:valid?)
+    end
 
-        full_result_html = File.read File.dirname(File.expand_path(__FILE__)) + '/templates/result.html'
-        full_result_html.gsub! '{{RESOURCE}}', result.resource
-        full_result_html.gsub! '{{EXCEPTIONS}}', exceptions_html
-        full_result_html.gsub! '{{HTML_CONTEXT}}', 'context'
-        full_result_html.gsub! '{{LINK}}', "#{result.resource}.html.txt"
+    def statistics
+      lines = []
 
-        full_result_html
-      end.join
-    end
+      lines << "Validated #{responses.size} pages.".yellow
+      lines << "All pages are valid.".green if invalid_responses.size == 0
+      lines << "#{x_pages_be(invalid_responses.size)} invalid.".red if invalid_responses.size > 0
 
-    def report
-      puts "Validated #{@valid_results.size + @invalid_results.size} HTML pages.".yellow
-      puts "#{x_pages_be(@valid_results.size)} valid.".green if @valid_results.size > 0
-      puts "#{x_pages_be(@invalid_results.size)} invalid.".red if @invalid_results.size > 0
-      puts 'Open .validation/results.html to view full results.'
-      puts
-    end
+      invalid_responses.each do |response|
+        lines << "  #{response.resource}:".red
+      
+        ([response.exceptions].flatten).each do |exception|
+          lines << "    - #{exception.strip}".red
+        end
+      end
 
-    private
+      lines.join("\n")
+    end
 
     def x_pages_be(size)
       if size <= 1
@@ -50,9 +47,5 @@ module Headhunter
         "#{size} pages are"
       end
     end
-
-    def random_name
-      (0...8).map { (65 + rand(26)).chr }.join
-    end
   end
 end

data/lib/headhunter/rack/capturing_middleware.rb

@@ -16,7 +16,7 @@ module Headhunter
         status, headers, response = rack_response
 
         if html = extract_html_from(response)
-          @hh.process!('unknown', html)
+          @hh.process('unknown', html)
         end
       end
 

data/lib/headhunter/runner.rb

@@ -16,12 +16,11 @@ module Headhunter
       @css_hunter     = CssHunter.new(stylesheets)
 
       @css_validator = CssValidator.new(stylesheets)
-      @css_validator.process!
     end
 
-    def process!(url, html)
-      @html_validator.process!(url, html)
-      @css_hunter.process!(url, html)
+    def process(url, html)
+      @html_validator.validate(url, html)
+      @css_hunter.process(url, html)
     end
 
     def clean_up!
@@ -31,11 +30,12 @@ module Headhunter
     end
 
     def report
-      @html_validator.prepare_results_html
+      puts [ @html_validator.statistics,
+             @css_validator.statistics,
+             @css_hunter.statistics
+           ].join "\n\n"
 
-      @html_validator.report
-      @css_validator.report
-      @css_hunter.report
+     puts
     end
 
     private
@@ -58,7 +58,7 @@ module Headhunter
     end
 
     def stylesheets
-      Dir["#{ASSETS_PATH}/*.css"]
+      Dir["#{::Rails.root}/#{ASSETS_PATH}/*.css"]
     end
   end
 end

data/lib/headhunter/version.rb

@@ -1,3 +1,3 @@
 module Headhunter
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: headhunter
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Joshua Muheim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-03 00:00:00.000000000 Z
+date: 2014-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -195,7 +195,6 @@ files:
 - lib/headhunter/css_hunter.rb
 - lib/headhunter/css_validator.rb
 - lib/headhunter/html_validator.rb
-- lib/headhunter/local_response.rb
 - lib/headhunter/rack/capturing_middleware.rb
 - lib/headhunter/rails.rb
 - lib/headhunter/runner.rb
@@ -203,6 +202,7 @@ files:
 - lib/headhunter/templates/results.html
 - lib/headhunter/version.rb
 - lib/headhunter.rb
+- lib/tidy/tidy
 - Gemfile
 - Gemfile.lock
 - Rakefile

data/lib/headhunter/local_response.rb

@@ -1,20 +0,0 @@
-require 'rexml/document'
-
-module Headhunter
-  class LocalResponse
-    attr_reader :body
-
-    def initialize(body)
-      @body = body
-      @headers = {'x-w3c-validator-status' => valid?}
-    end
-
-    def [](key)
-      @headers[key]
-    end
-
-    def valid?
-      REXML::Document.new(@body).root.each_element('//m:validity') { |e| return e.text == 'true' }
-    end
-  end
-end