html-proofer 1.6.0 → 2.0.0

data/lib/html/proofer/checkable.rb

@@ -1,34 +1,37 @@
+require 'addressable/uri'
+require_relative './utils'
+
 module HTML
   class Proofer
+    # Represents the superclass from which all checks derive.
     class Checkable
-      def initialize(obj, type, check)
-        @src = obj['src']
-        @href = obj['href']
-        @alt = obj['alt']
-        @name = obj['name']
-        @id = obj['id']
-        @rel = obj['rel']
+      include HTML::Utils
+      attr_reader :line
+
+      def initialize(obj, check)
+        obj.attributes.each_pair do |attribute, value|
+          next if attribute == 'data-proofer-ignore' # TODO: not quite sure why this doesn't work
+          instance_variable_set("@#{attribute}".to_sym, value.value)
+        end
 
         @data_ignore_proofer = obj['data-proofer-ignore']
         @content = obj.content
         @check = check
         @checked_paths = {}
-        @type = type
+        @type = self.class.name
+        @line = obj.line
 
         if @href && @check.options[:href_swap]
-          @check.options[:href_swap].each do |link, replace|
-            @href = @href.gsub(link, replace)
-          end
+          @href = swap(@href, @check.options[:href_swap])
         end
 
         # fix up missing protocols
-        @href.insert 0, "http:" if @href =~ /^\/\//
-        @src.insert 0, "http:" if @src =~ /^\/\//
-
+        @href.insert 0, 'http:' if @href =~ %r{^//}
+        @src.insert 0, 'http:' if @src =~ %r{^//}
       end
 
       def url
-        @src || @href || ""
+        @src || @srcset || @href || ''
       end
 
       def valid?
@@ -42,15 +45,15 @@ module HTML
       end
 
       def path
-        parts.path if !parts.nil?
+        CGI.unescape parts.path unless parts.nil?
       end
 
       def hash
-        parts.fragment if !parts.nil?
+        parts.fragment unless parts.nil?
       end
 
       def scheme
-        parts.scheme if !parts.nil?
+        parts.scheme unless parts.nil?
       end
 
       # path is to an external server
@@ -66,13 +69,13 @@ module HTML
         return true if @data_ignore_proofer
 
         case @type
-        when "favicon"
+        when 'FaviconCheckable'
           return true if url.match(/^data:image/)
-        when "link"
-          return true if ignores_pattern_check(@check.additional_href_ignores)
-        when "image"
+        when 'LinkCheckable'
+          return true if ignores_pattern_check(@check.href_ignores)
+        when 'ImageCheckable'
           return true if url.match(/^data:image/)
-          return true if ignores_pattern_check(@check.additional_alt_ignores)
+          return true if ignores_pattern_check(@check.alt_ignores)
         end
       end
 
@@ -83,7 +86,7 @@ module HTML
 
       # path is an anchor or a query
       def internal?
-        url.start_with? "#", "?"
+        url.start_with? '#', '?'
       end
 
       def file_path
@@ -102,7 +105,7 @@ module HTML
         file = File.join base, path
 
         # implicit index support
-        if File.directory? file and !unslashed_directory? file
+        if File.directory?(file) && !unslashed_directory?(file)
           file = File.join file, @check.options[:directory_index_file]
         end
 
@@ -111,7 +114,7 @@ module HTML
 
       # checks if a file exists relative to the current pwd
       def exists?
-        return @checked_paths[absolute_path] if @checked_paths.has_key? absolute_path
+        return @checked_paths[absolute_path] if @checked_paths.key? absolute_path
         @checked_paths[absolute_path] = File.exist? absolute_path
       end
 
@@ -132,9 +135,20 @@ module HTML
         false
       end
 
-      def unslashed_directory? file
-        File.directory? file and !file.end_with? File::SEPARATOR and !@check.options[:followlocation]
+      def unslashed_directory?(file)
+        File.directory?(file) && !file.end_with?(File::SEPARATOR) && !follow_location?
       end
+
+      def follow_location?
+        @check.options[:typhoeus] && @check.options[:typhoeus][:followlocation]
+      end
+
+      private
+
+      def real_attr(attr)
+        attr.to_s unless attr.nil? || attr.empty?
+      end
+
     end
   end
 end

data/lib/html/proofer/checks/favicon.rb

@@ -1,21 +1,21 @@
 # encoding: utf-8
 
-class Favicon < ::HTML::Proofer::Checkable
+class FaviconCheckable < ::HTML::Proofer::Checkable
   def rel
     @rel
   end
 end
 
-class Favicons < ::HTML::Proofer::Checks::Check
+class FaviconCheck < ::HTML::Proofer::CheckRunner
 
   def run
-    @html.xpath("//link[not(ancestor::pre or ancestor::code)]").each do |favicon|
-      favicon = Favicon.new favicon, "favicon", self
+    @html.xpath('//link[not(ancestor::pre or ancestor::code)]').each do |favicon|
+      favicon = FaviconCheckable.new favicon, self
       next if favicon.ignore?
-      return if favicon.rel.split(" ").last.eql? "icon"
+      return if favicon.rel.split(' ').last.eql? 'icon'
     end
 
-    self.add_issue "no favicon specified"
+    add_issue 'no favicon specified'
   end
 
 end

data/lib/html/proofer/checks/html.rb

@@ -1,24 +1,23 @@
 # encoding: utf-8
 
-class Html < ::HTML::Proofer::Checks::Check
+class HtmlCheck < ::HTML::Proofer::CheckRunner
 
   # new html5 tags (source: http://www.w3schools.com/html/html5_new_elements.asp)
-  HTML5_TAGS = %w(article aside bdi details dialog figcaption 
-                  figure footer header main mark menuitem meter 
-                  nav progress rp rt ruby section summary 
-                  time wbr datalist keygen output color date 
-                  datetime datetime-local email month number 
-                  range search tel time url week canvas 
+  HTML5_TAGS = %w(article aside bdi details dialog figcaption
+                  figure footer header main mark menuitem meter
+                  nav progress rp rt ruby section summary
+                  time wbr datalist keygen output color date
+                  datetime datetime-local email month number
+                  range search tel time url week canvas
                   svg audio embed source track video)
 
   def run
     @html.errors.each do |e|
+      # Nokogiri (or rather libxml2 underhood) only recognizes html4 tags,
+      # so we need to skip errors caused by the new tags in html5
+      next if HTML5_TAGS.include? e.to_s[/Tag ([\w-]+) invalid/o, 1]
 
-        # Nokogiri (or rather libxml2 underhood) only recognizes html4 tags,
-        # so we need to skip errors caused by the new tags in html5
-        next if HTML5_TAGS.include? e.to_s[/Tag ([\w-]+) invalid/o, 1]
-
-        self.add_issue(e.to_s)
+      add_issue(e.to_s)
     end
   end
 end

data/lib/html/proofer/checks/images.rb

@@ -1,19 +1,19 @@
 # encoding: utf-8
 
-class Image < ::HTML::Proofer::Checkable
+class ImageCheckable < ::HTML::Proofer::Checkable
 
   SCREEN_SHOT_REGEX = /Screen(?: |%20)Shot(?: |%20)\d+-\d+-\d+(?: |%20)at(?: |%20)\d+.\d+.\d+/
 
   def valid_alt_tag?
-    @alt and !@alt.empty?
+    @alt && !@alt.empty?
   end
 
   def terrible_filename?
-    @src =~ SCREEN_SHOT_REGEX
+    src =~ SCREEN_SHOT_REGEX
   end
 
   def src
-    @src unless @src.nil? || @src.empty?
+    real_attr(@src) || real_attr(@srcset)
   end
 
   def missing_src?
@@ -22,29 +22,29 @@ class Image < ::HTML::Proofer::Checkable
 
 end
 
-class Images < ::HTML::Proofer::Checks::Check
+class ImageCheck < ::HTML::Proofer::CheckRunner
   def run
-    @html.css("img").each do |i|
-      img = Image.new i, "image", self
+    @html.css('img').each do |i|
+      img = ImageCheckable.new i, self
 
       next if img.ignore?
 
       # screenshot filenames should return because of terrible names
-      next self.add_issue "image has a terrible filename (#{img.src})" if img.terrible_filename?
+      next add_issue("image has a terrible filename (#{img.src})", i.line) if img.terrible_filename?
 
       # does the image exist?
       if img.missing_src?
-        self.add_issue "image has no src attribute"
+        add_issue('image has no src or srcset attribute', i.line)
       else
         if img.remote?
           add_to_external_urls img.src
         else
-          self.add_issue("internal image #{img.src} does not exist") unless img.exists?
+          add_issue("internal image #{img.src} does not exist", i.line) unless img.exists?
         end
       end
 
       # check alt tag
-      self.add_issue "image #{img.src} does not have an alt attribute" unless img.valid_alt_tag?
+      add_issue("image #{img.src} does not have an alt attribute", i.line) unless img.valid_alt_tag?
     end
 
     external_urls

data/lib/html/proofer/checks/links.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
-
-class Link < ::HTML::Proofer::Checkable
+class LinkCheckable < ::HTML::Proofer::Checkable
 
   def href
     real_attr @href
@@ -15,26 +14,21 @@ class Link < ::HTML::Proofer::Checkable
   end
 
   def missing_href?
-    href.nil? and name.nil? and id.nil?
+    href.nil? && name.nil? && id.nil?
   end
 
   def placeholder?
     (id || name) && href.nil?
   end
 
-  private
-
-  def real_attr(attr)
-    attr unless attr.nil? || attr.empty?
-  end
-
 end
 
-class Links < ::HTML::Proofer::Checks::Check
+class LinkCheck < ::HTML::Proofer::CheckRunner
+  include HTML::Utils
 
   def run
-    @html.css("a, link").each do |l|
-      link = Link.new l, "link", self
+    @html.css('a, link').each do |l|
+      link = LinkCheckable.new l, self
 
       next if link.ignore?
       next if link.href =~ /^javascript:/ # can't put this in ignore? because the URI does not parse
@@ -42,22 +36,22 @@ class Links < ::HTML::Proofer::Checks::Check
 
       # is it even a valid URL?
       unless link.valid?
-        self.add_issue "#{link.href} is an invalid URL"
+        add_issue("#{link.href} is an invalid URL", l.line)
         next
       end
 
-      if link.scheme == "mailto"
-        self.add_issue "#{link.href} contains no email address" if link.path.empty?
-        self.add_issue "#{link.href} contain an invalid email address" unless link.path.include?("@")
+      if link.scheme == 'mailto'
+        add_issue("#{link.href} contains no email address", l.line) if link.path.empty?
+        add_issue("#{link.href} contain an invalid email address", l.line) unless link.path.include?('@')
       end
 
-      if link.scheme == "tel"
-        self.add_issue "#{link.href} contains no phone number" if link.path.empty?
+      if link.scheme == 'tel'
+        add_issue("#{link.href} contains no phone number", l.line) if link.path.empty?
       end
 
       # is there even a href?
       if link.missing_href?
-        self.add_issue("anchor has no href attribute")
+        add_issue('anchor has no href attribute', l.line)
         next
       end
 
@@ -69,26 +63,23 @@ class Links < ::HTML::Proofer::Checks::Check
         add_to_external_urls link.href
         next
       elsif !link.internal?
-        self.add_issue "internally linking to #{link.href}, which does not exist" unless link.exists?
+        add_issue("internally linking to #{link.href}, which does not exist", l.line) unless link.exists?
       end
 
       # does the local directory have a trailing slash?
       if link.unslashed_directory? link.absolute_path
-        self.add_issue("internally linking to a directory #{link.absolute_path} without trailing slash")
+        add_issue("internally linking to a directory #{link.absolute_path} without trailing slash", l.line)
         next
       end
 
       # verify the target hash
       if link.hash
         if link.internal?
-          self.add_issue "linking to internal hash ##{link.hash} that does not exist" unless hash_check @html, link.hash
-        elsif link.external?
-          unless link.exists?
-            self.add_issue "trying to find hash of #{link.href}, but #{link.absolute_path} does not exist"
-          else
-            target_html = HTML::Proofer.create_nokogiri link.absolute_path
-            self.add_issue "linking to #{link.href}, but #{link.hash} does not exist" unless hash_check target_html, link.hash
+          unless hash_check @html, link.hash
+            add_issue("linking to internal hash ##{link.hash} that does not exist", l.line)
           end
+        elsif link.external?
+          external_link_check(link)
         end
       end
     end
@@ -96,6 +87,17 @@ class Links < ::HTML::Proofer::Checks::Check
     external_urls
   end
 
+  def external_link_check(link)
+    if !link.exists?
+      add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", l.line)
+    else
+      target_html = create_nokogiri link.absolute_path
+      unless hash_check target_html, link.hash
+        add_issue("linking to #{link.href}, but #{link.hash} does not exist", link.line)
+      end
+    end
+  end
+
   def hash_check(html, href_hash)
     html.xpath("//*[@id='#{href_hash}']", "//*[@name='#{href_hash}']").length > 0
   end

data/lib/html/proofer/checks/scripts.rb

@@ -1,9 +1,9 @@
 # encoding: utf-8
 
-class Script < ::HTML::Proofer::Checkable
+class ScriptCheckable < ::HTML::Proofer::Checkable
 
   def src
-    @src unless @src.nil? || @src.empty?
+    real_attr @src
   end
 
   def missing_src?
@@ -16,23 +16,22 @@ class Script < ::HTML::Proofer::Checkable
 
 end
 
-class Scripts < ::HTML::Proofer::Checks::Check
+class ScriptCheck < ::HTML::Proofer::CheckRunner
   def run
-    @html.css("script").each do |s|
-      script = Script.new s, "script", self
+    @html.css('script').each do |s|
+      script = ScriptCheckable.new s, self
 
       next if script.ignore?
       next unless script.blank?
 
       # does the script exist?
       if script.missing_src?
-        self.add_issue "script is empty and has no src attribute"
+        add_issue('script is empty and has no src attribute', s.line)
       elsif script.remote?
         add_to_external_urls script.src
       else
-        self.add_issue("internal script #{script.src} does not exist") unless script.exists?
+        add_issue("internal script #{script.src} does not exist", s.line) unless script.exists?
       end
-
     end
 
     external_urls

data/lib/html/proofer/log.rb

@@ -0,0 +1,38 @@
+require 'yell'
+require 'colored'
+
+module HTML
+  class Proofer
+    class Log
+      include Yell::Loggable
+
+      def initialize(verbose)
+        log_level = verbose ? :debug : :info
+
+        @logger = Yell.new(:format => false, \
+                           :name => 'HTML::Proofer', \
+                           :level => "gte.#{log_level}") do |l|
+          l.adapter :stdout, :level => [:debug, :info, :warn]
+          l.adapter :stderr, :level => [:error, :fatal]
+        end
+      end
+
+      def log(level, color, message)
+        @logger.send level, colorize(color, message)
+      end
+
+      def colorize(color, message)
+        if $stdout.isatty && $stderr.isatty
+          Colored.colorize(message, foreground: color)
+        else
+          message
+        end
+      end
+
+      # dumb override to play nice with Typhoeus/Ethon
+      def debug(message = nil)
+        log(:debug, :yellow, message) unless message.nil?
+      end
+    end
+  end
+end

data/lib/html/proofer/url_validator.rb

@@ -0,0 +1,135 @@
+require 'typhoeus'
+require 'uri'
+require_relative './utils'
+
+module HTML
+  class Proofer
+    class UrlValidator
+      include Utils
+
+      attr_accessor :logger, :external_urls, :hydra
+
+      def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts)
+        @logger = logger
+        @external_urls = external_urls
+        @failed_tests = []
+        @options = options
+        @hydra = Typhoeus::Hydra.new(hydra_opts)
+        @typhoeus_opts = typhoeus_opts
+      end
+
+      def run
+        external_link_checker(external_urls)
+        @failed_tests
+      end
+
+      # Proofer runs faster if we pull out all the external URLs and run the checks
+      # at the end. Otherwise, we're halting the consuming process for every file during
+      # the check_directory_of_files process.
+      #
+      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
+      #
+      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
+      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
+      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
+      # not an option.
+      def external_link_checker(external_urls)
+        external_urls = Hash[external_urls.sort]
+
+        count = external_urls.length
+        check_text = "#{count} " << (count == 1 ? 'external link' : 'external links')
+        logger.log :info, :blue, "Checking #{check_text}..."
+
+        Ethon.logger = logger # log from Typhoeus/Ethon
+
+        url_processor(external_urls)
+
+        logger.log :debug, :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..."
+        hydra.run
+      end
+
+      def url_processor(external_urls)
+        external_urls.each_pair do |href, filenames|
+          href = clean_url(href)
+          if hash?(href) && @options[:check_external_hash]
+            queue_request(:get, href, filenames)
+          else
+            queue_request(:head, href, filenames)
+          end
+        end
+      end
+
+      def clean_url(href)
+        Addressable::URI.parse(href).normalize
+      end
+
+      def queue_request(method, href, filenames)
+        request = Typhoeus::Request.new(href, @typhoeus_opts.merge({ :method => method }))
+        request.on_complete { |response| response_handler(response, filenames) }
+        hydra.queue request
+      end
+
+      def response_handler(response, filenames)
+        effective_url = response.options[:effective_url]
+        href = response.request.base_url.to_s
+        method = response.request.options[:method]
+        response_code = response.code
+        debug_msg = "Received a #{response_code} for #{href}"
+        debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
+        logger.log :debug, :yellow, debug_msg
+
+        if response_code.between?(200, 299)
+          check_hash_in_2xx_response(href, effective_url, response, filenames)
+        elsif response.timed_out?
+          handle_timeout(filenames, response_code)
+        elsif method == :head
+          queue_request(:get, href, filenames)
+        else
+          return if @options[:only_4xx] && !response_code.between?(400, 499)
+          # Received a non-successful http response.
+          add_failed_tests filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code
+        end
+      end
+
+      # Even though the response was a success, we may have been asked to check
+      # if the hash on the URL exists on the page
+      def check_hash_in_2xx_response(href, effective_url, response, filenames)
+        return if @options[:only_4xx]
+        return unless @options[:check_external_hash]
+        return unless (hash = hash?(href))
+
+        body_doc = create_nokogiri(response.body)
+
+        # user-content is a special addition by GitHub.
+        xpath = %(//*[@name="#{hash}"]|//*[@id="#{hash}"])
+        if URI.parse(href).host.match(/github\.com/i)
+          xpath << %(|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])
+        end
+
+        return unless body_doc.xpath(xpath).empty?
+
+        add_failed_tests filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response.code
+      end
+
+      def handle_timeout
+        return if @options[:only_4xx]
+        add_failed_tests filenames, "External link #{href} failed: got a time out", response_code
+      end
+
+      def add_failed_tests(filenames, desc, status = nil)
+        if filenames.nil?
+          @failed_tests << CheckRunner::Issue.new('', desc, nil, status)
+        else
+          filenames.each { |f| @failed_tests << CheckRunner::Issue.new(f, desc, nil, status) }
+        end
+      end
+
+      def hash?(url)
+        URI.parse(url).fragment
+      rescue URI::InvalidURIError
+        nil
+      end
+
+    end
+  end
+end

data/lib/html/proofer/utils.rb

@@ -0,0 +1,24 @@
+require 'nokogiri'
+
+module HTML
+  module Utils
+    def create_nokogiri(path)
+      if File.exist? path
+        content = File.open(path).read
+      else
+        content = path
+      end
+
+      Nokogiri::HTML(content)
+    end
+    module_function :create_nokogiri
+
+    def swap(href, replacement)
+      replacement.each do |link, replace|
+        href = href.gsub(link, replace)
+      end
+      href
+    end
+    module_function :swap
+  end
+end

data/lib/html/proofer/version.rb

@@ -1,5 +1,5 @@
 module HTML
   class Proofer
-    VERSION = "1.6.0"
+    VERSION = '2.0.0'
   end
 end