html-proofer 1.6.0 → 2.0.0

data/lib/html/proofer.rb

@@ -1,40 +1,34 @@
-require 'nokogiri'
-require 'yell'
+def require_all(path)
+  glob = File.join(File.dirname(__FILE__), path, '*.rb')
+  Dir[glob].each do |f|
+    require f
+  end
+end
+
+require_all 'proofer'
+require_all 'proofer/check_runner'
+require_all 'proofer/checks'
+require_relative './proofer/utils'
+
 require 'parallel'
-require "addressable/uri"
 
 begin
-  require "awesome_print"
+  require 'awesome_print'
 rescue LoadError; end
 
-%w[
-  checkable
-  checks
-  issue
-  version
-].each { |r| require File.join(File.dirname(__FILE__), "proofer", r) }
-
 module HTML
 
-  def self.colorize(color, string)
-    if $stdout.isatty && $stderr.isatty
-      Colored.colorize(string, :foreground => color)
-    else
-      string
-    end
-  end
-
   class Proofer
-    include Yell::Loggable
+    include Utils
 
     attr_reader :options, :typhoeus_opts, :parallel_opts
 
-    def initialize(src, opts={})
+    def initialize(src, opts = {})
       @src = src
 
       @proofer_opts = {
-        :ext => ".html",
-        :favicon => false,
+        :ext => '.html',
+        :check_favicon => false,
         :href_swap => [],
         :href_ignore => [],
         :file_ignore => [],
@@ -43,184 +37,103 @@ module HTML
         :disable_external => false,
         :verbose => false,
         :only_4xx => false,
-        :directory_index_file => "index.html",
-        :validate_html => false,
-        :error_sort => :path
+        :directory_index_file => 'index.html',
+        :check_html => false,
+        :error_sort => :path,
+        :checks_to_ignore => []
       }
 
-      @typhoeus_opts = {
+      @typhoeus_opts = opts[:typhoeus] || {
         :followlocation => true
       }
+      opts.delete(:typhoeus)
+
+      @hydra_opts = opts[:hydra] || {}
+      opts.delete(:hydra)
 
       # fall back to parallel defaults
       @parallel_opts = opts[:parallel] || {}
       opts.delete(:parallel)
 
-      # Typhoeus won't let you pass in any non-Typhoeus option; if the option is not
-      # a proofer_opt, it must be for Typhoeus
-      opts.keys.each do |key|
-        if @proofer_opts[key].nil?
-          @typhoeus_opts[key] = opts[key]
-        end
-      end
-
-      @options = @proofer_opts.merge(@typhoeus_opts).merge(opts)
+      @options = @proofer_opts.merge(opts)
 
       @failed_tests = []
+    end
 
-      Yell.new({ :format => false, :name => "HTML::Proofer", :level => "gte.#{log_level}" }) do |l|
-        l.adapter :stdout, level: [:debug, :info, :warn]
-        l.adapter :stderr, level: [:error, :fatal]
-      end
+    def logger
+      @logger ||= HTML::Proofer::Log.new(@options[:verbose])
     end
 
     def run
-      unless @src.is_a? Array
-        external_urls = {}
-
-        logger.info HTML::colorize :white, "Running #{get_checks} checks on #{@src} on *#{@options[:ext]}... \n\n"
-
-        results = Parallel.map(files, @parallel_opts) do |path|
-          html = HTML::Proofer.create_nokogiri(path)
-          result = {:external_urls => {}, :failed_tests => []}
-
-          get_checks.each do |klass|
-            logger.debug HTML::colorize :blue, "Checking #{klass.to_s.downcase} on #{path} ..."
-            check =  Object.const_get(klass).new(@src, path, html, @options)
-            check.run
-            result[:external_urls].merge!(check.external_urls)
-            result[:failed_tests].concat(check.issues) if check.issues.length > 0
-          end
-          result
-        end
-
-        results.each do |item|
-          external_urls.merge!(item[:external_urls])
-          @failed_tests.concat(item[:failed_tests])
-        end
-
-        external_link_checker(external_urls) unless @options[:disable_external]
+      count = checks.length
+      check_text = "#{checks} " << (count == 1 ? 'check' : 'checks')
+      logger.log :info, :blue, "Running #{check_text} on #{@src} on *#{@options[:ext]}... \n\n"
 
-        logger.info HTML::colorize :green, "Ran on #{files.length} files!\n\n"
+      if @src.is_a?(Array) && !@options[:disable_external]
+        check_list_of_links
       else
-        external_urls = Hash[*@src.map{ |s| [s, nil] }.flatten]
-        external_link_checker(external_urls) unless @options[:disable_external]
+        check_directory_of_files
       end
 
       if @failed_tests.empty?
-        logger.info HTML::colorize :green, "HTML-Proofer finished successfully."
+        logger.log :info, :green, 'HTML-Proofer finished successfully.'
       else
-        matcher = nil
-
-        # always sort by the actual option, then path, to ensure consistent alphabetical (by filename) results
-        @failed_tests = @failed_tests.sort do |a,b|
-          comp = (a.send(@options[:error_sort]) <=> b.send(@options[:error_sort]))
-          comp.zero? ? (a.path <=> b.path) : comp
-        end
+        print_failed_tests
+      end
+    end
 
-        @failed_tests.each do |issue|
-          case @options[:error_sort]
-          when :path
-            if matcher != issue.path
-              logger.error HTML::colorize :blue, "- #{issue.path}"
-              matcher = issue.path
-            end
-            logger.error HTML::colorize :red, "  *  #{issue.desc}"
-          when :desc
-            if matcher != issue.desc
-              logger.error HTML::colorize :blue, "- #{issue.desc}"
-              matcher = issue.desc
-            end
-            logger.error HTML::colorize :red, "  *  #{issue.path}"
-          when :status
-            if matcher != issue.status
-              logger.error HTML::colorize :blue, "- #{issue.status}"
-              matcher = issue.status
-            end
-            logger.error HTML::colorize :red, "  *  #{issue.to_s}"
-          end
+    def check_list_of_links
+      if @options[:href_swap]
+        @src = @src.map do |external_url|
+          swap(external_url, @options[:href_swap])
         end
-
-        raise HTML::colorize :red, "HTML-Proofer found #{@failed_tests.length} failures!"
       end
+      external_urls = Hash[*@src.map { |s| [s, nil] }.flatten]
+      validate_urls(external_urls)
     end
 
-    # the hypothesis is that Proofer runs way faster if we pull out
-    # all the external URLs and run the checks at the end. Otherwise, we're halting
-    # the consuming process for every file. In addition, sorting the list lets
-    # libcurl keep connections to hosts alive. Finally, we'll make a HEAD request,
-    # rather than GETing all the contents
-    def external_link_checker(external_urls)
-      external_urls = Hash[external_urls.sort]
-
-      logger.info HTML::colorize :yellow, "Checking #{external_urls.length} external links..."
+    # Collects any external URLs found in a directory of files. Also collectes
+    # every failed test from check_files_for_internal_woes.
+    # Sends the external URLs to Typhoeus for batch processing.
+    def check_directory_of_files
+      external_urls = {}
+      results = check_files_for_internal_woes
 
-      Ethon.logger = logger # log from Typhoeus/Ethon
-
-      external_urls.each_pair do |href, filenames|
-        if has_hash? href && @options[:check_external_hash]
-          queue_request(:get, href, filenames)
-        else
-          queue_request(:head, href, filenames)
-        end
+      results.each do |item|
+        external_urls.merge!(item[:external_urls])
+        @failed_tests.concat(item[:failed_tests])
       end
-      logger.debug HTML::colorize :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..."
-      hydra.run
-    end
 
-    def queue_request(method, href, filenames)
-      request = Typhoeus::Request.new(href, @typhoeus_opts.merge({:method => method}))
-      request.on_complete { |response| response_handler(response, filenames) }
-      hydra.queue request
+      validate_urls(external_urls) unless @options[:disable_external]
+
+      logger.log :info, :blue, "Ran on #{files.length} files!\n\n"
     end
 
-    def response_handler(response, filenames)
-      effective_url = response.options[:effective_url]
-      href = response.request.base_url
-      method = response.request.options[:method]
-      response_code = response.code
-
-      debug_msg = "Received a #{response_code} for #{href}"
-      debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
-      logger.debug debug_msg
-
-      if response_code.between?(200, 299)
-        return if @options[:only_4xx]
-        return unless @options[:check_external_hash]
-        if hash = has_hash?(href)
-          body_doc = Nokogiri::HTML(response.body)
-          # user-content is a special addition by GitHub.
-          xpath = %$//*[@name="#{hash}"]|//*[@id="#{hash}"]$
-          xpath << %$|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"]$ if URI.parse(href).host.match(/github\.com/i)
-          if body_doc.xpath(xpath).empty?
-            add_failed_tests filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response_code
-          end
+    # Walks over each implemented check and runs them on the files, in parallel.
+    def check_files_for_internal_woes
+      Parallel.map(files, @parallel_opts) do |path|
+        html = create_nokogiri(path)
+        result = { :external_urls => {}, :failed_tests => [] }
+
+        checks.each do |klass|
+          logger.log :debug, :yellow, "Checking #{klass.to_s.downcase} on #{path} ..."
+          check = Object.const_get(klass).new(@src, path, html, @options)
+          check.run
+          result[:external_urls].merge!(check.external_urls)
+          result[:failed_tests].concat(check.issues) if check.issues.length > 0
         end
-      elsif response.timed_out?
-        return if @options[:only_4xx]
-        add_failed_tests filenames, "External link #{href} failed: got a time out", response_code
-      elsif (response_code == 405 || response_code == 420 || response_code == 503) && method == :head
-        # 420s usually come from rate limiting; let's ignore the query and try just the path with a GET
-        uri = URI(href)
-        queue_request(:get, uri.scheme + "://" + uri.host + uri.path, filenames)
-      # just be lazy; perform an explicit get request. some servers are apparently not configured to
-      # intercept HTTP HEAD
-      elsif method == :head
-        queue_request(:get, effective_url, filenames)
-      else
-        return if @options[:only_4xx] && !response_code.between?(400, 499)
-        # Received a non-successful http response.
-        add_failed_tests filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code
+        result
       end
     end
 
-    def hydra
-      @hydra ||= Typhoeus::Hydra.new
+    def validate_urls(external_urls)
+      url_validator = HTML::Proofer::UrlValidator.new(logger, external_urls, @options, @typhoeus_opts, @hydra_opts)
+      @failed_tests.concat(url_validator.run)
     end
 
     def files
       if File.directory? @src
-        pattern = File.join @src, "**", "*#{@options[:ext]}"
+        pattern = File.join(@src, '**', "*#{@options[:ext]}")
         files = Dir.glob(pattern).select { |fn| File.file? fn }
         files.reject { |f| ignore_file?(f) }
       elsif File.extname(@src) == @options[:ext]
@@ -231,49 +144,23 @@ module HTML
     end
 
     def ignore_file?(file)
-      @options[:file_ignore].each do |pattern|
-        if pattern.is_a? String
-          return pattern == file
-        elsif pattern.is_a? Regexp
-          return pattern =~ file
-        end
+      options[:file_ignore].each do |pattern|
+        return true if pattern.is_a?(String) && pattern == file
+        return true if pattern.is_a?(Regexp) && pattern =~ file
       end
 
       false
     end
 
-    def self.create_nokogiri(path)
-      content = File.open(path).read
-      Nokogiri::HTML(content)
-    end
-
-    def get_checks
-      checks = HTML::Proofer::Checks::Check.subclasses.map { |c| c.name }
-      checks.delete("Favicons") unless @options[:favicon]
-      checks.delete("Html") unless @options[:validate_html]
-      checks
-    end
-
-    def has_hash?(url)
-      begin
-        URI.parse(url).fragment
-      rescue URI::InvalidURIError
-        nil
-      end
-    end
-
-    def log_level
-      @options[:verbose] ? :debug : :info
-    end
-
-    def add_failed_tests(filenames, desc, status = nil)
-      if filenames.nil?
-        @failed_tests << Checks::Issue.new("", desc, status)
-      elsif
-        filenames.each { |f|
-          @failed_tests << Checks::Issue.new(f, desc, status)
-        }
+    def checks
+      return @checks unless @checks.nil?
+      @checks = HTML::Proofer::CheckRunner.checks.map(&:name)
+      @checks.delete('FaviconCheck') unless @options[:check_favicon]
+      @checks.delete('HtmlCheck') unless @options[:check_html]
+      @options[:checks_to_ignore].each do |ignored|
+        @checks.delete(ignored)
       end
+      @checks
     end
 
     def failed_tests
@@ -282,5 +169,14 @@ module HTML
       @failed_tests.each { |f| result << f.to_s }
       result
     end
+
+    def print_failed_tests
+      sorted_failures = HTML::Proofer::CheckRunner::SortedIssues.new(@failed_tests, @options[:error_sort], logger)
+
+      sorted_failures.sort_and_report
+      count = @failed_tests.length
+      failure_text = "#{count} " << (count == 1 ? 'failure' : 'failures')
+      fail logger.colorize :red, "HTML-Proofer found #{failure_text}!"
+    end
   end
 end

data/spec/html/proofer/command_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe 'Command test' do
+  it 'works with as-links' do
+    output = make_bin('--as-links www.github.com,foofoofoo.biz')
+    expect(output).to match('1 failure')
+  end
+
+  it 'works with alt-ignore' do
+    ignorableLinks = "#{FIXTURES_DIR}/images/ignorableAltViaOptions.html"
+    output = make_bin('--alt-ignore /wikimedia/,gpl.png', ignorableLinks)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with checks_to_ignore' do
+    external = "#{FIXTURES_DIR}/links/file.foo"
+    output = make_bin('--ext .foo --checks-to-ignore LinkCheck', external)
+    expect(output).to match('successfully')
+    expect(output).to_not match('LinkCheck')
+  end
+
+  it 'works with check-external-hash' do
+    brokenHashOnTheWeb = "#{FIXTURES_DIR}/links/brokenHashOnTheWeb.html"
+    output = make_bin('--check-external-hash', brokenHashOnTheWeb)
+    expect(output).to match('1 failure')
+  end
+
+  it 'works with directory-index-file' do
+    link_pointing_to_directory = "#{FIXTURES_DIR}/links/link_pointing_to_directory.html"
+    output = make_bin('--directory-index-file index.php', link_pointing_to_directory)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with disable-external' do
+    external = "#{FIXTURES_DIR}/links/brokenLinkExternal.html"
+    output = make_bin('--disable-external', external)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with ext' do
+    external = "#{FIXTURES_DIR}/links/file.foo"
+    output = make_bin('--ext .foo', external)
+    expect(output).to match('1 failure')
+    expect(output).to match('LinkCheck')
+  end
+
+  it 'works with file-ignore' do
+    external = "#{FIXTURES_DIR}/links/brokenHashInternal.html"
+    output = make_bin("--file-ignore #{external}", external)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with href-ignore' do
+    ignorableLinks = "#{FIXTURES_DIR}/links/ignorableLinksViaOptions.html"
+    output = make_bin('--href-ignore /^http:\/\//,/sdadsad/,../whaadadt.html', ignorableLinks)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with href-swap' do
+    translatedLink = "#{FIXTURES_DIR}/links/linkTranslatedViaHrefSwap.html"
+    output = make_bin('--href-swap "\A/articles/([\w-]+):\1.html"', translatedLink)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with only-4xx' do
+    brokenHashOnTheWeb = "#{FIXTURES_DIR}/links/brokenHashOnTheWeb.html"
+    output = make_bin('--only-4xx', brokenHashOnTheWeb)
+    expect(output).to match('successfully')
+  end
+
+  it 'works with check-favicon' do
+    broken = "#{FIXTURES_DIR}/favicon/favicon_broken.html"
+    output = make_bin('--check-favicon', broken)
+    expect(output).to match('1 failure')
+  end
+
+  it 'works with check-html' do
+    broken = "#{FIXTURES_DIR}/html/invalid_tag.html"
+    output = make_bin('--check-html', broken)
+    expect(output).to match('1 failure')
+  end
+end

data/spec/html/proofer/favicon_spec.rb

@@ -1,47 +1,47 @@
-require "spec_helper"
+require 'spec_helper'
 
-describe "Favicons test" do
-  it "ignores for absent favicon by default" do
+describe 'Favicons test' do
+  it 'ignores for absent favicon by default' do
     absent = "#{FIXTURES_DIR}/favicon/favicon_absent.html"
-    expect(make_proofer(absent).failed_tests).to eq []
+    expect(run_proofer(absent).failed_tests).to eq []
   end
 
-  it "fails for absent favicon" do
+  it 'fails for absent favicon' do
     absent = "#{FIXTURES_DIR}/favicon/favicon_absent.html"
-    proofer = make_proofer(absent, {:favicon => true})
-    expect(proofer.failed_tests.first).to match /no favicon specified/
+    proofer = run_proofer(absent, { :check_favicon => true })
+    expect(proofer.failed_tests.first).to match(/no favicon specified/)
   end
 
-  it "fails for absent favicon but present apple touch icon" do
+  it 'fails for absent favicon but present apple touch icon' do
     absent = "#{FIXTURES_DIR}/favicon/favicon_absent_apple.html"
-    proofer = make_proofer(absent, {:favicon => true})
+    proofer = run_proofer(absent, { :check_favicon => true })
     # Travis gives a different error message here for some reason
-    expect(proofer.failed_tests.last).to match /(internally linking to gpl.png, which does not exist|no favicon specified)/
+    expect(proofer.failed_tests.last).to match(/(internally linking to gpl.png, which does not exist|no favicon specified)/)
   end
 
-  it "fails for broken favicon" do
+  it 'fails for broken favicon' do
     broken = "#{FIXTURES_DIR}/favicon/favicon_broken.html"
-    proofer = make_proofer(broken, {:favicon => true})
+    proofer = run_proofer(broken, { :check_favicon => true })
 
-    expect(proofer.failed_tests.first).to match /internally linking to asdadaskdalsdk.png/
+    expect(proofer.failed_tests.first).to match(/internally linking to asdadaskdalsdk.png/)
   end
 
-  it "passes for present favicon" do
+  it 'passes for present favicon' do
     present = "#{FIXTURES_DIR}/favicon/favicon_present.html"
-    proofer = make_proofer(present, {:favicon => true})
+    proofer = run_proofer(present, { :check_favicon => true })
     expect(proofer.failed_tests).to eq []
   end
 
-  it "passes for present favicon with shortcut notation" do
+  it 'passes for present favicon with shortcut notation' do
     present = "#{FIXTURES_DIR}/favicon/favicon_present_shortcut.html"
-    proofer =  make_proofer(present, {:favicon => true})
+    proofer =  run_proofer(present, { :check_favicon => true })
     expect(proofer.failed_tests).to eq []
   end
 
-  it "fails for broken favicon with data-proofer-ignore" do
+  it 'fails for broken favicon with data-proofer-ignore' do
     broken_but_ignored = "#{FIXTURES_DIR}/favicon/favicon_broken_but_ignored.html"
-    proofer = make_proofer(broken_but_ignored, {:favicon => true})
-    expect(proofer.failed_tests.first).to match /no favicon specified/
+    proofer = run_proofer(broken_but_ignored, { :check_favicon => true })
+    expect(proofer.failed_tests.first).to match(/no favicon specified/)
   end
 
 end

data/spec/html/proofer/fixtures/images/srcSetCheck.html

@@ -0,0 +1,7 @@
+<picture>
+    <!--[if IE 9]><video style="display: none;"><![endif]-->
+    <source srcset="examples/images/extralarge.jpg" media="(min-width: 1000px)">
+    <source srcset="examples/images/large.jpg" media="(min-width: 800px)">
+    <!--[if IE 9]></video><![endif]-->
+    <img src="gpl.png" alt="Working">
+</picture>

data/spec/html/proofer/fixtures/images/srcSetIgnorable.html

@@ -0,0 +1,13 @@
+<html>
+
+<body>
+
+
+<img src="gpl.png"/>Relative to self
+
+
+<p>Blah blah blah. <img srcset="//upload.wikimedia.org/wikipedia/en/thumb/2/22/Heckert_GNU_white.svg/256px-Heckert_GNU_white.svg.png" /> </p>
+
+</body>
+
+</html>

data/spec/html/proofer/fixtures/images/srcSetMissingAlt.html

@@ -0,0 +1,7 @@
+<picture>
+    <!--[if IE 9]><video style="display: none;"><![endif]-->
+    <source srcset="examples/images/extralarge.jpg" media="(min-width: 1000px)">
+    <source srcset="examples/images/large.jpg" media="(min-width: 800px)">
+    <!--[if IE 9]></video><![endif]-->
+    <img srcset="gpl.png">
+</picture>

data/spec/html/proofer/fixtures/images/srcSetMissingImage.html

@@ -0,0 +1,7 @@
+<picture>
+    <!--[if IE 9]><video style="display: none;"><![endif]-->
+    <source srcset="examples/images/extralarge.jpg" media="(min-width: 1000px)">
+    <source srcset="examples/images/large.jpg" media="(min-width: 800px)">
+    <!--[if IE 9]></video><![endif]-->
+    <img srcset="notreal.png" alt="Missing image">
+</picture>

data/spec/html/proofer/fixtures/links/erstiebegru/314/210/303/237ung.html

@@ -0,0 +1 @@
+

data/spec/html/proofer/fixtures/links/erstiebegr/303/274/303/237ung.html

@@ -0,0 +1 @@
+

data/spec/html/proofer/fixtures/links/file.foo

@@ -0,0 +1,11 @@
+<html>
+
+<body>
+
+  <p>Blah blah blah. <a href="http://www.google.com">Working link!</a></p>
+
+
+<a href="http://www.asdo3IRJ395295jsingrkrg4.com">broken link!</a>
+</body>
+
+</html>

data/spec/html/proofer/fixtures/links/folder/multiples/catalog/file.html

@@ -0,0 +1,8 @@
+<html>
+
+<body>
+
+<a href="http://www.asdo3IRJ395295jsingrkrg4.com">broken link!</a>
+</body>
+
+</html>

data/spec/html/proofer/fixtures/links/folder/multiples/javadoc/file.html

@@ -0,0 +1,8 @@
+<html>
+
+<body>
+
+<a href="http://www.asdo3IRJ395295jsingrkrg4.com">broken link!</a>
+</body>
+
+</html>

data/spec/html/proofer/fixtures/links/nodupe.html

@@ -1 +1 @@
-<a href="https://help.github.com/articles/generating-ssh-keys#step-3-add-your-ssh-key-to-github">add ssh</a>
+<a href="https://help.github.com/articlezzz">add ssh</a>

data/spec/html/proofer/fixtures/links/redirected_error.html

@@ -0,0 +1 @@
+<a href="http://asia.cnet.com/blogs/geekonomics/post.htm?id=63009224">not real</a>

data/spec/html/proofer/fixtures/links/rootLink/rootLink.html

@@ -3,7 +3,6 @@
 <body>
 
   <p>Blah blah blah. <a href="/">I am root.</a></p>
-  <p>Blah blah blah. <a href="/folder">I am relative to root.</a></p>
 
 </body>
 

data/spec/html/proofer/fixtures/links/urlencoded-href.html

@@ -0,0 +1,2 @@
+<a href="erstiebegr%C3%BC%C3%9Fung.html">Erstiebegrüßung</a>
+<a href="erstiebegrüßung.html">Erstiebegrüßung</a>

data/spec/html/proofer/fixtures/links/utf8Link.html

@@ -0,0 +1,2 @@
+<a href="http://ben.balter.com/2014/09/29/source-disclosed-≠-open-source/">utf8</a>
+<a href="http://ben.balter.com/2014/09/29/source-disclosed-%E2%89%A0-open-source/">another utf8</a>

data/spec/html/proofer/fixtures/utils/lang-jp.html

@@ -0,0 +1 @@
+<html lang="jp">