html-proofer 4.0.0.rc3 → 4.0.0

data/lib/html_proofer/log.rb

@@ -1,21 +1,21 @@
 # frozen_string_literal: true
 
-require 'yell'
-require 'rainbow'
+require "yell"
+require "rainbow"
 
 module HTMLProofer
   class Log
     include Yell::Loggable
 
-    STDOUT_LEVELS = %i[debug info warn].freeze
-    STDERR_LEVELS = %i[error fatal].freeze
+    STDOUT_LEVELS = [:debug, :info, :warn].freeze
+    STDERR_LEVELS = [:error, :fatal].freeze
 
     def initialize(log_level)
       @logger = Yell.new(format: false, \
-                         name: 'HTMLProofer', \
-                         level: "gte.#{log_level}") do |l|
-        l.adapter :stdout, level: 'lte.warn'
-        l.adapter :stderr, level: 'gte.error'
+        name: "HTMLProofer", \
+        level: "gte.#{log_level}") do |l|
+        l.adapter(:stdout, level: "lte.warn")
+        l.adapter(:stderr, level: "gte.error")
       end
     end
 
@@ -24,23 +24,23 @@ module HTMLProofer
     end
 
     def log_with_color(level, message)
-      @logger.send level, colorize(level, message)
+      @logger.send(level, colorize(level, message))
     end
 
     def colorize(level, message)
       color = case level
-              when :debug
-                :cyan
-              when :info
-                :blue
-              when :warn
-                :yellow
-              when :error, :fatal
-                :red
-              end
+      when :debug
+        :cyan
+      when :info
+        :blue
+      when :warn
+        :yellow
+      when :error, :fatal
+        :red
+      end
 
       if (STDOUT_LEVELS.include?(level) && $stdout.isatty) || \
-         (STDERR_LEVELS.include?(level) && $stderr.isatty)
+          (STDERR_LEVELS.include?(level) && $stderr.isatty)
         Rainbow(message).send(color)
       else
         message

data/lib/html_proofer/reporter/cli.rb

@@ -1,29 +1,33 @@
 # frozen_string_literal: true
 
-class HTMLProofer::Reporter::Cli < HTMLProofer::Reporter
-  def report
-    msg = failures.each_with_object([]) do |(check_name, failures), arr|
-      str = ["For the #{check_name} check, the following failures were found:\n"]
+module HTMLProofer
+  class Reporter
+    class Cli < HTMLProofer::Reporter
+      def report
+        msg = failures.each_with_object([]) do |(check_name, failures), arr|
+          str = ["For the #{check_name} check, the following failures were found:\n"]
 
-      failures.each do |failure|
-        path_str = blank?(failure.path) ? '' : "In #{failure.path}"
+          failures.each do |failure|
+            path_str = blank?(failure.path) ? "" : "At #{failure.path}"
 
-        line_str = failure.line.nil? ? '' : " (line #{failure.line})"
+            line_str = failure.line.nil? ? "" : ":#{failure.line}"
 
-        path_and_line = [path_str, line_str].join
-        path_and_line = blank?(path_and_line) ? '' : "* #{path_and_line}:\n\n"
+            path_and_line = "#{path_str}#{line_str}"
+            path_and_line = blank?(path_and_line) ? "" : "* #{path_and_line}:\n\n"
 
-        status_str = failure.status.nil? ? '' : " (status code #{failure.status})"
+            status_str = failure.status.nil? ? "" : " (status code #{failure.status})"
 
-        indent = blank?(path_and_line) ? '* ' : '  '
-        str << <<~MSG
-          #{path_and_line}#{indent}#{failure.description}#{status_str}
-        MSG
-      end
+            indent = blank?(path_and_line) ? "* " : "  "
+            str << <<~MSG
+              #{path_and_line}#{indent}#{failure.description}#{status_str}
+            MSG
+          end
 
-      arr << str.join("\n")
-    end
+          arr << str.join("\n")
+        end
 
-    @logger.log(:error, msg.join("\n"))
+        @logger.log(:error, msg.join("\n"))
+      end
+    end
   end
 end

data/lib/html_proofer/reporter.rb

@@ -12,12 +12,12 @@ module HTMLProofer
 
     def failures=(failures)
       @failures = failures.group_by(&:check_name) \
-                          .transform_values { |issues| issues.sort_by { |issue| [issue.path, issue.line] } } \
-                          .sort
+        .transform_values { |issues| issues.sort_by { |issue| [issue.path, issue.line] } } \
+        .sort
     end
 
     def report
-      raise NotImplementedError, 'HTMLProofer::Reporter subclasses must implement #report'
+      raise NotImplementedError, "HTMLProofer::Reporter subclasses must implement #report"
     end
   end
 end

data/lib/html_proofer/runner.rb

@@ -5,9 +5,9 @@ module HTMLProofer
     include HTMLProofer::Utils
 
     attr_reader :options, :cache, :logger, :internal_urls, :external_urls, :checked_paths, :current_check
-    attr_accessor :current_path, :current_source, :reporter
+    attr_accessor :current_filename, :current_source, :reporter
 
-    URL_TYPES = %i[external internal].freeze
+    URL_TYPES = [:external, :internal].freeze
 
     def initialize(src, opts = {})
       @options = HTMLProofer::Configuration.generate_defaults(opts)
@@ -28,22 +28,23 @@ module HTMLProofer
 
       @current_check = nil
       @current_source = nil
-      @current_path = nil
+      @current_filename = nil
 
       @reporter = Reporter::Cli.new(logger: @logger)
     end
 
     def run
-      check_text = pluralize(checks.length, 'check', 'checks')
+      check_text = pluralize(checks.length, "check", "checks")
 
       if @type == :links
-        @logger.log :info, "Running #{check_text} (#{format_checks_list(checks)}) on #{@source} ... \n\n"
+        @logger.log(:info, "Running #{check_text} (#{format_checks_list(checks)}) on #{@source} ... \n\n")
         check_list_of_links unless @options[:disable_external]
       else
-        @logger.log :info, "Running #{check_text} (#{format_checks_list(checks)}) in #{@source} on *#{@options[:extensions].join(', ')} files...\n\n"
+        @logger.log(:info,
+          "Running #{check_text} (#{format_checks_list(checks)}) in #{@source} on *#{@options[:extensions].join(", ")} files...\n\n")
 
         check_files
-        @logger.log :info, "Ran on #{pluralize(files.length, 'file', 'files')}!\n\n"
+        @logger.log(:info, "Ran on #{pluralize(files.length, "file", "files")}!\n\n")
       end
 
       @cache.write
@@ -51,7 +52,7 @@ module HTMLProofer
       @reporter.failures = @failures
 
       if @failures.empty?
-        @logger.log :info, 'HTML-Proofer finished successfully.'
+        @logger.log(:info, "HTML-Proofer finished successfully.")
       else
         @failures.uniq!
         report_failed_checks
@@ -97,40 +98,38 @@ module HTMLProofer
     # Walks over each implemented check and runs them on the files, in parallel.
     def process_files
       if @options[:parallel][:enable]
-        Parallel.map(files, @options[:parallel]) { |path| load_file(path) }
+        Parallel.map(files, @options[:parallel]) { |file| load_file(file[:path], file[:source]) }
       else
-        files.map { |path| load_file(path) }
+        files.map do |file|
+          load_file(file[:path], file[:source])
+        end
       end
     end
 
-    def load_file(path)
+    def load_file(path, source)
       @html = create_nokogiri(path)
-      check_parsed(path)
+      check_parsed(path, source)
     end
 
     # Collects any external URLs found in a directory of files. Also collectes
     # every failed test from process_files.
-    def check_parsed(path)
+    def check_parsed(path, source)
       result = { internal_urls: {}, external_urls: {}, failures: [] }
 
-      @source = [@source] if @type == :file
-
-      @source.each do |current_source|
-        checks.each do |klass|
-          @current_source = current_source
-          @current_path = path
+      checks.each do |klass|
+        @current_source = source
+        @current_filename = path
 
-          check = Object.const_get(klass).new(self, @html)
-          @logger.log :debug, "Running #{check.short_name} in #{path}"
+        check = Object.const_get(klass).new(self, @html)
+        @logger.log(:debug, "Running #{check.short_name} in #{path}")
 
-          @current_check = check
+        @current_check = check
 
-          check.run
+        check.run
 
-          result[:external_urls].merge!(check.external_urls)
-          result[:internal_urls].merge!(check.internal_urls)
-          result[:failures].concat(check.failures)
-        end
+        result[:external_urls].merge!(check.external_urls) { |_key, old, current| old.concat(current) }
+        result[:internal_urls].merge!(check.internal_urls) { |_key, old, current| old.concat(current) }
+        result[:failures].concat(check.failures)
       end
       result
     end
@@ -148,15 +147,17 @@ module HTMLProofer
 
     def files
       @files ||= if @type == :directory
-                   @source.map do |src|
-                     pattern = File.join(src, '**', "*{#{@options[:extensions].join(',')}}")
-                     Dir.glob(pattern).select { |f| File.file?(f) && !ignore_file?(f) }
-                   end.flatten
-                 elsif @type == :file &&  @options[:extensions].include?(File.extname(@source))
-                   [@source].reject { |f| ignore_file?(f) }
-                 else
-                   []
-                 end
+        @source.map do |src|
+          pattern = File.join(src, "**", "*{#{@options[:extensions].join(",")}}")
+          Dir.glob(pattern).select do |f|
+            File.file?(f) && !ignore_file?(f)
+          end.map { |f| { source: src, path: f } }
+        end.flatten
+      elsif @type == :file && @options[:extensions].include?(File.extname(@source))
+        [@source].reject { |f| ignore_file?(f) }.map { |f| { source: f, path: f } }
+      else
+        []
+      end
     end
 
     def ignore_file?(file)
@@ -179,7 +180,7 @@ module HTMLProofer
     def checks
       return @checks if defined?(@checks) && !@checks.nil?
 
-      return (@checks = ['LinkCheck']) if @type == :links
+      return (@checks = ["LinkCheck"]) if @type == :links
 
       @checks = HTMLProofer::Check.subchecks(@options).map(&:name)
 
@@ -193,9 +194,9 @@ module HTMLProofer
     def report_failed_checks
       @reporter.report
 
-      failure_text = pluralize(@failures.length, 'failure', 'failures')
-      @logger.log :fatal, "\nHTML-Proofer found #{failure_text}!"
-      exit 1
+      failure_text = pluralize(@failures.length, "failure", "failures")
+      @logger.log(:fatal, "\nHTML-Proofer found #{failure_text}!")
+      exit(1)
     end
 
     # Set before_request callback.
@@ -227,19 +228,19 @@ module HTMLProofer
 
       existing_urls_count = @cache.size(type)
       cache_text = pluralize(existing_urls_count, "#{type} link", "#{type} links")
-      @logger.log :debug, "Found #{cache_text} in the cache"
+      @logger.log(:debug, "Found #{cache_text} in the cache")
 
       urls_to_check = @cache.retrieve_urls(ivar, type)
       urls_detected = pluralize(urls_to_check.count, "#{type} link", "#{type} links")
-      @logger.log :info, "Checking #{urls_detected}"
+      @logger.log(:info, "Checking #{urls_detected}")
 
       urls_to_check
     end
 
     private def format_checks_list(checks)
       checks.map do |check|
-        check.sub(/HTMLProofer::Check::/, '')
-      end.join(', ')
+        check.sub(/HTMLProofer::Check::/, "")
+      end.join(", ")
     end
   end
 end

data/lib/html_proofer/url_validator/external.rb

@@ -1,188 +1,193 @@
 # frozen_string_literal: true
 
-require 'typhoeus'
-require 'uri'
+require "typhoeus"
+require "uri"
 
 module HTMLProofer
-  class UrlValidator::External < UrlValidator
-    include HTMLProofer::Utils
+  class UrlValidator
+    class External < UrlValidator
+      include HTMLProofer::Utils
 
-    attr_reader :external_urls
-    attr_writer :before_request
+      attr_reader :external_urls
+      attr_writer :before_request
 
-    def initialize(runner, external_urls)
-      super(runner)
+      def initialize(runner, external_urls)
+        super(runner)
 
-      @external_urls = external_urls
-      @hydra = Typhoeus::Hydra.new(@runner.options[:hydra])
-      @before_request = []
+        @external_urls = external_urls
+        @hydra = Typhoeus::Hydra.new(@runner.options[:hydra])
+        @before_request = []
 
-      @paths_with_queries = {}
-    end
-
-    def validate
-      if @cache.enabled?
-        urls_to_check = @runner.load_external_cache
-        run_external_link_checker(urls_to_check)
-      else
-        run_external_link_checker(@external_urls)
+        @paths_with_queries = {}
       end
 
-      @failed_checks
-    end
-
-    # Proofer runs faster if we pull out all the external URLs and run the checks
-    # at the end. Otherwise, we're halting the consuming process for every file during
-    # `process_files`.
-    #
-    # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
-    #
-    # Finally, we'll first make a HEAD request, rather than GETing all the contents.
-    # If the HEAD fails, we'll fall back to GET, as some servers are not configured
-    # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
-    # not available as an option.
-    def run_external_link_checker(external_urls)
-      # Route log from Typhoeus/Ethon to our own logger
-      Ethon.logger = @logger
-
-      external_urls.each_pair do |external_url, metadata|
-        url = Attribute::Url.new(@runner, external_url, base_url: nil)
-
-        unless url.valid?
-          add_failure(metadata, "#{url} is an invalid URL", 0)
-          next
+      def validate
+        if @cache.enabled?
+          urls_to_check = @runner.load_external_cache
+          run_external_link_checker(urls_to_check)
+        else
+          run_external_link_checker(@external_urls)
         end
 
-        next unless new_url_query_values?(url)
-
-        method = if @runner.options[:check_external_hash] && url.hash?
-                   :get
-                 else
-                   :head
-                 end
-
-        queue_request(method, url, metadata)
+        @failed_checks
       end
 
-      @hydra.run
-    end
+      # Proofer runs faster if we pull out all the external URLs and run the checks
+      # at the end. Otherwise, we're halting the consuming process for every file during
+      # `process_files`.
+      #
+      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
+      #
+      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
+      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
+      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
+      # not available as an option.
+      def run_external_link_checker(external_urls)
+        # Route log from Typhoeus/Ethon to our own logger
+        Ethon.logger = @logger
+
+        external_urls.each_pair do |external_url, metadata|
+          url = Attribute::Url.new(@runner, external_url, base_url: nil)
+
+          unless url.valid?
+            add_failure(metadata, "#{url} is an invalid URL", 0)
+            next
+          end
+
+          next unless new_url_query_values?(url)
+
+          method = if @runner.options[:check_external_hash] && url.hash?
+            :get
+          else
+            :head
+          end
+
+          queue_request(method, url, metadata)
+        end
 
-    def queue_request(method, url, filenames)
-      opts = @runner.options[:typhoeus].merge(method: method)
-      request = Typhoeus::Request.new(url.url, opts)
-      @before_request.each do |callback|
-        callback.call(request)
+        @hydra.run
       end
-      request.on_complete { |response| response_handler(response, url, filenames) }
-      @hydra.queue request
-    end
 
-    def response_handler(response, url, filenames)
-      method = response.request.options[:method]
-      href = response.request.base_url.to_s
-      response_code = response.code
-      response.body.delete!("\x00")
-
-      @logger.log :debug, "Received a #{response_code} for #{href}"
-
-      return if @runner.options[:ignore_status_codes].include?(response_code)
-
-      if response_code.between?(200, 299)
-        @cache.add_external(href, filenames, response_code, 'OK') unless check_hash_in_2xx_response(href, url, response, filenames)
-      elsif response.timed_out?
-        handle_timeout(href, filenames, response_code)
-      elsif response_code.zero?
-        handle_connection_failure(href, filenames, response_code, response.status_message)
-      elsif method == :head # some servers don't support HEAD
-        queue_request(:get, url, filenames)
-      else
-        return if @runner.options[:only_4xx] && !response_code.between?(400, 499)
-
-        # Received a non-successful http response.
-        status_message = blank?(response.status_message) ? '' : ": #{response.status_message}"
-        msg = "External link #{href} failed#{status_message}"
-        add_failure(filenames, msg, response_code)
-        @cache.add_external(href, filenames, response_code, msg)
+      def queue_request(method, url, filenames)
+        opts = @runner.options[:typhoeus].merge(method: method)
+        request = Typhoeus::Request.new(url.url, opts)
+        @before_request.each do |callback|
+          callback.call(request)
+        end
+        request.on_complete { |response| response_handler(response, url, filenames) }
+        @hydra.queue(request)
       end
-    end
 
-    # Even though the response was a success, we may have been asked to check
-    # if the hash on the URL exists on the page
-    def check_hash_in_2xx_response(href, url, response, filenames)
-      return false if @runner.options[:only_4xx]
-      return false unless @runner.options[:check_external_hash]
-      return false unless url.hash?
-
-      hash = url.hash
-
-      body_doc = create_nokogiri(response.body)
-
-      unencoded_hash = Addressable::URI.unescape(hash)
-      xpath = [%(//*[@name="#{hash}"]|/*[@name="#{unencoded_hash}"]|//*[@id="#{hash}"]|//*[@id="#{unencoded_hash}"])]
-      # user-content is a special addition by GitHub.
-      if url.host =~ /github\.com/i
-        xpath << [%(//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])]
-        # when linking to a file on GitHub, like #L12-L34, only the first "L" portion
-        # will be identified as a linkable portion
-        xpath << [%(//td[@id="#{Regexp.last_match[1]}"])] if hash =~ /\A(L\d)+/
+      def response_handler(response, url, filenames)
+        method = response.request.options[:method]
+        href = response.request.base_url.to_s
+        response_code = response.code
+        response.body.delete!("\x00")
+
+        @logger.log(:debug, "Received a #{response_code} for #{href}")
+
+        return if @runner.options[:ignore_status_codes].include?(response_code)
+
+        if response_code.between?(200, 299)
+          @cache.add_external(href, filenames, response_code, "OK") unless check_hash_in_2xx_response(href, url,
+            response, filenames)
+        elsif response.timed_out?
+          handle_timeout(href, filenames, response_code)
+        elsif response_code.zero?
+          handle_connection_failure(href, filenames, response_code, response.status_message)
+        elsif method == :head # some servers don't support HEAD
+          queue_request(:get, url, filenames)
+        else
+          return if @runner.options[:only_4xx] && !response_code.between?(400, 499)
+
+          # Received a non-successful http response.
+          status_message = blank?(response.status_message) ? "" : ": #{response.status_message}"
+          msg = "External link #{href} failed#{status_message}"
+          add_failure(filenames, msg, response_code)
+          @cache.add_external(href, filenames, response_code, msg)
+        end
       end
 
-      return unless body_doc.xpath(xpath.join('|')).empty?
+      # Even though the response was a success, we may have been asked to check
+      # if the hash on the URL exists on the page
+      def check_hash_in_2xx_response(href, url, response, filenames)
+        return false if @runner.options[:only_4xx]
+        return false unless @runner.options[:check_external_hash]
+        return false unless url.hash?
+
+        hash = url.hash
+
+        body_doc = create_nokogiri(response.body)
+
+        unencoded_hash = Addressable::URI.unescape(hash)
+        xpath = [%(//*[@name="#{hash}"]|/*[@name="#{unencoded_hash}"]|//*[@id="#{hash}"]|//*[@id="#{unencoded_hash}"])]
+        # user-content is a special addition by GitHub.
+        if url.host =~ /github\.com/i
+          xpath << [%(//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])]
+          # when linking to a file on GitHub, like #L12-L34, only the first "L" portion
+          # will be identified as a linkable portion
+          xpath << [%(//td[@id="#{Regexp.last_match[1]}"])] if hash =~ /\A(L\d)+/
+        end
 
-      msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
-      add_failure(filenames, msg, response.code)
-      @cache.add_external(href, filenames, response.code, msg)
-      true
-    end
+        return unless body_doc.xpath(xpath.join("|")).empty?
 
-    def handle_timeout(href, filenames, response_code)
-      msg = "External link #{href} failed: got a time out (response code #{response_code})"
-      @cache.add_external(href, filenames, 0, msg)
-      return if @runner.options[:only_4xx]
+        msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
+        add_failure(filenames, msg, response.code)
+        @cache.add_external(href, filenames, response.code, msg)
+        true
+      end
 
-      add_failure(filenames, msg, response_code)
-    end
+      def handle_timeout(href, filenames, response_code)
+        msg = "External link #{href} failed: got a time out (response code #{response_code})"
+        @cache.add_external(href, filenames, 0, msg)
+        return if @runner.options[:only_4xx]
 
-    def handle_connection_failure(href, metadata, response_code, status_message)
-      msgs = [<<~MSG
-        External link #{href} failed with something very wrong.
-        It's possible libcurl couldn't connect to the server, or perhaps the request timed out.
-        Sometimes, making too many requests at once also breaks things.
-      MSG
-      ]
+        add_failure(filenames, msg, response_code)
+      end
 
-      msgs << "Either way, the return message from the server is: #{status_message}" unless blank?(status_message)
+      def handle_connection_failure(href, metadata, response_code, status_message)
+        msgs = [<<~MSG,
+          External link #{href} failed with something very wrong.
+          It's possible libcurl couldn't connect to the server, or perhaps the request timed out.
+          Sometimes, making too many requests at once also breaks things.
+        MSG
+        ]
 
-      msg = msgs.join("\n").chomp
+        msgs << "Either way, the return message from the server is: #{status_message}" unless blank?(status_message)
 
-      @cache.add_external(href, metadata, 0, msg)
-      return if @runner.options[:only_4xx]
+        msg = msgs.join("\n").chomp
 
-      add_failure(metadata, msg, response_code)
-    end
+        @cache.add_external(href, metadata, 0, msg)
+        return if @runner.options[:only_4xx]
 
-    def add_failure(metadata, description, status = nil)
-      if blank?(metadata) # possible if we're checking an array of links
-        @failed_checks << Failure.new('', 'Links > External', description, status: status)
-      else
-        metadata.each { |m| @failed_checks << Failure.new(m[:filename], 'Links > External', description, line: m[:line], status: status) }
+        add_failure(metadata, msg, response_code)
       end
-    end
 
-    # remember queries we've seen, ignore future ones
-    private def new_url_query_values?(url)
-      return true if (query_values = url.query_values).nil?
+      def add_failure(metadata, description, status = nil)
+        if blank?(metadata) # possible if we're checking an array of links
+          @failed_checks << Failure.new("", "Links > External", description, status: status)
+        else
+          metadata.each do |m|
+            @failed_checks << Failure.new(m[:filename], "Links > External", description, line: m[:line], status: status)
+          end
+        end
+      end
 
-      queries = query_values.keys.join('-')
-      domain_path = url.domain_path
-      if @paths_with_queries[domain_path].nil?
-        @paths_with_queries[domain_path] = [queries]
-        true
-      elsif !@paths_with_queries[domain_path].include?(queries)
-        @paths_with_queries[domain_path] << queries
-        true
-      else
-        false
+      # remember queries we've seen, ignore future ones
+      private def new_url_query_values?(url)
+        return true if (query_values = url.query_values).nil?
+
+        queries = query_values.keys.join("-")
+        domain_path = url.domain_path
+        if @paths_with_queries[domain_path].nil?
+          @paths_with_queries[domain_path] = [queries]
+          true
+        elsif !@paths_with_queries[domain_path].include?(queries)
+          @paths_with_queries[domain_path] << queries
+          true
+        else
+          false
+        end
       end
     end
   end