html-proofer 3.15.0 → 3.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 722b9c6547474c7a4654e46a3c7af273eaa8a77b7538592ab0df4b42dd787ea2
-  data.tar.gz: 240979d4d2d62dbdd76ab14d868c8c0ecf420e10c93ae1cdc5c8182106fef698
+  metadata.gz: 9b5a47189e872130e01a2080e6e8ddb1f7f22520098deaf941960194d2338b2b
+  data.tar.gz: 3543c42860956427e8c828861f76e1c5cd984d79054c091c6e5f97e0352b3137
 SHA512:
-  metadata.gz: 483c149a8b502277cb3d5d02f3aa4e009c27b1ef1a0c9f775b27410515a6654a013f434d018933c8991227d3826648902f0831613efddf3523e031a49dd586f8
-  data.tar.gz: 823bc37d94226a5dfed9d0db49ae6b8803ff74a719e8d3332b08d934bdaaa2c83374b41b5591fd6a714b6f8bca41a65bc6fa0ea099f6eadb5b17176ed5f61c56
+  metadata.gz: 1defd1fb2ee651015231b0ed4d1407ed56e25a907e0602ee9e175b23f3b88938e020f438d2171ceea8d52bc58984901543c97184a3da13c9b08cdaf62200a4aa
+  data.tar.gz: 97eb6120db724822830398a85f14faa4415638e965d21aef0703991e7a14d335f32df65d72bb877be6f0706f24154deea3f08a6bbeb2cdf5d7c392d03ce2af23

data/bin/htmlproofer

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-STDOUT.sync = true
+$stdout.sync = true
 
 $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 
@@ -41,6 +41,8 @@ Mercenary.program(:htmlproofer) do |p|
   p.option 'report_missing_names', '--report-missing-names', 'When `check_html` is enabled, HTML markup that are missing entity names are reported as errors (default: `false`)'
   p.option 'report_script_embeds', '--report-script-embeds', 'When `check_html` is enabled, `script` tags containing markup are reported as errors (default: `false`)'
   p.option 'report_missing_doctype', '--report-missing-doctype', 'When `check_html` is enabled, HTML markup with missing or out-of-order `DOCTYPE` are reported as errors (default: `false`)'
+  p.option 'report_eof_tags', '--report-eof-tags', 'When `check_html` is enabled, HTML markup with tags that are malformed are reported as errors (default: `false`)'
+  p.option 'report_mismatched_tags', '--report-mismatched-tags', 'When `check_html` is enabled, HTML markup with mismatched tags are reported as errors (default: `false`)'
   p.option 'log_level', '--log-level <level>', String, 'Sets the logging level, as determined by Yell. One of `:debug`, `:info`, `:warn`, `:error`, or `:fatal`. (default: `:info`)'
   p.option 'only_4xx', '--only-4xx', 'Only reports errors for links that fall within the 4xx status code range'
   p.option 'storage_dir', '--storage-dir PATH', String, 'Directory where to store the cache log (default: "tmp/.htmlproofer")'
@@ -82,6 +84,8 @@ Mercenary.program(:htmlproofer) do |p|
     options[:validation][:report_missing_names] = opts['report_missing_names'] unless opts['report_missing_names'].nil?
     options[:validation][:report_invalid_tags] = opts['report_invalid_tags'] unless opts['report_invalid_tags'].nil?
     options[:validation][:report_missing_doctype] = opts['report_missing_doctype'] unless opts['report_missing_doctype'].nil?
+    options[:validation][:report_eof_tags] = opts['report_eof_tags'] unless opts['report_eof_tags'].nil?
+    options[:validation][:report_mismatched_tags] = opts['report_mismatched_tags'] unless opts['report_mismatched_tags'].nil?
 
     options[:typhoeus] = HTMLProofer::Configuration.parse_json_option('typhoeus_config', opts['typhoeus_config']) unless opts['typhoeus_config'].nil?
 

data/lib/html-proofer.rb

@@ -2,7 +2,7 @@
 
 def require_all(path)
   dir = File.join(File.dirname(__FILE__), path)
-  Dir[File.join(dir, '*.rb')].each do |f|
+  Dir[File.join(dir, '*.rb')].sort.each do |f|
     require f
   end
 end

data/lib/html-proofer/cache.rb

@@ -59,6 +59,8 @@ module HTMLProofer
     end
 
     def add(url, filenames, status, msg = '')
+      return unless use_cache?
+
       data = {
         time: @cache_time,
         filenames: filenames,
@@ -92,12 +94,12 @@ module HTMLProofer
       del = 0
       @cache_log.delete_if do |url, _|
         url = clean_url(url)
-        if !found_urls.include?(url)
+        if found_urls.include?(url)
+          false
+        else
           @logger.log :debug, "Removing #{url} from cache check"
           del += 1
           true
-        else
-          false
         end
       end
 
@@ -115,14 +117,12 @@ module HTMLProofer
       @load.nil?
     end
 
-    def retrieve_urls(external_urls)
-      urls_to_check = detect_url_changes(external_urls)
+    def retrieve_urls(urls)
+      urls_to_check = detect_url_changes(urls)
       @cache_log.each_pair do |url, cache|
-        if within_timeframe?(cache['time'])
-          next if cache['message'].empty? # these were successes to skip
-        else
-          urls_to_check[url] = cache['filenames'] # recheck expired links
-        end
+        next if within_timeframe?(cache['time']) && cache['message'].empty? # these were successes to skip
+
+        urls_to_check[url] = cache['filenames'] # recheck expired links
       end
       urls_to_check
     end

data/lib/html-proofer/check.rb

@@ -3,20 +3,23 @@
 module HTMLProofer
   # Mostly handles issue management and collecting of external URLs.
   class Check
-    attr_reader :node, :html, :element, :src, :path, :options, :issues, :external_urls
+    attr_reader :node, :html, :element, :src, :path, :options, :issues, :internal_urls, :external_urls
 
-    def initialize(src, path, html, options)
+    def initialize(src, path, html, logger, cache, options)
       @src    = src
       @path   = path
       @html   = remove_ignored(html)
+      @logger = logger
+      @cache = cache
       @options = options
       @issues = []
+      @internal_urls = {}
       @external_urls = {}
     end
 
     def create_element(node)
       @node = node
-      Element.new(node, self)
+      Element.new(node, self, @logger)
     end
 
     def run
@@ -25,15 +28,20 @@ module HTMLProofer
 
     def add_issue(desc, line: nil, status: -1, content: nil)
       @issues << Issue.new(@path, desc, line: line, status: status, content: content)
+      false
+    end
+
+    def add_to_internal_urls(url, internal_url)
+      if @internal_urls[url]
+        @internal_urls[url] << internal_url
+      else
+        @internal_urls[url] = [internal_url]
+      end
     end
 
     def add_to_external_urls(url)
       return if @external_urls[url]
 
-      add_path_for_url(url)
-    end
-
-    def add_path_for_url(url)
       if @external_urls[url]
         @external_urls[url] << @path
       else

data/lib/html-proofer/check/html.rb

@@ -6,7 +6,9 @@ class HtmlCheck < ::HTMLProofer::Check
   INVALID_TAG_MSG = /Tag ([\w\-:]+) invalid/.freeze
   INVALID_PREFIX = /Namespace prefix/.freeze
   PARSE_ENTITY_REF = /htmlParseEntityRef: no name/.freeze
-  DOCTYPE_MSG = /The doctype must be the first token in the document/.freeze
+  DOCTYPE_MSG = /Expected a doctype token/.freeze
+  EOF_IN_TAG = /End of input in tag/.freeze
+  MISMATCHED_TAGS = /That tag isn't allowed here/.freeze
 
   def run
     @html.errors.each do |error|
@@ -24,6 +26,10 @@ class HtmlCheck < ::HTMLProofer::Check
       options[:validation][:report_missing_names]
     when DOCTYPE_MSG
       options[:validation][:report_missing_doctype]
+    when EOF_IN_TAG
+      options[:validation][:report_eof_tags]
+    when MISMATCHED_TAGS
+      options[:validation][:report_mismatched_tags]
     else
       true
     end

data/lib/html-proofer/check/links.rb

@@ -4,6 +4,8 @@ class LinkCheck < ::HTMLProofer::Check
   include HTMLProofer::Utils
 
   def missing_href?
+    return blank?(@link.src) if @node.name == 'source'
+
     blank?(@link.href) && blank?(@link.name) && blank?(@link.id)
   end
 
@@ -12,7 +14,7 @@ class LinkCheck < ::HTMLProofer::Check
   end
 
   def run
-    @html.css('a, link').each do |node|
+    @html.css('a, link, source').each do |node|
       @link = create_element(node)
       line = node.line
       content = node.to_s
@@ -49,23 +51,31 @@ class LinkCheck < ::HTMLProofer::Check
         # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx
         next if @link.respond_to?(:rel) && @link.rel == 'dns-prefetch'
 
-        add_to_external_urls(@link.href)
+        add_to_external_urls(@link.href || @link.src)
         next
-      elsif @link.internal? && !@link.exists?
-        add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content)
+      elsif @link.internal?
+        if @link.exists?
+          add_to_internal_urls(@link.href, InternalLink.new(@link, @path, line, content))
+        else
+          add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content)
+        end
       end
+    end
 
-      # does the local directory have a trailing slash?
-      if @link.unslashed_directory? @link.absolute_path
-        add_issue("internally linking to a directory #{@link.absolute_path} without trailing slash", line: line, content: content)
-        next
-      end
+    external_urls
+  end
 
-      # verify the target hash
-      handle_hash(@link, line, content) if @link.hash
+  def check_internal_link(link, line, content)
+    # does the local directory have a trailing slash?
+    if link.unslashed_directory?(link.absolute_path)
+      add_issue("internally linking to a directory #{link.absolute_path} without trailing slash", line: line, content: content)
+      return false
     end
 
-    external_urls
+    # verify the target hash
+    return handle_hash(link, line, content) if link.hash
+
+    true
   end
 
   def check_schemes(link, line, content)
@@ -94,23 +104,27 @@ class LinkCheck < ::HTMLProofer::Check
   end
 
   def handle_hash(link, line, content)
-    if link.internal? && !hash_check(link.html, link.hash)
-      add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content)
+    if link.internal? && !hash_exists?(link.html, link.hash) # rubocop:disable Style/GuardClause
+      return add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content)
     elsif link.external?
-      external_link_check(link, line, content)
+      return external_link_check(link, line, content)
     end
+
+    true
   end
 
   def external_link_check(link, line, content)
-    if !link.exists?
-      add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content)
+    if link.exists? # rubocop:disable Style/GuardClause
+      target_html = create_nokogiri(link.absolute_path)
+      return add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_exists?(target_html, link.hash)
     else
-      target_html = create_nokogiri link.absolute_path
-      add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_check target_html, link.hash
+      return add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content)
     end
+
+    true
   end
 
-  def hash_check(html, href_hash)
+  def hash_exists?(html, href_hash)
     decoded_href_hash = Addressable::URI.unescape(href_hash)
     fragment_ids = [href_hash, decoded_href_hash]
     # https://www.w3.org/TR/html5/single-page.html#scroll-to-fragid
@@ -148,7 +162,19 @@ class LinkCheck < ::HTMLProofer::Check
 
   class XpathFunctions
     def case_sensitive_equals(node_set, str_to_match)
-      node_set.find_all { |node| node.to_s. == str_to_match.to_s }
+      node_set.find_all { |node| node.to_s.== str_to_match.to_s }
+    end
+  end
+
+  class InternalLink
+    attr_reader :link, :href, :path, :line, :content
+
+    def initialize(link, path, line, content)
+      @link = link
+      @href = @link.href
+      @path = path
+      @line = line
+      @content = content
     end
   end
 end

data/lib/html-proofer/check/opengraph.rb

@@ -3,12 +3,12 @@
 class OpenGraphElement < ::HTMLProofer::Element
   attr_reader :src
 
-  def initialize(obj, check)
-    super(obj, check)
+  def initialize(obj, check, logger)
+    super(obj, check, logger)
     # Fake up src from the content attribute
     instance_variable_set('@src', @content)
 
-    @src.insert 0, 'http:' if @src =~ %r{^//}
+    @src.insert 0, 'http:' if %r{^//}.match?(@src)
   end
 end
 
@@ -23,7 +23,7 @@ class OpenGraphCheck < ::HTMLProofer::Check
 
   def run
     @html.css('meta[property="og:url"], meta[property="og:image"]').each do |m|
-      @opengraph = OpenGraphElement.new(m, self)
+      @opengraph = OpenGraphElement.new(m, self, @logger)
 
       next if @opengraph.ignore?
 

data/lib/html-proofer/configuration.rb

@@ -52,7 +52,9 @@ module HTMLProofer
       report_script_embeds: false,
       report_missing_names: false,
       report_invalid_tags: false,
-      report_missing_doctype: false
+      report_missing_doctype: false,
+      report_eof_tags: false,
+      report_mismatched_tags: false
     }.freeze
 
     CACHE_DEFAULTS = {}.freeze
@@ -78,7 +80,7 @@ module HTMLProofer
       begin
         JSON.parse(config)
       rescue StandardError
-        raise ArgumentError, "Option '" + option_name + "' did not contain valid JSON."
+        raise ArgumentError, "Option '#{option_name} did not contain valid JSON."
       end
     end
   end

data/lib/html-proofer/element.rb

@@ -10,12 +10,18 @@ module HTMLProofer
 
     attr_reader :id, :name, :alt, :href, :link, :src, :line, :data_proofer_ignore
 
-    def initialize(obj, check)
+    def initialize(obj, check, logger)
+      @logger = logger
       # Construct readable ivars for every element
-      obj.attributes.each_pair do |attribute, value|
-        name = attribute.tr('-:.', '_').to_s.to_sym
-        (class << self; self; end).send(:attr_reader, name)
-        instance_variable_set("@#{name}", value.value)
+      begin
+        obj.attributes.each_pair do |attribute, value|
+          name = attribute.tr('-:.;@', '_').to_s.to_sym
+          (class << self; self; end).send(:attr_reader, name)
+          instance_variable_set("@#{name}", value.value)
+        end
+      rescue NameError => e
+        @logger.log :error, "Attribute set `#{obj}` contains an error!"
+        raise e
       end
 
       @aria_hidden = defined?(@aria_hidden) && @aria_hidden == 'true' ? true : false
@@ -36,19 +42,19 @@ module HTMLProofer
 
       # fix up missing protocols
       if defined?(@href)
-        @href.insert(0, 'http:') if @href =~ %r{^//}
+        @href.insert(0, 'http:') if %r{^//}.match?(@href)
       else
         @href = nil
       end
 
       if defined?(@src)
-        @src.insert(0, 'http:') if @src =~ %r{^//}
+        @src.insert(0, 'http:') if %r{^//}.match?(@src)
       else
         @src = nil
       end
 
       if defined?(@srcset)
-        @srcset.insert(0, 'http:') if @srcset =~ %r{^//}
+        @srcset.insert(0, 'http:') if %r{^//}.match?(@srcset)
       else
         @srcset = nil
       end
@@ -99,12 +105,10 @@ module HTMLProofer
       return true if @data_proofer_ignore
       return true if @parent_ignorable
 
-      return true if url =~ /^javascript:/
+      return true if /^javascript:/.match?(url)
 
       # ignore base64 encoded images
-      if %w[ImageCheck FaviconCheck].include? @type
-        return true if url =~ /^data:image/
-      end
+      return true if %w[ImageCheck FaviconCheck].include?(@type) && /^data:image/.match?(url)
 
       # ignore user defined URLs
       return true if ignores_pattern_check(@check.options[:url_ignore])
@@ -172,7 +176,7 @@ module HTMLProofer
 
       path_dot_ext = path + @check.options[:extension] if @check.options[:assume_extension]
 
-      if path =~ %r{^/} # path relative to root
+      if %r{^/}.match?(path) # path relative to root
         if File.directory?(@check.src)
           base = @check.src
         else
@@ -181,7 +185,7 @@ module HTMLProofer
         end
       elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file
         base = File.dirname @check.path
-      elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # relative links in nested dir, path is a file
+      elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file
         base = File.dirname @check.path
       else # relative link, path is a directory
         base = @check.path
@@ -211,10 +215,11 @@ module HTMLProofer
 
     def ignores_pattern_check(links)
       links.each do |ignore|
-        if ignore.is_a? String
+        case ignore
+        when String
           return true if ignore == url
-        elsif ignore.is_a? Regexp
-          return true if ignore =~ url
+        when Regexp
+          return true if ignore&.match?(url)
         end
       end
 

data/lib/html-proofer/log.rb

@@ -7,16 +7,27 @@ module HTMLProofer
   class Log
     include Yell::Loggable
 
+    STDOUT_LEVELS = %i[debug info warn].freeze
+    STDERR_LEVELS = %i[error fatal].freeze
+
     def initialize(log_level)
       @logger = Yell.new(format: false, \
                          name: 'HTMLProofer', \
                          level: "gte.#{log_level}") do |l|
-        l.adapter :stdout, level: %i[debug info warn]
-        l.adapter :stderr, level: %i[error fatal]
+        l.adapter :stdout, level: 'lte.warn'
+        l.adapter :stderr, level: 'gte.error'
       end
     end
 
     def log(level, message)
+      log_with_color(level, message)
+    end
+
+    def log_with_color(level, message)
+      @logger.send level, colorize(level, message)
+    end
+
+    def colorize(level, message)
       color = case level
               when :debug
                 :cyan
@@ -28,15 +39,8 @@ module HTMLProofer
                 :red
               end
 
-      log_with_color(level, color, message)
-    end
-
-    def log_with_color(level, color, message)
-      @logger.send level, colorize(color, message)
-    end
-
-    def colorize(color, message)
-      if $stdout.isatty && $stderr.isatty
+      if (STDOUT_LEVELS.include?(level) && $stdout.isatty) || \
+         (STDERR_LEVELS.include?(level) && $stderr.isatty)
         Rainbow(message).send(color)
       else
         message

data/lib/html-proofer/middleware.rb

@@ -6,6 +6,7 @@ module HTMLProofer
 
     class InvalidHtmlError < StandardError
       def initialize(failures)
+        super
         @failures = failures
       end
 
@@ -21,7 +22,8 @@ module HTMLProofer
         allow_hash_href: true,
         check_external_hash: true,
         check_html: true,
-        url_ignore: [/.*/] # Don't try to check local files exist
+        url_ignore: [/.*/], # Don't try to check if local files exist
+        validation: { report_eof_tags: true }
       }
     end
 
@@ -52,7 +54,7 @@ module HTMLProofer
     def call(env)
       result = @app.call(env)
       return result if env['REQUEST_METHOD'] != 'GET'
-      return result if env['QUERY_STRING'] =~ /proofer-ignore/
+      return result if /proofer-ignore/.match?(env['QUERY_STRING'])
       return result if result.first != 200
 
       body = []

data/lib/html-proofer/runner.rb

@@ -4,7 +4,7 @@ module HTMLProofer
   class Runner
     include HTMLProofer::Utils
 
-    attr_reader :options, :external_urls, :failures
+    attr_reader :options, :internal_urls, :external_urls, :failures
 
     def initialize(src, opts = {})
       @src = src
@@ -20,6 +20,8 @@ module HTMLProofer
 
       @type = @options.delete(:type)
       @logger = HTMLProofer::Log.new(@options[:log_level])
+      @cache = Cache.new(@logger, @options[:cache])
+      @internal_link_checks = nil
 
       # Add swap patterns for internal domains
       unless @options[:internal_domains].empty?
@@ -30,7 +32,11 @@ module HTMLProofer
         end
       end
 
+      @internal_urls = {}
+      @internal_urls_to_paths = {}
+      @external_urls = {}
       @failures = []
+      @before_request = []
     end
 
     def run
@@ -45,7 +51,7 @@ module HTMLProofer
       end
 
       if @failures.empty?
-        @logger.log_with_color :info, :green, 'HTML-Proofer finished successfully.'
+        @logger.log :info, 'HTML-Proofer finished successfully.'
       else
         print_failed_tests
       end
@@ -58,15 +64,13 @@ module HTMLProofer
         end
       end
       @external_urls = Hash[*@src.map { |s| [s, nil] }.flatten]
-      validate_urls
+      validate_external_urls
     end
 
     # Collects any external URLs found in a directory of files. Also collectes
     # every failed test from process_files.
     # Sends the external URLs to Typhoeus for batch processing.
     def check_files
-      @external_urls = {}
-
       process_files.each do |item|
         @external_urls.merge!(item[:external_urls])
         @failures.concat(item[:failures])
@@ -77,9 +81,12 @@ module HTMLProofer
       # just not run those other checks at all.
       if @options[:external_only]
         @failures = []
-        validate_urls
+        validate_external_urls
       elsif !@options[:disable_external]
-        validate_urls
+        validate_external_urls
+        validate_internal_urls
+      else
+        validate_internal_urls
       end
     end
 
@@ -100,8 +107,21 @@ module HTMLProofer
       @src.each do |src|
         checks.each do |klass|
           @logger.log :debug, "Checking #{klass.to_s.downcase} on #{path} ..."
-          check = Object.const_get(klass).new(src, path, html, @options)
+          check = Object.const_get(klass).new(src, path, html, @logger, @cache, @options)
           check.run
+
+          if klass == 'LinkCheck'
+            @internal_link_checks = check
+            check.internal_urls.each_pair do |url, internal_urls|
+              if @internal_urls_to_paths[url]
+                @internal_urls_to_paths[url].concat(internal_urls.map(&:path))
+              else
+                @internal_urls_to_paths[url] = internal_urls.map(&:path)
+              end
+            end
+            @internal_urls.merge!(check.internal_urls)
+          end
+
           external_urls = check.external_urls
           external_urls = Hash[check.external_urls.map { |url, file| [swap(url, @options[:url_swap]), file] }] if @options[:url_swap]
           result[:external_urls].merge!(external_urls)
@@ -112,15 +132,35 @@ module HTMLProofer
     end
 
     def check_path(path)
-      check_parsed create_nokogiri(path), path
+      check_parsed(create_nokogiri(path), path)
     end
 
-    def validate_urls
-      url_validator = HTMLProofer::UrlValidator.new(@logger, @external_urls, @options)
+    def validate_external_urls
+      url_validator = HTMLProofer::UrlValidator.new(@logger, @cache, @external_urls, @options)
+      url_validator.before_request = @before_request
       @failures.concat(url_validator.run)
       @external_urls = url_validator.external_urls
     end
 
+    def validate_internal_urls
+      if @cache.use_cache?
+        urls_to_check = load_internal_cache
+
+        urls_to_check.each_pair do |url, internal_urls|
+          result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.line, internal_urls.first.content)
+          code = result ? 200 : 404
+          @cache.add(url, @internal_urls_to_paths[url].sort, code, '') # TODO: blank msg for now
+        end
+        @cache.write
+      else
+        @internal_urls.values.flatten.each do |internal_url|
+          @internal_link_checks.check_internal_link(internal_url.link, internal_url.line, internal_url.content)
+        end
+      end
+
+      @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero?
+    end
+
     def files
       @files ||= if @type == :directory
                    @src.map do |src|
@@ -147,6 +187,8 @@ module HTMLProofer
     def checks
       return @checks if defined?(@checks) && !@checks.nil?
 
+      return (@checks = ['LinkCheck']) if @type == :links
+
       @checks = HTMLProofer::Check.subchecks.map(&:name)
       @checks.delete('FaviconCheck') unless @options[:check_favicon]
       @checks.delete('HtmlCheck') unless @options[:check_html]
@@ -169,7 +211,31 @@ module HTMLProofer
       sorted_failures.sort_and_report
       count = @failures.length
       failure_text = pluralize(count, 'failure', 'failures')
-      raise @logger.colorize :red, "HTML-Proofer found #{failure_text}!"
+      raise @logger.colorize :fatal, "HTML-Proofer found #{failure_text}!"
+    end
+
+    # Set before_request callback.
+    #
+    # @example Set before_request.
+    #   request.before_request { |request| p "yay" }
+    #
+    # @param [ Block ] block The block to execute.
+    #
+    # @yield [ Typhoeus::Request ]
+    #
+    # @return [ Array<Block> ] All before_request blocks.
+    def before_request(&block)
+      @before_request ||= []
+      @before_request << block if block_given?
+      @before_request
+    end
+
+    def load_internal_cache
+      urls_to_check = @cache.retrieve_urls(@internal_urls)
+      cache_text = pluralize(urls_to_check.count, 'internal link', 'internal links')
+      @logger.log :info, "Found #{cache_text} in the cache..."
+
+      urls_to_check
     end
   end
 end

data/lib/html-proofer/url_validator.rb

@@ -10,21 +10,23 @@ module HTMLProofer
     include HTMLProofer::Utils
 
     attr_reader :external_urls
+    attr_writer :before_request
 
-    def initialize(logger, external_urls, options)
+    def initialize(logger, cache, external_urls, options)
       @logger = logger
       @external_urls = external_urls
       @failed_tests = []
       @options = options
       @hydra = Typhoeus::Hydra.new(@options[:hydra])
-      @cache = Cache.new(@logger, @options[:cache])
+      @cache = cache
+      @before_request = []
     end
 
     def run
       @external_urls = remove_query_values
 
       if @cache.use_cache?
-        urls_to_check = load_cache
+        urls_to_check = @cache.retrieve_urls(@external_urls)
         external_link_checker(urls_to_check)
         @cache.write
       else
@@ -41,11 +43,11 @@ module HTMLProofer
       iterable_external_urls = @external_urls.dup
       @external_urls.each_key do |url|
         uri = begin
-                Addressable::URI.parse(url)
-              rescue URI::Error, Addressable::URI::InvalidURIError
-                @logger.log :error, "#{url} is an invalid URL"
-                nil
-              end
+          Addressable::URI.parse(url)
+        rescue URI::Error, Addressable::URI::InvalidURIError
+          @logger.log :error, "#{url} is an invalid URL"
+          nil
+        end
         next if uri.nil? || uri.query.nil?
 
         iterable_external_urls.delete(url) unless new_url_query_values?(uri, paths_with_queries)
@@ -72,15 +74,6 @@ module HTMLProofer
       uri.host + uri.path
     end
 
-    def load_cache
-      cache_count = @cache.size
-      cache_text = pluralize(cache_count, 'link', 'links')
-
-      @logger.log :info, "Found #{cache_text} in the cache..."
-
-      @cache.retrieve_urls(@external_urls)
-    end
-
     # Proofer runs faster if we pull out all the external URLs and run the checks
     # at the end. Otherwise, we're halting the consuming process for every file during
     # `process_files`.
@@ -109,11 +102,11 @@ module HTMLProofer
     def establish_queue(external_urls)
       external_urls.each_pair do |url, filenames|
         url = begin
-                 clean_url(url)
-              rescue URI::Error, Addressable::URI::InvalidURIError
-                add_external_issue(filenames, "#{url} is an invalid URL")
-                next
-               end
+          clean_url(url)
+        rescue URI::Error, Addressable::URI::InvalidURIError
+          add_external_issue(filenames, "#{url} is an invalid URL")
+          next
+        end
 
         method = if hash?(url) && @options[:check_external_hash]
                    :get
@@ -127,16 +120,19 @@ module HTMLProofer
     def clean_url(href)
       # catch any obvious issues, like strings in port numbers
       parsed = Addressable::URI.parse(href)
-      if href !~ /^([!#{$&}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/
-        parsed.normalize
-      else
+      if href =~ /^([!#{$&}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/
         href
+      else
+        parsed.normalize
       end
     end
 
     def queue_request(method, href, filenames)
       opts = @options[:typhoeus].merge(method: method)
       request = Typhoeus::Request.new(href, opts)
+      @before_request.each do |callback|
+        callback.call(request)
+      end
       request.on_complete { |response| response_handler(response, filenames) }
       @hydra.queue request
     end

data/lib/html-proofer/utils.rb

@@ -5,7 +5,7 @@ require 'nokogumbo'
 module HTMLProofer
   module Utils
     def pluralize(count, single, plural)
-      "#{count} #{(count == 1 ? single : plural)}"
+      "#{count} #{count == 1 ? single : plural}"
     end
 
     def create_nokogiri(path)
@@ -15,7 +15,7 @@ module HTMLProofer
                   path
                 end
 
-      Nokogiri::HTML5(content)
+      Nokogiri::HTML5(content, max_errors: -1)
     end
 
     def swap(href, replacement)

data/lib/html-proofer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = '3.15.0'
+  VERSION = '3.17.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html-proofer
 version: !ruby/object:Gem::Version
-  version: 3.15.0
+  version: 3.17.0
 platform: ruby
 authors:
 - Garen Torikian
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-13 00:00:00.000000000 Z
+date: 2020-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -293,23 +293,23 @@ homepage: https://github.com/gjtorikian/html-proofer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: A set of tests to validate your HTML output. These tests check if your image
   references are legitimate, if they have alt tags, if your internal links are working,