html-proofer 3.14.1 → 3.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e652993805b4e62434a27fd09f20936c71d1abfa2588e1ea991ed00d5174cfa4
-  data.tar.gz: 1e4feee6f61dbadd8ed0d91e25950c0f643e3f29ee1cf881fa9bde6f37854901
+  metadata.gz: 39ffac5e6c23ab7b75d99dd69779be2109ed8b02dc4a5cb0a33740119cb66766
+  data.tar.gz: cdaf142dfde936fa1d88d410499c04fdbbb3b9814bc00ba6800ff3edb8acee50
 SHA512:
-  metadata.gz: a6955c42ec89662b66b83422555e0d43c837f2d11f9e67bf85f587b2301d7c24f6f0275e16b31c904ee9da2999dde8583994d532f5be7a2b2f838dda2dab6e1b
-  data.tar.gz: d261d16b8fcd86ceeac73eac6594b3c30a589015b5400ad785e879ce8b77ad7793fd85609063b6b57ffb08333ac07e1982a1b7cae449ab3e7cc5e0f1c36d6e5d
+  metadata.gz: 34a4253b2eb9674060583c0f9006f9c36204bfd8bbb3e347293874e8db3248153e1efcdb13cbb9a5fc7967aabe478773a6a33c3c2372b4e819ecc0c41af43f36
+  data.tar.gz: 34530aeb5fb2836568aa5f7ebea7f154ef8e58c82ac0803aeba5632f704e7aed6c8111a095546c56b1331fa3dc1b563b08eac7cff0205aee4056815ebd8a6478

data/bin/htmlproofer

@@ -20,13 +20,13 @@ Mercenary.program(:htmlproofer) do |p|
   p.option 'as_links', '--as-links', 'Assumes that `PATH` is a comma-separated array of links to check.'
   p.option 'alt_ignore', '--alt-ignore image1,[image2,...]', Array, 'A comma-separated list of Strings or RegExps containing `img`s whose missing `alt` tags are safe to ignore'
   p.option 'assume_extension', '--assume-extension', 'Automatically add extension (e.g. `.html`) to file paths, to allow extensionless URLs (as supported by Jekyll 3 and GitHub Pages) (default: `false`).'
-  p.option 'checks_to_ignore', '--checks-to-ignore check1,[check2,...]', Array, ' An array of Strings indicating which checks you\'d like to not perform.'
+  p.option 'checks_to_ignore', '--checks-to-ignore check1,[check2,...]', Array, 'A comma-separated list of Strings indicating which checks you do not want to run (default: `[]`)'
   p.option 'check_external_hash', '--check-external-hash', 'Checks whether external hashes exist (even if the webpage exists). This slows the checker down (default: `false`).'
   p.option 'check_favicon', '--check-favicon', 'Enables the favicon checker (default: `false`).'
-  p.option 'check_html', '--check-html', 'Enables HTML validation errors from Nokogiri (default: `false`).'
+  p.option 'check_html', '--check-html', 'Enables HTML validation errors from Nokogumbo (default: `false`).'
   p.option 'check_img_http', '--check-img-http', 'Fails an image if it\'s marked as `http` (default: `false`).'
   p.option 'check_opengraph', '--check-opengraph', 'Enables the Open Graph checker (default: `false`).'
-  p.option 'check_sri', '--check-sri', 'Check that `<link>` and `<script>` external resources do use SRI (default: `false`).'
+  p.option 'check_sri', '--check-sri', 'Check that `<link>` and `<script>` external resources use SRI (default: `false`).'
   p.option 'directory_index_file', '--directory-index-file <filename>', String, 'Sets the file to look for when a link refers to a directory. (default: `index.html`)'
   p.option 'disable_external', '--disable-external', 'If `true`, does not run the external link checker, which can take a lot of time (default: `false`)'
   p.option 'empty_alt_ignore', '--empty-alt-ignore', 'If `true`, ignores images with empty alt tags'
@@ -37,9 +37,12 @@ Mercenary.program(:htmlproofer) do |p|
   p.option 'file_ignore', '--file-ignore file1,[file2,...]', Array, 'A comma-separated list of Strings or RegExps containing file paths that are safe to ignore'
   p.option 'http_status_ignore', '--http-status-ignore 123,[xxx, ...]', Array, 'A comma-separated list of numbers representing status codes to ignore.'
   p.option 'internal_domains', '--internal-domains domain1,[domain2,...]', Array, 'A comma-separated list of Strings containing domains that will be treated as internal urls.'
-  p.option 'report_invalid_tags', '--report-invalid-tags', 'Ignore `check_html` errors associated with unknown markup (default: `false`)'
-  p.option 'report_missing_names', '--report-missing-names', 'Ignore `check_html` errors associated with missing entities (default: `false`)'
-  p.option 'report_script_embeds', '--report-script-embeds', 'Ignore `check_html` errors associated with `script`s (default: `false`)'
+  p.option 'report_invalid_tags', '--report-invalid-tags', 'When `check_html` is enabled, HTML markup that is unknown to Nokogumbo are reported as errors (default: `false`)'
+  p.option 'report_missing_names', '--report-missing-names', 'When `check_html` is enabled, HTML markup that are missing entity names are reported as errors (default: `false`)'
+  p.option 'report_script_embeds', '--report-script-embeds', 'When `check_html` is enabled, `script` tags containing markup are reported as errors (default: `false`)'
+  p.option 'report_missing_doctype', '--report-missing-doctype', 'When `check_html` is enabled, HTML markup with missing or out-of-order `DOCTYPE` are reported as errors (default: `false`)'
+  p.option 'report_eof_tags', '--report-eof-tags', 'When `check_html` is enabled, HTML markup with tags that are malformed are reported as errors (default: `false`)'
+  p.option 'report_mismatched_tags', '--report-mismatched-tags', 'When `check_html` is enabled, HTML markup with mismatched tags are reported as errors (default: `false`)'
   p.option 'log_level', '--log-level <level>', String, 'Sets the logging level, as determined by Yell. One of `:debug`, `:info`, `:warn`, `:error`, or `:fatal`. (default: `:info`)'
   p.option 'only_4xx', '--only-4xx', 'Only reports errors for links that fall within the 4xx status code range'
   p.option 'storage_dir', '--storage-dir PATH', String, 'Directory where to store the cache log (default: "tmp/.htmlproofer")'
@@ -80,6 +83,9 @@ Mercenary.program(:htmlproofer) do |p|
     options[:validation][:report_script_embeds] = opts['report_script_embeds'] unless opts['report_script_embeds'].nil?
     options[:validation][:report_missing_names] = opts['report_missing_names'] unless opts['report_missing_names'].nil?
     options[:validation][:report_invalid_tags] = opts['report_invalid_tags'] unless opts['report_invalid_tags'].nil?
+    options[:validation][:report_missing_doctype] = opts['report_missing_doctype'] unless opts['report_missing_doctype'].nil?
+    options[:validation][:report_eof_tags] = opts['report_eof_tags'] unless opts['report_eof_tags'].nil?
+    options[:validation][:report_mismatched_tags] = opts['report_mismatched_tags'] unless opts['report_mismatched_tags'].nil?
 
     options[:typhoeus] = HTMLProofer::Configuration.parse_json_option('typhoeus_config', opts['typhoeus_config']) unless opts['typhoeus_config'].nil?
 

data/lib/html-proofer.rb

@@ -2,7 +2,7 @@
 
 def require_all(path)
   dir = File.join(File.dirname(__FILE__), path)
-  Dir[File.join(dir, '*.rb')].each do |f|
+  Dir[File.join(dir, '*.rb')].sort.each do |f|
     require f
   end
 end
@@ -17,7 +17,7 @@ require 'fileutils'
 begin
   require 'awesome_print'
   require 'pry-byebug'
-rescue LoadError; end # rubocop:disable Lint/HandleExceptions
+rescue LoadError; end # rubocop:disable Lint/SuppressedException
 module HTMLProofer
   def self.check_file(file, options = {})
     raise ArgumentError unless file.is_a?(String)

data/lib/html-proofer/cache.rb

@@ -120,9 +120,8 @@ module HTMLProofer
       @cache_log.each_pair do |url, cache|
         if within_timeframe?(cache['time'])
           next if cache['message'].empty? # these were successes to skip
-        else
-          urls_to_check[url] = cache['filenames'] # recheck expired links
         end
+        urls_to_check[url] = cache['filenames'] # recheck expired links
       end
       urls_to_check
     end

data/lib/html-proofer/check.rb

@@ -5,10 +5,11 @@ module HTMLProofer
   class Check
     attr_reader :node, :html, :element, :src, :path, :options, :issues, :external_urls
 
-    def initialize(src, path, html, options)
+    def initialize(src, path, html, logger, options)
       @src    = src
       @path   = path
       @html   = remove_ignored(html)
+      @logger = logger
       @options = options
       @issues = []
       @external_urls = {}
@@ -16,7 +17,7 @@ module HTMLProofer
 
     def create_element(node)
       @node = node
-      Element.new(node, self)
+      Element.new(node, self, @logger)
     end
 
     def run

data/lib/html-proofer/check/html.rb

@@ -1,28 +1,37 @@
 # frozen_string_literal: true
 
 class HtmlCheck < ::HTMLProofer::Check
+  # tags embedded in scripts are used in templating languages: http://git.io/vOovv
   SCRIPT_EMBEDS_MSG = /Element script embeds close tag/.freeze
   INVALID_TAG_MSG = /Tag ([\w\-:]+) invalid/.freeze
   INVALID_PREFIX = /Namespace prefix/.freeze
   PARSE_ENTITY_REF = /htmlParseEntityRef: no name/.freeze
+  DOCTYPE_MSG = /Expected a doctype token/.freeze
+  EOF_IN_TAG = /End of input in tag/.freeze
+  MISMATCHED_TAGS = /That tag isn't allowed here/.freeze
 
   def run
     @html.errors.each do |error|
-      message = error.message
-      line    = error.line
-
-      if message =~ INVALID_TAG_MSG || message =~ INVALID_PREFIX
-        next unless options[:validation][:report_invalid_tags]
-      end
-
-      if message =~ PARSE_ENTITY_REF
-        next unless options[:validation][:report_missing_names]
-      end
-
-      # tags embedded in scripts are used in templating languages: http://git.io/vOovv
-      next if !options[:validation][:report_script_embeds] && message =~ SCRIPT_EMBEDS_MSG
+      add_issue(error.message, line: error.line) if report?(error.message)
+    end
+  end
 
-      add_issue(message, line: line)
+  def report?(message)
+    case message
+    when SCRIPT_EMBEDS_MSG
+      options[:validation][:report_script_embeds]
+    when INVALID_TAG_MSG, INVALID_PREFIX
+      options[:validation][:report_invalid_tags]
+    when PARSE_ENTITY_REF
+      options[:validation][:report_missing_names]
+    when DOCTYPE_MSG
+      options[:validation][:report_missing_doctype]
+    when EOF_IN_TAG
+      options[:validation][:report_eof_tags]
+    when MISMATCHED_TAGS
+      options[:validation][:report_mismatched_tags]
+    else
+      true
     end
   end
 end

data/lib/html-proofer/check/links.rb

@@ -34,7 +34,7 @@ class LinkCheck < ::HTMLProofer::Check
       if missing_href?
         next if @link.allow_missing_href?
         # HTML5 allows dropping the href: http://git.io/vBX0z
-        next if @html.internal_subset.name == 'html' && @html.internal_subset.external_id.nil?
+        next if @html.internal_subset.nil? || (@html.internal_subset.name == 'html' && @html.internal_subset.external_id.nil?)
 
         add_issue('anchor has no href attribute', line: line, content: content)
         next

data/lib/html-proofer/check/opengraph.rb

@@ -3,12 +3,12 @@
 class OpenGraphElement < ::HTMLProofer::Element
   attr_reader :src
 
-  def initialize(obj, check)
-    super(obj, check)
+  def initialize(obj, check, logger)
+    super(obj, check, logger)
     # Fake up src from the content attribute
     instance_variable_set('@src', @content)
 
-    @src.insert 0, 'http:' if @src =~ %r{^//}
+    @src.insert 0, 'http:' if %r{^//}.match?(@src)
   end
 end
 
@@ -23,7 +23,7 @@ class OpenGraphCheck < ::HTMLProofer::Check
 
   def run
     @html.css('meta[property="og:url"], meta[property="og:image"]').each do |m|
-      @opengraph = OpenGraphElement.new(m, self)
+      @opengraph = OpenGraphElement.new(m, self, @logger)
 
       next if @opengraph.ignore?
 

data/lib/html-proofer/configuration.rb

@@ -51,7 +51,10 @@ module HTMLProofer
     VALIDATION_DEFAULTS = {
       report_script_embeds: false,
       report_missing_names: false,
-      report_invalid_tags: false
+      report_invalid_tags: false,
+      report_missing_doctype: false,
+      report_eof_tags: false,
+      report_mismatched_tags: false
     }.freeze
 
     CACHE_DEFAULTS = {}.freeze

data/lib/html-proofer/element.rb

@@ -10,12 +10,18 @@ module HTMLProofer
 
     attr_reader :id, :name, :alt, :href, :link, :src, :line, :data_proofer_ignore
 
-    def initialize(obj, check)
+    def initialize(obj, check, logger)
+      @logger = logger
       # Construct readable ivars for every element
-      obj.attributes.each_pair do |attribute, value|
-        name = attribute.tr('-:.', '_').to_s.to_sym
-        (class << self; self; end).send(:attr_reader, name)
-        instance_variable_set("@#{name}", value.value)
+      begin
+        obj.attributes.each_pair do |attribute, value|
+          name = attribute.tr('-:.;@', '_').to_s.to_sym
+          (class << self; self; end).send(:attr_reader, name)
+          instance_variable_set("@#{name}", value.value)
+        end
+      rescue NameError => e
+        @logger.log :error, "Attribute set `#{obj}` contains an error!"
+        raise e
       end
 
       @aria_hidden = defined?(@aria_hidden) && @aria_hidden == 'true' ? true : false
@@ -36,19 +42,19 @@ module HTMLProofer
 
       # fix up missing protocols
       if defined?(@href)
-        @href.insert(0, 'http:') if @href =~ %r{^//}
+        @href.insert(0, 'http:') if %r{^//}.match?(@href)
       else
         @href = nil
       end
 
       if defined?(@src)
-        @src.insert(0, 'http:') if @src =~ %r{^//}
+        @src.insert(0, 'http:') if %r{^//}.match?(@src)
       else
         @src = nil
       end
 
       if defined?(@srcset)
-        @srcset.insert(0, 'http:') if @srcset =~ %r{^//}
+        @srcset.insert(0, 'http:') if %r{^//}.match?(@srcset)
       else
         @srcset = nil
       end
@@ -99,11 +105,11 @@ module HTMLProofer
       return true if @data_proofer_ignore
       return true if @parent_ignorable
 
-      return true if url =~ /^javascript:/
+      return true if /^javascript:/.match?(url)
 
       # ignore base64 encoded images
       if %w[ImageCheck FaviconCheck].include? @type
-        return true if url =~ /^data:image/
+        return true if /^data:image/.match?(url)
       end
 
       # ignore user defined URLs
@@ -172,7 +178,7 @@ module HTMLProofer
 
       path_dot_ext = path + @check.options[:extension] if @check.options[:assume_extension]
 
-      if path =~ %r{^/} # path relative to root
+      if %r{^/}.match?(path) # path relative to root
         if File.directory?(@check.src)
           base = @check.src
         else
@@ -211,10 +217,11 @@ module HTMLProofer
 
     def ignores_pattern_check(links)
       links.each do |ignore|
-        if ignore.is_a? String
+        case ignore
+        when String
           return true if ignore == url
-        elsif ignore.is_a? Regexp
-          return true if ignore =~ url
+        when Regexp
+          return true if ignore&.match?(url)
         end
       end
 

data/lib/html-proofer/log.rb

@@ -7,16 +7,27 @@ module HTMLProofer
   class Log
     include Yell::Loggable
 
+    STDOUT_LEVELS = %i[debug info warn].freeze
+    STDERR_LEVELS = %i[error fatal].freeze
+
     def initialize(log_level)
       @logger = Yell.new(format: false, \
                          name: 'HTMLProofer', \
                          level: "gte.#{log_level}") do |l|
-        l.adapter :stdout, level: %i[debug info warn]
-        l.adapter :stderr, level: %i[error fatal]
+        l.adapter :stdout, level: 'lte.warn'
+        l.adapter :stderr, level: 'gte.error'
       end
     end
 
     def log(level, message)
+      log_with_color(level, message)
+    end
+
+    def log_with_color(level, message)
+      @logger.send level, colorize(level, message)
+    end
+
+    def colorize(level, message)
       color = case level
               when :debug
                 :cyan
@@ -28,15 +39,8 @@ module HTMLProofer
                 :red
               end
 
-      log_with_color(level, color, message)
-    end
-
-    def log_with_color(level, color, message)
-      @logger.send level, colorize(color, message)
-    end
-
-    def colorize(color, message)
-      if $stdout.isatty && $stderr.isatty
+      if (STDOUT_LEVELS.include?(level) && $stdout.isatty) || \
+         (STDERR_LEVELS.include?(level) && $stderr.isatty)
         Rainbow(message).send(color)
       else
         message

data/lib/html-proofer/middleware.rb

@@ -21,7 +21,8 @@ module HTMLProofer
         allow_hash_href: true,
         check_external_hash: true,
         check_html: true,
-        url_ignore: [/.*/] # Don't try to check local files exist
+        url_ignore: [/.*/], # Don't try to check if local files exist
+        validation: { report_eof_tags: true }
       }
     end
 
@@ -52,7 +53,7 @@ module HTMLProofer
     def call(env)
       result = @app.call(env)
       return result if env['REQUEST_METHOD'] != 'GET'
-      return result if env['QUERY_STRING'] =~ /proofer-ignore/
+      return result if /proofer-ignore/.match?(env['QUERY_STRING'])
       return result if result.first != 200
 
       body = []
@@ -69,7 +70,7 @@ module HTMLProofer
           'response',
           Middleware.options
         ).check_parsed(
-          Nokogiri::HTML(clean_content(html)), 'response'
+          Nokogiri::HTML5(html, max_errors: -1), 'response'
         )
 
         raise InvalidHtmlError, parsed[:failures] unless parsed[:failures].empty?

data/lib/html-proofer/runner.rb

@@ -31,6 +31,7 @@ module HTMLProofer
       end
 
       @failures = []
+      @before_request = []
     end
 
     def run
@@ -45,7 +46,7 @@ module HTMLProofer
       end
 
       if @failures.empty?
-        @logger.log_with_color :info, :green, 'HTML-Proofer finished successfully.'
+        @logger.log :info, 'HTML-Proofer finished successfully.'
       else
         print_failed_tests
       end
@@ -100,7 +101,7 @@ module HTMLProofer
       @src.each do |src|
         checks.each do |klass|
           @logger.log :debug, "Checking #{klass.to_s.downcase} on #{path} ..."
-          check = Object.const_get(klass).new(src, path, html, @options)
+          check = Object.const_get(klass).new(src, path, html, @logger, @options)
           check.run
           external_urls = check.external_urls
           external_urls = Hash[check.external_urls.map { |url, file| [swap(url, @options[:url_swap]), file] }] if @options[:url_swap]
@@ -117,6 +118,7 @@ module HTMLProofer
 
     def validate_urls
       url_validator = HTMLProofer::UrlValidator.new(@logger, @external_urls, @options)
+      url_validator.before_request = @before_request
       @failures.concat(url_validator.run)
       @external_urls = url_validator.external_urls
     end
@@ -147,6 +149,8 @@ module HTMLProofer
     def checks
       return @checks if defined?(@checks) && !@checks.nil?
 
+      return (@checks = ['LinkCheck']) if @type == :links
+
       @checks = HTMLProofer::Check.subchecks.map(&:name)
       @checks.delete('FaviconCheck') unless @options[:check_favicon]
       @checks.delete('HtmlCheck') unless @options[:check_html]
@@ -169,7 +173,23 @@ module HTMLProofer
       sorted_failures.sort_and_report
       count = @failures.length
       failure_text = pluralize(count, 'failure', 'failures')
-      raise @logger.colorize :red, "HTML-Proofer found #{failure_text}!"
+      raise @logger.colorize :fatal, "HTML-Proofer found #{failure_text}!"
+    end
+
+    # Set before_request callback.
+    #
+    # @example Set before_request.
+    #   request.before_request { |request| p "yay" }
+    #
+    # @param [ Block ] block The block to execute.
+    #
+    # @yield [ Typhoeus::Request ]
+    #
+    # @return [ Array<Block> ] All before_request blocks.
+    def before_request(&block)
+      @before_request ||= []
+      @before_request << block if block_given?
+      @before_request
     end
   end
 end

data/lib/html-proofer/url_validator.rb

@@ -10,6 +10,7 @@ module HTMLProofer
     include HTMLProofer::Utils
 
     attr_reader :external_urls
+    attr_writer :before_request
 
     def initialize(logger, external_urls, options)
       @logger = logger
@@ -18,6 +19,7 @@ module HTMLProofer
       @options = options
       @hydra = Typhoeus::Hydra.new(@options[:hydra])
       @cache = Cache.new(@logger, @options[:cache])
+      @before_request = []
     end
 
     def run
@@ -137,6 +139,9 @@ module HTMLProofer
     def queue_request(method, href, filenames)
       opts = @options[:typhoeus].merge(method: method)
       request = Typhoeus::Request.new(href, opts)
+      @before_request.each do |callback|
+        callback.call(request)
+      end
       request.on_complete { |response| response_handler(response, filenames) }
       @hydra.queue request
     end
@@ -146,7 +151,7 @@ module HTMLProofer
       href = response.request.base_url.to_s
       method = response.request.options[:method]
       response_code = response.code
-      response.body.gsub!("\x00", '')
+      response.body.delete!("\x00")
 
       debug_msg = if filenames.nil?
                     "Received a #{response_code} for #{href}"

data/lib/html-proofer/utils.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
+require 'nokogumbo'
 
 module HTMLProofer
   module Utils
@@ -15,7 +15,7 @@ module HTMLProofer
                   path
                 end
 
-      Nokogiri::HTML(clean_content(content))
+      Nokogiri::HTML5(content, max_errors: -1)
     end
 
     def swap(href, replacement)
@@ -24,14 +24,5 @@ module HTMLProofer
       end
       href
     end
-
-    # address a problem with Nokogiri's parsing URL entities
-    # problem from http://git.io/vBYU1
-    # solution from http://git.io/vBYUi
-    def clean_content(string)
-      string.gsub(%r{(?:https?:)?//([^>]+)}i) do |url|
-        url.gsub(/&(?!amp;)/, '&')
-      end
-    end
   end
 end

data/lib/html-proofer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = '3.14.1'
+  VERSION = '3.16.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html-proofer
 version: !ruby/object:Gem::Version
-  version: 3.14.1
+  version: 3.16.0
 platform: ruby
 authors:
 - Garen Torikian
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-17 00:00:00.000000000 Z
+date: 2020-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -39,19 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: nokogumbo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -308,7 +308,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A set of tests to validate your HTML output. These tests check if your image