html-proofer 3.19.4 → 4.0.0.rc1

data/lib/html_proofer/check/favicon.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class HTMLProofer::Check::Favicon < HTMLProofer::Check
+  def run
+    found = false
+    @html.css('link').each do |node|
+      @favicon = create_element(node)
+
+      next if @favicon.ignore?
+
+      break if (found = @favicon.node['rel'].split.last.eql? 'icon')
+    end
+
+    return if immediate_redirect?
+
+    if found
+      if @favicon.url.remote?
+        add_to_external_urls(@favicon.url, @favicon.line)
+      elsif !@favicon.url.exists?
+        add_failure("internal favicon #{@favicon.url.raw_attribute} does not exist", line: @favicon.line, content: @favicon.content)
+      end
+    else
+      add_failure('no favicon provided')
+    end
+  end
+
+  private
+
+  # allow any instant-redirect meta tag
+  def immediate_redirect?
+    @html.xpath("//meta[@http-equiv='refresh']").attribute('content').value.start_with? '0;'
+  rescue StandardError
+    false
+  end
+end

data/lib/html_proofer/check/images.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+class HTMLProofer::Check::Images < HTMLProofer::Check
+  SCREEN_SHOT_REGEX = /Screen(?: |%20)Shot(?: |%20)\d+-\d+-\d+(?: |%20)at(?: |%20)\d+.\d+.\d+/.freeze
+
+  def run
+    @html.css('img').each do |node|
+      @img = create_element(node)
+
+      next if @img.ignore?
+
+      # screenshot filenames should return because of terrible names
+      add_failure("image has a terrible filename (#{@img.url.raw_attribute})", line: @img.line, content: @img.content) if terrible_filename?
+
+      # does the image exist?
+      if missing_src?
+        add_failure('image has no src or srcset attribute', line: @img.line, content: @img.content)
+      elsif @img.url.remote?
+        add_to_external_urls(@img.url, @img.line)
+      elsif !@img.url.exists? && !@img.multiple_srcsets?
+        add_failure("internal image #{@img.url.raw_attribute} does not exist", line: @img.line, content: @img.content)
+      elsif @img.multiple_srcsets?
+        srcsets = @img.srcset.split(',').map(&:strip)
+        srcsets.each do |srcset|
+          srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url)
+
+          if srcset_url.remote?
+            add_to_external_urls(srcset_url.url, @img.line)
+          elsif !srcset_url.exists?
+            add_failure("internal image #{srcset} does not exist", line: @img.line, content: @img.content)
+          end
+        end
+      end
+
+      add_failure("image #{@img.url.raw_attribute} does not have an alt attribute", line: @img.line, content: @img.content) if empty_alt_tag? && !ignore_missing_alt? && !ignore_alt?
+
+      add_failure("image #{@img.url.raw_attribute} uses the http scheme", line: @img.line, content: @img.content) if @runner.enforce_https? && @img.url.http?
+    end
+
+    external_urls
+  end
+
+  def ignore_missing_alt?
+    @runner.options[:ignore_missing_alt]
+  end
+
+  def ignore_alt?
+    @img.url.ignore? || @img.aria_hidden?
+  end
+
+  def empty_alt_tag?
+    @img.node['alt'].nil? || @img.node['alt'].strip.empty?
+  end
+
+  def terrible_filename?
+    @img.url.to_s =~ SCREEN_SHOT_REGEX
+  end
+
+  def missing_src?
+    blank?(@img.url.to_s)
+  end
+end

data/lib/html_proofer/check/links.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+class HTMLProofer::Check::Links < HTMLProofer::Check
+  def run
+    @html.css('a, link, source').each do |node|
+      @link = create_element(node)
+
+      next if @link.ignore?
+
+      if !allow_hash_href? && @link.node['href'] == '#'
+        add_failure('linking to internal hash #, which points to nowhere', line: @link.line, content: @link.content)
+        next
+      end
+
+      # is there even an href?
+      if blank?(@link.url.raw_attribute)
+        next if allow_missing_href?
+
+        add_failure("'#{@link.node.name}' tag is missing a reference", line: @link.line, content: @link.content)
+        next
+      end
+
+      # is it even a valid URL?
+      unless @link.url.valid?
+        add_failure("#{@link.href} is an invalid URL", line: @link.line, content: @link.content)
+        next
+      end
+
+      check_schemes
+
+      # intentionally down here because we still want valid? & missing_href? to execute
+      next if @link.url.non_http_remote?
+
+      if !@link.url.internal? && @link.url.remote?
+        check_sri if @runner.check_sri? && @link.link_tag?
+
+        # we need to skip these for now; although the domain main be valid,
+        # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx
+        next if @link.node['rel'] == 'dns-prefetch'
+
+        unless @link.url.path?
+          add_failure("#{@link.url.raw_attribute} is an invalid URL", line: @link.line, content: @link.content)
+          next
+        end
+
+        add_to_external_urls(@link.url, @link.line)
+      elsif @link.url.internal?
+        # does the local directory have a trailing slash?
+        add_failure("internally linking to a directory #{@link.url.raw_attribute} without trailing slash", line: @link.line, content: @link.content) if @link.url.unslashed_directory?(@link.url.absolute_path)
+
+        add_to_internal_urls(@link.url, @link.line)
+      end
+    end
+
+    external_urls
+  end
+
+  def allow_missing_href?
+    @runner.options[:allow_missing_href]
+  end
+
+  def allow_hash_href?
+    @runner.options[:allow_hash_href]
+  end
+
+  def check_schemes
+    case @link.url.scheme
+    when 'mailto'
+      handle_mailto
+    when 'tel'
+      handle_tel
+    when 'http'
+      return unless @runner.options[:enforce_https]
+
+      add_failure("#{@link.url.raw_attribute} is not an HTTPS link", line: @link.line, content: @link.content)
+    end
+  end
+
+  def handle_mailto
+    if @link.url.path.empty?
+      add_failure("#{@link.url.raw_attribute} contains no email address", line: @link.line, content: @link.content) unless ignore_empty_mailto?
+    elsif !/#{URI::MailTo::EMAIL_REGEXP}/o.match?(@link.url.path)
+      add_failure("#{@link.url.raw_attribute} contains an invalid email address", line: @link.line, content: @link.content)
+    end
+  end
+
+  def handle_tel
+    add_failure("#{@link.url.raw_attribute} contains no phone number", line: @link.line, content: @link.content) if @link.url.path.empty?
+  end
+
+  def ignore_empty_mailto?
+    @runner.options[:ignore_empty_mailto]
+  end
+
+  # Whitelist for affected elements from Subresource Integrity specification
+  # https://w3c.github.io/webappsec-subresource-integrity/#link-element-for-stylesheets
+  SRI_REL_TYPES = %(stylesheet)
+
+  def check_sri
+    return unless SRI_REL_TYPES.include?(@link.node['rel'])
+
+    if blank?(@link.node['integrity']) && blank?(@link.node['crossorigin'])
+      add_failure("SRI and CORS not provided in: #{@link.url.raw_attribute}", line: @link.line, content: @link.content)
+    elsif blank?(@link.node['integrity'])
+      add_failure("Integrity is missing in: #{@link.url.raw_attribute}", line: @link.line, content: @link.content)
+    elsif blank?(@link.node['crossorigin'])
+      add_failure("CORS not provided for external resource in: #{@link.link.url.raw_attribute}", line: @link.line, content: @link.content)
+    end
+  end
+
+  private def source_tag?
+    @link.node.name == 'source'
+  end
+
+  private def anchor_tag?
+    @link.node.name == 'a'
+  end
+end

data/lib/html_proofer/check/open_graph.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class HTMLProofer::Check::OpenGraph < HTMLProofer::Check
+  def run
+    @html.css('meta[property="og:url"], meta[property="og:image"]').each do |node|
+      @open_graph = create_element(node)
+
+      next if @open_graph.ignore?
+
+      # does the open_graph exist?
+      if missing_content?
+        add_failure('open graph has no content attribute', line: @open_graph.line, content: @open_graph.content)
+      elsif empty_content?
+        add_failure('open graph content attribute is empty', line: @open_graph.line, content: @open_graph.content)
+      elsif !@open_graph.url.valid?
+        add_failure("#{@open_graph.src} is an invalid URL", line: @open_graph.line)
+      elsif @open_graph.url.remote?
+        add_to_external_urls(@open_graph.url, @open_graph.line)
+      else
+        add_failure("internal open graph #{@open_graph.url.raw_attribute} does not exist", line: @open_graph.line, content: @open_graph.content) unless @open_graph.url.exists?
+      end
+    end
+
+    external_urls
+  end
+
+  private def missing_content?
+    @open_graph.node['content'].nil?
+  end
+
+  private def empty_content?
+    @open_graph.node['content'].empty?
+  end
+end

data/lib/html_proofer/check/scripts.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class HTMLProofer::Check::Scripts < HTMLProofer::Check
+  def run
+    @html.css('script').each do |node|
+      @script = create_element(node)
+
+      next if @script.ignore?
+      next unless @script.content.strip.empty?
+
+      # does the script exist?
+      if missing_src?
+        add_failure('script is empty and has no src attribute', line: @script.line, content: @script.content)
+      elsif @script.url.remote?
+        add_to_external_urls(@script.src, @script.line)
+        check_sri if @runner.check_sri?
+      elsif !@script.url.exists?
+        add_failure("internal script reference #{@script.src} does not exist", line: @script.line, content: @script.content)
+      end
+    end
+
+    external_urls
+  end
+
+  def missing_src?
+    @script.node['src'].nil?
+  end
+
+  def check_sri
+    if blank?(@script.node['integrity']) && blank?(@script.node['crossorigin'])
+      add_failure("SRI and CORS not provided in: #{@script.url.raw_attribute}", line: @script.line, content: @script.content)
+    elsif blank?(@script.node['integrity'])
+      add_failure("Integrity is missing in: #{@script.url.raw_attribute}", line: @script.line, content: @script.content)
+    elsif blank?(@script.node['crossorigin'])
+      add_failure("CORS not provided for external resource in: #{@script.url.raw_attribute}", line: @script.line, content: @script.content)
+    end
+  end
+end

data/lib/html_proofer/check.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  # Mostly handles issue management and collecting of external URLs.
+  class Check
+    include HTMLProofer::Utils
+
+    attr_reader :failures, :options, :internal_urls, :external_urls
+
+    def initialize(runner, html)
+      @runner = runner
+      @html   = remove_ignored(html)
+
+      @external_urls = {}
+      @internal_urls = {}
+      @failures = []
+    end
+
+    def create_element(node)
+      Element.new(@runner, node, base_url: base_url)
+    end
+
+    def run
+      raise NotImplementedError, 'HTMLProofer::Check subclasses must implement #run'
+    end
+
+    def add_failure(description, line: nil, status: nil, content: nil)
+      @failures << Failure.new(@runner.current_path, short_name, description, line: line, status: status, content: content)
+    end
+
+    def self.subchecks(runner_options)
+      # grab all known checks
+      checks = ObjectSpace.each_object(Class).select do |klass|
+        klass < self
+      end
+
+      # remove any checks not explicitly included
+      checks.each_with_object([]) do |check, arr|
+        next unless runner_options[:checks].include?(check.short_name)
+
+        arr << check
+      end
+    end
+
+    def short_name
+      self.class.name.split('::').last
+    end
+
+    def self.short_name
+      name.split('::').last
+    end
+
+    def add_to_internal_urls(url, line)
+      url_string = url.raw_attribute
+
+      @internal_urls[url_string] = [] if @internal_urls[url_string].nil?
+
+      metadata = {
+        source: @runner.current_source,
+        current_path: @runner.current_path,
+        line: line,
+        base_url: base_url,
+        found: nil
+      }
+      @internal_urls[url_string] << metadata
+    end
+
+    def add_to_external_urls(url, line)
+      url_string = url.to_s
+
+      @external_urls[url_string] = [] if @external_urls[url_string].nil?
+
+      @external_urls[url_string] << { filename: @runner.current_path, line: line }
+    end
+
+    private def base_url
+      return @base_url if defined?(@base_url)
+
+      return (@base_url = '') if (base = @html.at_css('base')).nil?
+
+      @base_url = base['href']
+    end
+
+    private def remove_ignored(html)
+      return if html.nil?
+
+      html.css('code, pre, tt').each(&:unlink)
+      html
+    end
+  end
+end

data/lib/{html-proofer → html_proofer}/configuration.rb

@@ -2,35 +2,27 @@
 
 module HTMLProofer
   module Configuration
-    require_relative 'version'
+    DEFAULT_TESTS = %w[Links Images Scripts].freeze
 
     PROOFER_DEFAULTS = {
+      allow_hash_href: true,
       allow_missing_href: false,
-      allow_hash_href: false,
-      alt_ignore: [],
-      assume_extension: false,
-      check_external_hash: false,
-      check_favicon: false,
-      check_html: false,
-      check_img_http: false,
-      check_opengraph: false,
-      checks_to_ignore: [],
-      check_sri: false,
+      assume_extension: '.html',
+      check_external_hash: true,
+      checks: DEFAULT_TESTS,
       directory_index_file: 'index.html',
       disable_external: false,
-      empty_alt_ignore: false,
-      enforce_https: false,
-      error_sort: :path,
-      extension: '.html',
-      external_only: false,
-      file_ignore: [],
-      http_status_ignore: [],
-      internal_domains: [],
-      log_level: :info,
       ignore_empty_mailto: false,
+      ignore_files: [],
+      ignore_missing_alt: false,
+      ignore_status_codes: [],
+      ignore_urls: [],
+      enforce_https: true,
+      extensions: ['.html'],
+      log_level: :info,
       only_4xx: false,
-      url_ignore: [],
-      url_swap: {}
+      swap_attributes: {},
+      swap_urls: {}
     }.freeze
 
     TYPHOEUS_DEFAULTS = {
@@ -47,19 +39,26 @@ module HTMLProofer
       max_concurrency: 50
     }.freeze
 
-    PARALLEL_DEFAULTS = {}.freeze
-
-    VALIDATION_DEFAULTS = {
-      report_script_embeds: false,
-      report_missing_names: false,
-      report_invalid_tags: false,
-      report_missing_doctype: false,
-      report_eof_tags: false,
-      report_mismatched_tags: false
+    PARALLEL_DEFAULTS = {
+      enable: true
     }.freeze
 
     CACHE_DEFAULTS = {}.freeze
 
+    def self.generate_defaults(opts)
+      options = PROOFER_DEFAULTS.merge(opts)
+
+      options[:typhoeus] = HTMLProofer::Configuration::TYPHOEUS_DEFAULTS.merge(opts[:typhoeus] || {})
+      options[:hydra] = HTMLProofer::Configuration::HYDRA_DEFAULTS.merge(opts[:hydra] || {})
+
+      options[:parallel] = HTMLProofer::Configuration::PARALLEL_DEFAULTS.merge(opts[:parallel] || {})
+      options[:cache] = HTMLProofer::Configuration::CACHE_DEFAULTS.merge(opts[:cache] || {})
+
+      options.delete(:src)
+
+      options
+    end
+
     def self.to_regex?(item)
       if item.start_with?('/') && item.end_with?('/')
         Regexp.new item[1...-1]

data/lib/html_proofer/element.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'addressable/uri'
+
+module HTMLProofer
+  # Represents the element currently being processed
+  class Element
+    include HTMLProofer::Utils
+
+    attr_reader :node, :url, :base_url, :line, :content
+
+    def initialize(runner, node, base_url: nil)
+      @runner = runner
+      @node = node
+
+      @base_url = base_url
+      @url = Attribute::Url.new(runner, link_attribute, base_url: base_url)
+
+      @line = node.line
+      @content = node.content
+    end
+
+    def link_attribute
+      meta_content || src || srcset || href
+    end
+
+    def meta_content
+      return nil unless meta_tag?
+      return swap_attributes('content') if attribute_swapped?
+
+      @node['content']
+    end
+
+    def meta_tag?
+      @node.name == 'meta'
+    end
+
+    def src
+      return nil if !img_tag? && !script_tag? && !source_tag?
+      return swap_attributes('src') if attribute_swapped?
+
+      @node['src']
+    end
+
+    def img_tag?
+      @node.name == 'img'
+    end
+
+    def script_tag?
+      @node.name == 'script'
+    end
+
+    def srcset
+      return nil if !img_tag? && !source_tag?
+      return swap_attributes('srcset') if attribute_swapped?
+
+      @node['srcset']
+    end
+
+    def source_tag?
+      @node.name == 'source'
+    end
+
+    def href
+      return nil if !a_tag? && !link_tag?
+      return swap_attributes('href') if attribute_swapped?
+
+      @node['href']
+    end
+
+    def a_tag?
+      @node.name == 'a'
+    end
+
+    def link_tag?
+      @node.name == 'link'
+    end
+
+    def aria_hidden?
+      @node.attributes['aria-hidden']&.value == 'true'
+    end
+
+    def multiple_srcsets?
+      !blank?(srcset) && srcset.split(',').size > 1
+    end
+
+    def ignore?
+      return true if @node.attributes['data-proofer-ignore']
+      return true if ancestors_ignorable?
+
+      return true if url&.ignore?
+
+      false
+    end
+
+    private def attribute_swapped?
+      return false if blank?(@runner.options[:swap_attributes])
+
+      attrs = @runner.options[:swap_attributes][@node.name]
+
+      return true unless blank?(attrs)
+    end
+
+    private def swap_attributes(old_attr)
+      attrs = @runner.options[:swap_attributes][@node.name]
+
+      new_attr = attrs.find do |(o, _)|
+        o == old_attr
+      end&.last
+
+      return nil if blank?(new_attr)
+
+      @node[new_attr]
+    end
+
+    private def ancestors_ignorable?
+      ancestors_attributes = @node.ancestors.map { |a| a.respond_to?(:attributes) && a.attributes }
+      ancestors_attributes.pop # remove document at the end
+      ancestors_attributes.any? { |a| !a['data-proofer-ignore'].nil? }
+    end
+  end
+end

data/lib/html_proofer/failure.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Failure
+    attr_reader :path, :check_name, :description, :status, :line, :content
+
+    def initialize(path, check_name, description, line: nil, status: nil, content: nil)
+      @path = path
+      @check_name = check_name
+      @description = description
+
+      @line = line
+      @status = status
+      @content = content
+    end
+  end
+end

data/lib/html_proofer/reporter/cli.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class HTMLProofer::Reporter::Cli < HTMLProofer::Reporter
+  def report
+    msg = failures.each_with_object([]) do |(check_name, failures), arr|
+      str = ["For the #{check_name} check, the following failures were found:\n"]
+
+      failures.each do |failure|
+        path_str = blank?(failure.path) ? '' : "In #{failure.path}"
+
+        line_str = failure.line.nil? ? '' : " (line #{failure.line})"
+
+        path_and_line = [path_str, line_str].join
+        path_and_line = blank?(path_and_line) ? '' : "* #{path_and_line}:\n\n"
+
+        status_str = failure.status.nil? ? '' : " (status code #{failure.status})"
+
+        indent = blank?(path_and_line) ? '* ' : '  '
+        str << <<~MSG
+          #{path_and_line}#{indent}#{failure.description}#{status_str}
+        MSG
+      end
+
+      arr << str.join("\n")
+    end
+
+    @logger.log(:error, msg.join("\n"))
+  end
+end

data/lib/html_proofer/reporter.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Reporter
+    include HTMLProofer::Utils
+
+    attr_reader :failures
+
+    def initialize(logger: nil)
+      @logger = logger
+    end
+
+    def failures=(failures)
+      @failures = failures.group_by(&:check_name) \
+                          .transform_values { |issues| issues.sort_by { |issue| [issue.path, issue.line] } } \
+                          .sort
+    end
+
+    def report
+      raise NotImplementedError, 'HTMLProofer::Reporter subclasses must implement #report'
+    end
+  end
+end