html-proofer 3.19.4 → 4.0.0

data/lib/html_proofer/check.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  # Mostly handles issue management and collecting of external URLs.
+  class Check
+    include HTMLProofer::Utils
+
+    attr_reader :failures, :options, :internal_urls, :external_urls
+
+    def initialize(runner, html)
+      @runner = runner
+      @html   = remove_ignored(html)
+
+      @external_urls = {}
+      @internal_urls = {}
+      @failures = []
+    end
+
+    def create_element(node)
+      Element.new(@runner, node, base_url: base_url)
+    end
+
+    def run
+      raise NotImplementedError, "HTMLProofer::Check subclasses must implement #run"
+    end
+
+    def add_failure(description, line: nil, status: nil, content: nil)
+      @failures << Failure.new(@runner.current_filename, short_name, description, line: line, status: status,
+        content: content)
+    end
+
+    def self.subchecks(runner_options)
+      # grab all known checks
+      checks = ObjectSpace.each_object(Class).select do |klass|
+        klass < self
+      end
+
+      # remove any checks not explicitly included
+      checks.each_with_object([]) do |check, arr|
+        next unless runner_options[:checks].include?(check.short_name)
+
+        arr << check
+      end
+    end
+
+    def short_name
+      self.class.name.split("::").last
+    end
+
+    def self.short_name
+      name.split("::").last
+    end
+
+    def add_to_internal_urls(url, line)
+      url_string = url.raw_attribute
+
+      @internal_urls[url_string] = [] if @internal_urls[url_string].nil?
+
+      metadata = {
+        source: @runner.current_source,
+        filename: @runner.current_filename,
+        line: line,
+        base_url: base_url,
+        found: false,
+      }
+      @internal_urls[url_string] << metadata
+    end
+
+    def add_to_external_urls(url, line)
+      url_string = url.to_s
+
+      @external_urls[url_string] = [] if @external_urls[url_string].nil?
+
+      @external_urls[url_string] << { filename: @runner.current_filename, line: line }
+    end
+
+    private def base_url
+      return @base_url if defined?(@base_url)
+
+      return (@base_url = "") if (base = @html.at_css("base")).nil?
+
+      @base_url = base["href"]
+    end
+
+    private def remove_ignored(html)
+      return if html.nil?
+
+      html.css("code, pre, tt").each(&:unlink)
+      html
+    end
+  end
+end

data/lib/html_proofer/configuration.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  module Configuration
+    DEFAULT_TESTS = ["Links", "Images", "Scripts"].freeze
+
+    PROOFER_DEFAULTS = {
+      allow_hash_href: true,
+      allow_missing_href: false,
+      assume_extension: ".html",
+      check_external_hash: true,
+      checks: DEFAULT_TESTS,
+      directory_index_file: "index.html",
+      disable_external: false,
+      ignore_empty_alt: true,
+      ignore_empty_mailto: false,
+      ignore_files: [],
+      ignore_missing_alt: false,
+      ignore_status_codes: [],
+      ignore_urls: [],
+      enforce_https: true,
+      extensions: [".html"],
+      log_level: :info,
+      only_4xx: false,
+      swap_attributes: {},
+      swap_urls: {},
+    }.freeze
+
+    TYPHOEUS_DEFAULTS = {
+      followlocation: true,
+      headers: {
+        "User-Agent" => "Mozilla/5.0 (compatible; HTML Proofer/#{HTMLProofer::VERSION}; +https://github.com/gjtorikian/html-proofer)",
+        "Accept" => "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5",
+      },
+      connecttimeout: 10,
+      timeout: 30,
+    }.freeze
+
+    HYDRA_DEFAULTS = {
+      max_concurrency: 50,
+    }.freeze
+
+    PARALLEL_DEFAULTS = {
+      enable: true,
+    }.freeze
+
+    CACHE_DEFAULTS = {}.freeze
+
+    def self.generate_defaults(opts)
+      options = PROOFER_DEFAULTS.merge(opts)
+
+      options[:typhoeus] = HTMLProofer::Configuration::TYPHOEUS_DEFAULTS.merge(opts[:typhoeus] || {})
+      options[:hydra] = HTMLProofer::Configuration::HYDRA_DEFAULTS.merge(opts[:hydra] || {})
+
+      options[:parallel] = HTMLProofer::Configuration::PARALLEL_DEFAULTS.merge(opts[:parallel] || {})
+      options[:cache] = HTMLProofer::Configuration::CACHE_DEFAULTS.merge(opts[:cache] || {})
+
+      options.delete(:src)
+
+      options
+    end
+
+    def self.to_regex?(item)
+      if item.start_with?("/") && item.end_with?("/")
+        Regexp.new(item[1...-1])
+      else
+        item
+      end
+    end
+
+    def self.parse_json_option(option_name, config, symbolize_names: true)
+      raise ArgumentError, "Must provide an option name in string format." unless option_name.is_a?(String)
+      raise ArgumentError, "Must provide an option name in string format." if option_name.strip.empty?
+
+      return {} if config.nil?
+
+      raise ArgumentError, "Must provide a JSON configuration in string format." unless config.is_a?(String)
+
+      return {} if config.strip.empty?
+
+      begin
+        JSON.parse(config, { symbolize_names: symbolize_names })
+      rescue StandardError
+        raise ArgumentError, "Option '#{option_name} did not contain valid JSON."
+      end
+    end
+  end
+end

data/lib/html_proofer/element.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "addressable/uri"
+
+module HTMLProofer
+  # Represents the element currently being processed
+  class Element
+    include HTMLProofer::Utils
+
+    attr_reader :node, :url, :base_url, :line, :content
+
+    def initialize(runner, node, base_url: nil)
+      @runner = runner
+      @node = node
+
+      @base_url = base_url
+      @url = Attribute::Url.new(runner, link_attribute, base_url: base_url)
+
+      @line = node.line
+      @content = node.content
+    end
+
+    def link_attribute
+      meta_content || src || srcset || href
+    end
+
+    def meta_content
+      return nil unless meta_tag?
+      return swap_attributes("content") if attribute_swapped?
+
+      @node["content"]
+    end
+
+    def meta_tag?
+      @node.name == "meta"
+    end
+
+    def src
+      return nil if !img_tag? && !script_tag? && !source_tag?
+      return swap_attributes("src") if attribute_swapped?
+
+      @node["src"]
+    end
+
+    def img_tag?
+      @node.name == "img"
+    end
+
+    def script_tag?
+      @node.name == "script"
+    end
+
+    def srcset
+      return nil if !img_tag? && !source_tag?
+      return swap_attributes("srcset") if attribute_swapped?
+
+      @node["srcset"]
+    end
+
+    def source_tag?
+      @node.name == "source"
+    end
+
+    def href
+      return nil if !a_tag? && !link_tag?
+      return swap_attributes("href") if attribute_swapped?
+
+      @node["href"]
+    end
+
+    def a_tag?
+      @node.name == "a"
+    end
+
+    def link_tag?
+      @node.name == "link"
+    end
+
+    def aria_hidden?
+      @node.attributes["aria-hidden"]&.value == "true"
+    end
+
+    def multiple_srcsets?
+      !blank?(srcset) && srcset.split(",").size > 1
+    end
+
+    def ignore?
+      return true if @node.attributes["data-proofer-ignore"]
+      return true if ancestors_ignorable?
+
+      return true if url&.ignore?
+
+      false
+    end
+
+    private def attribute_swapped?
+      return false if blank?(@runner.options[:swap_attributes])
+
+      attrs = @runner.options[:swap_attributes][@node.name]
+
+      return true unless blank?(attrs)
+    end
+
+    private def swap_attributes(old_attr)
+      attrs = @runner.options[:swap_attributes][@node.name]
+
+      new_attr = attrs.find do |(o, _)|
+        o == old_attr
+      end&.last
+
+      return nil if blank?(new_attr)
+
+      @node[new_attr]
+    end
+
+    private def ancestors_ignorable?
+      ancestors_attributes = @node.ancestors.map { |a| a.respond_to?(:attributes) && a.attributes }
+      ancestors_attributes.pop # remove document at the end
+      ancestors_attributes.any? { |a| !a["data-proofer-ignore"].nil? }
+    end
+  end
+end

data/lib/html_proofer/failure.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Failure
+    attr_reader :path, :check_name, :description, :status, :line, :content
+
+    def initialize(path, check_name, description, line: nil, status: nil, content: nil)
+      @path = path
+      @check_name = check_name
+      @description = description
+
+      @line = line
+      @status = status
+      @content = content
+    end
+  end
+end

data/lib/{html-proofer → html_proofer}/log.rb

@@ -1,21 +1,21 @@
 # frozen_string_literal: true
 
-require 'yell'
-require 'rainbow'
+require "yell"
+require "rainbow"
 
 module HTMLProofer
   class Log
     include Yell::Loggable
 
-    STDOUT_LEVELS = %i[debug info warn].freeze
-    STDERR_LEVELS = %i[error fatal].freeze
+    STDOUT_LEVELS = [:debug, :info, :warn].freeze
+    STDERR_LEVELS = [:error, :fatal].freeze
 
     def initialize(log_level)
       @logger = Yell.new(format: false, \
-                         name: 'HTMLProofer', \
-                         level: "gte.#{log_level}") do |l|
-        l.adapter :stdout, level: 'lte.warn'
-        l.adapter :stderr, level: 'gte.error'
+        name: "HTMLProofer", \
+        level: "gte.#{log_level}") do |l|
+        l.adapter(:stdout, level: "lte.warn")
+        l.adapter(:stderr, level: "gte.error")
       end
     end
 
@@ -24,23 +24,23 @@ module HTMLProofer
     end
 
     def log_with_color(level, message)
-      @logger.send level, colorize(level, message)
+      @logger.send(level, colorize(level, message))
     end
 
     def colorize(level, message)
       color = case level
-              when :debug
-                :cyan
-              when :info
-                :blue
-              when :warn
-                :yellow
-              when :error, :fatal
-                :red
-              end
+      when :debug
+        :cyan
+      when :info
+        :blue
+      when :warn
+        :yellow
+      when :error, :fatal
+        :red
+      end
 
       if (STDOUT_LEVELS.include?(level) && $stdout.isatty) || \
-         (STDERR_LEVELS.include?(level) && $stderr.isatty)
+          (STDERR_LEVELS.include?(level) && $stderr.isatty)
         Rainbow(message).send(color)
       else
         message

data/lib/html_proofer/reporter/cli.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Reporter
+    class Cli < HTMLProofer::Reporter
+      def report
+        msg = failures.each_with_object([]) do |(check_name, failures), arr|
+          str = ["For the #{check_name} check, the following failures were found:\n"]
+
+          failures.each do |failure|
+            path_str = blank?(failure.path) ? "" : "At #{failure.path}"
+
+            line_str = failure.line.nil? ? "" : ":#{failure.line}"
+
+            path_and_line = "#{path_str}#{line_str}"
+            path_and_line = blank?(path_and_line) ? "" : "* #{path_and_line}:\n\n"
+
+            status_str = failure.status.nil? ? "" : " (status code #{failure.status})"
+
+            indent = blank?(path_and_line) ? "* " : "  "
+            str << <<~MSG
+              #{path_and_line}#{indent}#{failure.description}#{status_str}
+            MSG
+          end
+
+          arr << str.join("\n")
+        end
+
+        @logger.log(:error, msg.join("\n"))
+      end
+    end
+  end
+end

data/lib/html_proofer/reporter.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Reporter
+    include HTMLProofer::Utils
+
+    attr_reader :failures
+
+    def initialize(logger: nil)
+      @logger = logger
+    end
+
+    def failures=(failures)
+      @failures = failures.group_by(&:check_name) \
+        .transform_values { |issues| issues.sort_by { |issue| [issue.path, issue.line] } } \
+        .sort
+    end
+
+    def report
+      raise NotImplementedError, "HTMLProofer::Reporter subclasses must implement #report"
+    end
+  end
+end

data/lib/html_proofer/runner.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Runner
+    include HTMLProofer::Utils
+
+    attr_reader :options, :cache, :logger, :internal_urls, :external_urls, :checked_paths, :current_check
+    attr_accessor :current_filename, :current_source, :reporter
+
+    URL_TYPES = [:external, :internal].freeze
+
+    def initialize(src, opts = {})
+      @options = HTMLProofer::Configuration.generate_defaults(opts)
+
+      @type = @options.delete(:type)
+      @source = src
+
+      @logger = HTMLProofer::Log.new(@options[:log_level])
+      @cache = Cache.new(self, @options[:cache])
+
+      @external_urls = {}
+      @internal_urls = {}
+      @failures = []
+
+      @before_request = []
+
+      @checked_paths = {}
+
+      @current_check = nil
+      @current_source = nil
+      @current_filename = nil
+
+      @reporter = Reporter::Cli.new(logger: @logger)
+    end
+
+    def run
+      check_text = pluralize(checks.length, "check", "checks")
+
+      if @type == :links
+        @logger.log(:info, "Running #{check_text} (#{format_checks_list(checks)}) on #{@source} ... \n\n")
+        check_list_of_links unless @options[:disable_external]
+      else
+        @logger.log(:info,
+          "Running #{check_text} (#{format_checks_list(checks)}) in #{@source} on *#{@options[:extensions].join(", ")} files...\n\n")
+
+        check_files
+        @logger.log(:info, "Ran on #{pluralize(files.length, "file", "files")}!\n\n")
+      end
+
+      @cache.write
+
+      @reporter.failures = @failures
+
+      if @failures.empty?
+        @logger.log(:info, "HTML-Proofer finished successfully.")
+      else
+        @failures.uniq!
+        report_failed_checks
+      end
+    end
+
+    def check_list_of_links
+      @external_urls = @source.uniq.each_with_object({}) do |link, hash|
+        url = Attribute::Url.new(self, link, base_url: nil).to_s
+
+        hash[url] = []
+      end
+
+      validate_external_urls
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    # Sends the collected external URLs to Typhoeus for batch processing.
+    def check_files
+      process_files.each do |result|
+        URL_TYPES.each do |url_type|
+          type = :"#{url_type}_urls"
+          ivar_name = "@#{type}"
+          ivar = instance_variable_get(ivar_name)
+
+          if ivar.empty?
+            instance_variable_set(ivar_name, result[type])
+          else
+            result[type].each do |url, metadata|
+              ivar[url] = [] if ivar[url].nil?
+              ivar[url].concat(metadata)
+            end
+          end
+        end
+        @failures.concat(result[:failures])
+      end
+
+      validate_external_urls unless @options[:disable_external]
+
+      validate_internal_urls
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    def process_files
+      if @options[:parallel][:enable]
+        Parallel.map(files, @options[:parallel]) { |file| load_file(file[:path], file[:source]) }
+      else
+        files.map do |file|
+          load_file(file[:path], file[:source])
+        end
+      end
+    end
+
+    def load_file(path, source)
+      @html = create_nokogiri(path)
+      check_parsed(path, source)
+    end
+
+    # Collects any external URLs found in a directory of files. Also collectes
+    # every failed test from process_files.
+    def check_parsed(path, source)
+      result = { internal_urls: {}, external_urls: {}, failures: [] }
+
+      checks.each do |klass|
+        @current_source = source
+        @current_filename = path
+
+        check = Object.const_get(klass).new(self, @html)
+        @logger.log(:debug, "Running #{check.short_name} in #{path}")
+
+        @current_check = check
+
+        check.run
+
+        result[:external_urls].merge!(check.external_urls) { |_key, old, current| old.concat(current) }
+        result[:internal_urls].merge!(check.internal_urls) { |_key, old, current| old.concat(current) }
+        result[:failures].concat(check.failures)
+      end
+      result
+    end
+
+    def validate_external_urls
+      external_url_validator = HTMLProofer::UrlValidator::External.new(self, @external_urls)
+      external_url_validator.before_request = @before_request
+      @failures.concat(external_url_validator.validate)
+    end
+
+    def validate_internal_urls
+      internal_link_validator = HTMLProofer::UrlValidator::Internal.new(self, @internal_urls)
+      @failures.concat(internal_link_validator.validate)
+    end
+
+    def files
+      @files ||= if @type == :directory
+        @source.map do |src|
+          pattern = File.join(src, "**", "*{#{@options[:extensions].join(",")}}")
+          Dir.glob(pattern).select do |f|
+            File.file?(f) && !ignore_file?(f)
+          end.map { |f| { source: src, path: f } }
+        end.flatten
+      elsif @type == :file && @options[:extensions].include?(File.extname(@source))
+        [@source].reject { |f| ignore_file?(f) }.map { |f| { source: f, path: f } }
+      else
+        []
+      end
+    end
+
+    def ignore_file?(file)
+      @options[:ignore_files].each do |pattern|
+        return true if pattern.is_a?(String) && pattern == file
+        return true if pattern.is_a?(Regexp) && pattern =~ file
+      end
+
+      false
+    end
+
+    def check_sri?
+      @options[:check_sri]
+    end
+
+    def enforce_https?
+      @options[:enforce_https]
+    end
+
+    def checks
+      return @checks if defined?(@checks) && !@checks.nil?
+
+      return (@checks = ["LinkCheck"]) if @type == :links
+
+      @checks = HTMLProofer::Check.subchecks(@options).map(&:name)
+
+      @checks
+    end
+
+    def failed_checks
+      @reporter.failures.flatten.select { |f| f.is_a?(Failure) }
+    end
+
+    def report_failed_checks
+      @reporter.report
+
+      failure_text = pluralize(@failures.length, "failure", "failures")
+      @logger.log(:fatal, "\nHTML-Proofer found #{failure_text}!")
+      exit(1)
+    end
+
+    # Set before_request callback.
+    #
+    # @example Set before_request.
+    #   request.before_request { |request| p "yay" }
+    #
+    # @param [ Block ] block The block to execute.
+    #
+    # @yield [ Typhoeus::Request ]
+    #
+    # @return [ Array<Block> ] All before_request blocks.
+    def before_request(&block)
+      @before_request ||= []
+      @before_request << block if block
+      @before_request
+    end
+
+    def load_internal_cache
+      load_cache(:internal)
+    end
+
+    def load_external_cache
+      load_cache(:external)
+    end
+
+    private def load_cache(type)
+      ivar = instance_variable_get("@#{type}_urls")
+
+      existing_urls_count = @cache.size(type)
+      cache_text = pluralize(existing_urls_count, "#{type} link", "#{type} links")
+      @logger.log(:debug, "Found #{cache_text} in the cache")
+
+      urls_to_check = @cache.retrieve_urls(ivar, type)
+      urls_detected = pluralize(urls_to_check.count, "#{type} link", "#{type} links")
+      @logger.log(:info, "Checking #{urls_detected}")
+
+      urls_to_check
+    end
+
+    private def format_checks_list(checks)
+      checks.map do |check|
+        check.sub(/HTMLProofer::Check::/, "")
+      end.join(", ")
+    end
+  end
+end