html-proofer 3.19.4 → 4.0.0.rc1

data/lib/html_proofer/runner.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Runner
+    include HTMLProofer::Utils
+
+    attr_reader :options, :cache, :logger, :internal_urls, :external_urls, :checked_paths, :current_check
+    attr_accessor :current_path, :current_source, :reporter
+
+    URL_TYPES = %i[external internal].freeze
+
+    def initialize(src, opts = {})
+      @options = HTMLProofer::Configuration.generate_defaults(opts)
+
+      @type = @options.delete(:type)
+      @source = src
+
+      @logger = HTMLProofer::Log.new(@options[:log_level])
+      @cache = Cache.new(self, @options[:cache])
+
+      @external_urls = {}
+      @internal_urls = {}
+      @failures = []
+
+      @before_request = []
+
+      @checked_paths = {}
+
+      @current_check = nil
+      @current_source = nil
+      @current_path = nil
+
+      @reporter = Reporter::Cli.new(logger: @logger)
+    end
+
+    def run
+      check_text = pluralize(checks.length, 'check', 'checks')
+
+      if @type == :links
+        @logger.log :info, "Running #{check_text} (#{format_checks_list(checks)}) on #{@source} ... \n\n"
+        check_list_of_links unless @options[:disable_external]
+      else
+        @logger.log :info, "Running #{check_text} (#{format_checks_list(checks)}) in #{@source} on *#{@options[:extensions].join(', ')} files...\n\n"
+
+        check_files
+        @logger.log :info, "Ran on #{pluralize(files.length, 'file', 'files')}!\n\n"
+      end
+
+      @cache.write
+
+      @reporter.failures = @failures
+
+      if @failures.empty?
+        @logger.log :info, 'HTML-Proofer finished successfully.'
+      else
+        @failures.uniq!
+        report_failed_checks
+      end
+    end
+
+    def check_list_of_links
+      @external_urls = @source.uniq.each_with_object({}) do |link, hash|
+        url = Attribute::Url.new(self, link, base_url: nil).to_s
+
+        hash[url] = []
+      end
+
+      validate_external_urls
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    # Sends the collected external URLs to Typhoeus for batch processing.
+    def check_files
+      process_files.each do |result|
+        URL_TYPES.each do |url_type|
+          type = :"#{url_type}_urls"
+          ivar_name = "@#{type}"
+          ivar = instance_variable_get(ivar_name)
+
+          if ivar.empty?
+            instance_variable_set(ivar_name, result[type])
+          else
+            result[type].each do |url, metadata|
+              ivar[url] = [] if ivar[url].nil?
+              ivar[url].concat(metadata)
+            end
+          end
+        end
+        @failures.concat(result[:failures])
+      end
+
+      validate_external_urls unless @options[:disable_external]
+
+      validate_internal_urls
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    def process_files
+      if @options[:parallel][:enable]
+        Parallel.map(files, @options[:parallel]) { |path| load_file(path) }
+      else
+        files.map { |path| load_file(path) }
+      end
+    end
+
+    def load_file(path)
+      @html = create_nokogiri(path)
+      check_parsed(path)
+    end
+
+    # Collects any external URLs found in a directory of files. Also collectes
+    # every failed test from process_files.
+    def check_parsed(path)
+      result = { internal_urls: {}, external_urls: {}, failures: [] }
+
+      @source = [@source] if @type == :file
+
+      @source.each do |current_source|
+        checks.each do |klass|
+          @current_source = current_source
+          @current_path = path
+
+          check = Object.const_get(klass).new(self, @html)
+          @logger.log :debug, "Running #{check.short_name} in #{path}"
+
+          @current_check = check
+
+          check.run
+
+          result[:external_urls].merge!(check.external_urls)
+          result[:internal_urls].merge!(check.internal_urls)
+          result[:failures].concat(check.failures)
+        end
+      end
+      result
+    end
+
+    def validate_external_urls
+      external_url_validator = HTMLProofer::UrlValidator::External.new(self, @external_urls)
+      external_url_validator.before_request = @before_request
+      @failures.concat(external_url_validator.validate)
+    end
+
+    def validate_internal_urls
+      internal_link_validator = HTMLProofer::UrlValidator::Internal.new(self, @internal_urls)
+      @failures.concat(internal_link_validator.validate)
+    end
+
+    def files
+      @files ||= if @type == :directory
+                   @source.map do |src|
+                     pattern = File.join(src, '**', "*{#{@options[:extensions].join(',')}}")
+                     Dir.glob(pattern).select { |f| File.file?(f) && !ignore_file?(f) }
+                   end.flatten
+                 elsif @type == :file &&  @options[:extensions].include?(File.extname(@source))
+                   [@source].reject { |f| ignore_file?(f) }
+                 else
+                   []
+                 end
+    end
+
+    def ignore_file?(file)
+      @options[:ignore_files].each do |pattern|
+        return true if pattern.is_a?(String) && pattern == file
+        return true if pattern.is_a?(Regexp) && pattern =~ file
+      end
+
+      false
+    end
+
+    def check_sri?
+      @options[:check_sri]
+    end
+
+    def enforce_https?
+      @options[:enforce_https]
+    end
+
+    def checks
+      return @checks if defined?(@checks) && !@checks.nil?
+
+      return (@checks = ['LinkCheck']) if @type == :links
+
+      @checks = HTMLProofer::Check.subchecks(@options).map(&:name)
+
+      @checks
+    end
+
+    def failed_checks
+      @reporter.failures.flatten.select { |f| f.is_a?(Failure) }
+    end
+
+    def report_failed_checks
+      @reporter.report
+
+      failure_text = pluralize(@failures.length, 'failure', 'failures')
+      @logger.log :fatal, "\nHTML-Proofer found #{failure_text}!"
+      exit 1
+    end
+
+    # Set before_request callback.
+    #
+    # @example Set before_request.
+    #   request.before_request { |request| p "yay" }
+    #
+    # @param [ Block ] block The block to execute.
+    #
+    # @yield [ Typhoeus::Request ]
+    #
+    # @return [ Array<Block> ] All before_request blocks.
+    def before_request(&block)
+      @before_request ||= []
+      @before_request << block if block
+      @before_request
+    end
+
+    def load_internal_cache
+      load_cache(:internal)
+    end
+
+    def load_external_cache
+      load_cache(:external)
+    end
+
+    private def load_cache(type)
+      ivar = instance_variable_get("@#{type}_urls")
+
+      existing_urls_count = @cache.size(type)
+      cache_text = pluralize(existing_urls_count, "#{type} link", "#{type} links")
+      @logger.log :debug, "Found #{cache_text} in the cache"
+
+      urls_to_check = @cache.retrieve_urls(ivar, type)
+      urls_detected = pluralize(urls_to_check.count, "#{type} link", "#{type} links")
+      @logger.log :info, "Checking #{urls_detected}"
+
+      urls_to_check
+    end
+
+    private def format_checks_list(checks)
+      checks.map do |check|
+        check.sub(/HTMLProofer::Check::/, '')
+      end.join(', ')
+    end
+  end
+end

data/lib/html_proofer/url_validator/external.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require 'typhoeus'
+require 'uri'
+
+module HTMLProofer
+  class UrlValidator::External < UrlValidator
+    include HTMLProofer::Utils
+
+    attr_reader :external_urls
+    attr_writer :before_request
+
+    def initialize(runner, external_urls)
+      super(runner)
+
+      @external_urls = external_urls
+      @hydra = Typhoeus::Hydra.new(@runner.options[:hydra])
+      @before_request = []
+
+      @paths_with_queries = {}
+    end
+
+    def validate
+      if @cache.enabled?
+        urls_to_check = @runner.load_external_cache
+        run_external_link_checker(urls_to_check)
+      else
+        run_external_link_checker(@external_urls)
+      end
+
+      @failed_checks
+    end
+
+    # Proofer runs faster if we pull out all the external URLs and run the checks
+    # at the end. Otherwise, we're halting the consuming process for every file during
+    # `process_files`.
+    #
+    # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
+    #
+    # Finally, we'll first make a HEAD request, rather than GETing all the contents.
+    # If the HEAD fails, we'll fall back to GET, as some servers are not configured
+    # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
+    # not available as an option.
+    def run_external_link_checker(external_urls)
+      # Route log from Typhoeus/Ethon to our own logger
+      Ethon.logger = @logger
+
+      external_urls.each_pair do |external_url, metadata|
+        url = Attribute::Url.new(@runner, external_url, base_url: nil)
+
+        unless url.valid?
+          add_failure(metadata, "#{url} is an invalid URL", 0)
+          next
+        end
+
+        next unless new_url_query_values?(url)
+
+        method = if @runner.options[:check_external_hash] && url.hash?
+                   :get
+                 else
+                   :head
+                 end
+
+        queue_request(method, url, metadata)
+      end
+
+      @hydra.run
+    end
+
+    def queue_request(method, url, filenames)
+      opts = @runner.options[:typhoeus].merge(method: method)
+      request = Typhoeus::Request.new(url.url, opts)
+      @before_request.each do |callback|
+        callback.call(request)
+      end
+      request.on_complete { |response| response_handler(response, url, filenames) }
+      @hydra.queue request
+    end
+
+    def response_handler(response, url, filenames)
+      method = response.request.options[:method]
+      href = response.request.base_url.to_s
+      response_code = response.code
+      response.body.delete!("\x00")
+
+      @logger.log :debug, "Received a #{response_code} for #{href}"
+
+      return if @runner.options[:ignore_status_codes].include?(response_code)
+
+      if response_code.between?(200, 299)
+        @cache.add_external(href, filenames, response_code, 'OK') unless check_hash_in_2xx_response(href, url, response, filenames)
+      elsif response.timed_out?
+        handle_timeout(href, filenames, response_code)
+      elsif response_code.zero?
+        handle_connection_failure(href, filenames, response_code, response.status_message)
+      elsif method == :head # some servers don't support HEAD
+        queue_request(:get, url, filenames)
+      else
+        return if @runner.options[:only_4xx] && !response_code.between?(400, 499)
+
+        # Received a non-successful http response.
+        status_message = blank?(response.status_message) ? '' : ": #{response.status_message}"
+        msg = "External link #{href} failed#{status_message}"
+        add_failure(filenames, msg, response_code)
+        @cache.add_external(href, filenames, response_code, msg)
+      end
+    end
+
+    # Even though the response was a success, we may have been asked to check
+    # if the hash on the URL exists on the page
+    def check_hash_in_2xx_response(href, url, response, filenames)
+      return false if @runner.options[:only_4xx]
+      return false unless @runner.options[:check_external_hash]
+      return false unless url.hash?
+
+      hash = url.hash
+
+      body_doc = create_nokogiri(response.body)
+
+      unencoded_hash = Addressable::URI.unescape(hash)
+      xpath = [%(//*[@name="#{hash}"]|/*[@name="#{unencoded_hash}"]|//*[@id="#{hash}"]|//*[@id="#{unencoded_hash}"])]
+      # user-content is a special addition by GitHub.
+      if url.host =~ /github\.com/i
+        xpath << [%(//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])]
+        # when linking to a file on GitHub, like #L12-L34, only the first "L" portion
+        # will be identified as a linkable portion
+        xpath << [%(//td[@id="#{Regexp.last_match[1]}"])] if hash =~ /\A(L\d)+/
+      end
+
+      return unless body_doc.xpath(xpath.join('|')).empty?
+
+      msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
+      add_failure(filenames, msg, response.code)
+      @cache.add_external(href, filenames, response.code, msg)
+      true
+    end
+
+    def handle_timeout(href, filenames, response_code)
+      msg = "External link #{href} failed: got a time out (response code #{response_code})"
+      @cache.add_external(href, filenames, 0, msg)
+      return if @runner.options[:only_4xx]
+
+      add_failure(filenames, msg, response_code)
+    end
+
+    def handle_connection_failure(href, metadata, response_code, status_message)
+      msgs = [<<~MSG
+        External link #{href} failed with something very wrong.
+        It's possible libcurl couldn't connect to the server, or perhaps the request timed out.
+        Sometimes, making too many requests at once also breaks things.
+      MSG
+      ]
+
+      msgs << "Either way, the return message from the server is: #{status_message}" unless blank?(status_message)
+
+      msg = msgs.join("\n").chomp
+
+      @cache.add_external(href, metadata, 0, msg)
+      return if @runner.options[:only_4xx]
+
+      add_failure(metadata, msg, response_code)
+    end
+
+    def add_failure(metadata, description, status = nil)
+      if blank?(metadata) # possible if we're checking an array of links
+        @failed_checks << Failure.new('', 'Links > External', description, status: status)
+      else
+        metadata.each { |m| @failed_checks << Failure.new(m[:filename], 'Links > External', description, line: m[:line], status: status) }
+      end
+    end
+
+    # remember queries we've seen, ignore future ones
+    private def new_url_query_values?(url)
+      return true if (query_values = url.query_values).nil?
+
+      queries = query_values.keys.join('-')
+      domain_path = url.domain_path
+      if @paths_with_queries[domain_path].nil?
+        @paths_with_queries[domain_path] = [queries]
+        true
+      elsif !@paths_with_queries[domain_path].include?(queries)
+        @paths_with_queries[domain_path] << queries
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/html_proofer/url_validator/internal.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class UrlValidator::Internal < UrlValidator
+    attr_reader :internal_urls
+
+    def initialize(runner, internal_urls)
+      super(runner)
+
+      @internal_urls = internal_urls
+    end
+
+    def validate
+      if @cache.enabled?
+        urls_to_check = @runner.load_internal_cache
+        run_internal_link_checker(urls_to_check)
+      else
+        run_internal_link_checker(@internal_urls)
+      end
+
+      @failed_checks
+    end
+
+    def run_internal_link_checker(links)
+      links.each_pair do |link, matched_files|
+        matched_files.each do |metadata|
+          url = HTMLProofer::Attribute::Url.new(@runner, link, base_url: metadata[:base_url])
+
+          @runner.current_source = metadata[:source]
+          @runner.current_path = metadata[:current_path]
+
+          unless file_exists?(url)
+            @failed_checks << Failure.new(@runner.current_path, 'Links > Internal', "internally linking to #{url}, which does not exist", line: metadata[:line], status: nil, content: nil)
+            @cache.add_internal(url.to_s, metadata, false)
+            next
+          end
+
+          unless hash_exists?(url)
+            @failed_checks << Failure.new(@runner.current_path, 'Links > Internal', "internally linking to #{url}; the file exists, but the hash '#{url.hash}' does not", line: metadata[:line], status: nil, content: nil)
+            @cache.add_internal(url.to_s, metadata, false)
+            next
+          end
+
+          @cache.add_internal(url.to_s, metadata, true)
+        end
+      end
+
+      @failed_checks
+    end
+
+    private def file_exists?(url)
+      absolute_path = url.absolute_path
+      return @runner.checked_paths[url.absolute_path] if @runner.checked_paths.key?(absolute_path)
+
+      @runner.checked_paths[url.absolute_path] = File.exist?(absolute_path)
+    end
+
+    # verify the target hash
+    private def hash_exists?(url)
+      href_hash = url.hash
+      return true if blank?(href_hash)
+
+      # prevents searching files we didn't ask about
+      return false unless url.known_extension?
+
+      decoded_href_hash = Addressable::URI.unescape(href_hash)
+      fragment_ids = [href_hash, decoded_href_hash]
+      # https://www.w3.org/TR/html5/single-page.html#scroll-to-fragid
+      fragment_ids.include?('top') || !find_fragments(fragment_ids, url).empty?
+    end
+
+    private def find_fragments(fragment_ids, url)
+      xpaths = fragment_ids.uniq.flat_map do |frag_id|
+        escaped_frag_id = "'#{frag_id.split("'").join("', \"'\", '")}', ''"
+        [
+          "//*[case_sensitive_equals(@id, concat(#{escaped_frag_id}))]",
+          "//*[case_sensitive_equals(@name, concat(#{escaped_frag_id}))]"
+        ]
+      end
+      xpaths << XpathFunctions.new
+
+      html = create_nokogiri(url.absolute_path)
+      html.xpath(*xpaths)
+    end
+  end
+end

data/lib/html_proofer/url_validator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class UrlValidator
+    include HTMLProofer::Utils
+
+    def initialize(runner)
+      @runner = runner
+
+      @cache = @runner.cache
+      @logger = @runner.logger
+
+      @failed_checks = []
+    end
+  end
+end

data/lib/{html-proofer → html_proofer}/utils.rb

@@ -8,21 +8,18 @@ module HTMLProofer
       "#{count} #{count == 1 ? single : plural}"
     end
 
+    def blank?(obj)
+      obj.nil? || obj.empty?
+    end
+
     def create_nokogiri(path)
       content = if File.exist?(path) && !File.directory?(path)
-                  File.open(path).read
+                  File.read(path)
                 else
                   path
                 end
 
       Nokogiri::HTML5(content, max_errors: -1)
     end
-
-    def swap(href, replacement)
-      replacement.each do |link, replace|
-        href = href.gsub(link, replace)
-      end
-      href
-    end
   end
 end

data/lib/{html-proofer → html_proofer}/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = '3.19.4'
+  VERSION = '4.0.0.rc1'
 end

data/lib/html_proofer/xpath_functions.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  # https://stackoverflow.com/a/8812293
+  class XpathFunctions
+    def case_sensitive_equals(node_set, str_to_match)
+      node_set.find_all { |node| node.to_s.== str_to_match.to_s }
+    end
+  end
+end

data/lib/html_proofer.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'zeitwerk'
+lib_dir = File.join(File.dirname(__dir__), 'lib')
+gem_loader = Zeitwerk::Loader.for_gem
+gem_loader.inflector.inflect(
+  'html_proofer' => 'HTMLProofer'
+)
+gem_loader.ignore(File.join(lib_dir, 'html-proofer.rb'))
+gem_loader.setup
+
+require 'html_proofer/version'
+
+require 'parallel'
+require 'fileutils'
+
+%w[awesome_print debug].each do |gem|
+  require gem
+rescue LoadError;  # rubocop:disable Lint/SuppressedException
+end
+module HTMLProofer
+  def self.check_file(file, options = {})
+    raise ArgumentError unless file.is_a?(String)
+    raise ArgumentError, "#{file} does not exist" unless File.exist?(file)
+
+    options[:type] = :file
+    HTMLProofer::Runner.new(file, options)
+  end
+
+  def self.check_directory(directory, options = {})
+    raise ArgumentError unless directory.is_a?(String)
+    raise ArgumentError, "#{directory} does not exist" unless Dir.exist?(directory)
+
+    options[:type] = :directory
+    HTMLProofer::Runner.new([directory], options)
+  end
+
+  def self.check_directories(directories, options = {})
+    raise ArgumentError unless directories.is_a?(Array)
+
+    options[:type] = :directory
+    directories.each do |directory|
+      raise ArgumentError, "#{directory} does not exist" unless Dir.exist?(directory)
+    end
+    HTMLProofer::Runner.new(directories, options)
+  end
+
+  def self.check_links(links, options = {})
+    raise ArgumentError unless links.is_a?(Array)
+
+    options[:type] = :links
+    HTMLProofer::Runner.new(links, options)
+  end
+end
+
+gem_loader.eager_load