html-proofer 3.19.4 → 4.4.0

data/lib/html_proofer/runner.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Runner
+    include HTMLProofer::Utils
+
+    attr_reader :options, :cache, :logger, :internal_urls, :external_urls, :checked_paths, :current_check
+    attr_accessor :current_filename, :current_source, :reporter
+
+    URL_TYPES = [:external, :internal].freeze
+
+    def initialize(src, opts = {})
+      @options = HTMLProofer::Configuration.generate_defaults(opts)
+
+      @type = @options.delete(:type)
+      @source = src
+
+      @logger = HTMLProofer::Log.new(@options[:log_level])
+      @cache = Cache.new(self, @options[:cache])
+
+      @external_urls = {}
+      @internal_urls = {}
+      @failures = []
+
+      @before_request = []
+
+      @checked_paths = {}
+
+      @current_check = nil
+      @current_source = nil
+      @current_filename = nil
+
+      @reporter = Reporter::Cli.new(logger: @logger)
+    end
+
+    def run
+      check_text = pluralize(checks.length, "check", "checks")
+
+      if @type == :links
+        @logger.log(:info, "Running #{check_text} (#{format_checks_list(checks)}) on #{@source} ... \n\n")
+        check_list_of_links unless @options[:disable_external]
+      else
+        @logger.log(:info,
+          "Running #{check_text} (#{format_checks_list(checks)}) in #{@source} on *#{@options[:extensions].join(", ")} files...\n\n")
+
+        check_files
+        @logger.log(:info, "Ran on #{pluralize(files.length, "file", "files")}!\n\n")
+      end
+
+      @cache.write
+
+      @reporter.failures = @failures
+
+      if @failures.empty?
+        @logger.log(:info, "HTML-Proofer finished successfully.")
+      else
+        @failures.uniq!
+        report_failed_checks
+      end
+    end
+
+    def check_list_of_links
+      @external_urls = @source.uniq.each_with_object({}) do |link, hash|
+        url = Attribute::Url.new(self, link, base_url: nil).to_s
+
+        hash[url] = []
+      end
+
+      validate_external_urls
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    # Sends the collected external URLs to Typhoeus for batch processing.
+    def check_files
+      process_files.each do |result|
+        URL_TYPES.each do |url_type|
+          type = :"#{url_type}_urls"
+          ivar_name = "@#{type}"
+          ivar = instance_variable_get(ivar_name)
+
+          if ivar.empty?
+            instance_variable_set(ivar_name, result[type])
+          else
+            result[type].each do |url, metadata|
+              ivar[url] = [] if ivar[url].nil?
+              ivar[url].concat(metadata)
+            end
+          end
+        end
+        @failures.concat(result[:failures])
+      end
+
+      validate_external_urls unless @options[:disable_external]
+
+      validate_internal_urls
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    def process_files
+      if @options[:parallel][:enable]
+        Parallel.map(files, @options[:parallel]) { |file| load_file(file[:path], file[:source]) }
+      else
+        files.map do |file|
+          load_file(file[:path], file[:source])
+        end
+      end
+    end
+
+    def load_file(path, source)
+      @html = create_nokogiri(path)
+      check_parsed(path, source)
+    end
+
+    # Collects any external URLs found in a directory of files. Also collectes
+    # every failed test from process_files.
+    def check_parsed(path, source)
+      result = { internal_urls: {}, external_urls: {}, failures: [] }
+
+      checks.each do |klass|
+        @current_source = source
+        @current_filename = path
+
+        check = Object.const_get(klass).new(self, @html)
+        @logger.log(:debug, "Running #{check.short_name} in #{path}")
+
+        @current_check = check
+
+        check.run
+
+        result[:external_urls].merge!(check.external_urls) { |_key, old, current| old.concat(current) }
+        result[:internal_urls].merge!(check.internal_urls) { |_key, old, current| old.concat(current) }
+        result[:failures].concat(check.failures)
+      end
+      result
+    end
+
+    def validate_external_urls
+      external_url_validator = HTMLProofer::UrlValidator::External.new(self, @external_urls)
+      external_url_validator.before_request = @before_request
+      @failures.concat(external_url_validator.validate)
+    end
+
+    def validate_internal_urls
+      internal_link_validator = HTMLProofer::UrlValidator::Internal.new(self, @internal_urls)
+      @failures.concat(internal_link_validator.validate)
+    end
+
+    def files
+      @files ||= if @type == :directory
+        @source.map do |src|
+          pattern = File.join(src, "**", "*{#{@options[:extensions].join(",")}}")
+          Dir.glob(pattern).select do |f|
+            File.file?(f) && !ignore_file?(f)
+          end.map { |f| { source: src, path: f } }
+        end.flatten
+      elsif @type == :file && @options[:extensions].include?(File.extname(@source))
+        [@source].reject { |f| ignore_file?(f) }.map { |f| { source: f, path: f } }
+      else
+        []
+      end
+    end
+
+    def ignore_file?(file)
+      @options[:ignore_files].each do |pattern|
+        return true if pattern.is_a?(String) && pattern == file
+        return true if pattern.is_a?(Regexp) && pattern =~ file
+      end
+
+      false
+    end
+
+    def check_sri?
+      @options[:check_sri]
+    end
+
+    def enforce_https?
+      @options[:enforce_https]
+    end
+
+    def checks
+      return @checks if defined?(@checks) && !@checks.nil?
+
+      return (@checks = ["LinkCheck"]) if @type == :links
+
+      @checks = HTMLProofer::Check.subchecks(@options).map(&:name)
+
+      @checks
+    end
+
+    def failed_checks
+      @reporter.failures.flatten.select { |f| f.is_a?(Failure) }
+    end
+
+    def report_failed_checks
+      @reporter.report
+
+      failure_text = pluralize(@failures.length, "failure", "failures")
+      @logger.log(:fatal, "\nHTML-Proofer found #{failure_text}!")
+      exit(1)
+    end
+
+    # Set before_request callback.
+    #
+    # @example Set before_request.
+    #   request.before_request { |request| p "yay" }
+    #
+    # @param [ Block ] block The block to execute.
+    #
+    # @yield [ Typhoeus::Request ]
+    #
+    # @return [ Array<Block> ] All before_request blocks.
+    def before_request(&block)
+      @before_request ||= []
+      @before_request << block if block
+      @before_request
+    end
+
+    def load_internal_cache
+      load_cache(:internal)
+    end
+
+    def load_external_cache
+      load_cache(:external)
+    end
+
+    private def load_cache(type)
+      ivar = instance_variable_get("@#{type}_urls")
+
+      existing_urls_count = @cache.size(type)
+      cache_text = pluralize(existing_urls_count, "#{type} link", "#{type} links")
+      @logger.log(:debug, "Found #{cache_text} in the cache")
+
+      urls_to_check = @cache.retrieve_urls(ivar, type)
+
+      urls_to_check
+    end
+
+    private def format_checks_list(checks)
+      checks.map do |check|
+        check.sub(/HTMLProofer::Check::/, "")
+      end.join(", ")
+    end
+  end
+end

data/lib/html_proofer/url_validator/external.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require "typhoeus"
+require "uri"
+
+module HTMLProofer
+  class UrlValidator
+    class External < UrlValidator
+      include HTMLProofer::Utils
+
+      attr_reader :external_urls
+      attr_writer :before_request
+
+      def initialize(runner, external_urls)
+        super(runner)
+
+        @external_urls = external_urls
+        @hydra = Typhoeus::Hydra.new(@runner.options[:hydra])
+        @before_request = []
+
+        @paths_with_queries = {}
+      end
+
+      def validate
+        urls_to_check = @cache.external_enabled? ? @runner.load_external_cache : @external_urls
+        urls_detected = pluralize(urls_to_check.count, "external link", "external links")
+        @logger.log(:info, "Checking #{urls_detected}")
+
+        run_external_link_checker(urls_to_check)
+
+        @failed_checks
+      end
+
+      # Proofer runs faster if we pull out all the external URLs and run the checks
+      # at the end. Otherwise, we're halting the consuming process for every file during
+      # `process_files`.
+      #
+      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
+      #
+      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
+      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
+      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
+      # not available as an option.
+      def run_external_link_checker(external_urls)
+        # Route log from Typhoeus/Ethon to our own logger
+        Ethon.logger = @logger
+
+        external_urls.each_pair do |external_url, metadata|
+          url = Attribute::Url.new(@runner, external_url, base_url: nil)
+
+          unless url.valid?
+            add_failure(metadata, "#{url} is an invalid URL", 0)
+            next
+          end
+
+          next unless new_url_query_values?(url)
+
+          method = if @runner.options[:check_external_hash] && url.hash?
+            :get
+          else
+            :head
+          end
+
+          queue_request(method, url, metadata)
+        end
+
+        @hydra.run
+      end
+
+      def queue_request(method, url, filenames)
+        opts = @runner.options[:typhoeus].merge(method: method)
+        request = Typhoeus::Request.new(url.url, opts)
+        @before_request.each do |callback|
+          callback.call(request)
+        end
+        request.on_complete { |response| response_handler(response, url, filenames) }
+        @hydra.queue(request)
+      end
+
+      def response_handler(response, url, filenames)
+        method = response.request.options[:method]
+        href = response.request.base_url.to_s
+        response_code = response.code
+        response.body.delete!("\x00")
+
+        @logger.log(:debug, "Received a #{response_code} for #{href}")
+
+        return if @runner.options[:ignore_status_codes].include?(response_code)
+
+        if response_code.between?(200, 299)
+          @cache.add_external(href, filenames, response_code, "OK", true) unless check_hash_in_2xx_response(href, url,
+            response, filenames)
+        elsif response.timed_out?
+          handle_timeout(href, filenames, response_code)
+        elsif response_code.zero?
+          handle_connection_failure(href, filenames, response_code, response.status_message)
+        elsif method == :head # some servers don't support HEAD
+          queue_request(:get, url, filenames)
+        else
+          return if @runner.options[:only_4xx] && !response_code.between?(400, 499)
+
+          # Received a non-successful http response.
+          status_message = blank?(response.status_message) ? "" : ": #{response.status_message}"
+          msg = "External link #{href} failed#{status_message}"
+          add_failure(filenames, msg, response_code)
+          @cache.add_external(href, filenames, response_code, msg, false)
+        end
+      end
+
+      # Even though the response was a success, we may have been asked to check
+      # if the hash on the URL exists on the page
+      def check_hash_in_2xx_response(href, url, response, filenames)
+        return false if @runner.options[:only_4xx]
+        return false unless @runner.options[:check_external_hash]
+        return false unless url.hash?
+
+        hash = url.hash
+
+        body_doc = create_nokogiri(response.body)
+
+        unencoded_hash = Addressable::URI.unescape(hash)
+        xpath = [%(//*[@name="#{hash}"]|/*[@name="#{unencoded_hash}"]|//*[@id="#{hash}"]|//*[@id="#{unencoded_hash}"])]
+        # user-content is a special addition by GitHub.
+        if url.host =~ /github\.com/i
+          xpath << [%(//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])]
+          # when linking to a file on GitHub, like #L12-L34, only the first "L" portion
+          # will be identified as a linkable portion
+          xpath << [%(//td[@id="#{Regexp.last_match[1]}"])] if hash =~ /\A(L\d)+/
+        end
+
+        return unless body_doc.xpath(xpath.join("|")).empty?
+
+        msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
+        add_failure(filenames, msg, response.code)
+        @cache.add_external(href, filenames, response.code, msg, false)
+        true
+      end
+
+      def handle_timeout(href, filenames, response_code)
+        msg = "External link #{href} failed: got a time out (response code #{response_code})"
+        @cache.add_external(href, filenames, 0, msg, false)
+        return if @runner.options[:only_4xx]
+
+        add_failure(filenames, msg, response_code)
+      end
+
+      def handle_connection_failure(href, metadata, response_code, status_message)
+        msgs = [<<~MSG,
+          External link #{href} failed with something very wrong.
+          It's possible libcurl couldn't connect to the server, or perhaps the request timed out.
+          Sometimes, making too many requests at once also breaks things.
+        MSG
+        ]
+
+        msgs << "Either way, the return message from the server is: #{status_message}" unless blank?(status_message)
+
+        msg = msgs.join("\n").chomp
+
+        @cache.add_external(href, metadata, 0, msg, false)
+        return if @runner.options[:only_4xx]
+
+        add_failure(metadata, msg, response_code)
+      end
+
+      def add_failure(metadata, description, status = nil)
+        if blank?(metadata) # possible if we're checking an array of links
+          @failed_checks << Failure.new("", "Links > External", description, status: status)
+        else
+          metadata.each do |m|
+            @failed_checks << Failure.new(m[:filename], "Links > External", description, line: m[:line], status: status)
+          end
+        end
+      end
+
+      # remember queries we've seen, ignore future ones
+      private def new_url_query_values?(url)
+        return true if (query_values = url.query_values).nil?
+
+        queries = query_values.keys.join("-")
+        domain_path = url.domain_path
+        if @paths_with_queries[domain_path].nil?
+          @paths_with_queries[domain_path] = [queries]
+          true
+        elsif !@paths_with_queries[domain_path].include?(queries)
+          @paths_with_queries[domain_path] << queries
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/html_proofer/url_validator/internal.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class UrlValidator
+    class Internal < UrlValidator
+      attr_reader :internal_urls
+
+      def initialize(runner, internal_urls)
+        super(runner)
+
+        @internal_urls = internal_urls
+      end
+
+      def validate
+        urls_to_check = @cache.internal_enabled? ? @runner.load_internal_cache : @internal_urls
+        urls_detected = pluralize(urls_to_check.count, "internal link", "internal links")
+        @logger.log(:info, "Checking #{urls_detected}")
+
+        run_internal_link_checker(urls_to_check)
+
+        @failed_checks
+      end
+
+      def run_internal_link_checker(links)
+        to_add = []
+        links.each_pair do |link, matched_files|
+          matched_files.each do |metadata|
+            url = HTMLProofer::Attribute::Url.new(@runner, link, base_url: metadata[:base_url])
+
+            @runner.current_source = metadata[:source]
+            @runner.current_filename = metadata[:filename]
+
+            unless file_exists?(url)
+              @failed_checks << Failure.new(@runner.current_filename, "Links > Internal",
+                "internally linking to #{url}, which does not exist", line: metadata[:line], status: nil, content: nil)
+              to_add << [url, metadata, false]
+              next
+            end
+
+            unless hash_exists?(url)
+              @failed_checks << Failure.new(@runner.current_filename, "Links > Internal",
+                "internally linking to #{url}; the file exists, but the hash '#{url.hash}' does not", line: metadata[:line], status: nil, content: nil)
+              to_add << [url, metadata, false]
+              next
+            end
+
+            to_add << [url, metadata, true]
+          end
+        end
+
+        # adding directly to the cache above results in an endless loop
+        to_add.each do |(url, metadata, exists)|
+          @cache.add_internal(url.to_s, metadata, exists)
+        end
+
+        @failed_checks
+      end
+
+      private def file_exists?(url)
+        absolute_path = url.absolute_path
+        return @runner.checked_paths[url.absolute_path] if @runner.checked_paths.key?(absolute_path)
+
+        @runner.checked_paths[url.absolute_path] = File.exist?(absolute_path)
+      end
+
+      # verify the target hash
+      private def hash_exists?(url)
+        href_hash = url.hash
+        return true if blank?(href_hash)
+        return true unless @runner.options[:check_internal_hash]
+
+        # prevents searching files we didn't ask about
+        return false unless url.known_extension?
+        return false unless url.has_hash?
+
+        decoded_href_hash = Addressable::URI.unescape(href_hash)
+        fragment_ids = [href_hash, decoded_href_hash]
+        # https://www.w3.org/TR/html5/single-page.html#scroll-to-fragid
+        fragment_ids.include?("top") || !find_fragments(fragment_ids, url).empty?
+      end
+
+      private def find_fragments(fragment_ids, url)
+        xpaths = fragment_ids.uniq.flat_map do |frag_id|
+          escaped_frag_id = "'#{frag_id.split("'").join("', \"'\", '")}', ''"
+          [
+            "//*[case_sensitive_equals(@id, concat(#{escaped_frag_id}))]",
+            "//*[case_sensitive_equals(@name, concat(#{escaped_frag_id}))]",
+          ]
+        end
+        xpaths << XpathFunctions.new
+
+        html = create_nokogiri(url.absolute_path)
+        html.xpath(*xpaths)
+      end
+    end
+  end
+end

data/lib/html_proofer/url_validator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class UrlValidator
+    include HTMLProofer::Utils
+
+    def initialize(runner)
+      @runner = runner
+
+      @cache = @runner.cache
+      @logger = @runner.logger
+
+      @failed_checks = []
+    end
+  end
+end

data/lib/{html-proofer → html_proofer}/utils.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
+require "nokogiri"
 
 module HTMLProofer
   module Utils
@@ -8,21 +8,18 @@ module HTMLProofer
       "#{count} #{count == 1 ? single : plural}"
     end
 
+    def blank?(obj)
+      obj.nil? || obj.empty?
+    end
+
     def create_nokogiri(path)
       content = if File.exist?(path) && !File.directory?(path)
-                  File.open(path).read
-                else
-                  path
-                end
+        File.read(path)
+      else
+        path
+      end
 
       Nokogiri::HTML5(content, max_errors: -1)
     end
-
-    def swap(href, replacement)
-      replacement.each do |link, replace|
-        href = href.gsub(link, replace)
-      end
-      href
-    end
   end
 end

data/lib/{html-proofer → html_proofer}/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = '3.19.4'
+  VERSION = "4.4.0"
 end

data/lib/html_proofer/xpath_functions.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  # https://stackoverflow.com/a/8812293
+  class XpathFunctions
+    def case_sensitive_equals(node_set, str_to_match)
+      node_set.find_all { |node| node.to_s.== str_to_match.to_s }
+    end
+  end
+end

data/lib/html_proofer.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+lib_dir = File.join(File.dirname(__dir__), "lib")
+gem_loader = Zeitwerk::Loader.for_gem
+gem_loader.inflector.inflect(
+  "html_proofer" => "HTMLProofer"
+)
+gem_loader.ignore(File.join(lib_dir, "html-proofer.rb"))
+gem_loader.setup
+
+require "html_proofer/version"
+
+require "parallel"
+require "fileutils"
+
+if ENV.fetch("DEBUG", false)
+  require "awesome_print"
+  require "debug"
+end
+
+module HTMLProofer
+  class << self
+    def check_file(file, options = {})
+      raise ArgumentError unless file.is_a?(String)
+      raise ArgumentError, "#{file} does not exist" unless File.exist?(file)
+
+      options[:type] = :file
+      HTMLProofer::Runner.new(file, options)
+    end
+
+    def check_directory(directory, options = {})
+      raise ArgumentError unless directory.is_a?(String)
+      raise ArgumentError, "#{directory} does not exist" unless Dir.exist?(directory)
+
+      options[:type] = :directory
+      HTMLProofer::Runner.new([directory], options)
+    end
+
+    def check_directories(directories, options = {})
+      raise ArgumentError unless directories.is_a?(Array)
+
+      options[:type] = :directory
+      directories.each do |directory|
+        raise ArgumentError, "#{directory} does not exist" unless Dir.exist?(directory)
+      end
+      HTMLProofer::Runner.new(directories, options)
+    end
+
+    def check_links(links, options = {})
+      raise ArgumentError unless links.is_a?(Array)
+
+      options[:type] = :links
+      HTMLProofer::Runner.new(links, options)
+    end
+  end
+end
+
+gem_loader.eager_load