html-proofer 3.19.4 → 4.0.0

data/lib/html_proofer/url_validator/external.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "typhoeus"
+require "uri"
+
+module HTMLProofer
+  class UrlValidator
+    class External < UrlValidator
+      include HTMLProofer::Utils
+
+      attr_reader :external_urls
+      attr_writer :before_request
+
+      def initialize(runner, external_urls)
+        super(runner)
+
+        @external_urls = external_urls
+        @hydra = Typhoeus::Hydra.new(@runner.options[:hydra])
+        @before_request = []
+
+        @paths_with_queries = {}
+      end
+
+      def validate
+        if @cache.enabled?
+          urls_to_check = @runner.load_external_cache
+          run_external_link_checker(urls_to_check)
+        else
+          run_external_link_checker(@external_urls)
+        end
+
+        @failed_checks
+      end
+
+      # Proofer runs faster if we pull out all the external URLs and run the checks
+      # at the end. Otherwise, we're halting the consuming process for every file during
+      # `process_files`.
+      #
+      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
+      #
+      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
+      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
+      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
+      # not available as an option.
+      def run_external_link_checker(external_urls)
+        # Route log from Typhoeus/Ethon to our own logger
+        Ethon.logger = @logger
+
+        external_urls.each_pair do |external_url, metadata|
+          url = Attribute::Url.new(@runner, external_url, base_url: nil)
+
+          unless url.valid?
+            add_failure(metadata, "#{url} is an invalid URL", 0)
+            next
+          end
+
+          next unless new_url_query_values?(url)
+
+          method = if @runner.options[:check_external_hash] && url.hash?
+            :get
+          else
+            :head
+          end
+
+          queue_request(method, url, metadata)
+        end
+
+        @hydra.run
+      end
+
+      def queue_request(method, url, filenames)
+        opts = @runner.options[:typhoeus].merge(method: method)
+        request = Typhoeus::Request.new(url.url, opts)
+        @before_request.each do |callback|
+          callback.call(request)
+        end
+        request.on_complete { |response| response_handler(response, url, filenames) }
+        @hydra.queue(request)
+      end
+
+      def response_handler(response, url, filenames)
+        method = response.request.options[:method]
+        href = response.request.base_url.to_s
+        response_code = response.code
+        response.body.delete!("\x00")
+
+        @logger.log(:debug, "Received a #{response_code} for #{href}")
+
+        return if @runner.options[:ignore_status_codes].include?(response_code)
+
+        if response_code.between?(200, 299)
+          @cache.add_external(href, filenames, response_code, "OK") unless check_hash_in_2xx_response(href, url,
+            response, filenames)
+        elsif response.timed_out?
+          handle_timeout(href, filenames, response_code)
+        elsif response_code.zero?
+          handle_connection_failure(href, filenames, response_code, response.status_message)
+        elsif method == :head # some servers don't support HEAD
+          queue_request(:get, url, filenames)
+        else
+          return if @runner.options[:only_4xx] && !response_code.between?(400, 499)
+
+          # Received a non-successful http response.
+          status_message = blank?(response.status_message) ? "" : ": #{response.status_message}"
+          msg = "External link #{href} failed#{status_message}"
+          add_failure(filenames, msg, response_code)
+          @cache.add_external(href, filenames, response_code, msg)
+        end
+      end
+
+      # Even though the response was a success, we may have been asked to check
+      # if the hash on the URL exists on the page
+      def check_hash_in_2xx_response(href, url, response, filenames)
+        return false if @runner.options[:only_4xx]
+        return false unless @runner.options[:check_external_hash]
+        return false unless url.hash?
+
+        hash = url.hash
+
+        body_doc = create_nokogiri(response.body)
+
+        unencoded_hash = Addressable::URI.unescape(hash)
+        xpath = [%(//*[@name="#{hash}"]|/*[@name="#{unencoded_hash}"]|//*[@id="#{hash}"]|//*[@id="#{unencoded_hash}"])]
+        # user-content is a special addition by GitHub.
+        if url.host =~ /github\.com/i
+          xpath << [%(//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])]
+          # when linking to a file on GitHub, like #L12-L34, only the first "L" portion
+          # will be identified as a linkable portion
+          xpath << [%(//td[@id="#{Regexp.last_match[1]}"])] if hash =~ /\A(L\d)+/
+        end
+
+        return unless body_doc.xpath(xpath.join("|")).empty?
+
+        msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
+        add_failure(filenames, msg, response.code)
+        @cache.add_external(href, filenames, response.code, msg)
+        true
+      end
+
+      def handle_timeout(href, filenames, response_code)
+        msg = "External link #{href} failed: got a time out (response code #{response_code})"
+        @cache.add_external(href, filenames, 0, msg)
+        return if @runner.options[:only_4xx]
+
+        add_failure(filenames, msg, response_code)
+      end
+
+      def handle_connection_failure(href, metadata, response_code, status_message)
+        msgs = [<<~MSG,
+          External link #{href} failed with something very wrong.
+          It's possible libcurl couldn't connect to the server, or perhaps the request timed out.
+          Sometimes, making too many requests at once also breaks things.
+        MSG
+        ]
+
+        msgs << "Either way, the return message from the server is: #{status_message}" unless blank?(status_message)
+
+        msg = msgs.join("\n").chomp
+
+        @cache.add_external(href, metadata, 0, msg)
+        return if @runner.options[:only_4xx]
+
+        add_failure(metadata, msg, response_code)
+      end
+
+      def add_failure(metadata, description, status = nil)
+        if blank?(metadata) # possible if we're checking an array of links
+          @failed_checks << Failure.new("", "Links > External", description, status: status)
+        else
+          metadata.each do |m|
+            @failed_checks << Failure.new(m[:filename], "Links > External", description, line: m[:line], status: status)
+          end
+        end
+      end
+
+      # remember queries we've seen, ignore future ones
+      private def new_url_query_values?(url)
+        return true if (query_values = url.query_values).nil?
+
+        queries = query_values.keys.join("-")
+        domain_path = url.domain_path
+        if @paths_with_queries[domain_path].nil?
+          @paths_with_queries[domain_path] = [queries]
+          true
+        elsif !@paths_with_queries[domain_path].include?(queries)
+          @paths_with_queries[domain_path] << queries
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/html_proofer/url_validator/internal.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class UrlValidator
+    class Internal < UrlValidator
+      attr_reader :internal_urls
+
+      def initialize(runner, internal_urls)
+        super(runner)
+
+        @internal_urls = internal_urls
+      end
+
+      def validate
+        if @cache.enabled?
+          urls_to_check = @runner.load_internal_cache
+          run_internal_link_checker(urls_to_check)
+        else
+          run_internal_link_checker(@internal_urls)
+        end
+
+        @failed_checks
+      end
+
+      def run_internal_link_checker(links)
+        to_add = []
+        links.each_pair do |link, matched_files|
+          matched_files.each do |metadata|
+            url = HTMLProofer::Attribute::Url.new(@runner, link, base_url: metadata[:base_url])
+
+            @runner.current_source = metadata[:source]
+            @runner.current_filename = metadata[:filename]
+
+            unless file_exists?(url)
+              @failed_checks << Failure.new(@runner.current_filename, "Links > Internal",
+                "internally linking to #{url}, which does not exist", line: metadata[:line], status: nil, content: nil)
+              to_add << [url, metadata, false]
+              next
+            end
+
+            unless hash_exists?(url)
+              @failed_checks << Failure.new(@runner.current_filename, "Links > Internal",
+                "internally linking to #{url}; the file exists, but the hash '#{url.hash}' does not", line: metadata[:line], status: nil, content: nil)
+              to_add << [url, metadata, false]
+              next
+            end
+
+            to_add << [url, metadata, true]
+          end
+        end
+
+        # adding directly to the cache above results in an endless loop
+        to_add.each do |(url, metadata, exists)|
+          @cache.add_internal(url.to_s, metadata, exists)
+        end
+
+        @failed_checks
+      end
+
+      private def file_exists?(url)
+        absolute_path = url.absolute_path
+        return @runner.checked_paths[url.absolute_path] if @runner.checked_paths.key?(absolute_path)
+
+        @runner.checked_paths[url.absolute_path] = File.exist?(absolute_path)
+      end
+
+      # verify the target hash
+      private def hash_exists?(url)
+        href_hash = url.hash
+        return true if blank?(href_hash)
+
+        # prevents searching files we didn't ask about
+        return false unless url.known_extension?
+
+        decoded_href_hash = Addressable::URI.unescape(href_hash)
+        fragment_ids = [href_hash, decoded_href_hash]
+        # https://www.w3.org/TR/html5/single-page.html#scroll-to-fragid
+        fragment_ids.include?("top") || !find_fragments(fragment_ids, url).empty?
+      end
+
+      private def find_fragments(fragment_ids, url)
+        xpaths = fragment_ids.uniq.flat_map do |frag_id|
+          escaped_frag_id = "'#{frag_id.split("'").join("', \"'\", '")}', ''"
+          [
+            "//*[case_sensitive_equals(@id, concat(#{escaped_frag_id}))]",
+            "//*[case_sensitive_equals(@name, concat(#{escaped_frag_id}))]",
+          ]
+        end
+        xpaths << XpathFunctions.new
+
+        html = create_nokogiri(url.absolute_path)
+        html.xpath(*xpaths)
+      end
+    end
+  end
+end

data/lib/html_proofer/url_validator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class UrlValidator
+    include HTMLProofer::Utils
+
+    def initialize(runner)
+      @runner = runner
+
+      @cache = @runner.cache
+      @logger = @runner.logger
+
+      @failed_checks = []
+    end
+  end
+end

data/lib/{html-proofer → html_proofer}/utils.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
+require "nokogiri"
 
 module HTMLProofer
   module Utils
@@ -8,21 +8,18 @@ module HTMLProofer
       "#{count} #{count == 1 ? single : plural}"
     end
 
+    def blank?(obj)
+      obj.nil? || obj.empty?
+    end
+
     def create_nokogiri(path)
       content = if File.exist?(path) && !File.directory?(path)
-                  File.open(path).read
-                else
-                  path
-                end
+        File.read(path)
+      else
+        path
+      end
 
       Nokogiri::HTML5(content, max_errors: -1)
     end
-
-    def swap(href, replacement)
-      replacement.each do |link, replace|
-        href = href.gsub(link, replace)
-      end
-      href
-    end
   end
 end

data/lib/{html-proofer → html_proofer}/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = '3.19.4'
+  VERSION = "4.0.0"
 end

data/lib/html_proofer/xpath_functions.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  # https://stackoverflow.com/a/8812293
+  class XpathFunctions
+    def case_sensitive_equals(node_set, str_to_match)
+      node_set.find_all { |node| node.to_s.== str_to_match.to_s }
+    end
+  end
+end

data/lib/html_proofer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+lib_dir = File.join(File.dirname(__dir__), "lib")
+gem_loader = Zeitwerk::Loader.for_gem
+gem_loader.inflector.inflect(
+  "html_proofer" => "HTMLProofer"
+)
+gem_loader.ignore(File.join(lib_dir, "html-proofer.rb"))
+gem_loader.setup
+
+require "html_proofer/version"
+
+require "parallel"
+require "fileutils"
+
+if ENV.fetch("DEBUG", false)
+  require "awesome_print"
+  require "debug"
+end
+
+module HTMLProofer
+  def self.check_file(file, options = {})
+    raise ArgumentError unless file.is_a?(String)
+    raise ArgumentError, "#{file} does not exist" unless File.exist?(file)
+
+    options[:type] = :file
+    HTMLProofer::Runner.new(file, options)
+  end
+
+  def self.check_directory(directory, options = {})
+    raise ArgumentError unless directory.is_a?(String)
+    raise ArgumentError, "#{directory} does not exist" unless Dir.exist?(directory)
+
+    options[:type] = :directory
+    HTMLProofer::Runner.new([directory], options)
+  end
+
+  def self.check_directories(directories, options = {})
+    raise ArgumentError unless directories.is_a?(Array)
+
+    options[:type] = :directory
+    directories.each do |directory|
+      raise ArgumentError, "#{directory} does not exist" unless Dir.exist?(directory)
+    end
+    HTMLProofer::Runner.new(directories, options)
+  end
+
+  def self.check_links(links, options = {})
+    raise ArgumentError unless links.is_a?(Array)
+
+    options[:type] = :links
+    HTMLProofer::Runner.new(links, options)
+  end
+end
+
+gem_loader.eager_load

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html-proofer
 version: !ruby/object:Gem::Version
-  version: 3.19.4
+  version: 4.0.0
 platform: ruby
 authors:
 - Garen Torikian
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-19 00:00:00.000000000 Z
+date: 2022-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -193,7 +207,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubocop-performance
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -258,23 +272,29 @@ extra_rdoc_files: []
 files:
 - bin/htmlproofer
 - lib/html-proofer.rb
-- lib/html-proofer/cache.rb
-- lib/html-proofer/check.rb
-- lib/html-proofer/check/favicon.rb
-- lib/html-proofer/check/html.rb
-- lib/html-proofer/check/images.rb
-- lib/html-proofer/check/links.rb
-- lib/html-proofer/check/opengraph.rb
-- lib/html-proofer/check/scripts.rb
-- lib/html-proofer/configuration.rb
-- lib/html-proofer/element.rb
-- lib/html-proofer/issue.rb
-- lib/html-proofer/log.rb
-- lib/html-proofer/middleware.rb
-- lib/html-proofer/runner.rb
-- lib/html-proofer/url_validator.rb
-- lib/html-proofer/utils.rb
-- lib/html-proofer/version.rb
+- lib/html_proofer.rb
+- lib/html_proofer/attribute.rb
+- lib/html_proofer/attribute/url.rb
+- lib/html_proofer/cache.rb
+- lib/html_proofer/check.rb
+- lib/html_proofer/check/favicon.rb
+- lib/html_proofer/check/images.rb
+- lib/html_proofer/check/links.rb
+- lib/html_proofer/check/open_graph.rb
+- lib/html_proofer/check/scripts.rb
+- lib/html_proofer/configuration.rb
+- lib/html_proofer/element.rb
+- lib/html_proofer/failure.rb
+- lib/html_proofer/log.rb
+- lib/html_proofer/reporter.rb
+- lib/html_proofer/reporter/cli.rb
+- lib/html_proofer/runner.rb
+- lib/html_proofer/url_validator.rb
+- lib/html_proofer/url_validator/external.rb
+- lib/html_proofer/url_validator/internal.rb
+- lib/html_proofer/utils.rb
+- lib/html_proofer/version.rb
+- lib/html_proofer/xpath_functions.rb
 homepage: https://github.com/gjtorikian/html-proofer
 licenses:
 - MIT
@@ -299,7 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.13
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: A set of tests to validate your HTML output. These tests check if your image