html-proofer 3.19.4 → 4.0.0.rc1

data/lib/html-proofer/runner.rb

@@ -1,249 +0,0 @@
-# frozen_string_literal: true
-
-module HTMLProofer
-  class Runner
-    include HTMLProofer::Utils
-
-    attr_reader :options, :internal_urls, :external_urls, :failures
-
-    def initialize(src, opts = {})
-      @src = src
-
-      @options = HTMLProofer::Configuration::PROOFER_DEFAULTS.merge(opts)
-
-      @options[:typhoeus] = HTMLProofer::Configuration::TYPHOEUS_DEFAULTS.merge(opts[:typhoeus] || {})
-      @options[:hydra] = HTMLProofer::Configuration::HYDRA_DEFAULTS.merge(opts[:hydra] || {})
-
-      @options[:parallel] = HTMLProofer::Configuration::PARALLEL_DEFAULTS.merge(opts[:parallel] || {})
-      @options[:validation] = HTMLProofer::Configuration::VALIDATION_DEFAULTS.merge(opts[:validation] || {})
-      @options[:cache] = HTMLProofer::Configuration::CACHE_DEFAULTS.merge(opts[:cache] || {})
-
-      @type = @options.delete(:type)
-      @logger = HTMLProofer::Log.new(@options[:log_level])
-      @cache = Cache.new(@logger, @options[:cache])
-      @internal_link_checks = nil
-
-      # Add swap patterns for internal domains
-      unless @options[:internal_domains].empty?
-        @options[:internal_domains].each do |dom|
-          @options[:url_swap][Regexp.new("^http://#{dom}")] = ''
-          @options[:url_swap][Regexp.new("^https://#{dom}")] = ''
-          @options[:url_swap][Regexp.new("^//#{dom}")] = ''
-        end
-      end
-
-      @internal_urls = {}
-      @internal_urls_to_paths = {}
-      @external_urls = {}
-      @failures = []
-      @before_request = []
-    end
-
-    def run
-      if @type == :links
-        @logger.log :info, "Running #{checks} on #{@src}... \n\n"
-        check_list_of_links unless @options[:disable_external]
-      else
-        @logger.log :info, "Running #{checks} on #{@src} on *#{@options[:extension]}... \n\n"
-        check_files
-        file_text = pluralize(files.length, 'file', 'files')
-        @logger.log :info, "Ran on #{file_text}!\n\n"
-      end
-
-      if @failures.empty?
-        @logger.log :info, 'HTML-Proofer finished successfully.'
-      else
-        @failures.uniq!
-        print_failed_tests
-      end
-    end
-
-    def check_list_of_links
-      if @options[:url_swap]
-        @src = @src.map do |url|
-          swap(url, @options[:url_swap])
-        end
-      end
-      @external_urls = @src.each_with_object({}) do |url, hash|
-        hash[url] = nil
-      end
-      validate_external_urls
-    end
-
-    # Collects any external URLs found in a directory of files. Also collectes
-    # every failed test from process_files.
-    # Sends the external URLs to Typhoeus for batch processing.
-    def check_files
-      process_files.each do |item|
-        @external_urls.merge!(item[:external_urls])
-        @failures.concat(item[:failures])
-      end
-
-      # TODO: lazy. if we're checking only external links,
-      # we'll just trash all the failed tests. really, we should
-      # just not run those other checks at all.
-      if @options[:external_only]
-        @failures = []
-        validate_external_urls
-      elsif !@options[:disable_external]
-        validate_external_urls
-        validate_internal_urls
-      else
-        validate_internal_urls
-      end
-    end
-
-    # Walks over each implemented check and runs them on the files, in parallel.
-    def process_files
-      if @options[:parallel].empty?
-        files.map { |path| check_path(path) }
-      else
-        Parallel.map(files, @options[:parallel]) { |path| check_path(path) }
-      end
-    end
-
-    def check_parsed(html, path)
-      result = { external_urls: {}, failures: [] }
-
-      @src = [@src] if @type == :file
-
-      @src.each do |src|
-        checks.each do |klass|
-          @logger.log :debug, "Checking #{klass.to_s.downcase} on #{path} ..."
-          check = Object.const_get(klass).new(src, path, html, @logger, @cache, @options)
-          check.run
-
-          if klass == 'LinkCheck'
-            @internal_link_checks = check
-            check.internal_urls.each_pair do |url, internal_urls|
-              if @internal_urls_to_paths[url]
-                @internal_urls_to_paths[url].concat(internal_urls.map(&:path))
-              else
-                @internal_urls_to_paths[url] = internal_urls.map(&:path)
-              end
-            end
-            @internal_urls.merge!(check.internal_urls)
-          end
-
-          external_urls = check.external_urls
-          external_urls = check.external_urls.transform_keys { |url| swap(url, @options[:url_swap]) } if @options[:url_swap]
-          result[:external_urls].merge!(external_urls)
-          result[:failures].concat(check.issues)
-        end
-      end
-      result
-    end
-
-    def check_path(path)
-      check_parsed(create_nokogiri(path), path)
-    end
-
-    def validate_external_urls
-      url_validator = HTMLProofer::UrlValidator.new(@logger, @cache, @external_urls, @options)
-      url_validator.before_request = @before_request
-      @failures.concat(url_validator.run)
-      @external_urls = url_validator.external_urls
-    end
-
-    def validate_internal_urls
-      if @cache.use_cache?
-        urls_to_check = load_internal_cache
-
-        urls_to_check.each_pair do |url, internal_urls|
-          # pulled from cache
-          internal_urls = @internal_urls[url] unless internal_urls.first.is_a?(LinkCheck::InternalLink)
-
-          result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.path, internal_urls.first.line, internal_urls.first.content)
-          code = result ? 200 : 404
-          @cache.add(url, @internal_urls_to_paths[url].sort, code, '') # TODO: blank msg for now
-        end
-        @cache.write
-      else
-        @internal_urls.values.flatten.each do |internal_url|
-          result = @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content)
-          next if result
-
-          @failures.concat(@internal_link_checks.issues) unless @internal_link_checks.issues.length.zero?
-        end
-      end
-    end
-
-    def files
-      @files ||= if @type == :directory
-                   @src.map do |src|
-                     pattern = File.join(src, '**', "*#{@options[:extension]}")
-                     files = Dir.glob(pattern).select { |fn| File.file? fn }
-                     files.reject { |f| ignore_file?(f) }
-                   end.flatten
-                 elsif @type == :file && File.extname(@src) == @options[:extension]
-                   [@src].reject { |f| ignore_file?(f) }
-                 else
-                   []
-                 end
-    end
-
-    def ignore_file?(file)
-      @options[:file_ignore].each do |pattern|
-        return true if pattern.is_a?(String) && pattern == file
-        return true if pattern.is_a?(Regexp) && pattern =~ file
-      end
-
-      false
-    end
-
-    def checks
-      return @checks if defined?(@checks) && !@checks.nil?
-
-      return (@checks = ['LinkCheck']) if @type == :links
-
-      @checks = HTMLProofer::Check.subchecks.map(&:name)
-      @checks.delete('FaviconCheck') unless @options[:check_favicon]
-      @checks.delete('HtmlCheck') unless @options[:check_html]
-      @checks.delete('OpenGraphCheck') unless @options[:check_opengraph]
-      @options[:checks_to_ignore].each { |ignored| @checks.delete(ignored) }
-      @checks
-    end
-
-    def failed_tests
-      result = []
-      return result if @failures.empty?
-
-      @failures.each { |f| result << f.to_s }
-      result
-    end
-
-    def print_failed_tests
-      sorted_failures = SortedIssues.new(@failures, @options[:error_sort], @logger)
-
-      sorted_failures.sort_and_report
-      count = @failures.length
-      failure_text = pluralize(count, 'failure', 'failures')
-      @logger.log :fatal, "\nHTML-Proofer found #{failure_text}!"
-      exit 1
-    end
-
-    # Set before_request callback.
-    #
-    # @example Set before_request.
-    #   request.before_request { |request| p "yay" }
-    #
-    # @param [ Block ] block The block to execute.
-    #
-    # @yield [ Typhoeus::Request ]
-    #
-    # @return [ Array<Block> ] All before_request blocks.
-    def before_request(&block)
-      @before_request ||= []
-      @before_request << block if block
-      @before_request
-    end
-
-    def load_internal_cache
-      urls_to_check = @cache.retrieve_urls(@internal_urls, :internal)
-      cache_text = pluralize(urls_to_check.count, 'internal link', 'internal links')
-      @logger.log :info, "Found #{cache_text} in the cache..."
-
-      urls_to_check
-    end
-  end
-end

data/lib/html-proofer/url_validator.rb

@@ -1,237 +0,0 @@
-# frozen_string_literal: true
-
-require 'typhoeus'
-require 'uri'
-require_relative './utils'
-require_relative './cache'
-
-module HTMLProofer
-  class UrlValidator
-    include HTMLProofer::Utils
-
-    attr_reader :external_urls
-    attr_writer :before_request
-
-    def initialize(logger, cache, external_urls, options)
-      @logger = logger
-      @external_urls = external_urls
-      @failed_tests = []
-      @options = options
-      @hydra = Typhoeus::Hydra.new(@options[:hydra])
-      @cache = cache
-      @before_request = []
-    end
-
-    def run
-      @external_urls = remove_query_values
-
-      if @cache.use_cache?
-        urls_to_check = @cache.retrieve_urls(@external_urls, :external)
-        external_link_checker(urls_to_check)
-        @cache.write
-      else
-        external_link_checker(@external_urls)
-      end
-
-      @failed_tests
-    end
-
-    def remove_query_values
-      return nil if @external_urls.nil?
-
-      paths_with_queries = {}
-      iterable_external_urls = @external_urls.dup
-      @external_urls.each_key do |url|
-        uri = begin
-          Addressable::URI.parse(url)
-        rescue URI::Error, Addressable::URI::InvalidURIError
-          @logger.log :error, "#{url} is an invalid URL"
-          nil
-        end
-        next if uri.nil? || uri.query.nil?
-
-        iterable_external_urls.delete(url) unless new_url_query_values?(uri, paths_with_queries)
-      end
-      iterable_external_urls
-    end
-
-    # remember queries we've seen, ignore future ones
-    def new_url_query_values?(uri, paths_with_queries)
-      queries = uri.query_values.keys.join('-')
-      domain_path = extract_domain_path(uri)
-      if paths_with_queries[domain_path].nil?
-        paths_with_queries[domain_path] = [queries]
-        true
-      elsif !paths_with_queries[domain_path].include?(queries)
-        paths_with_queries[domain_path] << queries
-        true
-      else
-        false
-      end
-    end
-
-    def extract_domain_path(uri)
-      uri.host + uri.path
-    end
-
-    # Proofer runs faster if we pull out all the external URLs and run the checks
-    # at the end. Otherwise, we're halting the consuming process for every file during
-    # `process_files`.
-    #
-    # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
-    #
-    # Finally, we'll first make a HEAD request, rather than GETing all the contents.
-    # If the HEAD fails, we'll fall back to GET, as some servers are not configured
-    # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
-    # not available as an option.
-    def external_link_checker(external_urls)
-      external_urls = external_urls.sort.to_h
-
-      count = external_urls.length
-      check_text = pluralize(count, 'external link', 'external links')
-      @logger.log :info, "Checking #{check_text}..."
-
-      # Route log from Typhoeus/Ethon to our own logger
-      Ethon.logger = @logger
-
-      establish_queue(external_urls)
-
-      @hydra.run
-    end
-
-    def establish_queue(external_urls)
-      external_urls.each_pair do |url, filenames|
-        url = begin
-          clean_url(url)
-        rescue URI::Error, Addressable::URI::InvalidURIError
-          add_external_issue(filenames, "#{url} is an invalid URL")
-          next
-        end
-
-        method = if hash?(url) && @options[:check_external_hash]
-                   :get
-                 else
-                   :head
-                 end
-        queue_request(method, url, filenames)
-      end
-    end
-
-    def clean_url(href)
-      # catch any obvious issues, like strings in port numbers
-      parsed = Addressable::URI.parse(href)
-      if href =~ /^([!#{Regexp.last_match(0)}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/
-        href
-      else
-        parsed.normalize
-      end
-    end
-
-    def queue_request(method, href, filenames)
-      opts = @options[:typhoeus].merge(method: method)
-      request = Typhoeus::Request.new(href, opts)
-      @before_request.each do |callback|
-        callback.call(request)
-      end
-      request.on_complete { |response| response_handler(response, filenames) }
-      @hydra.queue request
-    end
-
-    def response_handler(response, filenames)
-      effective_url = response.options[:effective_url]
-      href = response.request.base_url.to_s
-      method = response.request.options[:method]
-      response_code = response.code
-      response.body.delete!("\x00")
-
-      debug_msg = if filenames.nil?
-                    "Received a #{response_code} for #{href}"
-                  else
-                    "Received a #{response_code} for #{href}  in #{filenames.join(' ')}"
-                  end
-
-      @logger.log :debug, debug_msg
-
-      return if @options[:http_status_ignore].include?(response_code)
-
-      if response_code.between?(200, 299)
-        @cache.add(href, filenames, response_code) unless check_hash_in_2xx_response(href, effective_url, response, filenames)
-      elsif response.timed_out?
-        handle_timeout(href, filenames, response_code)
-      elsif response_code.zero?
-        handle_failure(effective_url, filenames, response_code, response.return_message)
-      elsif method == :head
-        queue_request(:get, href, filenames)
-      else
-        return if @options[:only_4xx] && !response_code.between?(400, 499)
-
-        # Received a non-successful http response.
-        msg = "External link #{href} failed: #{response_code} #{response.return_message}"
-        add_external_issue(filenames, msg, response_code)
-        @cache.add(href, filenames, response_code, msg)
-      end
-    end
-
-    # Even though the response was a success, we may have been asked to check
-    # if the hash on the URL exists on the page
-    def check_hash_in_2xx_response(href, effective_url, response, filenames)
-      return false if @options[:only_4xx]
-      return false unless @options[:check_external_hash]
-      return false unless (hash = hash?(href))
-
-      body_doc = create_nokogiri(response.body)
-
-      unencoded_hash = Addressable::URI.unescape(hash)
-      xpath = [%(//*[@name="#{hash}"]|/*[@name="#{unencoded_hash}"]|//*[@id="#{hash}"]|//*[@id="#{unencoded_hash}"])]
-      # user-content is a special addition by GitHub.
-      if URI.parse(href).host =~ /github\.com/i
-        xpath << [%(//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])]
-        # when linking to a file on GitHub, like #L12-L34, only the first "L" portion
-        # will be identified as a linkable portion
-        xpath << [%(//td[@id="#{Regexp.last_match[1]}"])] if hash =~ /\A(L\d)+/
-      end
-
-      return unless body_doc.xpath(xpath.join('|')).empty?
-
-      msg = "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not"
-      add_external_issue(filenames, msg, response.code)
-      @cache.add(href, filenames, response.code, msg)
-      true
-    end
-
-    def handle_timeout(href, filenames, response_code)
-      msg = "External link #{href} failed: got a time out (response code #{response_code})"
-      @cache.add(href, filenames, 0, msg)
-      return if @options[:only_4xx]
-
-      add_external_issue(filenames, msg, response_code)
-    end
-
-    def handle_failure(href, filenames, response_code, return_message)
-      msg = "External link #{href} failed: response code #{response_code} means something's wrong.
-             It's possible libcurl couldn't connect to the server or perhaps the request timed out.
-             Sometimes, making too many requests at once also breaks things.
-             Either way, the return message (if any) from the server is: #{return_message}"
-      @cache.add(href, filenames, 0, msg)
-      return if @options[:only_4xx]
-
-      add_external_issue(filenames, msg, response_code)
-    end
-
-    def add_external_issue(filenames, desc, status = nil)
-      # possible if we're checking an array of links
-      if filenames.nil?
-        @failed_tests << Issue.new('', desc, status: status)
-      else
-        filenames.each { |f| @failed_tests << Issue.new(f, desc, status: status) }
-      end
-    end
-
-    # Does the URL have a hash?
-    def hash?(url)
-      URI.parse(url).fragment
-    rescue URI::InvalidURIError
-      false
-    end
-  end
-end