html-proofer 2.6.4 → 3.0.0

data/lib/html-proofer/log.rb

@@ -0,0 +1,49 @@
+require 'yell'
+require 'colored'
+
+module HTMLProofer
+  class Log
+    include Yell::Loggable
+
+    def initialize(log_level)
+      @logger = Yell.new(:format => false, \
+                         :name => 'HTMLProofer', \
+                         :level => "gte.#{log_level}") do |l|
+        l.adapter :stdout, :level => [:debug, :info, :warn]
+        l.adapter :stderr, :level => [:error, :fatal]
+      end
+    end
+
+    def log(level, message)
+      color = case level
+              when :debug
+                :light_blue
+              when :info
+                :blue
+              when :warn
+                :yellow
+              when :error, :fatal
+                :red
+              end
+
+      log_with_color(level, color, message)
+    end
+
+    def log_with_color(level, color, message)
+      @logger.send level, colorize(color, message)
+    end
+
+    def colorize(color, message)
+      if $stdout.isatty && $stderr.isatty
+        Colored.colorize(message, foreground: color)
+      else
+        message
+      end
+    end
+
+    # dumb override to play nice with Typhoeus/Ethon
+    def debug(message = nil)
+      log(:debug, message) unless message.nil?
+    end
+  end
+end

data/lib/html-proofer/runner.rb

@@ -0,0 +1,160 @@
+module HTMLProofer
+  class Runner
+    include HTMLProofer::Utils
+
+    attr_reader :options, :external_urls
+
+    def initialize(src, opts = {})
+      @src = src
+
+      @options = HTMLProofer::Configuration::PROOFER_DEFAULTS.merge(opts)
+
+      @options[:typhoeus] = HTMLProofer::Configuration::TYPHOEUS_DEFAULTS.merge(opts[:typhoeus] || {})
+      @options[:hydra] = HTMLProofer::Configuration::HYDRA_DEFAULTS.merge(opts[:hydra] || {})
+
+      @options[:parallel] = HTMLProofer::Configuration::PARALLEL_DEFAULTS.merge(opts[:parallel] || {})
+      @options[:validation] = HTMLProofer::Configuration::VALIDATION_DEFAULTS.merge(opts[:validation] || {})
+      @options[:cache] = HTMLProofer::Configuration::CACHE_DEFAULTS.merge(opts[:cache] || {})
+
+      @type = @options.delete(:type)
+      @logger = HTMLProofer::Log.new(@options[:log_level])
+
+      if !@options[:cache].empty? && !File.exist?(STORAGE_DIR)
+        FileUtils.mkdir_p(STORAGE_DIR)
+      end
+
+      @failures = []
+    end
+
+    def run
+      @logger.log :info, "Running #{checks} on #{@src} on *#{@options[:extension]}... \n\n"
+
+      if @type == :links
+        check_list_of_links unless @options[:disable_external]
+      else
+        check_files
+        file_text = pluralize(files.length, 'file', 'files')
+        @logger.log :info, "Ran on #{file_text}!\n\n"
+      end
+
+      if @failures.empty?
+        @logger.log_with_color :info, :green, 'HTML-Proofer finished successfully.'
+      else
+        print_failed_tests
+      end
+    end
+
+    def check_list_of_links
+      if @options[:url_swap]
+        @src = @src.map do |url|
+          swap(url, @options[:url_swap])
+        end
+      end
+      @external_urls = Hash[*@src.map { |s| [s, nil] }.flatten]
+      validate_urls
+    end
+
+    # Collects any external URLs found in a directory of files. Also collectes
+    # every failed test from process_files.
+    # Sends the external URLs to Typhoeus for batch processing.
+    def check_files
+      @external_urls = {}
+
+      process_files.each do |item|
+        @external_urls.merge!(item[:external_urls])
+        @failures.concat(item[:failures])
+      end
+
+      # TODO: lazy. if we're checking only external links,
+      # we'll just trash all the failed tests. really, we should
+      # just not run those other checks at all.
+      if @options[:external_only]
+        @failures = []
+        validate_urls
+      elsif !@options[:disable_external]
+        validate_urls
+      end
+    end
+
+    # Walks over each implemented check and runs them on the files, in parallel.
+    def process_files
+      if @options[:parallel].empty?
+        files.map { |path| check_path(path) }
+      else
+        Parallel.map(files, @options[:parallel]) { |path| check_path(path) }
+      end
+    end
+
+    def check_path(path)
+      result = { :external_urls => {}, :failures => [] }
+      html = create_nokogiri(path)
+
+      @src = [@src] if @type == :file
+
+      @src.each do |src|
+        checks.each do |klass|
+          @logger.log :debug, "Checking #{klass.to_s.downcase} on #{path} ..."
+          check = Object.const_get(klass).new(src, path, html, @options)
+          check.run
+          result[:external_urls].merge!(check.external_urls)
+          result[:failures].concat(check.issues)
+        end
+      end
+      result
+    end
+
+    def validate_urls
+      url_validator = HTMLProofer::UrlValidator.new(@logger, @external_urls, @options)
+      @failures.concat(url_validator.run)
+      @external_urls = url_validator.external_urls
+    end
+
+    def files
+      @files ||= if @type == :directory
+                   @src.map do |src|
+                     pattern = File.join(src, '**', "*#{@options[:extension]}")
+                     files = Dir.glob(pattern).select { |fn| File.file? fn }
+                     files.reject { |f| ignore_file?(f) }
+                   end.flatten
+                 elsif @type == :file && File.extname(@src) == @options[:extension]
+                   [@src].reject { |f| ignore_file?(f) }
+                 else
+                   []
+                 end
+    end
+
+    def ignore_file?(file)
+      @options[:file_ignore].each do |pattern|
+        return true if pattern.is_a?(String) && pattern == file
+        return true if pattern.is_a?(Regexp) && pattern =~ file
+      end
+
+      false
+    end
+
+    def checks
+      return @checks unless @checks.nil?
+      @checks = HTMLProofer::Check.subchecks.map(&:name)
+      @checks.delete('FaviconCheck') unless @options[:check_favicon]
+      @checks.delete('HtmlCheck') unless @options[:check_html]
+      @options[:checks_to_ignore].each { |ignored| @checks.delete(ignored) }
+      @checks
+    end
+
+    def failed_tests
+      result = []
+      return result if @failures.empty?
+      @failures.each { |f| result << f.to_s }
+      result
+    end
+
+    def print_failed_tests
+      sorted_failures = SortedIssues.new(@failures, @options[:error_sort], @logger)
+
+      sorted_failures.sort_and_report
+      count = @failures.length
+      failure_text = pluralize(count, 'failure', 'failures')
+      fail @logger.colorize :red, "HTML-Proofer found #{failure_text}!"
+    end
+  end
+end

data/lib/html-proofer/url_validator.rb

@@ -0,0 +1,218 @@
+require 'typhoeus'
+require 'uri'
+require_relative './utils'
+require_relative './cache'
+
+module HTMLProofer
+  class UrlValidator
+    include HTMLProofer::Utils
+
+    attr_reader :external_urls
+
+    def initialize(logger, external_urls, options)
+      @logger = logger
+      @external_urls = external_urls
+      @failed_tests = []
+      @options = options
+      @hydra = Typhoeus::Hydra.new(@options[:hydra])
+      @cache = Cache.new(@logger, @options[:cache])
+    end
+
+    def run
+      @external_urls = remove_query_values
+
+      if @cache.use_cache?
+        urls_to_check = load_cache
+        external_link_checker(urls_to_check)
+        @cache.write
+      else
+        external_link_checker(@external_urls)
+      end
+
+      @failed_tests
+    end
+
+    def remove_query_values
+      return nil if @external_urls.nil?
+      paths_with_queries = {}
+      iterable_external_urls = @external_urls.dup
+      @external_urls.keys.each do |url|
+        uri = begin
+                Addressable::URI.parse(url)
+              rescue URI::Error, Addressable::URI::InvalidURIError
+                @logger.log :error, "#{url} is an invalid URL"
+                nil
+              end
+        next if uri.nil? || uri.query.nil?
+        iterable_external_urls.delete(url) unless new_url_query_values?(uri, paths_with_queries)
+      end
+      iterable_external_urls
+    end
+
+    # remember queries we've seen, ignore future ones
+    def new_url_query_values?(uri, paths_with_queries)
+      queries = uri.query_values.keys.join('-')
+      domain_path = extract_domain_path(uri)
+      if paths_with_queries[domain_path].nil?
+        paths_with_queries[domain_path] = [queries]
+        true
+      elsif !paths_with_queries[domain_path].include?(queries)
+        paths_with_queries[domain_path] << queries
+        true
+      else
+        false
+      end
+    end
+
+    def extract_domain_path(uri)
+      uri.host + uri.path
+    end
+
+    def load_cache
+      cache_count = @cache.size
+      cache_text = pluralize(cache_count, 'link', 'links')
+
+      @logger.log :info, "Found #{cache_text} in the cache..."
+
+      @cache.retrieve_urls(@external_urls)
+    end
+
+    # Proofer runs faster if we pull out all the external URLs and run the checks
+    # at the end. Otherwise, we're halting the consuming process for every file during
+    # `process_files`.
+    #
+    # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
+    #
+    # Finally, we'll first make a HEAD request, rather than GETing all the contents.
+    # If the HEAD fails, we'll fall back to GET, as some servers are not configured
+    # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
+    # not available as an option.
+    def external_link_checker(external_urls)
+      external_urls = Hash[external_urls.sort]
+
+      count = external_urls.length
+      check_text = pluralize(count, 'external link', 'external links')
+      @logger.log :info, "Checking #{check_text}..."
+
+      # Route log from Typhoeus/Ethon to our own logger
+      Ethon.logger = @logger
+
+      establish_queue(external_urls)
+
+      @hydra.run
+    end
+
+    def establish_queue(external_urls)
+      external_urls.each_pair do |url, filenames|
+        url = begin
+                 clean_url(url)
+               rescue URI::Error, Addressable::URI::InvalidURIError
+                 add_external_issue(filenames, "#{url} is an invalid URL")
+                 next
+               end
+
+        method = if hash?(url) && @options[:check_external_hash]
+                   :get
+                 else
+                   :head
+                 end
+        queue_request(method, url, filenames)
+      end
+    end
+
+    def clean_url(href)
+      Addressable::URI.parse(href).normalize
+    end
+
+    def queue_request(method, href, filenames)
+      opts = @options[:typhoeus].merge({ :method => method })
+      request = Typhoeus::Request.new(href, opts)
+      request.on_complete { |response| response_handler(response, filenames) }
+      @hydra.queue request
+    end
+
+    def response_handler(response, filenames)
+      effective_url = response.options[:effective_url]
+      href = response.request.base_url.to_s
+      method = response.request.options[:method]
+      response_code = response.code
+
+      debug_msg = "Received a #{response_code} for #{href}"
+      debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
+      @logger.log :debug, debug_msg
+
+      return if @options[:http_status_ignore].include?(response_code)
+
+      if response_code.between?(200, 299)
+        unless check_hash_in_2xx_response(href, effective_url, response, filenames)
+          @cache.add(href, filenames, response_code)
+        end
+      elsif response.timed_out?
+        handle_timeout(href, filenames, response_code)
+      elsif response_code == 0
+        handle_failure(href, filenames, response_code)
+      elsif method == :head
+        queue_request(:get, href, filenames)
+      else
+        return if @options[:only_4xx] && !response_code.between?(400, 499)
+        # Received a non-successful http response.
+        msg = "External link #{href} failed: #{response_code} #{response.return_message}"
+        add_external_issue(filenames, msg, response_code)
+        @cache.add(href, filenames, response_code, msg)
+      end
+    end
+
+    # Even though the response was a success, we may have been asked to check
+    # if the hash on the URL exists on the page
+    def check_hash_in_2xx_response(href, effective_url, response, filenames)
+      return false if @options[:only_4xx]
+      return false unless @options[:check_external_hash]
+      return false unless (hash = hash?(href))
+
+      body_doc = create_nokogiri(response.body)
+
+      # user-content is a special addition by GitHub.
+      xpath = %(//*[@name="#{hash}"]|//*[@id="#{hash}"])
+      if URI.parse(href).host.match(/github\.com/i)
+        xpath << %(|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])
+      end
+
+      return unless body_doc.xpath(xpath).empty?
+
+      msg = "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not"
+      add_external_issue(filenames, msg, response.code)
+      @cache.add(href, filenames, response.code, msg)
+      true
+    end
+
+    def handle_timeout(href, filenames, response_code)
+      msg = "External link #{href} failed: got a time out (response code #{response_code})"
+      @cache.add(href, filenames, 0, msg)
+      return if @options[:only_4xx]
+      add_external_issue(filenames, msg, response_code)
+    end
+
+    def handle_failure(href, filenames, response_code)
+      msg = "External link #{href} failed: response code #{response_code} means something's wrong"
+      @cache.add(href, filenames, 0, msg)
+      return if @options[:only_4xx]
+      add_external_issue(filenames, msg, response_code)
+    end
+
+    def add_external_issue(filenames, desc, status = nil)
+      # possible if we're checking an array of links
+      if filenames.nil?
+        @failed_tests << Issue.new('', desc, status: status)
+      else
+        filenames.each { |f| @failed_tests << Issue.new(f, desc, status: status) }
+      end
+    end
+
+    # Does the URL have a hash?
+    def hash?(url)
+      URI.parse(url).fragment
+    rescue URI::InvalidURIError
+      false
+    end
+  end
+end

data/lib/html-proofer/utils.rb

@@ -0,0 +1,40 @@
+require 'nokogiri'
+
+module HTMLProofer
+  module Utils
+    STORAGE_DIR = File.join('tmp', '.htmlproofer')
+
+    def pluralize(count, single, plural)
+      "#{count} " << (count == 1 ? single : plural)
+    end
+
+    def create_nokogiri(path)
+      if File.exist? path
+        content = File.open(path).read
+      else
+        content = path
+      end
+
+      Nokogiri::HTML(clean_content(content))
+    end
+    module_function :create_nokogiri
+
+    def swap(href, replacement)
+      replacement.each do |link, replace|
+        href = href.gsub(link, replace)
+      end
+      href
+    end
+    module_function :swap
+
+    # address a problem with Nokogiri's parsing URL entities
+    # problem from http://git.io/vBYU1
+    # solution from http://git.io/vBYUi
+    def clean_content(string)
+      string.gsub(%r{https?://([^>]+)}i) do |url|
+        url.gsub(/&(?!amp;)/, '&amp;')
+      end
+    end
+    module_function :clean_content
+  end
+end