html-proofer 2.5.2 → 3.0.0

data/lib/html/proofer/checks/links.rb

@@ -1,124 +0,0 @@
-class LinkCheckable < ::HTML::Proofer::Checkable
-
-  def href
-    real_attr @href
-  end
-
-  def id
-    real_attr @id
-  end
-
-  def name
-    real_attr @name
-  end
-
-  def missing_href?
-    href.nil? && name.nil? && id.nil?
-  end
-
-  def placeholder?
-    (id || name) && href.nil?
-  end
-end
-
-class LinkCheck < ::HTML::Proofer::CheckRunner
-  include HTML::Proofer::Utils
-
-  def run
-    @html.css('a, link').each do |node|
-      link = LinkCheckable.new(node, self)
-      line = node.line
-
-      next if link.ignore?
-      next if link.href =~ /^javascript:/ # can't put this in ignore? because the URI does not parse
-      next if link.placeholder?
-
-      # is it even a valid URL?
-      unless link.valid?
-        add_issue("#{link.href} is an invalid URL", line)
-        next
-      end
-
-      check_schemes(link, line)
-
-      # is there even a href?
-      if link.missing_href?
-        add_issue('anchor has no href attribute', line)
-        next
-      end
-
-      # intentionally here because we still want valid? & missing_href? to execute
-      next if link.non_http_remote?
-
-      # does the file even exist?
-      if link.remote?
-        add_to_external_urls(link.href, line)
-        next
-      elsif !link.internal?
-        add_issue("internally linking to #{link.href}, which does not exist", line) unless link.exists?
-      end
-
-      # does the local directory have a trailing slash?
-      if link.unslashed_directory? link.absolute_path
-        add_issue("internally linking to a directory #{link.absolute_path} without trailing slash", line)
-        next
-      end
-
-      # verify the target hash
-      handle_hash(link, line) if link.hash
-    end
-
-    external_urls
-  end
-
-  def check_schemes(link, line)
-    case link.scheme
-    when 'mailto'
-      handle_mailto(link, line)
-    when 'tel'
-      handle_tel(link, line)
-    when 'http'
-      add_issue("#{link.href} is not an HTTPS link", line) if @options[:enforce_https]
-    end
-  end
-
-  def handle_mailto(link, line)
-    if link.path.empty?
-      add_issue("#{link.href} contains no email address", line)
-    elsif !link.path.include?('@')
-      add_issue("#{link.href} contains an invalid email address", line)
-    end
-  end
-
-  def handle_tel(link, line)
-    add_issue("#{link.href} contains no phone number", line) if link.path.empty?
-  end
-
-  def handle_hash(link, line)
-    if link.internal?
-      unless hash_check @html, link.hash
-        add_issue("linking to internal hash ##{link.hash} that does not exist", line)
-      end
-    elsif link.external?
-      external_link_check(link, line)
-    end
-  end
-
-  def external_link_check(link, line)
-    if !link.exists?
-      add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line)
-    else
-      target_html = create_nokogiri link.absolute_path
-      unless hash_check target_html, link.hash
-        add_issue("linking to #{link.href}, but #{link.hash} does not exist", line)
-      end
-    end
-  end
-
-  def hash_check(html, href_hash)
-    html.xpath("//*[case_insensitive_equals(@id, '#{href_hash}')]", \
-               "//*[case_insensitive_equals(@name, '#{href_hash}')]", \
-               HTML::Proofer::XpathFunctions.new).length > 0
-  end
-
-end

data/lib/html/proofer/checks/scripts.rb

@@ -1,40 +0,0 @@
-# encoding: utf-8
-
-class ScriptCheckable < ::HTML::Proofer::Checkable
-
-  def src
-    real_attr @src
-  end
-
-  def missing_src?
-    !src
-  end
-
-  def blank?
-    @text.strip.empty?
-  end
-
-end
-
-class ScriptCheck < ::HTML::Proofer::CheckRunner
-  def run
-    @html.css('script').each do |node|
-      script = ScriptCheckable.new(node, self)
-      line = node.line
-
-      next if script.ignore?
-      next unless script.blank?
-
-      # does the script exist?
-      if script.missing_src?
-        add_issue('script is empty and has no src attribute', line)
-      elsif script.remote?
-        add_to_external_urls(script.src, line)
-      else
-        add_issue("internal script #{script.src} does not exist", line) unless script.exists?
-      end
-    end
-
-    external_urls
-  end
-end

data/lib/html/proofer/log.rb

@@ -1,42 +0,0 @@
-require 'yell'
-require 'colored'
-
-module HTML
-  class Proofer
-    class Log
-      include Yell::Loggable
-
-      def initialize(verbose, verbosity = nil)
-        log_level = if verbosity.nil?
-                      verbose ? :debug : :info
-                    else
-                      verbosity
-                    end
-
-        @logger = Yell.new(:format => false, \
-                           :name => 'HTML::Proofer', \
-                           :level => "gte.#{log_level}") do |l|
-          l.adapter :stdout, :level => [:debug, :info, :warn]
-          l.adapter :stderr, :level => [:error, :fatal]
-        end
-      end
-
-      def log(level, color, message)
-        @logger.send level, colorize(color, message)
-      end
-
-      def colorize(color, message)
-        if $stdout.isatty && $stderr.isatty
-          Colored.colorize(message, foreground: color)
-        else
-          message
-        end
-      end
-
-      # dumb override to play nice with Typhoeus/Ethon
-      def debug(message = nil)
-        log(:debug, :yellow, message) unless message.nil?
-      end
-    end
-  end
-end

data/lib/html/proofer/url_validator.rb

@@ -1,179 +0,0 @@
-require 'typhoeus'
-require 'uri'
-require_relative './utils'
-
-module HTML
-  class Proofer
-    class UrlValidator
-      include HTML::Proofer::Utils
-
-      attr_accessor :logger, :external_urls, :iterable_external_urls, :hydra
-
-      def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts)
-        @logger = logger
-        @external_urls = external_urls
-        @iterable_external_urls = {}
-        @failed_tests = []
-        @options = options
-        @hydra = Typhoeus::Hydra.new(hydra_opts)
-        @typhoeus_opts = typhoeus_opts
-        @external_domain_paths_with_queries = {}
-      end
-
-      def run
-        @iterable_external_urls = remove_query_values
-        external_link_checker(@iterable_external_urls)
-        @failed_tests
-      end
-
-      def remove_query_values
-        return if @external_urls.nil?
-        iterable_external_urls = @external_urls.dup
-        @external_urls.keys.each do |url|
-          uri = begin
-                  Addressable::URI.parse(url)
-                rescue URI::Error, Addressable::URI::InvalidURIError
-                  @logger.log :error, :red, "#{url} is an invalid URL"
-                  nil
-                end
-          next if uri.nil? || uri.query.nil?
-          iterable_external_urls.delete(url) unless new_url_query_values?(uri)
-        end
-        iterable_external_urls
-      end
-
-      # remember queries we've seen, ignore future ones
-      def new_url_query_values?(uri)
-        queries = uri.query_values.keys.join('-')
-        domain_path = extract_domain_path(uri)
-        if @external_domain_paths_with_queries[domain_path].nil?
-          @external_domain_paths_with_queries[domain_path] = [queries]
-          true
-        elsif !@external_domain_paths_with_queries[domain_path].include?(queries)
-          @external_domain_paths_with_queries[domain_path] << queries
-          true
-        else
-          false
-        end
-      end
-
-      def extract_domain_path(uri)
-        uri.host + uri.path
-      end
-
-      # Proofer runs faster if we pull out all the external URLs and run the checks
-      # at the end. Otherwise, we're halting the consuming process for every file during
-      # the check_directory_of_files process.
-      #
-      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
-      #
-      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
-      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
-      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
-      # not an option.
-      def external_link_checker(external_urls)
-        external_urls = Hash[external_urls.sort]
-
-        count = external_urls.length
-        check_text = "#{count} " << (count == 1 ? 'external link' : 'external links')
-        logger.log :info, :blue, "Checking #{check_text}..."
-
-        Ethon.logger = logger # log from Typhoeus/Ethon
-
-        url_processor(external_urls)
-
-        logger.log :debug, :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..."
-        hydra.run
-      end
-
-      def url_processor(external_urls)
-        external_urls.each_pair do |href, filenames|
-          href = begin
-                   clean_url(href)
-                 rescue URI::Error, Addressable::URI::InvalidURIError
-                   add_external_issue(filenames, "#{href} is an invalid URL")
-                   next
-                 end
-
-          if hash?(href) && @options[:check_external_hash]
-            queue_request(:get, href, filenames)
-          else
-            queue_request(:head, href, filenames)
-          end
-        end
-      end
-
-      def clean_url(href)
-        Addressable::URI.parse(href).normalize
-      end
-
-      def queue_request(method, href, filenames)
-        request = Typhoeus::Request.new(href, @typhoeus_opts.merge({ :method => method }))
-        request.on_complete { |response| response_handler(response, filenames) }
-        hydra.queue request
-      end
-
-      def response_handler(response, filenames)
-        effective_url = response.options[:effective_url]
-        href = response.request.base_url.to_s
-        method = response.request.options[:method]
-        response_code = response.code
-
-        debug_msg = "Received a #{response_code} for #{href}"
-        debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
-        logger.log :debug, :yellow, debug_msg
-
-        if response_code.between?(200, 299)
-          check_hash_in_2xx_response(href, effective_url, response, filenames)
-        elsif response.timed_out?
-          handle_timeout(href, filenames, response_code)
-        elsif method == :head
-          queue_request(:get, href, filenames)
-        else
-          return if @options[:only_4xx] && !response_code.between?(400, 499)
-          # Received a non-successful http response.
-          add_external_issue(filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code)
-        end
-      end
-
-      # Even though the response was a success, we may have been asked to check
-      # if the hash on the URL exists on the page
-      def check_hash_in_2xx_response(href, effective_url, response, filenames)
-        return if @options[:only_4xx]
-        return unless @options[:check_external_hash]
-        return unless (hash = hash?(href))
-
-        body_doc = create_nokogiri(response.body)
-
-        # user-content is a special addition by GitHub.
-        xpath = %(//*[@name="#{hash}"]|//*[@id="#{hash}"])
-        if URI.parse(href).host.match(/github\.com/i)
-          xpath << %(|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])
-        end
-
-        return unless body_doc.xpath(xpath).empty?
-
-        add_external_issue filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response.code
-      end
-
-      def handle_timeout(href, filenames, response_code)
-        return if @options[:only_4xx]
-        add_external_issue filenames, "External link #{href} failed: got a time out", response_code
-      end
-
-      def add_external_issue(filenames, desc, status = nil)
-        if filenames.nil?
-          @failed_tests << CheckRunner::Issue.new('', desc, nil, status)
-        else
-          filenames.each { |f| @failed_tests << CheckRunner::Issue.new(f, desc, nil, status) }
-        end
-      end
-
-      def hash?(url)
-        URI.parse(url).fragment
-      rescue URI::InvalidURIError
-        nil
-      end
-    end
-  end
-end

data/lib/html/proofer/utils.rb

@@ -1,26 +0,0 @@
-require 'nokogiri'
-
-module HTML
-  class Proofer
-    module Utils
-      def create_nokogiri(path)
-        if File.exist? path
-          content = File.open(path).read
-        else
-          content = path
-        end
-
-        Nokogiri::HTML(content)
-      end
-      module_function :create_nokogiri
-
-      def swap(href, replacement)
-        replacement.each do |link, replace|
-          href = href.gsub(link, replace)
-        end
-        href
-      end
-      module_function :swap
-    end
-  end
-end

data/lib/html/proofer/version.rb

@@ -1,5 +0,0 @@
-module HTML
-  class Proofer
-    VERSION = '2.5.2'
-  end
-end

data/lib/html/proofer/xpathfunctions.rb

@@ -1,9 +0,0 @@
-module HTML
-  class Proofer
-    class XpathFunctions
-      def case_insensitive_equals(node_set, str_to_match)
-        node_set.find_all { |node| node.to_s.downcase == str_to_match.to_s.downcase }
-      end
-    end
-  end
-end

data/lib/html/proofer.rb

@@ -1,202 +0,0 @@
-def require_all(path)
-  glob = File.join(File.dirname(__FILE__), path, '*.rb')
-  Dir[glob].each do |f|
-    require f
-  end
-end
-
-require_all 'proofer'
-require_all 'proofer/check_runner'
-require_all 'proofer/checks'
-require_relative './proofer/utils'
-require_relative './proofer/xpathfunctions'
-
-require 'parallel'
-
-begin
-  require 'awesome_print'
-rescue LoadError; end
-
-module HTML
-
-  class Proofer
-    include HTML::Proofer::Utils
-
-    attr_reader :options, :typhoeus_opts, :hydra_opts, :parallel_opts, :validation_opts, :external_urls, :iterable_external_urls
-
-    TYPHOEUS_DEFAULTS = {
-      :followlocation => true,
-      :headers => {
-        'User-Agent' => "Mozilla/5.0 (compatible; HTML Proofer/#{VERSION}; +https://github.com/gjtorikian/html-proofer)"
-      }
-    }
-
-    def initialize(src, opts = {})
-      @src = src
-
-      if opts[:verbose]
-        warn '`@options[:verbose]` will be removed in a future 3.x.x release: http://git.io/vGHHh'
-      end
-      if opts[:href_ignore]
-        warn '`@options[:href_ignore]` will be renamed in a future 3.x.x release: http://git.io/vGHHy'
-      end
-
-      @proofer_opts = {
-        :ext => '.html',
-        :check_favicon => false,
-        :href_swap => [],
-        :href_ignore => [],
-        :file_ignore => [],
-        :url_ignore => [],
-        :check_external_hash => false,
-        :alt_ignore => [],
-        :empty_alt_ignore => false,
-        :enforce_https => false,
-        :disable_external => false,
-        :verbose => false,
-        :only_4xx => false,
-        :directory_index_file => 'index.html',
-        :check_html => false,
-        :error_sort => :path,
-        :checks_to_ignore => []
-      }
-
-      @typhoeus_opts = TYPHOEUS_DEFAULTS.merge(opts[:typhoeus] || {})
-      opts.delete(:typhoeus)
-
-      @hydra_opts = opts[:hydra] || {}
-      opts.delete(:hydra)
-
-      # fall back to parallel defaults
-      @parallel_opts = opts[:parallel] || {}
-      opts.delete(:parallel)
-
-      @validation_opts = opts[:validation] || {}
-      opts.delete(:validation)
-
-      @options = @proofer_opts.merge(opts)
-
-      @failed_tests = []
-    end
-
-    def logger
-      @logger ||= HTML::Proofer::Log.new(@options[:verbose], @options[:verbosity])
-    end
-
-    def run
-      count = checks.length
-      check_text = "#{checks} " << (count == 1 ? 'check' : 'checks')
-      logger.log :info, :blue, "Running #{check_text} on #{@src} on *#{@options[:ext]}... \n\n"
-
-      if @src.is_a?(Array) && !@options[:disable_external]
-        check_list_of_links
-      else
-        check_directory_of_files
-      end
-
-      if @failed_tests.empty?
-        logger.log :info, :green, 'HTML-Proofer finished successfully.'
-      else
-        print_failed_tests
-      end
-    end
-
-    def check_list_of_links
-      if @options[:href_swap]
-        @src = @src.map do |url|
-          swap(url, @options[:href_swap])
-        end
-      end
-      @external_urls = Hash[*@src.map { |s| [s, nil] }.flatten]
-      validate_urls
-    end
-
-    # Collects any external URLs found in a directory of files. Also collectes
-    # every failed test from check_files_for_internal_woes.
-    # Sends the external URLs to Typhoeus for batch processing.
-    def check_directory_of_files
-      @external_urls = {}
-      results = check_files_for_internal_woes
-
-      results.each do |item|
-        @external_urls.merge!(item[:external_urls])
-        @failed_tests.concat(item[:failed_tests])
-      end
-
-      validate_urls unless @options[:disable_external]
-
-      logger.log :info, :blue, "Ran on #{files.length} files!\n\n"
-    end
-
-    # Walks over each implemented check and runs them on the files, in parallel.
-    def check_files_for_internal_woes
-      Parallel.map(files, @parallel_opts) do |path|
-        html = create_nokogiri(path)
-        result = { :external_urls => {}, :failed_tests => [] }
-
-        checks.each do |klass|
-          logger.log :debug, :yellow, "Checking #{klass.to_s.downcase} on #{path} ..."
-          check = Object.const_get(klass).new(@src, path, html, @options, @typhoeus_opts, @hydra_opts, @parallel_opts, @validation_opts)
-          check.run
-          result[:external_urls].merge!(check.external_urls)
-          result[:failed_tests].concat(check.issues) if check.issues.length > 0
-        end
-        result
-      end
-    end
-
-    def validate_urls
-      url_validator = HTML::Proofer::UrlValidator.new(logger, @external_urls, @options, @typhoeus_opts, @hydra_opts)
-      @failed_tests.concat(url_validator.run)
-      @iterable_external_urls = url_validator.iterable_external_urls
-    end
-
-    def files
-      if File.directory? @src
-        pattern = File.join(@src, '**', "*#{@options[:ext]}")
-        files = Dir.glob(pattern).select { |fn| File.file? fn }
-        files.reject { |f| ignore_file?(f) }
-      elsif File.extname(@src) == @options[:ext]
-        [@src].reject { |f| ignore_file?(f) }
-      else
-        []
-      end
-    end
-
-    def ignore_file?(file)
-      options[:file_ignore].each do |pattern|
-        return true if pattern.is_a?(String) && pattern == file
-        return true if pattern.is_a?(Regexp) && pattern =~ file
-      end
-
-      false
-    end
-
-    def checks
-      return @checks unless @checks.nil?
-      @checks = HTML::Proofer::CheckRunner.checks.map(&:name)
-      @checks.delete('FaviconCheck') unless @options[:check_favicon]
-      @checks.delete('HtmlCheck') unless @options[:check_html]
-      @options[:checks_to_ignore].each do |ignored|
-        @checks.delete(ignored)
-      end
-      @checks
-    end
-
-    def failed_tests
-      return [] if @failed_tests.empty?
-      result = []
-      @failed_tests.each { |f| result << f.to_s }
-      result
-    end
-
-    def print_failed_tests
-      sorted_failures = HTML::Proofer::CheckRunner::SortedIssues.new(@failed_tests, @options[:error_sort], logger)
-
-      sorted_failures.sort_and_report
-      count = @failed_tests.length
-      failure_text = "#{count} " << (count == 1 ? 'failure' : 'failures')
-      fail logger.colorize :red, "HTML-Proofer found #{failure_text}!"
-    end
-  end
-end