html-proofer 2.6.4 → 3.0.0

data/lib/html/proofer/checkable.rb

@@ -1,168 +0,0 @@
-require 'addressable/uri'
-require_relative './utils'
-
-module HTML
-  class Proofer
-    # Represents the superclass from which all checks derive.
-    class Checkable
-      include HTML::Proofer::Utils
-
-      attr_reader :line
-
-      def initialize(obj, check)
-        obj.attributes.each_pair do |attribute, value|
-          instance_variable_set("@#{attribute.tr('-:.', '_')}".to_sym, value.value)
-        end
-
-        @text = obj.content
-        @check = check
-        @checked_paths = {}
-        @type = self.class.name
-        @line = obj.line
-
-        if @href && @check.options[:href_swap]
-          @href = swap(@href, @check.options[:href_swap])
-        end
-
-        # fix up missing protocols
-        @href.insert 0, 'http:' if @href =~ %r{^//}
-        @src.insert 0, 'http:' if @src =~ %r{^//}
-      end
-
-      def url
-        @src || @srcset || @href || ''
-      end
-
-      def valid?
-        !parts.nil?
-      end
-
-      def parts
-        @parts ||= Addressable::URI.parse url
-      rescue URI::Error, Addressable::URI::InvalidURIError
-        @parts = nil
-      end
-
-      def path
-        Addressable::URI.unencode parts.path unless parts.nil?
-      end
-
-      def hash
-        parts.fragment unless parts.nil?
-      end
-
-      def scheme
-        parts.scheme unless parts.nil?
-      end
-
-      # path is to an external server
-      def remote?
-        %w( http https ).include? scheme
-      end
-
-      def non_http_remote?
-        !scheme.nil? && !remote?
-      end
-
-      def ignore?
-        return true if @data_proofer_ignore
-
-        # ignore base64 encoded images
-        if %w(ImageCheckable FaviconCheckable).include? @type
-          return true if url.match(/^data:image/)
-        end
-
-        # ignore user defined URLs
-        return true if ignores_pattern_check(@check.url_ignores)
-
-        # ignore user defined hrefs
-        if 'LinkCheckable' == @type
-          return true if ignores_pattern_check(@check.href_ignores)
-        end
-
-        # ignore user defined alts
-        if 'ImageCheckable' == @type
-          return true if ignores_pattern_check(@check.alt_ignores)
-        end
-      end
-
-      def ignore_empty_alt?
-        @check.empty_alt_ignore
-      end
-
-      def allow_hash_href?
-        @check.allow_hash_href
-      end
-
-      # path is external to the file
-      def external?
-        !internal?
-      end
-
-      # path is an anchor or a query
-      def internal?
-        url.start_with? '#', '?'
-      end
-
-      def file_path
-        return if path.nil?
-
-        if path =~ %r{^/} # path relative to root
-          base = File.directory?(@check.src) ? @check.src : File.dirname(@check.src)
-        elsif File.exist?(File.expand_path path, @check.src) # relative links, path is a file
-          base = File.dirname @check.path
-        elsif File.exist?(File.join(File.dirname(@check.path), path)) # relative links in nested dir, path is a file
-          base = File.dirname @check.path
-        else # relative link, path is a directory
-          base = @check.path
-        end
-
-        file = File.join base, path
-
-        # implicit index support
-        if File.directory?(file) && !unslashed_directory?(file)
-          file = File.join file, @check.options[:directory_index_file]
-        end
-
-        file
-      end
-
-      # checks if a file exists relative to the current pwd
-      def exists?
-        return @checked_paths[absolute_path] if @checked_paths.key? absolute_path
-        @checked_paths[absolute_path] = File.exist? absolute_path
-      end
-
-      def absolute_path
-        path = file_path || @check.path
-        File.expand_path path, Dir.pwd
-      end
-
-      def ignores_pattern_check(links)
-        links.each do |ignore|
-          if ignore.is_a? String
-            return true if ignore == url
-          elsif ignore.is_a? Regexp
-            return true if ignore =~ url
-          end
-        end
-
-        false
-      end
-
-      def unslashed_directory?(file)
-        File.directory?(file) && !file.end_with?(File::SEPARATOR) && !follow_location?
-      end
-
-      def follow_location?
-        @check.typhoeus_opts && @check.typhoeus_opts[:followlocation]
-      end
-
-      private
-
-      def real_attr(attr)
-        attr.to_s unless attr.nil? || attr.empty?
-      end
-    end
-  end
-end

data/lib/html/proofer/checks/html.rb

@@ -1,46 +0,0 @@
-# encoding: utf-8
-
-class HtmlCheck < ::HTML::Proofer::CheckRunner
-  # new html5 tags (source: http://www.w3schools.com/html/html5_new_elements.asp)
-  # and svg child tags (source: https://developer.mozilla.org/en-US/docs/Web/SVG/Element)
-  HTML5_TAGS = %w(article aside bdi details dialog figcaption
-                  figure footer header main mark menuitem meter
-                  nav progress rp rt ruby section summary
-                  time wbr datalist keygen output color date
-                  datetime datetime-local email month number
-                  range search tel time url week canvas
-                  svg audio embed source track video
-                  altGlyph altGlyphDef altGlyphItem animate
-                  animateColor animateMotion animateTransform
-                  circle clipPath color-profile cursor defs
-                  desc ellipse feBlend feColorMatrix
-                  feComponentTransfer feComposite feConvolveMatrix
-                  feDiffuseLighting feDisplacementMap feDistantLight
-                  feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur
-                  feImage feMerge feMergeNode feMorphology feOffset
-                  fePointLight feSpecularLighting feSpotLight feTile
-                  feTurbulence filter font font-face font-face-format
-                  font-face-name font-face-src font-face-uri
-                  foreignObject g glyph glyphRef hkern image line
-                  linearGradient marker mask metadata missing-glyph
-                  mpath path pattern polygon polyline radialGradient
-                  rect set stop switch symbol text textPath tref tspan use
-                  view vkern)
-
-  SCRIPT_EMBEDS_MSG = /Element script embeds close tag/
-
-  def run
-    @html.errors.each do |error|
-      message = error.message
-      line    = error.line
-      # Nokogiri (or rather libxml2 underhood) only recognizes html4 tags,
-      # so we need to skip errors caused by the new tags in html5
-      next if HTML5_TAGS.include? message[/Tag ([\w-]+) invalid/o, 1]
-
-      # tags embedded in scripts are used in templating languages: http://git.io/vOovv
-      next if @validation_opts[:ignore_script_embeds] && message =~ SCRIPT_EMBEDS_MSG
-
-      add_issue(message, line)
-    end
-  end
-end

data/lib/html/proofer/checks/images.rb

@@ -1,54 +0,0 @@
-# encoding: utf-8
-
-class ImageCheckable < ::HTML::Proofer::Checkable
-  SCREEN_SHOT_REGEX = /Screen(?: |%20)Shot(?: |%20)\d+-\d+-\d+(?: |%20)at(?: |%20)\d+.\d+.\d+/
-
-  attr_reader :alt
-
-  def empty_alt_tag?
-    alt.strip.empty?
-  end
-
-  def terrible_filename?
-    src =~ SCREEN_SHOT_REGEX
-  end
-
-  def src
-    real_attr(@src) || real_attr(@srcset)
-  end
-
-  def missing_src?
-    !src
-  end
-end
-
-class ImageCheck < ::HTML::Proofer::CheckRunner
-  def run
-    @html.css('img').each do |node|
-      img = ImageCheckable.new(node, self)
-      line = node.line
-
-      next if img.ignore?
-
-      # screenshot filenames should return because of terrible names
-      next add_issue("image has a terrible filename (#{img.src})", line) if img.terrible_filename?
-
-      # does the image exist?
-      if img.missing_src?
-        add_issue('image has no src or srcset attribute', line)
-      else
-        if img.remote?
-          add_to_external_urls(img.src, line)
-        else
-          add_issue("internal image #{img.src} does not exist", line) unless img.exists?
-        end
-      end
-
-      if img.alt.nil? || (img.empty_alt_tag? && !img.ignore_empty_alt?)
-        add_issue("image #{img.src} does not have an alt attribute", line)
-      end
-    end
-
-    external_urls
-  end
-end

data/lib/html/proofer/checks/scripts.rb

@@ -1,40 +0,0 @@
-# encoding: utf-8
-
-class ScriptCheckable < ::HTML::Proofer::Checkable
-
-  def src
-    real_attr @src
-  end
-
-  def missing_src?
-    !src
-  end
-
-  def blank?
-    @text.strip.empty?
-  end
-
-end
-
-class ScriptCheck < ::HTML::Proofer::CheckRunner
-  def run
-    @html.css('script').each do |node|
-      script = ScriptCheckable.new(node, self)
-      line = node.line
-
-      next if script.ignore?
-      next unless script.blank?
-
-      # does the script exist?
-      if script.missing_src?
-        add_issue('script is empty and has no src attribute', line)
-      elsif script.remote?
-        add_to_external_urls(script.src, line)
-      else
-        add_issue("internal script #{script.src} does not exist", line) unless script.exists?
-      end
-    end
-
-    external_urls
-  end
-end

data/lib/html/proofer/configuration.rb

@@ -1,48 +0,0 @@
-module HTML
-  class Proofer
-    module Configuration
-      require_relative 'version'
-
-      PROOFER_DEFAULTS = {
-        :allow_hash_href => false,
-        :alt_ignore => [],
-        :check_external_hash => false,
-        :check_favicon => false,
-        :check_html => false,
-        :checks_to_ignore => [],
-        :directory_index_file => 'index.html',
-        :disable_external => false,
-        :empty_alt_ignore => false,
-        :enforce_https => false,
-        :error_sort => :path,
-        :ext => '.html',
-        :external_only => false,
-        :file_ignore => [],
-        :href_ignore => [],
-        :href_swap => [],
-        :only_4xx => false,
-        :url_ignore => [],
-        :verbose => false
-      }
-
-      TYPHOEUS_DEFAULTS = {
-        :followlocation => true,
-        :headers => {
-          'User-Agent' => "Mozilla/5.0 (compatible; HTML Proofer/#{HTML::Proofer::VERSION}; +https://github.com/gjtorikian/html-proofer)"
-        }
-      }
-
-      HYDRA_DEFAULTS = {
-        :max_concurrency => 50
-      }
-
-      def self.to_regex?(item)
-        if item.start_with?('/') && item.end_with?('/')
-          Regexp.new item[1...-1]
-        else
-          item
-        end
-      end
-    end
-  end
-end

data/lib/html/proofer/log.rb

@@ -1,42 +0,0 @@
-require 'yell'
-require 'colored'
-
-module HTML
-  class Proofer
-    class Log
-      include Yell::Loggable
-
-      def initialize(verbose, verbosity = nil)
-        log_level = if verbosity.nil?
-                      verbose ? :debug : :info
-                    else
-                      verbosity
-                    end
-
-        @logger = Yell.new(:format => false, \
-                           :name => 'HTML::Proofer', \
-                           :level => "gte.#{log_level}") do |l|
-          l.adapter :stdout, :level => [:debug, :info, :warn]
-          l.adapter :stderr, :level => [:error, :fatal]
-        end
-      end
-
-      def log(level, color, message)
-        @logger.send level, colorize(color, message)
-      end
-
-      def colorize(color, message)
-        if $stdout.isatty && $stderr.isatty
-          Colored.colorize(message, foreground: color)
-        else
-          message
-        end
-      end
-
-      # dumb override to play nice with Typhoeus/Ethon
-      def debug(message = nil)
-        log(:debug, :yellow, message) unless message.nil?
-      end
-    end
-  end
-end

data/lib/html/proofer/url_validator.rb

@@ -1,222 +0,0 @@
-require 'typhoeus'
-require 'uri'
-require_relative './utils'
-require_relative './cache'
-
-module HTML
-  class Proofer
-    class UrlValidator
-      include HTML::Proofer::Utils
-
-      attr_accessor :logger, :external_urls, :iterable_external_urls, :hydra
-
-      def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts)
-        @logger = logger
-        @external_urls = external_urls
-        @iterable_external_urls = {}
-        @failed_tests = []
-        @options = options
-        @hydra = Typhoeus::Hydra.new(hydra_opts)
-        @typhoeus_opts = typhoeus_opts
-        @external_domain_paths_with_queries = {}
-        @cache = Cache.new(@logger, @options[:cache])
-      end
-
-      def run
-        @iterable_external_urls = remove_query_values
-
-        if @cache.exists && @cache.load
-          cache_count = @cache.cache_log.length
-          cache_text = pluralize(cache_count, 'link', 'links')
-
-          logger.log :info, :blue, "Found #{cache_text} in the cache..."
-
-          urls_to_check = @cache.detect_url_changes(@iterable_external_urls)
-
-          @cache.cache_log.each_pair do |url, cache|
-            if @cache.within_timeframe?(cache['time'])
-              next if cache['message'].empty? # these were successes to skip
-              urls_to_check[url] = cache['filenames'] # these are failures to retry
-            else
-              urls_to_check[url] = cache['filenames'] # pass or fail, recheck expired links
-            end
-          end
-
-          external_link_checker(urls_to_check)
-        else
-          external_link_checker(@iterable_external_urls)
-        end
-
-        @cache.write
-        @failed_tests
-      end
-
-      def remove_query_values
-        return nil if @external_urls.nil?
-        iterable_external_urls = @external_urls.dup
-        @external_urls.keys.each do |url|
-          uri = begin
-                  Addressable::URI.parse(url)
-                rescue URI::Error, Addressable::URI::InvalidURIError
-                  @logger.log :error, :red, "#{url} is an invalid URL"
-                  nil
-                end
-          next if uri.nil? || uri.query.nil?
-          iterable_external_urls.delete(url) unless new_url_query_values?(uri)
-        end
-        iterable_external_urls
-      end
-
-      # remember queries we've seen, ignore future ones
-      def new_url_query_values?(uri)
-        queries = uri.query_values.keys.join('-')
-        domain_path = extract_domain_path(uri)
-        if @external_domain_paths_with_queries[domain_path].nil?
-          @external_domain_paths_with_queries[domain_path] = [queries]
-          true
-        elsif !@external_domain_paths_with_queries[domain_path].include?(queries)
-          @external_domain_paths_with_queries[domain_path] << queries
-          true
-        else
-          false
-        end
-      end
-
-      def extract_domain_path(uri)
-        uri.host + uri.path
-      end
-
-      # Proofer runs faster if we pull out all the external URLs and run the checks
-      # at the end. Otherwise, we're halting the consuming process for every file during
-      # the check_directory_of_files process.
-      #
-      # In addition, sorting the list lets libcurl keep connections to the same hosts alive.
-      #
-      # Finally, we'll first make a HEAD request, rather than GETing all the contents.
-      # If the HEAD fails, we'll fall back to GET, as some servers are not configured
-      # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is
-      # not an option.
-      def external_link_checker(external_urls)
-        external_urls = Hash[external_urls.sort]
-
-        count = external_urls.length
-        check_text = pluralize(count, 'external link', 'external links')
-        logger.log :info, :blue, "Checking #{check_text}..."
-
-        Ethon.logger = logger # log from Typhoeus/Ethon
-
-        url_processor(external_urls)
-
-        logger.log :debug, :yellow, "Running requests for:"
-        logger.log :debug, :yellow, "###\n" + external_urls.keys.join("\n") + "\n###"
-
-        hydra.run
-      end
-
-      def url_processor(external_urls)
-        external_urls.each_pair do |href, filenames|
-          href = begin
-                   clean_url(href)
-                 rescue URI::Error, Addressable::URI::InvalidURIError
-                   add_external_issue(filenames, "#{href} is an invalid URL")
-                   next
-                 end
-
-          if hash?(href) && @options[:check_external_hash]
-            queue_request(:get, href, filenames)
-          else
-            queue_request(:head, href, filenames)
-          end
-        end
-      end
-
-      def clean_url(href)
-        Addressable::URI.parse(href).normalize
-      end
-
-      def queue_request(method, href, filenames)
-        request = Typhoeus::Request.new(href, @typhoeus_opts.merge({ :method => method }))
-        request.on_complete { |response| response_handler(response, filenames) }
-        hydra.queue request
-      end
-
-      def response_handler(response, filenames)
-        effective_url = response.options[:effective_url]
-        href = response.request.base_url.to_s
-        method = response.request.options[:method]
-        response_code = response.code
-
-        debug_msg = "Received a #{response_code} for #{href}"
-        debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
-        logger.log :debug, :yellow, debug_msg
-
-        if response_code.between?(200, 299)
-          check_hash_in_2xx_response(href, effective_url, response, filenames)
-          @cache.add(href, filenames, response_code)
-        elsif response.timed_out?
-          handle_timeout(href, filenames, response_code)
-        elsif response_code == 0
-          handle_failure(href, filenames, response_code)
-        elsif method == :head
-          queue_request(:get, href, filenames)
-        else
-          return if @options[:only_4xx] && !response_code.between?(400, 499)
-          # Received a non-successful http response.
-          msg = "External link #{href} failed: #{response_code} #{response.return_message}"
-          add_external_issue(filenames, msg, response_code)
-          @cache.add(href, filenames, response_code, msg)
-        end
-      end
-
-      # Even though the response was a success, we may have been asked to check
-      # if the hash on the URL exists on the page
-      def check_hash_in_2xx_response(href, effective_url, response, filenames)
-        return if @options[:only_4xx]
-        return unless @options[:check_external_hash]
-        return unless (hash = hash?(href))
-
-        body_doc = create_nokogiri(response.body)
-
-        # user-content is a special addition by GitHub.
-        xpath = %(//*[@name="#{hash}"]|//*[@id="#{hash}"])
-        if URI.parse(href).host.match(/github\.com/i)
-          xpath << %(|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"])
-        end
-
-        return unless body_doc.xpath(xpath).empty?
-
-        msg = "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not"
-        add_external_issue(filenames, msg, response.code)
-        @cache.add(href, filenames, response.code, msg)
-      end
-
-      def handle_timeout(href, filenames, response_code)
-        msg = "External link #{href} failed: got a time out (response code #{response_code})"
-        @cache.add(href, filenames, 0, msg)
-        return if @options[:only_4xx]
-        add_external_issue(filenames, msg, response_code)
-      end
-
-      def handle_failure(href, filenames, response_code)
-        msg = "External link #{href} failed: response code #{response_code} means something's wrong"
-        @cache.add(href, filenames, 0, msg)
-        return if @options[:only_4xx]
-        add_external_issue(filenames, msg, response_code)
-      end
-
-      def add_external_issue(filenames, desc, status = nil)
-        if filenames.nil?
-          @failed_tests << CheckRunner::Issue.new('', desc, nil, status)
-        else
-          filenames.each { |f| @failed_tests << CheckRunner::Issue.new(f, desc, nil, status) }
-        end
-      end
-
-      def hash?(url)
-        URI.parse(url).fragment
-      rescue URI::InvalidURIError
-        nil
-      end
-    end
-  end
-end