html-proofer 2.3.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e5d2a78c7071f5534b4fb0d22c34337d9b92a6c
-  data.tar.gz: 4e89db1b52488116724d07608297d3abfb6581bb
+  metadata.gz: 1ce6c96e75683a8da4de60754d83aef138221d02
+  data.tar.gz: b341f1ee5b7237e58cba8e775dfaa152eaeeb0a5
 SHA512:
-  metadata.gz: 84e99216d622692010d127c80f4cf4a128b5391535169418a9715d99f1d0ccdfcbe3edf976ee769ce2d5021b1989c6e232d00de72ae8574f82eb29b925a0e269
-  data.tar.gz: 57e675836bb4a138dcfd84a0751772bbd99d05d3605dc8c1f922197683fe92a65563f0618393720a5ed3f44101025977f94f2dbf7cba96161def098148ad98ce
+  metadata.gz: 000ff0816076d1e8b738526f380e88309fdc04d93fa203a43d62960696064062b7942647a5f79be0f8e7239ca69e67f08f669866352cabf67152a6f95beb643b
+  data.tar.gz: 1863ee52843f2d5071170aaed79ce2ae9f6902ffb1a99d93021b746eb9d3d792d460d4caff2f703907291915b0bc8861af8967cd4f5a394e0a6604aa23e0dec0

data/bin/htmlproofer

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+STDOUT.sync = true
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w( .. lib ))
+
+require 'html-proofer'
+require 'mercenary'
+
+Mercenary.program(:htmlproofer) do |p|
+  p.version HTMLProofer::VERSION
+  p.description %(Test your rendered HTML files to make sure they're accurate.)
+  p.syntax 'htmlproofer PATH [options]'
+
+  p.description 'Runs the HTML-Proofer suite on the files in PATH. For more details, see the README.'
+
+  p.option 'allow_hash_href', '--allow-hash-href', 'If `true`, ignores the `href` `#`'
+  p.option 'as_links', '--as-links', 'Assumes that `PATH` is a comma-separated array of links to check.'
+  p.option 'alt_ignore', '--alt-ignore image1,[image2,...]', Array, 'A comma-separated list of Strings or RegExps containing `img`s whose missing `alt` tags are safe to ignore'
+  p.option 'assume_extension', '--assume-extension', 'Automatically add extension (e.g. `.html`) to file paths, to allow extensionless URLs (as supported by Jekyll 3 and GitHub Pages) (default: `false`).'
+  p.option 'checks_to_ignore', '--checks-to-ignore check1,[check2,...]', Array, ' An array of Strings indicating which checks you\'d like to not perform.'
+  p.option 'check_external_hash', '--check-external-hash', 'Checks whether external hashes exist (even if the website exists). This slows the checker down (default: `false`).'
+  p.option 'check_favicon', '--check-favicon', 'Enables the favicon checker (default: `false`).'
+  p.option 'check_html', '--check-html', 'Enables HTML validation errors from Nokogiri (default: `false`).'
+  p.option 'directory_index_file', '--directory-index-file', String, 'Sets the file to look for when a link refers to a directory. (default: `index.html`)'
+  p.option 'disable_external', '--disable-external', 'If `true`, does not run the external link checker, which can take a lot of time (default: `false`)'
+  p.option 'empty_alt_ignore', '--empty-alt-ignore', 'If `true`, ignores images with empty alt tags'
+  p.option 'error_sort', '--error-sort SORT', 'Defines the sort order for error output. Can be `:path`, `:desc`, or `:status` (default: `path`).'
+  p.option 'enforce_https', '--enforce-https', 'Fails a link if it\'s not marked as `https` (default: `false`).'
+  p.option 'extension', '--extension EXT', String, 'The extension of your HTML files including the dot. (default: `.html`)'
+  p.option 'external_only', '--external_only', 'Only checks problems with external references'
+  p.option 'file_ignore', '--file-ignore file1,[file2,...]', Array, 'A comma-separated list of Strings or RegExps containing file paths that are safe to ignore'
+  p.option 'http_status_ignore', '--http-status-ignore 123,[xxx, ...]', Array, 'A comma-separated list of numbers representing status codes to ignore.'
+  p.option 'report_invalid_tags', '--report-invalid-tags', 'Ignore `check_html` errors associated with unknown markup (default: `false`)'
+  p.option 'report_script_embeds', '--report-script-embeds', 'Ignore `check_html` errors associated with `script`s (default: `false`)'
+  p.option 'log_level', '--log-level <level>', String, 'Sets the logging level, as determined by Yell'
+  p.option 'only_4xx', '--only-4xx', 'Only reports errors for links that fall within the 4xx status code range'
+  p.option 'timeframe', '--timeframe <time>', String, 'A string representing the caching timeframe.'
+  p.option 'url_ignore', '--url-ignore link1,[link2,...]', Array, 'A comma-separated list of Strings or RegExps containing URLs that are safe to ignore. It affects all HTML attributes. Note that non-HTTP(S) URIs are always ignored'
+  p.option 'url_swap', '--url-swap re:string,[re:string,...]', Array, 'A comma-separated list containing key-value pairs of `RegExp => String`. It transforms URLs that match `RegExp` into `String` via `gsub`.'
+
+  p.action do |args, opts|
+    args = ['.'] if args.empty?
+    path = args.first
+
+    options = {}
+
+    # prepare everything to go to proofer
+    p.options.select { |o| !opts[o.config_key].nil? }.each do |option|
+      if option.return_type.to_s == 'Array' # TODO: is_a? doesn't work here?
+        opts[option.config_key] = opts[option.config_key].map { |i| HTMLProofer::Configuration.to_regex?(i) }
+      end
+      options[option.config_key.to_sym] = opts[option.config_key]
+    end
+
+    # some minor manipulation of a special option
+    unless opts['url_swap'].nil?
+      options[:url_swap] = {}
+      opts['url_swap'].each do |s|
+        pair = s.split(':', 2)
+        options[:url_swap][Regexp.new(pair[0])] = pair[1]
+      end
+    end
+
+    options[:error_sort] = opts['error-sort'].to_sym unless opts['error-sort'].nil?
+    options[:log_level] = opts['log_level'].to_sym unless opts['log_level'].nil?
+
+    # FIXME: this is gross
+    options[:validation] = {}
+    options[:validation][:report_script_embeds] = opts['report_script_embeds']
+    options[:validation][:report_invalid_tags] = opts['report_invalid_tags']
+
+    options[:cache] = {}
+    options[:cache][:timeframe] = opts['timeframe'] unless opts['timeframe'].nil?
+
+    paths = path.split(',')
+    if opts['as_links']
+      links = path.delete(' ').split(',')
+      HTMLProofer.check_links(links, options).run
+    elsif File.directory?(paths.first)
+      HTMLProofer.check_directories(paths, options).run
+    else
+      HTMLProofer.check_file(path, options).run
+    end
+  end
+end

data/lib/html-proofer/cache.rb

@@ -0,0 +1,153 @@
+require_relative 'utils'
+
+require 'json'
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/date'
+require 'active_support/core_ext/numeric/time'
+
+module HTMLProofer
+  class Cache
+    include HTMLProofer::Utils
+
+    CACHE_LOG = File.join(STORAGE_DIR, 'cache.log')
+
+    attr_reader :exists, :cache_log
+
+    def initialize(logger, options)
+      @logger = logger
+      @cache_log = {}
+
+      if options.nil? || options.empty?
+        define_singleton_method('use_cache?') { false }
+      else
+        define_singleton_method('use_cache?') { true }
+        @parsed_timeframe = parsed_timeframe(options[:timeframe])
+      end
+
+      @cache_time = Time.now
+
+      if File.exist?(CACHE_LOG)
+        contents = File.read(CACHE_LOG)
+        @cache_log = contents.empty? ? {} : JSON.parse(contents)
+      end
+    end
+
+    def within_timeframe?(time)
+      (@parsed_timeframe..@cache_time).cover?(time)
+    end
+
+    def urls
+      @cache_log['urls'] || []
+    end
+
+    def size
+      @cache_log.length
+    end
+
+    def parsed_timeframe(timeframe)
+      time, date = timeframe.match(/(\d+)(\D)/).captures
+      time = time.to_f
+      case date
+      when 'M'
+        time.months.ago
+      when 'w'
+        time.weeks.ago
+      when 'd'
+        time.days.ago
+      when 'h'
+        time.hours.ago
+      else
+        fail ArgumentError, "#{date} is not a valid timeframe!"
+      end
+    end
+
+    def add(url, filenames, status, msg = '')
+      data = {
+                :time => @cache_time,
+                :filenames => filenames,
+                :status => status,
+                :message => msg
+             }
+
+      @cache_log[clean_url(url)] = data
+    end
+
+    def detect_url_changes(found)
+      existing_urls = @cache_log.keys.map { |url| clean_url(url) }
+      found_urls = found.keys.map { |url| clean_url(url) }
+
+      # prepare to add new URLs detected
+      additions = found.reject do |url, _|
+        url = clean_url(url)
+        if existing_urls.include?(url)
+          true
+        else
+          @logger.log :debug, "Adding #{url} to cache check"
+          false
+        end
+      end
+
+      new_link_count = additions.length
+      new_link_text = pluralize(new_link_count, 'link', 'links')
+      @logger.log :info, "Adding #{new_link_text} to the cache..."
+
+      # remove from cache URLs that no longer exist
+      del = 0
+      @cache_log.delete_if do |url, _|
+        url = clean_url(url)
+        if !found_urls.include?(url)
+          @logger.log :debug, "Removing #{url} from cache check"
+          del += 1
+          true
+        else
+          false
+        end
+      end
+
+      del_link_text = pluralize(del, 'link', 'links')
+      @logger.log :info, "Removing #{del_link_text} from the cache..."
+
+      additions
+    end
+
+    def write
+      File.write(CACHE_LOG, @cache_log.to_json)
+    end
+
+    def load?
+      @load.nil?
+    end
+
+    def retrieve_urls(external_urls)
+      urls_to_check = detect_url_changes(external_urls)
+      @cache_log.each_pair do |url, cache|
+        if within_timeframe?(cache['time'])
+          next if cache['message'].empty? # these were successes to skip
+          urls_to_check[url] = cache['filenames'] # these are failures to retry
+        else
+          urls_to_check[url] = cache['filenames'] # pass or fail, recheck expired links
+        end
+      end
+      urls_to_check
+    end
+
+    # FIXME: there seems to be some discrepenacy where Typhoeus occasionally adds
+    # a trailing slash to URL strings, which causes issues with the cache
+    def slashless_url(url)
+      url.chomp('/')
+    end
+
+    # FIXME: it seems that Typhoeus actually acts on escaped URLs,
+    # but there's no way to get at that information, and the cache
+    # stores unescaped URLs. Because of this, some links, such as
+    # github.com/search/issues?q=is:open+is:issue+fig are not matched
+    # as github.com/search/issues?q=is%3Aopen+is%3Aissue+fig
+    def unescape_url(url)
+      Addressable::URI.unescape(url)
+    end
+
+    def clean_url(url)
+      slashless_url(unescape_url(url))
+    end
+  end
+end

data/lib/html-proofer/check/favicon.rb

@@ -0,0 +1,15 @@
+class FaviconCheck < ::HTMLProofer::Check
+  def run
+    found = false
+    @html.xpath('//link[not(ancestor::pre or ancestor::code)]').each do |node|
+      favicon = create_element(node)
+      next if favicon.ignore?
+      found = true if favicon.rel.split(' ').last.eql? 'icon'
+      break if found
+    end
+
+    return if found
+
+    add_issue 'no favicon specified'
+  end
+end

data/lib/html-proofer/check/html.rb

@@ -0,0 +1,21 @@
+class HtmlCheck < ::HTMLProofer::Check
+  SCRIPT_EMBEDS_MSG = /Element script embeds close tag/
+  INVALID_TAG_MSG = /Tag ([\w\-:]+) invalid/
+  INVALID_PREFIX = /Namespace prefix/
+
+  def run
+    @html.errors.each do |error|
+      message = error.message
+      line    = error.line
+
+      if message =~ INVALID_TAG_MSG || message =~ INVALID_PREFIX
+        next unless options[:validation][:report_invalid_tags]
+      end
+
+      # tags embedded in scripts are used in templating languages: http://git.io/vOovv
+      next if !options[:validation][:report_script_embeds] && message =~ SCRIPT_EMBEDS_MSG
+
+      add_issue(message, line: line)
+    end
+  end
+end

data/lib/html-proofer/check/images.rb

@@ -0,0 +1,47 @@
+class ImageCheck < ::HTMLProofer::Check
+  SCREEN_SHOT_REGEX = /Screen(?: |%20)Shot(?: |%20)\d+-\d+-\d+(?: |%20)at(?: |%20)\d+.\d+.\d+/
+
+  def empty_alt_tag?
+    @img.alt.strip.empty?
+  end
+
+  def terrible_filename?
+    @img.url =~ SCREEN_SHOT_REGEX
+  end
+
+  def missing_src?
+    blank?(@img.url)
+  end
+
+  def run
+    @html.css('img').each do |node|
+      @img = create_element(node)
+      line = node.line
+
+      next if @img.ignore?
+
+      # screenshot filenames should return because of terrible names
+      if terrible_filename?
+        add_issue("image has a terrible filename (#{@img.url})", line: line)
+        next
+      end
+
+      # does the image exist?
+      if missing_src?
+        add_issue('image has no src or srcset attribute', line: line)
+      else
+        if @img.remote?
+          add_to_external_urls(@img.url, line)
+        elsif !@img.exists?
+          add_issue("internal image #{@img.url} does not exist", line: line)
+        end
+      end
+
+      if @img.alt.nil? || (empty_alt_tag? && !@img.ignore_empty_alt?)
+        add_issue("image #{@img.url} does not have an alt attribute", line: line)
+      end
+    end
+
+    external_urls
+  end
+end

data/lib/html-proofer/check/links.rb

@@ -0,0 +1,120 @@
+class LinkCheck < ::HTMLProofer::Check
+  include HTMLProofer::Utils
+
+  def missing_href?
+    blank?(@link.href) && blank?(@link.name) && blank?(@link.id)
+  end
+
+  def placeholder?
+    (!blank?(@link.id) || !blank?(@link.name)) && @link.href.nil?
+  end
+
+  def run
+    @html.css('a, link').each do |node|
+      @link = create_element(node)
+      line = @node.line
+
+      next if @link.ignore?
+
+      next if placeholder?
+      next if @link.allow_hash_href? && @link.href == '#'
+
+      # is it even a valid URL?
+      unless @link.valid?
+        add_issue("#{@link.href} is an invalid URL", line: line)
+        next
+      end
+
+      check_schemes(@link, line)
+
+      # is there even an href?
+      if missing_href?
+        # HTML5 allows dropping the href: http://git.io/vBX0z
+        next if @html.internal_subset.name == 'html' && @html.internal_subset.external_id.nil?
+        add_issue('anchor has no href attribute', line: line)
+        next
+      end
+
+      # intentionally here because we still want valid? & missing_href? to execute
+      next if @link.non_http_remote?
+      # does the file even exist?
+      if @link.remote?
+        add_to_external_urls(@link.href, line)
+        next
+      elsif !@link.internal? && !@link.exists?
+        add_issue("internally linking to #{@link.href}, which does not exist", line: line)
+      end
+
+      # does the local directory have a trailing slash?
+      if @link.unslashed_directory? @link.absolute_path
+        add_issue("internally linking to a directory #{@link.absolute_path} without trailing slash", line: line)
+        next
+      end
+
+      # verify the target hash
+      handle_hash(@link, line) if @link.hash
+    end
+
+    external_urls
+  end
+
+  def check_schemes(link, line)
+    case link.scheme
+    when 'mailto'
+      handle_mailto(link, line)
+    when 'tel'
+      handle_tel(link, line)
+    when 'http'
+      return unless @options[:enforce_https]
+      add_issue("#{link.href} is not an HTTPS link", line: line)
+    end
+  end
+
+  def handle_mailto(link, line)
+    if link.path.empty?
+      add_issue("#{link.href} contains no email address", line: line)
+    elsif !link.path.include?('@')
+      add_issue("#{link.href} contains an invalid email address", line: line)
+    end
+  end
+
+  def handle_tel(link, line)
+    add_issue("#{link.href} contains no phone number", line: line) if link.path.empty?
+  end
+
+  def handle_hash(link, line)
+    if link.internal?
+      unless hash_check @html, link.hash
+        add_issue("linking to internal hash ##{link.hash} that does not exist", line: line)
+      end
+    elsif link.external?
+      external_link_check(link, line)
+    end
+  end
+
+  def external_link_check(link, line)
+    if !link.exists?
+      add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line)
+    else
+      target_html = create_nokogiri link.absolute_path
+      unless hash_check target_html, link.hash
+        add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line)
+      end
+    end
+  end
+
+  def hash_check(html, href_hash)
+    decoded_href_hash = URI.decode(href_hash)
+    html.xpath("//*[case_insensitive_equals(@id, '#{href_hash}')]", \
+               "//*[case_insensitive_equals(@name, '#{href_hash}')]", \
+               "//*[case_insensitive_equals(@id, '#{decoded_href_hash}')]", \
+               "//*[case_insensitive_equals(@name, '#{decoded_href_hash}')]", \
+               XpathFunctions.new).length > 0
+  end
+
+  class XpathFunctions
+    def case_insensitive_equals(node_set, str_to_match)
+      node_set.find_all {|node| node.to_s.downcase == str_to_match.to_s.downcase }
+    end
+  end
+end

data/lib/html-proofer/check/scripts.rb

@@ -0,0 +1,28 @@
+class ScriptCheck < ::HTMLProofer::Check
+  attr_reader :src
+
+  def missing_src?
+    !@script.src
+  end
+
+  def run
+    @html.css('script').each do |node|
+      @script = create_element(node)
+      line = node.line
+
+      next if @script.ignore?
+      next unless node.text.strip.empty?
+
+      # does the script exist?
+      if missing_src?
+        add_issue('script is empty and has no src attribute', line: line)
+      elsif @script.remote?
+        add_to_external_urls(@script.src, line)
+      elsif !@script.exists?
+        add_issue("internal script #{@script.src} does not exist", line: line)
+      end
+    end
+
+    external_urls
+  end
+end

data/lib/html-proofer/check.rb

@@ -0,0 +1,63 @@
+module HTMLProofer
+  # Mostly handles issue management and collecting of external URLs.
+  class Check
+    attr_reader :node, :element, :src, :path, :options, :issues, :external_urls
+
+    def initialize(src, path, html, options)
+      @src    = src
+      @path   = path
+      @html   = remove_ignored(html)
+      @options = options
+      @issues = []
+      @external_urls = {}
+    end
+
+    def create_element(node)
+      @node = node
+      Element.new(node, self)
+    end
+
+    def run
+      fail NotImplementedError, 'HTMLProofer::Check subclasses must implement #run'
+    end
+
+    def add_issue(desc, line: nil, status: -1)
+      @issues << Issue.new(@path, desc, line: line, status: status)
+    end
+
+    def add_to_external_urls(url, _)
+      return if @external_urls[url]
+      add_path_for_url(url)
+    end
+
+    def add_path_for_url(url)
+      if @external_urls[url]
+        @external_urls[url] << @path
+      else
+        @external_urls[url] = [@path]
+      end
+    end
+
+    def self.subchecks
+      classes = []
+
+      ObjectSpace.each_object(Class) do |c|
+        next unless c.superclass == self
+        classes << c
+      end
+
+      classes
+    end
+
+    def blank?(attr)
+      attr.nil? || attr.empty?
+    end
+
+    private
+
+    def remove_ignored(html)
+      html.css('code, pre, tt').each(&:unlink)
+      html
+    end
+  end
+end

data/lib/html-proofer/configuration.rb

@@ -0,0 +1,56 @@
+module HTMLProofer
+  module Configuration
+    require_relative 'version'
+
+    PROOFER_DEFAULTS = {
+      :allow_hash_href => false,
+      :alt_ignore => [],
+      :assume_extension => false,
+      :check_external_hash => false,
+      :check_favicon => false,
+      :check_html => false,
+      :checks_to_ignore => [],
+      :directory_index_file => 'index.html',
+      :disable_external => false,
+      :empty_alt_ignore => false,
+      :enforce_https => false,
+      :error_sort => :path,
+      :extension => '.html',
+      :external_only => false,
+      :file_ignore => [],
+      :http_status_ignore => [],
+      :log_level => :info,
+      :only_4xx => false,
+      :url_ignore => [],
+      :url_swap => []
+    }
+
+    TYPHOEUS_DEFAULTS = {
+      :followlocation => true,
+      :headers => {
+        'User-Agent' => "Mozilla/5.0 (compatible; HTML Proofer/#{HTMLProofer::VERSION}; +https://github.com/gjtorikian/html-proofer)"
+      }
+    }
+
+    HYDRA_DEFAULTS = {
+      :max_concurrency => 50
+    }
+
+    PARALLEL_DEFAULTS = {}
+
+    VALIDATION_DEFAULTS = {
+      :report_script_embeds => false,
+      :report_invalid_tags => false
+    }
+
+    CACHE_DEFAULTS = {}
+
+    def self.to_regex?(item)
+      if item.start_with?('/') && item.end_with?('/')
+        Regexp.new item[1...-1]
+      else
+        item
+      end
+    end
+  end
+end