html-proofer 3.19.4 → 4.4.0

data/lib/html_proofer/cache.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+require "date"
+require "json"
+require "uri"
+
+module HTMLProofer
+  class Cache
+    include HTMLProofer::Utils
+
+    CACHE_VERSION = 2
+
+    DEFAULT_STORAGE_DIR = File.join("tmp", ".htmlproofer")
+    DEFAULT_CACHE_FILE_NAME = "cache.json"
+
+    URI_REGEXP = URI::DEFAULT_PARSER.make_regexp
+
+    attr_reader :exists, :cache_log, :storage_dir, :cache_file
+
+    def initialize(runner, options)
+      @runner = runner
+      @logger = @runner.logger
+
+      @cache_datetime = Time.now
+      @cache_time = @cache_datetime.to_time
+
+      if blank?(options)
+        define_singleton_method(:enabled?) { false }
+        define_singleton_method(:external_enabled?) { false }
+        define_singleton_method(:internal_enabled?) { false }
+      else
+        # we still consider the cache as enabled, regardless of the specic timeframes
+        define_singleton_method(:enabled?) { true }
+        setup_cache!(options)
+
+        @external_timeframe = parsed_timeframe(options[:timeframe][:external])
+        define_singleton_method(:external_enabled?) { !@external_timeframe.nil? }
+        @internal_timeframe = parsed_timeframe(options[:timeframe][:internal])
+        define_singleton_method(:internal_enabled?) { !@internal_timeframe.nil? }
+      end
+    end
+
+    def parsed_timeframe(timeframe)
+      return nil if timeframe.nil?
+
+      time, date = timeframe.match(/(\d+)(\D)/).captures
+      time = time.to_i
+      case date
+      when "M"
+        time_ago(time, :months)
+      when "w"
+        time_ago(time, :weeks)
+      when "d"
+        time_ago(time, :days)
+      when "h"
+        time_ago(time, :hours)
+      else
+        raise ArgumentError, "#{date} is not a valid timeframe!"
+      end
+    end
+
+    def add_internal(url, metadata, found)
+      return unless internal_enabled?
+
+      @cache_log[:internal][url] = { time: @cache_time, metadata: [] } if @cache_log[:internal][url].nil?
+
+      @cache_log[:internal][url][:metadata] << construct_internal_link_metadata(metadata, found)
+    end
+
+    def add_external(url, filenames, status_code, msg, found)
+      return unless external_enabled?
+
+      clean_url = cleaned_url(url)
+      @cache_log[:external][clean_url] =
+        { time: @cache_time.to_s, found: found, status_code: status_code, message: msg, metadata: filenames }
+    end
+
+    def detect_url_changes(urls_detected, type)
+      determine_deletions(urls_detected, type)
+
+      additions = determine_additions(urls_detected, type)
+
+      additions
+    end
+
+    def write
+      return unless enabled?
+
+      File.write(@cache_file, @cache_log.to_json)
+    end
+
+    def retrieve_urls(urls_detected, type)
+      # if there are no urls, bail
+      return {} if urls_detected.empty?
+
+      urls_detected = urls_detected.transform_keys do |url|
+        cleaned_url(url)
+      end
+
+      urls_to_check = detect_url_changes(urls_detected, type)
+
+      urls_to_check
+    end
+
+    def within_external_timeframe?(time)
+      within_timeframe?(time, @external_timeframe)
+    end
+
+    def within_internal_timeframe?(time)
+      within_timeframe?(time, @internal_timeframe)
+    end
+
+    def empty?
+      blank?(@cache_log) || (@cache_log[:internal].empty? && @cache_log[:external].empty?)
+    end
+
+    def size(type)
+      @cache_log[type].size
+    end
+
+    private def construct_internal_link_metadata(metadata, found)
+      {
+        source: metadata[:source],
+        filename: metadata[:filename],
+        line: metadata[:line],
+        base_url: metadata[:base_url],
+        found: found,
+      }
+    end
+
+    # prepare to add new URLs detected
+    private def determine_additions(urls_detected, type)
+      additions = type == :external ? determine_external_additions(urls_detected) : determine_internal_additions(urls_detected)
+
+      new_link_count = additions.length
+      new_link_text = pluralize(new_link_count, "new #{type} link", "new #{type} links")
+      @logger.log(:debug, "Adding #{new_link_text} to the cache")
+
+      additions
+    end
+
+    private def determine_external_additions(urls_detected)
+      urls_detected.reject do |url, _metadata|
+        if @cache_log[:external].include?(url)
+          found = @cache_log[:external][url][:found] # if this is false, we're trying again
+          unless found
+            @logger.log(:debug, "Adding #{url} to external cache (not found)")
+          end
+          found
+        else
+          @logger.log(:debug, "Adding #{url} to external cache")
+          false
+        end
+      end
+    end
+
+    private def determine_internal_additions(urls_detected)
+      urls_detected.each_with_object({}) do |(url, detected_metadata), hsh|
+        # url is not even in cache
+        if @cache_log[:internal][url].nil?
+          @logger.log(:debug, "Adding #{url} to internal cache")
+          hsh[url] = detected_metadata
+          next
+        end
+
+        # detect metadata additions
+        # NOTE: the time-stamp for the whole url key will not be updated,
+        # so that it reflects the earliest time any of the metadata was checked
+        cache_metadata = @cache_log[:internal][url][:metadata]
+        metadata_additions = detected_metadata.reject do |detected|
+          existing_cache_metadata = cache_metadata.find { |cached, _| cached[:filename] == detected[:filename] }
+          # cache for this url, from an existing path, exists as found
+          found = !existing_cache_metadata.nil? && !existing_cache_metadata.empty? && existing_cache_metadata[:found]
+          unless found
+            @logger.log(:debug, "Adding #{detected} to internal cache for #{url}")
+          end
+          found
+        end
+
+        if metadata_additions.empty?
+          next
+        end
+
+        hsh[url] = metadata_additions
+        # remove from the cache the detected metadata additions as they correspond to failures to be rechecked
+        # (this works assuming the detected url metadata have "found" set to false)
+        @cache_log[:internal][url][:metadata] = cache_metadata.difference(metadata_additions)
+      end
+    end
+
+    # remove from cache URLs that no longer exist
+    private def determine_deletions(urls_detected, type)
+      deletions = 0
+
+      @cache_log[type].delete_if do |url, cache|
+        expired_timeframe = type == :external ? !within_external_timeframe?(cache[:time]) : !within_internal_timeframe?(cache[:time])
+        if expired_timeframe
+          @logger.log(:debug, "Removing #{url} from #{type} cache (expired timeframe)")
+          deletions += 1
+          true
+        elsif urls_detected.include?(url)
+          false
+        elsif url_matches_type?(url, type)
+          @logger.log(:debug, "Removing #{url} from #{type} cache (not detected anymore)")
+          deletions += 1
+          true
+        end
+      end
+
+      del_link_text = pluralize(deletions, "outdated #{type} link", "outdated #{type} links")
+      @logger.log(:debug, "Removing #{del_link_text} from the cache")
+    end
+
+    private def setup_cache!(options)
+      default_structure = {
+        version: CACHE_VERSION,
+        internal: {},
+        external: {},
+      }
+
+      @storage_dir = options[:storage_dir] || DEFAULT_STORAGE_DIR
+
+      FileUtils.mkdir_p(storage_dir) unless Dir.exist?(storage_dir)
+
+      cache_file_name = options[:cache_file] || DEFAULT_CACHE_FILE_NAME
+
+      @cache_file = File.join(storage_dir, cache_file_name)
+
+      return (@cache_log = default_structure) unless File.exist?(@cache_file)
+
+      contents = File.read(@cache_file)
+
+      return (@cache_log = default_structure) if blank?(contents)
+
+      log = JSON.parse(contents, symbolize_names: true)
+
+      old_cache = (cache_version = log[:version]).nil?
+      @cache_log = if old_cache # previous cache version, create a new one
+        default_structure
+      elsif cache_version != CACHE_VERSION
+      # if cache version is newer...do something
+      else
+        log[:internal] = log[:internal].transform_keys(&:to_s)
+        log[:external] = log[:external].transform_keys(&:to_s)
+        log
+      end
+    end
+
+    # https://github.com/rails/rails/blob/3872bc0e54d32e8bf3a6299b0bfe173d94b072fc/activesupport/lib/active_support/duration.rb#L112-L117
+    SECONDS_PER_HOUR   = 3600
+    SECONDS_PER_DAY    = 86400
+    SECONDS_PER_WEEK   = 604800
+    SECONDS_PER_MONTH  = 2629746  # 1/12 of a gregorian year
+
+    private def time_ago(measurement, unit)
+      case unit
+      when :months
+        @cache_datetime - (SECONDS_PER_MONTH * measurement)
+      when :weeks
+        @cache_datetime - (SECONDS_PER_WEEK * measurement)
+      when :days
+        @cache_datetime - (SECONDS_PER_DAY * measurement)
+      when :hours
+        @cache_datetime - Rational(SECONDS_PER_HOUR * measurement)
+      end.to_time
+    end
+
+    private def url_matches_type?(url, type)
+      return true if type == :internal && url !~ URI_REGEXP
+      return true if type == :external && url =~ URI_REGEXP
+    end
+
+    private def cleaned_url(url)
+      cleaned_url = escape_unescape(url)
+
+      return cleaned_url unless cleaned_url.end_with?("/", "#", "?") && cleaned_url.length > 1
+
+      cleaned_url[0..-2]
+    end
+
+    private def escape_unescape(url)
+      Addressable::URI.parse(url).normalize.to_s
+    end
+
+    private def within_timeframe?(current_time, parsed_timeframe)
+      return false if current_time.nil? || parsed_timeframe.nil?
+
+      current_time = Time.parse(current_time) if current_time.is_a?(String)
+      (parsed_timeframe..@cache_time).cover?(current_time)
+    end
+  end
+end

data/lib/html_proofer/check/favicon.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Check
+    class Favicon < HTMLProofer::Check
+      def run
+        found = false
+        @html.css("link").each do |node|
+          @favicon = create_element(node)
+
+          next if @favicon.ignore?
+
+          break if (found = @favicon.node["rel"].split.last.eql?("icon"))
+        end
+
+        return if immediate_redirect?
+
+        if found
+          if @favicon.url.protocol_relative?
+            add_failure("favicon link #{@favicon.url} is a protocol-relative URL, use explicit https:// instead",
+              line: @favicon.line, content: @favicon.content)
+          elsif @favicon.url.remote?
+            add_to_external_urls(@favicon.url, @favicon.line)
+          elsif !@favicon.url.exists?
+            add_failure("internal favicon #{@favicon.url.raw_attribute} does not exist", line: @favicon.line,
+              content: @favicon.content)
+          end
+        else
+          add_failure("no favicon provided")
+        end
+      end
+
+      private
+
+      # allow any instant-redirect meta tag
+      def immediate_redirect?
+        @html.xpath("//meta[@http-equiv='refresh']").attribute("content").value.start_with?("0;")
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/html_proofer/check/images.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Check
+    class Images < HTMLProofer::Check
+      SCREEN_SHOT_REGEX = /Screen(?: |%20)Shot(?: |%20)\d+-\d+-\d+(?: |%20)at(?: |%20)\d+.\d+.\d+/.freeze
+
+      def run
+        @html.css("img, source").each do |node|
+          @img = create_element(node)
+
+          next if @img.ignore?
+
+          # screenshot filenames should return because of terrible names
+          add_failure("image has a terrible filename (#{@img.url.raw_attribute})", line: @img.line,
+            content: @img.content) if terrible_filename?
+
+          # does the image exist?
+          if missing_src?
+            add_failure("image has no src or srcset attribute", line: @img.line, content: @img.content)
+          elsif @img.url.protocol_relative?
+            add_failure("image link #{@img.url} is a protocol-relative URL, use explicit https:// instead",
+              line: @img.line, content: @img.content)
+          elsif @img.url.remote?
+            add_to_external_urls(@img.url, @img.line)
+          elsif !@img.url.exists? && !@img.multiple_srcsets? && !@img.multiple_sizes?
+            add_failure("internal image #{@img.url.raw_attribute} does not exist", line: @img.line,
+              content: @img.content)
+          elsif @img.multiple_srcsets? || @img.multiple_sizes?
+            @img.srcsets_wo_sizes.each do |srcset|
+              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url, extract_size: true)
+
+              if srcset_url.protocol_relative?
+                add_failure("image link #{srcset_url.url} is a protocol-relative URL, use explicit https:// instead",
+                  line: @img.line, content: @img.content)
+              elsif srcset_url.remote?
+                add_to_external_urls(srcset_url.url, @img.line)
+              elsif !srcset_url.exists?
+                add_failure("internal image #{srcset} does not exist", line: @img.line, content: @img.content)
+              end
+            end
+          end
+
+          # if this is an img element, check that the alt attribute is present
+          if @img.img_tag? && !ignore_element?
+            if missing_alt_tag? && !ignore_missing_alt?
+              add_failure("image #{@img.url.raw_attribute} does not have an alt attribute", line: @img.line,
+                content: @img.content)
+            elsif (empty_alt_tag? || alt_all_spaces?) && !ignore_empty_alt?
+              add_failure("image #{@img.url.raw_attribute} has an alt attribute, but no content", line: @img.line,
+                content: @img.content)
+            end
+          end
+
+          add_failure("image #{@img.url.raw_attribute} uses the http scheme", line: @img.line,
+            content: @img.content) if @runner.enforce_https? && @img.url.http?
+        end
+
+        external_urls
+      end
+
+      def ignore_missing_alt?
+        @runner.options[:ignore_missing_alt]
+      end
+
+      def ignore_empty_alt?
+        @runner.options[:ignore_empty_alt]
+      end
+
+      def ignore_element?
+        @img.url.ignore? || @img.aria_hidden?
+      end
+
+      def missing_alt_tag?
+        @img.node["alt"].nil?
+      end
+
+      def empty_alt_tag?
+        !missing_alt_tag? && @img.node["alt"].empty?
+      end
+
+      def empty_whitespace_alt_tag?
+        !missing_alt_tag? && @img.node["alt"].strip.empty?
+      end
+
+      def alt_all_spaces?
+        !missing_alt_tag? && @img.node["alt"].split.all?(" ")
+      end
+
+      def terrible_filename?
+        @img.url.to_s =~ SCREEN_SHOT_REGEX
+      end
+
+      def missing_src?
+        blank?(@img.url.to_s)
+      end
+    end
+  end
+end

data/lib/html_proofer/check/links.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Check
+    class Links < HTMLProofer::Check
+      def run
+        @html.css("a, link").each do |node|
+          @link = create_element(node)
+
+          next if @link.ignore?
+
+          if !allow_hash_href? && @link.node["href"] == "#"
+            add_failure("linking to internal hash #, which points to nowhere", line: @link.line, content: @link.content)
+            next
+          end
+
+          # is there even an href?
+          if blank?(@link.url.raw_attribute)
+            next if allow_missing_href?
+
+            add_failure("'#{@link.node.name}' tag is missing a reference", line: @link.line, content: @link.content)
+            next
+          end
+
+          # is it even a valid URL?
+          unless @link.url.valid?
+            add_failure("#{@link.href} is an invalid URL", line: @link.line, content: @link.content)
+            next
+          end
+
+          if @link.url.protocol_relative?
+            add_failure("#{@link.url} is a protocol-relative URL, use explicit https:// instead",
+              line: @link.line, content: @link.content)
+            next
+          end
+
+          check_schemes
+
+          # intentionally down here because we still want valid? & missing_href? to execute
+          next if @link.url.non_http_remote?
+
+          if !@link.url.internal? && @link.url.remote?
+            check_sri if @runner.check_sri? && @link.link_tag?
+
+            # we need to skip these for now; although the domain main be valid,
+            # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx
+            next if @link.node["rel"] == "dns-prefetch"
+
+            unless @link.url.path?
+              add_failure("#{@link.url.raw_attribute} is an invalid URL", line: @link.line, content: @link.content)
+              next
+            end
+
+            add_to_external_urls(@link.url, @link.line)
+          elsif @link.url.internal?
+            # does the local directory have a trailing slash?
+            if @link.url.unslashed_directory?(@link.url.absolute_path)
+              add_failure("internally linking to a directory #{@link.url.raw_attribute} without trailing slash",
+                line: @link.line, content: @link.content)
+              next
+            end
+
+            add_to_internal_urls(@link.url, @link.line)
+          end
+        end
+      end
+
+      def allow_missing_href?
+        @runner.options[:allow_missing_href]
+      end
+
+      def allow_hash_href?
+        @runner.options[:allow_hash_href]
+      end
+
+      def check_schemes
+        case @link.url.scheme
+        when "mailto"
+          handle_mailto
+        when "tel"
+          handle_tel
+        when "http"
+          return unless @runner.options[:enforce_https]
+
+          add_failure("#{@link.url.raw_attribute} is not an HTTPS link", line: @link.line, content: @link.content)
+        end
+      end
+
+      def handle_mailto
+        if @link.url.path.empty?
+          add_failure("#{@link.url.raw_attribute} contains no email address", line: @link.line,
+            content: @link.content) unless ignore_empty_mailto?
+        elsif !/#{URI::MailTo::EMAIL_REGEXP}/o.match?(@link.url.path)
+          add_failure("#{@link.url.raw_attribute} contains an invalid email address", line: @link.line,
+            content: @link.content)
+        end
+      end
+
+      def handle_tel
+        add_failure("#{@link.url.raw_attribute} contains no phone number", line: @link.line,
+          content: @link.content) if @link.url.path.empty?
+      end
+
+      def ignore_empty_mailto?
+        @runner.options[:ignore_empty_mailto]
+      end
+
+      # Allowed elements from Subresource Integrity specification
+      # https://w3c.github.io/webappsec-subresource-integrity/#link-element-for-stylesheets
+      SRI_REL_TYPES = %(stylesheet)
+
+      def check_sri
+        return unless SRI_REL_TYPES.include?(@link.node["rel"])
+
+        if blank?(@link.node["integrity"]) && blank?(@link.node["crossorigin"])
+          add_failure("SRI and CORS not provided in: #{@link.url.raw_attribute}", line: @link.line,
+            content: @link.content)
+        elsif blank?(@link.node["integrity"])
+          add_failure("Integrity is missing in: #{@link.url.raw_attribute}", line: @link.line, content: @link.content)
+        elsif blank?(@link.node["crossorigin"])
+          add_failure("CORS not provided for external resource in: #{@link.link.url.raw_attribute}", line: @link.line,
+            content: @link.content)
+        end
+      end
+
+      private def source_tag?
+        @link.node.name == "source"
+      end
+
+      private def anchor_tag?
+        @link.node.name == "a"
+      end
+    end
+  end
+end

data/lib/html_proofer/check/open_graph.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Check
+    class OpenGraph < HTMLProofer::Check
+      def run
+        @html.css('meta[property="og:url"], meta[property="og:image"]').each do |node|
+          @open_graph = create_element(node)
+
+          next if @open_graph.ignore?
+
+          # does the open_graph exist?
+          if missing_content?
+            add_failure("open graph has no content attribute", line: @open_graph.line, content: @open_graph.content)
+          elsif empty_content?
+            add_failure("open graph content attribute is empty", line: @open_graph.line, content: @open_graph.content)
+          elsif !@open_graph.url.valid?
+            add_failure("#{@open_graph.src} is an invalid URL", line: @open_graph.line)
+          elsif @open_graph.url.protocol_relative?
+            add_failure("open graph link #{@open_graph.url} is a protocol-relative URL, use explicit https:// instead",
+              line: @open_graph.line, content: @open_graph.content)
+          elsif @open_graph.url.remote?
+            add_to_external_urls(@open_graph.url, @open_graph.line)
+          else
+            add_failure("internal open graph #{@open_graph.url.raw_attribute} does not exist", line: @open_graph.line,
+              content: @open_graph.content) unless @open_graph.url.exists?
+          end
+        end
+
+        external_urls
+      end
+
+      private def missing_content?
+        @open_graph.node["content"].nil?
+      end
+
+      private def empty_content?
+        @open_graph.node["content"].empty?
+      end
+    end
+  end
+end

data/lib/html_proofer/check/scripts.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module HTMLProofer
+  class Check
+    class Scripts < HTMLProofer::Check
+      def run
+        @html.css("script").each do |node|
+          @script = create_element(node)
+
+          next if @script.ignore?
+          next unless @script.content.strip.empty?
+
+          # does the script exist?
+          if missing_src?
+            add_failure("script is empty and has no src attribute", line: @script.line, content: @script.content)
+          elsif @script.url.protocol_relative?
+            add_failure("script link #{@script.url} is a protocol-relative URL, use explicit https:// instead",
+              line: @script.line, content: @script.content)
+          elsif @script.url.remote?
+            add_to_external_urls(@script.url, @script.line)
+            check_sri if @runner.check_sri?
+          elsif !@script.url.exists?
+            add_failure("internal script reference #{@script.src} does not exist", line: @script.line,
+              content: @script.content)
+          end
+        end
+
+        external_urls
+      end
+
+      def missing_src?
+        @script.node["src"].nil?
+      end
+
+      def check_sri
+        if blank?(@script.node["integrity"]) && blank?(@script.node["crossorigin"])
+          add_failure("SRI and CORS not provided in: #{@script.url.raw_attribute}", line: @script.line,
+            content: @script.content)
+        elsif blank?(@script.node["integrity"])
+          add_failure("Integrity is missing in: #{@script.url.raw_attribute}", line: @script.line,
+            content: @script.content)
+        elsif blank?(@script.node["crossorigin"])
+          add_failure("CORS not provided for external resource in: #{@script.url.raw_attribute}", line: @script.line,
+            content: @script.content)
+        end
+      end
+    end
+  end
+end