html-proofer 4.2.0 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 916656620e369529ae71be63c4316ba627afcf97d2f60b4ce963e375f772a7b8
-  data.tar.gz: 83e751f1ec8bf453c2612cf1d34ef24e3de1d7620d740be1f8c392627f39cc85
+  metadata.gz: e8afbf9c7f55055c38b77971166e852d6f7180481d1e9fb27f84e071c582d1a7
+  data.tar.gz: 88db5f8e0bb8059ff3f6c50c8fcb4320bf0f6a4e1ef6fc053671591ff44a529f
 SHA512:
-  metadata.gz: c396512e2262ba0bbeab69ee602c42c522cdec4f7b9036e81808011e4b6d47e06188baeaa86524361166af90ce1751b3f08e4075b437b28eab8c4c6bec065630
-  data.tar.gz: e2e660641cca01f1501b69dda405808624ec03b4ebaa0a9c12618f82e77314f94adbc459ec343898f548be5676f763d11d69bf5cbbebf154148490e49b304f83
+  metadata.gz: 3f765689616f7884c1a33685dceb08a0786c851839d4ec612023d3ed746ed7c7b7e9087bd91d8360d795fd54a2077fdbf9af9df2c0b25d246a9cc0a064579501
+  data.tar.gz: 441a4f2b13e237505f9402a53fee5713a68d680824e6a976da42f26bfe1f57a0bdf661f6c3cafc45cb0fa28623c56d679c75044b10dd69db64ac824b7d911885

data/bin/htmlproofer

@@ -15,22 +15,22 @@ Mercenary.program(:htmlproofer) do |p|
 
   p.description 'Runs the HTML-Proofer suite on the files in PATH. For more details, see the README.'
 
-  p.option 'allow_hash_href', '--allow-hash-href', 'If `true`, assumes `href="#"` anchors are valid'
-  p.option 'allow_missing_href', '--allow-missing-href', 'If `true`, does not flag `a` tags missing `href`. In HTML5, this is technically allowed, but could also be human error.'
+  p.option 'allow_hash_href', '--allow-hash-href=<true|false>',  'String', 'If `true`, assumes `href="#"` anchors are valid (default: `true`)'
+  p.option 'allow_missing_href', '--allow-missing-href=<true|false>',  'String', 'If `true`, does not flag `a` tags missing `href`. In HTML5, this is technically allowed, but could also be human error. (default: `false`)'
   p.option 'as_links', '--as-links', 'Assumes that `PATH` is a comma-separated array of links to check.'
   p.option 'assume_extension', '--assume-extension <ext>', 'Automatically add specified extension to files for internal links, to allow extensionless URLs (as supported by most servers) (default: `.html`).'
   p.option 'checks', '--checks check1,[check2,...]', Array, 'A comma-separated list of Strings indicating which checks you want to run (default: `["Links", "Images", "Scripts"]`)'
-  p.option 'check_external_hash', '--check-external-hash', 'Checks whether external hashes exist (even if the webpage exists) (default: `true`).'
-  p.option 'check_internal_hash', '--check-internal-hash', 'Checks whether internal hashes exist (even if the webpage exists) (default: `true`).'
-  p.option 'check_sri', '--check-sri', 'Check that `<link>` and `<script>` external resources use SRI (default: `false`).'
+  p.option 'check_external_hash', '--check-external-hash=<true|false>', 'String', 'Checks whether external hashes exist (even if the webpage exists) (default: `true`).'
+  p.option 'check_internal_hash', '--check-internal-hash=<true|false>',  'String', 'Checks whether internal hashes exist (even if the webpage exists) (default: `true`).'
+  p.option 'check_sri', '--check-sri=<true|false>', 'String', 'Check that `<link>` and `<script>` external resources use SRI (default: `false`).'
   p.option 'directory_index_file', '--directory-index-file <filename>', String, 'Sets the file to look for when a link refers to a directory. (default: `index.html`)'
-  p.option 'disable_external', '--disable-external', 'If `true`, does not run the external link checker (default: `false`)'
-  p.option 'enforce_https', '--enforce-https <false>', String, 'Fails a link if it\'s not marked as `https` (default: `true`).'
+  p.option 'disable_external', '--disable-external=<true|false>', String, 'If `true`, does not run the external link checker (default: `false`)'
+  p.option 'enforce_https', '--enforce-https=<true|false>', String, 'Fails a link if it\'s not marked as `https` (default: `true`).'
   p.option 'extensions', '--extensions ext1,[ext2,...[', Array, 'A comma-separated list of Strings indicating the file extensions you would like to check (including the dot) (default: `.html`)'
-  p.option 'ignore_empty_alt', '--ignore-empty-alt', ' If `true`, ignores images with empty/missing alt tags (in other words, `<img alt>` and `<img alt="">` are valid; set this to `false` to flag those)'
+  p.option 'ignore_empty_alt', '--ignore-empty-alt=<true|false>', 'String', 'If `true`, ignores images with empty/missing alt tags (in other words, `<img alt>` and `<img alt="">` are valid; set this to `false` to flag those) (default: `true`)'
+  p.option 'ignore_empty_mailto', '--ignore-empty-mailto=<true|false>',  'String', 'If `true`, allows `mailto:` `href`s which do not contain an email address (default: `false`)'
   p.option 'ignore_files', '--ignore-files file1,[file2,...]', Array, 'A comma-separated list of Strings or RegExps containing file paths that are safe to ignore'
-  p.option 'ignore_empty_mailto', '--ignore-empty-mailto', 'If `true`, allows `mailto:` `href`s which do not contain an email address'
-  p.option 'ignore_missing_alt', '--ignore-missing-alt', 'If `true`, ignores images with missing alt tags'
+  p.option 'ignore_missing_alt', '--ignore-missing-alt=<true|false>',  'String', 'If `true`, ignores images with missing alt tags (default: `false`)'
   p.option 'ignore_status_codes', '--ignore-status-codes 123,[xxx, ...]', Array, 'A comma-separated list of numbers representing status codes to ignore.'
   p.option 'ignore_urls', '--ignore-urls link1,[link2,...]', Array, 'A comma-separated list of Strings or RegExps containing URLs that are safe to ignore. This affects all HTML attributes, such as `alt` tags on images.'
   p.option 'log_level', '--log-level <level>', String, 'Sets the logging level, as determined by Yell. One of `:debug`, `:info`, `:warn`, `:error`, or `:fatal`. (default: `:info`)'
@@ -68,7 +68,15 @@ Mercenary.program(:htmlproofer) do |p|
       end
     end
 
-    options[:enforce_https] = false if opts['enforce_https'] == "false"
+    # check booleans
+    [:allow_hash_href, :allow_missing_href, :check_external_hash, :check_internal_hash, :check_sri, :disable_external, :enforce_https, :ignore_empty_alt, :ignore_empty_mailto, :ignore_missing_alt].each do |option|
+      next if (val = opts[option.to_s]).nil?
+      if val == "false"
+        options[option] = false
+      else
+        options[option] = true
+      end
+    end
 
     options[:log_level] = opts['log_level'].to_sym unless opts['log_level'].nil?
 

data/lib/html_proofer/attribute/url.rb

@@ -3,18 +3,20 @@
 module HTMLProofer
   class Attribute
     class Url < HTMLProofer::Attribute
-      attr_reader :url
+      attr_reader :url, :size
 
       REMOTE_SCHEMES = ["http", "https"].freeze
 
-      def initialize(runner, link_attribute, base_url: nil)
+      def initialize(runner, link_attribute, base_url: nil, extract_size: false)
         super
 
         if @raw_attribute.nil?
           @url = nil
         else
           @url = @raw_attribute.delete("\u200b").strip
+          @url, @size = @url.split(/\s+/) if extract_size
           @url = Addressable::URI.join(base_url, @url).to_s unless blank?(base_url)
+          @url = "" if @url.nil?
 
           swap_urls!
           clean_url!
@@ -30,6 +32,7 @@ module HTMLProofer
 
       def known_extension?
         return true if hash_link?
+        return true if path.end_with?("/")
 
         ext = File.extname(path)
 
@@ -201,6 +204,10 @@ module HTMLProofer
         url.start_with?("#")
       end
 
+      def has_hash?
+        url.include?("#")
+      end
+
       def param_link?
         url.start_with?("?")
       end

data/lib/html_proofer/cache.rb

@@ -26,12 +26,17 @@ module HTMLProofer
 
       if blank?(options)
         define_singleton_method(:enabled?) { false }
+        define_singleton_method(:external_enabled?) { false }
+        define_singleton_method(:internal_enabled?) { false }
       else
+        # we still consider the cache as enabled, regardless of the specic timeframes
         define_singleton_method(:enabled?) { true }
         setup_cache!(options)
 
         @external_timeframe = parsed_timeframe(options[:timeframe][:external])
+        define_singleton_method(:external_enabled?) { !@external_timeframe.nil? }
         @internal_timeframe = parsed_timeframe(options[:timeframe][:internal])
+        define_singleton_method(:internal_enabled?) { !@internal_timeframe.nil? }
       end
     end
 
@@ -55,17 +60,15 @@ module HTMLProofer
     end
 
     def add_internal(url, metadata, found)
-      return unless enabled?
+      return unless internal_enabled?
 
       @cache_log[:internal][url] = { time: @cache_time, metadata: [] } if @cache_log[:internal][url].nil?
 
       @cache_log[:internal][url][:metadata] << construct_internal_link_metadata(metadata, found)
     end
 
-    def add_external(url, filenames, status_code, msg)
-      return unless enabled?
-
-      found = status_code.between?(200, 299)
+    def add_external(url, filenames, status_code, msg, found)
+      return unless external_enabled?
 
       clean_url = cleaned_url(url)
       @cache_log[:external][clean_url] =
@@ -73,10 +76,10 @@ module HTMLProofer
     end
 
     def detect_url_changes(urls_detected, type)
-      additions = determine_additions(urls_detected, type)
-
       determine_deletions(urls_detected, type)
 
+      additions = determine_additions(urls_detected, type)
+
       additions
     end
 
@@ -96,13 +99,6 @@ module HTMLProofer
 
       urls_to_check = detect_url_changes(urls_detected, type)
 
-      @cache_log[type].each_pair do |url, cache|
-        within_timeframe = type == :external ? within_external_timeframe?(cache[:time]) : within_internal_timeframe?(cache[:time])
-        next if within_timeframe
-
-        urls_to_check[url] = cache[:metadata] # recheck expired links
-      end
-
       urls_to_check
     end
 
@@ -146,7 +142,11 @@ module HTMLProofer
     private def determine_external_additions(urls_detected)
       urls_detected.reject do |url, _metadata|
         if @cache_log[:external].include?(url)
-          @cache_log[:external][url][:found] # if this is false, we're trying again
+          found = @cache_log[:external][url][:found] # if this is false, we're trying again
+          unless found
+            @logger.log(:debug, "Adding #{url} to external cache (not found)")
+          end
+          found
         else
           @logger.log(:debug, "Adding #{url} to external cache")
           false
@@ -155,28 +155,36 @@ module HTMLProofer
     end
 
     private def determine_internal_additions(urls_detected)
-      urls_detected.each_with_object({}) do |(url, metadata), hsh|
+      urls_detected.each_with_object({}) do |(url, detected_metadata), hsh|
         # url is not even in cache
         if @cache_log[:internal][url].nil?
-          hsh[url] = metadata
+          @logger.log(:debug, "Adding #{url} to internal cache")
+          hsh[url] = detected_metadata
           next
         end
 
+        # detect metadata additions
+        # NOTE: the time-stamp for the whole url key will not be updated,
+        # so that it reflects the earliest time any of the metadata was checked
         cache_metadata = @cache_log[:internal][url][:metadata]
-        incoming_metadata = urls_detected[url].each_with_object([]) do |incoming_url, arr|
-          existing_cache_metadata = cache_metadata.find { |k, _| k[:filename] == incoming_url[:filename] }
-
+        metadata_additions = detected_metadata.reject do |detected|
+          existing_cache_metadata = cache_metadata.find { |cached, _| cached[:filename] == detected[:filename] }
           # cache for this url, from an existing path, exists as found
-          if !existing_cache_metadata.nil? && !existing_cache_metadata.empty? && existing_cache_metadata[:found]
-            metadata.find { |m| m[:filename] == existing_cache_metadata[:filename] }[:found] = true
-            next
+          found = !existing_cache_metadata.nil? && !existing_cache_metadata.empty? && existing_cache_metadata[:found]
+          unless found
+            @logger.log(:debug, "Adding #{detected} to internal cache for #{url}")
           end
+          found
+        end
 
-          @logger.log(:debug, "Adding #{incoming_url} to internal cache")
-          arr << incoming_url
+        if metadata_additions.empty?
+          next
         end
 
-        hsh[url] = incoming_metadata
+        hsh[url] = metadata_additions
+        # remove from the cache the detected metadata additions as they correspond to failures to be rechecked
+        # (this works assuming the detected url metadata have "found" set to false)
+        @cache_log[:internal][url][:metadata] = cache_metadata.difference(metadata_additions)
       end
     end
 
@@ -184,11 +192,16 @@ module HTMLProofer
     private def determine_deletions(urls_detected, type)
       deletions = 0
 
-      @cache_log[type].delete_if do |url, _|
-        if urls_detected.include?(url)
+      @cache_log[type].delete_if do |url, cache|
+        expired_timeframe = type == :external ? !within_external_timeframe?(cache[:time]) : !within_internal_timeframe?(cache[:time])
+        if expired_timeframe
+          @logger.log(:debug, "Removing #{url} from #{type} cache (expired timeframe)")
+          deletions += 1
+          true
+        elsif urls_detected.include?(url)
           false
         elsif url_matches_type?(url, type)
-          @logger.log(:debug, "Removing #{url} from #{type} cache")
+          @logger.log(:debug, "Removing #{url} from #{type} cache (not detected anymore)")
           deletions += 1
           true
         end

data/lib/html_proofer/check/images.rb

@@ -25,7 +25,7 @@ module HTMLProofer
               content: @img.content)
           elsif @img.multiple_srcsets?
             @img.srcsets.each do |srcset|
-              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url)
+              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url, extract_size: true)
 
               if srcset_url.remote?
                 add_to_external_urls(srcset_url.url, @img.line)
@@ -35,7 +35,7 @@ module HTMLProofer
             end
           elsif @img.multiple_sizes?
             @img.srcsets_wo_sizes.each do |srcset|
-              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url)
+              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url, extract_size: true)
 
               if srcset_url.remote?
                 add_to_external_urls(srcset_url.url, @img.line)

data/lib/html_proofer/runner.rb

@@ -231,8 +231,6 @@ module HTMLProofer
       @logger.log(:debug, "Found #{cache_text} in the cache")
 
       urls_to_check = @cache.retrieve_urls(ivar, type)
-      urls_detected = pluralize(urls_to_check.count, "#{type} link", "#{type} links")
-      @logger.log(:info, "Checking #{urls_detected}")
 
       urls_to_check
     end

data/lib/html_proofer/url_validator/external.rb

@@ -22,12 +22,11 @@ module HTMLProofer
       end
 
       def validate
-        if @cache.enabled?
-          urls_to_check = @runner.load_external_cache
-          run_external_link_checker(urls_to_check)
-        else
-          run_external_link_checker(@external_urls)
-        end
+        urls_to_check = @cache.external_enabled? ? @runner.load_external_cache : @external_urls
+        urls_detected = pluralize(urls_to_check.count, "external link", "external links")
+        @logger.log(:info, "Checking #{urls_detected}")
+
+        run_external_link_checker(urls_to_check)
 
         @failed_checks
       end
@@ -89,7 +88,7 @@ module HTMLProofer
         return if @runner.options[:ignore_status_codes].include?(response_code)
 
         if response_code.between?(200, 299)
-          @cache.add_external(href, filenames, response_code, "OK") unless check_hash_in_2xx_response(href, url,
+          @cache.add_external(href, filenames, response_code, "OK", true) unless check_hash_in_2xx_response(href, url,
             response, filenames)
         elsif response.timed_out?
           handle_timeout(href, filenames, response_code)
@@ -104,7 +103,7 @@ module HTMLProofer
           status_message = blank?(response.status_message) ? "" : ": #{response.status_message}"
           msg = "External link #{href} failed#{status_message}"
           add_failure(filenames, msg, response_code)
-          @cache.add_external(href, filenames, response_code, msg)
+          @cache.add_external(href, filenames, response_code, msg, false)
         end
       end
 
@@ -133,13 +132,13 @@ module HTMLProofer
 
         msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
         add_failure(filenames, msg, response.code)
-        @cache.add_external(href, filenames, response.code, msg)
+        @cache.add_external(href, filenames, response.code, msg, false)
         true
       end
 
       def handle_timeout(href, filenames, response_code)
         msg = "External link #{href} failed: got a time out (response code #{response_code})"
-        @cache.add_external(href, filenames, 0, msg)
+        @cache.add_external(href, filenames, 0, msg, false)
         return if @runner.options[:only_4xx]
 
         add_failure(filenames, msg, response_code)
@@ -157,7 +156,7 @@ module HTMLProofer
 
         msg = msgs.join("\n").chomp
 
-        @cache.add_external(href, metadata, 0, msg)
+        @cache.add_external(href, metadata, 0, msg, false)
         return if @runner.options[:only_4xx]
 
         add_failure(metadata, msg, response_code)

data/lib/html_proofer/url_validator/internal.rb

@@ -12,12 +12,11 @@ module HTMLProofer
       end
 
       def validate
-        if @cache.enabled?
-          urls_to_check = @runner.load_internal_cache
-          run_internal_link_checker(urls_to_check)
-        else
-          run_internal_link_checker(@internal_urls)
-        end
+        urls_to_check = @cache.internal_enabled? ? @runner.load_internal_cache : @internal_urls
+        urls_detected = pluralize(urls_to_check.count, "internal link", "internal links")
+        @logger.log(:info, "Checking #{urls_detected}")
+
+        run_internal_link_checker(urls_to_check)
 
         @failed_checks
       end
@@ -72,6 +71,7 @@ module HTMLProofer
 
         # prevents searching files we didn't ask about
         return false unless url.known_extension?
+        return false unless url.has_hash?
 
         decoded_href_hash = Addressable::URI.unescape(href_hash)
         fragment_ids = [href_hash, decoded_href_hash]

data/lib/html_proofer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = "4.2.0"
+  VERSION = "4.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html-proofer
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.3.0
 platform: ruby
 authors:
 - Garen Torikian
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-16 00:00:00.000000000 Z
+date: 2022-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable