html-proofer 4.1.0 → 4.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c827c0b89b31ad51e1a4b177f190bdcbbb8f8fd933f235420be60da28e43cfb6
-  data.tar.gz: e47a4bf7944efc46622762720036c846c1ccb82acf339d99a93d7fade05eb1d9
+  metadata.gz: 285bb041bc95bc47d55de09a492e5d802a4c2a5dadf539bb7742f0d8138e2e23
+  data.tar.gz: dddc7a66aab15685917621d28600133b4824ce0c27ad204d7750252f717fe33e
 SHA512:
-  metadata.gz: c4b46613b516b0fbbaf52747d812dd390598b2bd3db6d45e342d2946ed012a7da2d47f36a0bae9a7c8b09fd484d7d0ecf2e4d12714f1044bb0ef2cfd5bd1d6da
-  data.tar.gz: '005058a0df180ce5619c2733438189842645840e1b94b6209bd3b2b6751b6cdd5f30b2adaa4aaed1e797e1332cfaf98388d54e06bf4497f5eb6f9796be71a1d8'
+  metadata.gz: bd983c2df18c4def1ded8f6caadc5917142d34021b9e873091a7040bd596b8efc05cee56b3fb3330ceab441f601d7c305e6b7e28a821fbb8d20ee7a85ccbf7a4
+  data.tar.gz: 966a38245a5bb725838b5c63214b5abc133c53e90fb927118b64a4da2a383d3c993e32994fd7eff0df473e15e8d23788737d716b00c8931977e82acb33114e58

data/bin/htmlproofer

@@ -15,21 +15,22 @@ Mercenary.program(:htmlproofer) do |p|
 
   p.description 'Runs the HTML-Proofer suite on the files in PATH. For more details, see the README.'
 
-  p.option 'allow_hash_href', '--allow-hash-href', 'If `true`, assumes `href="#"` anchors are valid'
-  p.option 'allow_missing_href', '--allow-missing-href', 'If `true`, does not flag `a` tags missing `href`. In HTML5, this is technically allowed, but could also be human error.'
+  p.option 'allow_hash_href', '--allow-hash-href=<true|false>',  'String', 'If `true`, assumes `href="#"` anchors are valid (default: `true`)'
+  p.option 'allow_missing_href', '--allow-missing-href=<true|false>',  'String', 'If `true`, does not flag `a` tags missing `href`. In HTML5, this is technically allowed, but could also be human error. (default: `false`)'
   p.option 'as_links', '--as-links', 'Assumes that `PATH` is a comma-separated array of links to check.'
   p.option 'assume_extension', '--assume-extension <ext>', 'Automatically add specified extension to files for internal links, to allow extensionless URLs (as supported by most servers) (default: `.html`).'
   p.option 'checks', '--checks check1,[check2,...]', Array, 'A comma-separated list of Strings indicating which checks you want to run (default: `["Links", "Images", "Scripts"]`)'
-  p.option 'check_external_hash', '--check-external-hash', 'Checks whether external hashes exist (even if the webpage exists) (default: `true`).'
-  p.option 'check_sri', '--check-sri', 'Check that `<link>` and `<script>` external resources use SRI (default: `false`).'
+  p.option 'check_external_hash', '--check-external-hash=<true|false>', 'String', 'Checks whether external hashes exist (even if the webpage exists) (default: `true`).'
+  p.option 'check_internal_hash', '--check-internal-hash=<true|false>',  'String', 'Checks whether internal hashes exist (even if the webpage exists) (default: `true`).'
+  p.option 'check_sri', '--check-sri=<true|false>', 'String', 'Check that `<link>` and `<script>` external resources use SRI (default: `false`).'
   p.option 'directory_index_file', '--directory-index-file <filename>', String, 'Sets the file to look for when a link refers to a directory. (default: `index.html`)'
-  p.option 'disable_external', '--disable-external', 'If `true`, does not run the external link checker (default: `false`)'
-  p.option 'enforce_https', '--enforce-https <false>', String, 'Fails a link if it\'s not marked as `https` (default: `true`).'
+  p.option 'disable_external', '--disable-external=<true|false>', String, 'If `true`, does not run the external link checker (default: `false`)'
+  p.option 'enforce_https', '--enforce-https=<true|false>', String, 'Fails a link if it\'s not marked as `https` (default: `true`).'
   p.option 'extensions', '--extensions ext1,[ext2,...[', Array, 'A comma-separated list of Strings indicating the file extensions you would like to check (including the dot) (default: `.html`)'
-  p.option 'ignore_empty_alt', '--ignore-empty-alt', ' If `true`, ignores images with empty/missing alt tags (in other words, `<img alt>` and `<img alt="">` are valid; set this to `false` to flag those)'
+  p.option 'ignore_empty_alt', '--ignore-empty-alt=<true|false>', 'String', 'If `true`, ignores images with empty/missing alt tags (in other words, `<img alt>` and `<img alt="">` are valid; set this to `false` to flag those) (default: `true`)'
+  p.option 'ignore_empty_mailto', '--ignore-empty-mailto=<true|false>',  'String', 'If `true`, allows `mailto:` `href`s which do not contain an email address (default: `false`)'
   p.option 'ignore_files', '--ignore-files file1,[file2,...]', Array, 'A comma-separated list of Strings or RegExps containing file paths that are safe to ignore'
-  p.option 'ignore_empty_mailto', '--ignore-empty-mailto', 'If `true`, allows `mailto:` `href`s which do not contain an email address'
-  p.option 'ignore_missing_alt', '--ignore-missing-alt', 'If `true`, ignores images with missing alt tags'
+  p.option 'ignore_missing_alt', '--ignore-missing-alt=<true|false>',  'String', 'If `true`, ignores images with missing alt tags (default: `false`)'
   p.option 'ignore_status_codes', '--ignore-status-codes 123,[xxx, ...]', Array, 'A comma-separated list of numbers representing status codes to ignore.'
   p.option 'ignore_urls', '--ignore-urls link1,[link2,...]', Array, 'A comma-separated list of Strings or RegExps containing URLs that are safe to ignore. This affects all HTML attributes, such as `alt` tags on images.'
   p.option 'log_level', '--log-level <level>', String, 'Sets the logging level, as determined by Yell. One of `:debug`, `:info`, `:warn`, `:error`, or `:fatal`. (default: `:info`)'
@@ -67,7 +68,15 @@ Mercenary.program(:htmlproofer) do |p|
       end
     end
 
-    options[:enforce_https] = false if opts['enforce_https'] == "false"
+    # check booleans
+    [:allow_hash_href, :allow_missing_href, :check_external_hash, :check_internal_hash, :check_sri, :disable_external, :enforce_https, :ignore_empty_alt, :ignore_empty_mailto, :ignore_missing_alt].each do |option|
+      next if (val = opts[option.to_s]).nil?
+      if val == "false"
+        options[option] = false
+      else
+        options[option] = true
+      end
+    end
 
     options[:log_level] = opts['log_level'].to_sym unless opts['log_level'].nil?
 

data/lib/html_proofer/attribute/url.rb

@@ -3,18 +3,20 @@
 module HTMLProofer
   class Attribute
     class Url < HTMLProofer::Attribute
-      attr_reader :url
+      attr_reader :url, :size
 
       REMOTE_SCHEMES = ["http", "https"].freeze
 
-      def initialize(runner, link_attribute, base_url: nil)
+      def initialize(runner, link_attribute, base_url: nil, extract_size: false)
         super
 
         if @raw_attribute.nil?
           @url = nil
         else
           @url = @raw_attribute.delete("\u200b").strip
+          @url, @size = @url.split(/\s+/) if extract_size
           @url = Addressable::URI.join(base_url, @url).to_s unless blank?(base_url)
+          @url = "" if @url.nil?
 
           swap_urls!
           clean_url!
@@ -30,6 +32,7 @@ module HTMLProofer
 
       def known_extension?
         return true if hash_link?
+        return true if path.end_with?("/")
 
         ext = File.extname(path)
 
@@ -201,6 +204,10 @@ module HTMLProofer
         url.start_with?("#")
       end
 
+      def has_hash?
+        url.include?("#")
+      end
+
       def param_link?
         url.start_with?("?")
       end

data/lib/html_proofer/cache.rb

@@ -26,12 +26,17 @@ module HTMLProofer
 
       if blank?(options)
         define_singleton_method(:enabled?) { false }
+        define_singleton_method(:external_enabled?) { false }
+        define_singleton_method(:internal_enabled?) { false }
       else
+        # we still consider the cache as enabled, regardless of the specic timeframes
         define_singleton_method(:enabled?) { true }
         setup_cache!(options)
 
         @external_timeframe = parsed_timeframe(options[:timeframe][:external])
+        define_singleton_method(:external_enabled?) { !@external_timeframe.nil? }
         @internal_timeframe = parsed_timeframe(options[:timeframe][:internal])
+        define_singleton_method(:internal_enabled?) { !@internal_timeframe.nil? }
       end
     end
 
@@ -55,17 +60,15 @@ module HTMLProofer
     end
 
     def add_internal(url, metadata, found)
-      return unless enabled?
+      return unless internal_enabled?
 
       @cache_log[:internal][url] = { time: @cache_time, metadata: [] } if @cache_log[:internal][url].nil?
 
       @cache_log[:internal][url][:metadata] << construct_internal_link_metadata(metadata, found)
     end
 
-    def add_external(url, filenames, status_code, msg)
-      return unless enabled?
-
-      found = status_code.between?(200, 299)
+    def add_external(url, filenames, status_code, msg, found)
+      return unless external_enabled?
 
       clean_url = cleaned_url(url)
       @cache_log[:external][clean_url] =
@@ -73,10 +76,10 @@ module HTMLProofer
     end
 
     def detect_url_changes(urls_detected, type)
-      additions = determine_additions(urls_detected, type)
-
       determine_deletions(urls_detected, type)
 
+      additions = determine_additions(urls_detected, type)
+
       additions
     end
 
@@ -96,13 +99,6 @@ module HTMLProofer
 
       urls_to_check = detect_url_changes(urls_detected, type)
 
-      @cache_log[type].each_pair do |url, cache|
-        within_timeframe = type == :external ? within_external_timeframe?(cache[:time]) : within_internal_timeframe?(cache[:time])
-        next if within_timeframe
-
-        urls_to_check[url] = cache[:metadata] # recheck expired links
-      end
-
       urls_to_check
     end
 
@@ -146,7 +142,11 @@ module HTMLProofer
     private def determine_external_additions(urls_detected)
       urls_detected.reject do |url, _metadata|
         if @cache_log[:external].include?(url)
-          @cache_log[:external][url][:found] # if this is false, we're trying again
+          found = @cache_log[:external][url][:found] # if this is false, we're trying again
+          unless found
+            @logger.log(:debug, "Adding #{url} to external cache (not found)")
+          end
+          found
         else
           @logger.log(:debug, "Adding #{url} to external cache")
           false
@@ -155,28 +155,36 @@ module HTMLProofer
     end
 
     private def determine_internal_additions(urls_detected)
-      urls_detected.each_with_object({}) do |(url, metadata), hsh|
+      urls_detected.each_with_object({}) do |(url, detected_metadata), hsh|
         # url is not even in cache
         if @cache_log[:internal][url].nil?
-          hsh[url] = metadata
+          @logger.log(:debug, "Adding #{url} to internal cache")
+          hsh[url] = detected_metadata
           next
         end
 
+        # detect metadata additions
+        # NOTE: the time-stamp for the whole url key will not be updated,
+        # so that it reflects the earliest time any of the metadata was checked
         cache_metadata = @cache_log[:internal][url][:metadata]
-        incoming_metadata = urls_detected[url].each_with_object([]) do |incoming_url, arr|
-          existing_cache_metadata = cache_metadata.find { |k, _| k[:filename] == incoming_url[:filename] }
-
+        metadata_additions = detected_metadata.reject do |detected|
+          existing_cache_metadata = cache_metadata.find { |cached, _| cached[:filename] == detected[:filename] }
           # cache for this url, from an existing path, exists as found
-          if !existing_cache_metadata.nil? && !existing_cache_metadata.empty? && existing_cache_metadata[:found]
-            metadata.find { |m| m[:filename] == existing_cache_metadata[:filename] }[:found] = true
-            next
+          found = !existing_cache_metadata.nil? && !existing_cache_metadata.empty? && existing_cache_metadata[:found]
+          unless found
+            @logger.log(:debug, "Adding #{detected} to internal cache for #{url}")
           end
+          found
+        end
 
-          @logger.log(:debug, "Adding #{incoming_url} to internal cache")
-          arr << incoming_url
+        if metadata_additions.empty?
+          next
         end
 
-        hsh[url] = incoming_metadata
+        hsh[url] = metadata_additions
+        # remove from the cache the detected metadata additions as they correspond to failures to be rechecked
+        # (this works assuming the detected url metadata have "found" set to false)
+        @cache_log[:internal][url][:metadata] = cache_metadata.difference(metadata_additions)
       end
     end
 
@@ -184,11 +192,16 @@ module HTMLProofer
     private def determine_deletions(urls_detected, type)
       deletions = 0
 
-      @cache_log[type].delete_if do |url, _|
-        if urls_detected.include?(url)
+      @cache_log[type].delete_if do |url, cache|
+        expired_timeframe = type == :external ? !within_external_timeframe?(cache[:time]) : !within_internal_timeframe?(cache[:time])
+        if expired_timeframe
+          @logger.log(:debug, "Removing #{url} from #{type} cache (expired timeframe)")
+          deletions += 1
+          true
+        elsif urls_detected.include?(url)
           false
         elsif url_matches_type?(url, type)
-          @logger.log(:debug, "Removing #{url} from #{type} cache")
+          @logger.log(:debug, "Removing #{url} from #{type} cache (not detected anymore)")
           deletions += 1
           true
         end

data/lib/html_proofer/check/images.rb

@@ -23,19 +23,9 @@ module HTMLProofer
           elsif !@img.url.exists? && !@img.multiple_srcsets? && !@img.multiple_sizes?
             add_failure("internal image #{@img.url.raw_attribute} does not exist", line: @img.line,
               content: @img.content)
-          elsif @img.multiple_srcsets?
-            @img.srcsets.each do |srcset|
-              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url)
-
-              if srcset_url.remote?
-                add_to_external_urls(srcset_url.url, @img.line)
-              elsif !srcset_url.exists?
-                add_failure("internal image #{srcset} does not exist", line: @img.line, content: @img.content)
-              end
-            end
-          elsif @img.multiple_sizes?
+          elsif @img.multiple_srcsets? || @img.multiple_sizes?
             @img.srcsets_wo_sizes.each do |srcset|
-              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url)
+              srcset_url = HTMLProofer::Attribute::Url.new(@runner, srcset, base_url: @img.base_url, extract_size: true)
 
               if srcset_url.remote?
                 add_to_external_urls(srcset_url.url, @img.line)

data/lib/html_proofer/configuration.rb

@@ -9,6 +9,7 @@ module HTMLProofer
       allow_missing_href: false,
       assume_extension: ".html",
       check_external_hash: true,
+      check_internal_hash: true,
       checks: DEFAULT_TESTS,
       directory_index_file: "index.html",
       disable_external: false,

data/lib/html_proofer/runner.rb

@@ -231,8 +231,6 @@ module HTMLProofer
       @logger.log(:debug, "Found #{cache_text} in the cache")
 
       urls_to_check = @cache.retrieve_urls(ivar, type)
-      urls_detected = pluralize(urls_to_check.count, "#{type} link", "#{type} links")
-      @logger.log(:info, "Checking #{urls_detected}")
 
       urls_to_check
     end

data/lib/html_proofer/url_validator/external.rb

@@ -22,12 +22,11 @@ module HTMLProofer
       end
 
       def validate
-        if @cache.enabled?
-          urls_to_check = @runner.load_external_cache
-          run_external_link_checker(urls_to_check)
-        else
-          run_external_link_checker(@external_urls)
-        end
+        urls_to_check = @cache.external_enabled? ? @runner.load_external_cache : @external_urls
+        urls_detected = pluralize(urls_to_check.count, "external link", "external links")
+        @logger.log(:info, "Checking #{urls_detected}")
+
+        run_external_link_checker(urls_to_check)
 
         @failed_checks
       end
@@ -89,7 +88,7 @@ module HTMLProofer
         return if @runner.options[:ignore_status_codes].include?(response_code)
 
         if response_code.between?(200, 299)
-          @cache.add_external(href, filenames, response_code, "OK") unless check_hash_in_2xx_response(href, url,
+          @cache.add_external(href, filenames, response_code, "OK", true) unless check_hash_in_2xx_response(href, url,
             response, filenames)
         elsif response.timed_out?
           handle_timeout(href, filenames, response_code)
@@ -104,7 +103,7 @@ module HTMLProofer
           status_message = blank?(response.status_message) ? "" : ": #{response.status_message}"
           msg = "External link #{href} failed#{status_message}"
           add_failure(filenames, msg, response_code)
-          @cache.add_external(href, filenames, response_code, msg)
+          @cache.add_external(href, filenames, response_code, msg, false)
         end
       end
 
@@ -133,13 +132,13 @@ module HTMLProofer
 
         msg = "External link #{href} failed: #{url.sans_hash} exists, but the hash '#{hash}' does not"
         add_failure(filenames, msg, response.code)
-        @cache.add_external(href, filenames, response.code, msg)
+        @cache.add_external(href, filenames, response.code, msg, false)
         true
       end
 
       def handle_timeout(href, filenames, response_code)
         msg = "External link #{href} failed: got a time out (response code #{response_code})"
-        @cache.add_external(href, filenames, 0, msg)
+        @cache.add_external(href, filenames, 0, msg, false)
         return if @runner.options[:only_4xx]
 
         add_failure(filenames, msg, response_code)
@@ -157,7 +156,7 @@ module HTMLProofer
 
         msg = msgs.join("\n").chomp
 
-        @cache.add_external(href, metadata, 0, msg)
+        @cache.add_external(href, metadata, 0, msg, false)
         return if @runner.options[:only_4xx]
 
         add_failure(metadata, msg, response_code)

data/lib/html_proofer/url_validator/internal.rb

@@ -12,12 +12,11 @@ module HTMLProofer
       end
 
       def validate
-        if @cache.enabled?
-          urls_to_check = @runner.load_internal_cache
-          run_internal_link_checker(urls_to_check)
-        else
-          run_internal_link_checker(@internal_urls)
-        end
+        urls_to_check = @cache.internal_enabled? ? @runner.load_internal_cache : @internal_urls
+        urls_detected = pluralize(urls_to_check.count, "internal link", "internal links")
+        @logger.log(:info, "Checking #{urls_detected}")
+
+        run_internal_link_checker(urls_to_check)
 
         @failed_checks
       end
@@ -68,9 +67,11 @@ module HTMLProofer
       private def hash_exists?(url)
         href_hash = url.hash
         return true if blank?(href_hash)
+        return true unless @runner.options[:check_internal_hash]
 
         # prevents searching files we didn't ask about
         return false unless url.known_extension?
+        return false unless url.has_hash?
 
         decoded_href_hash = Addressable::URI.unescape(href_hash)
         fragment_ids = [href_hash, decoded_href_hash]

data/lib/html_proofer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTMLProofer
-  VERSION = "4.1.0"
+  VERSION = "4.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html-proofer
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.3.1
 platform: ruby
 authors:
 - Garen Torikian
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2022-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable