broken_link_finder 0.9.3 → 0.11.1

data/lib/broken_link_finder/link_manager.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Class responsible for handling the link collection logic.
+  class LinkManager
+    # Used for mapping pages to broken links.
+    attr_reader :broken_links
+
+    # Used for mapping pages to ignored links.
+    attr_reader :ignored_links
+
+    # Used to record crawl statistics e.g. duration etc.
+    attr_reader :crawl_stats
+
+    # Used to map a link (as is) to its absolute (crawlable) form.
+    attr_reader :broken_link_map
+
+    # Used to prevent crawling a broken link twice.
+    attr_reader :all_broken_links
+
+    # Used to prevent crawling an intact link twice.
+    attr_reader :all_intact_links
+
+    # Used for building crawl statistics.
+    attr_reader :all_ignored_links
+
+    # Returns a new LinkManager instance with empty link collections.
+    def initialize(sort)
+      raise "Sort by either :page or :link, not #{sort}" \
+      unless %i[page link].include?(sort)
+
+      @sort = sort
+      @lock = Mutex.new
+
+      empty # Initialises the link collections.
+    end
+
+    # Initialise/empty the link collection objects.
+    def empty
+      @broken_links      = {}
+      @ignored_links     = {}
+      @crawl_stats       = {}
+      @broken_link_map   = {}
+      @all_broken_links  = Set.new
+      @all_intact_links  = Set.new
+      @all_ignored_links = Set.new
+    end
+
+    # Append key => [value] to the broken link collections.
+    # If map: true, then the link will also be recorded in @broken_link_map.
+    def append_broken_link(doc, link, map: true)
+      key, value = get_key_value(doc.url, link)
+
+      @lock.synchronize do
+        @broken_links[key] = [] unless @broken_links[key]
+        @broken_links[key] << value
+
+        @all_broken_links << link
+
+        @broken_link_map[link] = link.make_absolute(doc) if map
+      end
+    end
+
+    # Remove the broken link from the necessary collections.
+    def remove_broken_link(link)
+      @lock.synchronize do
+        if @sort == :page
+          @broken_links.each { |_k, links| links.delete(link) }
+          @broken_links.delete_if { |_k, links| links.empty? }
+        else
+          @broken_links.delete(link)
+        end
+
+        @all_broken_links.delete(link)
+        @all_intact_links << link
+      end
+    end
+
+    # Append key => [value] to the ignored link collections.
+    def append_ignored_link(url, link)
+      key, value = get_key_value(url, link)
+
+      @lock.synchronize do
+        @ignored_links[key] = [] unless @ignored_links[key]
+        @ignored_links[key] << value
+
+        @all_ignored_links << link
+      end
+    end
+
+    # Append link to @all_intact_links.
+    def append_intact_link(link)
+      @lock.synchronize { @all_intact_links << link }
+    end
+
+    # Sorts the link collection's keys and values alphabetically.
+    def sort
+      @broken_links.values.map(&:uniq!)
+      @ignored_links.values.map(&:uniq!)
+
+      @broken_links  = @broken_links.sort_by  { |k, _v| k }.to_h
+      @ignored_links = @ignored_links.sort_by { |k, _v| k }.to_h
+
+      @broken_links.each  { |_k, v| v.sort! }
+      @ignored_links.each { |_k, v| v.sort! }
+    end
+
+    # Tally's up various statistics about the crawl and its links.
+    def tally(url:, pages_crawled:, start:)
+      @crawl_stats[:url]               = url
+      @crawl_stats[:pages_crawled]     = pages_crawled
+      @crawl_stats[:num_pages]         = pages_crawled.size
+      @crawl_stats[:num_links]         = (
+        @all_broken_links.size + @all_intact_links.size + @all_ignored_links.size
+      )
+      @crawl_stats[:num_broken_links]  = @all_broken_links.size
+      @crawl_stats[:num_intact_links]  = @all_intact_links.size
+      @crawl_stats[:num_ignored_links] = @all_ignored_links.size
+      @crawl_stats[:duration]          = Time.now - start
+    end
+
+    private
+
+    # Returns the correct key value depending on the @sort type.
+    # @sort == :page ? [url, link] : [link, url]
+    def get_key_value(url, link)
+      case @sort
+      when :page
+        [url, link]
+      when :link
+        [link, url]
+      else
+        raise "Unsupported sort type: #{sort}"
+      end
+    end
+  end
+end

data/lib/broken_link_finder/reporter/html_reporter.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Class responsible for reporting in a HTML format.
+  class HTMLReporter < Reporter
+    # Returns a new HTMLReporter instance.
+    # stream is any Object that responds to :puts and :print.
+    def initialize(stream, sort,
+                   broken_links, ignored_links,
+                   broken_link_map, crawl_stats)
+      super
+    end
+
+    # Pretty print a report detailing the full link summary.
+    def call(broken_verbose: true, ignored_verbose: false)
+      puts '<div class="broken_link_finder_report">'
+
+      report_crawl_summary
+      report_broken_links(verbose: broken_verbose)
+      report_ignored_links(verbose: ignored_verbose)
+
+      puts '</div>'
+
+      nil
+    end
+
+    private
+
+    # Report a summary of the overall crawl.
+    def report_crawl_summary
+      puts format(
+        '<p class="crawl_summary">Crawled <a href="%s">%s</a><br />%s page(s) containing %s unique link(s) in %s seconds</p>',
+        @crawl_stats[:url],
+        @crawl_stats[:url],
+        @crawl_stats[:num_pages],
+        @crawl_stats[:num_links],
+        @crawl_stats[:duration]&.truncate(2)
+      )
+    end
+
+    # Report a summary of the broken links.
+    def report_broken_links(verbose: true)
+      puts '<div class="broken_links">'
+
+      if @broken_links.empty?
+        puts_summary 'Good news, there are no broken links!', type: :broken
+      else
+        num_pages, num_links = get_hash_stats(@broken_links)
+        puts_summary "Found #{num_links} unique broken link(s) across #{num_pages} page(s):", type: :broken
+
+        @broken_links.each do |key, values|
+          puts_group(key, type: :broken) # Puts the opening <p> element.
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts_group_item value, type: :broken }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts_group_item values[i], type: :broken }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, remove --concise to see them all<br />"
+          end
+
+          puts '</p>'
+        end
+      end
+
+      puts '</div>'
+    end
+
+    # Report a summary of the ignored links.
+    def report_ignored_links(verbose: false)
+      puts '<div class="ignored_links">'
+
+      if @ignored_links.any?
+        num_pages, num_links = get_hash_stats(@ignored_links)
+        puts_summary "Ignored #{num_links} unique unsupported link(s) across #{num_pages} page(s), which you should check manually:", type: :ignored
+
+        @ignored_links.each do |key, values|
+          puts_group(key, type: :ignored) # Puts the opening <p> element.
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts_group_item value, type: :ignored }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts_group_item values[i], type: :ignored }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, use --verbose to see them all<br />"
+          end
+
+          puts '</p>'
+        end
+      end
+
+      puts '</div>'
+    end
+
+    def puts_summary(text, type:)
+      klass = (type == :broken) ? 'broken_links_summary' : 'ignored_links_summary'
+      puts "<p class=\"#{klass}\">#{text}</p>"
+    end
+
+    def puts_group(link, type:)
+      href = build_url(link)
+      a_element = "<a href=\"#{href}\">#{link}</a>"
+
+      case type
+      when :broken
+        msg = sort_by_page? ?
+          "The following broken links were found on '#{a_element}':" :
+          "The broken link '#{a_element}' was found on the following pages:"
+        klass = 'broken_links_group'
+      when :ignored
+        msg = sort_by_page? ?
+          "The following links were ignored on '#{a_element}':" :
+          "The link '#{a_element}' was ignored on the following pages:"
+        klass = 'ignored_links_group'
+      else
+        raise "type: must be :broken or :ignored, not: #{type}"
+      end
+
+      puts "<p class=\"#{klass}\">"
+      puts msg + '<br />'
+    end
+
+    def puts_group_item(value, type:)
+      klass = (type == :broken) ? 'broken_links_group_item' : 'ignored_links_group_item'
+      puts "<a class=\"#{klass}\" href=\"#{build_url(value)}\">#{value}</a><br />"
+    end
+
+    def build_url(link)
+      href = @broken_link_map[link]
+      href || link
+    end
+
+    alias_method :report, :call
+  end
+end

data/lib/broken_link_finder/reporter/reporter.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Generic reporter class to be inherited from by format specific reporters.
+  class Reporter
+    # The amount of pages/links to display when verbose is false.
+    NUM_VALUES = 3
+
+    # Returns a new Reporter instance.
+    # stream is any Object that responds to :puts and :print.
+    def initialize(stream, sort,
+                   broken_links, ignored_links,
+                   broken_link_map, crawl_stats)
+      unless stream.respond_to?(:puts) && stream.respond_to?(:print)
+        raise 'stream must respond_to? :puts and :print'
+      end
+      raise "sort by either :page or :link, not #{sort}" \
+      unless %i[page link].include?(sort)
+
+      @stream          = stream
+      @sort            = sort
+      @broken_links    = broken_links
+      @ignored_links   = ignored_links
+      @broken_link_map = broken_link_map
+      @crawl_stats     = crawl_stats
+    end
+
+    # Pretty print a report detailing the full link summary.
+    def call(broken_verbose: true, ignored_verbose: false)
+      raise 'Not implemented by parent class'
+    end
+
+    protected
+
+    # Return true if the sort is by page.
+    def sort_by_page?
+      @sort == :page
+    end
+
+    # Returns the key/value statistics of hash e.g. the number of keys and
+    # combined values. The hash should be of the format: { 'str' => [...] }.
+    # Use like: `num_pages, num_links = get_hash_stats(links)`.
+    def get_hash_stats(hash)
+      num_keys   = hash.keys.length
+      num_values = hash.values.flatten.uniq.length
+
+      sort_by_page? ?
+        [num_keys, num_values] :
+        [num_values, num_keys]
+    end
+
+    # Prints the text. Defaults to a blank line.
+    def print(text = '')
+      @stream.print(text)
+    end
+
+    # Prints the text + \n. Defaults to a blank line.
+    def puts(text = '')
+      @stream.puts(text)
+    end
+
+    # Prints text + \n\n.
+    def putsn(text)
+      puts(text)
+      puts
+    end
+
+    # Prints \n + text + \n.
+    def nputs(text)
+      puts
+      puts(text)
+    end
+
+    alias_method :report, :call
+  end
+end

data/lib/broken_link_finder/reporter/text_reporter.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Class responsible for reporting in a text format.
+  class TextReporter < Reporter
+    # Returns a new TextReporter instance.
+    # stream is any Object that responds to :puts and :print.
+    def initialize(stream, sort,
+                   broken_links, ignored_links,
+                   broken_link_map, crawl_stats)
+      super
+    end
+
+    # Pretty print a report detailing the full link summary.
+    def call(broken_verbose: true, ignored_verbose: false)
+      report_crawl_summary
+      report_broken_links(verbose: broken_verbose)
+      report_ignored_links(verbose: ignored_verbose)
+
+      nil
+    end
+
+    private
+
+    # Report a summary of the overall crawl.
+    def report_crawl_summary
+      puts "Crawled #{@crawl_stats[:url]}"
+      putsn format(
+        '%s page(s) containing %s unique link(s) in %s seconds',
+        @crawl_stats[:num_pages],
+        @crawl_stats[:num_links],
+        @crawl_stats[:duration]&.truncate(2)
+      )
+    end
+
+    # Report a summary of the broken links.
+    def report_broken_links(verbose: true)
+      if @broken_links.empty?
+        puts 'Good news, there are no broken links!'
+      else
+        num_pages, num_links = get_hash_stats(@broken_links)
+        puts "Found #{num_links} unique broken link(s) across #{num_pages} page(s):"
+
+        @broken_links.each do |key, values|
+          msg = sort_by_page? ?
+            "The following broken links were found on '#{key}':" :
+            "The broken link '#{key}' was found on the following pages:"
+          nputs msg
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts value }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts values[i] }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, remove --concise to see them all"
+          end
+        end
+      end
+    end
+
+    # Report a summary of the ignored links.
+    def report_ignored_links(verbose: false)
+      if @ignored_links.any?
+        num_pages, num_links = get_hash_stats(@ignored_links)
+        nputs "Ignored #{num_links} unique unsupported link(s) across #{num_pages} page(s), which you should check manually:"
+
+        @ignored_links.each do |key, values|
+          msg = sort_by_page? ?
+            "The following links were ignored on '#{key}':" :
+            "The link '#{key}' was ignored on the following pages:"
+          nputs msg
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts value }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts values[i] }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, use --verbose to see them all"
+          end
+        end
+      end
+    end
+
+    alias_method :report, :call
+  end
+end

data/lib/broken_link_finder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrokenLinkFinder
-  VERSION = '0.9.3'
+  VERSION = '0.11.1'
 end

data/lib/broken_link_finder/wgit_extensions.rb

@@ -1,11 +1,31 @@
 # frozen_string_literal: true
 
-# We extract all the Document's links, not just the links to other webpages.
-Wgit::Document.define_extension(
+# Define a method on each doc for recording unparsable links.
+# Unparsable links are recorded as broken links by Finder.
+class Wgit::Document
+  def unparsable_links
+    @unparsable_links ||= []
+  end
+end
+
+# Returns a Wgit::Url or nil (if link is unparsable).
+# A proc is preferrable to a function to avoid polluting the global namespace.
+parse_link = lambda do |doc, link|
+  Wgit::Url.new(link)
+rescue StandardError
+  doc.unparsable_links << link
+  nil
+end
+
+# We extract all the Document's links e.g. <a>, <img>, <script>, <link> etc.
+Wgit::Document.define_extractor(
   :all_links,
-  '//*/@href | //*/@src', # Any element with a href or src attribute.
+  '//*/@href | //*/@src', # Any element's href or src attribute URL.
   singleton: false,
   text_content_only: true
-) do |links|
-  links.uniq.to_urls
+) do |links, doc|
+  links
+    .uniq
+    .map { |link| parse_link.call(doc, link) }
+    .compact
 end