broken_link_finder 0.9.4 → 0.12.0

data/lib/broken_link_finder/reporter/html_reporter.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Class responsible for reporting in a HTML format.
+  class HTMLReporter < Reporter
+    # Returns a new HTMLReporter instance.
+    # stream is any Object that responds to :puts and :print.
+    def initialize(stream, sort,
+                   broken_links, ignored_links,
+                   broken_link_map, crawl_stats)
+      super
+    end
+
+    # Pretty print a report detailing the full link summary.
+    def call(broken_verbose: true, ignored_verbose: false)
+      puts '<div class="broken_link_finder_report">'
+
+      report_crawl_summary
+      report_broken_links(verbose: broken_verbose)
+      report_ignored_links(verbose: ignored_verbose)
+
+      puts '</div>'
+
+      nil
+    end
+
+    private
+
+    # Report a summary of the overall crawl.
+    def report_crawl_summary
+      puts format(
+        '<p class="crawl_summary">Crawled <a href="%s">%s</a><br />%s page(s) containing %s unique link(s) in %s seconds</p>',
+        @crawl_stats[:url],
+        @crawl_stats[:url],
+        @crawl_stats[:num_pages],
+        @crawl_stats[:num_links],
+        @crawl_stats[:duration]&.truncate(2)
+      )
+    end
+
+    # Report a summary of the broken links.
+    def report_broken_links(verbose: true)
+      puts '<div class="broken_links">'
+
+      if @broken_links.empty?
+        puts_summary 'Good news, there are no broken links!', type: :broken
+      else
+        num_pages, num_links = get_hash_stats(@broken_links)
+        puts_summary "Found #{num_links} unique broken link(s) across #{num_pages} page(s):", type: :broken
+
+        @broken_links.each do |key, values|
+          puts_group(key, type: :broken) # Puts the opening <p> element.
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts_group_item value, type: :broken }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts_group_item values[i], type: :broken }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, remove --concise to see them all<br />"
+          end
+
+          puts '</p>'
+        end
+      end
+
+      puts '</div>'
+    end
+
+    # Report a summary of the ignored links.
+    def report_ignored_links(verbose: false)
+      puts '<div class="ignored_links">'
+
+      if @ignored_links.any?
+        num_pages, num_links = get_hash_stats(@ignored_links)
+        puts_summary "Ignored #{num_links} unique unsupported link(s) across #{num_pages} page(s), which you should check manually:", type: :ignored
+
+        @ignored_links.each do |key, values|
+          puts_group(key, type: :ignored) # Puts the opening <p> element.
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts_group_item value, type: :ignored }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts_group_item values[i], type: :ignored }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, use --verbose to see them all<br />"
+          end
+
+          puts '</p>'
+        end
+      end
+
+      puts '</div>'
+    end
+
+    def puts_summary(text, type:)
+      klass = (type == :broken) ? 'broken_links_summary' : 'ignored_links_summary'
+      puts "<p class=\"#{klass}\">#{text}</p>"
+    end
+
+    def puts_group(link, type:)
+      href = build_url(link)
+      a_element = "<a href=\"#{href}\">#{link}</a>"
+
+      case type
+      when :broken
+        msg = sort_by_page? ?
+          "The following broken links were found on '#{a_element}':" :
+          "The broken link '#{a_element}' was found on the following pages:"
+        klass = 'broken_links_group'
+      when :ignored
+        msg = sort_by_page? ?
+          "The following links were ignored on '#{a_element}':" :
+          "The link '#{a_element}' was ignored on the following pages:"
+        klass = 'ignored_links_group'
+      else
+        raise "type: must be :broken or :ignored, not: #{type}"
+      end
+
+      puts "<p class=\"#{klass}\">"
+      puts msg + '<br />'
+    end
+
+    def puts_group_item(value, type:)
+      klass = (type == :broken) ? 'broken_links_group_item' : 'ignored_links_group_item'
+      puts "<a class=\"#{klass}\" href=\"#{build_url(value)}\">#{value}</a><br />"
+    end
+
+    def build_url(link)
+      href = @broken_link_map[link]
+      href || link
+    end
+
+    alias_method :report, :call
+  end
+end

data/lib/broken_link_finder/reporter/reporter.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Generic reporter class to be inherited from by format specific reporters.
+  class Reporter
+    # The amount of pages/links to display when verbose is false.
+    NUM_VALUES = 3
+
+    # Returns a new Reporter instance.
+    # stream is any Object that responds to :puts and :print.
+    def initialize(stream, sort,
+                   broken_links, ignored_links,
+                   broken_link_map, crawl_stats)
+      unless stream.respond_to?(:puts) && stream.respond_to?(:print)
+        raise 'stream must respond_to? :puts and :print'
+      end
+      raise "sort by either :page or :link, not #{sort}" \
+      unless %i[page link].include?(sort)
+
+      @stream          = stream
+      @sort            = sort
+      @broken_links    = broken_links
+      @ignored_links   = ignored_links
+      @broken_link_map = broken_link_map
+      @crawl_stats     = crawl_stats
+    end
+
+    # Pretty print a report detailing the full link summary.
+    def call(broken_verbose: true, ignored_verbose: false)
+      raise 'Not implemented by parent class'
+    end
+
+    protected
+
+    # Return true if the sort is by page.
+    def sort_by_page?
+      @sort == :page
+    end
+
+    # Returns the key/value statistics of hash e.g. the number of keys and
+    # combined values. The hash should be of the format: { 'str' => [...] }.
+    # Use like: `num_pages, num_links = get_hash_stats(links)`.
+    def get_hash_stats(hash)
+      num_keys   = hash.keys.length
+      num_values = hash.values.flatten.uniq.length
+
+      sort_by_page? ?
+        [num_keys, num_values] :
+        [num_values, num_keys]
+    end
+
+    # Prints the text. Defaults to a blank line.
+    def print(text = '')
+      @stream.print(text)
+    end
+
+    # Prints the text + \n. Defaults to a blank line.
+    def puts(text = '')
+      @stream.puts(text)
+    end
+
+    # Prints text + \n\n.
+    def putsn(text)
+      puts(text)
+      puts
+    end
+
+    # Prints \n + text + \n.
+    def nputs(text)
+      puts
+      puts(text)
+    end
+
+    alias_method :report, :call
+  end
+end

data/lib/broken_link_finder/reporter/text_reporter.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Class responsible for reporting in a text format.
+  class TextReporter < Reporter
+    # Returns a new TextReporter instance.
+    # stream is any Object that responds to :puts and :print.
+    def initialize(stream, sort,
+                   broken_links, ignored_links,
+                   broken_link_map, crawl_stats)
+      super
+    end
+
+    # Pretty print a report detailing the full link summary.
+    def call(broken_verbose: true, ignored_verbose: false)
+      report_crawl_summary
+      report_broken_links(verbose: broken_verbose)
+      report_ignored_links(verbose: ignored_verbose)
+
+      nil
+    end
+
+    private
+
+    # Report a summary of the overall crawl.
+    def report_crawl_summary
+      puts "Crawled #{@crawl_stats[:url]}"
+      putsn format(
+        '%s page(s) containing %s unique link(s) in %s seconds',
+        @crawl_stats[:num_pages],
+        @crawl_stats[:num_links],
+        @crawl_stats[:duration]&.truncate(2)
+      )
+    end
+
+    # Report a summary of the broken links.
+    def report_broken_links(verbose: true)
+      if @broken_links.empty?
+        puts 'Good news, there are no broken links!'
+      else
+        num_pages, num_links = get_hash_stats(@broken_links)
+        puts "Found #{num_links} unique broken link(s) across #{num_pages} page(s):"
+
+        @broken_links.each do |key, values|
+          msg = sort_by_page? ?
+            "The following broken links were found on '#{key}':" :
+            "The broken link '#{key}' was found on the following pages:"
+          nputs msg
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts value }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts values[i] }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, remove --concise to see them all"
+          end
+        end
+      end
+    end
+
+    # Report a summary of the ignored links.
+    def report_ignored_links(verbose: false)
+      if @ignored_links.any?
+        num_pages, num_links = get_hash_stats(@ignored_links)
+        nputs "Ignored #{num_links} unique unsupported link(s) across #{num_pages} page(s), which you should check manually:"
+
+        @ignored_links.each do |key, values|
+          msg = sort_by_page? ?
+            "The following links were ignored on '#{key}':" :
+            "The link '#{key}' was ignored on the following pages:"
+          nputs msg
+
+          if verbose || (values.length <= NUM_VALUES)
+            values.each { |value| puts value }
+          else # Only print N values and summarise the rest.
+            NUM_VALUES.times { |i| puts values[i] }
+
+            objects = sort_by_page? ? 'link(s)' : 'page(s)'
+            puts "+ #{values.length - NUM_VALUES} other #{objects}, use --verbose to see them all"
+          end
+        end
+      end
+    end
+
+    alias_method :report, :call
+  end
+end

data/lib/broken_link_finder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrokenLinkFinder
-  VERSION = '0.9.4'
+  VERSION = '0.12.0'
 end

data/lib/broken_link_finder/wgit_extensions.rb

@@ -1,11 +1,31 @@
 # frozen_string_literal: true
 
-# We extract all the Document's links, not just the links to other webpages.
-Wgit::Document.define_extension(
+# Define a method on each doc for recording unparsable links.
+# Unparsable links are recorded as broken links by Finder.
+class Wgit::Document
+  def unparsable_links
+    @unparsable_links ||= []
+  end
+end
+
+# Returns a Wgit::Url or nil (if link is unparsable).
+# A proc is preferrable to a function to avoid polluting the global namespace.
+parse_link = lambda do |doc, link|
+  Wgit::Url.new(link)
+rescue StandardError
+  doc.unparsable_links << link
+  nil
+end
+
+# Define a custom extractor for all page links we're interested in checking.
+Wgit::Document.define_extractor(
   :all_links,
-  '//*/@href | //*/@src', # Any element with a href or src attribute.
+  lambda { BrokenLinkFinder::link_xpath },
   singleton: false,
   text_content_only: true
-) do |links|
-  links.uniq.to_urls
+) do |links, doc|
+  links
+    .uniq
+    .map { |link| parse_link.call(doc, link) }
+    .compact
 end

data/lib/broken_link_finder/xpath.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module BrokenLinkFinder
+  # Extract all the Document's <body> links e.g. <a>, <img>, <script> etc.
+  DEFAULT_LINK_XPATH = '/html/body//*/@href | /html/body//*/@src'
+
+  @link_xpath = DEFAULT_LINK_XPATH
+
+  class << self
+    # The xpath used to extract links from a crawled page.
+    # Can be overridden as required.
+    attr_accessor :link_xpath
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.12.0
 platform: ruby
 authors:
 - Michael Telford
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-11-02 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -100,42 +100,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: '0.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: '0.20'
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: wgit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '0.10'
 description: Finds a website's broken links using the 'wgit' gem and reports back
   to you with a summary.
 email: michael.telford@live.com
@@ -159,15 +159,22 @@ files:
 - exe/broken_link_finder
 - lib/broken_link_finder.rb
 - lib/broken_link_finder/finder.rb
-- lib/broken_link_finder/reporter.rb
+- lib/broken_link_finder/link_manager.rb
+- lib/broken_link_finder/reporter/html_reporter.rb
+- lib/broken_link_finder/reporter/reporter.rb
+- lib/broken_link_finder/reporter/text_reporter.rb
 - lib/broken_link_finder/version.rb
 - lib/broken_link_finder/wgit_extensions.rb
+- lib/broken_link_finder/xpath.rb
 - load.rb
 homepage: https://github.com/michaeltelford/broken-link-finder
 licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/michaeltelford/broken-link-finder
+  changelog_uri: https://github.com/michaeltelford/broken-link-finder/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/michaeltelford/broken-link-finder/issues
+  documentation_uri: https://www.rubydoc.info/gems/broken_link_finder
   allowed_push_host: https://rubygems.org
 post_install_message: Added the executable 'broken_link_finder' to $PATH
 rdoc_options: []
@@ -184,9 +191,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Finds a website's broken links and reports back to you with a summary.
 test_files: []