broken_link_finder 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d7f59b84300909c1325badc759f20ae3e87026dc615e4f8c8a3a9e4c5e2a5b3
-  data.tar.gz: add7905653a1036b2b1e6939671bf59dbfef3718336b2943c023f95c9d18e5c1
+  metadata.gz: d3f285779c735d254089d0b757d78a5fe1e1a082e0cbe21f591f499f2da54ba6
+  data.tar.gz: 3f62093f5589eb15df77f2289c86f826c1d7750401713b4854d26686e44dd745
 SHA512:
-  metadata.gz: 85d406b55aa0431bb6c2a0b7a52f2cc6b3cfa526d11a2969f2339d72af8cbeb69fc7d0c3c17d7bc723a998c0aa650f120843b107d7a4030db515be74883f41d2
-  data.tar.gz: 194273bfdcd99eff722b83dd3a86748d5e8a5875c48db8f7281cdce7fd856e93d6e764fcd2ded030c9a4ed22898277e386b0b61a02282a79599269545d903bdb
+  metadata.gz: 05d14ed145636e0a711b19519370b66189b39854a032ac1131c222d34f2a304c82cbcaefb7e60ed976ed0bd94a4a0dc191d7c80f885d7fd9811fe9ed3a8aafcb
+  data.tar.gz: a0ff34531ec08bd8abe134e7214ac74e3cf90b0d84d57f0268395b8ce2f55a31f8e0b8efeace50fa594e1a5af8b1afb5302fbac4cf7820579c2150d4ad63133c

data/CHANGELOG.md

@@ -9,6 +9,17 @@
 - ...
 ---
 
+## v0.9.0
+### Added
+- The `version` command to the executable.
+- The `--threads` aka `-t` option to the executable's `crawl` command to control crawl speed vs. resource usage.
+### Changed/Removed
+- Changed the default number of maximum threads for a recursive crawl from 30 to 100. Users will see a speed boost with increased resource usage as a result. This is configurable using the new `crawl` command option e.g. `--threads 30`.
+### Fixed
+- Several bugs by updating the `wgit` dependancy.
+- A bug in the report logic causing an incorrect link count.
+---
+
 ## v0.8.1
 ### Added
 - ...

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.8.1)
+    broken_link_finder (0.9.0)
       thor (= 0.20.3)
       thread (= 0.2)
-      wgit (= 0.0.16)
+      wgit (= 0.0.17)
 
 GEM
   remote: https://rubygems.org/
@@ -42,7 +42,7 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    wgit (0.0.16)
+    wgit (0.0.17)
       addressable (~> 2.6.0)
       mongo (~> 2.9.0)
       nokogiri (~> 1.10.3)

data/README.md

@@ -39,6 +39,10 @@ Or install it yourself as:
 
     $ gem install broken_link_finder
 
+Finally, verify the installation with:
+
+    $ broken_link_finder version
+
 ## Usage
 
 You can check for broken links via the library or executable.

data/Rakefile

@@ -15,6 +15,16 @@ task :help do
   system "bundle exec rake -D"
 end
 
+desc "Run the setup script"
+task :setup do
+  system "./bin/setup"
+end
+
+desc "Run the development console"
+task :console do
+  system "./bin/console"
+end
+
 desc "Compile all project Ruby files with warnings."
 task :compile do
   paths = Dir["**/*.rb", "**/*.gemspec", 'exe/broken_link_finder']

data/benchmark.rb

@@ -18,6 +18,6 @@ puts "Links crawled: #{finder.total_links_crawled}"
 # Site: 9.732416
 # Multi-threading crawl_site now yields the same time as a single page
 
-# https://meos.ch/ site crawl - post all link recording functionality
+# Large site crawl - post all link recording functionality
 # Pre:  608 seconds with 7665 links crawled
 # Post: 355 seconds with 1099 links crawled

data/bin/console

@@ -5,20 +5,29 @@ require "pry"
 require "byebug"
 require "broken_link_finder"
 require 'wgit/core_ext'
+require 'logger'
 require 'httplog'
 
+logger = Logger.new(STDOUT)
+logger.formatter = proc do |severity, datetime, progname, msg|
+  "#{msg}\n"
+end
+
 # Monkey patch all Net:HTTP network calls and log them.
 HttpLog.configure do |config|
+  config.enabled        = true
+  config.logger         = logger
+
   config.log_connect    = false
   config.log_request    = true
   config.log_headers    = false
   config.log_data       = false
   config.log_status     = true
   config.log_response   = false
-  config.log_benchmark  = true
+  config.log_benchmark  = false
 
-  config.compact_log    = true
-  config.json_log       = true
+  config.compact_log    = false
+  config.json_log       = false
 end
 
 # Call reload to load all recent code changes.
@@ -33,9 +42,12 @@ end
 
 # You can add fixtures and/or initialization code here...
 reload
+
 url = "http://txti.es/"
 by_page = Finder.new
 by_link = Finder.new sort: :link
 finder = by_page
 
+puts "\nbroken_link_finder v#{BrokenLinkFinder::VERSION}"
+
 binding.pry

data/broken_link_finder.gemspec

@@ -45,7 +45,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "httplog", "~> 1.3"
   spec.add_development_dependency "memory_profiler", "~> 0.9"
 
-  spec.add_runtime_dependency "wgit", "0.0.16"
+  spec.add_runtime_dependency "wgit", "0.0.17"
   spec.add_runtime_dependency "thread", "0.2"
   spec.add_runtime_dependency "thor", "0.20.3"
 end

data/exe/broken_link_finder

@@ -6,18 +6,20 @@ require 'thor'
 
 class BrokenLinkFinderCLI < Thor
   desc 'crawl [URL]', 'Find broken links at the URL'
-  option :recursive, type: :boolean, aliases: [:r], desc: 'Crawl the entire site.'
-  option :sort_by_link, type: :boolean, aliases: [:l], desc: 'Makes report more concise if there are more pages crawled than broken links found. Use with -r on medium/large sites.'
-  option :verbose, type: :boolean, aliases: [:v], desc: 'Display all ignored links.'
-  option :concise, type: :boolean, aliases: [:c], desc: 'Display only a summary of broken links.'
+  option :recursive, type: :boolean, aliases: [:r], default: false, desc: 'Crawl the entire site.'
+  option :threads, type: :numeric, aliases: [:t], default: BrokenLinkFinder::DEFAULT_MAX_THREADS, desc: 'Max number of threads to use when crawling recursively; 1 thread per web page.'
+  option :sort_by_link, type: :boolean, aliases: [:l], default: false, desc: 'Makes report more concise if there are more pages crawled than broken links found. Use with -r on medium/large sites.'
+  option :verbose, type: :boolean, aliases: [:v], default: false, desc: 'Display all ignored links.'
+  option :concise, type: :boolean, aliases: [:c], default: false, desc: 'Display only a summary of broken links.'
   def crawl(url)
     url = "http://#{url}" unless url.start_with?('http')
 
-    sort_by           = options[:sort_by_link]  ? :link : :page
-    broken_verbose    = options[:concise]       ? false : true
-    ignored_verbose   = options[:verbose]       ? true  : false
+    sort_by           = options[:sort_by_link] ? :link : :page
+    max_threads       = options[:threads]
+    broken_verbose    = !options[:concise]
+    ignored_verbose   = options[:verbose]
 
-    finder = BrokenLinkFinder::Finder.new(sort: sort_by)
+    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads: max_threads)
     options[:recursive] ? finder.crawl_site(url) : finder.crawl_page(url)
     finder.pretty_print_link_report(
       broken_verbose: broken_verbose,
@@ -26,6 +28,11 @@ class BrokenLinkFinderCLI < Thor
   rescue Exception => ex
     puts "An error has occurred: #{ex.message}"
   end
+
+  desc 'version', 'Display the currently installed version'
+  def version
+    puts "broken_link_finder v#{BrokenLinkFinder::VERSION}"
+  end
 end
 
 BrokenLinkFinderCLI.start(ARGV)

data/lib/broken_link_finder/finder.rb

@@ -4,19 +4,18 @@ require 'thread/pool'
 require 'set'
 
 module BrokenLinkFinder
-  # Alias for BrokenLinkFinder::Finder.new, don't use this if you want to
-  # override the max_threads variable.
-  def self.new(sort: :page)
-    Finder.new(sort: sort)
+  DEFAULT_MAX_THREADS = 100.freeze
+
+  # Alias for BrokenLinkFinder::Finder.new.
+  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS)
+    Finder.new(sort: sort, max_threads: max_threads)
   end
 
   class Finder
-    DEFAULT_MAX_THREADS = 30.freeze
-
-    attr_reader :broken_links, :ignored_links, :total_links_crawled
+    attr_reader :sort, :broken_links, :ignored_links, :total_links_crawled, :max_threads
 
     # Creates a new Finder instance.
-    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS)
+    def initialize(sort: :page, max_threads: BrokenLinkFinder::DEFAULT_MAX_THREADS)
       unless [:page, :link].include?(sort)
         raise "sort by either :page or :link, not #{sort}"
       end
@@ -44,10 +43,11 @@ module BrokenLinkFinder
     def crawl_url(url)
       clear_links
 
-      # Ensure the given page url is valid.
       url = Wgit::Url.new(url)
       doc = @crawler.crawl_url(url)
-      raise "Invalid URL: #{url}" unless doc
+
+      # Ensure the given page url is valid.
+      raise "Invalid or broken URL: #{url}" unless doc
 
       # Get all page links and determine which are broken.
       find_broken_links(doc)
@@ -70,23 +70,24 @@ module BrokenLinkFinder
       crawled_pages = []
 
       # Crawl the site's HTML web pages looking for links.
-      @crawler.crawl_site(url) do |doc|
-        # Ensure the given website url is valid.
-        raise "Invalid URL: #{url}" if doc.url == url and doc.empty?
+      orig_doc = @crawler.crawl_site(url) do |doc|
         crawled_pages << doc.url
-
-        # Get all page links and determine which are broken.
         next unless doc
+
+        # Start a thread for each page, checking for broken links.
         pool.process { find_broken_links(doc) }
       end
 
+      # Ensure the given website url is valid.
+      raise "Invalid or broken URL: #{url}" if orig_doc.nil?
+
       # Wait for all threads to finish.
       pool.shutdown
 
       sort_links
       set_total_links_crawled
 
-      [@broken_links.any?, crawled_pages]
+      [@broken_links.any?, crawled_pages.uniq]
     end
 
     # Pretty prints the link report into a stream e.g. STDOUT or a file,
@@ -133,7 +134,7 @@ module BrokenLinkFinder
         end
 
         # The link hasn't been processed before so we crawl it.
-        link_url = link.is_relative? ? doc.url.to_base.concat(link) : link
+        link_url = get_absolute_link(doc, link)
         link_doc = @crawler.crawl_url(link_url)
 
         # Determine if the crawled link is broken or not.
@@ -149,6 +150,11 @@ module BrokenLinkFinder
       nil
     end
 
+    # Returns the link in absolute form so it can be crawled.
+    def get_absolute_link(doc, link)
+      link.is_relative? ? doc.base_url(link: link).concat(link) : link
+    end
+
     # Returns true if the link is/contains a broken anchor.
     def has_broken_anchor(doc)
       raise "link document is nil" unless doc
@@ -200,10 +206,13 @@ module BrokenLinkFinder
 
     # Sort keys and values alphabetically.
     def sort_links
-      @broken_links = @broken_links.sort_by { |k, v| k }.to_h
+      @broken_links.values.map  { |v| v.uniq! }
+      @ignored_links.values.map { |v| v.uniq! }
+
+      @broken_links  = @broken_links.sort_by  { |k, v| k }.to_h
       @ignored_links = @ignored_links.sort_by { |k, v| k }.to_h
 
-      @broken_links.each { |k, v| v.sort! }
+      @broken_links.each  { |k, v| v.sort! }
       @ignored_links.each { |k, v| v.sort! }
     end
 

data/lib/broken_link_finder/reporter.rb

@@ -11,16 +11,17 @@ module BrokenLinkFinder
         raise "sort by either :page or :link, not #{sort}"
       end
 
-      @stream = stream
-      @sort = sort
-      @broken_links = broken_links
-      @ignored_links = ignored_links
+      @stream         = stream
+      @sort           = sort
+      @broken_links   = broken_links
+      @ignored_links  = ignored_links
     end
 
     # Pretty print a report detailing the link summary.
     def pretty_print_link_report(broken_verbose: true, ignored_verbose: false)
       report_broken_links(verbose: broken_verbose)
       report_ignored_links(verbose: ignored_verbose)
+      nil
     end
 
     private

data/lib/broken_link_finder/version.rb

@@ -1,3 +1,3 @@
 module BrokenLinkFinder
-  VERSION = "0.8.1"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Michael Telford
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-21 00:00:00.000000000 Z
+date: 2019-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 0.0.17
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.16
+        version: 0.0.17
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement