broken_link_finder 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be656fb33a6363c5da6bb1cd05e52f8bc9b1f7223825bd7dc4ada8af2bdea1d2
-  data.tar.gz: 2b4db95eaf086c10ac6f7528ec63c426ead3507f9adc4ecf0fd177c828d487f2
+  metadata.gz: 3e926c0997f5dcb805a9cdb4bf367237e3dff3284259395f6f38fa6b23a69172
+  data.tar.gz: 5c4020b8d1ae713a34593107580e82e3b3c06c75ec9a675e6cb536c07add1aca
 SHA512:
-  metadata.gz: 59bdd686ff0cce9359e51415011ba0521404834790ba7eec304d29d085bbdfafb3c1c0f8fd63bd235d41c0aed61278a2211f05fdb0f3d9c15d31a53f1b18b877
-  data.tar.gz: 27986886e3fa6ab4123027ff3067633787470c00401001942f7b28f4ba87fcfe38a11513caa2ade8dc5bfed0efb8a9db4af9749130c8960ac7ccb84a169fc154
+  metadata.gz: c988b4c602441da10fcf602ca14e528ad8ce2e076e961659e10130c3393c4d2a0729a1c51e4611445f3d20c27402d65ae2cab8bb937e1009b4af1123462aaa01
+  data.tar.gz: 448fbc03310b0378a91e2a2bf3db95bd33cc4cd2fe51af36c55e0409e05482c5beed9783ee6465b3747d8e25f4afd3fdfa72c0604834ab2b0df070921605a183

data/CHANGELOG.md

@@ -9,6 +9,15 @@
 - ...
 ---
 
+## v0.8.0
+### Added
+- Logic to prevent re-crawling links for more efficiency.
+### Changed/Removed
+- Updated the `wgit` gem which fixes a bug in `crawl_site` and adds support for IRI's.
+### Fixed
+- Bug where an error from the executable wasn't being rescued.
+---
+
 ## v0.7.0
 ### Added
 - Added the `--verbose` flag to the executable for displaying all ignored links.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.7.0)
+    broken_link_finder (0.8.0)
       thor (= 0.20.3)
       thread (= 0.2)
-      wgit (= 0.0.13)
+      wgit (= 0.0.15)
 
 GEM
   remote: https://rubygems.org/
@@ -24,7 +24,7 @@ GEM
     method_source (0.9.2)
     mini_portile2 (2.4.0)
     minitest (5.11.3)
-    mongo (2.8.0)
+    mongo (2.9.1)
       bson (>= 4.4.2, < 5.0.0)
     nokogiri (1.10.4)
       mini_portile2 (~> 2.4.0)
@@ -42,8 +42,9 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    wgit (0.0.13)
-      mongo (~> 2.8.0)
+    wgit (0.0.15)
+      addressable (~> 2.6.0)
+      mongo (~> 2.9.0)
       nokogiri (~> 1.10.3)
 
 PLATFORMS

data/benchmark.rb

@@ -1,18 +1,23 @@
-require 'broken_link_finder'
+require_relative './lib/broken_link_finder'
 require 'benchmark'
 require 'memory_profiler'
 
-url = ARGV[0] || "http://txti.es"
+url = ARGV[0] || 'http://txti.es'
 finder = BrokenLinkFinder::Finder.new
 
-puts Benchmark.measure { finder.crawl_page url }
+# puts Benchmark.measure { finder.crawl_page url }
 puts Benchmark.measure { finder.crawl_site url }
+puts "Links crawled: #{finder.total_links_crawled}"
 
-# http://txti.es
-# Pre  threading: 17.591528
-# Post threading: 7.508828 :-)
+# http://txti.es page crawl
+# Pre  threading: 17.5 seconds
+# Post threading: 7.5  seconds
 
-# http://txti.es
+# http://txti.es post threading - page vs site crawl
 # Page: 9.526981
 # Site: 9.732416
-# Multi-threading crawl_site now yields the same time as a single page.
+# Multi-threading crawl_site now yields the same time as a single page
+
+# https://meos.ch/ site crawl - post all link recording functionality
+# Pre:  608 seconds with 7665 links crawled
+# Post: 355 seconds with 1099 links crawled

data/broken_link_finder.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.post_install_message = "Added the executable 'broken_link_finder' to $PATH"
 
-  spec.required_ruby_version = '~> 2.5' # Only works with ruby 2.5.x
+  spec.required_ruby_version = '~> 2.5'
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
@@ -45,7 +45,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "httplog", "~> 1.3"
   spec.add_development_dependency "memory_profiler", "~> 0.9"
 
-  spec.add_runtime_dependency "wgit", "0.0.13"
+  spec.add_runtime_dependency "wgit", "0.0.15"
   spec.add_runtime_dependency "thread", "0.2"
   spec.add_runtime_dependency "thor", "0.20.3"
 end

data/exe/broken_link_finder

@@ -23,6 +23,8 @@ class BrokenLinkFinderCLI < Thor
       broken_verbose: broken_verbose,
       ignored_verbose: ignored_verbose
     )
+  rescue Exception => ex
+    puts "An error has occurred: #{ex.message}"
   end
 end
 

data/lib/broken_link_finder/finder.rb

@@ -1,6 +1,7 @@
 require_relative 'reporter'
 require 'wgit'
 require 'thread/pool'
+require 'set'
 
 module BrokenLinkFinder
   # Alias for BrokenLinkFinder::Finder.new, don't use this if you want to
@@ -12,7 +13,7 @@ module BrokenLinkFinder
   class Finder
     DEFAULT_MAX_THREADS = 30.freeze
 
-    attr_reader :broken_links, :ignored_links
+    attr_reader :broken_links, :ignored_links, :total_links_crawled
 
     # Creates a new Finder instance.
     def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS)
@@ -20,18 +21,21 @@ module BrokenLinkFinder
         raise "sort by either :page or :link, not #{sort}"
       end
 
-      @sort = sort
+      @sort        = sort
       @max_threads = max_threads
-      @lock = Mutex.new
-      @crawler = Wgit::Crawler.new
+      @lock        = Mutex.new
+      @crawler     = Wgit::Crawler.new
 
       clear_links
     end
 
     # Clear/empty the link collection Hashes.
     def clear_links
-      @broken_links = {}
-      @ignored_links = {}
+      @broken_links        = {}
+      @ignored_links       = {}
+      @total_links_crawled = 0
+      @all_broken_links    = Set.new
+      @all_intact_links    = Set.new
     end
 
     # Finds broken links within a single page and appends them to the
@@ -39,9 +43,9 @@ module BrokenLinkFinder
     # Access the broken links with Finder#broken_links.
     def crawl_url(url)
       clear_links
-      url = Wgit::Url.new(url)
 
       # Ensure the given page url is valid.
+      url = Wgit::Url.new(url)
       doc = @crawler.crawl_url(url)
       raise "Invalid URL: #{url}" unless doc
 
@@ -49,6 +53,8 @@ module BrokenLinkFinder
       find_broken_links(doc)
 
       sort_links
+      set_total_links_crawled
+
       @broken_links.any?
     end
 
@@ -58,6 +64,7 @@ module BrokenLinkFinder
     # Access the broken links with Finder#broken_links.
     def crawl_site(url)
       clear_links
+
       url = Wgit::Url.new(url)
       pool = Thread.pool(@max_threads)
       crawled_pages = []
@@ -73,8 +80,12 @@ module BrokenLinkFinder
         pool.process { find_broken_links(doc) }
       end
 
-      pool.shutdown # Wait for all threads to finish.
+      # Wait for all threads to finish.
+      pool.shutdown
+
       sort_links
+      set_total_links_crawled
+
       [@broken_links.any?, crawled_pages]
     end
 
@@ -101,7 +112,7 @@ module BrokenLinkFinder
 
     # Finds which links are unsupported or broken and records the details.
     def find_broken_links(doc)
-      # Process the Document's links before checking if they're broke.
+      # Report and reject any non supported links.
       links = doc.all_links.
         reject do |link|
           if !link.is_relative? and !link.start_with?('http')
@@ -113,13 +124,25 @@ module BrokenLinkFinder
 
       # Iterate over the supported links checking if they're broken or not.
       links.each do |link|
+        # Check if the link has already been processed previously.
+        next if @all_intact_links.include?(link)
+
+        if @all_broken_links.include?(link)
+          append_broken_link(doc.url, link)
+          next
+        end
+
+        # The link hasn't been processed before so we crawl it.
         link_url = link.is_relative? ? doc.url.to_base.concat(link) : link
         link_doc = @crawler.crawl_url(link_url)
 
+        # Determine if the crawled link is broken or not.
         if @crawler.last_response.is_a?(Net::HTTPNotFound) or
             link_doc.nil? or
             has_broken_anchor(link_doc)
           append_broken_link(doc.url, link)
+        else
+          @lock.synchronize { @all_intact_links << link }
         end
       end
 
@@ -140,17 +163,21 @@ module BrokenLinkFinder
     # Append key => [value] to @broken_links.
     def append_broken_link(url, link)
       key, value = get_key_value(url, link)
+
       @lock.synchronize do
         unless @broken_links[key]
           @broken_links[key] = []
         end
         @broken_links[key] << value
+
+        @all_broken_links  << link
       end
     end
 
     # Append key => [value] to @ignored_links.
     def append_ignored_link(url, link)
       key, value = get_key_value(url, link)
+
       @lock.synchronize do
         unless @ignored_links[key]
           @ignored_links[key] = []
@@ -180,6 +207,11 @@ module BrokenLinkFinder
       @ignored_links.each { |k, v| v.sort! }
     end
 
+    # Sets and returns the total number of links crawled.
+    def set_total_links_crawled
+      @total_links_crawled = @all_broken_links.size + @all_intact_links.size
+    end
+
     alias_method :crawl_page, :crawl_url
     alias_method :pretty_print_link_summary, :pretty_print_link_report
   end

data/lib/broken_link_finder/version.rb

@@ -1,3 +1,3 @@
 module BrokenLinkFinder
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Michael Telford
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-08-13 00:00:00.000000000 Z
+date: 2019-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 0.0.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 0.0.15
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement