broken_link_finder 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b9617c6016a37c3afeec665d3f5d70e8574e8cf28100f4c661ee95fd7f7ccee
-  data.tar.gz: cf517abd6b554f3cdf01b535dfd4417dd226fdde8edae4556190b0da69cd3bda
+  metadata.gz: 0f89fd142f329ee9b5df8de5a01abd2976cab0b16a690e50d98a0775a94eb937
+  data.tar.gz: 81e1db2b0a7f2e76e0113b4086fea5a41a6bca6b8f51d79d849007fe73fb3c34
 SHA512:
-  metadata.gz: 691714c983790ff8d8a004e3c92f1edae1a50e44e4771f123d177904364c65679c034580a3b182209a18b2baa83277b88c571e0cddcf131c638fcc106cc031a5
-  data.tar.gz: '0093753cea2740b13b5b8fda6dde5046e76ce4e8e7cf93e3be7ae99cf7bd6e6aa164218105fe81b4a35ddf4622717f01d69455be9dc65b738e7ae32c33341c32'
+  metadata.gz: d8d9d745fdd025ce30a6c082730b878deb409bd7342d82f63dfbffbbecb5fa75f4ddadea2acace87904255199cb32b53277662d573534079d7d6575d2e80e03b
+  data.tar.gz: 813926476b8fed62c28a46decbff72018134f547f1618dd28418d6ea31f55bbc55bd43d0cd47bfc36fd8a476e0d65f4dd21d31d6bd79aeb3317d5282bbe9c46b

data/.gitignore

@@ -10,3 +10,4 @@
 main.rb
 .DS_Store
 .byebug_history
+benchmark.rb

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.2.1)
-      wgit (> 0)
+    broken_link_finder (0.3.0)
+      thread (= 0.2)
+      wgit (= 0.0.9)
 
 GEM
   remote: https://rubygems.org/
@@ -28,6 +29,7 @@ GEM
     public_suffix (3.1.0)
     rake (10.5.0)
     safe_yaml (1.0.5)
+    thread (0.2.0)
     webmock (3.5.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/README.md

@@ -35,14 +35,14 @@ require 'broken_link_finder'
 
 finder = BrokenLinkFinder::Finder.new
 finder.crawl_site "http://txti.es" # Also, see Finder#crawl_page for a single webpage.
-finder.pretty_print_broken_links # Also, see Finder#broken_links for a Hash.
+finder.pretty_print_broken_links # Also, see Finder#broken_links for a Hash of links.
 ```
 
 Then execute the script with:
 
     $ ruby main.rb
 
-The output should look something like:
+If broken links are found then the output should look something like:
 
 ```text
 Below is a breakdown of the different pages and their broken links...
@@ -60,7 +60,6 @@ http://imgur.com
 
 ## TODO
 
-- Speed boost.
 - Create a `broken_link_finder` executable.
 - Add logger functionality (especially useful in the console during development).
 
@@ -68,7 +67,7 @@ http://imgur.com
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release[origin]`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release[origin]`, which will create a git tag for the version, push git commits and tags, and push the `*.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/broken_link_finder.gemspec

@@ -42,5 +42,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "byebug", "~> 11.0"
   spec.add_development_dependency "webmock", "~> 3.5"
 
-  spec.add_runtime_dependency "wgit", "> 0"
+  spec.add_runtime_dependency "wgit", "0.0.9"
+  spec.add_runtime_dependency "thread", "0.2"
 end

data/lib/broken_link_finder/finder.rb

@@ -1,13 +1,18 @@
 require 'wgit'
+require 'thread/pool'
 
 module BrokenLinkFinder
   class Finder
+    DEFAULT_MAX_THREADS = 30.freeze
+
     attr_reader :broken_links
 
     # Create a new Finder instance.
-    def initialize
-      @broken_links = {}
+    def initialize(max_threads: DEFAULT_MAX_THREADS)
+      @max_threads = max_threads
+      @lock = Mutex.new
       @crawler = Wgit::Crawler.new
+      @broken_links = {}
     end
 
     # Clear/empty the @broken_links Hash.
@@ -16,30 +21,36 @@ module BrokenLinkFinder
     end
 
     # Finds broken links within an entire site and appends them to the
-    # @broken_links array.
+    # @broken_links array. Returns a tuple containing a Boolean of true if
+    # at least one broken link was found and an Array of all pages crawled.
+    # Access the broken links with Finder#broken_links.
     def crawl_site(url)
       clear_broken_links
       url = Wgit::Url.new(url)
-
+      pool = Thread.pool(@max_threads)
       crawled_pages = []
+
       @crawler.crawl_site(url) do |doc|
         # Ensure the given website url is valid.
         raise "Invalid URL: #{url}" if doc.url == url and doc.empty?
 
-        # Ensure we only process each page once.
+        # Ensure we only process each page once. For example, /about.html might
+        # be linked to several times throughout the entire site.
         next if crawled_pages.include?(doc.url)
         crawled_pages << doc.url
 
         # Get all page links and determine which are broken.
         next unless doc
-        find_broken_links(doc)
+        pool.process { find_broken_links(doc) }
       end
 
-      !@broken_links.empty?
+      pool.shutdown
+      [!@broken_links.empty?, crawled_pages]
     end
 
     # Finds broken links within a single page and appends them to the
-    # @broken_links array.
+    # @broken_links array. Returns true if at least one broken link was found.
+    # Access the broken links with Finder#broken_links.
     def crawl_url(url)
       clear_broken_links
       url = Wgit::Url.new(url)
@@ -55,12 +66,12 @@ module BrokenLinkFinder
     end
 
     # Pretty prints the contents of broken_links into a stream e.g. Kernel
-    # (STDOUT) or a file. 
+    # (STDOUT) or a file - anything that respond_to? :puts.
     # Returns true if there were broken links and vice versa. 
     def pretty_print_broken_links(stream = Kernel)
       raise "stream must respond_to? :puts" unless stream.respond_to? :puts
-      
-      if (@broken_links.empty?)
+
+      if @broken_links.empty?
         stream.puts("Good news, there are no broken links!")
         false
       else
@@ -81,7 +92,7 @@ broken links...")
 
     private
 
-    # Finds which links are broken and append the details to @broken_links.
+    # Finds which links are broken and appends the details to @broken_links.
     def find_broken_links(doc)
       links = doc.internal_full_links + doc.external_links
       links.each do |link|
@@ -94,10 +105,12 @@ broken links...")
 
     # Append url => [link] to @broken_links.
     def append_broken_link(url, link)
-      unless @broken_links[url]
-        @broken_links[url] = []
+      @lock.synchronize do
+        unless @broken_links[url]
+          @broken_links[url] = []
+        end
+        @broken_links[url] << link
       end
-      @broken_links[url] << link
     end
 
     alias_method :crawl_page, :crawl_url

data/lib/broken_link_finder/version.rb

@@ -1,3 +1,3 @@
 module BrokenLinkFinder
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Michael Telford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-04 00:00:00.000000000 Z
+date: 2019-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -98,16 +98,30 @@ dependencies:
   name: wgit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.9
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.2'
 description: Finds a website's broken links using the 'wgit' gem and reports back
   to you with a summary.
 email: michael.telford@live.com