broken_link_finder 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 715744f5d7676d5d4ba2cddb80f0f8109f79a7f92689c3ff3088a52f307f5f1f
-  data.tar.gz: 7026f6037f0d710d8dab3bc710ddf7b202594c25ac8a8522398e62af3f4e78dd
+  metadata.gz: 110068a5db9454d69709454f50ade3f667c0e63dad6d101c23bf991a04f770eb
+  data.tar.gz: 0fd46fed486bf382935d68020fcae9391c288b5d5742ce19ba61d2896e2eebe7
 SHA512:
-  metadata.gz: 7844d0c6d2c39752a98dcb9a7eb455f6492012321be71ff40f49641b7adc3658f4f541a35afc9ca1a9d6ce330472c4f166db0895acc1e1d6ecad53f86af7f0ac
-  data.tar.gz: f5224db527d4636e9006ea332813d9cd133ea221506aa7a45edecc6fd230f212e21f4db8d1757a070c753876d6b003329a381b0f9a48663c966318b9fb2d1c86
+  metadata.gz: c9c744045ee462b8981ab418a5c3acdb9162c36e14c8603391c1f79e9247872de877a83ce2a7dd5f8779d8774a044562054445410666a5af518d11f5fded3f22
+  data.tar.gz: e4e0e2b2d19596493564a361fef0a98afdddcefb395309dba139df6bd508a12385ca16e47004bd503c585954454d9608449995b4ed6009926e0b856d11962181

data/CHANGELOG.md

@@ -9,6 +9,15 @@
 - ...
 ---
 
+## v0.9.2
+### Added
+- ...
+### Changed/Removed
+- Updated `wgit` gem to version 0.4.0 which brings a speed boost to crawls.
+### Fixed
+- ...
+---
+
 ## v0.9.1
 ### Added
 - `BrokenLinkFinder::Finder.crawl_site` alias: `crawl_r`.

data/Gemfile

@@ -2,5 +2,7 @@
 
 source 'https://rubygems.org'
 
+ruby '~> 2.5'
+
 # Specify your gem's dependencies in broken_link_finder.gemspec
 gemspec

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.9.1)
-      thor (= 0.20.3)
-      thread (= 0.2)
-      wgit (= 0.2.0)
+    broken_link_finder (0.9.2)
+      thor (~> 0.20.3)
+      thread (~> 0.2.0)
+      wgit (~> 0.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -16,15 +16,16 @@ GEM
     coderay (1.1.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    hashdiff (0.4.0)
-    httplog (1.3.2)
-      rack (>= 1.0)
-      rainbow (>= 2.0.0)
-    memory_profiler (0.9.14)
+    ethon (0.12.0)
+      ffi (>= 1.3.0)
+    ffi (1.11.1)
+    hashdiff (1.0.0)
+    maxitest (3.4.0)
+      minitest (>= 5.0.0, < 5.13.0)
     method_source (0.9.2)
     mini_portile2 (2.4.0)
-    minitest (5.11.3)
-    mongo (2.9.1)
+    minitest (5.12.2)
+    mongo (2.9.2)
       bson (>= 4.4.2, < 5.0.0)
     nokogiri (1.10.4)
       mini_portile2 (~> 2.4.0)
@@ -32,20 +33,21 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     public_suffix (3.1.0)
-    rack (2.0.7)
-    rainbow (3.0.0)
     rake (10.5.0)
     safe_yaml (1.0.5)
     thor (0.20.3)
-    thread (0.2.0)
-    webmock (3.5.1)
+    thread (0.2.2)
+    typhoeus (1.3.1)
+      ethon (>= 0.9.0)
+    webmock (3.7.6)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
-    wgit (0.2.0)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    wgit (0.4.0)
       addressable (~> 2.6.0)
       mongo (~> 2.9.0)
       nokogiri (~> 1.10.3)
+      typhoeus (~> 1.3.1)
 
 PLATFORMS
   ruby
@@ -54,12 +56,13 @@ DEPENDENCIES
   broken_link_finder!
   bundler (~> 2.0)
   byebug (~> 11.0)
-  httplog (~> 1.3)
-  memory_profiler (~> 0.9)
-  minitest (~> 5.0)
+  maxitest (~> 3.3)
   pry (~> 0.12)
   rake (~> 10.0)
-  webmock (~> 3.5)
+  webmock (~> 3.6)
+
+RUBY VERSION
+   ruby 2.5.3p105
 
 BUNDLED WITH
    2.0.1

data/README.md

@@ -19,9 +19,11 @@ In a nutshell, only HTTP(S) based links can be successfully verified by `broken_
 
 See the [usage](#Usage) section below on how to check which links have been ignored during a crawl.
 
+With that said, the usual array of HTTP URL features are supported including anchors/fragments, query strings and IRI's (non ASCII based URL's).
+
 ## Made Possible By
 
-`broken_link_finder` relies heavily on the `wgit` Ruby gem. See its [repository](https://github.com/michaeltelford/wgit) for more details.
+`broken_link_finder` relies heavily on the `wgit` Ruby gem by the same author. See its [repository](https://github.com/michaeltelford/wgit) for more details.
 
 ## Installation
 

data/benchmark.rb

@@ -2,7 +2,6 @@
 
 require_relative './lib/broken_link_finder'
 require 'benchmark'
-require 'memory_profiler'
 
 url = ARGV[0] || 'http://txti.es'
 finder = BrokenLinkFinder::Finder.new

data/bin/console

@@ -4,30 +4,20 @@
 require 'bundler/setup'
 require 'pry'
 require 'byebug'
-require 'logger'
-require 'httplog'
 require 'broken_link_finder'
 
-logger = Logger.new(STDOUT)
-logger.formatter = proc do |_severity, _datetime, _progname, msg|
-  "#{msg}\n"
-end
-
-# Monkey patch all Net:HTTP network calls and log them.
-HttpLog.configure do |config|
-  config.enabled        = true
-  config.logger         = logger
-
-  config.log_connect    = false
-  config.log_request    = true
-  config.log_headers    = false
-  config.log_data       = false
-  config.log_status     = true
-  config.log_response   = false
-  config.log_benchmark  = false
-
-  config.compact_log    = false
-  config.json_log       = false
+# Monkey patch and log all HTTP requests made during the console.
+module Typhoeus
+  singleton_class.class_eval do
+    alias_method :orig_get, :get
+  end
+
+  def self.get(base_url, options = {})
+    puts "[typhoeus] Sending GET: #{base_url}"
+    resp = orig_get(base_url, options)
+    puts "[typhoeus] Status: #{resp.code} (#{resp.body.length} bytes in #{resp.total_time} seconds)"
+    resp
+  end
 end
 
 # Call reload to load all recent code changes.
@@ -48,6 +38,7 @@ by_page = Finder.new
 by_link = Finder.new sort: :link
 finder = by_page
 
+# Start the console.
 puts "\nbroken_link_finder v#{BrokenLinkFinder::VERSION}"
 
 binding.pry

data/broken_link_finder.gemspec

@@ -39,14 +39,12 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'byebug', '~> 11.0'
-  spec.add_development_dependency 'httplog', '~> 1.3'
-  spec.add_development_dependency 'memory_profiler', '~> 0.9'
-  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'maxitest', '~> 3.3'
   spec.add_development_dependency 'pry', '~> 0.12'
   spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'webmock', '~> 3.5'
+  spec.add_development_dependency 'webmock', '~> 3.6'
 
-  spec.add_runtime_dependency 'thor', '0.20.3'
-  spec.add_runtime_dependency 'thread', '0.2'
-  spec.add_runtime_dependency 'wgit', '0.2.0'
+  spec.add_runtime_dependency 'thor', '~> 0.20.3'
+  spec.add_runtime_dependency 'thread', '~> 0.2.0'
+  spec.add_runtime_dependency 'wgit', '~> 0.4.0'
 end

data/lib/broken_link_finder/finder.rb

@@ -39,7 +39,7 @@ module BrokenLinkFinder
 
     # Finds broken links within a single page and appends them to the
     # @broken_links array. Returns true if at least one broken link was found.
-    # Access the broken links with Finder#broken_links.
+    # Access the broken links afterwards with Finder#broken_links.
     def crawl_url(url)
       clear_links
 
@@ -61,7 +61,7 @@ module BrokenLinkFinder
     # Finds broken links within an entire site and appends them to the
     # @broken_links array. Returns a tuple containing a Boolean of true if
     # at least one broken link was found and an Array of all pages crawled.
-    # Access the broken links with Finder#broken_links.
+    # Access the broken links afterwards with Finder#broken_links.
     def crawl_site(url)
       clear_links
 
@@ -70,7 +70,7 @@ module BrokenLinkFinder
       crawled_pages = []
 
       # Crawl the site's HTML web pages looking for links.
-      orig_doc = @crawler.crawl_site(url) do |doc|
+      externals = @crawler.crawl_site(url) do |doc|
         crawled_pages << doc.url
         next unless doc
 
@@ -79,7 +79,7 @@ module BrokenLinkFinder
       end
 
       # Ensure the given website url is valid.
-      raise "Invalid or broken URL: #{url}" if orig_doc.nil?
+      raise "Invalid or broken URL: #{url}" unless externals
 
       # Wait for all threads to finish.
       pool.shutdown
@@ -113,15 +113,7 @@ module BrokenLinkFinder
 
     # Finds which links are unsupported or broken and records the details.
     def find_broken_links(doc)
-      # Report and reject any non supported links.
-      links = doc.all_links
-                 .reject do |link|
-                    if link.is_absolute? && !link.start_with?('http')
-                      append_ignored_link(doc.url, link)
-                      true
-                    end
-                  end
-                 .uniq
+      links = get_supported_links(doc)
 
       # Iterate over the supported links checking if they're broken or not.
       links.each do |link|
@@ -134,11 +126,10 @@ module BrokenLinkFinder
         end
 
         # The link hasn't been processed before so we crawl it.
-        link_url = get_absolute_link(doc, link)
-        link_doc = @crawler.crawl_url(link_url)
+        link_doc = crawl_link(doc, link)
 
         # Determine if the crawled link is broken or not.
-        if  @crawler.last_response.is_a?(Net::HTTPNotFound) ||
+        if  @crawler.last_response.code == 404 ||
             link_doc.nil? ||
             has_broken_anchor(link_doc)
           append_broken_link(doc.url, link)
@@ -150,6 +141,24 @@ module BrokenLinkFinder
       nil
     end
 
+    # Report and reject any non supported links. Any link that is absolute and
+    # doesn't start with 'http' is unsupported e.g. 'mailto:blah' etc.
+    def get_supported_links(doc)
+      doc.all_links
+         .reject do |link|
+           if link.is_absolute? && !link.start_with?('http')
+             append_ignored_link(doc.url, link)
+             true
+           end
+         end
+    end
+
+    # Makes the link absolute and crawls it, returning its Wgit::Document.
+    def crawl_link(doc, link)
+      link = get_absolute_link(doc, link)
+      @crawler.crawl_url(link)
+    end
+
     # Returns the link in absolute form so it can be crawled.
     def get_absolute_link(doc, link)
       link.is_relative? ? doc.base_url(link: link).concat(link) : link

data/lib/broken_link_finder/reporter.rb

@@ -12,10 +12,10 @@ module BrokenLinkFinder
       raise "sort by either :page or :link, not #{sort}" \
       unless %i[page link].include?(sort)
 
-      @stream         = stream
-      @sort           = sort
-      @broken_links   = broken_links
-      @ignored_links  = ignored_links
+      @stream        = stream
+      @sort          = sort
+      @broken_links  = broken_links
+      @ignored_links = ignored_links
     end
 
     # Pretty print a report detailing the link summary.

data/lib/broken_link_finder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrokenLinkFinder
-  VERSION = '0.9.1'
+  VERSION = '0.9.2'
 end

data/lib/broken_link_finder/wgit_extensions.rb

@@ -7,5 +7,5 @@ Wgit::Document.define_extension(
   singleton: false,
   text_content_only: true
 ) do |links|
-  links&.map(&:to_url)&.uniq
+  links.uniq.to_urls
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Michael Telford
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-22 00:00:00.000000000 Z
+date: 2019-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -39,47 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '11.0'
 - !ruby/object:Gem::Dependency
-  name: httplog
+  name: maxitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: memory_profiler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -114,56 +86,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.20.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.20.3
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: wgit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.4.0
 description: Finds a website's broken links using the 'wgit' gem and reports back
   to you with a summary.
 email: michael.telford@live.com