broken_link_finder 0.12.1 → 0.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88b1e96f1de644a1a3c06ba7cc0ee1b53f75a3de6686b343e55028e8fa69da9f
-  data.tar.gz: e399ca05a4b0b9b2c0644b2846fa9dc6be6acd664e1bdc58758eb9ca7a5543cd
+  metadata.gz: 4e60e87e2f9f1ae05d4ebe58066169e389994fba88ea3393a9dc5c3d04df20b6
+  data.tar.gz: 8019a1671fb811d6bf67feefb044f77be18dc04b0904b1b4e43cb258ee0d5fa3
 SHA512:
-  metadata.gz: 57a1604358b0297b66604d1fc5a60a9d1bda05aa9bd5f6b91135ddc2aec4a6eb703c00ef4d905ac156170b190bf500481ce56cf6319f07e8b57447cca4c6a210
-  data.tar.gz: f4b88e66c9c4fcd2bcbca2fe882abdede7c531e1d5e752a2ac986e39cf51d87714852dcb6e7e8e4870b623d54b468cc8f3ec88c253e7182c1fe89c0af91366a4
+  metadata.gz: 6b16be9cbbd35ea468ff879a833b8a20cc6557b6bf1f2c0c79ae7243fc8a75d0f1b2d19ff9030dc112816871b232f85cbe49e344ee0ada405062b731961912a8
+  data.tar.gz: cf4f43c4d5369f218e1700b941fd5a4b3681aaafc8131e1394b734b72d343b6ee743690f15c115f711ccf5e133fe91777224fd943b04ae89e46e3ca06f09e424

data/.ruby-version

@@ -1 +1 @@
-3.0.2
+3.3.0

data/CHANGELOG.md

@@ -9,6 +9,26 @@
 - ...
 ---
 
+## v0.12.3
+### Added
+- Added `BrokenLinkFinder::Finder#manager` getter method.
+### Changed/Removed
+- Updated production dependencies including `wgit` (to `v0.12.0`).
+- Updated `Wgit::Url`s to look like Strings when inspected.
+### Fixed
+- ...
+---
+
+## v0.12.2
+### Added
+- Updated to Ruby 3.3 and updated production dependencies including Wgit (v0.11.0)
+- Added `--js` and `--js-delay` flag options to the executable. This allows JS parsing to update a page's DOM before it get crawled.
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
 ## v0.12.1
 ### Added
 - Support for Ruby 3.

data/Gemfile.lock

@@ -1,66 +1,70 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.12.1)
-      thor (~> 0.20)
+    broken_link_finder (0.12.3)
+      thor (~> 1.3)
       thread (~> 0.2)
-      wgit (~> 0.10)
+      wgit (~> 0.12)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    bson (4.12.1)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
+    bson (5.0.1)
     byebug (11.1.3)
-    cliver (0.3.2)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    crack (0.4.5)
+    concurrent-ruby (1.3.4)
+    crack (1.0.0)
+      bigdecimal
       rexml
-    ethon (0.15.0)
+    ethon (0.16.0)
       ffi (>= 1.15.0)
-    ferrum (0.11)
+    ferrum (0.15)
       addressable (~> 2.5)
-      cliver (~> 0.3)
       concurrent-ruby (~> 1.1)
-      websocket-driver (>= 0.6, < 0.8)
-    ffi (1.15.4)
-    hashdiff (1.0.1)
+      webrick (~> 1.7)
+      websocket-driver (~> 0.7)
+    ffi (1.17.0)
+    hashdiff (1.1.1)
     maxitest (3.7.0)
       minitest (>= 5.0.0, < 5.15.0)
-    method_source (1.0.0)
-    mini_portile2 (2.6.1)
+    method_source (1.1.0)
+    mini_portile2 (2.8.7)
     minitest (5.14.4)
-    mongo (2.17.0)
-      bson (>= 4.8.2, < 5.0.0)
-    nokogiri (1.12.5)
-      mini_portile2 (~> 2.6.1)
+    mongo (2.21.0)
+      bson (>= 4.14.1, < 6.0.0)
+    nokogiri (1.16.7)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    pry (0.14.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.6)
-    racc (1.6.0)
-    rake (13.0.6)
-    rexml (3.2.5)
-    thor (0.20.3)
+    public_suffix (6.0.1)
+    racc (1.8.1)
+    rake (13.2.1)
+    rexml (3.3.9)
+    thor (1.3.2)
     thread (0.2.2)
-    typhoeus (1.4.0)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
-    webmock (3.14.0)
+    webmock (3.24.0)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.5)
+    webrick (1.8.2)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    wgit (0.10.2)
-      addressable (~> 2.6)
-      ferrum (~> 0.8)
-      mongo (~> 2.9)
-      nokogiri (~> 1.10)
-      typhoeus (~> 1.3)
+    wgit (0.12.0)
+      addressable (~> 2.8)
+      base64 (~> 0.2)
+      ferrum (~> 0.14)
+      mongo (~> 2.19)
+      nokogiri (~> 1.15)
+      typhoeus (~> 1.4)
 
 PLATFORMS
   ruby
@@ -75,7 +79,7 @@ DEPENDENCIES
   webmock (~> 3.6)
 
 RUBY VERSION
-   ruby 3.0.2p107
+   ruby 3.3.0p0
 
 BUNDLED WITH
-   2.2.22
+   2.5.3

data/broken_link_finder.gemspec

@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'webmock', '~> 3.6'
 
-  spec.add_runtime_dependency 'thor', '~> 0.20'
+  spec.add_runtime_dependency 'thor', '~> 1.3'
   spec.add_runtime_dependency 'thread', '~> 0.2'
-  spec.add_runtime_dependency 'wgit', '~> 0.10'
+  spec.add_runtime_dependency 'wgit', '~> 0.12'
 end

data/exe/broken_link_finder

@@ -9,7 +9,9 @@ class BrokenLinkFinderCLI < Thor
   desc 'crawl [URL]', 'Find broken links at the URL'
   option :recursive, type: :boolean, aliases: [:r], default: false, desc: 'Crawl the entire site.'
   option :threads, type: :numeric, aliases: [:t], default: BrokenLinkFinder::DEFAULT_MAX_THREADS, desc: 'Max number of threads to use when crawling recursively; 1 thread per web page.'
-  option :xpath, type: :string, aliases: [:x], default: BrokenLinkFinder::DEFAULT_LINK_XPATH
+  option :xpath, type: :string, aliases: [:x], default: BrokenLinkFinder::DEFAULT_LINK_XPATH, desc: 'The xpath to extract links with, before checking if broken'
+  option :js, type: :boolean, default: false, desc: 'Run the Javascript on a page before crawling the HTML, requires Chrome/Chromium to be installed to $PATH'
+  option :js_delay, type: :numeric, default: 1, desc: "The seconds of delay time given to a page's Javascript for it to update the DOM, requires the --js flag"
   option :html, type: :boolean, aliases: [:h], default: false, desc: 'Produce a HTML report (instead of text)'
   option :sort_by_link, type: :boolean, aliases: [:l], default: false, desc: 'Makes report more concise if there are more pages crawled than broken links found. Use with -r on medium/large sites.'
   option :verbose, type: :boolean, aliases: [:v], default: false, desc: 'Display all ignored links.'
@@ -22,15 +24,17 @@ class BrokenLinkFinderCLI < Thor
     max_threads     = options[:threads]
     broken_verbose  = !options[:concise]
     ignored_verbose = options[:verbose]
+    parse_js        = options[:js]
+    parse_js_delay  = options[:js_delay]
 
     BrokenLinkFinder.link_xpath = options[:xpath]
-    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads: max_threads)
+    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads:) do |crawler|
+      crawler.parse_javascript = parse_js
+      crawler.parse_javascript_delay = parse_js_delay
+    end
+
     options[:recursive] ? finder.crawl_site(url) : finder.crawl_page(url)
-    finder.report(
-      type:            report_type,
-      broken_verbose:  broken_verbose,
-      ignored_verbose: ignored_verbose
-    )
+    finder.report(type: report_type, broken_verbose:, ignored_verbose:)
 
     exit 0
   rescue StandardError => e

data/lib/broken_link_finder/finder.rb

@@ -5,8 +5,8 @@ module BrokenLinkFinder
   SERVER_WAIT_TIME    = 0.5 # Used by Finder#retry_broken_links.
 
   # Alias for BrokenLinkFinder::Finder.new.
-  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS)
-    Finder.new(sort: sort, max_threads: max_threads)
+  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS, &block)
+    Finder.new(sort: sort, max_threads: max_threads, &block)
   end
 
   # Class responsible for finding broken links on a page or site.
@@ -17,8 +17,14 @@ module BrokenLinkFinder
     # The max number of threads created during #crawl_site - one thread per page.
     attr_reader :max_threads
 
+    # The underlying Wgit::Crawler used by this instance of Finder.
+    attr_reader :crawler
+
+    # The underlying link manager used by this instance of Finder.
+    attr_reader :manager
+
     # Returns a new Finder instance.
-    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS)
+    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS, &block)
       raise "Sort by either :page or :link, not #{sort}" \
       unless %i[page link].include?(sort)
 
@@ -26,6 +32,8 @@ module BrokenLinkFinder
       @max_threads = max_threads
       @crawler     = Wgit::Crawler.new
       @manager     = BrokenLinkFinder::LinkManager.new(@sort)
+
+      yield @crawler if block_given?
     end
 
     # Returns the current broken links.
@@ -56,7 +64,7 @@ module BrokenLinkFinder
       doc = @crawler.crawl(url.dup)
 
       # Ensure the given page url is valid.
-      raise "Invalid or broken URL: #{url}" unless doc
+      raise "Invalid or broken URL: #{url}" if doc.empty?
 
       # Get all page links and determine which are broken.
       find_broken_links(doc)
@@ -207,7 +215,7 @@ module BrokenLinkFinder
 
     # Return if the crawled link is broken or not.
     def link_broken?(doc)
-      doc.nil? || @crawler.last_response.not_found? || has_broken_anchor(doc)
+      doc.empty? || @crawler.last_response.not_found? || has_broken_anchor(doc)
     end
 
     # Returns true if the link is/contains a broken anchor/fragment.

data/lib/broken_link_finder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrokenLinkFinder
-  VERSION = '0.12.1'
+  VERSION = '0.12.3'
 end

data/lib/broken_link_finder/wgit_extensions.rb

@@ -1,5 +1,12 @@
 # frozen_string_literal: true
 
+# Make Wgit::Urls look like Strings when inspected.
+class Wgit::Url
+  def inspect
+    to_s.inspect
+  end
+end
+
 # Define a method on each doc for recording unparsable links.
 # Unparsable links are recorded as broken links by Finder.
 class Wgit::Document

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.3
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-22 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
 description: Finds a website's broken links using the 'wgit' gem and reports back
   to you with a summary.
 email: michael.telford@live.com
@@ -194,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Finds a website's broken links and reports back to you with a summary.