broken_link_finder 0.12.1 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88b1e96f1de644a1a3c06ba7cc0ee1b53f75a3de6686b343e55028e8fa69da9f
-  data.tar.gz: e399ca05a4b0b9b2c0644b2846fa9dc6be6acd664e1bdc58758eb9ca7a5543cd
+  metadata.gz: 2279e54526bbf0b0a68072c060e480bc5efe3e8cd9deefd028789716c736cf77
+  data.tar.gz: 667e1e86207dd74d594e5e57411076cc21713f15f549adb5a0c904ed5633902a
 SHA512:
-  metadata.gz: 57a1604358b0297b66604d1fc5a60a9d1bda05aa9bd5f6b91135ddc2aec4a6eb703c00ef4d905ac156170b190bf500481ce56cf6319f07e8b57447cca4c6a210
-  data.tar.gz: f4b88e66c9c4fcd2bcbca2fe882abdede7c531e1d5e752a2ac986e39cf51d87714852dcb6e7e8e4870b623d54b468cc8f3ec88c253e7182c1fe89c0af91366a4
+  metadata.gz: 3dcc51f65cfb9b869f77f5becb0861b617b2b5424ad0890ff238f18be1f8cc5d70173dc9dd3a98d18754986dc8a2c578e66c7db533fbddcf8673a1ca8c1c90dd
+  data.tar.gz: 7a4f7ee5cd856a2713bfa89f250aaf5bf0458818604cdc89ecd683af23337b2bb7b695b51b3f7ce134f3d696e9e447b2c1eaff97ca41b6f683dc85e9ee0dfb83

data/.ruby-version

@@ -1 +1 @@
-3.0.2
+3.3.0

data/CHANGELOG.md

@@ -9,6 +9,16 @@
 - ...
 ---
 
+## v0.12.2
+### Added
+- Updated to Ruby 3.3 and updated production dependencies including Wgit (v0.11)
+- Added `--js` and `--js-delay` flag options to the executable. This allows JS parsing to update a page's DOM before it get crawled.
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
 ## v0.12.1
 ### Added
 - Support for Ruby 3.

data/Gemfile.lock

@@ -1,66 +1,66 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.12.1)
-      thor (~> 0.20)
+    broken_link_finder (0.12.2)
+      thor (~> 1.3)
       thread (~> 0.2)
-      wgit (~> 0.10)
+      wgit (~> 0.11)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    bson (4.12.1)
+    addressable (2.8.6)
+      public_suffix (>= 2.0.2, < 6.0)
+    bson (4.15.0)
     byebug (11.1.3)
-    cliver (0.3.2)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     crack (0.4.5)
       rexml
-    ethon (0.15.0)
+    ethon (0.16.0)
       ffi (>= 1.15.0)
-    ferrum (0.11)
+    ferrum (0.14)
       addressable (~> 2.5)
-      cliver (~> 0.3)
       concurrent-ruby (~> 1.1)
+      webrick (~> 1.7)
       websocket-driver (>= 0.6, < 0.8)
-    ffi (1.15.4)
-    hashdiff (1.0.1)
+    ffi (1.16.3)
+    hashdiff (1.1.0)
     maxitest (3.7.0)
       minitest (>= 5.0.0, < 5.15.0)
     method_source (1.0.0)
-    mini_portile2 (2.6.1)
+    mini_portile2 (2.8.5)
     minitest (5.14.4)
-    mongo (2.17.0)
-      bson (>= 4.8.2, < 5.0.0)
-    nokogiri (1.12.5)
-      mini_portile2 (~> 2.6.1)
+    mongo (2.19.3)
+      bson (>= 4.14.1, < 5.0.0)
+    nokogiri (1.16.0)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    pry (0.14.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.6)
-    racc (1.6.0)
-    rake (13.0.6)
-    rexml (3.2.5)
-    thor (0.20.3)
+    public_suffix (5.0.4)
+    racc (1.7.3)
+    rake (13.1.0)
+    rexml (3.2.6)
+    thor (1.3.0)
     thread (0.2.2)
-    typhoeus (1.4.0)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
-    webmock (3.14.0)
+    webmock (3.19.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.5)
+    webrick (1.8.1)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    wgit (0.10.2)
-      addressable (~> 2.6)
-      ferrum (~> 0.8)
-      mongo (~> 2.9)
-      nokogiri (~> 1.10)
-      typhoeus (~> 1.3)
+    wgit (0.11.0)
+      addressable (~> 2.8)
+      ferrum (~> 0.14)
+      mongo (~> 2.19)
+      nokogiri (~> 1.15)
+      typhoeus (~> 1.4)
 
 PLATFORMS
   ruby
@@ -75,7 +75,7 @@ DEPENDENCIES
   webmock (~> 3.6)
 
 RUBY VERSION
-   ruby 3.0.2p107
+   ruby 3.3.0p0
 
 BUNDLED WITH
-   2.2.22
+   2.5.3

data/broken_link_finder.gemspec

@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'webmock', '~> 3.6'
 
-  spec.add_runtime_dependency 'thor', '~> 0.20'
+  spec.add_runtime_dependency 'thor', '~> 1.3'
   spec.add_runtime_dependency 'thread', '~> 0.2'
-  spec.add_runtime_dependency 'wgit', '~> 0.10'
+  spec.add_runtime_dependency 'wgit', '~> 0.11'
 end

data/exe/broken_link_finder

@@ -9,7 +9,9 @@ class BrokenLinkFinderCLI < Thor
   desc 'crawl [URL]', 'Find broken links at the URL'
   option :recursive, type: :boolean, aliases: [:r], default: false, desc: 'Crawl the entire site.'
   option :threads, type: :numeric, aliases: [:t], default: BrokenLinkFinder::DEFAULT_MAX_THREADS, desc: 'Max number of threads to use when crawling recursively; 1 thread per web page.'
-  option :xpath, type: :string, aliases: [:x], default: BrokenLinkFinder::DEFAULT_LINK_XPATH
+  option :xpath, type: :string, aliases: [:x], default: BrokenLinkFinder::DEFAULT_LINK_XPATH, desc: 'The xpath to extract links with, before checking if broken'
+  option :js, type: :boolean, default: false, desc: 'Run the Javascript on a page before crawling the HTML, requires Chrome/Chromium to be installed to $PATH'
+  option :js_delay, type: :numeric, default: 1, desc: "The seconds of delay time given to a page's Javascript for it to update the DOM, requires the --js flag"
   option :html, type: :boolean, aliases: [:h], default: false, desc: 'Produce a HTML report (instead of text)'
   option :sort_by_link, type: :boolean, aliases: [:l], default: false, desc: 'Makes report more concise if there are more pages crawled than broken links found. Use with -r on medium/large sites.'
   option :verbose, type: :boolean, aliases: [:v], default: false, desc: 'Display all ignored links.'
@@ -22,15 +24,17 @@ class BrokenLinkFinderCLI < Thor
     max_threads     = options[:threads]
     broken_verbose  = !options[:concise]
     ignored_verbose = options[:verbose]
+    parse_js        = options[:js]
+    parse_js_delay  = options[:js_delay]
 
     BrokenLinkFinder.link_xpath = options[:xpath]
-    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads: max_threads)
+    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads:) do |crawler|
+      crawler.parse_javascript = parse_js
+      crawler.parse_javascript_delay = parse_js_delay
+    end
+
     options[:recursive] ? finder.crawl_site(url) : finder.crawl_page(url)
-    finder.report(
-      type:            report_type,
-      broken_verbose:  broken_verbose,
-      ignored_verbose: ignored_verbose
-    )
+    finder.report(type: report_type, broken_verbose:, ignored_verbose:)
 
     exit 0
   rescue StandardError => e

data/lib/broken_link_finder/finder.rb

@@ -5,8 +5,8 @@ module BrokenLinkFinder
   SERVER_WAIT_TIME    = 0.5 # Used by Finder#retry_broken_links.
 
   # Alias for BrokenLinkFinder::Finder.new.
-  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS)
-    Finder.new(sort: sort, max_threads: max_threads)
+  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS, &block)
+    Finder.new(sort: sort, max_threads: max_threads, &block)
   end
 
   # Class responsible for finding broken links on a page or site.
@@ -17,8 +17,11 @@ module BrokenLinkFinder
     # The max number of threads created during #crawl_site - one thread per page.
     attr_reader :max_threads
 
+    # The underlying Wgit::Crawler used by this instance of Finder.
+    attr_reader :crawler
+
     # Returns a new Finder instance.
-    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS)
+    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS, &block)
       raise "Sort by either :page or :link, not #{sort}" \
       unless %i[page link].include?(sort)
 
@@ -26,6 +29,8 @@ module BrokenLinkFinder
       @max_threads = max_threads
       @crawler     = Wgit::Crawler.new
       @manager     = BrokenLinkFinder::LinkManager.new(@sort)
+
+      yield @crawler if block_given?
     end
 
     # Returns the current broken links.

data/lib/broken_link_finder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrokenLinkFinder
-  VERSION = '0.12.1'
+  VERSION = '0.12.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.2
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-22 00:00:00.000000000 Z
+date: 2024-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
 description: Finds a website's broken links using the 'wgit' gem and reports back
   to you with a summary.
 email: michael.telford@live.com
@@ -194,7 +194,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Finds a website's broken links and reports back to you with a summary.