broken_link_finder 0.12.0 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24ca9c7a6071b07f5ab3132c9c79c4628570c9c3e157b77a27a05cdc0578ac6e
-  data.tar.gz: 6668eb430c8296e1439f56c242e7e08a27733605d724ec1c5cfa638dcfaa8b52
+  metadata.gz: 2279e54526bbf0b0a68072c060e480bc5efe3e8cd9deefd028789716c736cf77
+  data.tar.gz: 667e1e86207dd74d594e5e57411076cc21713f15f549adb5a0c904ed5633902a
 SHA512:
-  metadata.gz: 1d1cdc47ade4651b8bc2df01212364ba938ee73269bf53e7278519ecd374247291c932abfa73a031973403ed55d360bc9d14b5c60ba312aca4b32837b5064294
-  data.tar.gz: f56308da4b9d7a4a39afd43808f77d2b6f2fbbf00f17502d2d889de504bcc82ee1858fb673333b11693a65ad73a4f5fb65a97b15955443e8268b1e0ab08b4e51
+  metadata.gz: 3dcc51f65cfb9b869f77f5becb0861b617b2b5424ad0890ff238f18be1f8cc5d70173dc9dd3a98d18754986dc8a2c578e66c7db533fbddcf8673a1ca8c1c90dd
+  data.tar.gz: 7a4f7ee5cd856a2713bfa89f250aaf5bf0458818604cdc89ecd683af23337b2bb7b695b51b3f7ce134f3d696e9e447b2c1eaff97ca41b6f683dc85e9ee0dfb83

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+3.3.0

data/CHANGELOG.md

@@ -9,6 +9,25 @@
 - ...
 ---
 
+## v0.12.2
+### Added
+- Updated to Ruby 3.3 and updated production dependencies including Wgit (v0.11)
+- Added `--js` and `--js-delay` flag options to the executable. This allows JS parsing to update a page's DOM before it get crawled.
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
+## v0.12.1
+### Added
+- Support for Ruby 3.
+### Changed/Removed
+- Removed support for Ruby 2.5 (as it's too old).
+### Fixed
+- ...
+---
+
 ## v0.12.0
 ### Added
 - `BrokenLinkFinder::link_xpath` and `link_xpath=` methods so you can customise how links are extracted from each crawled page using the API.

data/Gemfile

@@ -2,7 +2,7 @@
 
 source 'https://rubygems.org'
 
-ruby '~> 2.5'
+ruby '>= 2.6', '< 4'
 
 # Specify your gem's dependencies in broken_link_finder.gemspec
 gemspec

data/Gemfile.lock

@@ -1,66 +1,66 @@
 PATH
   remote: .
   specs:
-    broken_link_finder (0.12.0)
-      thor (~> 0.20)
+    broken_link_finder (0.12.2)
+      thor (~> 1.3)
       thread (~> 0.2)
-      wgit (~> 0.10)
+      wgit (~> 0.11)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    bson (4.12.0)
+    addressable (2.8.6)
+      public_suffix (>= 2.0.2, < 6.0)
+    bson (4.15.0)
     byebug (11.1.3)
-    cliver (0.3.2)
     coderay (1.1.3)
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.2.2)
     crack (0.4.5)
       rexml
-    ethon (0.12.0)
-      ffi (>= 1.3.0)
-    ferrum (0.11)
+    ethon (0.16.0)
+      ffi (>= 1.15.0)
+    ferrum (0.14)
       addressable (~> 2.5)
-      cliver (~> 0.3)
       concurrent-ruby (~> 1.1)
+      webrick (~> 1.7)
       websocket-driver (>= 0.6, < 0.8)
-    ffi (1.15.0)
-    hashdiff (1.0.1)
-    maxitest (3.6.0)
-      minitest (>= 5.0.0, < 5.14.0)
+    ffi (1.16.3)
+    hashdiff (1.1.0)
+    maxitest (3.7.0)
+      minitest (>= 5.0.0, < 5.15.0)
     method_source (1.0.0)
-    mini_portile2 (2.5.0)
-    minitest (5.13.0)
-    mongo (2.14.0)
-      bson (>= 4.8.2, < 5.0.0)
-    nokogiri (1.11.2)
-      mini_portile2 (~> 2.5.0)
+    mini_portile2 (2.8.5)
+    minitest (5.14.4)
+    mongo (2.19.3)
+      bson (>= 4.14.1, < 5.0.0)
+    nokogiri (1.16.0)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    pry (0.14.0)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.6)
-    racc (1.5.2)
-    rake (13.0.3)
-    rexml (3.2.4)
-    thor (0.20.3)
+    public_suffix (5.0.4)
+    racc (1.7.3)
+    rake (13.1.0)
+    rexml (3.2.6)
+    thor (1.3.0)
     thread (0.2.2)
-    typhoeus (1.4.0)
+    typhoeus (1.4.1)
       ethon (>= 0.9.0)
-    webmock (3.12.2)
-      addressable (>= 2.3.6)
+    webmock (3.19.1)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.3)
+    webrick (1.8.1)
+    websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    wgit (0.10.0)
-      addressable (~> 2.6)
-      ferrum (~> 0.8)
-      mongo (~> 2.9)
-      nokogiri (~> 1.10)
-      typhoeus (~> 1.3)
+    wgit (0.11.0)
+      addressable (~> 2.8)
+      ferrum (~> 0.14)
+      mongo (~> 2.19)
+      nokogiri (~> 1.15)
+      typhoeus (~> 1.4)
 
 PLATFORMS
   ruby
@@ -75,7 +75,7 @@ DEPENDENCIES
   webmock (~> 3.6)
 
 RUBY VERSION
-   ruby 2.7.0p0
+   ruby 3.3.0p0
 
 BUNDLED WITH
-   2.1.4
+   2.5.3

data/README.md

@@ -35,7 +35,7 @@ Only MRI Ruby is tested and supported, but `broken_link_finder` may work with ot
 
 Currently, the required MRI Ruby version is:
 
-`~> 2.5` (a.k.a.) `>= 2.5 && < 3`
+`ruby '>= 2.6', '< 4'`
 
 ### Using Bundler
 

data/broken_link_finder.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
   spec.post_install_message = "Added the executable 'broken_link_finder' to $PATH"
 
-  spec.required_ruby_version = '~> 2.5'
+  spec.required_ruby_version = '>= 2.6', '< 4'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'byebug', '~> 11.0'
@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'webmock', '~> 3.6'
 
-  spec.add_runtime_dependency 'thor', '~> 0.20'
+  spec.add_runtime_dependency 'thor', '~> 1.3'
   spec.add_runtime_dependency 'thread', '~> 0.2'
-  spec.add_runtime_dependency 'wgit', '~> 0.10'
+  spec.add_runtime_dependency 'wgit', '~> 0.11'
 end

data/exe/broken_link_finder

@@ -9,7 +9,9 @@ class BrokenLinkFinderCLI < Thor
   desc 'crawl [URL]', 'Find broken links at the URL'
   option :recursive, type: :boolean, aliases: [:r], default: false, desc: 'Crawl the entire site.'
   option :threads, type: :numeric, aliases: [:t], default: BrokenLinkFinder::DEFAULT_MAX_THREADS, desc: 'Max number of threads to use when crawling recursively; 1 thread per web page.'
-  option :xpath, type: :string, aliases: [:x], default: BrokenLinkFinder::DEFAULT_LINK_XPATH
+  option :xpath, type: :string, aliases: [:x], default: BrokenLinkFinder::DEFAULT_LINK_XPATH, desc: 'The xpath to extract links with, before checking if broken'
+  option :js, type: :boolean, default: false, desc: 'Run the Javascript on a page before crawling the HTML, requires Chrome/Chromium to be installed to $PATH'
+  option :js_delay, type: :numeric, default: 1, desc: "The seconds of delay time given to a page's Javascript for it to update the DOM, requires the --js flag"
   option :html, type: :boolean, aliases: [:h], default: false, desc: 'Produce a HTML report (instead of text)'
   option :sort_by_link, type: :boolean, aliases: [:l], default: false, desc: 'Makes report more concise if there are more pages crawled than broken links found. Use with -r on medium/large sites.'
   option :verbose, type: :boolean, aliases: [:v], default: false, desc: 'Display all ignored links.'
@@ -22,15 +24,17 @@ class BrokenLinkFinderCLI < Thor
     max_threads     = options[:threads]
     broken_verbose  = !options[:concise]
     ignored_verbose = options[:verbose]
+    parse_js        = options[:js]
+    parse_js_delay  = options[:js_delay]
 
     BrokenLinkFinder.link_xpath = options[:xpath]
-    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads: max_threads)
+    finder = BrokenLinkFinder::Finder.new(sort: sort_by, max_threads:) do |crawler|
+      crawler.parse_javascript = parse_js
+      crawler.parse_javascript_delay = parse_js_delay
+    end
+
     options[:recursive] ? finder.crawl_site(url) : finder.crawl_page(url)
-    finder.report(
-      type:            report_type,
-      broken_verbose:  broken_verbose,
-      ignored_verbose: ignored_verbose
-    )
+    finder.report(type: report_type, broken_verbose:, ignored_verbose:)
 
     exit 0
   rescue StandardError => e

data/lib/broken_link_finder/finder.rb

@@ -5,8 +5,8 @@ module BrokenLinkFinder
   SERVER_WAIT_TIME    = 0.5 # Used by Finder#retry_broken_links.
 
   # Alias for BrokenLinkFinder::Finder.new.
-  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS)
-    Finder.new(sort: sort, max_threads: max_threads)
+  def self.new(sort: :page, max_threads: DEFAULT_MAX_THREADS, &block)
+    Finder.new(sort: sort, max_threads: max_threads, &block)
   end
 
   # Class responsible for finding broken links on a page or site.
@@ -17,8 +17,11 @@ module BrokenLinkFinder
     # The max number of threads created during #crawl_site - one thread per page.
     attr_reader :max_threads
 
+    # The underlying Wgit::Crawler used by this instance of Finder.
+    attr_reader :crawler
+
     # Returns a new Finder instance.
-    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS)
+    def initialize(sort: :page, max_threads: DEFAULT_MAX_THREADS, &block)
       raise "Sort by either :page or :link, not #{sort}" \
       unless %i[page link].include?(sort)
 
@@ -26,6 +29,8 @@ module BrokenLinkFinder
       @max_threads = max_threads
       @crawler     = Wgit::Crawler.new
       @manager     = BrokenLinkFinder::LinkManager.new(@sort)
+
+      yield @crawler if block_given?
     end
 
     # Returns the current broken links.

data/lib/broken_link_finder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrokenLinkFinder
-  VERSION = '0.12.0'
+  VERSION = '0.12.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: broken_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.2
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2024-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
 description: Finds a website's broken links using the 'wgit' gem and reports back
   to you with a summary.
 email: michael.telford@live.com
@@ -182,16 +182,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+  - - "<"
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Finds a website's broken links and reports back to you with a summary.