deadfinder 1.7.1 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98d859458bc87d32005ad05d4e3f90103aab430037b4b439951911e760eea968
-  data.tar.gz: 347a37265add728676f196a8d313dfda37a54b48eb45030f6ff01dba15b152d6
+  metadata.gz: 4d72852a66ecdb612a93361fa42c367d9c2505966016bf4ca0a76a099e6ab235
+  data.tar.gz: 19116106505f9c3060dce40406b8bc478f4ce01e49eccfddaa192fc981adcef2
 SHA512:
-  metadata.gz: dc059649e3a8cfd4e644651db38e1082b4b1060624cd2d1d6d43064c43c87c836583912c6430c00f0799ee77526917d2357286cd2129c4e6176e62457c6dc007
-  data.tar.gz: 17d62439486da8d4f2efe8ce957744c8fca8ee23a5670fb7aa128d9b688e4b7dcf7c8689355553431a171ac14e2c2a6d58515a9ec9a2c4d4ea1f5bfc40c482f2
+  metadata.gz: 5b00c09cbe273376dd5cc60fb964d1a74146e6d98e5a21656d91202b792b6bcac3576ba825b3fc3d144f58f2e76a34ad08658f925d0469824b420fac1e2ea153
+  data.tar.gz: 4ae07f18471a1aa71018533259e14d99af53799e1f65479a6d953f44d7d5f182c97393747cb502a910878311ff70ec85a6e007c63aea20c8c58b78798f03b3cc

data/lib/deadfinder/cli.rb

@@ -15,7 +15,7 @@ module DeadFinder
     class_option :headers, aliases: :H, default: [], type: :array,
                            desc: 'Custom HTTP headers to send with initial request'
     class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
-    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.7.1;)', type: :string,
+    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.9.0;)', type: :string,
                               desc: 'User-Agent string to use for requests'
     class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
     class_option :proxy_auth, default: '', type: :string, desc: 'Proxy server authentication credentials'
@@ -24,6 +24,9 @@ module DeadFinder
     class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
     class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
     class_option :debug, default: false, type: :boolean, desc: 'Debug mode'
+    class_option :limit, default: 0, type: :numeric, desc: 'Limit the number of URLs to scan'
+    class_option :coverage, default: false, type: :boolean, desc: 'Enable coverage tracking and reporting'
+    class_option :visualize, default: '', type: :string, desc: 'Generate a visualization of the scan results (e.g., report.png)'
 
     def self.exit_on_failure?
       true
@@ -31,22 +34,30 @@ module DeadFinder
 
     desc 'pipe', 'Scan the URLs from STDIN. (e.g., cat urls.txt | deadfinder pipe)'
     def pipe
-      DeadFinder.run_pipe options
+      opts = options.dup
+      opts['coverage'] = true if opts['visualize'] && !opts['visualize'].empty?
+      DeadFinder.run_pipe opts
     end
 
     desc 'file <FILE>', 'Scan the URLs from File. (e.g., deadfinder file urls.txt)'
     def file(filename)
-      DeadFinder.run_file filename, options
+      opts = options.dup
+      opts['coverage'] = true if opts['visualize'] && !opts['visualize'].empty?
+      DeadFinder.run_file filename, opts
     end
 
     desc 'url <URL>', 'Scan the Single URL.'
     def url(url)
-      DeadFinder.run_url url, options
+      opts = options.dup
+      opts['coverage'] = true if opts['visualize'] && !opts['visualize'].empty?
+      DeadFinder.run_url url, opts
     end
 
     desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
     def sitemap(sitemap)
-      DeadFinder.run_sitemap sitemap, options
+      opts = options.dup
+      opts['coverage'] = true if opts['visualize'] && !opts['visualize'].empty?
+      DeadFinder.run_sitemap sitemap, opts
     end
 
     desc 'completion <SHELL>', 'Generate completion script for shell.'

data/lib/deadfinder/runner.rb

@@ -83,6 +83,15 @@ module DeadFinder
       jobs.close
 
       (1..jobs_size).each { ~results }
+
+      # Log coverage summary if tracking was enabled
+      if options['coverage'] && DeadFinder.coverage_data[target] && DeadFinder.coverage_data[target][:total] > 0
+        total = DeadFinder.coverage_data[target][:total]
+        dead = DeadFinder.coverage_data[target][:dead]
+        percentage = ((dead.to_f / total) * 100).round(2)
+        DeadFinder::Logger.sub_info "Coverage: #{dead}/#{total} URLs are dead links (#{percentage}%)"
+      end
+
       DeadFinder::Logger.sub_complete 'Task completed'
     rescue StandardError => e
       DeadFinder::Logger.error "[#{e}] #{target}"
@@ -94,6 +103,12 @@ module DeadFinder
           # Skip if already cached
         else
           CACHE_SET[j] = true
+          # Track total URLs tested for coverage calculation (only if coverage flag is enabled)
+          if options['coverage']
+            DeadFinder.coverage_data[target] ||= { total: 0, dead: 0, status_counts: Hash.new(0) }
+            DeadFinder.coverage_data[target][:total] += 1
+          end
+
           begin
             CACHE_QUE[j] = true
             uri = URI.parse(j)
@@ -114,11 +129,23 @@ module DeadFinder
               CACHE_QUE[j] = false
               DeadFinder.output[target] ||= []
               DeadFinder.output[target] << j
+              # Track dead URLs for coverage calculation (only if coverage flag is enabled)
+              if options['coverage']
+                DeadFinder.coverage_data[target][:dead] += 1
+                DeadFinder.coverage_data[target][:status_counts][status_code] += 1
+              end
             else
               DeadFinder::Logger.verbose_ok "[#{status_code}] #{j}" if options['verbose']
+              # Track status for successful URLs
+              DeadFinder.coverage_data[target][:status_counts][status_code] += 1 if options['coverage']
             end
           rescue StandardError => e
             DeadFinder::Logger.verbose "[#{e}] #{j}" if options['verbose']
+            # Consider errored URLs as dead for coverage calculation (only if coverage flag is enabled)
+            if options['coverage']
+              DeadFinder.coverage_data[target][:dead] += 1
+              DeadFinder.coverage_data[target][:status_counts]['error'] += 1
+            end
           end
         end
         results << j

data/lib/deadfinder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadFinder
-  VERSION = '1.7.1'
+  VERSION = '1.9.0'
 end

data/lib/deadfinder/visualizer.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'chunky_png'
+
+module DeadFinder
+  # Visualizer module for generating images from scan results
+  module Visualizer
+    def self.generate(data, output_path)
+      # Extract summary data
+      summary = data[:summary]
+      return if summary.nil?
+
+      total_tested = summary[:total_tested]
+      return if total_tested.nil? || total_tested.zero?
+
+      # Create a new image with transparent background
+      png = ChunkyPNG::Image.new(500, 300, ChunkyPNG::Color::TRANSPARENT)
+
+      # Draw stacked bar chart for status code distribution
+      status_counts = data[:summary][:overall_status_counts] || {}
+      bar_height = 70
+      current_y = 110
+
+      # Sort statuses by count descending
+      sorted_statuses = status_counts.sort_by { |_, v| -v }
+
+      sorted_statuses.each do |status, count|
+        height = (count.to_f / total_tested * bar_height).to_i
+        next if height.zero?
+
+        color = case status.to_s
+                when '200' then ChunkyPNG::Color.rgb(0, 255, 0) # Green for 200
+                when /^3\d{2}$/ then ChunkyPNG::Color.rgb(255, 165, 0) # Orange for 3xx
+                when /^4\d{2}$/ then ChunkyPNG::Color.rgb(255, 0, 0)   # Red for 4xx
+                when /^5\d{2}$/ then ChunkyPNG::Color.rgb(128, 0, 128) # Purple for 5xx
+                else ChunkyPNG::Color.rgb(128, 128, 128)               # Gray for others/error
+                end
+
+        (current_y..(current_y + height - 1)).each do |y|
+          (20..480).each do |x|
+            png[x, y] = color
+          end
+        end
+        current_y += height
+      end
+
+      # Draw rounded outline around the bar area
+      r = 10
+      x1 = 10
+      y1 = 100
+      x2 = 490
+      y2 = 190
+
+      # Top line
+      ((x1 + r)..(x2 - r)).each do |x|
+        png[x, y1] = ChunkyPNG::Color.rgba(0, 0, 0, 128)
+        # Bottom line
+        png[x, y2] = ChunkyPNG::Color.rgba(0, 0, 0, 128)
+      end
+      # Left line
+      ((y1 + r)..(y2 - r)).each do |y|
+        png[x1, y] = ChunkyPNG::Color.rgba(0, 0, 0, 128)
+        # Right line
+        png[x2, y] = ChunkyPNG::Color.rgba(0, 0, 0, 128)
+      end
+
+      # Corners: quarter circles
+      # Top-left
+      (0..90).each do |angle|
+        rad = angle * Math::PI / 180
+        cx = x1 + r
+        cy = y1 + r
+        px = cx + (r * Math.cos(rad))
+        py = cy + (r * Math.sin(rad))
+        png[px.to_i, py.to_i] = ChunkyPNG::Color.rgba(0, 0, 0, 128) if px >= x1 && py >= y1
+      end
+      # Top-right
+      (90..180).each do |angle|
+        rad = angle * Math::PI / 180
+        cx = x2 - r
+        cy = y1 + r
+        px = cx + (r * Math.cos(rad))
+        py = cy + (r * Math.sin(rad))
+        png[px.to_i, py.to_i] = ChunkyPNG::Color.rgba(0, 0, 0, 128) if px <= x2 && py >= y1
+      end
+      # Bottom-left
+      (270..360).each do |angle|
+        rad = angle * Math::PI / 180
+        cx = x1 + r
+        cy = y2 - r
+        px = cx + (r * Math.cos(rad))
+        py = cy + (r * Math.sin(rad))
+        png[px.to_i, py.to_i] = ChunkyPNG::Color.rgba(0, 0, 0, 128) if px >= x1 && py <= y2
+      end
+      # Bottom-right
+      (180..270).each do |angle|
+        rad = angle * Math::PI / 180
+        cx = x2 - r
+        cy = y2 - r
+        px = cx + (r * Math.cos(rad))
+        py = cy + (r * Math.sin(rad))
+        png[px.to_i, py.to_i] = ChunkyPNG::Color.rgba(0, 0, 0, 128) if px <= x2 && py <= y2
+      end
+
+      # Save the image
+      png.save(output_path, :fast_rgba)
+    end
+  end
+end

data/lib/deadfinder.rb

@@ -7,6 +7,7 @@ require 'nokogiri'
 require 'deadfinder/utils'
 require 'deadfinder/logger'
 require 'deadfinder/runner'
+require 'deadfinder/visualizer'
 require 'deadfinder/cli'
 require 'deadfinder/version'
 require 'concurrent-edge'
@@ -29,6 +30,15 @@ module DeadFinder
     @output = val
   end
 
+  @coverage_data = {}
+  def self.coverage_data
+    @coverage_data
+  end
+
+  def self.coverage_data=(val)
+    @coverage_data = val
+  end
+
   def self.run_pipe(options)
     run_with_input(options) { $stdin.gets&.chomp }
   end
@@ -48,8 +58,10 @@ module DeadFinder
     app = Runner.new
     base_uri = URI(sitemap_url)
     sitemap = SitemapParser.new(sitemap_url, recurse: true)
-    DeadFinder::Logger.info "Found #{sitemap.to_a.size} URLs from #{sitemap_url}"
-    sitemap.to_a.each do |url|
+    urls = sitemap.to_a
+    urls = urls.first(options['limit']) if options['limit'].positive?
+    DeadFinder::Logger.info "Found #{urls.size} URLs from #{sitemap_url}"
+    urls.each do |url|
       turl = generate_url(url, base_uri)
       run_with_target(turl, options, app)
     end
@@ -60,7 +72,9 @@ module DeadFinder
     DeadFinder::Logger.apply_options(options)
     DeadFinder::Logger.info 'Reading input'
     app = Runner.new
-    Array(yield).each do |target|
+    targets = Array(yield)
+    targets = targets.first(options['limit']) if options['limit'].positive?
+    targets.each do |target|
       run_with_target(target, options, app)
     end
     gen_output(options)
@@ -71,30 +85,89 @@ module DeadFinder
     app.run(target, options)
   end
 
-  def self.gen_output(options)
-    return if options['output'].empty?
+  def self.calculate_coverage
+    coverage_summary = {}
+    total_all_tested = 0
+    total_all_dead = 0
+    overall_status_counts = Hash.new(0)
+
+    coverage_data.each do |target, data|
+      total = data[:total]
+      dead = data[:dead]
+      status_counts = data[:status_counts] || {}
+      coverage_percentage = total.positive? ? ((dead.to_f / total) * 100).round(2) : 0.0
+
+      coverage_summary[target] = {
+        total_tested: total,
+        dead_links: dead,
+        coverage_percentage: coverage_percentage,
+        status_counts: status_counts
+      }
+
+      total_all_tested += total
+      total_all_dead += dead
+      status_counts.each { |code, count| overall_status_counts[code] += count }
+    end
 
+    overall_coverage = total_all_tested.positive? ? ((total_all_dead.to_f / total_all_tested) * 100).round(2) : 0.0
+
+    {
+      targets: coverage_summary,
+      summary: {
+        total_tested: total_all_tested,
+        total_dead: total_all_dead,
+        overall_coverage_percentage: overall_coverage,
+        overall_status_counts: overall_status_counts
+      }
+    }
+  end
+
+  def self.gen_output(options)
     output_data = DeadFinder.output.to_h
     format = options['output_format'].to_s.downcase
+    # Include coverage data only if coverage flag is enabled and data exists
+    coverage_info = calculate_coverage if options['coverage'] && coverage_data.any? && coverage_data.values.any? { |v| v[:total].positive? }
 
-    content = case format
-              when 'yaml', 'yml'
-                output_data.to_yaml
-              when 'csv'
-                generate_csv(output_data)
-              else
-                JSON.pretty_generate(output_data)
-              end
+    unless options['output'].to_s.empty?
+      content = case format
+                when 'yaml', 'yml'
+                  output_with_coverage = coverage_info ? { 'dead_links' => output_data, 'coverage' => coverage_info } : output_data
+                  output_with_coverage.to_yaml
+                when 'csv'
+                  generate_csv(output_data, coverage_info)
+                else
+                  output_with_coverage = coverage_info ? { 'dead_links' => output_data, 'coverage' => coverage_info } : output_data
+                  JSON.pretty_generate(output_with_coverage)
+                end
+      File.write(options['output'], content)
+    end
+
+    return unless options['visualize'] && !options['visualize'].empty? && coverage_info
 
-    File.write(options['output'], content)
+    DeadFinder::Visualizer.generate(coverage_info, options['visualize'])
   end
 
-  def self.generate_csv(output_data)
+  def self.generate_csv(output_data, coverage_info = nil)
     CSV.generate do |csv|
       csv << %w[target url]
       output_data.each do |target, urls|
         Array(urls).each { |url| csv << [target, url] }
       end
+
+      # Add coverage information as additional rows if available
+      if coverage_info
+        csv << [] # Empty row separator
+        csv << ['Coverage Report']
+        csv << %w[target total_tested dead_links coverage_percentage]
+        coverage_info[:targets].each do |target, data|
+          csv << [target, data[:total_tested], data[:dead_links], "#{data[:coverage_percentage]}%"]
+        end
+        csv << [] # Empty row separator
+        csv << ['Overall Summary']
+        csv << %w[total_tested total_dead overall_coverage_percentage]
+        summary = coverage_info[:summary]
+        csv << [summary[:total_tested], summary[:total_dead], "#{summary[:overall_coverage_percentage]}%"]
+      end
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.9.0
 platform: ruby
 authors:
 - hahwul
 bindir: bin
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -151,38 +151,72 @@ dependencies:
         version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: chunky_png
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: cyclonedx-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
 description: Find dead-links (broken links). Dead link (broken link) means a link
   within a web page that cannot be connected. These links can have a negative impact
   to SEO and Security. This tool makes it easy to identify and modify.
@@ -202,6 +236,7 @@ files:
 - lib/deadfinder/url_pattern_matcher.rb
 - lib/deadfinder/utils.rb
 - lib/deadfinder/version.rb
+- lib/deadfinder/visualizer.rb
 homepage: https://www.hahwul.com/projects/deadfinder/
 licenses:
 - MIT
@@ -222,7 +257,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Find dead-links (broken links)
 test_files: []