deadfinder 1.7.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04f339906ad4505079153aea6164c57c48ef0d70540002cd49e9f56adddde355
-  data.tar.gz: 2483fc250e5de54e3cd4349c6a91cf305372c4acf7297fb286eb7a9140678d81
+  metadata.gz: 5c4f184329cd4314f65259f1f4b316c50ad3953bd52eeb81fbac45907eca25ab
+  data.tar.gz: d6e046f05ca1088d3a9a593fce99cc52dc582687e9982b68b1b604d3e1573918
 SHA512:
-  metadata.gz: 0f2577176f086c4ad6406b1c60e5e82a4e711e83902fd4088424d993e67de5d0a17fd9fea6e40cd8a8808a5944738393d0fd8ea7d109f19ba939db13155fc9c9
-  data.tar.gz: beb4467d411b85e42b569a552321da6e9138fab877b4ebb2b1f8b852cd2e30b99e57a8bca8e12a5774fd83679c5a50bfdf3c3cdc21f66d716a38dd4e8833fdc7
+  metadata.gz: 9c8dd4b7c1047808c8ac57228f9b1ab7ea02b948e8f42c37766a189058b7b44cb50822b4d4632faf45b27046364bdb08b0cc679c7ed2c8a95f2dacef818085aa
+  data.tar.gz: af74057d1c8e03e69e47253b07f685796f3978e9c5da75a33efd33ae7d8f004f147555a03c529c33bd871fd7830b887cc27c1a9e851f93d8725daa24036cefef

data/lib/deadfinder/cli.rb

@@ -15,7 +15,7 @@ module DeadFinder
     class_option :headers, aliases: :H, default: [], type: :array,
                            desc: 'Custom HTTP headers to send with initial request'
     class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
-    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.7.0;)', type: :string,
+    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.8.0;)', type: :string,
                               desc: 'User-Agent string to use for requests'
     class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
     class_option :proxy_auth, default: '', type: :string, desc: 'Proxy server authentication credentials'
@@ -24,6 +24,8 @@ module DeadFinder
     class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
     class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
     class_option :debug, default: false, type: :boolean, desc: 'Debug mode'
+    class_option :limit, default: 0, type: :numeric, desc: 'Limit the number of URLs to scan'
+    class_option :coverage, default: false, type: :boolean, desc: 'Enable coverage tracking and reporting'
 
     def self.exit_on_failure?
       true

data/lib/deadfinder/runner.rb

@@ -83,6 +83,15 @@ module DeadFinder
       jobs.close
 
       (1..jobs_size).each { ~results }
+      
+      # Log coverage summary if tracking was enabled
+      if options['coverage'] && DeadFinder.coverage_data[target] && DeadFinder.coverage_data[target][:total] > 0
+        total = DeadFinder.coverage_data[target][:total]
+        dead = DeadFinder.coverage_data[target][:dead]
+        percentage = ((dead.to_f / total) * 100).round(2)
+        DeadFinder::Logger.sub_info "Coverage: #{dead}/#{total} URLs are dead links (#{percentage}%)"
+      end
+      
       DeadFinder::Logger.sub_complete 'Task completed'
     rescue StandardError => e
       DeadFinder::Logger.error "[#{e}] #{target}"
@@ -94,6 +103,12 @@ module DeadFinder
           # Skip if already cached
         else
           CACHE_SET[j] = true
+          # Track total URLs tested for coverage calculation (only if coverage flag is enabled)
+          if options['coverage']
+            DeadFinder.coverage_data[target] ||= { total: 0, dead: 0 }
+            DeadFinder.coverage_data[target][:total] += 1
+          end
+          
           begin
             CACHE_QUE[j] = true
             uri = URI.parse(j)
@@ -114,11 +129,15 @@ module DeadFinder
               CACHE_QUE[j] = false
               DeadFinder.output[target] ||= []
               DeadFinder.output[target] << j
+              # Track dead URLs for coverage calculation (only if coverage flag is enabled)
+              DeadFinder.coverage_data[target][:dead] += 1 if options['coverage']
             else
               DeadFinder::Logger.verbose_ok "[#{status_code}] #{j}" if options['verbose']
             end
           rescue StandardError => e
             DeadFinder::Logger.verbose "[#{e}] #{j}" if options['verbose']
+            # Consider errored URLs as dead for coverage calculation (only if coverage flag is enabled)
+            DeadFinder.coverage_data[target][:dead] += 1 if options['coverage']
           end
         end
         results << j

data/lib/deadfinder/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadFinder
-  VERSION = '1.7.0'
+  VERSION = '1.8.0'
 end

data/lib/deadfinder.rb

@@ -29,6 +29,15 @@ module DeadFinder
     @output = val
   end
 
+  @coverage_data = {}
+  def self.coverage_data
+    @coverage_data
+  end
+
+  def self.coverage_data=(val)
+    @coverage_data = val
+  end
+
   def self.run_pipe(options)
     run_with_input(options) { $stdin.gets&.chomp }
   end
@@ -40,6 +49,7 @@ module DeadFinder
   def self.run_url(url, options)
     DeadFinder::Logger.apply_options(options)
     run_with_target(url, options)
+    gen_output(options)
   end
 
   def self.run_sitemap(sitemap_url, options)
@@ -47,8 +57,10 @@ module DeadFinder
     app = Runner.new
     base_uri = URI(sitemap_url)
     sitemap = SitemapParser.new(sitemap_url, recurse: true)
-    DeadFinder::Logger.info "Found #{sitemap.to_a.size} URLs from #{sitemap_url}"
-    sitemap.to_a.each do |url|
+    urls = sitemap.to_a
+    urls = urls.first(options['limit']) if options['limit'].positive?
+    DeadFinder::Logger.info "Found #{urls.size} URLs from #{sitemap_url}"
+    urls.each do |url|
       turl = generate_url(url, base_uri)
       run_with_target(turl, options, app)
     end
@@ -59,7 +71,9 @@ module DeadFinder
     DeadFinder::Logger.apply_options(options)
     DeadFinder::Logger.info 'Reading input'
     app = Runner.new
-    Array(yield).each do |target|
+    targets = Array(yield)
+    targets = targets.first(options['limit']) if options['limit'].positive?
+    targets.each do |target|
       run_with_target(target, options, app)
     end
     gen_output(options)
@@ -70,30 +84,82 @@ module DeadFinder
     app.run(target, options)
   end
 
+  def self.calculate_coverage
+    coverage_summary = {}
+    total_all_tested = 0
+    total_all_dead = 0
+
+    coverage_data.each do |target, data|
+      total = data[:total]
+      dead = data[:dead]
+      coverage_percentage = total.positive? ? ((dead.to_f / total) * 100).round(2) : 0.0
+
+      coverage_summary[target] = {
+        total_tested: total,
+        dead_links: dead,
+        coverage_percentage: coverage_percentage
+      }
+
+      total_all_tested += total
+      total_all_dead += dead
+    end
+
+    overall_coverage = total_all_tested.positive? ? ((total_all_dead.to_f / total_all_tested) * 100).round(2) : 0.0
+
+    {
+      targets: coverage_summary,
+      summary: {
+        total_tested: total_all_tested,
+        total_dead: total_all_dead,
+        overall_coverage_percentage: overall_coverage
+      }
+    }
+  end
+
   def self.gen_output(options)
     return if options['output'].empty?
 
     output_data = DeadFinder.output.to_h
     format = options['output_format'].to_s.downcase
 
+    # Include coverage data only if coverage flag is enabled and data exists
+    coverage_info = calculate_coverage if options['coverage'] && coverage_data.any? && coverage_data.values.any? { |v| v[:total].positive? }
+
     content = case format
               when 'yaml', 'yml'
-                output_data.to_yaml
+                output_with_coverage = coverage_info ? { 'dead_links' => output_data, 'coverage' => coverage_info } : output_data
+                output_with_coverage.to_yaml
               when 'csv'
-                generate_csv(output_data)
+                generate_csv(output_data, coverage_info)
               else
-                JSON.pretty_generate(output_data)
+                output_with_coverage = coverage_info ? { 'dead_links' => output_data, 'coverage' => coverage_info } : output_data
+                JSON.pretty_generate(output_with_coverage)
               end
 
     File.write(options['output'], content)
   end
 
-  def self.generate_csv(output_data)
+  def self.generate_csv(output_data, coverage_info = nil)
     CSV.generate do |csv|
       csv << %w[target url]
       output_data.each do |target, urls|
         Array(urls).each { |url| csv << [target, url] }
       end
+
+      # Add coverage information as additional rows if available
+      if coverage_info
+        csv << [] # Empty row separator
+        csv << ['Coverage Report']
+        csv << %w[target total_tested dead_links coverage_percentage]
+        coverage_info[:targets].each do |target, data|
+          csv << [target, data[:total_tested], data[:dead_links], "#{data[:coverage_percentage]}%"]
+        end
+        csv << [] # Empty row separator
+        csv << ['Overall Summary']
+        csv << %w[total_tested total_dead overall_coverage_percentage]
+        summary = coverage_info[:summary]
+        csv << [summary[:total_tested], summary[:total_dead], "#{summary[:overall_coverage_percentage]}%"]
+      end
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.8.0
 platform: ruby
 authors:
 - hahwul
 bindir: bin
 cert_chain: []
-date: 2025-03-12 00:00:00.000000000 Z
+date: 2025-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -53,42 +53,42 @@ dependencies:
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.11'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.13.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: '1.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.13.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: '1.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: open-uri
   requirement: !ruby/object:Gem::Requirement
@@ -153,22 +153,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement