deadfinder 1.6.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a9dba525c92f09cb14019fe890f73dc85eaa975179ac19efca534f0fdc76b14
-  data.tar.gz: fee3168a2912dacd9a2740b1aaf2c495b0b68b48b4d024c208523d8a8a5bb5e1
+  metadata.gz: 04f339906ad4505079153aea6164c57c48ef0d70540002cd49e9f56adddde355
+  data.tar.gz: 2483fc250e5de54e3cd4349c6a91cf305372c4acf7297fb286eb7a9140678d81
 SHA512:
-  metadata.gz: be806719c1ed5ca905885222d485f376c7e1a61dff0388f812e05d52e279e21315af08fdcfb5b66e9773c88be2f1a5c8c434eba62e6340f91db76ea29f4643db
-  data.tar.gz: d966f9237d2a4f62ba98684346b045c49bd14b024723eda22ac5f0c9c2fb0e62fc358d6607d924c0cdd80cd10e8e5f4eb37a13a067188d8394d142fe0476001b
+  metadata.gz: 0f2577176f086c4ad6406b1c60e5e82a4e711e83902fd4088424d993e67de5d0a17fd9fea6e40cd8a8808a5944738393d0fd8ea7d109f19ba939db13155fc9c9
+  data.tar.gz: beb4467d411b85e42b569a552321da6e9138fab877b4ebb2b1f8b852cd2e30b99e57a8bca8e12a5774fd83679c5a50bfdf3c3cdc21f66d716a38dd4e8833fdc7

data/bin/deadfinder

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require 'deadfinder'
-DeadFinder.start(ARGV)
+DeadFinder::CLI.start(ARGV)

data/lib/deadfinder/cli.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'thor'
+require 'deadfinder'
+require 'deadfinder/completion'
+
+module DeadFinder
+  # CLI class for handling command-line interactions
+  class CLI < Thor
+    class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
+    class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
+    class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
+    class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
+    class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
+    class_option :headers, aliases: :H, default: [], type: :array,
+                           desc: 'Custom HTTP headers to send with initial request'
+    class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
+    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.7.0;)', type: :string,
+                              desc: 'User-Agent string to use for requests'
+    class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
+    class_option :proxy_auth, default: '', type: :string, desc: 'Proxy server authentication credentials'
+    class_option :match, aliases: :m, default: '', type: :string, desc: 'Match the URL with the given pattern'
+    class_option :ignore, aliases: :i, default: '', type: :string, desc: 'Ignore the URL with the given pattern'
+    class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
+    class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
+    class_option :debug, default: false, type: :boolean, desc: 'Debug mode'
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc 'pipe', 'Scan the URLs from STDIN. (e.g., cat urls.txt | deadfinder pipe)'
+    def pipe
+      DeadFinder.run_pipe options
+    end
+
+    desc 'file <FILE>', 'Scan the URLs from File. (e.g., deadfinder file urls.txt)'
+    def file(filename)
+      DeadFinder.run_file filename, options
+    end
+
+    desc 'url <URL>', 'Scan the Single URL.'
+    def url(url)
+      DeadFinder.run_url url, options
+    end
+
+    desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
+    def sitemap(sitemap)
+      DeadFinder.run_sitemap sitemap, options
+    end
+
+    desc 'completion <SHELL>', 'Generate completion script for shell.'
+    def completion(shell)
+      unless %w[bash zsh fish].include?(shell)
+        DeadFinder::Logger.error "Unsupported shell: #{shell}"
+        return
+      end
+      case shell
+      when 'bash'
+        puts DeadFinder::Completion.bash
+      when 'zsh'
+        puts DeadFinder::Completion.zsh
+      when 'fish'
+        puts DeadFinder::Completion.fish
+      end
+    end
+
+    desc 'version', 'Show version.'
+    def version
+      DeadFinder::Logger.info "deadfinder #{DeadFinder::VERSION}"
+    end
+  end
+end

data/lib/deadfinder/completion.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module DeadFinder
+  # Module for completion script generation
+  module Completion
+    def self.bash
+      <<~BASH
+        _deadfinder_completions()
+        {
+          local cur prev opts
+          COMPREPLY=()
+          cur="${COMP_WORDS[COMP_CWORD]}"
+          opts="--include30x --concurrency --timeout --output --output_format --headers --worker_headers --user_agent --proxy --proxy_auth --match --ignore --silent --verbose --debug"
+
+          COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+          return 0
+        }
+        complete -F _deadfinder_completions deadfinder
+      BASH
+    end
+
+    def self.zsh
+      <<~ZSH
+        #compdef deadfinder
+
+        _arguments \\
+          '--include30x[Include 30x redirections]' \\
+          '--concurrency[Number of concurrency]:number' \\
+          '--timeout[Timeout in seconds]:number' \\
+          '--output[File to write result]:file' \\
+          '--output_format[Output format]:string' \\
+          '--headers[Custom HTTP headers]:array' \\
+          '--worker_headers[Custom HTTP headers for workers]:array' \\
+          '--user_agent[User-Agent string]:string' \\
+          '--proxy[Proxy server]:string' \\
+          '--proxy_auth[Proxy server authentication]:string' \\
+          '--match[Match URL pattern]:string' \\
+          '--ignore[Ignore URL pattern]:string' \\
+          '--silent[Silent mode]' \\
+          '--verbose[Verbose mode]' \\
+          '--debug[Debug mode]'
+      ZSH
+    end
+
+    def self.fish
+      <<~FISH
+        complete -c deadfinder -l include30x -d 'Include 30x redirections'
+        complete -c deadfinder -l concurrency -d 'Number of concurrency' -a '(seq 1 100)'
+        complete -c deadfinder -l timeout -d 'Timeout in seconds' -a '(seq 1 60)'
+        complete -c deadfinder -l output -d 'File to write result' -r
+        complete -c deadfinder -l output_format -d 'Output format' -r
+        complete -c deadfinder -l headers -d 'Custom HTTP headers' -r
+        complete -c deadfinder -l worker_headers -d 'Custom HTTP headers for workers' -r
+        complete -c deadfinder -l user_agent -d 'User-Agent string' -r
+        complete -c deadfinder -l proxy -d 'Proxy server' -r
+        complete -c deadfinder -l proxy_auth -d 'Proxy server authentication' -r
+        complete -c deadfinder -l match -d 'Match URL pattern' -r
+        complete -c deadfinder -l ignore -d 'Ignore URL pattern' -r
+        complete -c deadfinder -l silent -d 'Silent mode'
+        complete -c deadfinder -l verbose -d 'Verbose mode'
+        complete -c deadfinder -l debug -d 'Debug mode'
+      FISH
+    end
+  end
+end

data/lib/deadfinder/http_client.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'openssl'
+
+module DeadFinder
+  # HTTP client module
+  module HttpClient
+    def self.create(uri, options)
+      begin
+        proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
+      rescue URI::InvalidURIError => e
+        DeadFinder::Logger.error "Invalid proxy URI: #{options['proxy']} - #{e.message}"
+        proxy_uri = nil # or handle the error as appropriate
+      end
+      http = if proxy_uri
+               Net::HTTP.new(uri.host, uri.port,
+                             proxy_uri.host, proxy_uri.port,
+                             proxy_uri.user, proxy_uri.password)
+             else
+               Net::HTTP.new(uri.host, uri.port)
+             end
+      http.use_ssl = (uri.scheme == 'https')
+      http.read_timeout = options['timeout'].to_i if options['timeout']
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
+
+      if options['proxy_auth'] && proxy_uri
+        proxy_user, proxy_pass = options['proxy_auth'].split(':', 2)
+        http.proxy_user = proxy_user
+        http.proxy_pass = proxy_pass
+      end
+
+      http
+    end
+  end
+end

data/lib/deadfinder/logger.rb

@@ -2,53 +2,102 @@
 
 require 'colorize'
 
-class Logger
-  @silent = false
-  @mutex = Mutex.new
+module DeadFinder
+  class Logger
+    @silent = false
+    @verbose = false
+    @debug = false
+    @mutex = Mutex.new
 
-  def self.set_silent
-    @mutex.synchronize { @silent = true }
-  end
+    def self.apply_options(options)
+      set_silent if options['silent']
+      set_verbose if options['verbose']
+      set_debug if options['debug']
+    end
 
-  def self.unset_silent
-    @mutex.synchronize { @silent = false }
-  end
+    def self.set_silent
+      @mutex.synchronize { @silent = true }
+    end
 
-  def self.silent?
-    @mutex.synchronize { @silent }
-  end
+    def self.set_verbose
+      @mutex.synchronize { @verbose = true }
+    end
 
-  def self.log(prefix, text, color)
-    return if silent?
+    def self.set_debug
+      @mutex.synchronize { @debug = true }
+    end
 
-    puts prefix.colorize(color) + text.to_s.colorize(:"light_#{color}")
-  end
+    def self.unset_debug
+      @mutex.synchronize { @debug = false }
+    end
 
-  def self.info(text)
-    log('ℹ ', text, :blue)
-  end
+    def self.unset_verbose
+      @mutex.synchronize { @verbose = false }
+    end
 
-  def self.error(text)
-    log('⚠︎ ', text, :red)
-  end
+    def self.debug?
+      @mutex.synchronize { @debug }
+    end
 
-  def self.target(text)
-    log('► ', text, :green)
-  end
+    def self.verbose?
+      @mutex.synchronize { @verbose }
+    end
 
-  def self.sub_info(text)
-    log('  ● ', text, :blue)
-  end
+    def self.unset_silent
+      @mutex.synchronize { @silent = false }
+    end
 
-  def self.sub_done(text)
-    log('  ✓ ', text, :blue)
-  end
+    def self.silent?
+      @mutex.synchronize { @silent }
+    end
 
-  def self.found(text)
-    log('  ✘ ', text, :red)
-  end
+    def self.log(prefix, text, color)
+      return if silent?
+
+      puts prefix.colorize(color) + text.to_s
+    end
+
+    def self.sub_log(prefix, is_end, text, color)
+      return if silent?
+
+      indent = is_end ? '  └── ' : '  ├── '
+      puts indent.colorize(color) + prefix.colorize(color) + text.to_s
+    end
+
+    def self.debug(text)
+      log('❀ ', text, :yellow) if debug?
+    end
+
+    def self.info(text)
+      log('ℹ ', text, :blue)
+    end
+
+    def self.error(text)
+      log('⚠︎ ', text, :red)
+    end
+
+    def self.target(text)
+      log('► ', text, :green)
+    end
+
+    def self.sub_info(text)
+      log('  ● ', text, :blue)
+    end
+
+    def self.sub_complete(text)
+      sub_log('● ', true, text, :blue)
+    end
+
+    def self.found(text)
+      sub_log('✘ ', false, text, :red)
+    end
+
+    def self.verbose(text)
+      sub_log('➜ ', false, text, :yellow) if verbose?
+    end
 
-  def self.verbose(text)
-    log('  ➜ ', text, :yellow)
+    def self.verbose_ok(text)
+      sub_log('✓ ', false, text, :green) if verbose?
+    end
   end
 end

data/lib/deadfinder/runner.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'concurrent-edge'
+require 'nokogiri'
+require 'open-uri'
+require 'net/http'
+require 'openssl'
+require 'deadfinder/logger'
+require 'deadfinder/http_client'
+require 'deadfinder/url_pattern_matcher'
+
+module DeadFinder
+  # Runner class for executing the main logic
+  class Runner
+    def default_options
+      {
+        'concurrency' => 50,
+        'timeout' => 10,
+        'output' => '',
+        'output_format' => 'json',
+        'headers' => [],
+        'worker_headers' => [],
+        'silent' => true,
+        'verbose' => false,
+        'include30x' => false,
+        'proxy' => '',
+        'proxy_auth' => '',
+        'match' => '',
+        'ignore' => '',
+      }
+    end
+
+    def run(target, options)
+      DeadFinder::Logger.apply_options(options)
+      headers = options['headers'].each_with_object({}) do |header, hash|
+        kv = header.split(': ')
+        hash[kv[0]] = kv[1]
+      rescue StandardError
+      end
+      page = Nokogiri::HTML(URI.open(target, headers))
+      links = extract_links(page)
+
+      DeadFinder::Logger.debug "#{CACHE_QUE.size} URLs in queue, #{CACHE_SET.size} URLs in cache"
+
+      if options['match'] != ''
+        begin
+          links.each do |type, urls|
+        links[type] = urls.select { |url| DeadFinder::UrlPatternMatcher.match?(url, options['match']) }
+          end
+        rescue RegexpError => e
+          DeadFinder::Logger.error "Invalid match pattern: #{e.message}"
+        end
+      end
+
+      if options['ignore'] != ''
+        begin
+          links.each do |type, urls|
+            links[type] = urls.reject { |url| DeadFinder::UrlPatternMatcher.ignore?(url, options['ignore']) }
+          end
+        rescue RegexpError => e
+          DeadFinder::Logger.error "Invalid match pattern: #{e.message}"
+        end
+      end
+
+      total_links_count = links.values.flatten.length
+      link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }
+                       .compact.join(' / ')
+      DeadFinder::Logger.sub_info "Discovered #{total_links_count} URLs, currently checking them. [#{link_info}]" unless link_info.empty?
+
+      jobs = Channel.new(buffer: :buffered, capacity: 1000)
+      results = Channel.new(buffer: :buffered, capacity: 1000)
+
+      (1..options['concurrency']).each do |w|
+        Channel.go { worker(w, jobs, results, target, options) }
+      end
+
+      links.values.flatten.uniq.each do |node|
+        result = generate_url(node, target)
+        jobs << result unless result.nil?
+      end
+
+      jobs_size = jobs.size
+      jobs.close
+
+      (1..jobs_size).each { ~results }
+      DeadFinder::Logger.sub_complete 'Task completed'
+    rescue StandardError => e
+      DeadFinder::Logger.error "[#{e}] #{target}"
+    end
+
+    def worker(_id, jobs, results, target, options)
+      jobs.each do |j|
+        if CACHE_SET[j]
+          # Skip if already cached
+        else
+          CACHE_SET[j] = true
+          begin
+            CACHE_QUE[j] = true
+            uri = URI.parse(j)
+            http = HttpClient.create(uri, options)
+
+            request = Net::HTTP::Get.new(uri.request_uri)
+            request['User-Agent'] = options['user_agent']
+            options['worker_headers']&.each do |header|
+              key, value = header.split(':', 2)
+              request[key.strip] = value.strip
+            end
+
+            response = http.request(request)
+            status_code = response.code.to_i
+
+            if status_code >= 400 || (status_code >= 300 && options['include30x'])
+              DeadFinder::Logger.found "[#{status_code}] #{j}"
+              CACHE_QUE[j] = false
+              DeadFinder.output[target] ||= []
+              DeadFinder.output[target] << j
+            else
+              DeadFinder::Logger.verbose_ok "[#{status_code}] #{j}" if options['verbose']
+            end
+          rescue StandardError => e
+            DeadFinder::Logger.verbose "[#{e}] #{j}" if options['verbose']
+          end
+        end
+        results << j
+      end
+    end
+
+    private
+
+    def extract_links(page)
+      {
+        anchor: page.css('a').map { |element| element['href'] }.compact,
+        script: page.css('script').map { |element| element['src'] }.compact,
+        link: page.css('link').map { |element| element['href'] }.compact,
+        iframe: page.css('iframe').map { |element| element['src'] }.compact,
+        form: page.css('form').map { |element| element['action'] }.compact,
+        object: page.css('object').map { |element| element['data'] }.compact,
+        embed: page.css('embed').map { |element| element['src'] }.compact
+      }
+    end
+  end
+end

data/lib/deadfinder/url_pattern_matcher.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+module DeadFinder
+  # URL pattern matcher module
+  module UrlPatternMatcher
+    def self.match?(url, pattern)
+      Timeout.timeout(1) { Regexp.new(pattern).match?(url) }
+    rescue Timeout::Error
+      false
+    end
+
+    def self.ignore?(url, pattern)
+      Timeout.timeout(1) { Regexp.new(pattern).match?(url) }
+    rescue Timeout::Error
+      false
+    end
+  end
+end

data/lib/deadfinder/version.rb

@@ -1,3 +1,5 @@
 # frozen_string_literal: true
 
-VERSION = '1.6.0'
+module DeadFinder
+  VERSION = '1.7.0'
+end

data/lib/deadfinder.rb

@@ -6,6 +6,8 @@ require 'open-uri'
 require 'nokogiri'
 require 'deadfinder/utils'
 require 'deadfinder/logger'
+require 'deadfinder/runner'
+require 'deadfinder/cli'
 require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
@@ -13,245 +15,85 @@ require 'json'
 require 'yaml'
 require 'csv'
 
-Channel = Concurrent::Channel
-CACHE_SET = Concurrent::Map.new
-CACHE_QUE = Concurrent::Map.new
-OUTPUT = {}
+module DeadFinder
+  Channel = Concurrent::Channel
+  CACHE_SET = Concurrent::Map.new
+  CACHE_QUE = Concurrent::Map.new
 
-class DeadFinderRunner
-  def default_options
-    {
-      'concurrency' => 50,
-      'timeout' => 10,
-      'output' => '',
-      'output_format' => 'json',
-      'headers' => [],
-      'worker_headers' => [],
-      'silent' => true,
-      'verbose' => false,
-      'include30x' => false
-    }
+  @output = {}
+  def self.output
+    @output
   end
 
-  def run(target, options)
-    Logger.set_silent if options['silent']
-    headers = options['headers'].each_with_object({}) do |header, hash|
-      kv = header.split(': ')
-      hash[kv[0]] = kv[1]
-    rescue StandardError
-    end
-    page = Nokogiri::HTML(URI.open(target, headers))
-    links = extract_links(page)
-
-    total_links_count = links.values.flatten.length
-    # Generate link info string for non-empty link types
-    link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }.compact.join(' / ')
-
-    # Log the information if there are any links
-    Logger.sub_info "Found #{total_links_count} URLs. [#{link_info}]" unless link_info.empty?
-    Logger.sub_info 'Checking'
-
-    jobs = Channel.new(buffer: :buffered, capacity: 1000)
-    results = Channel.new(buffer: :buffered, capacity: 1000)
-
-    (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results, target, options) }
-    end
-
-    links.values.flatten.uniq.each do |node|
-      result = generate_url(node, target)
-      jobs << result unless result.nil?
-    end
-
-    jobs_size = jobs.size
-    jobs.close
-
-    (1..jobs_size).each do
-      ~results
-    end
-    Logger.sub_done 'Done'
-  rescue StandardError => e
-    Logger.error "[#{e}] #{target}"
+  def self.output=(val)
+    @output = val
   end
 
-  def worker(_id, jobs, results, target, options)
-    jobs.each do |j|
-      if CACHE_SET[j]
-        Logger.found "[404 Not Found] #{j}" unless CACHE_QUE[j]
-      else
-        CACHE_SET[j] = true
-        begin
-          CACHE_QUE[j] = true
-          uri = URI.parse(j)
-
-          # Create HTTP request with timeout and headers
-          proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
-          http = if proxy_uri
-                   Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
-                 else
-                   Net::HTTP.new(uri.host, uri.port)
-                 end
-          http.use_ssl = (uri.scheme == 'https')
-          http.read_timeout = options['timeout'].to_i if options['timeout']
-
-          # Set SSL verification mode
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
-
-          request = Net::HTTP::Get.new(uri.request_uri)
-
-          # Add User-Agent header
-          request['User-Agent'] = options['user_agent']
-
-          # Add worker headers if provided
-          options['worker_headers']&.each do |header|
-            key, value = header.split(':', 2)
-            request[key.strip] = value.strip
-          end
-
-          response = http.request(request)
-          status_code = response.code.to_i
-          Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
-
-          if status_code >= 400 || (status_code >= 300 && options['include30x'])
-            Logger.found "[#{status_code} #{response.message}] #{j}"
-            CACHE_QUE[j] = false
-            OUTPUT[target] ||= []
-            OUTPUT[target] << j
-          end
-        rescue StandardError => e
-          Logger.verbose "[#{e}] #{j}" if options['verbose']
-        end
-      end
-      results << j
-    end
+  def self.run_pipe(options)
+    run_with_input(options) { $stdin.gets&.chomp }
   end
 
-  private
-
-  def extract_links(page)
-    {
-      anchor: page.css('a').map { |element| element['href'] }.compact,
-      script: page.css('script').map { |element| element['src'] }.compact,
-      link: page.css('link').map { |element| element['href'] }.compact,
-      iframe: page.css('iframe').map { |element| element['src'] }.compact,
-      form: page.css('form').map { |element| element['action'] }.compact,
-      object: page.css('object').map { |element| element['data'] }.compact,
-      embed: page.css('embed').map { |element| element['src'] }.compact
-    }
+  def self.run_file(filename, options)
+    run_with_input(options) { File.foreach(filename).map(&:chomp) }
   end
-end
 
-def run_pipe(options)
-  Logger.set_silent if options['silent']
-
-  Logger.info 'Reading from STDIN'
-  app = DeadFinderRunner.new
-  while $stdin.gets
-    target = $LAST_READ_LINE.chomp
-    Logger.target "Checking: #{target}"
-    app.run target, options
+  def self.run_url(url, options)
+    DeadFinder::Logger.apply_options(options)
+    run_with_target(url, options)
   end
-  gen_output(options)
-end
-
-def run_file(filename, options)
-  Logger.set_silent if options['silent']
 
-  Logger.info "Reading: #{filename}"
-  app = DeadFinderRunner.new
-  File.foreach(filename) do |line|
-    target = line.chomp
-    Logger.target "Checking: #{target}"
-    app.run target, options
+  def self.run_sitemap(sitemap_url, options)
+    DeadFinder::Logger.apply_options(options)
+    app = Runner.new
+    base_uri = URI(sitemap_url)
+    sitemap = SitemapParser.new(sitemap_url, recurse: true)
+    DeadFinder::Logger.info "Found #{sitemap.to_a.size} URLs from #{sitemap_url}"
+    sitemap.to_a.each do |url|
+      turl = generate_url(url, base_uri)
+      run_with_target(turl, options, app)
+    end
+    gen_output(options)
   end
-  gen_output(options)
-end
-
-def run_url(url, options)
-  Logger.set_silent if options['silent']
 
-  Logger.target "Checking: #{url}"
-  app = DeadFinderRunner.new
-  app.run url, options
-  gen_output(options)
-end
+  def self.run_with_input(options)
+    DeadFinder::Logger.apply_options(options)
+    DeadFinder::Logger.info 'Reading input'
+    app = Runner.new
+    Array(yield).each do |target|
+      run_with_target(target, options, app)
+    end
+    gen_output(options)
+  end
 
-def run_sitemap(sitemap_url, options)
-  Logger.set_silent if options['silent']
-  Logger.info "Parsing sitemap: #{sitemap_url}"
-  app = DeadFinderRunner.new
-  base_uri = URI(sitemap_url)
-  sitemap = SitemapParser.new sitemap_url, { recurse: true }
-  sitemap.to_a.each do |url|
-    turl = generate_url url, base_uri
-    Logger.target "Checking: #{turl}"
-    app.run turl, options
+  def self.run_with_target(target, options, app = Runner.new)
+    DeadFinder::Logger.target "Fetching #{target}"
+    app.run(target, options)
   end
-  gen_output(options)
-end
 
-def gen_output(options)
-  return if options['output'].empty?
+  def self.gen_output(options)
+    return if options['output'].empty?
 
-  output_data = OUTPUT.to_h
-  format = options['output_format'].to_s.downcase
+    output_data = DeadFinder.output.to_h
+    format = options['output_format'].to_s.downcase
 
-  content = case format
-            when 'yaml', 'yml'
-              output_data.to_yaml
-            when 'csv'
-              CSV.generate do |csv|
-                csv << ['target', 'url']
-                output_data.each do |target, urls|
-                  Array(urls).each do |url|
-                    csv << [target, url]
-                  end
-                end
+    content = case format
+              when 'yaml', 'yml'
+                output_data.to_yaml
+              when 'csv'
+                generate_csv(output_data)
+              else
+                JSON.pretty_generate(output_data)
               end
-            else
-              JSON.pretty_generate(output_data)
-            end
 
-  File.write(options['output'], content)
-end
-
-class DeadFinder < Thor
-  class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
-  class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
-  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
-  class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
-  class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
-  class_option :headers, aliases: :H, default: [], type: :array,
-                         desc: 'Custom HTTP headers to send with initial request'
-  class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
-  class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.6.0;)', type: :string,
-                            desc: 'User-Agent string to use for requests'
-  class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
-  class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
-  class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
-
-  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
-  def pipe
-    run_pipe options
-  end
-
-  desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
-  def file(filename)
-    run_file filename, options
-  end
-
-  desc 'url <URL>', 'Scan the Single URL.'
-  def url(url)
-    run_url url, options
-  end
-
-  desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
-  def sitemap(sitemap)
-    run_sitemap sitemap, options
+    File.write(options['output'], content)
   end
 
-  desc 'version', 'Show version.'
-  def version
-    Logger.info "deadfinder #{VERSION}"
+  def self.generate_csv(output_data)
+    CSV.generate do |csv|
+      csv << %w[target url]
+      output_data.each do |target, urls|
+        Array(urls).each { |url| csv << [target, url] }
+      end
+    end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.0
 platform: ruby
 authors:
 - hahwul
 bindir: bin
 cert_chain: []
-date: 2025-02-17 00:00:00.000000000 Z
+date: 2025-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -169,6 +169,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Find dead-links (broken links). Dead link (broken link) means a link
   within a web page that cannot be connected. These links can have a negative impact
   to SEO and Security. This tool makes it easy to identify and modify.
@@ -180,10 +194,15 @@ extra_rdoc_files: []
 files:
 - bin/deadfinder
 - lib/deadfinder.rb
+- lib/deadfinder/cli.rb
+- lib/deadfinder/completion.rb
+- lib/deadfinder/http_client.rb
 - lib/deadfinder/logger.rb
+- lib/deadfinder/runner.rb
+- lib/deadfinder/url_pattern_matcher.rb
 - lib/deadfinder/utils.rb
 - lib/deadfinder/version.rb
-homepage: https://www.hahwul.com
+homepage: https://www.hahwul.com/projects/deadfinder/
 licenses:
 - MIT
 metadata: