deadfinder 1.6.1 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8c1697ba3269001737ec44650a3f27ffd73cbf5467bd110df7e09cc7fc85ba5
-  data.tar.gz: f7ac73c33b9862cdbaaa661712d4969a999d73e12cdf4e6e725f9280fe67203b
+  metadata.gz: 98d859458bc87d32005ad05d4e3f90103aab430037b4b439951911e760eea968
+  data.tar.gz: 347a37265add728676f196a8d313dfda37a54b48eb45030f6ff01dba15b152d6
 SHA512:
-  metadata.gz: c59678fbca26d0caa131cc31c931ddb43712bae58f389fb8b5207eb4564904e8ed29b746645ec2802d7379be2f98e502a87207dbcf5d56bb5dfbe5d8b211448d
-  data.tar.gz: ddb8e07bda9415278724e09b559bc3e76d49c56a9bc8143d7a5e4426914c6034884a9b90bccf34982c2097d79e28d80833bf1025beee16b3d8dfe9e1b3a9f2d6
+  metadata.gz: dc059649e3a8cfd4e644651db38e1082b4b1060624cd2d1d6d43064c43c87c836583912c6430c00f0799ee77526917d2357286cd2129c4e6176e62457c6dc007
+  data.tar.gz: 17d62439486da8d4f2efe8ce957744c8fca8ee23a5670fb7aa128d9b688e4b7dcf7c8689355553431a171ac14e2c2a6d58515a9ec9a2c4d4ea1f5bfc40c482f2

data/lib/deadfinder/cli.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'thor'
+require 'deadfinder'
+require 'deadfinder/completion'
+
+module DeadFinder
+  # CLI class for handling command-line interactions
+  class CLI < Thor
+    class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
+    class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
+    class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
+    class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
+    class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
+    class_option :headers, aliases: :H, default: [], type: :array,
+                           desc: 'Custom HTTP headers to send with initial request'
+    class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
+    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.7.1;)', type: :string,
+                              desc: 'User-Agent string to use for requests'
+    class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
+    class_option :proxy_auth, default: '', type: :string, desc: 'Proxy server authentication credentials'
+    class_option :match, aliases: :m, default: '', type: :string, desc: 'Match the URL with the given pattern'
+    class_option :ignore, aliases: :i, default: '', type: :string, desc: 'Ignore the URL with the given pattern'
+    class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
+    class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
+    class_option :debug, default: false, type: :boolean, desc: 'Debug mode'
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc 'pipe', 'Scan the URLs from STDIN. (e.g., cat urls.txt | deadfinder pipe)'
+    def pipe
+      DeadFinder.run_pipe options
+    end
+
+    desc 'file <FILE>', 'Scan the URLs from File. (e.g., deadfinder file urls.txt)'
+    def file(filename)
+      DeadFinder.run_file filename, options
+    end
+
+    desc 'url <URL>', 'Scan the Single URL.'
+    def url(url)
+      DeadFinder.run_url url, options
+    end
+
+    desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
+    def sitemap(sitemap)
+      DeadFinder.run_sitemap sitemap, options
+    end
+
+    desc 'completion <SHELL>', 'Generate completion script for shell.'
+    def completion(shell)
+      unless %w[bash zsh fish].include?(shell)
+        DeadFinder::Logger.error "Unsupported shell: #{shell}"
+        return
+      end
+      case shell
+      when 'bash'
+        puts DeadFinder::Completion.bash
+      when 'zsh'
+        puts DeadFinder::Completion.zsh
+      when 'fish'
+        puts DeadFinder::Completion.fish
+      end
+    end
+
+    desc 'version', 'Show version.'
+    def version
+      DeadFinder::Logger.info "deadfinder #{DeadFinder::VERSION}"
+    end
+  end
+end

data/lib/deadfinder/completion.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module DeadFinder
+  # Module for completion script generation
+  module Completion
+    def self.bash
+      <<~BASH
+        _deadfinder_completions()
+        {
+          local cur prev opts
+          COMPREPLY=()
+          cur="${COMP_WORDS[COMP_CWORD]}"
+          opts="--include30x --concurrency --timeout --output --output_format --headers --worker_headers --user_agent --proxy --proxy_auth --match --ignore --silent --verbose --debug"
+
+          COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+          return 0
+        }
+        complete -F _deadfinder_completions deadfinder
+      BASH
+    end
+
+    def self.zsh
+      <<~ZSH
+        #compdef deadfinder
+
+        _arguments \\
+          '--include30x[Include 30x redirections]' \\
+          '--concurrency[Number of concurrency]:number' \\
+          '--timeout[Timeout in seconds]:number' \\
+          '--output[File to write result]:file' \\
+          '--output_format[Output format]:string' \\
+          '--headers[Custom HTTP headers]:array' \\
+          '--worker_headers[Custom HTTP headers for workers]:array' \\
+          '--user_agent[User-Agent string]:string' \\
+          '--proxy[Proxy server]:string' \\
+          '--proxy_auth[Proxy server authentication]:string' \\
+          '--match[Match URL pattern]:string' \\
+          '--ignore[Ignore URL pattern]:string' \\
+          '--silent[Silent mode]' \\
+          '--verbose[Verbose mode]' \\
+          '--debug[Debug mode]'
+      ZSH
+    end
+
+    def self.fish
+      <<~FISH
+        complete -c deadfinder -l include30x -d 'Include 30x redirections'
+        complete -c deadfinder -l concurrency -d 'Number of concurrency' -a '(seq 1 100)'
+        complete -c deadfinder -l timeout -d 'Timeout in seconds' -a '(seq 1 60)'
+        complete -c deadfinder -l output -d 'File to write result' -r
+        complete -c deadfinder -l output_format -d 'Output format' -r
+        complete -c deadfinder -l headers -d 'Custom HTTP headers' -r
+        complete -c deadfinder -l worker_headers -d 'Custom HTTP headers for workers' -r
+        complete -c deadfinder -l user_agent -d 'User-Agent string' -r
+        complete -c deadfinder -l proxy -d 'Proxy server' -r
+        complete -c deadfinder -l proxy_auth -d 'Proxy server authentication' -r
+        complete -c deadfinder -l match -d 'Match URL pattern' -r
+        complete -c deadfinder -l ignore -d 'Ignore URL pattern' -r
+        complete -c deadfinder -l silent -d 'Silent mode'
+        complete -c deadfinder -l verbose -d 'Verbose mode'
+        complete -c deadfinder -l debug -d 'Debug mode'
+      FISH
+    end
+  end
+end

data/lib/deadfinder/http_client.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'openssl'
+
+module DeadFinder
+  # HTTP client module
+  module HttpClient
+    def self.create(uri, options)
+      begin
+        proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
+      rescue URI::InvalidURIError => e
+        DeadFinder::Logger.error "Invalid proxy URI: #{options['proxy']} - #{e.message}"
+        proxy_uri = nil # or handle the error as appropriate
+      end
+      http = if proxy_uri
+               Net::HTTP.new(uri.host, uri.port,
+                             proxy_uri.host, proxy_uri.port,
+                             proxy_uri.user, proxy_uri.password)
+             else
+               Net::HTTP.new(uri.host, uri.port)
+             end
+      http.use_ssl = (uri.scheme == 'https')
+      http.read_timeout = options['timeout'].to_i if options['timeout']
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
+
+      if options['proxy_auth'] && proxy_uri
+        proxy_user, proxy_pass = options['proxy_auth'].split(':', 2)
+        http.proxy_user = proxy_user
+        http.proxy_pass = proxy_pass
+      end
+
+      http
+    end
+  end
+end

data/lib/deadfinder/logger.rb

@@ -2,53 +2,102 @@
 
 require 'colorize'
 
-class Logger
-  @silent = false
-  @mutex = Mutex.new
+module DeadFinder
+  class Logger
+    @silent = false
+    @verbose = false
+    @debug = false
+    @mutex = Mutex.new
 
-  def self.set_silent
-    @mutex.synchronize { @silent = true }
-  end
+    def self.apply_options(options)
+      set_silent if options['silent']
+      set_verbose if options['verbose']
+      set_debug if options['debug']
+    end
 
-  def self.unset_silent
-    @mutex.synchronize { @silent = false }
-  end
+    def self.set_silent
+      @mutex.synchronize { @silent = true }
+    end
 
-  def self.silent?
-    @mutex.synchronize { @silent }
-  end
+    def self.set_verbose
+      @mutex.synchronize { @verbose = true }
+    end
 
-  def self.log(prefix, text, color)
-    return if silent?
+    def self.set_debug
+      @mutex.synchronize { @debug = true }
+    end
 
-    puts prefix.colorize(color) + text.to_s.colorize(:"light_#{color}")
-  end
+    def self.unset_debug
+      @mutex.synchronize { @debug = false }
+    end
 
-  def self.info(text)
-    log('ℹ ', text, :blue)
-  end
+    def self.unset_verbose
+      @mutex.synchronize { @verbose = false }
+    end
 
-  def self.error(text)
-    log('⚠︎ ', text, :red)
-  end
+    def self.debug?
+      @mutex.synchronize { @debug }
+    end
 
-  def self.target(text)
-    log('► ', text, :green)
-  end
+    def self.verbose?
+      @mutex.synchronize { @verbose }
+    end
 
-  def self.sub_info(text)
-    log('  ● ', text, :blue)
-  end
+    def self.unset_silent
+      @mutex.synchronize { @silent = false }
+    end
 
-  def self.sub_done(text)
-    log('  ✓ ', text, :blue)
-  end
+    def self.silent?
+      @mutex.synchronize { @silent }
+    end
 
-  def self.found(text)
-    log('  ✘ ', text, :red)
-  end
+    def self.log(prefix, text, color)
+      return if silent?
+
+      puts prefix.colorize(color) + text.to_s
+    end
+
+    def self.sub_log(prefix, is_end, text, color)
+      return if silent?
+
+      indent = is_end ? '  └── ' : '  ├── '
+      puts indent.colorize(color) + prefix.colorize(color) + text.to_s
+    end
+
+    def self.debug(text)
+      log('❀ ', text, :yellow) if debug?
+    end
+
+    def self.info(text)
+      log('ℹ ', text, :blue)
+    end
+
+    def self.error(text)
+      log('⚠︎ ', text, :red)
+    end
+
+    def self.target(text)
+      log('► ', text, :green)
+    end
+
+    def self.sub_info(text)
+      log('  ● ', text, :blue)
+    end
+
+    def self.sub_complete(text)
+      sub_log('● ', true, text, :blue)
+    end
+
+    def self.found(text)
+      sub_log('✘ ', false, text, :red)
+    end
+
+    def self.verbose(text)
+      sub_log('➜ ', false, text, :yellow) if verbose?
+    end
 
-  def self.verbose(text)
-    log('  ➜ ', text, :yellow)
+    def self.verbose_ok(text)
+      sub_log('✓ ', false, text, :green) if verbose?
+    end
   end
 end

data/lib/deadfinder/runner.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'concurrent-edge'
+require 'nokogiri'
+require 'open-uri'
+require 'net/http'
+require 'openssl'
+require 'deadfinder/logger'
+require 'deadfinder/http_client'
+require 'deadfinder/url_pattern_matcher'
+
+module DeadFinder
+  # Runner class for executing the main logic
+  class Runner
+    def default_options
+      {
+        'concurrency' => 50,
+        'timeout' => 10,
+        'output' => '',
+        'output_format' => 'json',
+        'headers' => [],
+        'worker_headers' => [],
+        'silent' => true,
+        'verbose' => false,
+        'include30x' => false,
+        'proxy' => '',
+        'proxy_auth' => '',
+        'match' => '',
+        'ignore' => '',
+      }
+    end
+
+    def run(target, options)
+      DeadFinder::Logger.apply_options(options)
+      headers = options['headers'].each_with_object({}) do |header, hash|
+        kv = header.split(': ')
+        hash[kv[0]] = kv[1]
+      rescue StandardError
+      end
+      page = Nokogiri::HTML(URI.open(target, headers))
+      links = extract_links(page)
+
+      DeadFinder::Logger.debug "#{CACHE_QUE.size} URLs in queue, #{CACHE_SET.size} URLs in cache"
+
+      if options['match'] != ''
+        begin
+          links.each do |type, urls|
+        links[type] = urls.select { |url| DeadFinder::UrlPatternMatcher.match?(url, options['match']) }
+          end
+        rescue RegexpError => e
+          DeadFinder::Logger.error "Invalid match pattern: #{e.message}"
+        end
+      end
+
+      if options['ignore'] != ''
+        begin
+          links.each do |type, urls|
+            links[type] = urls.reject { |url| DeadFinder::UrlPatternMatcher.ignore?(url, options['ignore']) }
+          end
+        rescue RegexpError => e
+          DeadFinder::Logger.error "Invalid match pattern: #{e.message}"
+        end
+      end
+
+      total_links_count = links.values.flatten.length
+      link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }
+                       .compact.join(' / ')
+      DeadFinder::Logger.sub_info "Discovered #{total_links_count} URLs, currently checking them. [#{link_info}]" unless link_info.empty?
+
+      jobs = Channel.new(buffer: :buffered, capacity: 1000)
+      results = Channel.new(buffer: :buffered, capacity: 1000)
+
+      (1..options['concurrency']).each do |w|
+        Channel.go { worker(w, jobs, results, target, options) }
+      end
+
+      links.values.flatten.uniq.each do |node|
+        result = generate_url(node, target)
+        jobs << result unless result.nil?
+      end
+
+      jobs_size = jobs.size
+      jobs.close
+
+      (1..jobs_size).each { ~results }
+      DeadFinder::Logger.sub_complete 'Task completed'
+    rescue StandardError => e
+      DeadFinder::Logger.error "[#{e}] #{target}"
+    end
+
+    def worker(_id, jobs, results, target, options)
+      jobs.each do |j|
+        if CACHE_SET[j]
+          # Skip if already cached
+        else
+          CACHE_SET[j] = true
+          begin
+            CACHE_QUE[j] = true
+            uri = URI.parse(j)
+            http = HttpClient.create(uri, options)
+
+            request = Net::HTTP::Get.new(uri.request_uri)
+            request['User-Agent'] = options['user_agent']
+            options['worker_headers']&.each do |header|
+              key, value = header.split(':', 2)
+              request[key.strip] = value.strip
+            end
+
+            response = http.request(request)
+            status_code = response.code.to_i
+
+            if status_code >= 400 || (status_code >= 300 && options['include30x'])
+              DeadFinder::Logger.found "[#{status_code}] #{j}"
+              CACHE_QUE[j] = false
+              DeadFinder.output[target] ||= []
+              DeadFinder.output[target] << j
+            else
+              DeadFinder::Logger.verbose_ok "[#{status_code}] #{j}" if options['verbose']
+            end
+          rescue StandardError => e
+            DeadFinder::Logger.verbose "[#{e}] #{j}" if options['verbose']
+          end
+        end
+        results << j
+      end
+    end
+
+    private
+
+    def extract_links(page)
+      {
+        anchor: page.css('a').map { |element| element['href'] }.compact,
+        script: page.css('script').map { |element| element['src'] }.compact,
+        link: page.css('link').map { |element| element['href'] }.compact,
+        iframe: page.css('iframe').map { |element| element['src'] }.compact,
+        form: page.css('form').map { |element| element['action'] }.compact,
+        object: page.css('object').map { |element| element['data'] }.compact,
+        embed: page.css('embed').map { |element| element['src'] }.compact
+      }
+    end
+  end
+end

data/lib/deadfinder/url_pattern_matcher.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+module DeadFinder
+  # URL pattern matcher module
+  module UrlPatternMatcher
+    def self.match?(url, pattern)
+      Timeout.timeout(1) { Regexp.new(pattern).match?(url) }
+    rescue Timeout::Error
+      false
+    end
+
+    def self.ignore?(url, pattern)
+      Timeout.timeout(1) { Regexp.new(pattern).match?(url) }
+    rescue Timeout::Error
+      false
+    end
+  end
+end

data/lib/deadfinder/version.rb

@@ -1,3 +1,5 @@
 # frozen_string_literal: true
 
-VERSION = '1.6.1'
+module DeadFinder
+  VERSION = '1.7.1'
+end

data/lib/deadfinder.rb

@@ -6,6 +6,8 @@ require 'open-uri'
 require 'nokogiri'
 require 'deadfinder/utils'
 require 'deadfinder/logger'
+require 'deadfinder/runner'
+require 'deadfinder/cli'
 require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
@@ -27,165 +29,48 @@ module DeadFinder
     @output = val
   end
 
-  class Runner
-    def default_options
-      {
-        'concurrency' => 50,
-        'timeout' => 10,
-        'output' => '',
-        'output_format' => 'json',
-        'headers' => [],
-        'worker_headers' => [],
-        'silent' => true,
-        'verbose' => false,
-        'include30x' => false
-      }
-    end
-
-    def run(target, options)
-      Logger.set_silent if options['silent']
-      headers = options['headers'].each_with_object({}) do |header, hash|
-        kv = header.split(': ')
-        hash[kv[0]] = kv[1]
-      rescue StandardError
-      end
-      page = Nokogiri::HTML(URI.open(target, headers))
-      links = extract_links(page)
-
-      total_links_count = links.values.flatten.length
-      link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }
-                       .compact.join(' / ')
-      Logger.sub_info "Found #{total_links_count} URLs. [#{link_info}]" unless link_info.empty?
-      Logger.sub_info 'Checking'
-
-      jobs = Channel.new(buffer: :buffered, capacity: 1000)
-      results = Channel.new(buffer: :buffered, capacity: 1000)
-
-      (1..options['concurrency']).each do |w|
-        Channel.go { worker(w, jobs, results, target, options) }
-      end
-
-      links.values.flatten.uniq.each do |node|
-        result = generate_url(node, target)
-        jobs << result unless result.nil?
-      end
-
-      jobs_size = jobs.size
-      jobs.close
-
-      (1..jobs_size).each { ~results }
-      Logger.sub_done 'Done'
-    rescue StandardError => e
-      Logger.error "[#{e}] #{target}"
-    end
-
-    def worker(_id, jobs, results, target, options)
-      jobs.each do |j|
-        if CACHE_SET[j]
-          Logger.found "[404 Not Found] #{j}" unless CACHE_QUE[j]
-        else
-          CACHE_SET[j] = true
-          begin
-            CACHE_QUE[j] = true
-            uri = URI.parse(j)
-            proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
-            http = if proxy_uri
-                     Net::HTTP.new(uri.host, uri.port,
-                                   proxy_uri.host, proxy_uri.port,
-                                   proxy_uri.user, proxy_uri.password)
-                   else
-                     Net::HTTP.new(uri.host, uri.port)
-                   end
-            http.use_ssl = (uri.scheme == 'https')
-            http.read_timeout = options['timeout'].to_i if options['timeout']
-            http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
-
-            request = Net::HTTP::Get.new(uri.request_uri)
-            request['User-Agent'] = options['user_agent']
-            options['worker_headers']&.each do |header|
-              key, value = header.split(':', 2)
-              request[key.strip] = value.strip
-            end
-
-            response = http.request(request)
-            status_code = response.code.to_i
-            Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
-
-            if status_code >= 400 || (status_code >= 300 && options['include30x'])
-              Logger.found "[#{status_code} #{response.message}] #{j}"
-              CACHE_QUE[j] = false
-              DeadFinder.output[target] ||= []
-              DeadFinder.output[target] << j
-            end
-          rescue StandardError => e
-            Logger.verbose "[#{e}] #{j}" if options['verbose']
-          end
-        end
-        results << j
-      end
-    end
-
-    private
-
-    def extract_links(page)
-      {
-        anchor: page.css('a').map { |element| element['href'] }.compact,
-        script: page.css('script').map { |element| element['src'] }.compact,
-        link: page.css('link').map { |element| element['href'] }.compact,
-        iframe: page.css('iframe').map { |element| element['src'] }.compact,
-        form: page.css('form').map { |element| element['action'] }.compact,
-        object: page.css('object').map { |element| element['data'] }.compact,
-        embed: page.css('embed').map { |element| element['src'] }.compact
-      }
-    end
-  end
-
   def self.run_pipe(options)
-    Logger.set_silent if options['silent']
-    Logger.info 'Reading from STDIN'
-    app = Runner.new
-    while $stdin.gets
-      target = $LAST_READ_LINE.chomp
-      Logger.target "Checking: #{target}"
-      app.run target, options
-    end
-    gen_output(options)
+    run_with_input(options) { $stdin.gets&.chomp }
   end
 
   def self.run_file(filename, options)
-    Logger.set_silent if options['silent']
-    Logger.info "Reading: #{filename}"
-    app = Runner.new
-    File.foreach(filename) do |line|
-      target = line.chomp
-      Logger.target "Checking: #{target}"
-      app.run target, options
-    end
-    gen_output(options)
+    run_with_input(options) { File.foreach(filename).map(&:chomp) }
   end
 
   def self.run_url(url, options)
-    Logger.set_silent if options['silent']
-    Logger.target "Checking: #{url}"
-    app = Runner.new
-    app.run url, options
+    DeadFinder::Logger.apply_options(options)
+    run_with_target(url, options)
     gen_output(options)
   end
 
   def self.run_sitemap(sitemap_url, options)
-    Logger.set_silent if options['silent']
-    Logger.info "Parsing sitemap: #{sitemap_url}"
+    DeadFinder::Logger.apply_options(options)
     app = Runner.new
     base_uri = URI(sitemap_url)
-    sitemap = SitemapParser.new sitemap_url, { recurse: true }
+    sitemap = SitemapParser.new(sitemap_url, recurse: true)
+    DeadFinder::Logger.info "Found #{sitemap.to_a.size} URLs from #{sitemap_url}"
     sitemap.to_a.each do |url|
       turl = generate_url(url, base_uri)
-      Logger.target "Checking: #{turl}"
-      app.run turl, options
+      run_with_target(turl, options, app)
     end
     gen_output(options)
   end
 
+  def self.run_with_input(options)
+    DeadFinder::Logger.apply_options(options)
+    DeadFinder::Logger.info 'Reading input'
+    app = Runner.new
+    Array(yield).each do |target|
+      run_with_target(target, options, app)
+    end
+    gen_output(options)
+  end
+
+  def self.run_with_target(target, options, app = Runner.new)
+    DeadFinder::Logger.target "Fetching #{target}"
+    app.run(target, options)
+  end
+
   def self.gen_output(options)
     return if options['output'].empty?
 
@@ -196,12 +81,7 @@ module DeadFinder
               when 'yaml', 'yml'
                 output_data.to_yaml
               when 'csv'
-                CSV.generate do |csv|
-                  csv << %w[target url]
-                  output_data.each do |target, urls|
-                    Array(urls).each { |url| csv << [target, url] }
-                  end
-                end
+                generate_csv(output_data)
               else
                 JSON.pretty_generate(output_data)
               end
@@ -209,44 +89,12 @@ module DeadFinder
     File.write(options['output'], content)
   end
 
-  class CLI < Thor
-    class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
-    class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
-    class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
-    class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
-    class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
-    class_option :headers, aliases: :H, default: [], type: :array,
-                           desc: 'Custom HTTP headers to send with initial request'
-    class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
-    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.6.1;)', type: :string,
-                              desc: 'User-Agent string to use for requests'
-    class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
-    class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
-    class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
-
-    desc 'pipe', 'Scan the URLs from STDIN. (e.g., cat urls.txt | deadfinder pipe)'
-    def pipe
-      DeadFinder.run_pipe options
-    end
-
-    desc 'file <FILE>', 'Scan the URLs from File. (e.g., deadfinder file urls.txt)'
-    def file(filename)
-      DeadFinder.run_file filename, options
-    end
-
-    desc 'url <URL>', 'Scan the Single URL.'
-    def url(url)
-      DeadFinder.run_url url, options
-    end
-
-    desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
-    def sitemap(sitemap)
-      DeadFinder.run_sitemap sitemap, options
-    end
-
-    desc 'version', 'Show version.'
-    def version
-      Logger.info "deadfinder #{VERSION}"
+  def self.generate_csv(output_data)
+    CSV.generate do |csv|
+      csv << %w[target url]
+      output_data.each do |target, urls|
+        Array(urls).each { |url| csv << [target, url] }
+      end
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.1
 platform: ruby
 authors:
 - hahwul
 bindir: bin
 cert_chain: []
-date: 2025-02-17 00:00:00.000000000 Z
+date: 2025-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -53,42 +53,42 @@ dependencies:
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.11'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.13.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: '1.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.13.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: '1.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: open-uri
   requirement: !ruby/object:Gem::Requirement
@@ -194,7 +194,12 @@ extra_rdoc_files: []
 files:
 - bin/deadfinder
 - lib/deadfinder.rb
+- lib/deadfinder/cli.rb
+- lib/deadfinder/completion.rb
+- lib/deadfinder/http_client.rb
 - lib/deadfinder/logger.rb
+- lib/deadfinder/runner.rb
+- lib/deadfinder/url_pattern_matcher.rb
 - lib/deadfinder/utils.rb
 - lib/deadfinder/version.rb
 homepage: https://www.hahwul.com/projects/deadfinder/