deadfinder 1.5.1 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ddb8f1182cbb94fdb1c270ff6d0af2ae176fdb2376a52496fc535c69a1c576c
-  data.tar.gz: 5f49781224441d738fade165e0fae6c851c6739de2959e8c32601ebd9495e16c
+  metadata.gz: d8c1697ba3269001737ec44650a3f27ffd73cbf5467bd110df7e09cc7fc85ba5
+  data.tar.gz: f7ac73c33b9862cdbaaa661712d4969a999d73e12cdf4e6e725f9280fe67203b
 SHA512:
-  metadata.gz: 5875750e01d1b909390a566b94a4daeeee5e2df3da81528d2c2cf4f831ef92c5e1d46732bbe4425cf5b0ba751136faa9ac7cb0dffeb6beefa0a4fabfa805e2b7
-  data.tar.gz: bc62403a89307870b0166f530928f8ce1c951e131818b3ac55bc6d879e5e8273189dc120ee51740fccb3eaf2f473bc5d9bcfbb3441ae17c1793fe83f9c0de716
+  metadata.gz: c59678fbca26d0caa131cc31c931ddb43712bae58f389fb8b5207eb4564904e8ed29b746645ec2802d7379be2f98e502a87207dbcf5d56bb5dfbe5d8b211448d
+  data.tar.gz: ddb8e07bda9415278724e09b559bc3e76d49c56a9bc8143d7a5e4426914c6034884a9b90bccf34982c2097d79e28d80833bf1025beee16b3d8dfe9e1b3a9f2d6

data/bin/deadfinder

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require 'deadfinder'
-DeadFinder.start(ARGV)
+DeadFinder::CLI.start(ARGV)

data/lib/deadfinder/utils.rb

@@ -4,41 +4,24 @@ require 'uri'
 
 def generate_url(text, base_url)
   node = text.to_s
+  return node if node.start_with?('http://', 'https://')
+
   begin
-    unless node.start_with?('http://', 'https://')
-      uri = URI(base_url)
-      if node.start_with? '//'
-        return "#{uri.scheme}:#{node}"
-      elsif node.start_with? '/'
-        return "#{uri.scheme}://#{uri.host}#{node}"
-      elsif ignore_scheme? node
-        return nil
-      else
-        return "#{extract_directory(uri)}#{node}"
-      end
+    uri = URI(base_url)
+    if node.start_with?('//')
+      "#{uri.scheme}:#{node}"
+    elsif node.start_with?('/')
+      "#{uri.scheme}://#{uri.host}#{node}"
+    elsif ignore_scheme?(node)
+      nil
+    else
+      URI.join(base_url, node).to_s
     end
   rescue StandardError
-    # puts e
+    nil
   end
-  node
 end
 
 def ignore_scheme?(url)
   url.start_with?('mailto:', 'tel:', 'sms:', 'data:', 'file:')
 end
-
-def extract_directory(uri)
-  return "#{uri.scheme}://#{uri.host}#{uri.path}" if uri.path.end_with?('/')
-
-  path_components = uri.path.split('/')
-  path_components.last
-  path_components.pop
-
-  directory_path = path_components.join('/')
-
-  if directory_path.start_with?('/')
-    "#{uri.scheme}://#{uri.host}#{directory_path}/"
-  else
-    "#{uri.scheme}://#{uri.host}/#{directory_path}/"
-  end
-end

data/lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.5.1'
+VERSION = '1.6.1'

data/lib/deadfinder.rb

@@ -10,224 +10,243 @@ require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
 require 'json'
+require 'yaml'
+require 'csv'
 
-Channel = Concurrent::Channel
-CACHE_SET = Concurrent::Map.new
-CACHE_QUE = Concurrent::Map.new
-OUTPUT = {}
-
-class DeadFinderRunner
-  def default_options
-    {
-      'concurrency' => 50,
-      'timeout' => 10,
-      'output' => '',
-      'headers' => [],
-      'worker_headers' => [],
-      'silent' => true,
-      'verbose' => false,
-      'include30x' => false
-    }
+module DeadFinder
+  Channel = Concurrent::Channel
+  CACHE_SET = Concurrent::Map.new
+  CACHE_QUE = Concurrent::Map.new
+
+  @output = {}
+  def self.output
+    @output
   end
 
-  def run(target, options)
-    Logger.set_silent if options['silent']
-    headers = options['headers'].each_with_object({}) do |header, hash|
-      kv = header.split(': ')
-      hash[kv[0]] = kv[1]
-    rescue StandardError
+  def self.output=(val)
+    @output = val
+  end
+
+  class Runner
+    def default_options
+      {
+        'concurrency' => 50,
+        'timeout' => 10,
+        'output' => '',
+        'output_format' => 'json',
+        'headers' => [],
+        'worker_headers' => [],
+        'silent' => true,
+        'verbose' => false,
+        'include30x' => false
+      }
     end
-    page = Nokogiri::HTML(URI.open(target, headers))
-    links = extract_links(page)
 
-    total_links_count = links.values.flatten.length
-    # Generate link info string for non-empty link types
-    link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }.compact.join(' / ')
+    def run(target, options)
+      Logger.set_silent if options['silent']
+      headers = options['headers'].each_with_object({}) do |header, hash|
+        kv = header.split(': ')
+        hash[kv[0]] = kv[1]
+      rescue StandardError
+      end
+      page = Nokogiri::HTML(URI.open(target, headers))
+      links = extract_links(page)
 
-    # Log the information if there are any links
-    Logger.sub_info "Found #{total_links_count} URLs. [#{link_info}]" unless link_info.empty?
-    Logger.sub_info 'Checking'
+      total_links_count = links.values.flatten.length
+      link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }
+                       .compact.join(' / ')
+      Logger.sub_info "Found #{total_links_count} URLs. [#{link_info}]" unless link_info.empty?
+      Logger.sub_info 'Checking'
 
-    jobs = Channel.new(buffer: :buffered, capacity: 1000)
-    results = Channel.new(buffer: :buffered, capacity: 1000)
+      jobs = Channel.new(buffer: :buffered, capacity: 1000)
+      results = Channel.new(buffer: :buffered, capacity: 1000)
 
-    (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results, target, options) }
-    end
+      (1..options['concurrency']).each do |w|
+        Channel.go { worker(w, jobs, results, target, options) }
+      end
 
-    links.values.flatten.uniq.each do |node|
-      result = generate_url(node, target)
-      jobs << result unless result.nil?
-    end
+      links.values.flatten.uniq.each do |node|
+        result = generate_url(node, target)
+        jobs << result unless result.nil?
+      end
 
-    jobs_size = jobs.size
-    jobs.close
+      jobs_size = jobs.size
+      jobs.close
 
-    (1..jobs_size).each do
-      ~results
+      (1..jobs_size).each { ~results }
+      Logger.sub_done 'Done'
+    rescue StandardError => e
+      Logger.error "[#{e}] #{target}"
     end
-    Logger.sub_done 'Done'
-  rescue StandardError => e
-    Logger.error "[#{e}] #{target}"
-  end
 
-  def worker(_id, jobs, results, target, options)
-    jobs.each do |j|
-      if CACHE_SET[j]
-        Logger.found "[404 Not Found] #{j}" unless CACHE_QUE[j]
-      else
-        CACHE_SET[j] = true
-        begin
-          CACHE_QUE[j] = true
-          uri = URI.parse(j)
-
-          # Create HTTP request with timeout and headers
-          proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
-          http = if proxy_uri
-                   Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
-                 else
-                   Net::HTTP.new(uri.host, uri.port)
-                 end
-          http.use_ssl = (uri.scheme == 'https')
-          http.read_timeout = options['timeout'].to_i if options['timeout']
-
-          # Set SSL verification mode
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
-
-          request = Net::HTTP::Get.new(uri.request_uri)
-
-          # Add User-Agent header
-          request['User-Agent'] = options['user_agent']
-
-          # Add worker headers if provided
-          options['worker_headers']&.each do |header|
-            key, value = header.split(':', 2)
-            request[key.strip] = value.strip
+    def worker(_id, jobs, results, target, options)
+      jobs.each do |j|
+        if CACHE_SET[j]
+          Logger.found "[404 Not Found] #{j}" unless CACHE_QUE[j]
+        else
+          CACHE_SET[j] = true
+          begin
+            CACHE_QUE[j] = true
+            uri = URI.parse(j)
+            proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
+            http = if proxy_uri
+                     Net::HTTP.new(uri.host, uri.port,
+                                   proxy_uri.host, proxy_uri.port,
+                                   proxy_uri.user, proxy_uri.password)
+                   else
+                     Net::HTTP.new(uri.host, uri.port)
+                   end
+            http.use_ssl = (uri.scheme == 'https')
+            http.read_timeout = options['timeout'].to_i if options['timeout']
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
+
+            request = Net::HTTP::Get.new(uri.request_uri)
+            request['User-Agent'] = options['user_agent']
+            options['worker_headers']&.each do |header|
+              key, value = header.split(':', 2)
+              request[key.strip] = value.strip
+            end
+
+            response = http.request(request)
+            status_code = response.code.to_i
+            Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
+
+            if status_code >= 400 || (status_code >= 300 && options['include30x'])
+              Logger.found "[#{status_code} #{response.message}] #{j}"
+              CACHE_QUE[j] = false
+              DeadFinder.output[target] ||= []
+              DeadFinder.output[target] << j
+            end
+          rescue StandardError => e
+            Logger.verbose "[#{e}] #{j}" if options['verbose']
           end
-
-          response = http.request(request)
-          status_code = response.code.to_i
-          Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
-
-          if status_code >= 400 || (status_code >= 300 && options['include30x'])
-            Logger.found "[#{status_code} #{response.message}] #{j}"
-            CACHE_QUE[j] = false
-            OUTPUT[target] ||= []
-            OUTPUT[target] << j
-          end
-        rescue StandardError => e
-          Logger.verbose "[#{e}] #{j}" if options['verbose']
         end
+        results << j
       end
-      results << j
     end
-  end
 
-  private
-
-  def extract_links(page)
-    {
-      anchor: page.css('a').map { |element| element['href'] }.compact,
-      script: page.css('script').map { |element| element['src'] }.compact,
-      link: page.css('link').map { |element| element['href'] }.compact,
-      iframe: page.css('iframe').map { |element| element['src'] }.compact,
-      form: page.css('form').map { |element| element['action'] }.compact,
-      object: page.css('object').map { |element| element['data'] }.compact,
-      embed: page.css('embed').map { |element| element['src'] }.compact
-    }
+    private
+
+    def extract_links(page)
+      {
+        anchor: page.css('a').map { |element| element['href'] }.compact,
+        script: page.css('script').map { |element| element['src'] }.compact,
+        link: page.css('link').map { |element| element['href'] }.compact,
+        iframe: page.css('iframe').map { |element| element['src'] }.compact,
+        form: page.css('form').map { |element| element['action'] }.compact,
+        object: page.css('object').map { |element| element['data'] }.compact,
+        embed: page.css('embed').map { |element| element['src'] }.compact
+      }
+    end
   end
-end
 
-def run_pipe(options)
-  Logger.set_silent if options['silent']
-
-  Logger.info 'Reading from STDIN'
-  app = DeadFinderRunner.new
-  while $stdin.gets
-    target = $LAST_READ_LINE.chomp
-    Logger.target "Checking: #{target}"
-    app.run target, options
+  def self.run_pipe(options)
+    Logger.set_silent if options['silent']
+    Logger.info 'Reading from STDIN'
+    app = Runner.new
+    while $stdin.gets
+      target = $LAST_READ_LINE.chomp
+      Logger.target "Checking: #{target}"
+      app.run target, options
+    end
+    gen_output(options)
   end
-  gen_output(options)
-end
-
-def run_file(filename, options)
-  Logger.set_silent if options['silent']
 
-  Logger.info "Reading: #{filename}"
-  app = DeadFinderRunner.new
-  File.foreach(filename) do |line|
-    target = line.chomp
-    Logger.target "Checking: #{target}"
-    app.run target, options
+  def self.run_file(filename, options)
+    Logger.set_silent if options['silent']
+    Logger.info "Reading: #{filename}"
+    app = Runner.new
+    File.foreach(filename) do |line|
+      target = line.chomp
+      Logger.target "Checking: #{target}"
+      app.run target, options
+    end
+    gen_output(options)
   end
-  gen_output(options)
-end
-
-def run_url(url, options)
-  Logger.set_silent if options['silent']
-
-  Logger.target "Checking: #{url}"
-  app = DeadFinderRunner.new
-  app.run url, options
-  gen_output(options)
-end
 
-def run_sitemap(sitemap_url, options)
-  Logger.set_silent if options['silent']
-  Logger.info "Parsing sitemap: #{sitemap_url}"
-  app = DeadFinderRunner.new
-  base_uri = URI(sitemap_url)
-  sitemap = SitemapParser.new sitemap_url, { recurse: true }
-  sitemap.to_a.each do |url|
-    turl = generate_url url, base_uri
-    Logger.target "Checking: #{turl}"
-    app.run turl, options
+  def self.run_url(url, options)
+    Logger.set_silent if options['silent']
+    Logger.target "Checking: #{url}"
+    app = Runner.new
+    app.run url, options
+    gen_output(options)
   end
-  gen_output(options)
-end
 
-def gen_output(options)
-  output_data = OUTPUT.to_h
-  File.write(options['output'], JSON.pretty_generate(output_data)) unless options['output'].empty?
-end
-
-class DeadFinder < Thor
-  class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
-  class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
-  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
-  class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
-  class_option :headers, aliases: :H, default: [], type: :array,
-                         desc: 'Custom HTTP headers to send with initial request'
-  class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
-  class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.5.1;)', type: :string,
-                            desc: 'User-Agent string to use for requests'
-  class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
-  class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
-  class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
-
-  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
-  def pipe
-    run_pipe options
+  def self.run_sitemap(sitemap_url, options)
+    Logger.set_silent if options['silent']
+    Logger.info "Parsing sitemap: #{sitemap_url}"
+    app = Runner.new
+    base_uri = URI(sitemap_url)
+    sitemap = SitemapParser.new sitemap_url, { recurse: true }
+    sitemap.to_a.each do |url|
+      turl = generate_url(url, base_uri)
+      Logger.target "Checking: #{turl}"
+      app.run turl, options
+    end
+    gen_output(options)
   end
 
-  desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
-  def file(filename)
-    run_file filename, options
+  def self.gen_output(options)
+    return if options['output'].empty?
+
+    output_data = DeadFinder.output.to_h
+    format = options['output_format'].to_s.downcase
+
+    content = case format
+              when 'yaml', 'yml'
+                output_data.to_yaml
+              when 'csv'
+                CSV.generate do |csv|
+                  csv << %w[target url]
+                  output_data.each do |target, urls|
+                    Array(urls).each { |url| csv << [target, url] }
+                  end
+                end
+              else
+                JSON.pretty_generate(output_data)
+              end
+
+    File.write(options['output'], content)
   end
 
-  desc 'url <URL>', 'Scan the Single URL.'
-  def url(url)
-    run_url url, options
-  end
+  class CLI < Thor
+    class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
+    class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
+    class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
+    class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
+    class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
+    class_option :headers, aliases: :H, default: [], type: :array,
+                           desc: 'Custom HTTP headers to send with initial request'
+    class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
+    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.6.1;)', type: :string,
+                              desc: 'User-Agent string to use for requests'
+    class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
+    class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
+    class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
+
+    desc 'pipe', 'Scan the URLs from STDIN. (e.g., cat urls.txt | deadfinder pipe)'
+    def pipe
+      DeadFinder.run_pipe options
+    end
 
-  desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
-  def sitemap(sitemap)
-    run_sitemap sitemap, options
-  end
+    desc 'file <FILE>', 'Scan the URLs from File. (e.g., deadfinder file urls.txt)'
+    def file(filename)
+      DeadFinder.run_file filename, options
+    end
 
-  desc 'version', 'Show version.'
-  def version
-    Logger.info "deadfinder #{VERSION}"
+    desc 'url <URL>', 'Scan the Single URL.'
+    def url(url)
+      DeadFinder.run_url url, options
+    end
+
+    desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
+    def sitemap(sitemap)
+      DeadFinder.run_sitemap sitemap, options
+    end
+
+    desc 'version', 'Show version.'
+    def version
+      Logger.info "deadfinder #{VERSION}"
+    end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.6.1
 platform: ruby
 authors:
 - hahwul
 bindir: bin
 cert_chain: []
-date: 2024-12-26 00:00:00.000000000 Z
+date: 2025-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -169,6 +169,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Find dead-links (broken links). Dead link (broken link) means a link
   within a web page that cannot be connected. These links can have a negative impact
   to SEO and Security. This tool makes it easy to identify and modify.
@@ -183,7 +197,7 @@ files:
 - lib/deadfinder/logger.rb
 - lib/deadfinder/utils.rb
 - lib/deadfinder/version.rb
-homepage: https://www.hahwul.com
+homepage: https://www.hahwul.com/projects/deadfinder/
 licenses:
 - MIT
 metadata:
@@ -203,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Find dead-links (broken links)
 test_files: []