deadfinder 1.6.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a9dba525c92f09cb14019fe890f73dc85eaa975179ac19efca534f0fdc76b14
-  data.tar.gz: fee3168a2912dacd9a2740b1aaf2c495b0b68b48b4d024c208523d8a8a5bb5e1
+  metadata.gz: d8c1697ba3269001737ec44650a3f27ffd73cbf5467bd110df7e09cc7fc85ba5
+  data.tar.gz: f7ac73c33b9862cdbaaa661712d4969a999d73e12cdf4e6e725f9280fe67203b
 SHA512:
-  metadata.gz: be806719c1ed5ca905885222d485f376c7e1a61dff0388f812e05d52e279e21315af08fdcfb5b66e9773c88be2f1a5c8c434eba62e6340f91db76ea29f4643db
-  data.tar.gz: d966f9237d2a4f62ba98684346b045c49bd14b024723eda22ac5f0c9c2fb0e62fc358d6607d924c0cdd80cd10e8e5f4eb37a13a067188d8394d142fe0476001b
+  metadata.gz: c59678fbca26d0caa131cc31c931ddb43712bae58f389fb8b5207eb4564904e8ed29b746645ec2802d7379be2f98e502a87207dbcf5d56bb5dfbe5d8b211448d
+  data.tar.gz: ddb8e07bda9415278724e09b559bc3e76d49c56a9bc8143d7a5e4426914c6034884a9b90bccf34982c2097d79e28d80833bf1025beee16b3d8dfe9e1b3a9f2d6

data/bin/deadfinder

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require 'deadfinder'
-DeadFinder.start(ARGV)
+DeadFinder::CLI.start(ARGV)

data/lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.6.0'
+VERSION = '1.6.1'

data/lib/deadfinder.rb

@@ -13,245 +13,240 @@ require 'json'
 require 'yaml'
 require 'csv'
 
-Channel = Concurrent::Channel
-CACHE_SET = Concurrent::Map.new
-CACHE_QUE = Concurrent::Map.new
-OUTPUT = {}
-
-class DeadFinderRunner
-  def default_options
-    {
-      'concurrency' => 50,
-      'timeout' => 10,
-      'output' => '',
-      'output_format' => 'json',
-      'headers' => [],
-      'worker_headers' => [],
-      'silent' => true,
-      'verbose' => false,
-      'include30x' => false
-    }
+module DeadFinder
+  Channel = Concurrent::Channel
+  CACHE_SET = Concurrent::Map.new
+  CACHE_QUE = Concurrent::Map.new
+
+  @output = {}
+  def self.output
+    @output
   end
 
-  def run(target, options)
-    Logger.set_silent if options['silent']
-    headers = options['headers'].each_with_object({}) do |header, hash|
-      kv = header.split(': ')
-      hash[kv[0]] = kv[1]
-    rescue StandardError
+  def self.output=(val)
+    @output = val
+  end
+
+  class Runner
+    def default_options
+      {
+        'concurrency' => 50,
+        'timeout' => 10,
+        'output' => '',
+        'output_format' => 'json',
+        'headers' => [],
+        'worker_headers' => [],
+        'silent' => true,
+        'verbose' => false,
+        'include30x' => false
+      }
     end
-    page = Nokogiri::HTML(URI.open(target, headers))
-    links = extract_links(page)
 
-    total_links_count = links.values.flatten.length
-    # Generate link info string for non-empty link types
-    link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }.compact.join(' / ')
+    def run(target, options)
+      Logger.set_silent if options['silent']
+      headers = options['headers'].each_with_object({}) do |header, hash|
+        kv = header.split(': ')
+        hash[kv[0]] = kv[1]
+      rescue StandardError
+      end
+      page = Nokogiri::HTML(URI.open(target, headers))
+      links = extract_links(page)
 
-    # Log the information if there are any links
-    Logger.sub_info "Found #{total_links_count} URLs. [#{link_info}]" unless link_info.empty?
-    Logger.sub_info 'Checking'
+      total_links_count = links.values.flatten.length
+      link_info = links.map { |type, urls| "#{type}:#{urls.length}" if urls.length.positive? }
+                       .compact.join(' / ')
+      Logger.sub_info "Found #{total_links_count} URLs. [#{link_info}]" unless link_info.empty?
+      Logger.sub_info 'Checking'
 
-    jobs = Channel.new(buffer: :buffered, capacity: 1000)
-    results = Channel.new(buffer: :buffered, capacity: 1000)
+      jobs = Channel.new(buffer: :buffered, capacity: 1000)
+      results = Channel.new(buffer: :buffered, capacity: 1000)
 
-    (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results, target, options) }
-    end
+      (1..options['concurrency']).each do |w|
+        Channel.go { worker(w, jobs, results, target, options) }
+      end
 
-    links.values.flatten.uniq.each do |node|
-      result = generate_url(node, target)
-      jobs << result unless result.nil?
-    end
+      links.values.flatten.uniq.each do |node|
+        result = generate_url(node, target)
+        jobs << result unless result.nil?
+      end
 
-    jobs_size = jobs.size
-    jobs.close
+      jobs_size = jobs.size
+      jobs.close
 
-    (1..jobs_size).each do
-      ~results
+      (1..jobs_size).each { ~results }
+      Logger.sub_done 'Done'
+    rescue StandardError => e
+      Logger.error "[#{e}] #{target}"
     end
-    Logger.sub_done 'Done'
-  rescue StandardError => e
-    Logger.error "[#{e}] #{target}"
-  end
 
-  def worker(_id, jobs, results, target, options)
-    jobs.each do |j|
-      if CACHE_SET[j]
-        Logger.found "[404 Not Found] #{j}" unless CACHE_QUE[j]
-      else
-        CACHE_SET[j] = true
-        begin
-          CACHE_QUE[j] = true
-          uri = URI.parse(j)
-
-          # Create HTTP request with timeout and headers
-          proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
-          http = if proxy_uri
-                   Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
-                 else
-                   Net::HTTP.new(uri.host, uri.port)
-                 end
-          http.use_ssl = (uri.scheme == 'https')
-          http.read_timeout = options['timeout'].to_i if options['timeout']
-
-          # Set SSL verification mode
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
-
-          request = Net::HTTP::Get.new(uri.request_uri)
-
-          # Add User-Agent header
-          request['User-Agent'] = options['user_agent']
-
-          # Add worker headers if provided
-          options['worker_headers']&.each do |header|
-            key, value = header.split(':', 2)
-            request[key.strip] = value.strip
-          end
+    def worker(_id, jobs, results, target, options)
+      jobs.each do |j|
+        if CACHE_SET[j]
+          Logger.found "[404 Not Found] #{j}" unless CACHE_QUE[j]
+        else
+          CACHE_SET[j] = true
+          begin
+            CACHE_QUE[j] = true
+            uri = URI.parse(j)
+            proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
+            http = if proxy_uri
+                     Net::HTTP.new(uri.host, uri.port,
+                                   proxy_uri.host, proxy_uri.port,
+                                   proxy_uri.user, proxy_uri.password)
+                   else
+                     Net::HTTP.new(uri.host, uri.port)
+                   end
+            http.use_ssl = (uri.scheme == 'https')
+            http.read_timeout = options['timeout'].to_i if options['timeout']
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
+
+            request = Net::HTTP::Get.new(uri.request_uri)
+            request['User-Agent'] = options['user_agent']
+            options['worker_headers']&.each do |header|
+              key, value = header.split(':', 2)
+              request[key.strip] = value.strip
+            end
 
-          response = http.request(request)
-          status_code = response.code.to_i
-          Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
+            response = http.request(request)
+            status_code = response.code.to_i
+            Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
 
-          if status_code >= 400 || (status_code >= 300 && options['include30x'])
-            Logger.found "[#{status_code} #{response.message}] #{j}"
-            CACHE_QUE[j] = false
-            OUTPUT[target] ||= []
-            OUTPUT[target] << j
+            if status_code >= 400 || (status_code >= 300 && options['include30x'])
+              Logger.found "[#{status_code} #{response.message}] #{j}"
+              CACHE_QUE[j] = false
+              DeadFinder.output[target] ||= []
+              DeadFinder.output[target] << j
+            end
+          rescue StandardError => e
+            Logger.verbose "[#{e}] #{j}" if options['verbose']
           end
-        rescue StandardError => e
-          Logger.verbose "[#{e}] #{j}" if options['verbose']
         end
+        results << j
       end
-      results << j
     end
-  end
 
-  private
-
-  def extract_links(page)
-    {
-      anchor: page.css('a').map { |element| element['href'] }.compact,
-      script: page.css('script').map { |element| element['src'] }.compact,
-      link: page.css('link').map { |element| element['href'] }.compact,
-      iframe: page.css('iframe').map { |element| element['src'] }.compact,
-      form: page.css('form').map { |element| element['action'] }.compact,
-      object: page.css('object').map { |element| element['data'] }.compact,
-      embed: page.css('embed').map { |element| element['src'] }.compact
-    }
+    private
+
+    def extract_links(page)
+      {
+        anchor: page.css('a').map { |element| element['href'] }.compact,
+        script: page.css('script').map { |element| element['src'] }.compact,
+        link: page.css('link').map { |element| element['href'] }.compact,
+        iframe: page.css('iframe').map { |element| element['src'] }.compact,
+        form: page.css('form').map { |element| element['action'] }.compact,
+        object: page.css('object').map { |element| element['data'] }.compact,
+        embed: page.css('embed').map { |element| element['src'] }.compact
+      }
+    end
   end
-end
-
-def run_pipe(options)
-  Logger.set_silent if options['silent']
 
-  Logger.info 'Reading from STDIN'
-  app = DeadFinderRunner.new
-  while $stdin.gets
-    target = $LAST_READ_LINE.chomp
-    Logger.target "Checking: #{target}"
-    app.run target, options
+  def self.run_pipe(options)
+    Logger.set_silent if options['silent']
+    Logger.info 'Reading from STDIN'
+    app = Runner.new
+    while $stdin.gets
+      target = $LAST_READ_LINE.chomp
+      Logger.target "Checking: #{target}"
+      app.run target, options
+    end
+    gen_output(options)
   end
-  gen_output(options)
-end
 
-def run_file(filename, options)
-  Logger.set_silent if options['silent']
+  def self.run_file(filename, options)
+    Logger.set_silent if options['silent']
+    Logger.info "Reading: #{filename}"
+    app = Runner.new
+    File.foreach(filename) do |line|
+      target = line.chomp
+      Logger.target "Checking: #{target}"
+      app.run target, options
+    end
+    gen_output(options)
+  end
 
-  Logger.info "Reading: #{filename}"
-  app = DeadFinderRunner.new
-  File.foreach(filename) do |line|
-    target = line.chomp
-    Logger.target "Checking: #{target}"
-    app.run target, options
+  def self.run_url(url, options)
+    Logger.set_silent if options['silent']
+    Logger.target "Checking: #{url}"
+    app = Runner.new
+    app.run url, options
+    gen_output(options)
   end
-  gen_output(options)
-end
 
-def run_url(url, options)
-  Logger.set_silent if options['silent']
+  def self.run_sitemap(sitemap_url, options)
+    Logger.set_silent if options['silent']
+    Logger.info "Parsing sitemap: #{sitemap_url}"
+    app = Runner.new
+    base_uri = URI(sitemap_url)
+    sitemap = SitemapParser.new sitemap_url, { recurse: true }
+    sitemap.to_a.each do |url|
+      turl = generate_url(url, base_uri)
+      Logger.target "Checking: #{turl}"
+      app.run turl, options
+    end
+    gen_output(options)
+  end
 
-  Logger.target "Checking: #{url}"
-  app = DeadFinderRunner.new
-  app.run url, options
-  gen_output(options)
-end
+  def self.gen_output(options)
+    return if options['output'].empty?
 
-def run_sitemap(sitemap_url, options)
-  Logger.set_silent if options['silent']
-  Logger.info "Parsing sitemap: #{sitemap_url}"
-  app = DeadFinderRunner.new
-  base_uri = URI(sitemap_url)
-  sitemap = SitemapParser.new sitemap_url, { recurse: true }
-  sitemap.to_a.each do |url|
-    turl = generate_url url, base_uri
-    Logger.target "Checking: #{turl}"
-    app.run turl, options
-  end
-  gen_output(options)
-end
+    output_data = DeadFinder.output.to_h
+    format = options['output_format'].to_s.downcase
 
-def gen_output(options)
-  return if options['output'].empty?
-
-  output_data = OUTPUT.to_h
-  format = options['output_format'].to_s.downcase
-
-  content = case format
-            when 'yaml', 'yml'
-              output_data.to_yaml
-            when 'csv'
-              CSV.generate do |csv|
-                csv << ['target', 'url']
-                output_data.each do |target, urls|
-                  Array(urls).each do |url|
-                    csv << [target, url]
+    content = case format
+              when 'yaml', 'yml'
+                output_data.to_yaml
+              when 'csv'
+                CSV.generate do |csv|
+                  csv << %w[target url]
+                  output_data.each do |target, urls|
+                    Array(urls).each { |url| csv << [target, url] }
                   end
                 end
+              else
+                JSON.pretty_generate(output_data)
               end
-            else
-              JSON.pretty_generate(output_data)
-            end
 
-  File.write(options['output'], content)
-end
-
-class DeadFinder < Thor
-  class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
-  class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
-  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
-  class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
-  class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
-  class_option :headers, aliases: :H, default: [], type: :array,
-                         desc: 'Custom HTTP headers to send with initial request'
-  class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
-  class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.6.0;)', type: :string,
-                            desc: 'User-Agent string to use for requests'
-  class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
-  class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
-  class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
-
-  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
-  def pipe
-    run_pipe options
+    File.write(options['output'], content)
   end
 
-  desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
-  def file(filename)
-    run_file filename, options
-  end
+  class CLI < Thor
+    class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
+    class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
+    class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
+    class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write result (e.g., json, yaml, csv)'
+    class_option :output_format, aliases: :f, default: 'json', type: :string, desc: 'Output format'
+    class_option :headers, aliases: :H, default: [], type: :array,
+                           desc: 'Custom HTTP headers to send with initial request'
+    class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
+    class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.6.1;)', type: :string,
+                              desc: 'User-Agent string to use for requests'
+    class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
+    class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
+    class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
+
+    desc 'pipe', 'Scan the URLs from STDIN. (e.g., cat urls.txt | deadfinder pipe)'
+    def pipe
+      DeadFinder.run_pipe options
+    end
 
-  desc 'url <URL>', 'Scan the Single URL.'
-  def url(url)
-    run_url url, options
-  end
+    desc 'file <FILE>', 'Scan the URLs from File. (e.g., deadfinder file urls.txt)'
+    def file(filename)
+      DeadFinder.run_file filename, options
+    end
 
-  desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
-  def sitemap(sitemap)
-    run_sitemap sitemap, options
-  end
+    desc 'url <URL>', 'Scan the Single URL.'
+    def url(url)
+      DeadFinder.run_url url, options
+    end
 
-  desc 'version', 'Show version.'
-  def version
-    Logger.info "deadfinder #{VERSION}"
+    desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
+    def sitemap(sitemap)
+      DeadFinder.run_sitemap sitemap, options
+    end
+
+    desc 'version', 'Show version.'
+    def version
+      Logger.info "deadfinder #{VERSION}"
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - hahwul
@@ -169,6 +169,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Find dead-links (broken links). Dead link (broken link) means a link
   within a web page that cannot be connected. These links can have a negative impact
   to SEO and Security. This tool makes it easy to identify and modify.
@@ -183,7 +197,7 @@ files:
 - lib/deadfinder/logger.rb
 - lib/deadfinder/utils.rb
 - lib/deadfinder/version.rb
-homepage: https://www.hahwul.com
+homepage: https://www.hahwul.com/projects/deadfinder/
 licenses:
 - MIT
 metadata: