deadfinder 1.2.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 694fc6108a1fde665a3e8b265071409ac483434b093c94e3eebb25324e4bb300
-  data.tar.gz: 1bd04447be853ff381263915fe2ddcaca3418699828d5e97feb674623c41c8ba
+  metadata.gz: e1e747b78068c1590c88f49acc42ca917a3c26684564c4465449181d5fea4125
+  data.tar.gz: 19f6a77d77bb0b3e2daba04ec364b1626eebbe30ec9973753fb75c1cb78ffc2e
 SHA512:
-  metadata.gz: 42eb575191bc6fa72cca03b0a94a8f20db82242c19a93bc0fcc9dfe8872c03c2cfe1accf68c480b0432f7c73df4ee0b1782ad08906d1e4bdea35dfc268d837b3
-  data.tar.gz: 39d5838080f9afd84cf3272f85a4cc6e26cf663da10b76a7e9fcfe1f63ac1b41ca973a21e454743fbc1011f9955b3c9ec488e0a586df94bdaca5e01f2ea5bec2
+  metadata.gz: ddc7a3974fcbf62d78ce739862f1686b070bfea6f26ae534c5fe18b49fb48432f7483567ab109ef4b97d40916460be1b08524dc8615d5f537564555df817c781
+  data.tar.gz: b676e1009517b67da7cff2dc10f26a50f6635d1abfeab6e036092d2b63eff55fa758c3cb0ee2b63ec34157901f00708a2d0f4282e99e5fc61057f70ad939e23f

data/lib/deadfinder/logger.rb

@@ -7,6 +7,10 @@ class Logger
     puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue)
   end
 
+  def self.error(text)
+    puts '⚠︎ '.colorize(:red) + text.to_s
+  end
+
   def self.target(text)
     puts '► '.colorize(:green) + text.to_s.colorize(:light_green)
   end

data/lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.2.0'
+VERSION = '1.2.2'

data/lib/deadfinder.rb

@@ -19,38 +19,42 @@ Output = {}
 
 class DeadFinderRunner
   def run(target, options)
-    page = Nokogiri::HTML(URI.open(target))
-
-    nodeset_a = page.css('a')
-    link_a = nodeset_a.map { |element| element['href'] }.compact
-    nodeset_script = page.css('script')
-    link_script = nodeset_script.map { |element| element['src'] }.compact
-    nodeset_link = page.css('link')
-    link_link = nodeset_link.map { |element| element['href'] }.compact
-
-    link_merged = []
-    link_merged.concat link_a, link_script, link_link
-
-    Logger.target target
-    Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
-    Logger.sub_info 'Checking'
-    jobs    = Channel.new(buffer: :buffered, capacity: 1000)
-    results = Channel.new(buffer: :buffered, capacity: 1000)
-
-    (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results, target, options) }
-    end
+    begin
+      page = Nokogiri::HTML(URI.open(target))
+
+      nodeset_a = page.css('a')
+      link_a = nodeset_a.map { |element| element['href'] }.compact
+      nodeset_script = page.css('script')
+      link_script = nodeset_script.map { |element| element['src'] }.compact
+      nodeset_link = page.css('link')
+      link_link = nodeset_link.map { |element| element['href'] }.compact
+
+      link_merged = []
+      link_merged.concat link_a, link_script, link_link
+
+      Logger.target target
+      Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
+      Logger.sub_info 'Checking'
+      jobs    = Channel.new(buffer: :buffered, capacity: 1000)
+      results = Channel.new(buffer: :buffered, capacity: 1000)
+
+      (1..options['concurrency']).each do |w|
+        Channel.go { worker(w, jobs, results, target, options) }
+      end
 
-    link_a.uniq.each do |node|
-      result = generate_url node, target
-      jobs << result
-    end
-    jobs.close
+      link_merged.uniq.each do |node|
+        result = generate_url node, target
+        jobs << result
+      end
+      jobs.close
 
-    (1..link_a.uniq.length).each do
-      ~results
+      (1..link_merged.uniq.length).each do
+        ~results
+      end
+      Logger.sub_done 'Done'
+    rescue => e 
+      Logger.error "[#{e}] #{target}"
     end
-    Logger.sub_done 'Done'
   end
 
   def worker(_id, jobs, results, target, options)
@@ -102,9 +106,11 @@ end
 
 def run_sitemap(sitemap_url, options)
   app = DeadFinderRunner.new
+  base_uri = URI(sitemap_url)
   sitemap = SitemapParser.new sitemap_url, { recurse: true }
   sitemap.to_a.each do |url|
-    app.run url, options
+    turl = generate_url url, base_uri
+    app.run turl, options
   end
   gen_output
 end
@@ -114,8 +120,8 @@ def gen_output
 end
 
 class DeadFinder < Thor
-  class_option :concurrency, aliases: :c, default: 20, type: :numeric
-  class_option :timeout, aliases: :t, default: 10, type: :numeric
+  class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Set Concurrncy'
+  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Set HTTP Timeout'
   class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.2
 platform: ruby
 authors:
 - hahwul
@@ -122,9 +122,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.6.0
-description: Dead link (broken link) means a link within a web page that cannot be
-  connected. These links can have a security negative impact with SEO. This tool makes
-  it easy to identify and modify.
+description: Find dead-links (broken links). Dead link (broken link) means a link
+  within a web page that cannot be connected. These links can have a negative impact
+  to SEO and Security. This tool makes it easy to identify and modify.
 email: hahwul@gmail.com
 executables:
 - deadfinder