deadfinder 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 248abe3bb3c321a29cb164930132c0a61ad9e3f7987a8cfb863eebe614d0350f
-  data.tar.gz: bbd932e255500d98dbd2171d5fe3fb66a820846bb5801f4b6b1b1b74905675e7
+  metadata.gz: 694fc6108a1fde665a3e8b265071409ac483434b093c94e3eebb25324e4bb300
+  data.tar.gz: 1bd04447be853ff381263915fe2ddcaca3418699828d5e97feb674623c41c8ba
 SHA512:
-  metadata.gz: 4b722245eafc6373f2776db13377cbb391ab9a4c4cffb7b226e206fb8c8f7a58d1474345a7fef41b7cc14cc350ef3549db95f3359b7df7b3bcd43d0d3d580dac
-  data.tar.gz: 5b9706ea35279e5f7c881d1839bb030ab73fb412a474a3cc9ac01837a256ff121b17154246a9ef6173a7410f29e611f52cf66dff9eff7a14c7176b27a9ab00c1
+  metadata.gz: 42eb575191bc6fa72cca03b0a94a8f20db82242c19a93bc0fcc9dfe8872c03c2cfe1accf68c480b0432f7c73df4ee0b1782ad08906d1e4bdea35dfc268d837b3
+  data.tar.gz: 39d5838080f9afd84cf3272f85a4cc6e26cf663da10b76a7e9fcfe1f63ac1b41ca973a21e454743fbc1011f9955b3c9ec488e0a586df94bdaca5e01f2ea5bec2

data/lib/deadfinder/logger.rb

@@ -15,6 +15,10 @@ class Logger
     puts '  ● '.colorize(:blue) + text.to_s.colorize(:light_blue)
   end
 
+  def self.sub_done(text)
+    puts '  ✓ '.colorize(:blue) + text.to_s.colorize(:light_blue)
+  end
+
   def self.found(text)
     puts "  ✘ #{text}".colorize(:red)
   end

data/lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.1.1'
+VERSION = '1.2.0'

data/lib/deadfinder.rb

@@ -9,22 +9,36 @@ require 'deadfinder/logger'
 require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
+require 'set'
+require 'json'
 
 Channel = Concurrent::Channel
+CacheSet = Set.new
+CacheQue = {}
+Output = {}
 
 class DeadFinderRunner
   def run(target, options)
     page = Nokogiri::HTML(URI.open(target))
-    nodeset = page.css('a')
-    link_a = nodeset.map { |element| element['href'] }.compact
+
+    nodeset_a = page.css('a')
+    link_a = nodeset_a.map { |element| element['href'] }.compact
+    nodeset_script = page.css('script')
+    link_script = nodeset_script.map { |element| element['src'] }.compact
+    nodeset_link = page.css('link')
+    link_link = nodeset_link.map { |element| element['href'] }.compact
+
+    link_merged = []
+    link_merged.concat link_a, link_script, link_link
+
     Logger.target target
-    Logger.sub_info "Found #{link_a.length} point"
+    Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
     Logger.sub_info 'Checking'
-    jobs    = Channel.new(buffer: :buffered, capacity: 100)
-    results = Channel.new(buffer: :buffered, capacity: 100)
+    jobs    = Channel.new(buffer: :buffered, capacity: 1000)
+    results = Channel.new(buffer: :buffered, capacity: 1000)
 
     (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results) }
+      Channel.go { worker(w, jobs, results, target, options) }
     end
 
     link_a.uniq.each do |node|
@@ -36,15 +50,26 @@ class DeadFinderRunner
     (1..link_a.uniq.length).each do
       ~results
     end
-    Logger.sub_info 'Done'
+    Logger.sub_done 'Done'
   end
 
-  def worker(_id, jobs, results)
+  def worker(_id, jobs, results, target, options)
     jobs.each do |j|
-      begin
-        URI.open(j)
-      rescue StandardError => e
-        Logger.found "[#{e}] #{j}" if e.to_s.include? '404 Not Found'
+      if !CacheSet.include? j
+        CacheSet.add j
+        begin
+          CacheQue[j] = true
+          URI.open(j, read_timeout: options['timeout'])
+        rescue StandardError => e
+          if e.to_s.include? '404 Not Found'
+            Logger.found "[#{e}] #{j}"
+            CacheQue[j] = false
+            Output[target] = [] if Output[target].nil?
+            Output[target].push j
+          end
+        end
+      elsif !CacheQue[j]
+        Logger.found "[404 Not Found] #{j}"
       end
       results << j
     end
@@ -57,6 +82,7 @@ def run_pipe(options)
     target = $LAST_READ_LINE.gsub("\n", '')
     app.run target, options
   end
+  gen_output
 end
 
 def run_file(filename, options)
@@ -65,11 +91,13 @@ def run_file(filename, options)
     target = line.gsub("\n", '')
     app.run target, options
   end
+  gen_output
 end
 
 def run_url(url, options)
   app = DeadFinderRunner.new
   app.run url, options
+  gen_output
 end
 
 def run_sitemap(sitemap_url, options)
@@ -78,10 +106,17 @@ def run_sitemap(sitemap_url, options)
   sitemap.to_a.each do |url|
     app.run url, options
   end
+  gen_output
+end
+
+def gen_output
+  File.write options['output'], Output.to_json if options['output'] != ''
 end
 
 class DeadFinder < Thor
   class_option :concurrency, aliases: :c, default: 20, type: :numeric
+  class_option :timeout, aliases: :t, default: 10, type: :numeric
+  class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - hahwul
@@ -94,6 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: set
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
 description: Dead link (broken link) means a link within a web page that cannot be
   connected. These links can have a security negative impact with SEO. This tool makes
   it easy to identify and modify.