deadfinder 1.1.2 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e08daa4368e4c3e041e642f1708619637ea618c2937c6e36f2aba841f152ec4
-  data.tar.gz: 5162fb9eb973b7638ece6bcac4e6b2c731f3a6bd654f842f1aeeeed286cee4ed
+  metadata.gz: cccf1eac41e7cb13f5391bd8191931b6225f3b16dd23a2e4432816d93730be2e
+  data.tar.gz: dddb0d51bc6b00507ae8622945114d373e7366714a74ebc4ede95057db1224f2
 SHA512:
-  metadata.gz: aa82c3b87b81f8f5293819dbd0e242e57e7fda14fe8820197aa4e32d6c774ea0c379adda7ef49e88865fc00f32cb5cce36c6fae711ac019e2d5be921e1747d65
-  data.tar.gz: 952de90ed7b150e106fe33ec203140c162f2f74c941400328fc3da9c41866111f940043b53d9560903fdb822337ff8973bfc70740ad937cdc04428e5598e37fb
+  metadata.gz: d273d3d94cce7a98185dfda1713b43ddf632b19b11bff3f93c1c68bdbf3306c65825411b95f2b4ab3a517bfb88ca3db2f888a4b4a84814058a030b7c62098a0c
+  data.tar.gz: ca7d125f5c7a89275672418546e6913af1765bdc0eaaa8ce324be94c24e4b84989df9ec4fae371d6a4a7a160c97be47db2633b8de04be0eb98dfdc4ea2009978

data/lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.1.2'
+VERSION = '1.2.1'

data/lib/deadfinder.rb

@@ -10,10 +10,12 @@ require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
 require 'set'
+require 'json'
 
 Channel = Concurrent::Channel
 CacheSet = Set.new
 CacheQue = {}
+Output = {}
 
 class DeadFinderRunner
   def run(target, options)
@@ -27,7 +29,7 @@ class DeadFinderRunner
     link_link = nodeset_link.map { |element| element['href'] }.compact
 
     link_merged = []
-    link_merged = link_merged.concat link_a, link_script, link_link
+    link_merged.concat link_a, link_script, link_link
 
     Logger.target target
     Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
@@ -36,7 +38,7 @@ class DeadFinderRunner
     results = Channel.new(buffer: :buffered, capacity: 1000)
 
     (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results, options) }
+      Channel.go { worker(w, jobs, results, target, options) }
     end
 
     link_a.uniq.each do |node|
@@ -51,24 +53,25 @@ class DeadFinderRunner
     Logger.sub_done 'Done'
   end
 
-  def worker(_id, jobs, results, options)
+  def worker(_id, jobs, results, target, options)
     jobs.each do |j|
       if !CacheSet.include? j
         CacheSet.add j
         begin
           CacheQue[j] = true
-          URI.open(j, :read_timeout => options['timeout'])
+          URI.open(j, read_timeout: options['timeout'])
         rescue StandardError => e
-          Logger.found "[#{e}] #{j}" if e.to_s.include? '404 Not Found'
-          CacheQue[j] = false
+          if e.to_s.include? '404 Not Found'
+            Logger.found "[#{e}] #{j}"
+            CacheQue[j] = false
+            Output[target] = [] if Output[target].nil?
+            Output[target].push j
+          end
         end
-        results << j
-      else 
-        if !CacheQue[j] 
-          Logger.found "[404 Not Found] #{j}"
-        end
-        results << j
+      elsif !CacheQue[j]
+        Logger.found "[404 Not Found] #{j}"
       end
+      results << j
     end
   end
 end
@@ -79,6 +82,7 @@ def run_pipe(options)
     target = $LAST_READ_LINE.gsub("\n", '')
     app.run target, options
   end
+  gen_output
 end
 
 def run_file(filename, options)
@@ -87,11 +91,13 @@ def run_file(filename, options)
     target = line.gsub("\n", '')
     app.run target, options
   end
+  gen_output
 end
 
 def run_url(url, options)
   app = DeadFinderRunner.new
   app.run url, options
+  gen_output
 end
 
 def run_sitemap(sitemap_url, options)
@@ -100,11 +106,17 @@ def run_sitemap(sitemap_url, options)
   sitemap.to_a.each do |url|
     app.run url, options
   end
+  gen_output
+end
+
+def gen_output
+  File.write options['output'], Output.to_json if options['output'] != ''
 end
 
 class DeadFinder < Thor
-  class_option :concurrency, aliases: :c, default: 20, type: :numeric
-  class_option :timeout, aliases: :t, default: 10, type: :numeric
+  class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Set Concurrncy'
+  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Set HTTP Timeout'
+  class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.1
 platform: ruby
 authors:
 - hahwul
@@ -94,9 +94,37 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
-description: Dead link (broken link) means a link within a web page that cannot be
-  connected. These links can have a security negative impact with SEO. This tool makes
-  it easy to identify and modify.
+- !ruby/object:Gem::Dependency
+  name: set
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+description: Find dead-links (broken links). Dead link (broken link) means a link
+  within a web page that cannot be connected. These links can have a negative impact
+  to SEO and Security. This tool makes it easy to identify and modify.
 email: hahwul@gmail.com
 executables:
 - deadfinder