deadfinder 1.2.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e6ecbd2b8b19644f0e44e876216650b464b148fa60ba78d60c67f6c338139cb
-  data.tar.gz: fd35d65a065c726eb47011d6416601d7c62fa21f37a37a8fa44018291c0ec940
+  metadata.gz: 1f66dee5b8a32028d31fbd56996f59d2c388673ba4b5220135a8b7193f43917a
+  data.tar.gz: 5e3afcafadd5199573614a4de74b446bca6df3a6edb2272459989a468eace160
 SHA512:
-  metadata.gz: 99ec928fa2c7a24888f6c02e82039caddfecc7d9912055b5c5b82fb66be20cd91a1cb6f194afb1dbdd0a063fa33d048955c64d95363ff1698156f5ad69157cef
-  data.tar.gz: 8a85ad4a0c74c06b934295f5c0ccb08e8e915e688b04c6c3bf54df998a547f1fcb0cdccc96229888e09c7d893ab0a267bb4351e2eea2feb678f2a48747082be1
+  metadata.gz: d040d1d57abf50e7eee69a67be3833ba33775f92b51dbc6efc02817cdf7fa7f21f285513f787c7de7a7e7881e4caef9e130bcfc5f072fe73152487bca223490d
+  data.tar.gz: d4b85b1a4568fc2279cd68ac82df4df7c3918d28656b6839d16c6348128b47179868fecbbb42f3ae12da84f280e027390f87beddc20a03878f5c8599f1436317

data/lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.2.4'
+VERSION = '1.3.0'

data/lib/deadfinder.rb

@@ -19,42 +19,53 @@ Output = {}
 
 class DeadFinderRunner
   def run(target, options)
-    begin
+    page = nil
+
+    if options['headers'].length.positive?
+      headers = {}
+      options['headers'].each do |header|
+        kv = header.split ': '
+        headers[kv[0]] = kv[1]
+      rescue StandardError
+      end
+
+      page = Nokogiri::HTML(URI.open(target, headers))
+    else
       page = Nokogiri::HTML(URI.open(target))
+    end
 
-      nodeset_a = page.css('a')
-      link_a = nodeset_a.map { |element| element['href'] }.compact
-      nodeset_script = page.css('script')
-      link_script = nodeset_script.map { |element| element['src'] }.compact
-      nodeset_link = page.css('link')
-      link_link = nodeset_link.map { |element| element['href'] }.compact
+    nodeset_a = page.css('a')
+    link_a = nodeset_a.map { |element| element['href'] }.compact
+    nodeset_script = page.css('script')
+    link_script = nodeset_script.map { |element| element['src'] }.compact
+    nodeset_link = page.css('link')
+    link_link = nodeset_link.map { |element| element['href'] }.compact
 
-      link_merged = []
-      link_merged.concat link_a, link_script, link_link
+    link_merged = []
+    link_merged.concat link_a, link_script, link_link
 
-      Logger.target target
-      Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
-      Logger.sub_info 'Checking'
-      jobs    = Channel.new(buffer: :buffered, capacity: 1000)
-      results = Channel.new(buffer: :buffered, capacity: 1000)
+    Logger.target target
+    Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
+    Logger.sub_info 'Checking'
+    jobs    = Channel.new(buffer: :buffered, capacity: 1000)
+    results = Channel.new(buffer: :buffered, capacity: 1000)
 
-      (1..options['concurrency']).each do |w|
-        Channel.go { worker(w, jobs, results, target, options) }
-      end
+    (1..options['concurrency']).each do |w|
+      Channel.go { worker(w, jobs, results, target, options) }
+    end
 
-      link_merged.uniq.each do |node|
-        result = generate_url node, target
-        jobs << result
-      end
-      jobs.close
+    link_merged.uniq.each do |node|
+      result = generate_url node, target
+      jobs << result
+    end
+    jobs.close
 
-      (1..link_merged.uniq.length).each do
-        ~results
-      end
-      Logger.sub_done 'Done'
-    rescue => e 
-      Logger.error "[#{e}] #{target}"
+    (1..link_merged.uniq.length).each do
+      ~results
     end
+    Logger.sub_done 'Done'
+  rescue StandardError => e
+    Logger.error "[#{e}] #{target}"
   end
 
   def worker(_id, jobs, results, target, options)
@@ -120,9 +131,10 @@ def gen_output
 end
 
 class DeadFinder < Thor
-  class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Set Concurrncy'
-  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Set HTTP Timeout'
-  class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
+  class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Number of concurrncy'
+  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
+  class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
+  class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.3.0
 platform: ruby
 authors:
 - hahwul
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-09 00:00:00.000000000 Z
+date: 2022-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -39,89 +39,89 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.6.0
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
-  name: open-uri
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.13.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
-  name: sitemap-parser
+  name: open-uri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: set
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: set
+  name: sitemap-parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
-  name: json
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 1.2.0
 description: Find dead-links (broken links). Dead link (broken link) means a link
   within a web page that cannot be connected. These links can have a negative impact
   to SEO and Security. This tool makes it easy to identify and modify.