deadfinder 1.2.1 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cccf1eac41e7cb13f5391bd8191931b6225f3b16dd23a2e4432816d93730be2e
4
- data.tar.gz: dddb0d51bc6b00507ae8622945114d373e7366714a74ebc4ede95057db1224f2
3
+ metadata.gz: 0b18bed09ea30afe6efed9e0795f774bb8e97a5658c9d1c1092e625b2950345a
4
+ data.tar.gz: c5c9010c7d30764b559fa3bddffc3e737c87797aa046d50cb5ad84d74766137b
5
5
  SHA512:
6
- metadata.gz: d273d3d94cce7a98185dfda1713b43ddf632b19b11bff3f93c1c68bdbf3306c65825411b95f2b4ab3a517bfb88ca3db2f888a4b4a84814058a030b7c62098a0c
7
- data.tar.gz: ca7d125f5c7a89275672418546e6913af1765bdc0eaaa8ce324be94c24e4b84989df9ec4fae371d6a4a7a160c97be47db2633b8de04be0eb98dfdc4ea2009978
6
+ metadata.gz: 02a6a785560cb6e781b6a2c4f375e1daccbf0f4f00e5b4ee97e2d5e787b0fcb381013361aba96d618dea1e78f2b92f1290b60720e0c976d1fe57720100015169
7
+ data.tar.gz: 2eb418ff24d686ef057dec9811ea63a973199ab814a231221325f9908a2f83f5b19dc99cc73f96f5b0b50da852324352a31907130fcf88a02c94f33455203521
@@ -7,6 +7,10 @@ class Logger
7
7
  puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue)
8
8
  end
9
9
 
10
+ def self.error(text)
11
+ puts '⚠︎ '.colorize(:red) + text.to_s
12
+ end
13
+
10
14
  def self.target(text)
11
15
  puts '► '.colorize(:green) + text.to_s.colorize(:light_green)
12
16
  end
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- VERSION = '1.2.1'
3
+ VERSION = '1.2.3'
data/lib/deadfinder.rb CHANGED
@@ -19,38 +19,42 @@ Output = {}
19
19
 
20
20
  class DeadFinderRunner
21
21
  def run(target, options)
22
- page = Nokogiri::HTML(URI.open(target))
23
-
24
- nodeset_a = page.css('a')
25
- link_a = nodeset_a.map { |element| element['href'] }.compact
26
- nodeset_script = page.css('script')
27
- link_script = nodeset_script.map { |element| element['src'] }.compact
28
- nodeset_link = page.css('link')
29
- link_link = nodeset_link.map { |element| element['href'] }.compact
30
-
31
- link_merged = []
32
- link_merged.concat link_a, link_script, link_link
33
-
34
- Logger.target target
35
- Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
36
- Logger.sub_info 'Checking'
37
- jobs = Channel.new(buffer: :buffered, capacity: 1000)
38
- results = Channel.new(buffer: :buffered, capacity: 1000)
39
-
40
- (1..options['concurrency']).each do |w|
41
- Channel.go { worker(w, jobs, results, target, options) }
42
- end
22
+ begin
23
+ page = Nokogiri::HTML(URI.open(target))
24
+
25
+ nodeset_a = page.css('a')
26
+ link_a = nodeset_a.map { |element| element['href'] }.compact
27
+ nodeset_script = page.css('script')
28
+ link_script = nodeset_script.map { |element| element['src'] }.compact
29
+ nodeset_link = page.css('link')
30
+ link_link = nodeset_link.map { |element| element['href'] }.compact
31
+
32
+ link_merged = []
33
+ link_merged.concat link_a, link_script, link_link
34
+
35
+ Logger.target target
36
+ Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
37
+ Logger.sub_info 'Checking'
38
+ jobs = Channel.new(buffer: :buffered, capacity: 1000)
39
+ results = Channel.new(buffer: :buffered, capacity: 1000)
40
+
41
+ (1..options['concurrency']).each do |w|
42
+ Channel.go { worker(w, jobs, results, target, options) }
43
+ end
43
44
 
44
- link_a.uniq.each do |node|
45
- result = generate_url node, target
46
- jobs << result
47
- end
48
- jobs.close
45
+ link_merged.uniq.each do |node|
46
+ result = generate_url node, target
47
+ jobs << result
48
+ end
49
+ jobs.close
49
50
 
50
- (1..link_a.uniq.length).each do
51
- ~results
51
+ (1..link_merged.uniq.length).each do
52
+ ~results
53
+ end
54
+ Logger.sub_done 'Done'
55
+ rescue => e
56
+ Logger.error "[#{e}] #{target}"
52
57
  end
53
- Logger.sub_done 'Done'
54
58
  end
55
59
 
56
60
  def worker(_id, jobs, results, target, options)
@@ -102,9 +106,11 @@ end
102
106
 
103
107
  def run_sitemap(sitemap_url, options)
104
108
  app = DeadFinderRunner.new
109
+ base_uri = URI(sitemap_url)
105
110
  sitemap = SitemapParser.new sitemap_url, { recurse: true }
106
111
  sitemap.to_a.each do |url|
107
- app.run url, options
112
+ turl = generate_url url, base_uri
113
+ app.run turl, options
108
114
  end
109
115
  gen_output
110
116
  end
@@ -124,19 +130,19 @@ class DeadFinder < Thor
124
130
  run_pipe options
125
131
  end
126
132
 
127
- desc 'file', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
133
+ desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
128
134
  def file(filename)
129
135
  Logger.info 'File mode'
130
136
  run_file filename, options
131
137
  end
132
138
 
133
- desc 'url', 'Scan the Single URL.'
139
+ desc 'url <URL>', 'Scan the Single URL.'
134
140
  def url(url)
135
141
  Logger.info 'Single URL mode'
136
142
  run_url url, options
137
143
  end
138
144
 
139
- desc 'sitemap', 'Scan the URLs from sitemap.'
145
+ desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
140
146
  def sitemap(sitemap)
141
147
  Logger.info 'Sitemap mode'
142
148
  run_sitemap sitemap, options
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deadfinder
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - hahwul
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-09-26 00:00:00.000000000 Z
11
+ date: 2022-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: colorize