deadfinder 1.2.1 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/deadfinder/logger.rb +4 -0
- data/lib/deadfinder/version.rb +1 -1
- data/lib/deadfinder.rb +39 -33
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b18bed09ea30afe6efed9e0795f774bb8e97a5658c9d1c1092e625b2950345a
|
4
|
+
data.tar.gz: c5c9010c7d30764b559fa3bddffc3e737c87797aa046d50cb5ad84d74766137b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 02a6a785560cb6e781b6a2c4f375e1daccbf0f4f00e5b4ee97e2d5e787b0fcb381013361aba96d618dea1e78f2b92f1290b60720e0c976d1fe57720100015169
|
7
|
+
data.tar.gz: 2eb418ff24d686ef057dec9811ea63a973199ab814a231221325f9908a2f83f5b19dc99cc73f96f5b0b50da852324352a31907130fcf88a02c94f33455203521
|
data/lib/deadfinder/logger.rb
CHANGED
data/lib/deadfinder/version.rb
CHANGED
data/lib/deadfinder.rb
CHANGED
@@ -19,38 +19,42 @@ Output = {}
|
|
19
19
|
|
20
20
|
class DeadFinderRunner
|
21
21
|
def run(target, options)
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
22
|
+
begin
|
23
|
+
page = Nokogiri::HTML(URI.open(target))
|
24
|
+
|
25
|
+
nodeset_a = page.css('a')
|
26
|
+
link_a = nodeset_a.map { |element| element['href'] }.compact
|
27
|
+
nodeset_script = page.css('script')
|
28
|
+
link_script = nodeset_script.map { |element| element['src'] }.compact
|
29
|
+
nodeset_link = page.css('link')
|
30
|
+
link_link = nodeset_link.map { |element| element['href'] }.compact
|
31
|
+
|
32
|
+
link_merged = []
|
33
|
+
link_merged.concat link_a, link_script, link_link
|
34
|
+
|
35
|
+
Logger.target target
|
36
|
+
Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
|
37
|
+
Logger.sub_info 'Checking'
|
38
|
+
jobs = Channel.new(buffer: :buffered, capacity: 1000)
|
39
|
+
results = Channel.new(buffer: :buffered, capacity: 1000)
|
40
|
+
|
41
|
+
(1..options['concurrency']).each do |w|
|
42
|
+
Channel.go { worker(w, jobs, results, target, options) }
|
43
|
+
end
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
45
|
+
link_merged.uniq.each do |node|
|
46
|
+
result = generate_url node, target
|
47
|
+
jobs << result
|
48
|
+
end
|
49
|
+
jobs.close
|
49
50
|
|
50
|
-
|
51
|
-
|
51
|
+
(1..link_merged.uniq.length).each do
|
52
|
+
~results
|
53
|
+
end
|
54
|
+
Logger.sub_done 'Done'
|
55
|
+
rescue => e
|
56
|
+
Logger.error "[#{e}] #{target}"
|
52
57
|
end
|
53
|
-
Logger.sub_done 'Done'
|
54
58
|
end
|
55
59
|
|
56
60
|
def worker(_id, jobs, results, target, options)
|
@@ -102,9 +106,11 @@ end
|
|
102
106
|
|
103
107
|
def run_sitemap(sitemap_url, options)
|
104
108
|
app = DeadFinderRunner.new
|
109
|
+
base_uri = URI(sitemap_url)
|
105
110
|
sitemap = SitemapParser.new sitemap_url, { recurse: true }
|
106
111
|
sitemap.to_a.each do |url|
|
107
|
-
|
112
|
+
turl = generate_url url, base_uri
|
113
|
+
app.run turl, options
|
108
114
|
end
|
109
115
|
gen_output
|
110
116
|
end
|
@@ -124,19 +130,19 @@ class DeadFinder < Thor
|
|
124
130
|
run_pipe options
|
125
131
|
end
|
126
132
|
|
127
|
-
desc 'file', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
|
133
|
+
desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
|
128
134
|
def file(filename)
|
129
135
|
Logger.info 'File mode'
|
130
136
|
run_file filename, options
|
131
137
|
end
|
132
138
|
|
133
|
-
desc 'url', 'Scan the Single URL.'
|
139
|
+
desc 'url <URL>', 'Scan the Single URL.'
|
134
140
|
def url(url)
|
135
141
|
Logger.info 'Single URL mode'
|
136
142
|
run_url url, options
|
137
143
|
end
|
138
144
|
|
139
|
-
desc 'sitemap', 'Scan the URLs from sitemap.'
|
145
|
+
desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
|
140
146
|
def sitemap(sitemap)
|
141
147
|
Logger.info 'Sitemap mode'
|
142
148
|
run_sitemap sitemap, options
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deadfinder
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hahwul
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-09
|
11
|
+
date: 2022-10-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: colorize
|