deadfinder 1.2.4 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/deadfinder/version.rb +1 -1
- data/lib/deadfinder.rb +43 -31
- metadata +20 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1f66dee5b8a32028d31fbd56996f59d2c388673ba4b5220135a8b7193f43917a
|
4
|
+
data.tar.gz: 5e3afcafadd5199573614a4de74b446bca6df3a6edb2272459989a468eace160
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d040d1d57abf50e7eee69a67be3833ba33775f92b51dbc6efc02817cdf7fa7f21f285513f787c7de7a7e7881e4caef9e130bcfc5f072fe73152487bca223490d
|
7
|
+
data.tar.gz: d4b85b1a4568fc2279cd68ac82df4df7c3918d28656b6839d16c6348128b47179868fecbbb42f3ae12da84f280e027390f87beddc20a03878f5c8599f1436317
|
data/lib/deadfinder/version.rb
CHANGED
data/lib/deadfinder.rb
CHANGED
@@ -19,42 +19,53 @@ Output = {}
|
|
19
19
|
|
20
20
|
class DeadFinderRunner
|
21
21
|
def run(target, options)
|
22
|
-
|
22
|
+
page = nil
|
23
|
+
|
24
|
+
if options['headers'].length.positive?
|
25
|
+
headers = {}
|
26
|
+
options['headers'].each do |header|
|
27
|
+
kv = header.split ': '
|
28
|
+
headers[kv[0]] = kv[1]
|
29
|
+
rescue StandardError
|
30
|
+
end
|
31
|
+
|
32
|
+
page = Nokogiri::HTML(URI.open(target, headers))
|
33
|
+
else
|
23
34
|
page = Nokogiri::HTML(URI.open(target))
|
35
|
+
end
|
24
36
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
37
|
+
nodeset_a = page.css('a')
|
38
|
+
link_a = nodeset_a.map { |element| element['href'] }.compact
|
39
|
+
nodeset_script = page.css('script')
|
40
|
+
link_script = nodeset_script.map { |element| element['src'] }.compact
|
41
|
+
nodeset_link = page.css('link')
|
42
|
+
link_link = nodeset_link.map { |element| element['href'] }.compact
|
31
43
|
|
32
|
-
|
33
|
-
|
44
|
+
link_merged = []
|
45
|
+
link_merged.concat link_a, link_script, link_link
|
34
46
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
47
|
+
Logger.target target
|
48
|
+
Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
|
49
|
+
Logger.sub_info 'Checking'
|
50
|
+
jobs = Channel.new(buffer: :buffered, capacity: 1000)
|
51
|
+
results = Channel.new(buffer: :buffered, capacity: 1000)
|
40
52
|
|
41
|
-
|
42
|
-
|
43
|
-
|
53
|
+
(1..options['concurrency']).each do |w|
|
54
|
+
Channel.go { worker(w, jobs, results, target, options) }
|
55
|
+
end
|
44
56
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
57
|
+
link_merged.uniq.each do |node|
|
58
|
+
result = generate_url node, target
|
59
|
+
jobs << result
|
60
|
+
end
|
61
|
+
jobs.close
|
50
62
|
|
51
|
-
|
52
|
-
|
53
|
-
end
|
54
|
-
Logger.sub_done 'Done'
|
55
|
-
rescue => e
|
56
|
-
Logger.error "[#{e}] #{target}"
|
63
|
+
(1..link_merged.uniq.length).each do
|
64
|
+
~results
|
57
65
|
end
|
66
|
+
Logger.sub_done 'Done'
|
67
|
+
rescue StandardError => e
|
68
|
+
Logger.error "[#{e}] #{target}"
|
58
69
|
end
|
59
70
|
|
60
71
|
def worker(_id, jobs, results, target, options)
|
@@ -120,9 +131,10 @@ def gen_output
|
|
120
131
|
end
|
121
132
|
|
122
133
|
class DeadFinder < Thor
|
123
|
-
class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: '
|
124
|
-
class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: '
|
125
|
-
class_option :output, aliases: :o, default: '', type: :string, desc: '
|
134
|
+
class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Number of concurrncy'
|
135
|
+
class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
|
136
|
+
class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
|
137
|
+
class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
|
126
138
|
|
127
139
|
desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
|
128
140
|
def pipe
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deadfinder
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hahwul
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-10-
|
11
|
+
date: 2022-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: colorize
|
@@ -39,89 +39,89 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.6.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: json
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 2.6.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 2.6.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: nokogiri
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 1.13.0
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 1.13.0
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: open-uri
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 0.
|
75
|
+
version: 0.2.0
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: 0.
|
82
|
+
version: 0.2.0
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: set
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 1.
|
89
|
+
version: 1.0.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: 1.
|
96
|
+
version: 1.0.0
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
98
|
+
name: sitemap-parser
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
103
|
+
version: 0.5.0
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
110
|
+
version: 0.5.0
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
112
|
+
name: thor
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
115
|
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 2.
|
117
|
+
version: 1.2.0
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 2.
|
124
|
+
version: 1.2.0
|
125
125
|
description: Find dead-links (broken links). Dead link (broken link) means a link
|
126
126
|
within a web page that cannot be connected. These links can have a negative impact
|
127
127
|
to SEO and Security. This tool makes it easy to identify and modify.
|