deadfinder 1.1.2 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/deadfinder/version.rb +1 -1
- data/lib/deadfinder.rb +24 -12
- metadata +29 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 694fc6108a1fde665a3e8b265071409ac483434b093c94e3eebb25324e4bb300
|
4
|
+
data.tar.gz: 1bd04447be853ff381263915fe2ddcaca3418699828d5e97feb674623c41c8ba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42eb575191bc6fa72cca03b0a94a8f20db82242c19a93bc0fcc9dfe8872c03c2cfe1accf68c480b0432f7c73df4ee0b1782ad08906d1e4bdea35dfc268d837b3
|
7
|
+
data.tar.gz: 39d5838080f9afd84cf3272f85a4cc6e26cf663da10b76a7e9fcfe1f63ac1b41ca973a21e454743fbc1011f9955b3c9ec488e0a586df94bdaca5e01f2ea5bec2
|
data/lib/deadfinder/version.rb
CHANGED
data/lib/deadfinder.rb
CHANGED
@@ -10,10 +10,12 @@ require 'deadfinder/version'
|
|
10
10
|
require 'concurrent-edge'
|
11
11
|
require 'sitemap-parser'
|
12
12
|
require 'set'
|
13
|
+
require 'json'
|
13
14
|
|
14
15
|
Channel = Concurrent::Channel
|
15
16
|
CacheSet = Set.new
|
16
17
|
CacheQue = {}
|
18
|
+
Output = {}
|
17
19
|
|
18
20
|
class DeadFinderRunner
|
19
21
|
def run(target, options)
|
@@ -27,7 +29,7 @@ class DeadFinderRunner
|
|
27
29
|
link_link = nodeset_link.map { |element| element['href'] }.compact
|
28
30
|
|
29
31
|
link_merged = []
|
30
|
-
link_merged
|
32
|
+
link_merged.concat link_a, link_script, link_link
|
31
33
|
|
32
34
|
Logger.target target
|
33
35
|
Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
|
@@ -36,7 +38,7 @@ class DeadFinderRunner
|
|
36
38
|
results = Channel.new(buffer: :buffered, capacity: 1000)
|
37
39
|
|
38
40
|
(1..options['concurrency']).each do |w|
|
39
|
-
Channel.go { worker(w, jobs, results, options) }
|
41
|
+
Channel.go { worker(w, jobs, results, target, options) }
|
40
42
|
end
|
41
43
|
|
42
44
|
link_a.uniq.each do |node|
|
@@ -51,24 +53,25 @@ class DeadFinderRunner
|
|
51
53
|
Logger.sub_done 'Done'
|
52
54
|
end
|
53
55
|
|
54
|
-
def worker(_id, jobs, results, options)
|
56
|
+
def worker(_id, jobs, results, target, options)
|
55
57
|
jobs.each do |j|
|
56
58
|
if !CacheSet.include? j
|
57
59
|
CacheSet.add j
|
58
60
|
begin
|
59
61
|
CacheQue[j] = true
|
60
|
-
URI.open(j, :
|
62
|
+
URI.open(j, read_timeout: options['timeout'])
|
61
63
|
rescue StandardError => e
|
62
|
-
|
63
|
-
|
64
|
+
if e.to_s.include? '404 Not Found'
|
65
|
+
Logger.found "[#{e}] #{j}"
|
66
|
+
CacheQue[j] = false
|
67
|
+
Output[target] = [] if Output[target].nil?
|
68
|
+
Output[target].push j
|
69
|
+
end
|
64
70
|
end
|
65
|
-
|
66
|
-
|
67
|
-
if !CacheQue[j]
|
68
|
-
Logger.found "[404 Not Found] #{j}"
|
69
|
-
end
|
70
|
-
results << j
|
71
|
+
elsif !CacheQue[j]
|
72
|
+
Logger.found "[404 Not Found] #{j}"
|
71
73
|
end
|
74
|
+
results << j
|
72
75
|
end
|
73
76
|
end
|
74
77
|
end
|
@@ -79,6 +82,7 @@ def run_pipe(options)
|
|
79
82
|
target = $LAST_READ_LINE.gsub("\n", '')
|
80
83
|
app.run target, options
|
81
84
|
end
|
85
|
+
gen_output
|
82
86
|
end
|
83
87
|
|
84
88
|
def run_file(filename, options)
|
@@ -87,11 +91,13 @@ def run_file(filename, options)
|
|
87
91
|
target = line.gsub("\n", '')
|
88
92
|
app.run target, options
|
89
93
|
end
|
94
|
+
gen_output
|
90
95
|
end
|
91
96
|
|
92
97
|
def run_url(url, options)
|
93
98
|
app = DeadFinderRunner.new
|
94
99
|
app.run url, options
|
100
|
+
gen_output
|
95
101
|
end
|
96
102
|
|
97
103
|
def run_sitemap(sitemap_url, options)
|
@@ -100,11 +106,17 @@ def run_sitemap(sitemap_url, options)
|
|
100
106
|
sitemap.to_a.each do |url|
|
101
107
|
app.run url, options
|
102
108
|
end
|
109
|
+
gen_output
|
110
|
+
end
|
111
|
+
|
112
|
+
def gen_output
|
113
|
+
File.write options['output'], Output.to_json if options['output'] != ''
|
103
114
|
end
|
104
115
|
|
105
116
|
class DeadFinder < Thor
|
106
117
|
class_option :concurrency, aliases: :c, default: 20, type: :numeric
|
107
118
|
class_option :timeout, aliases: :t, default: 10, type: :numeric
|
119
|
+
class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
|
108
120
|
|
109
121
|
desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
|
110
122
|
def pipe
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deadfinder
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hahwul
|
@@ -94,6 +94,34 @@ dependencies:
|
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 1.2.0
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: set
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.0.0
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.0.0
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: json
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 2.6.0
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 2.6.0
|
97
125
|
description: Dead link (broken link) means a link within a web page that cannot be
|
98
126
|
connected. These links can have a security negative impact with SEO. This tool makes
|
99
127
|
it easy to identify and modify.
|