deadfinder 1.3.5 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/lib/deadfinder/logger.rb +20 -6
  3. data/lib/deadfinder/version.rb +1 -1
  4. data/lib/deadfinder.rb +46 -9
  5. metadata +54 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c4cee202abc3ad85072d0b9cfd7d7b2029eb539236cffb1d3b2fd220136186c3
4
- data.tar.gz: 3013f0ed97adfd00061325405141f6bba346957063d6ef7813408aa331cf3232
3
+ metadata.gz: cf75290cf2187f96b42398ac743fd60fc203616a2cd763c4de85a179b6dc4ae6
4
+ data.tar.gz: cd894f1ac871e057ac3df4e1d0b89b6bf08c9625f9089c9a0213e94bc4f43010
5
5
  SHA512:
6
- metadata.gz: 69bf37ab49d7464b5345fd8833fca11a3efd003958faa2f5406fc46b00ce17557b19b3c497086304070beb13cf22eeaf18974651cb88c77329ac056a7969bf6a
7
- data.tar.gz: 51435a7e833e6ee91d2f986265c4f885419ce59864f0f994f9deb28d384ce95595e97060837232115f1a5bdc9cb00961f9e93d792fafd11213d03050b29819f1
6
+ metadata.gz: d9e2c6ee268a3ce006c3a23c123852cf23e75b59218876aa0fd1a0ef779e07fb04994b781bb97d231b01db549903eb206c11b1a15774fcf8bce73423d7c1870b
7
+ data.tar.gz: dbe9ab7a014687e2fb34cb893cecb236c842c6d8dd3b9dcc5107c96630fe9b123684608e047db182a6f94eb9393281410b7c7831cb0c560d72ba529da11e49d1
data/lib/deadfinder/logger.rb CHANGED
@@ -3,27 +3,41 @@
3
3
  require 'colorize'
4
4
 
5
5
  class Logger
6
+ @silent = false
7
+
8
+ def self.set_silent
9
+ @silent = true
10
+ end
11
+
12
+ def self.silent?
13
+ @silent
14
+ end
15
+
6
16
  def self.info(text)
7
- puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue)
17
+ puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue) unless silent?
8
18
  end
9
19
 
10
20
  def self.error(text)
11
- puts '⚠︎ '.colorize(:red) + text.to_s
21
+ puts '⚠︎ '.colorize(:red) + text.to_s unless silent?
12
22
  end
13
23
 
14
24
  def self.target(text)
15
- puts '► '.colorize(:green) + text.to_s.colorize(:light_green)
25
+ puts '► '.colorize(:green) + text.to_s.colorize(:light_green) unless silent?
16
26
  end
17
27
 
18
28
  def self.sub_info(text)
19
- puts ' ● '.colorize(:blue) + text.to_s.colorize(:light_blue)
29
+ puts ' ● '.colorize(:blue) + text.to_s.colorize(:light_blue) unless silent?
20
30
  end
21
31
 
22
32
  def self.sub_done(text)
23
- puts ' ✓ '.colorize(:blue) + text.to_s.colorize(:light_blue)
33
+ puts ' ✓ '.colorize(:blue) + text.to_s.colorize(:light_blue) unless silent?
24
34
  end
25
35
 
26
36
  def self.found(text)
27
- puts " ✘ #{text}".colorize(:red)
37
+ puts " ✘ #{text}".colorize(:red) unless silent?
38
+ end
39
+
40
+ def self.verbose(text)
41
+ puts ' ➜ '.colorize(:yellow) + text.to_s.colorize(:light_yellow) unless silent?
28
42
  end
29
43
  end
data/lib/deadfinder/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- VERSION = '1.3.5'
3
+ VERSION = '1.4.0'
data/lib/deadfinder.rb CHANGED
@@ -9,7 +9,6 @@ require 'deadfinder/logger'
9
9
  require 'deadfinder/version'
10
10
  require 'concurrent-edge'
11
11
  require 'sitemap-parser'
12
- require 'set'
13
12
  require 'json'
14
13
 
15
14
  Channel = Concurrent::Channel
@@ -18,7 +17,20 @@ CacheQue = Concurrent::Map.new
18
17
  Output = Concurrent::Map.new
19
18
 
20
19
  class DeadFinderRunner
20
+ def default_options
21
+ {
22
+ 'concurrency' => 50,
23
+ 'timeout' => 10,
24
+ 'output' => '',
25
+ 'headers' => [],
26
+ 'silent' => true,
27
+ 'verbose' => false,
28
+ 'include30x' => false
29
+ }
30
+ end
31
+
21
32
  def run(target, options)
33
+ Logger.set_silent if options['silent']
22
34
  headers = options['headers'].each_with_object({}) do |header, hash|
23
35
  kv = header.split(': ')
24
36
  hash[kv[0]] = kv[1]
@@ -66,14 +78,26 @@ class DeadFinderRunner
66
78
  CacheSet[j] = true
67
79
  begin
68
80
  CacheQue[j] = true
69
- URI.open(j, read_timeout: options['timeout'])
70
- rescue StandardError => e
71
- if e.to_s.include? '404 Not Found'
72
- Logger.found "[#{e}] #{j}"
81
+ uri = URI.parse(j)
82
+
83
+ # Create HTTP request with timeout and headers
84
+ http = Net::HTTP.new(uri.host, uri.port)
85
+ http.use_ssl = (uri.scheme == 'https')
86
+ http.read_timeout = options['timeout'].to_i if options['timeout']
87
+
88
+ request = Net::HTTP::Get.new(uri.request_uri)
89
+ response = http.request(request)
90
+ status_code = response.code.to_i
91
+ Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
92
+
93
+ if status_code >= 400 || (status_code >= 300 && options['include30x'])
94
+ Logger.found "[#{status_code} #{response.message}] #{j}"
73
95
  CacheQue[j] = false
74
96
  Output[target] ||= []
75
97
  Output[target] << j
76
98
  end
99
+ rescue StandardError => e
100
+ Logger.verbose "[#{e}] #{j}" if options['verbose']
77
101
  end
78
102
  end
79
103
  results << j
@@ -96,35 +120,49 @@ class DeadFinderRunner
96
120
  end
97
121
 
98
122
  def run_pipe(options)
123
+ Logger.set_silent if options['silent']
124
+
125
+ Logger.info 'Reading from STDIN'
99
126
  app = DeadFinderRunner.new
100
127
  while $stdin.gets
101
128
  target = $LAST_READ_LINE.chomp
129
+ Logger.target "Checking: #{target}"
102
130
  app.run target, options
103
131
  end
104
132
  gen_output(options)
105
133
  end
106
134
 
107
135
  def run_file(filename, options)
136
+ Logger.set_silent if options['silent']
137
+
138
+ Logger.info "Reading: #{filename}"
108
139
  app = DeadFinderRunner.new
109
140
  File.foreach(filename) do |line|
110
141
  target = line.chomp
142
+ Logger.target "Checking: #{target}"
111
143
  app.run target, options
112
144
  end
113
145
  gen_output(options)
114
146
  end
115
147
 
116
148
  def run_url(url, options)
149
+ Logger.set_silent if options['silent']
150
+
151
+ Logger.target "Checking: #{url}"
117
152
  app = DeadFinderRunner.new
118
153
  app.run url, options
119
154
  gen_output(options)
120
155
  end
121
156
 
122
157
  def run_sitemap(sitemap_url, options)
158
+ Logger.set_silent if options['silent']
159
+ Logger.info "Parsing sitemap: #{sitemap_url}"
123
160
  app = DeadFinderRunner.new
124
161
  base_uri = URI(sitemap_url)
125
162
  sitemap = SitemapParser.new sitemap_url, { recurse: true }
126
163
  sitemap.to_a.each do |url|
127
164
  turl = generate_url url, base_uri
165
+ Logger.target "Checking: #{turl}"
128
166
  app.run turl, options
129
167
  end
130
168
  gen_output(options)
@@ -135,32 +173,31 @@ def gen_output(options)
135
173
  end
136
174
 
137
175
  class DeadFinder < Thor
176
+ class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
138
177
  class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
139
178
  class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
140
179
  class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
141
180
  class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
181
+ class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
182
+ class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
142
183
 
143
184
  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
144
185
  def pipe
145
- Logger.info 'Pipe mode'
146
186
  run_pipe options
147
187
  end
148
188
 
149
189
  desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
150
190
  def file(filename)
151
- Logger.info 'File mode'
152
191
  run_file filename, options
153
192
  end
154
193
 
155
194
  desc 'url <URL>', 'Scan the Single URL.'
156
195
  def url(url)
157
- Logger.info 'Single URL mode'
158
196
  run_url url, options
159
197
  end
160
198
 
161
199
  desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
162
200
  def sitemap(sitemap)
163
- Logger.info 'Sitemap mode'
164
201
  run_sitemap sitemap, options
165
202
  end
166
203
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deadfinder
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.5
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - hahwul
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-09-25 00:00:00.000000000 Z
11
+ date: 2024-10-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: colorize
@@ -17,6 +17,9 @@ dependencies:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: 0.8.0
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 0.8.0
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
@@ -24,6 +27,9 @@ dependencies:
24
27
  - - "~>"
25
28
  - !ruby/object:Gem::Version
26
29
  version: 0.8.0
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 0.8.0
27
33
  - !ruby/object:Gem::Dependency
28
34
  name: concurrent-ruby-edge
29
35
  requirement: !ruby/object:Gem::Requirement
@@ -31,6 +37,9 @@ dependencies:
31
37
  - - "~>"
32
38
  - !ruby/object:Gem::Version
33
39
  version: 0.6.0
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 0.6.0
34
43
  type: :runtime
35
44
  prerelease: false
36
45
  version_requirements: !ruby/object:Gem::Requirement
@@ -38,6 +47,9 @@ dependencies:
38
47
  - - "~>"
39
48
  - !ruby/object:Gem::Version
40
49
  version: 0.6.0
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 0.6.0
41
53
  - !ruby/object:Gem::Dependency
42
54
  name: json
43
55
  requirement: !ruby/object:Gem::Requirement
@@ -45,6 +57,9 @@ dependencies:
45
57
  - - "~>"
46
58
  - !ruby/object:Gem::Version
47
59
  version: 2.6.0
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: 2.6.0
48
63
  type: :runtime
49
64
  prerelease: false
50
65
  version_requirements: !ruby/object:Gem::Requirement
@@ -52,6 +67,9 @@ dependencies:
52
67
  - - "~>"
53
68
  - !ruby/object:Gem::Version
54
69
  version: 2.6.0
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: 2.6.0
55
73
  - !ruby/object:Gem::Dependency
56
74
  name: nokogiri
57
75
  requirement: !ruby/object:Gem::Requirement
@@ -59,6 +77,9 @@ dependencies:
59
77
  - - "~>"
60
78
  - !ruby/object:Gem::Version
61
79
  version: 1.13.0
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: 1.13.0
62
83
  type: :runtime
63
84
  prerelease: false
64
85
  version_requirements: !ruby/object:Gem::Requirement
@@ -66,6 +87,9 @@ dependencies:
66
87
  - - "~>"
67
88
  - !ruby/object:Gem::Version
68
89
  version: 1.13.0
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: 1.13.0
69
93
  - !ruby/object:Gem::Dependency
70
94
  name: open-uri
71
95
  requirement: !ruby/object:Gem::Requirement
@@ -73,6 +97,9 @@ dependencies:
73
97
  - - "~>"
74
98
  - !ruby/object:Gem::Version
75
99
  version: 0.2.0
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: 0.2.0
76
103
  type: :runtime
77
104
  prerelease: false
78
105
  version_requirements: !ruby/object:Gem::Requirement
@@ -80,20 +107,29 @@ dependencies:
80
107
  - - "~>"
81
108
  - !ruby/object:Gem::Version
82
109
  version: 0.2.0
110
+ - - ">="
111
+ - !ruby/object:Gem::Version
112
+ version: 0.2.0
83
113
  - !ruby/object:Gem::Dependency
84
114
  name: set
85
115
  requirement: !ruby/object:Gem::Requirement
86
116
  requirements:
87
117
  - - "~>"
88
118
  - !ruby/object:Gem::Version
89
- version: 1.0.0
119
+ version: 1.1.0
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: 1.1.0
90
123
  type: :runtime
91
124
  prerelease: false
92
125
  version_requirements: !ruby/object:Gem::Requirement
93
126
  requirements:
94
127
  - - "~>"
95
128
  - !ruby/object:Gem::Version
96
- version: 1.0.0
129
+ version: 1.1.0
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: 1.1.0
97
133
  - !ruby/object:Gem::Dependency
98
134
  name: sitemap-parser
99
135
  requirement: !ruby/object:Gem::Requirement
@@ -101,6 +137,9 @@ dependencies:
101
137
  - - "~>"
102
138
  - !ruby/object:Gem::Version
103
139
  version: 0.5.0
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: 0.5.0
104
143
  type: :runtime
105
144
  prerelease: false
106
145
  version_requirements: !ruby/object:Gem::Requirement
@@ -108,6 +147,9 @@ dependencies:
108
147
  - - "~>"
109
148
  - !ruby/object:Gem::Version
110
149
  version: 0.5.0
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: 0.5.0
111
153
  - !ruby/object:Gem::Dependency
112
154
  name: thor
113
155
  requirement: !ruby/object:Gem::Requirement
@@ -115,6 +157,9 @@ dependencies:
115
157
  - - "~>"
116
158
  - !ruby/object:Gem::Version
117
159
  version: 1.2.0
160
+ - - ">="
161
+ - !ruby/object:Gem::Version
162
+ version: 1.2.0
118
163
  type: :runtime
119
164
  prerelease: false
120
165
  version_requirements: !ruby/object:Gem::Requirement
@@ -122,6 +167,9 @@ dependencies:
122
167
  - - "~>"
123
168
  - !ruby/object:Gem::Version
124
169
  version: 1.2.0
170
+ - - ">="
171
+ - !ruby/object:Gem::Version
172
+ version: 1.2.0
125
173
  description: Find dead-links (broken links). Dead link (broken link) means a link
126
174
  within a web page that cannot be connected. These links can have a negative impact
127
175
  to SEO and Security. This tool makes it easy to identify and modify.
@@ -150,14 +198,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
150
198
  requirements:
151
199
  - - ">="
152
200
  - !ruby/object:Gem::Version
153
- version: '0'
201
+ version: 3.3.0
154
202
  required_rubygems_version: !ruby/object:Gem::Requirement
155
203
  requirements:
156
204
  - - ">="
157
205
  - !ruby/object:Gem::Version
158
206
  version: '0'
159
207
  requirements: []
160
- rubygems_version: 3.5.3
208
+ rubygems_version: 3.5.16
161
209
  signing_key:
162
210
  specification_version: 4
163
211
  summary: Find dead-links (broken links)