deadfinder 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 694fc6108a1fde665a3e8b265071409ac483434b093c94e3eebb25324e4bb300
4
- data.tar.gz: 1bd04447be853ff381263915fe2ddcaca3418699828d5e97feb674623c41c8ba
3
+ metadata.gz: e1e747b78068c1590c88f49acc42ca917a3c26684564c4465449181d5fea4125
4
+ data.tar.gz: 19f6a77d77bb0b3e2daba04ec364b1626eebbe30ec9973753fb75c1cb78ffc2e
5
5
  SHA512:
6
- metadata.gz: 42eb575191bc6fa72cca03b0a94a8f20db82242c19a93bc0fcc9dfe8872c03c2cfe1accf68c480b0432f7c73df4ee0b1782ad08906d1e4bdea35dfc268d837b3
7
- data.tar.gz: 39d5838080f9afd84cf3272f85a4cc6e26cf663da10b76a7e9fcfe1f63ac1b41ca973a21e454743fbc1011f9955b3c9ec488e0a586df94bdaca5e01f2ea5bec2
6
+ metadata.gz: ddc7a3974fcbf62d78ce739862f1686b070bfea6f26ae534c5fe18b49fb48432f7483567ab109ef4b97d40916460be1b08524dc8615d5f537564555df817c781
7
+ data.tar.gz: b676e1009517b67da7cff2dc10f26a50f6635d1abfeab6e036092d2b63eff55fa758c3cb0ee2b63ec34157901f00708a2d0f4282e99e5fc61057f70ad939e23f
@@ -7,6 +7,10 @@ class Logger
7
7
  puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue)
8
8
  end
9
9
 
10
+ def self.error(text)
11
+ puts '⚠︎ '.colorize(:red) + text.to_s
12
+ end
13
+
10
14
  def self.target(text)
11
15
  puts '► '.colorize(:green) + text.to_s.colorize(:light_green)
12
16
  end
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- VERSION = '1.2.0'
3
+ VERSION = '1.2.2'
data/lib/deadfinder.rb CHANGED
@@ -19,38 +19,42 @@ Output = {}
19
19
 
20
20
  class DeadFinderRunner
21
21
  def run(target, options)
22
- page = Nokogiri::HTML(URI.open(target))
23
-
24
- nodeset_a = page.css('a')
25
- link_a = nodeset_a.map { |element| element['href'] }.compact
26
- nodeset_script = page.css('script')
27
- link_script = nodeset_script.map { |element| element['src'] }.compact
28
- nodeset_link = page.css('link')
29
- link_link = nodeset_link.map { |element| element['href'] }.compact
30
-
31
- link_merged = []
32
- link_merged.concat link_a, link_script, link_link
33
-
34
- Logger.target target
35
- Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
36
- Logger.sub_info 'Checking'
37
- jobs = Channel.new(buffer: :buffered, capacity: 1000)
38
- results = Channel.new(buffer: :buffered, capacity: 1000)
39
-
40
- (1..options['concurrency']).each do |w|
41
- Channel.go { worker(w, jobs, results, target, options) }
42
- end
22
+ begin
23
+ page = Nokogiri::HTML(URI.open(target))
24
+
25
+ nodeset_a = page.css('a')
26
+ link_a = nodeset_a.map { |element| element['href'] }.compact
27
+ nodeset_script = page.css('script')
28
+ link_script = nodeset_script.map { |element| element['src'] }.compact
29
+ nodeset_link = page.css('link')
30
+ link_link = nodeset_link.map { |element| element['href'] }.compact
31
+
32
+ link_merged = []
33
+ link_merged.concat link_a, link_script, link_link
34
+
35
+ Logger.target target
36
+ Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
37
+ Logger.sub_info 'Checking'
38
+ jobs = Channel.new(buffer: :buffered, capacity: 1000)
39
+ results = Channel.new(buffer: :buffered, capacity: 1000)
40
+
41
+ (1..options['concurrency']).each do |w|
42
+ Channel.go { worker(w, jobs, results, target, options) }
43
+ end
43
44
 
44
- link_a.uniq.each do |node|
45
- result = generate_url node, target
46
- jobs << result
47
- end
48
- jobs.close
45
+ link_merged.uniq.each do |node|
46
+ result = generate_url node, target
47
+ jobs << result
48
+ end
49
+ jobs.close
49
50
 
50
- (1..link_a.uniq.length).each do
51
- ~results
51
+ (1..link_merged.uniq.length).each do
52
+ ~results
53
+ end
54
+ Logger.sub_done 'Done'
55
+ rescue => e
56
+ Logger.error "[#{e}] #{target}"
52
57
  end
53
- Logger.sub_done 'Done'
54
58
  end
55
59
 
56
60
  def worker(_id, jobs, results, target, options)
@@ -102,9 +106,11 @@ end
102
106
 
103
107
  def run_sitemap(sitemap_url, options)
104
108
  app = DeadFinderRunner.new
109
+ base_uri = URI(sitemap_url)
105
110
  sitemap = SitemapParser.new sitemap_url, { recurse: true }
106
111
  sitemap.to_a.each do |url|
107
- app.run url, options
112
+ turl = generate_url url, base_uri
113
+ app.run turl, options
108
114
  end
109
115
  gen_output
110
116
  end
@@ -114,8 +120,8 @@ def gen_output
114
120
  end
115
121
 
116
122
  class DeadFinder < Thor
117
- class_option :concurrency, aliases: :c, default: 20, type: :numeric
118
- class_option :timeout, aliases: :t, default: 10, type: :numeric
123
+ class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Set Concurrncy'
124
+ class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Set HTTP Timeout'
119
125
  class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
120
126
 
121
127
  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deadfinder
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - hahwul
@@ -122,9 +122,9 @@ dependencies:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
124
  version: 2.6.0
125
- description: Dead link (broken link) means a link within a web page that cannot be
126
- connected. These links can have a security negative impact with SEO. This tool makes
127
- it easy to identify and modify.
125
+ description: Find dead-links (broken links). Dead link (broken link) means a link
126
+ within a web page that cannot be connected. These links can have a negative impact
127
+ to SEO and Security. This tool makes it easy to identify and modify.
128
128
  email: hahwul@gmail.com
129
129
  executables:
130
130
  - deadfinder