deadfinder 1.2.3 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/lib/deadfinder/version.rb +1 -1
  3. data/lib/deadfinder.rb +43 -31
  4. metadata +20 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0b18bed09ea30afe6efed9e0795f774bb8e97a5658c9d1c1092e625b2950345a
4
- data.tar.gz: c5c9010c7d30764b559fa3bddffc3e737c87797aa046d50cb5ad84d74766137b
3
+ metadata.gz: 1f66dee5b8a32028d31fbd56996f59d2c388673ba4b5220135a8b7193f43917a
4
+ data.tar.gz: 5e3afcafadd5199573614a4de74b446bca6df3a6edb2272459989a468eace160
5
5
  SHA512:
6
- metadata.gz: 02a6a785560cb6e781b6a2c4f375e1daccbf0f4f00e5b4ee97e2d5e787b0fcb381013361aba96d618dea1e78f2b92f1290b60720e0c976d1fe57720100015169
7
- data.tar.gz: 2eb418ff24d686ef057dec9811ea63a973199ab814a231221325f9908a2f83f5b19dc99cc73f96f5b0b50da852324352a31907130fcf88a02c94f33455203521
6
+ metadata.gz: d040d1d57abf50e7eee69a67be3833ba33775f92b51dbc6efc02817cdf7fa7f21f285513f787c7de7a7e7881e4caef9e130bcfc5f072fe73152487bca223490d
7
+ data.tar.gz: d4b85b1a4568fc2279cd68ac82df4df7c3918d28656b6839d16c6348128b47179868fecbbb42f3ae12da84f280e027390f87beddc20a03878f5c8599f1436317
data/lib/deadfinder/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- VERSION = '1.2.3'
3
+ VERSION = '1.3.0'
data/lib/deadfinder.rb CHANGED
@@ -19,42 +19,53 @@ Output = {}
19
19
 
20
20
  class DeadFinderRunner
21
21
  def run(target, options)
22
- begin
22
+ page = nil
23
+
24
+ if options['headers'].length.positive?
25
+ headers = {}
26
+ options['headers'].each do |header|
27
+ kv = header.split ': '
28
+ headers[kv[0]] = kv[1]
29
+ rescue StandardError
30
+ end
31
+
32
+ page = Nokogiri::HTML(URI.open(target, headers))
33
+ else
23
34
  page = Nokogiri::HTML(URI.open(target))
35
+ end
24
36
 
25
- nodeset_a = page.css('a')
26
- link_a = nodeset_a.map { |element| element['href'] }.compact
27
- nodeset_script = page.css('script')
28
- link_script = nodeset_script.map { |element| element['src'] }.compact
29
- nodeset_link = page.css('link')
30
- link_link = nodeset_link.map { |element| element['href'] }.compact
37
+ nodeset_a = page.css('a')
38
+ link_a = nodeset_a.map { |element| element['href'] }.compact
39
+ nodeset_script = page.css('script')
40
+ link_script = nodeset_script.map { |element| element['src'] }.compact
41
+ nodeset_link = page.css('link')
42
+ link_link = nodeset_link.map { |element| element['href'] }.compact
31
43
 
32
- link_merged = []
33
- link_merged.concat link_a, link_script, link_link
44
+ link_merged = []
45
+ link_merged.concat link_a, link_script, link_link
34
46
 
35
- Logger.target target
36
- Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
37
- Logger.sub_info 'Checking'
38
- jobs = Channel.new(buffer: :buffered, capacity: 1000)
39
- results = Channel.new(buffer: :buffered, capacity: 1000)
47
+ Logger.target target
48
+ Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
49
+ Logger.sub_info 'Checking'
50
+ jobs = Channel.new(buffer: :buffered, capacity: 1000)
51
+ results = Channel.new(buffer: :buffered, capacity: 1000)
40
52
 
41
- (1..options['concurrency']).each do |w|
42
- Channel.go { worker(w, jobs, results, target, options) }
43
- end
53
+ (1..options['concurrency']).each do |w|
54
+ Channel.go { worker(w, jobs, results, target, options) }
55
+ end
44
56
 
45
- link_merged.uniq.each do |node|
46
- result = generate_url node, target
47
- jobs << result
48
- end
49
- jobs.close
57
+ link_merged.uniq.each do |node|
58
+ result = generate_url node, target
59
+ jobs << result
60
+ end
61
+ jobs.close
50
62
 
51
- (1..link_merged.uniq.length).each do
52
- ~results
53
- end
54
- Logger.sub_done 'Done'
55
- rescue => e
56
- Logger.error "[#{e}] #{target}"
63
+ (1..link_merged.uniq.length).each do
64
+ ~results
57
65
  end
66
+ Logger.sub_done 'Done'
67
+ rescue StandardError => e
68
+ Logger.error "[#{e}] #{target}"
58
69
  end
59
70
 
60
71
  def worker(_id, jobs, results, target, options)
@@ -120,9 +131,10 @@ def gen_output
120
131
  end
121
132
 
122
133
  class DeadFinder < Thor
123
- class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Set Concurrncy'
124
- class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Set HTTP Timeout'
125
- class_option :output, aliases: :o, default: '', type: :string, desc: 'Save JSON Result'
134
+ class_option :concurrency, aliases: :c, default: 20, type: :numeric, desc: 'Number of concurrncy'
135
+ class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
136
+ class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
137
+ class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
126
138
 
127
139
  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
128
140
  def pipe
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deadfinder
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.3
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - hahwul
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-10-09 00:00:00.000000000 Z
11
+ date: 2022-10-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: colorize
@@ -39,89 +39,89 @@ dependencies:
39
39
  - !ruby/object:Gem::Version
40
40
  version: 0.6.0
41
41
  - !ruby/object:Gem::Dependency
42
- name: nokogiri
42
+ name: json
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: 1.13.0
47
+ version: 2.6.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: 1.13.0
54
+ version: 2.6.0
55
55
  - !ruby/object:Gem::Dependency
56
- name: open-uri
56
+ name: nokogiri
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: 0.2.0
61
+ version: 1.13.0
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: 0.2.0
68
+ version: 1.13.0
69
69
  - !ruby/object:Gem::Dependency
70
- name: sitemap-parser
70
+ name: open-uri
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: 0.5.0
75
+ version: 0.2.0
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: 0.5.0
82
+ version: 0.2.0
83
83
  - !ruby/object:Gem::Dependency
84
- name: thor
84
+ name: set
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: 1.2.0
89
+ version: 1.0.0
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: 1.2.0
96
+ version: 1.0.0
97
97
  - !ruby/object:Gem::Dependency
98
- name: set
98
+ name: sitemap-parser
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: 1.0.0
103
+ version: 0.5.0
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: 1.0.0
110
+ version: 0.5.0
111
111
  - !ruby/object:Gem::Dependency
112
- name: json
112
+ name: thor
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: 2.6.0
117
+ version: 1.2.0
118
118
  type: :runtime
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: 2.6.0
124
+ version: 1.2.0
125
125
  description: Find dead-links (broken links). Dead link (broken link) means a link
126
126
  within a web page that cannot be connected. These links can have a negative impact
127
127
  to SEO and Security. This tool makes it easy to identify and modify.