yawast 0.6.0 → 0.7.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 83f5d74a8796ef331534669c079965a7f0b558a6
4
- data.tar.gz: b98c0de6ec0b415fcbfe8ad5e6121834d6428c38
3
+ metadata.gz: 1a0010955fcbfb843d4eaf927682a22df138b4a6
4
+ data.tar.gz: df672ffa6576e142c62fe0904f07ef90cc54c612
5
5
  SHA512:
6
- metadata.gz: 69a364e0ea8fe6f53dc9cea28efca7520aa0ea5faf92d554ba04906a67c76362820f9772dc5fa477ebcc7b151e07e16033a2027031ec1d8bb3fb59a50075dc06
7
- data.tar.gz: 32705ac1392cdfc4bc5ae3b9f6ac6671f0355b60555fd279fa4374d11215c8a1e2a631098da57aea9f49cdb219f96627a1be30157352d91095a28f43090356b0
6
+ metadata.gz: d1152a1e138492093e18834d8d542b4438a3904703a9449e6092857212db4d04048afeb9329e49bb591e217c1be82d06c1451a986a5e3cd517b66933dbacf018
7
+ data.tar.gz: 6ab13462bab4524b4e8c0347d0065b7947799ad6ab4ef6bed5bf38dadf090ad2d85e9bc1384a73247d302820ce034ea949824565960b93fedeaabc52797cbea1
data/.travis.yml CHANGED
@@ -1,8 +1,11 @@
1
1
  language: ruby
2
2
  rvm:
3
- - 2.2.8
4
3
  - 2.3.5
5
4
  - 2.4.2
5
+ - 2.6.0
6
+ before_install:
7
+ - gem update --system
8
+ - gem install bundler
6
9
  script:
7
10
  - bundle exec rake
8
11
  - bundle exec rake submitcodeclimate
data/CHANGELOG.md CHANGED
@@ -1,4 +1,19 @@
1
- ## 0.6.0 - In Development
1
+ ## 0.7.0 - In Development
2
+
3
+ * [#38](https://github.com/adamcaudill/yawast/issues/38) - Report Generation Mode (work in progress)
4
+ * [#133](https://github.com/adamcaudill/yawast/issues/133) - Include a Timestamp In Output
5
+ * [#134](https://github.com/adamcaudill/yawast/issues/134) - Add options to DNS command
6
+ * [#135](https://github.com/adamcaudill/yawast/issues/135) - Incomplete Certificate Chain Warning
7
+ * [#137](https://github.com/adamcaudill/yawast/issues/137) - Warn on TLS 1.0
8
+ * [#138](https://github.com/adamcaudill/yawast/issues/138) - Warn on Symantec Roots
9
+ * [#139](https://github.com/adamcaudill/yawast/issues/139) - Add Spider Option
10
+ * [#140](https://github.com/adamcaudill/yawast/issues/140) - Save output on cancel
11
+ * [#141](https://github.com/adamcaudill/yawast/issues/141) - Flag --internalssl as Deprecated
12
+ * [#130](https://github.com/adamcaudill/yawast/issues/130) - Bug: HSTS Error leads to printing HTML
13
+ * [#132](https://github.com/adamcaudill/yawast/issues/132) - Bug: Typo in SSL Output
14
+ * [#142](https://github.com/adamcaudill/yawast/issues/142) - Bug: Error In Collecting DNS Information
15
+
16
+ ## 0.6.0 - 2018-01-16
2
17
 
3
18
  * [#54](https://github.com/adamcaudill/yawast/issues/54) - Check for Python version in Server header
4
19
  * [#59](https://github.com/adamcaudill/yawast/issues/59) - SSL Labs: Display Certificate Chain
data/LICENSE CHANGED
@@ -1,6 +1,6 @@
1
1
  BSD 3-Clause License
2
2
 
3
- Copyright (c) 2013-2017, Adam Caudill <adam@adamcaudill.com>
3
+ Copyright (c) 2013-2019, Adam Caudill <adam@adamcaudill.com>
4
4
  All rights reserved.
5
5
 
6
6
  Redistribution and use in source and binary forms, with or without
data/bin/yawast CHANGED
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
+ # encoding: UTF-8
2
3
 
3
4
  require 'commander/import'
4
5
  require File.dirname(__FILE__) + '/../lib/yawast'
@@ -26,6 +27,8 @@ command :scan do |c|
26
27
  c.option '--proxy STRING', String, 'HTTP Proxy Server (such as Burp Suite)'
27
28
  c.option '--cookie STRING', String, 'Session cookie'
28
29
  c.option '--nodns', 'Disable DNS checks'
30
+ c.option '--spider', 'Spider the site'
31
+ c.option '--output STRING', String, 'Output JSON file'
29
32
 
30
33
  c.action do |args, options|
31
34
  Yawast::Commands::Scan.process(args, options)
@@ -43,6 +46,7 @@ command :head do |c|
43
46
  c.option '--proxy STRING', String, 'HTTP Proxy Server (such as Burp Suite)'
44
47
  c.option '--cookie STRING', String, 'Session cookie'
45
48
  c.option '--nodns', 'Disable DNS checks'
49
+ c.option '--output STRING', String, 'Output JSON file'
46
50
 
47
51
  c.action do |args, options|
48
52
  Yawast::Commands::Head.process(args, options)
@@ -79,6 +83,10 @@ command :dns do |c|
79
83
  c.syntax = './yawast dns URL'
80
84
  c.description = 'Gets information about the server DNS configuration'
81
85
 
86
+ c.option '--srv', 'Scan for known SRV DNS Records'
87
+ c.option '--subdomains', 'Search for Common Subdomains'
88
+ c.option '--output STRING', String, 'Output JSON file'
89
+
82
90
  c.action do |args, options|
83
91
  Yawast::Commands::DNS.process(args, options)
84
92
  end
data/lib/commands/dns.rb CHANGED
@@ -6,10 +6,15 @@ module Yawast
6
6
 
7
7
  Yawast.header
8
8
 
9
+ if options.output != nil
10
+ Yawast::Shared::Output.setup uri, options
11
+ end
12
+
9
13
  puts "Scanning: #{uri}"
10
14
  puts
11
15
 
12
16
  Yawast::Scanner::Plugins::DNS::Generic.dns_info uri, options
17
+ Yawast::Shared::Output.write_file
13
18
  end
14
19
  end
15
20
  end
data/lib/scanner/core.rb CHANGED
@@ -14,10 +14,15 @@ module Yawast
14
14
 
15
15
  print_header
16
16
 
17
+ if options.output != nil
18
+ Yawast::Shared::Output.setup @uri, options
19
+ end
20
+
17
21
  ssl_redirect = Yawast::Scanner::Plugins::SSL::SSL.check_for_ssl_redirect @uri
18
22
  if ssl_redirect
19
23
  @uri = ssl_redirect
20
24
  puts "Server redirects to TLS: Scanning: #{@uri}"
25
+ Yawast::Shared::Output.log_value 'server_tls_redirect', @uri
21
26
  end
22
27
 
23
28
  Yawast::Scanner::Plugins::SSL::SSL.set_openssl_options
@@ -42,6 +47,7 @@ module Yawast
42
47
 
43
48
  #cache the HEAD result, so that we can minimize hits
44
49
  head = get_head
50
+ Yawast::Shared::Output.log_hash 'http', 'head', 'raw', head.to_hash
45
51
  Yawast::Scanner::Generic.head_info(head, @uri)
46
52
 
47
53
  #perfom SSL checks
@@ -63,6 +69,10 @@ module Yawast
63
69
  Yawast::Scanner::Generic.check_options(@uri)
64
70
  Yawast::Scanner::Generic.check_trace(@uri)
65
71
 
72
+ if options.spider
73
+ Yawast::Scanner::Plugins::Spider::Spider.spider(@uri)
74
+ end
75
+
66
76
  #check for common directories
67
77
  if options.dir
68
78
  Yawast::Scanner::Plugins::Http::DirectorySearch.search @uri, options.dirrecursive, options.dirlistredir
@@ -75,6 +85,7 @@ module Yawast
75
85
  # less than 24 hours. if a scan is that long, we have bigger problems
76
86
  elapsed_time = Time.at(Time.now - start_time).utc.strftime('%H:%M:%S')
77
87
 
88
+ Yawast::Shared::Output.write_file
78
89
  puts "Scan complete (#{elapsed_time})."
79
90
  rescue => e
80
91
  Yawast::Utilities.puts_error "Fatal Error: Can not continue. (#{e.class}: #{e.message})"
@@ -24,6 +24,7 @@ module Yawast
24
24
  Yawast::Utilities.puts_info 'HEAD:'
25
25
  head.each do |k, v|
26
26
  Yawast::Utilities.puts_info "\t\t#{k}: #{v}"
27
+ Yawast::Shared::Output.log_value 'http', 'head', k, v
27
28
 
28
29
  server = v if k.downcase == 'server'
29
30
  powered_by = v if k.downcase == 'x-powered-by'
@@ -40,7 +41,11 @@ module Yawast
40
41
 
41
42
  if k.downcase == 'set-cookie'
42
43
  #this chunk of magic manages to properly split cookies, when multiple are sent together
43
- v.gsub(/(,([^;,]*=)|,$)/) { "\r\n#{$2}" }.split(/\r\n/).each { |c| cookies.push(c) }
44
+ v.gsub(/(,([^;,]*=)|,$)/) { "\r\n#{$2}" }.split(/\r\n/).each do |c|
45
+ cookies.push(c)
46
+
47
+ Yawast::Shared::Output.log_append_value 'http', 'head', 'cookies', c
48
+ end
44
49
  end
45
50
  end
46
51
  puts ''
@@ -163,11 +168,13 @@ module Yawast
163
168
 
164
169
  if res['Public'] != nil
165
170
  Yawast::Utilities.puts_info "Public HTTP Verbs (OPTIONS): #{res['Public']}"
171
+ Yawast::Shared::Output.log_value 'http', 'options', 'public', res['Public']
166
172
 
167
173
  puts ''
168
174
  end
169
175
  if res['Allow'] != nil
170
176
  Yawast::Utilities.puts_info "Allow HTTP Verbs (OPTIONS): #{res['Allow']}"
177
+ Yawast::Shared::Output.log_value 'http', 'options', 'allow', res['Allow']
171
178
 
172
179
  puts ''
173
180
  end
@@ -187,6 +194,9 @@ module Yawast
187
194
 
188
195
  puts ''
189
196
  end
197
+
198
+ Yawast::Shared::Output.log_value 'http', 'trace', 'raw', res.body
199
+ Yawast::Shared::Output.log_value 'http', 'trace', 'code', res.code
190
200
  end
191
201
  end
192
202
 
@@ -203,6 +213,11 @@ module Yawast
203
213
 
204
214
  puts ''
205
215
  end
216
+
217
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'raw', res.body
218
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'code', res.code
219
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'content-type', res['Content-Type']
220
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'length', res.body.length
206
221
  end
207
222
  end
208
223
  end
@@ -35,6 +35,8 @@ module Yawast
35
35
  cname = get_cname_record(domain)
36
36
  if cname != nil
37
37
  Yawast::Utilities.puts_info "\t\tCAA (#{domain}): CNAME Found: -> #{cname}"
38
+ Yawast::Shared::Output.log_value 'dns', 'caa', domain, "CNAME: #{cname}"
39
+
38
40
  chase_domain cname.to_s
39
41
  else
40
42
  print_caa_record domain
@@ -66,6 +68,8 @@ module Yawast
66
68
  # check for RDATA
67
69
  if rec.rdata != nil
68
70
  Yawast::Utilities.puts_info "\t\tCAA (#{domain}): #{rec.rdata}"
71
+
72
+ Yawast::Shared::Output.log_append_value 'dns', 'caa', domain, rec.rdata
69
73
  else
70
74
  Yawast::Utilities.puts_error "\t\tCAA (#{domain}): Invalid Response: #{ans.answer}"
71
75
  end
@@ -73,6 +77,8 @@ module Yawast
73
77
  else
74
78
  # no answer, so no records
75
79
  Yawast::Utilities.puts_info "\t\tCAA (#{domain}): No Records Found"
80
+
81
+ Yawast::Shared::Output.log_value 'dns', 'caa', domain, 'nil'
76
82
  end
77
83
  end
78
84
  end
@@ -21,9 +21,13 @@ module Yawast
21
21
 
22
22
  Yawast::Utilities.puts_info "\t\t#{ip.address} (#{host_name})"
23
23
 
24
+ Yawast::Shared::Output.log_value 'dns', 'a', ip.address, host_name
25
+
24
26
  # if address is private, force internal SSL mode, don't show links
25
27
  if IPAddr.new(ip.address.to_s, Socket::AF_INET).private?
26
28
  options.internalssl = true
29
+
30
+ Yawast::Shared::Output.log_value 'force_internal_ssl', true
27
31
  else
28
32
  #show network info
29
33
  Yawast::Utilities.puts_info "\t\t\t#{get_network_info(ip.address)}"
@@ -45,6 +49,8 @@ module Yawast
45
49
 
46
50
  Yawast::Utilities.puts_info "\t\t#{ip.address} (#{host_name})"
47
51
 
52
+ Yawast::Shared::Output.log_value 'dns', 'aaaa', ip.address, host_name
53
+
48
54
  # if address is private, force internal SSL mode, don't show links
49
55
  if IPAddr.new(ip.address.to_s, Socket::AF_INET6).private?
50
56
  options.internalssl = true
@@ -61,16 +67,24 @@ module Yawast
61
67
  unless txt.empty?
62
68
  txt.each do |rec|
63
69
  Yawast::Utilities.puts_info "\t\tTXT: #{rec.data}"
70
+
71
+ Yawast::Shared::Output.log_append_value 'dns', 'txt', uri.host, rec.data
64
72
  end
65
73
  end
66
74
 
67
75
  #check for higher-level TXT records, if we aren't already at the top
68
76
  if root_domain != uri.host
69
- txt = resv.getresources(root_domain, Resolv::DNS::Resource::IN::TXT)
70
- unless txt.empty?
71
- txt.each do |rec|
72
- Yawast::Utilities.puts_info "\t\tTXT (#{root_domain}): #{rec.data}"
77
+ begin
78
+ txt = resv.getresources(root_domain, Resolv::DNS::Resource::IN::TXT)
79
+ unless txt.empty?
80
+ txt.each do |rec|
81
+ Yawast::Utilities.puts_info "\t\tTXT (#{root_domain}): #{rec.data}"
82
+
83
+ Yawast::Shared::Output.log_append_value 'dns', 'txt', root_domain, rec.data
84
+ end
73
85
  end
86
+ rescue => e
87
+ Yawast::Utilities.puts_error "\t\tTXT: #{root_domain} (Error: #{e.message})"
74
88
  end
75
89
  end
76
90
 
@@ -81,6 +95,8 @@ module Yawast
81
95
  ip = resv.getaddress rec.exchange
82
96
 
83
97
  Yawast::Utilities.puts_info "\t\tMX: #{rec.exchange} (#{rec.preference}) - #{ip} (#{get_network_info(ip.to_s)})"
98
+
99
+ Yawast::Shared::Output.log_value 'dns', 'mx', rec.exchange, ip
84
100
  rescue => e
85
101
  Yawast::Utilities.puts_error "\t\tMX: #{rec.exchange} (#{rec.preference}) - Error: #{e.message})"
86
102
  end
@@ -96,6 +112,8 @@ module Yawast
96
112
  ip = resv.getaddress rec.exchange
97
113
 
98
114
  Yawast::Utilities.puts_info "\t\tMX (#{root_domain}): #{rec.exchange} (#{rec.preference}) - #{ip} (#{get_network_info(ip.to_s)})"
115
+
116
+ Yawast::Shared::Output.log_value 'dns', 'mx', rec.exchange, ip
99
117
  rescue => e
100
118
  Yawast::Utilities.puts_error "\t\tMX (#{root_domain}): #{rec.exchange} (#{rec.preference}) - Error: #{e.message})"
101
119
  end
@@ -109,6 +127,8 @@ module Yawast
109
127
  ip = resv.getaddress rec.name
110
128
 
111
129
  Yawast::Utilities.puts_info "\t\tNS: #{rec.name} - #{ip} (#{get_network_info(ip.to_s)})"
130
+
131
+ Yawast::Shared::Output.log_value 'dns', 'ns', rec.name, ip
112
132
  end
113
133
  end
114
134
 
@@ -145,6 +165,8 @@ module Yawast
145
165
  ip = resv.getaddress rec.target
146
166
 
147
167
  Yawast::Utilities.puts_info "\t\tSRV: #{host}: #{rec.target}:#{rec.port} - #{ip} (#{get_network_info(ip.to_s)})"
168
+
169
+ Yawast::Shared::Output.log_value 'dns', 'srv', host, "#{rec.target}:#{rec.port}"
148
170
  end
149
171
  end
150
172
  rescue
@@ -169,6 +191,8 @@ module Yawast
169
191
  else
170
192
  Yawast::Utilities.puts_info "\t\tA: #{host}: #{ip.address} (#{get_network_info(ip.address)})"
171
193
  end
194
+
195
+ Yawast::Shared::Output.log_value 'dns', 'subdomain', host, ip.address
172
196
  end
173
197
  end
174
198
  rescue
@@ -194,6 +218,8 @@ module Yawast
194
218
  ret = "#{network_info['as_country_code']} - #{network_info['as_description']}"
195
219
  @netinfo[ip] = ret
196
220
 
221
+ Yawast::Shared::Output.log_value 'dns', 'asn_info', ip, ret
222
+
197
223
  return ret
198
224
  rescue => e
199
225
  @netinfo_failed = true
@@ -99,10 +99,12 @@ module Yawast
99
99
 
100
100
  if res.code == '200'
101
101
  @results.push "\tFound: '#{uri}'"
102
+ Yawast::Shared::Output.log_append_value 'http', 'http_dir', uri
102
103
 
103
104
  load_queue uri if @recursive
104
105
  elsif res.code == '301' && @list_redirects
105
106
  @results.push "\tFound Redirect: '#{uri} -> '#{res['Location']}'"
107
+ Yawast::Shared::Output.log_value 'http', 'http_dir_redirect', uri, res['Location']
106
108
  end
107
109
  rescue => e
108
110
  unless e.message.include?('end of file') || e.message.include?('getaddrinfo')
@@ -163,6 +163,7 @@ module Yawast
163
163
 
164
164
  if res.code == '200'
165
165
  @results.push "'#{uri.path}' found: #{uri}"
166
+ Yawast::Shared::Output.log_append_value 'http', 'http_file', uri
166
167
  end
167
168
  rescue => e
168
169
  unless e.message.include?('end of file') || e.message.include?('getaddrinfo')
@@ -71,6 +71,8 @@ module Yawast
71
71
 
72
72
  if version != nil && version[0] != nil
73
73
  Yawast::Utilities.puts_warn "Apache Tomcat Version Found: #{version[0]}"
74
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_version', version[0]
75
+
74
76
  puts "\t\t\"curl -X XYZ #{uri}\""
75
77
 
76
78
  puts ''
@@ -80,8 +82,8 @@ module Yawast
80
82
  end
81
83
 
82
84
  def self.check_tomcat_manager(uri)
83
- check_tomcat_manager_paths uri, 'manager', 'Manager'
84
- check_tomcat_manager_paths uri, 'host-manager', 'Host Manager'
85
+ check_tomcat_manager_paths uri.copy, 'manager', 'Manager'
86
+ check_tomcat_manager_paths uri.copy, 'host-manager', 'Host Manager'
85
87
  end
86
88
 
87
89
  def self.check_tomcat_manager_paths(uri, base_path, manager)
@@ -93,16 +95,19 @@ module Yawast
93
95
  if ret.include? '<tt>conf/tomcat-users.xml</tt>'
94
96
  #this will get Tomcat 7+
95
97
  Yawast::Utilities.puts_warn "Apache Tomcat #{manager} page found: #{uri}"
98
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_mgr', manager, uri
96
99
  check_tomcat_manager_passwords uri, manager
97
100
 
98
101
  puts ''
99
102
  else
100
103
  #check for Tomcat 6 and below
104
+ uri = uri.copy
101
105
  uri.path = "/#{base_path}"
102
106
  ret = Yawast::Shared::Http.get(uri)
103
107
 
104
108
  if ret.include? '<tt>conf/tomcat-users.xml</tt>'
105
109
  Yawast::Utilities.puts_warn "Apache Tomcat #{manager} page found: #{uri}"
110
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_mgr', manager, uri
106
111
  check_tomcat_manager_passwords uri, manager
107
112
 
108
113
  puts ''
@@ -125,6 +130,8 @@ module Yawast
125
130
  if ret.include?('<font size="+2">Tomcat Web Application Manager</font>') ||
126
131
  ret.include?('<font size="+2">Tomcat Virtual Host Manager</font>')
127
132
  Yawast::Utilities.puts_vuln "Apache Tomcat #{manager} weak password: #{credentials}"
133
+
134
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_mgr_pwd', uri, credentials
128
135
  end
129
136
  end
130
137
 
@@ -133,9 +140,13 @@ module Yawast
133
140
  uri.path = "/#{SecureRandom.hex}.jsp/"
134
141
  uri.query = '' if uri.query != nil
135
142
 
143
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'path', uri
144
+
136
145
  # we'll use this to verify that it actually worked
137
146
  check_value = SecureRandom.hex
138
147
 
148
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'check_value', check_value
149
+
139
150
  # upload the JSP file
140
151
  req_data = "<% out.println(\"#{check_value}\");%>"
141
152
  Yawast::Shared::Http.put(uri, req_data)
@@ -143,8 +154,14 @@ module Yawast
143
154
  # check to see of we get check_value back
144
155
  uri.path = uri.path.chomp('/')
145
156
  res = Yawast::Shared::Http.get(uri)
157
+
158
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'body', res
159
+
146
160
  if res.include? check_value
147
161
  Yawast::Utilities.puts_vuln "Apache Tomcat PUT RCE (CVE-2017-12615): #{uri}"
162
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'vulnerable', true
163
+ else
164
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'vulnerable', false
148
165
  end
149
166
  end
150
167
 
@@ -159,9 +176,12 @@ module Yawast
159
176
  search.push '/struts2-rest-showcase/'
160
177
 
161
178
  search.each do |path|
179
+ uri = uri.copy
162
180
  uri.path = path
163
181
 
164
182
  ret = Yawast::Shared::Http.get_status_code uri
183
+ Yawast::Shared::Output.log_value 'apache', 'struts2_sample_files', uri, ret
184
+
165
185
  if ret == 200
166
186
  Yawast::Utilities.puts_warn "Apache Struts2 Sample Files: #{uri}"
167
187
  end
@@ -176,6 +196,7 @@ module Yawast
176
196
 
177
197
  if ret.include? search
178
198
  Yawast::Utilities.puts_vuln "#{search} page found: #{uri}"
199
+ Yawast::Shared::Output.log_value 'apache', 'page_search', search, uri
179
200
  puts ''
180
201
  end
181
202
  end
@@ -48,6 +48,9 @@ module Yawast
48
48
  if res.code == 200
49
49
  Yawast::Utilities.puts_vuln 'ASP.NET Debugging Enabled'
50
50
  end
51
+
52
+ Yawast::Shared::Output.log_value 'http', 'asp_net_debug', 'raw', res.body
53
+ Yawast::Shared::Output.log_value 'http', 'asp_net_debug', 'code', res.code
51
54
  end
52
55
  end
53
56
  end
@@ -0,0 +1,65 @@
1
+ require 'nokogiri'
2
+
3
+ module Yawast
4
+ module Scanner
5
+ module Plugins
6
+ module Spider
7
+ class Spider
8
+ def self.spider(uri)
9
+ @uri = uri.copy
10
+
11
+ @workers = []
12
+ @results = Queue.new
13
+
14
+ @links = []
15
+ @links.push @uri.to_s
16
+ puts 'Spidering site...'
17
+ get_links @uri
18
+
19
+ results = Thread.new do
20
+ begin
21
+ while true
22
+ if @results.length > 0
23
+ out = @results.pop(true)
24
+ Yawast::Utilities.puts_info out
25
+ Yawast::Shared::Output.log_append_value 'spider', 'get', out
26
+ end
27
+ end
28
+ rescue ThreadError
29
+ #do nothing
30
+ end
31
+ end
32
+
33
+ @workers.map(&:join)
34
+ results.terminate
35
+
36
+ puts
37
+ end
38
+
39
+ def self.get_links(uri)
40
+ # get the page, and work out from there
41
+ res = Yawast::Shared::Http.get_with_code uri
42
+ doc = Nokogiri::HTML res[:body]
43
+
44
+ results = doc.css('a').map { |link| link['href'] }
45
+
46
+ results.each do |link|
47
+ # check to see if this link is in scope
48
+ if link.to_s.include?(@uri.to_s) && res[:code] == '200'
49
+ # check to see if we've already seen this one
50
+ unless @links.include? link.to_s
51
+ @links.push link.to_s
52
+ @results.push "#{link.to_s}"
53
+
54
+ @workers.push Thread.new {
55
+ get_links URI.parse(link)
56
+ }
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
@@ -45,20 +45,25 @@ module Yawast
45
45
 
46
46
  Yawast::Utilities.puts_info "HSTS Preload: Chrome - #{chrome}; Firefox - #{firefox}; Tor - #{tor}"
47
47
  rescue => e
48
- Yawast::Utilities.puts_error "Error getting HSTS preload information: #{e.message}"
48
+ if e.message.include? 'unexpected token'
49
+ #this means we have a parsing error - don't need to include the entire message
50
+ Yawast::Utilities.puts_error "Error getting HSTS preload information: #{e.message.truncate(30)}"
51
+ else
52
+ Yawast::Utilities.puts_error "Error getting HSTS preload information: #{e.message}"
53
+ end
49
54
  end
50
55
  end
51
56
 
52
57
  def self.set_openssl_options
53
- #change certain defaults, to make things work better
54
- #we prefer RSA, to avoid issues with small DH keys
58
+ # change certain defaults, to make things work better
59
+ # we prefer RSA, to avoid issues with small DH keys
55
60
  OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ciphers] = 'RSA:ALL:COMPLEMENTOFALL'
56
61
  OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:verify_mode] = OpenSSL::SSL::VERIFY_NONE
57
62
  OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options] = OpenSSL::SSL::OP_ALL
58
63
  end
59
64
 
60
65
  def self.check_for_ssl_redirect(uri)
61
- #check to see if the site redirects to SSL by default
66
+ # check to see if the site redirects to SSL by default
62
67
  if uri.scheme != 'https'
63
68
  head = Yawast::Shared::Http.head(uri)
64
69
 
@@ -67,11 +72,11 @@ module Yawast
67
72
  location = URI.parse(head['Location'])
68
73
 
69
74
  if location.scheme == 'https'
70
- #we run this through extract_uri as it performs a few checks we need
75
+ # we run this through extract_uri as it performs a few checks we need
71
76
  return Yawast::Shared::Uri.extract_uri location.to_s
72
77
  end
73
78
  rescue
74
- #we don't care if this fails
79
+ # we don't care if this fails
75
80
  end
76
81
  end
77
82
  end
@@ -94,8 +99,11 @@ module Yawast
94
99
  ssl.connect
95
100
 
96
101
  # this provides a bunch of useful info, that's already formatted
97
- # instead of building this manually, we'll let OpenSSL do the work
98
- puts ssl.session.to_text
102
+ # instead of building this manually, we'll let OpenSSL do the
103
+ session_info = ssl.session.to_text
104
+ puts session_info
105
+
106
+ Yawast::Shared::Output.log_value 'ssl', 'session', 'info', session_info
99
107
 
100
108
  puts
101
109
  end
@@ -103,6 +111,68 @@ module Yawast
103
111
  Yawast::Utilities.puts_error "SSL Information: Error Getting Details: #{e.message}"
104
112
  end
105
113
  end
114
+
115
+ def self.check_symantec_root(hash)
116
+ roots = ['08297a4047dba23680c731db6e317653ca7848e1bebd3a0b0179a707f92cf178',
117
+ '2399561127a57125de8cefea610ddf2fa078b5c8067f4e828290bfb860e84b3c',
118
+ '2834991cf677466d22baac3b0055e5b911d9a9e55f5b85ba02dc566782c30e8a',
119
+ '2930bd09a07126bdc17288d4f2ad84645ec948607907a97b5ed0b0b05879ef69',
120
+ '2f274e48aba4ac7b765933101775506dc30ee38ef6acd5c04932cfe041234220',
121
+ '309b4a87f6ca56c93169aaa99c6d988854d7892bd5437e2d07b29cbeda55d35d',
122
+ '3266967e59cd68008d9dd320811185c704205e8d95fdd84f1c7b311e6704fc32',
123
+ '341de98b1392abf7f4ab90a960cf25d4bd6ec65b9a51ce6ed067d00ec7ce9b7f',
124
+ '363f3c849eab03b0a2a0f636d7b86d04d3ac7fcfe26a0a9121ab9795f6e176df',
125
+ '37d51006c512eaab626421f1ec8c92013fc5f82ae98ee533eb4619b8deb4d06c',
126
+ '3a43e220fe7f3ea9653d1e21742eac2b75c20fd8980305bc502caf8c2d9b41a1',
127
+ '3f9f27d583204b9e09c8a3d2066c4b57d3a2479c3693650880505698105dbce9',
128
+ '44640a0a0e4d000fbd574d2b8a07bdb4d1dfed3b45baaba76f785778c7011961',
129
+ '4b03f45807ad70f21bfc2cae71c9fde4604c064cf5ffb686bae5dbaad7fdd34c',
130
+ '53dfdfa4e297fcfe07594e8c62d5b8ab06b32c7549f38a163094fd6429d5da43',
131
+ '5b38bd129e83d5a0cad23921089490d50d4aae370428f8ddfffffa4c1564e184',
132
+ '5edb7ac43b82a06a8761e8d7be4979ebf2611f7dd79bf91c1c6b566a219ed766',
133
+ '5f0b62eab5e353ea6521651658fbb65359f443280a4afbd104d77d10f9f04c07',
134
+ '614fd18da1490560cdad1196e2492ab7062eab1a67b3a30f1d0585a7d6ba6824',
135
+ '69ddd7ea90bb57c93e135dc85ea6fcd5480b603239bdc454fc758b2a26cf7f79',
136
+ '76ef4762e573206006cbc338b17ca4bc200574a11928d90c3ef31c5e803e6c6f',
137
+ '83ce3c1229688a593d485f81973c0f9195431eda37cc5e36430e79c7a888638b',
138
+ '87c678bfb8b25f38f7e97b336956bbcf144bbacaa53647e61a2325bc1055316b',
139
+ '8d722f81a9c113c0791df136a2966db26c950a971db46b4199f4ea54b78bfb9f',
140
+ '8dbb5a7c06c20ef62dd912a36740992ff6e1e8583d42ede257c3affd7c769399',
141
+ '8f9e2751dcd574e9ba90e744ea92581fd0af640ae86ac1ce2198c90f96b44823',
142
+ '92a9d9833fe1944db366e8bfae7a95b6480c2d6c6c2a1be65d4236b608fca1bb',
143
+ '944554239d91ed9efedcf906d5e8113160b46fc816dc6bdc77b89da29b6562b9',
144
+ '9acfab7e43c8d880d06b262a94deeee4b4659989c3d0caf19baf6405e41ab7df',
145
+ '9d190b2e314566685be8a889e27aa8c7d7ae1d8aaddba3c1ecf9d24863cd34b9',
146
+ '9e503738722e0a104cf659ff9f92f0b5b3662acd112d4664d1e7db93abf46a59',
147
+ 'a0234f3bc8527ca5628eec81ad5d69895da5680dc91d1cb8477f33f878b95b0b',
148
+ 'a0459b9f63b22559f5fa5d4c6db3f9f72ff19342033578f073bf1d1b46cbb912',
149
+ 'a4310d50af18a6447190372a86afaf8b951ffb431d837f1e5688b45971ed1557',
150
+ 'a4b6b3996fc2f306b3fd8681bd63413d8c5009cc4fa329c2ccf0e2fa1b140305',
151
+ 'b32396746453442f353e616292bb20bbaa5d23b546450fdb9c54b8386167d529',
152
+ 'b478b812250df878635c2aa7ec7d155eaa625ee82916e2cd294361886cd1fbd4',
153
+ 'bb6ce72f0e64bfd93ade14b1becf8c41e7bc927cafb477a3a95878c01aa26c3e',
154
+ 'bcff2ab03578ebbfb219b65e854cf26a3d8dfe6d1acf3e765b8636827b81eaee',
155
+ 'c38dcb38959393358691ea4d4f3ce495ce748996e64ed1891d897a0fc4dd55c6',
156
+ 'c4fa68f8270924c300cbc0d3615a7b88e82231749cf6522452272222c9f0a83e',
157
+ 'ca2d82a08677072f8ab6764ff035676cfe3e5e325e012172df3f92096db79b85',
158
+ 'cb627d18b58ad56dde331a30456bc65c601a4e9b18dedcea08e7daaa07815ff0',
159
+ 'cb6b05d9e8e57cd882b10b4db70de4bb1de42ba48a7bd0318b635bf6e7781a9d',
160
+ 'cbb02707160f4f77291a27561448691ca5901808e5f36e758449a862aa5272cb',
161
+ 'cbb5af185e942a2402f9eacbc0ed5bb876eea3c1223623d00447e4f3ba554b65',
162
+ 'cf56ff46a4a186109dd96584b5eeb58a510c4275b0e5f94f40bbae865e19f673',
163
+ 'd17cd8ecd586b712238a482ce46fa5293970742f276d8ab6a9e46ee0288f3355',
164
+ 'ddcef1660de3b06996507f56168865a20b43cda89cf7e8735a82b83bba00c498',
165
+ 'e389360d0fdbaeb3d250584b4730314e222f39c156a020144e8d960561791506',
166
+ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
167
+ 'e6b8f8766485f807ae7f8dac1670461f07c0a13eef3a1ff717538d7abad391b4',
168
+ 'eb04cf5eb1f39afa762f2bb120f296cba520c1b97db1589565b81cb9a17b7244',
169
+ 'ebf3c02a8789b1fb7d511995d663b72906d913ce0d5e10568a8a77e2586167e7',
170
+ 'f5074a8f5b9a5b8142f34abe152f60364d770eae75ee3eeceb45b6b996509788',
171
+ 'f59db3f45d57fcec94ccd516e6c8ccb20dd4363feb2c44d8656e95f50fdd8df8',
172
+ 'fe863d0822fe7a2353fa484d5924e875656d3dc9fb58771f6f616f9d571bc592',
173
+ 'ff856a2d251dcd88d36656f450126798cfabaade40799c722de4d2b5db36a73a']
174
+ return roots.include? hash
175
+ end
106
176
  end
107
177
  end
108
178
  end