yawast 0.6.0 → 0.7.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 83f5d74a8796ef331534669c079965a7f0b558a6
4
- data.tar.gz: b98c0de6ec0b415fcbfe8ad5e6121834d6428c38
3
+ metadata.gz: 1a0010955fcbfb843d4eaf927682a22df138b4a6
4
+ data.tar.gz: df672ffa6576e142c62fe0904f07ef90cc54c612
5
5
  SHA512:
6
- metadata.gz: 69a364e0ea8fe6f53dc9cea28efca7520aa0ea5faf92d554ba04906a67c76362820f9772dc5fa477ebcc7b151e07e16033a2027031ec1d8bb3fb59a50075dc06
7
- data.tar.gz: 32705ac1392cdfc4bc5ae3b9f6ac6671f0355b60555fd279fa4374d11215c8a1e2a631098da57aea9f49cdb219f96627a1be30157352d91095a28f43090356b0
6
+ metadata.gz: d1152a1e138492093e18834d8d542b4438a3904703a9449e6092857212db4d04048afeb9329e49bb591e217c1be82d06c1451a986a5e3cd517b66933dbacf018
7
+ data.tar.gz: 6ab13462bab4524b4e8c0347d0065b7947799ad6ab4ef6bed5bf38dadf090ad2d85e9bc1384a73247d302820ce034ea949824565960b93fedeaabc52797cbea1
data/.travis.yml CHANGED
@@ -1,8 +1,11 @@
1
1
  language: ruby
2
2
  rvm:
3
- - 2.2.8
4
3
  - 2.3.5
5
4
  - 2.4.2
5
+ - 2.6.0
6
+ before_install:
7
+ - gem update --system
8
+ - gem install bundler
6
9
  script:
7
10
  - bundle exec rake
8
11
  - bundle exec rake submitcodeclimate
data/CHANGELOG.md CHANGED
@@ -1,4 +1,19 @@
1
- ## 0.6.0 - In Development
1
+ ## 0.7.0 - In Development
2
+
3
+ * [#38](https://github.com/adamcaudill/yawast/issues/38) - Report Generation Mode (work in progress)
4
+ * [#133](https://github.com/adamcaudill/yawast/issues/133) - Include a Timestamp In Output
5
+ * [#134](https://github.com/adamcaudill/yawast/issues/134) - Add options to DNS command
6
+ * [#135](https://github.com/adamcaudill/yawast/issues/135) - Incomplete Certificate Chain Warning
7
+ * [#137](https://github.com/adamcaudill/yawast/issues/137) - Warn on TLS 1.0
8
+ * [#138](https://github.com/adamcaudill/yawast/issues/138) - Warn on Symantec Roots
9
+ * [#139](https://github.com/adamcaudill/yawast/issues/139) - Add Spider Option
10
+ * [#140](https://github.com/adamcaudill/yawast/issues/140) - Save output on cancel
11
+ * [#141](https://github.com/adamcaudill/yawast/issues/141) - Flag --internalssl as Deprecated
12
+ * [#130](https://github.com/adamcaudill/yawast/issues/130) - Bug: HSTS Error leads to printing HTML
13
+ * [#132](https://github.com/adamcaudill/yawast/issues/132) - Bug: Typo in SSL Output
14
+ * [#142](https://github.com/adamcaudill/yawast/issues/142) - Bug: Error In Collecting DNS Information
15
+
16
+ ## 0.6.0 - 2018-01-16
2
17
 
3
18
  * [#54](https://github.com/adamcaudill/yawast/issues/54) - Check for Python version in Server header
4
19
  * [#59](https://github.com/adamcaudill/yawast/issues/59) - SSL Labs: Display Certificate Chain
data/LICENSE CHANGED
@@ -1,6 +1,6 @@
1
1
  BSD 3-Clause License
2
2
 
3
- Copyright (c) 2013-2017, Adam Caudill <adam@adamcaudill.com>
3
+ Copyright (c) 2013-2019, Adam Caudill <adam@adamcaudill.com>
4
4
  All rights reserved.
5
5
 
6
6
  Redistribution and use in source and binary forms, with or without
data/bin/yawast CHANGED
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
+ # encoding: UTF-8
2
3
 
3
4
  require 'commander/import'
4
5
  require File.dirname(__FILE__) + '/../lib/yawast'
@@ -26,6 +27,8 @@ command :scan do |c|
26
27
  c.option '--proxy STRING', String, 'HTTP Proxy Server (such as Burp Suite)'
27
28
  c.option '--cookie STRING', String, 'Session cookie'
28
29
  c.option '--nodns', 'Disable DNS checks'
30
+ c.option '--spider', 'Spider the site'
31
+ c.option '--output STRING', String, 'Output JSON file'
29
32
 
30
33
  c.action do |args, options|
31
34
  Yawast::Commands::Scan.process(args, options)
@@ -43,6 +46,7 @@ command :head do |c|
43
46
  c.option '--proxy STRING', String, 'HTTP Proxy Server (such as Burp Suite)'
44
47
  c.option '--cookie STRING', String, 'Session cookie'
45
48
  c.option '--nodns', 'Disable DNS checks'
49
+ c.option '--output STRING', String, 'Output JSON file'
46
50
 
47
51
  c.action do |args, options|
48
52
  Yawast::Commands::Head.process(args, options)
@@ -79,6 +83,10 @@ command :dns do |c|
79
83
  c.syntax = './yawast dns URL'
80
84
  c.description = 'Gets information about the server DNS configuration'
81
85
 
86
+ c.option '--srv', 'Scan for known SRV DNS Records'
87
+ c.option '--subdomains', 'Search for Common Subdomains'
88
+ c.option '--output STRING', String, 'Output JSON file'
89
+
82
90
  c.action do |args, options|
83
91
  Yawast::Commands::DNS.process(args, options)
84
92
  end
data/lib/commands/dns.rb CHANGED
@@ -6,10 +6,15 @@ module Yawast
6
6
 
7
7
  Yawast.header
8
8
 
9
+ if options.output != nil
10
+ Yawast::Shared::Output.setup uri, options
11
+ end
12
+
9
13
  puts "Scanning: #{uri}"
10
14
  puts
11
15
 
12
16
  Yawast::Scanner::Plugins::DNS::Generic.dns_info uri, options
17
+ Yawast::Shared::Output.write_file
13
18
  end
14
19
  end
15
20
  end
data/lib/scanner/core.rb CHANGED
@@ -14,10 +14,15 @@ module Yawast
14
14
 
15
15
  print_header
16
16
 
17
+ if options.output != nil
18
+ Yawast::Shared::Output.setup @uri, options
19
+ end
20
+
17
21
  ssl_redirect = Yawast::Scanner::Plugins::SSL::SSL.check_for_ssl_redirect @uri
18
22
  if ssl_redirect
19
23
  @uri = ssl_redirect
20
24
  puts "Server redirects to TLS: Scanning: #{@uri}"
25
+ Yawast::Shared::Output.log_value 'server_tls_redirect', @uri
21
26
  end
22
27
 
23
28
  Yawast::Scanner::Plugins::SSL::SSL.set_openssl_options
@@ -42,6 +47,7 @@ module Yawast
42
47
 
43
48
  #cache the HEAD result, so that we can minimize hits
44
49
  head = get_head
50
+ Yawast::Shared::Output.log_hash 'http', 'head', 'raw', head.to_hash
45
51
  Yawast::Scanner::Generic.head_info(head, @uri)
46
52
 
47
53
  #perfom SSL checks
@@ -63,6 +69,10 @@ module Yawast
63
69
  Yawast::Scanner::Generic.check_options(@uri)
64
70
  Yawast::Scanner::Generic.check_trace(@uri)
65
71
 
72
+ if options.spider
73
+ Yawast::Scanner::Plugins::Spider::Spider.spider(@uri)
74
+ end
75
+
66
76
  #check for common directories
67
77
  if options.dir
68
78
  Yawast::Scanner::Plugins::Http::DirectorySearch.search @uri, options.dirrecursive, options.dirlistredir
@@ -75,6 +85,7 @@ module Yawast
75
85
  # less than 24 hours. if a scan is that long, we have bigger problems
76
86
  elapsed_time = Time.at(Time.now - start_time).utc.strftime('%H:%M:%S')
77
87
 
88
+ Yawast::Shared::Output.write_file
78
89
  puts "Scan complete (#{elapsed_time})."
79
90
  rescue => e
80
91
  Yawast::Utilities.puts_error "Fatal Error: Can not continue. (#{e.class}: #{e.message})"
@@ -24,6 +24,7 @@ module Yawast
24
24
  Yawast::Utilities.puts_info 'HEAD:'
25
25
  head.each do |k, v|
26
26
  Yawast::Utilities.puts_info "\t\t#{k}: #{v}"
27
+ Yawast::Shared::Output.log_value 'http', 'head', k, v
27
28
 
28
29
  server = v if k.downcase == 'server'
29
30
  powered_by = v if k.downcase == 'x-powered-by'
@@ -40,7 +41,11 @@ module Yawast
40
41
 
41
42
  if k.downcase == 'set-cookie'
42
43
  #this chunk of magic manages to properly split cookies, when multiple are sent together
43
- v.gsub(/(,([^;,]*=)|,$)/) { "\r\n#{$2}" }.split(/\r\n/).each { |c| cookies.push(c) }
44
+ v.gsub(/(,([^;,]*=)|,$)/) { "\r\n#{$2}" }.split(/\r\n/).each do |c|
45
+ cookies.push(c)
46
+
47
+ Yawast::Shared::Output.log_append_value 'http', 'head', 'cookies', c
48
+ end
44
49
  end
45
50
  end
46
51
  puts ''
@@ -163,11 +168,13 @@ module Yawast
163
168
 
164
169
  if res['Public'] != nil
165
170
  Yawast::Utilities.puts_info "Public HTTP Verbs (OPTIONS): #{res['Public']}"
171
+ Yawast::Shared::Output.log_value 'http', 'options', 'public', res['Public']
166
172
 
167
173
  puts ''
168
174
  end
169
175
  if res['Allow'] != nil
170
176
  Yawast::Utilities.puts_info "Allow HTTP Verbs (OPTIONS): #{res['Allow']}"
177
+ Yawast::Shared::Output.log_value 'http', 'options', 'allow', res['Allow']
171
178
 
172
179
  puts ''
173
180
  end
@@ -187,6 +194,9 @@ module Yawast
187
194
 
188
195
  puts ''
189
196
  end
197
+
198
+ Yawast::Shared::Output.log_value 'http', 'trace', 'raw', res.body
199
+ Yawast::Shared::Output.log_value 'http', 'trace', 'code', res.code
190
200
  end
191
201
  end
192
202
 
@@ -203,6 +213,11 @@ module Yawast
203
213
 
204
214
  puts ''
205
215
  end
216
+
217
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'raw', res.body
218
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'code', res.code
219
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'content-type', res['Content-Type']
220
+ Yawast::Shared::Output.log_value 'http', 'propfind', 'length', res.body.length
206
221
  end
207
222
  end
208
223
  end
@@ -35,6 +35,8 @@ module Yawast
35
35
  cname = get_cname_record(domain)
36
36
  if cname != nil
37
37
  Yawast::Utilities.puts_info "\t\tCAA (#{domain}): CNAME Found: -> #{cname}"
38
+ Yawast::Shared::Output.log_value 'dns', 'caa', domain, "CNAME: #{cname}"
39
+
38
40
  chase_domain cname.to_s
39
41
  else
40
42
  print_caa_record domain
@@ -66,6 +68,8 @@ module Yawast
66
68
  # check for RDATA
67
69
  if rec.rdata != nil
68
70
  Yawast::Utilities.puts_info "\t\tCAA (#{domain}): #{rec.rdata}"
71
+
72
+ Yawast::Shared::Output.log_append_value 'dns', 'caa', domain, rec.rdata
69
73
  else
70
74
  Yawast::Utilities.puts_error "\t\tCAA (#{domain}): Invalid Response: #{ans.answer}"
71
75
  end
@@ -73,6 +77,8 @@ module Yawast
73
77
  else
74
78
  # no answer, so no records
75
79
  Yawast::Utilities.puts_info "\t\tCAA (#{domain}): No Records Found"
80
+
81
+ Yawast::Shared::Output.log_value 'dns', 'caa', domain, 'nil'
76
82
  end
77
83
  end
78
84
  end
@@ -21,9 +21,13 @@ module Yawast
21
21
 
22
22
  Yawast::Utilities.puts_info "\t\t#{ip.address} (#{host_name})"
23
23
 
24
+ Yawast::Shared::Output.log_value 'dns', 'a', ip.address, host_name
25
+
24
26
  # if address is private, force internal SSL mode, don't show links
25
27
  if IPAddr.new(ip.address.to_s, Socket::AF_INET).private?
26
28
  options.internalssl = true
29
+
30
+ Yawast::Shared::Output.log_value 'force_internal_ssl', true
27
31
  else
28
32
  #show network info
29
33
  Yawast::Utilities.puts_info "\t\t\t#{get_network_info(ip.address)}"
@@ -45,6 +49,8 @@ module Yawast
45
49
 
46
50
  Yawast::Utilities.puts_info "\t\t#{ip.address} (#{host_name})"
47
51
 
52
+ Yawast::Shared::Output.log_value 'dns', 'aaaa', ip.address, host_name
53
+
48
54
  # if address is private, force internal SSL mode, don't show links
49
55
  if IPAddr.new(ip.address.to_s, Socket::AF_INET6).private?
50
56
  options.internalssl = true
@@ -61,16 +67,24 @@ module Yawast
61
67
  unless txt.empty?
62
68
  txt.each do |rec|
63
69
  Yawast::Utilities.puts_info "\t\tTXT: #{rec.data}"
70
+
71
+ Yawast::Shared::Output.log_append_value 'dns', 'txt', uri.host, rec.data
64
72
  end
65
73
  end
66
74
 
67
75
  #check for higher-level TXT records, if we aren't already at the top
68
76
  if root_domain != uri.host
69
- txt = resv.getresources(root_domain, Resolv::DNS::Resource::IN::TXT)
70
- unless txt.empty?
71
- txt.each do |rec|
72
- Yawast::Utilities.puts_info "\t\tTXT (#{root_domain}): #{rec.data}"
77
+ begin
78
+ txt = resv.getresources(root_domain, Resolv::DNS::Resource::IN::TXT)
79
+ unless txt.empty?
80
+ txt.each do |rec|
81
+ Yawast::Utilities.puts_info "\t\tTXT (#{root_domain}): #{rec.data}"
82
+
83
+ Yawast::Shared::Output.log_append_value 'dns', 'txt', root_domain, rec.data
84
+ end
73
85
  end
86
+ rescue => e
87
+ Yawast::Utilities.puts_error "\t\tTXT: #{root_domain} (Error: #{e.message})"
74
88
  end
75
89
  end
76
90
 
@@ -81,6 +95,8 @@ module Yawast
81
95
  ip = resv.getaddress rec.exchange
82
96
 
83
97
  Yawast::Utilities.puts_info "\t\tMX: #{rec.exchange} (#{rec.preference}) - #{ip} (#{get_network_info(ip.to_s)})"
98
+
99
+ Yawast::Shared::Output.log_value 'dns', 'mx', rec.exchange, ip
84
100
  rescue => e
85
101
  Yawast::Utilities.puts_error "\t\tMX: #{rec.exchange} (#{rec.preference}) - Error: #{e.message})"
86
102
  end
@@ -96,6 +112,8 @@ module Yawast
96
112
  ip = resv.getaddress rec.exchange
97
113
 
98
114
  Yawast::Utilities.puts_info "\t\tMX (#{root_domain}): #{rec.exchange} (#{rec.preference}) - #{ip} (#{get_network_info(ip.to_s)})"
115
+
116
+ Yawast::Shared::Output.log_value 'dns', 'mx', rec.exchange, ip
99
117
  rescue => e
100
118
  Yawast::Utilities.puts_error "\t\tMX (#{root_domain}): #{rec.exchange} (#{rec.preference}) - Error: #{e.message})"
101
119
  end
@@ -109,6 +127,8 @@ module Yawast
109
127
  ip = resv.getaddress rec.name
110
128
 
111
129
  Yawast::Utilities.puts_info "\t\tNS: #{rec.name} - #{ip} (#{get_network_info(ip.to_s)})"
130
+
131
+ Yawast::Shared::Output.log_value 'dns', 'ns', rec.name, ip
112
132
  end
113
133
  end
114
134
 
@@ -145,6 +165,8 @@ module Yawast
145
165
  ip = resv.getaddress rec.target
146
166
 
147
167
  Yawast::Utilities.puts_info "\t\tSRV: #{host}: #{rec.target}:#{rec.port} - #{ip} (#{get_network_info(ip.to_s)})"
168
+
169
+ Yawast::Shared::Output.log_value 'dns', 'srv', host, "#{rec.target}:#{rec.port}"
148
170
  end
149
171
  end
150
172
  rescue
@@ -169,6 +191,8 @@ module Yawast
169
191
  else
170
192
  Yawast::Utilities.puts_info "\t\tA: #{host}: #{ip.address} (#{get_network_info(ip.address)})"
171
193
  end
194
+
195
+ Yawast::Shared::Output.log_value 'dns', 'subdomain', host, ip.address
172
196
  end
173
197
  end
174
198
  rescue
@@ -194,6 +218,8 @@ module Yawast
194
218
  ret = "#{network_info['as_country_code']} - #{network_info['as_description']}"
195
219
  @netinfo[ip] = ret
196
220
 
221
+ Yawast::Shared::Output.log_value 'dns', 'asn_info', ip, ret
222
+
197
223
  return ret
198
224
  rescue => e
199
225
  @netinfo_failed = true
@@ -99,10 +99,12 @@ module Yawast
99
99
 
100
100
  if res.code == '200'
101
101
  @results.push "\tFound: '#{uri}'"
102
+ Yawast::Shared::Output.log_append_value 'http', 'http_dir', uri
102
103
 
103
104
  load_queue uri if @recursive
104
105
  elsif res.code == '301' && @list_redirects
105
106
  @results.push "\tFound Redirect: '#{uri} -> '#{res['Location']}'"
107
+ Yawast::Shared::Output.log_value 'http', 'http_dir_redirect', uri, res['Location']
106
108
  end
107
109
  rescue => e
108
110
  unless e.message.include?('end of file') || e.message.include?('getaddrinfo')
@@ -163,6 +163,7 @@ module Yawast
163
163
 
164
164
  if res.code == '200'
165
165
  @results.push "'#{uri.path}' found: #{uri}"
166
+ Yawast::Shared::Output.log_append_value 'http', 'http_file', uri
166
167
  end
167
168
  rescue => e
168
169
  unless e.message.include?('end of file') || e.message.include?('getaddrinfo')
@@ -71,6 +71,8 @@ module Yawast
71
71
 
72
72
  if version != nil && version[0] != nil
73
73
  Yawast::Utilities.puts_warn "Apache Tomcat Version Found: #{version[0]}"
74
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_version', version[0]
75
+
74
76
  puts "\t\t\"curl -X XYZ #{uri}\""
75
77
 
76
78
  puts ''
@@ -80,8 +82,8 @@ module Yawast
80
82
  end
81
83
 
82
84
  def self.check_tomcat_manager(uri)
83
- check_tomcat_manager_paths uri, 'manager', 'Manager'
84
- check_tomcat_manager_paths uri, 'host-manager', 'Host Manager'
85
+ check_tomcat_manager_paths uri.copy, 'manager', 'Manager'
86
+ check_tomcat_manager_paths uri.copy, 'host-manager', 'Host Manager'
85
87
  end
86
88
 
87
89
  def self.check_tomcat_manager_paths(uri, base_path, manager)
@@ -93,16 +95,19 @@ module Yawast
93
95
  if ret.include? '<tt>conf/tomcat-users.xml</tt>'
94
96
  #this will get Tomcat 7+
95
97
  Yawast::Utilities.puts_warn "Apache Tomcat #{manager} page found: #{uri}"
98
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_mgr', manager, uri
96
99
  check_tomcat_manager_passwords uri, manager
97
100
 
98
101
  puts ''
99
102
  else
100
103
  #check for Tomcat 6 and below
104
+ uri = uri.copy
101
105
  uri.path = "/#{base_path}"
102
106
  ret = Yawast::Shared::Http.get(uri)
103
107
 
104
108
  if ret.include? '<tt>conf/tomcat-users.xml</tt>'
105
109
  Yawast::Utilities.puts_warn "Apache Tomcat #{manager} page found: #{uri}"
110
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_mgr', manager, uri
106
111
  check_tomcat_manager_passwords uri, manager
107
112
 
108
113
  puts ''
@@ -125,6 +130,8 @@ module Yawast
125
130
  if ret.include?('<font size="+2">Tomcat Web Application Manager</font>') ||
126
131
  ret.include?('<font size="+2">Tomcat Virtual Host Manager</font>')
127
132
  Yawast::Utilities.puts_vuln "Apache Tomcat #{manager} weak password: #{credentials}"
133
+
134
+ Yawast::Shared::Output.log_value 'apache', 'tomcat_mgr_pwd', uri, credentials
128
135
  end
129
136
  end
130
137
 
@@ -133,9 +140,13 @@ module Yawast
133
140
  uri.path = "/#{SecureRandom.hex}.jsp/"
134
141
  uri.query = '' if uri.query != nil
135
142
 
143
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'path', uri
144
+
136
145
  # we'll use this to verify that it actually worked
137
146
  check_value = SecureRandom.hex
138
147
 
148
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'check_value', check_value
149
+
139
150
  # upload the JSP file
140
151
  req_data = "<% out.println(\"#{check_value}\");%>"
141
152
  Yawast::Shared::Http.put(uri, req_data)
@@ -143,8 +154,14 @@ module Yawast
143
154
  # check to see of we get check_value back
144
155
  uri.path = uri.path.chomp('/')
145
156
  res = Yawast::Shared::Http.get(uri)
157
+
158
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'body', res
159
+
146
160
  if res.include? check_value
147
161
  Yawast::Utilities.puts_vuln "Apache Tomcat PUT RCE (CVE-2017-12615): #{uri}"
162
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'vulnerable', true
163
+ else
164
+ Yawast::Shared::Output.log_value 'apache', 'cve_2017_12615', 'vulnerable', false
148
165
  end
149
166
  end
150
167
 
@@ -159,9 +176,12 @@ module Yawast
159
176
  search.push '/struts2-rest-showcase/'
160
177
 
161
178
  search.each do |path|
179
+ uri = uri.copy
162
180
  uri.path = path
163
181
 
164
182
  ret = Yawast::Shared::Http.get_status_code uri
183
+ Yawast::Shared::Output.log_value 'apache', 'struts2_sample_files', uri, ret
184
+
165
185
  if ret == 200
166
186
  Yawast::Utilities.puts_warn "Apache Struts2 Sample Files: #{uri}"
167
187
  end
@@ -176,6 +196,7 @@ module Yawast
176
196
 
177
197
  if ret.include? search
178
198
  Yawast::Utilities.puts_vuln "#{search} page found: #{uri}"
199
+ Yawast::Shared::Output.log_value 'apache', 'page_search', search, uri
179
200
  puts ''
180
201
  end
181
202
  end
@@ -48,6 +48,9 @@ module Yawast
48
48
  if res.code == 200
49
49
  Yawast::Utilities.puts_vuln 'ASP.NET Debugging Enabled'
50
50
  end
51
+
52
+ Yawast::Shared::Output.log_value 'http', 'asp_net_debug', 'raw', res.body
53
+ Yawast::Shared::Output.log_value 'http', 'asp_net_debug', 'code', res.code
51
54
  end
52
55
  end
53
56
  end
@@ -0,0 +1,65 @@
1
+ require 'nokogiri'
2
+
3
+ module Yawast
4
+ module Scanner
5
+ module Plugins
6
+ module Spider
7
+ class Spider
8
+ def self.spider(uri)
9
+ @uri = uri.copy
10
+
11
+ @workers = []
12
+ @results = Queue.new
13
+
14
+ @links = []
15
+ @links.push @uri.to_s
16
+ puts 'Spidering site...'
17
+ get_links @uri
18
+
19
+ results = Thread.new do
20
+ begin
21
+ while true
22
+ if @results.length > 0
23
+ out = @results.pop(true)
24
+ Yawast::Utilities.puts_info out
25
+ Yawast::Shared::Output.log_append_value 'spider', 'get', out
26
+ end
27
+ end
28
+ rescue ThreadError
29
+ #do nothing
30
+ end
31
+ end
32
+
33
+ @workers.map(&:join)
34
+ results.terminate
35
+
36
+ puts
37
+ end
38
+
39
+ def self.get_links(uri)
40
+ # get the page, and work out from there
41
+ res = Yawast::Shared::Http.get_with_code uri
42
+ doc = Nokogiri::HTML res[:body]
43
+
44
+ results = doc.css('a').map { |link| link['href'] }
45
+
46
+ results.each do |link|
47
+ # check to see if this link is in scope
48
+ if link.to_s.include?(@uri.to_s) && res[:code] == '200'
49
+ # check to see if we've already seen this one
50
+ unless @links.include? link.to_s
51
+ @links.push link.to_s
52
+ @results.push "#{link.to_s}"
53
+
54
+ @workers.push Thread.new {
55
+ get_links URI.parse(link)
56
+ }
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
@@ -45,20 +45,25 @@ module Yawast
45
45
 
46
46
  Yawast::Utilities.puts_info "HSTS Preload: Chrome - #{chrome}; Firefox - #{firefox}; Tor - #{tor}"
47
47
  rescue => e
48
- Yawast::Utilities.puts_error "Error getting HSTS preload information: #{e.message}"
48
+ if e.message.include? 'unexpected token'
49
+ #this means we have a parsing error - don't need to include the entire message
50
+ Yawast::Utilities.puts_error "Error getting HSTS preload information: #{e.message.truncate(30)}"
51
+ else
52
+ Yawast::Utilities.puts_error "Error getting HSTS preload information: #{e.message}"
53
+ end
49
54
  end
50
55
  end
51
56
 
52
57
  def self.set_openssl_options
53
- #change certain defaults, to make things work better
54
- #we prefer RSA, to avoid issues with small DH keys
58
+ # change certain defaults, to make things work better
59
+ # we prefer RSA, to avoid issues with small DH keys
55
60
  OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ciphers] = 'RSA:ALL:COMPLEMENTOFALL'
56
61
  OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:verify_mode] = OpenSSL::SSL::VERIFY_NONE
57
62
  OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options] = OpenSSL::SSL::OP_ALL
58
63
  end
59
64
 
60
65
  def self.check_for_ssl_redirect(uri)
61
- #check to see if the site redirects to SSL by default
66
+ # check to see if the site redirects to SSL by default
62
67
  if uri.scheme != 'https'
63
68
  head = Yawast::Shared::Http.head(uri)
64
69
 
@@ -67,11 +72,11 @@ module Yawast
67
72
  location = URI.parse(head['Location'])
68
73
 
69
74
  if location.scheme == 'https'
70
- #we run this through extract_uri as it performs a few checks we need
75
+ # we run this through extract_uri as it performs a few checks we need
71
76
  return Yawast::Shared::Uri.extract_uri location.to_s
72
77
  end
73
78
  rescue
74
- #we don't care if this fails
79
+ # we don't care if this fails
75
80
  end
76
81
  end
77
82
  end
@@ -94,8 +99,11 @@ module Yawast
94
99
  ssl.connect
95
100
 
96
101
  # this provides a bunch of useful info, that's already formatted
97
- # instead of building this manually, we'll let OpenSSL do the work
98
- puts ssl.session.to_text
102
+ # instead of building this manually, we'll let OpenSSL do the
103
+ session_info = ssl.session.to_text
104
+ puts session_info
105
+
106
+ Yawast::Shared::Output.log_value 'ssl', 'session', 'info', session_info
99
107
 
100
108
  puts
101
109
  end
@@ -103,6 +111,68 @@ module Yawast
103
111
  Yawast::Utilities.puts_error "SSL Information: Error Getting Details: #{e.message}"
104
112
  end
105
113
  end
114
+
115
+ def self.check_symantec_root(hash)
116
+ roots = ['08297a4047dba23680c731db6e317653ca7848e1bebd3a0b0179a707f92cf178',
117
+ '2399561127a57125de8cefea610ddf2fa078b5c8067f4e828290bfb860e84b3c',
118
+ '2834991cf677466d22baac3b0055e5b911d9a9e55f5b85ba02dc566782c30e8a',
119
+ '2930bd09a07126bdc17288d4f2ad84645ec948607907a97b5ed0b0b05879ef69',
120
+ '2f274e48aba4ac7b765933101775506dc30ee38ef6acd5c04932cfe041234220',
121
+ '309b4a87f6ca56c93169aaa99c6d988854d7892bd5437e2d07b29cbeda55d35d',
122
+ '3266967e59cd68008d9dd320811185c704205e8d95fdd84f1c7b311e6704fc32',
123
+ '341de98b1392abf7f4ab90a960cf25d4bd6ec65b9a51ce6ed067d00ec7ce9b7f',
124
+ '363f3c849eab03b0a2a0f636d7b86d04d3ac7fcfe26a0a9121ab9795f6e176df',
125
+ '37d51006c512eaab626421f1ec8c92013fc5f82ae98ee533eb4619b8deb4d06c',
126
+ '3a43e220fe7f3ea9653d1e21742eac2b75c20fd8980305bc502caf8c2d9b41a1',
127
+ '3f9f27d583204b9e09c8a3d2066c4b57d3a2479c3693650880505698105dbce9',
128
+ '44640a0a0e4d000fbd574d2b8a07bdb4d1dfed3b45baaba76f785778c7011961',
129
+ '4b03f45807ad70f21bfc2cae71c9fde4604c064cf5ffb686bae5dbaad7fdd34c',
130
+ '53dfdfa4e297fcfe07594e8c62d5b8ab06b32c7549f38a163094fd6429d5da43',
131
+ '5b38bd129e83d5a0cad23921089490d50d4aae370428f8ddfffffa4c1564e184',
132
+ '5edb7ac43b82a06a8761e8d7be4979ebf2611f7dd79bf91c1c6b566a219ed766',
133
+ '5f0b62eab5e353ea6521651658fbb65359f443280a4afbd104d77d10f9f04c07',
134
+ '614fd18da1490560cdad1196e2492ab7062eab1a67b3a30f1d0585a7d6ba6824',
135
+ '69ddd7ea90bb57c93e135dc85ea6fcd5480b603239bdc454fc758b2a26cf7f79',
136
+ '76ef4762e573206006cbc338b17ca4bc200574a11928d90c3ef31c5e803e6c6f',
137
+ '83ce3c1229688a593d485f81973c0f9195431eda37cc5e36430e79c7a888638b',
138
+ '87c678bfb8b25f38f7e97b336956bbcf144bbacaa53647e61a2325bc1055316b',
139
+ '8d722f81a9c113c0791df136a2966db26c950a971db46b4199f4ea54b78bfb9f',
140
+ '8dbb5a7c06c20ef62dd912a36740992ff6e1e8583d42ede257c3affd7c769399',
141
+ '8f9e2751dcd574e9ba90e744ea92581fd0af640ae86ac1ce2198c90f96b44823',
142
+ '92a9d9833fe1944db366e8bfae7a95b6480c2d6c6c2a1be65d4236b608fca1bb',
143
+ '944554239d91ed9efedcf906d5e8113160b46fc816dc6bdc77b89da29b6562b9',
144
+ '9acfab7e43c8d880d06b262a94deeee4b4659989c3d0caf19baf6405e41ab7df',
145
+ '9d190b2e314566685be8a889e27aa8c7d7ae1d8aaddba3c1ecf9d24863cd34b9',
146
+ '9e503738722e0a104cf659ff9f92f0b5b3662acd112d4664d1e7db93abf46a59',
147
+ 'a0234f3bc8527ca5628eec81ad5d69895da5680dc91d1cb8477f33f878b95b0b',
148
+ 'a0459b9f63b22559f5fa5d4c6db3f9f72ff19342033578f073bf1d1b46cbb912',
149
+ 'a4310d50af18a6447190372a86afaf8b951ffb431d837f1e5688b45971ed1557',
150
+ 'a4b6b3996fc2f306b3fd8681bd63413d8c5009cc4fa329c2ccf0e2fa1b140305',
151
+ 'b32396746453442f353e616292bb20bbaa5d23b546450fdb9c54b8386167d529',
152
+ 'b478b812250df878635c2aa7ec7d155eaa625ee82916e2cd294361886cd1fbd4',
153
+ 'bb6ce72f0e64bfd93ade14b1becf8c41e7bc927cafb477a3a95878c01aa26c3e',
154
+ 'bcff2ab03578ebbfb219b65e854cf26a3d8dfe6d1acf3e765b8636827b81eaee',
155
+ 'c38dcb38959393358691ea4d4f3ce495ce748996e64ed1891d897a0fc4dd55c6',
156
+ 'c4fa68f8270924c300cbc0d3615a7b88e82231749cf6522452272222c9f0a83e',
157
+ 'ca2d82a08677072f8ab6764ff035676cfe3e5e325e012172df3f92096db79b85',
158
+ 'cb627d18b58ad56dde331a30456bc65c601a4e9b18dedcea08e7daaa07815ff0',
159
+ 'cb6b05d9e8e57cd882b10b4db70de4bb1de42ba48a7bd0318b635bf6e7781a9d',
160
+ 'cbb02707160f4f77291a27561448691ca5901808e5f36e758449a862aa5272cb',
161
+ 'cbb5af185e942a2402f9eacbc0ed5bb876eea3c1223623d00447e4f3ba554b65',
162
+ 'cf56ff46a4a186109dd96584b5eeb58a510c4275b0e5f94f40bbae865e19f673',
163
+ 'd17cd8ecd586b712238a482ce46fa5293970742f276d8ab6a9e46ee0288f3355',
164
+ 'ddcef1660de3b06996507f56168865a20b43cda89cf7e8735a82b83bba00c498',
165
+ 'e389360d0fdbaeb3d250584b4730314e222f39c156a020144e8d960561791506',
166
+ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
167
+ 'e6b8f8766485f807ae7f8dac1670461f07c0a13eef3a1ff717538d7abad391b4',
168
+ 'eb04cf5eb1f39afa762f2bb120f296cba520c1b97db1589565b81cb9a17b7244',
169
+ 'ebf3c02a8789b1fb7d511995d663b72906d913ce0d5e10568a8a77e2586167e7',
170
+ 'f5074a8f5b9a5b8142f34abe152f60364d770eae75ee3eeceb45b6b996509788',
171
+ 'f59db3f45d57fcec94ccd516e6c8ccb20dd4363feb2c44d8656e95f50fdd8df8',
172
+ 'fe863d0822fe7a2353fa484d5924e875656d3dc9fb58771f6f616f9d571bc592',
173
+ 'ff856a2d251dcd88d36656f450126798cfabaade40799c722de4d2b5db36a73a']
174
+ return roots.include? hash
175
+ end
106
176
  end
107
177
  end
108
178
  end