site-inspector 1.0.2 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +8 -0
  3. data/.rubocop.yml +42 -0
  4. data/.rubocop_todo.yml +139 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +9 -0
  7. data/Gemfile +7 -0
  8. data/Guardfile +10 -0
  9. data/README.md +189 -0
  10. data/Rakefile +10 -0
  11. data/bin/site-inspector +50 -22
  12. data/lib/cliver/dependency_ext.rb +24 -0
  13. data/lib/site-inspector.rb +62 -615
  14. data/lib/site-inspector/cache.rb +10 -51
  15. data/lib/site-inspector/checks/accessibility.rb +135 -0
  16. data/lib/site-inspector/checks/check.rb +54 -0
  17. data/lib/site-inspector/checks/content.rb +85 -0
  18. data/lib/site-inspector/checks/cookies.rb +45 -0
  19. data/lib/site-inspector/checks/dns.rb +138 -0
  20. data/lib/site-inspector/checks/headers.rb +68 -0
  21. data/lib/site-inspector/checks/hsts.rb +81 -0
  22. data/lib/site-inspector/checks/https.rb +40 -0
  23. data/lib/site-inspector/checks/sniffer.rb +67 -0
  24. data/lib/site-inspector/checks/wappalyzer.rb +62 -0
  25. data/lib/site-inspector/checks/whois.rb +36 -0
  26. data/lib/site-inspector/disk_cache.rb +42 -0
  27. data/lib/site-inspector/domain.rb +271 -0
  28. data/lib/site-inspector/endpoint.rb +217 -0
  29. data/lib/site-inspector/rails_cache.rb +13 -0
  30. data/lib/site-inspector/version.rb +5 -0
  31. data/package-lock.json +505 -0
  32. data/package.json +23 -0
  33. data/script/bootstrap +2 -0
  34. data/script/cibuild +11 -0
  35. data/script/console +3 -0
  36. data/script/pa11y-version +10 -0
  37. data/script/release +38 -0
  38. data/site-inspector.gemspec +42 -0
  39. data/spec/checks/site_inspector_endpoint_accessibility_spec.rb +84 -0
  40. data/spec/checks/site_inspector_endpoint_check_spec.rb +42 -0
  41. data/spec/checks/site_inspector_endpoint_content_spec.rb +117 -0
  42. data/spec/checks/site_inspector_endpoint_cookies_spec.rb +73 -0
  43. data/spec/checks/site_inspector_endpoint_dns_spec.rb +184 -0
  44. data/spec/checks/site_inspector_endpoint_headers_spec.rb +65 -0
  45. data/spec/checks/site_inspector_endpoint_hsts_spec.rb +92 -0
  46. data/spec/checks/site_inspector_endpoint_https_spec.rb +49 -0
  47. data/spec/checks/site_inspector_endpoint_sniffer_spec.rb +150 -0
  48. data/spec/checks/site_inspector_endpoint_wappalyzer_spec.rb +34 -0
  49. data/spec/checks/site_inspector_endpoint_whois_spec.rb +26 -0
  50. data/spec/fixtures/wappalyzer.json +125 -0
  51. data/spec/site_inspector_cache_spec.rb +15 -0
  52. data/spec/site_inspector_disk_cache_spec.rb +39 -0
  53. data/spec/site_inspector_domain_spec.rb +271 -0
  54. data/spec/site_inspector_endpoint_spec.rb +252 -0
  55. data/spec/site_inspector_spec.rb +48 -0
  56. data/spec/spec_helper.rb +19 -0
  57. metadata +204 -63
  58. data/lib/site-inspector/compliance.rb +0 -19
  59. data/lib/site-inspector/dns.rb +0 -92
  60. data/lib/site-inspector/headers.rb +0 -59
  61. data/lib/site-inspector/sniffer.rb +0 -26
@@ -0,0 +1,184 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+ require 'dnsruby'
5
+
6
+ describe SiteInspector::Endpoint::Dns do
7
+ subject do
8
+ stub_request(:head, 'http://github.com/').to_return(status: 200)
9
+ endpoint = SiteInspector::Endpoint.new('http://github.com')
10
+ described_class.new(endpoint)
11
+ end
12
+
13
+ it 'inits the resolver' do
14
+ expect(described_class.resolver.class).to eql(Dnsruby::Resolver)
15
+ end
16
+
17
+ # NOTE: these tests makes external calls
18
+ context 'live tests' do
19
+ it 'runs the query' do
20
+ expect(subject.query).not_to be_empty
21
+ end
22
+
23
+ context 'resolv' do
24
+ it 'returns the IP' do
25
+ expect(subject.ip).to include('140.82.')
26
+ end
27
+
28
+ it 'returns the hostname' do
29
+ expect(subject.hostname.sld).to eql('github')
30
+ end
31
+ end
32
+ end
33
+
34
+ context 'stubbed tests' do
35
+ before do
36
+ record = Dnsruby::RR.create type: 'A', address: '1.2.3.4', name: 'test'
37
+ allow(subject).to receive(:records) { [record] }
38
+ allow(subject).to receive(:query).and_return([])
39
+ end
40
+
41
+ it 'returns the records' do
42
+ expect(subject.records.count).to be(1)
43
+ expect(subject.records.first.class).to eql(Dnsruby::RR::IN::A)
44
+ end
45
+
46
+ it 'knows if a record exists' do
47
+ expect(subject.has_record?('A')).to be(true)
48
+ expect(subject.has_record?('CNAME')).to be(false)
49
+ end
50
+
51
+ it 'knows if a domain supports dnssec' do
52
+ expect(subject.dnssec?).to be(false)
53
+
54
+ # via https://github.com/alexdalitz/dnsruby/blob/master/test/tc_dnskey.rb
55
+ input = 'example.com. 86400 IN DNSKEY 256 3 5 ( AQPSKmynfzW4kyBv015MUG2DeIQ3' \
56
+ 'Cbl+BBZH4b/0PY1kxkmvHjcZc8no' \
57
+ 'kfzj31GajIQKY+5CptLr3buXA10h' \
58
+ 'WqTkF7H6RfoRqXQeogmMHfpftf6z' \
59
+ 'Mv1LyBUgia7za6ZEzOJBOztyvhjL' \
60
+ '742iU/TpPSEDhm2SNKLijfUppn1U' \
61
+ 'aNvv4w== )'
62
+
63
+ record = Dnsruby::RR.create input
64
+ allow(subject).to receive(:records) { [record] }
65
+
66
+ expect(subject.dnssec?).to be(true)
67
+ end
68
+
69
+ it 'knows if a domain supports ipv6' do
70
+ expect(subject.ipv6?).to be(false)
71
+
72
+ input = {
73
+ type: 'AAAA',
74
+ name: 'test',
75
+ address: '102:304:506:708:90a:b0c:d0e:ff10'
76
+ }
77
+ record = Dnsruby::RR.create input
78
+ allow(subject).to receive(:records) { [record] }
79
+
80
+ expect(subject.ipv6?).to be(true)
81
+ end
82
+
83
+ it "knows it's not a localhost address" do
84
+ expect(subject.localhost?).to be(false)
85
+ end
86
+
87
+ context 'hostname detection' do
88
+ it 'lists cnames' do
89
+ records = []
90
+
91
+ records.push Dnsruby::RR.create(
92
+ type: 'CNAME',
93
+ domainname: 'example.com',
94
+ name: 'example'
95
+ )
96
+
97
+ records.push Dnsruby::RR.create(
98
+ type: 'CNAME',
99
+ domainname: 'github.com',
100
+ name: 'github'
101
+ )
102
+
103
+ allow(subject).to receive(:records) { records }
104
+
105
+ expect(subject.cnames.count).to be(2)
106
+ expect(subject.cnames.first.sld).to eql('example')
107
+ end
108
+
109
+ it "knows when a domain doesn't have a cdn" do
110
+ expect(subject.cdn?).to be(false)
111
+ end
112
+
113
+ it 'detects CDNs' do
114
+ records = [Dnsruby::RR.create(
115
+ type: 'CNAME',
116
+ domainname: 'foo.cloudfront.net',
117
+ name: 'example'
118
+ )]
119
+ allow(subject).to receive(:records) { records }
120
+
121
+ expect(subject.send(:detect_by_hostname, 'cdn')).to be(:cloudfront)
122
+ expect(subject.cdn).to be(:cloudfront)
123
+ expect(subject.cdn?).to be(true)
124
+ end
125
+
126
+ it 'builds that path to a data file' do
127
+ path = subject.send(:data_path, 'foo')
128
+ expected = File.expand_path '../../lib/data/foo.yml', File.dirname(__FILE__)
129
+ expect(path).to eql(expected)
130
+ end
131
+
132
+ it 'loads data files' do
133
+ data = subject.send(:load_data, 'cdn')
134
+ expect(data.keys).to include('cloudfront')
135
+ end
136
+
137
+ it "knows when a domain isn't cloud" do
138
+ expect(subject.cloud?).to be(false)
139
+ end
140
+
141
+ it 'detects cloud providers' do
142
+ records = [Dnsruby::RR.create(
143
+ type: 'CNAME',
144
+ domainname: 'foo.herokuapp.com',
145
+ name: 'example'
146
+ )]
147
+ allow(subject).to receive(:records) { records }
148
+
149
+ expect(subject.send(:detect_by_hostname, 'cloud')).to be(:heroku)
150
+ expect(subject.cloud_provider).to be(:heroku)
151
+ expect(subject.cloud?).to be(true)
152
+ end
153
+
154
+ it "knows when a domain doesn't have google apps" do
155
+ expect(subject.google_apps?).to be(false)
156
+ end
157
+
158
+ it 'knows when a domain is using google apps' do
159
+ records = [Dnsruby::RR.create(
160
+ type: 'MX',
161
+ exchange: 'mx1.google.com',
162
+ name: 'example',
163
+ preference: 10
164
+ )]
165
+ allow(subject).to receive(:records) { records }
166
+ expect(subject.google_apps?).to be(true)
167
+ end
168
+ end
169
+ end
170
+
171
+ context 'localhost' do
172
+ before do
173
+ allow(subject).to receive(:ip).and_return('127.0.0.1')
174
+ end
175
+
176
+ it "knows it's a localhost address" do
177
+ expect(subject.localhost?).to be(true)
178
+ end
179
+
180
+ it 'returns a LocalhostError' do
181
+ expect(subject.to_h).to eql(error: SiteInspector::Endpoint::Dns::LocalhostError)
182
+ end
183
+ end
184
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ describe SiteInspector::Endpoint::Headers do
6
+ subject do
7
+ stub_request(:head, 'http://example.com/')
8
+ .to_return(status: 200, headers: { foo: 'bar' })
9
+ endpoint = SiteInspector::Endpoint.new('http://example.com')
10
+ described_class.new(endpoint)
11
+ end
12
+
13
+ def stub_header(header, value)
14
+ allow(subject).to receive(:headers) { { header => value } }
15
+ end
16
+
17
+ it 'parses the headers' do
18
+ expect(subject.headers.count).to be(1)
19
+ expect(subject.headers.keys).to include('foo')
20
+ end
21
+
22
+ it 'returns a header' do
23
+ expect(subject['foo']).to eql('bar')
24
+ expect(subject.headers['foo']).to eql('bar')
25
+ end
26
+
27
+ it 'knows the server' do
28
+ stub_header 'server', 'foo'
29
+ expect(subject.server).to eql('foo')
30
+ end
31
+
32
+ it 'knows if a server has an xss protection header' do
33
+ stub_header 'x-xss-protection', 'foo'
34
+ expect(subject.xss_protection).to eql('foo')
35
+ end
36
+
37
+ it 'validates xss-protection' do
38
+ stub_header 'x-xss-protection', 'foo'
39
+ expect(subject.xss_protection?).to be(false)
40
+
41
+ stub_header 'x-xss-protection', '1; mode=block'
42
+ expect(subject.xss_protection?).to be(true)
43
+ end
44
+
45
+ it 'checks for clickjack proetection' do
46
+ expect(subject.click_jacking_protection?).to be(false)
47
+ stub_header 'x-frame-options', 'foo'
48
+ expect(subject.click_jacking_protection).to eql('foo')
49
+ expect(subject.click_jacking_protection?).to be(true)
50
+ end
51
+
52
+ it 'checks for CSP' do
53
+ expect(subject.content_security_policy?).to be(false)
54
+ stub_header 'content-security-policy', 'foo'
55
+ expect(subject.content_security_policy).to eql('foo')
56
+ expect(subject.content_security_policy?).to be(true)
57
+ end
58
+
59
+ it 'checks for strict-transport-security' do
60
+ expect(subject.strict_transport_security?).to be(false)
61
+ stub_header 'strict-transport-security', 'foo'
62
+ expect(subject.strict_transport_security).to eql('foo')
63
+ expect(subject.strict_transport_security?).to be(true)
64
+ end
65
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ describe SiteInspector::Endpoint::Hsts do
6
+ subject do
7
+ headers = { 'strict-transport-security' => 'max-age=31536000; includeSubDomains;' }
8
+ stub_request(:head, 'http://example.com/')
9
+ .to_return(status: 200, headers: headers)
10
+ endpoint = SiteInspector::Endpoint.new('http://example.com')
11
+ described_class.new(endpoint)
12
+ end
13
+
14
+ def stub_header(value)
15
+ allow(subject).to receive(:header) { value }
16
+ end
17
+
18
+ it 'returns the headers' do
19
+ expect(subject.send(:headers).class).to eql(SiteInspector::Endpoint::Headers)
20
+ end
21
+
22
+ it 'returns the HSTS header' do
23
+ expect(subject.send(:header)).to eql('max-age=31536000; includeSubDomains;')
24
+ end
25
+
26
+ it 'parses the directives' do
27
+ expect(subject.send(:directives).count).to be(2)
28
+ expect(subject.send(:directives).first).to eql('max-age=31536000')
29
+ expect(subject.send(:directives).last).to eql('includeSubDomains')
30
+ end
31
+
32
+ it 'parses pairs' do
33
+ expect(subject.send(:pairs).keys).to include(:"max-age")
34
+ expect(subject.send(:pairs)[:"max-age"]).to eql('31536000')
35
+ end
36
+
37
+ it 'knows if the header is valid' do
38
+ expect(subject.valid?).to be(true)
39
+
40
+ allow(subject).to receive(:pairs).and_return(['fo o' => 'bar'])
41
+ expect(subject.valid?).to be(false)
42
+
43
+ allow(subject).to receive(:pairs).and_return(["fo'o" => 'bar'])
44
+ expect(subject.valid?).to be(false)
45
+ end
46
+
47
+ it 'knows the max age' do
48
+ expect(subject.max_age).to be(31_536_000)
49
+ end
50
+
51
+ it 'knows if subdomains are included' do
52
+ expect(subject.include_subdomains?).to be(true)
53
+ allow(subject).to receive(:pairs).and_return(foo: 'bar')
54
+ expect(subject.include_subdomains?).to be(false)
55
+ end
56
+
57
+ it "knows if it's preloaded" do
58
+ expect(subject.preload?).to be(false)
59
+ allow(subject).to receive(:pairs).and_return(preload: nil)
60
+ expect(subject.preload?).to be(true)
61
+ end
62
+
63
+ it "knows if it's enabled" do
64
+ expect(subject.enabled?).to be(true)
65
+
66
+ allow(subject).to receive(:pairs).and_return("max-age": 0)
67
+ expect(subject.preload?).to be(false)
68
+
69
+ allow(subject).to receive(:pairs).and_return(foo: 'bar')
70
+ expect(subject.preload?).to be(false)
71
+ end
72
+
73
+ it "knows if it's preload ready" do
74
+ expect(subject.preload_ready?).to be(false)
75
+
76
+ pairs = { "max-age": 10_886_401, preload: nil, includesubdomains: nil }
77
+ allow(subject).to receive(:pairs) { pairs }
78
+ expect(subject.preload_ready?).to be(true)
79
+
80
+ pairs = { "max-age": 10_886_401, includesubdomains: nil }
81
+ allow(subject).to receive(:pairs) { pairs }
82
+ expect(subject.preload_ready?).to be(false)
83
+
84
+ pairs = { "max-age": 10_886_401, preload: nil, includesubdomains: nil }
85
+ allow(subject).to receive(:pairs) { pairs }
86
+ expect(subject.preload_ready?).to be(true)
87
+
88
+ pairs = { "max-age": 5, preload: nil, includesubdomains: nil }
89
+ allow(subject).to receive(:pairs) { pairs }
90
+ expect(subject.preload_ready?).to be(false)
91
+ end
92
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ describe SiteInspector::Endpoint::Https do
6
+ subject do
7
+ stub_request(:head, 'https://example.com/')
8
+ .to_return(status: 200)
9
+ endpoint = SiteInspector::Endpoint.new('https://example.com')
10
+ allow(endpoint.response).to receive(:return_code).and_return(:ok)
11
+ described_class.new(endpoint)
12
+ end
13
+
14
+ it 'knows the scheme' do
15
+ expect(subject.send(:scheme)).to eql('https')
16
+ end
17
+
18
+ it 'knows if the scheme is https' do
19
+ expect(subject.scheme?).to be(true)
20
+ allow(subject).to receive(:scheme).and_return('http')
21
+ expect(subject.scheme?).to be(false)
22
+ end
23
+
24
+ it "knows if it's valid" do
25
+ expect(subject.valid?).to be(true)
26
+ end
27
+
28
+ it "knows when there's a bad chain" do
29
+ expect(subject.bad_chain?).to be(false)
30
+
31
+ url = Addressable::URI.parse('https://example.com')
32
+ response = Typhoeus::Response.new(return_code: :ssl_cacert)
33
+ response.request = Typhoeus::Request.new(url)
34
+
35
+ allow(subject).to receive(:response) { response }
36
+ expect(subject.bad_chain?).to be(true)
37
+ end
38
+
39
+ it "knows when there's a bad name" do
40
+ expect(subject.bad_name?).to be(false)
41
+
42
+ url = Addressable::URI.parse('https://example.com')
43
+ response = Typhoeus::Response.new(return_code: :peer_failed_verification)
44
+ response.request = Typhoeus::Request.new(url)
45
+
46
+ allow(subject).to receive(:response) { response }
47
+ expect(subject.bad_name?).to be(true)
48
+ end
49
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ describe SiteInspector::Endpoint::Sniffer do
6
+ def stub_header(header, value)
7
+ allow(subject.endpoint.headers).to receive(:headers) { { header => value } }
8
+ end
9
+
10
+ def set_cookie(key, value)
11
+ cookies = [
12
+ CGI::Cookie.new(
13
+ 'name' => 'foo',
14
+ 'value' => 'bar',
15
+ 'domain' => 'example.com',
16
+ 'path' => '/'
17
+ ),
18
+ CGI::Cookie.new(
19
+ 'name' => key,
20
+ 'value' => value,
21
+ 'domain' => 'example.com',
22
+ 'path' => '/'
23
+ )
24
+ ].map(&:to_s)
25
+
26
+ stub_request(:get, 'http://example.com/')
27
+ .to_return(status: 200, body: '')
28
+
29
+ stub_request(:head, 'http://example.com/')
30
+ .to_return(status: 200, headers: { 'set-cookie' => cookies })
31
+ end
32
+
33
+ context 'stubbed body' do
34
+ subject do
35
+ body = <<-BODY
36
+ <html>
37
+ <head>
38
+ <link rel='stylesheet' href='/wp-content/themes/foo/style.css type='text/css' media='all' />
39
+ </head>
40
+ <body>
41
+ <h1>Some page</h1>
42
+ <script>
43
+ jQuery(); googletag.pubads();
44
+ </script>
45
+ <script>
46
+ var _gaq=[['_setAccount','UA-12345678-1'],['_trackPageview']];
47
+ (function(d,t){var g=d.createElement(t),s=d.getElementsByTagName(t)[0];
48
+ g.src=('https:'==location.protocol?'//ssl':'//www')+'.google-analytics.com/ga.js';
49
+ s.parentNode.insertBefore(g,s)}(document,'script'));
50
+ </script>
51
+ </body>
52
+ </html>
53
+ BODY
54
+
55
+ stub_request(:get, 'http://example.com/')
56
+ .to_return(status: 200, body: body)
57
+
58
+ stub_request(:head, 'http://example.com/')
59
+ .to_return(status: 200)
60
+ endpoint = SiteInspector::Endpoint.new('http://example.com')
61
+ described_class.new(endpoint)
62
+ end
63
+
64
+ it 'sniffs' do
65
+ sniff = subject.send(:sniff, :cms)
66
+ expect(sniff).to be(:wordpress)
67
+ end
68
+
69
+ it 'detects the CMS' do
70
+ expect(subject.framework).to be(:wordpress)
71
+ end
72
+
73
+ it 'detects the analytics' do
74
+ expect(subject.analytics).to be(:google_analytics)
75
+ end
76
+
77
+ it 'detects javascript' do
78
+ expect(subject.javascript).to be(:jquery)
79
+ end
80
+
81
+ it 'detects advertising' do
82
+ expect(subject.advertising).to be(:adsense)
83
+ end
84
+
85
+ it 'knows wordpress is open source' do
86
+ expect(subject.open_source?).to be(true)
87
+ end
88
+ end
89
+
90
+ context 'no body' do
91
+ subject do
92
+ endpoint = SiteInspector::Endpoint.new('http://example.com')
93
+ described_class.new(endpoint)
94
+ end
95
+
96
+ it "knows when something isn't open source" do
97
+ set_cookie('foo', 'bar')
98
+ expect(subject.open_source?).to be(false)
99
+ end
100
+
101
+ it 'detects PHP' do
102
+ set_cookie('PHPSESSID', '1234')
103
+ expect(subject.framework).to be(:php)
104
+ expect(subject.open_source?).to be(true)
105
+ end
106
+
107
+ it 'detects Expression Engine' do
108
+ set_cookie('exp_csrf_token', '1234')
109
+ expect(subject.framework).to be(:expression_engine)
110
+ expect(subject.open_source?).to be(true)
111
+ end
112
+
113
+ it 'detects cowboy' do
114
+ stub_request(:get, 'http://example.com/')
115
+ .to_return(status: 200, body: '')
116
+
117
+ stub_request(:head, 'http://example.com/')
118
+ .to_return(status: 200, headers: { 'server' => 'Cowboy' })
119
+
120
+ expect(subject.framework).to be(:cowboy)
121
+ expect(subject.open_source?).to be(true)
122
+ end
123
+
124
+ it 'detects ColdFusion' do
125
+ cookies = [
126
+ CGI::Cookie.new(
127
+ 'name' => 'CFID',
128
+ 'value' => '1234',
129
+ 'domain' => 'example.com',
130
+ 'path' => '/'
131
+ ),
132
+ CGI::Cookie.new(
133
+ 'name' => 'CFTOKEN',
134
+ 'value' => '5678',
135
+ 'domain' => 'example.com',
136
+ 'path' => '/'
137
+ )
138
+ ].map(&:to_s)
139
+
140
+ stub_request(:get, 'http://example.com/')
141
+ .to_return(status: 200, body: '')
142
+
143
+ stub_request(:head, 'http://example.com/')
144
+ .to_return(status: 200, headers: { 'set-cookie' => cookies })
145
+
146
+ expect(subject.framework).to be(:coldfusion)
147
+ expect(subject.open_source?).to be(false)
148
+ end
149
+ end
150
+ end