site-inspector 3.1.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +34 -0
- data/.ruby-version +1 -1
- data/Gemfile +1 -1
- data/Guardfile +1 -1
- data/README.md +6 -1
- data/Rakefile +2 -2
- data/bin/site-inspector +15 -15
- data/lib/cliver/dependency_ext.rb +21 -0
- data/lib/site-inspector.rb +13 -11
- data/lib/site-inspector/checks/accessibility.rb +27 -17
- data/lib/site-inspector/checks/check.rb +1 -3
- data/lib/site-inspector/checks/content.rb +6 -6
- data/lib/site-inspector/checks/cookies.rb +6 -8
- data/lib/site-inspector/checks/dns.rb +21 -20
- data/lib/site-inspector/checks/headers.rb +12 -13
- data/lib/site-inspector/checks/hsts.rb +8 -9
- data/lib/site-inspector/checks/https.rb +3 -5
- data/lib/site-inspector/checks/sniffer.rb +8 -9
- data/lib/site-inspector/domain.rb +28 -32
- data/lib/site-inspector/endpoint.rb +31 -32
- data/lib/site-inspector/version.rb +1 -1
- data/script/cibuild +3 -1
- data/script/pa11y-version +9 -0
- data/site-inspector.gemspec +25 -25
- data/spec/checks/site_inspector_endpoint_accessibility_spec.rb +31 -30
- data/spec/checks/site_inspector_endpoint_check_spec.rb +10 -11
- data/spec/checks/site_inspector_endpoint_content_spec.rb +43 -44
- data/spec/checks/site_inspector_endpoint_cookies_spec.rb +30 -31
- data/spec/checks/site_inspector_endpoint_dns_spec.rb +72 -77
- data/spec/checks/site_inspector_endpoint_headers_spec.rb +26 -27
- data/spec/checks/site_inspector_endpoint_hsts_spec.rb +26 -27
- data/spec/checks/site_inspector_endpoint_https_spec.rb +11 -12
- data/spec/checks/site_inspector_endpoint_sniffer_spec.rb +56 -57
- data/spec/site_inspector_cache_spec.rb +6 -6
- data/spec/site_inspector_disk_cache_spec.rb +9 -9
- data/spec/site_inspector_domain_spec.rb +132 -136
- data/spec/site_inspector_endpoint_spec.rb +108 -108
- data/spec/site_inspector_spec.rb +17 -18
- data/spec/spec_helper.rb +3 -3
- metadata +21 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6884b52732376fcd208b761e6b441f52c8ff383d
|
4
|
+
data.tar.gz: a6d6268ce73a44108f4ec490da07d7b23257981d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c0c672eb141da7b5522b4ea9191024a13d08f37c3d91b11b8771f649596ff38f587cc98a95ce6cf6bb80f6f060d71068a51c45d132517e7a6a4237f7cb54d9d8
|
7
|
+
data.tar.gz: a1251d75b522286e2ff9889b347b4bf73440b2932f89acc9b4840d1b2c40ad7cbded89feca80e278f8120f9b1976d658860a75bd747c25921df4b55efcd365e8
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
Metrics/LineLength:
|
2
|
+
Enabled: false
|
3
|
+
|
4
|
+
Style/AlignHash:
|
5
|
+
EnforcedHashRocketStyle: table
|
6
|
+
EnforcedColonStyle: table
|
7
|
+
|
8
|
+
Lint/EndAlignment:
|
9
|
+
AlignWith: variable
|
10
|
+
AutoCorrect: true
|
11
|
+
|
12
|
+
Metrics/MethodLength:
|
13
|
+
Enabled: false
|
14
|
+
|
15
|
+
Metrics/AbcSize:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Style/Documentation:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Style/FileName:
|
22
|
+
Enabled: false
|
23
|
+
|
24
|
+
Metrics/CyclomaticComplexity:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Style/DoubleNegation:
|
28
|
+
Enabled: false
|
29
|
+
|
30
|
+
Metrics/ClassLength:
|
31
|
+
Enabled: false
|
32
|
+
|
33
|
+
Style/ClassVars:
|
34
|
+
Enabled: false
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.3.0
|
data/Gemfile
CHANGED
data/Guardfile
CHANGED
data/README.md
CHANGED
@@ -150,7 +150,7 @@ Uses the `pa11y` CLI to run automated accessibility tests. Requires `node`. To i
|
|
150
150
|
```ruby
|
151
151
|
class SiteInspector
|
152
152
|
class Endpoint
|
153
|
-
class Mention
|
153
|
+
class Mention < Check
|
154
154
|
def mentions_ben?
|
155
155
|
endpoint.content.body =~ /ben/i
|
156
156
|
end
|
@@ -159,6 +159,11 @@ class SiteInspector
|
|
159
159
|
end
|
160
160
|
```
|
161
161
|
|
162
|
+
This check can then be used as follows:
|
163
|
+
```
|
164
|
+
domain.canonical_endpoint.mention.mentions_ben?
|
165
|
+
```
|
166
|
+
|
162
167
|
Checks can call the `endpoint` object, which, contains the request, response, and other checks. Custom checks are automatically exposed as endpoint methods.
|
163
168
|
|
164
169
|
## Contributing
|
data/Rakefile
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
require './lib/site-inspector'
|
2
2
|
require 'rspec/core/rake_task'
|
3
3
|
|
4
|
-
desc
|
4
|
+
desc 'Run specs'
|
5
5
|
RSpec::Core::RakeTask.new do |t|
|
6
6
|
t.pattern = 'spec/**/*_spec.rb'
|
7
|
-
t.rspec_opts = [
|
7
|
+
t.rspec_opts = ['--order', 'rand', '--color']
|
8
8
|
end
|
data/bin/site-inspector
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'mercenary'
|
4
|
+
require 'oj'
|
5
5
|
require 'yaml'
|
6
6
|
require 'colorator'
|
7
|
-
require_relative
|
7
|
+
require_relative '../lib/site-inspector'
|
8
8
|
|
9
9
|
def stringify_keys_deep!(h)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
h.keys.each do |k|
|
11
|
+
ks = k.respond_to?(:to_s) ? k.to_s : k
|
12
|
+
h[ks] = h.delete k # Preserve order even when k == ks
|
13
|
+
stringify_keys_deep! h[ks] if h[ks].is_a? Hash
|
14
|
+
end
|
15
15
|
end
|
16
16
|
|
17
17
|
Mercenary.program(:"site-inspector") do |p|
|
18
18
|
p.version SiteInspector::VERSION
|
19
19
|
p.description "Returns information about a domain's technology and capabilities"
|
20
|
-
p.syntax
|
20
|
+
p.syntax 'site-inspector <command> <domain> [options]'
|
21
21
|
|
22
22
|
p.command(:inspect) do |c|
|
23
|
-
c.syntax
|
24
|
-
c.description
|
23
|
+
c.syntax 'inspect <domain> [options]'
|
24
|
+
c.description 'inspects a domain'
|
25
25
|
c.option 'json', '-j', '--json', 'JSON encode the output'
|
26
26
|
c.option 'all', '-a', '--all', 'return results for all endpoints (defaults to only the canonical endpoint)'
|
27
27
|
|
@@ -30,7 +30,7 @@ Mercenary.program(:"site-inspector") do |p|
|
|
30
30
|
end
|
31
31
|
|
32
32
|
c.action do |args, options|
|
33
|
-
next c.logger.fatal
|
33
|
+
next c.logger.fatal 'Must specify a domain' if args.length != 1
|
34
34
|
|
35
35
|
# Build our domain hash as requested
|
36
36
|
domain = SiteInspector.inspect(args[0])
|
@@ -38,15 +38,15 @@ Mercenary.program(:"site-inspector") do |p|
|
|
38
38
|
json = Oj.dump(hash, indent: 2, mode: :compat)
|
39
39
|
|
40
40
|
# Dump the JSON and run
|
41
|
-
next puts json if options[
|
41
|
+
next puts json if options['json']
|
42
42
|
|
43
43
|
# This is a dirty, dirty hack, but it's a simple way to stringify keys recursively
|
44
44
|
# And format the output in a human-readable way
|
45
45
|
yaml = YAML.dump Oj.load(json)
|
46
46
|
|
47
47
|
# Colorize bools
|
48
|
-
yaml.gsub!
|
49
|
-
yaml.gsub!
|
48
|
+
yaml.gsub!(/\: (true|ok)$/, ': ' + 'true'.green)
|
49
|
+
yaml.gsub!(/\: false$/, ': ' + 'false'.red)
|
50
50
|
|
51
51
|
puts yaml
|
52
52
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Cliver
|
2
|
+
class Dependency
|
3
|
+
# Memoized shortcut for detect
|
4
|
+
# Returns the path to the detected dependency
|
5
|
+
# Raises an error if the dependency was not satisfied
|
6
|
+
def path
|
7
|
+
@detected_path ||= detect!
|
8
|
+
end
|
9
|
+
|
10
|
+
# Returns the version of the resolved dependency
|
11
|
+
def version
|
12
|
+
return @detected_version if defined? @detected_version
|
13
|
+
version = installed_versions.find { |p, _v| p == path }
|
14
|
+
@detected_version = version.nil? ? nil : version[1]
|
15
|
+
end
|
16
|
+
|
17
|
+
def major_version
|
18
|
+
version.split('.').first if version
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/site-inspector.rb
CHANGED
@@ -21,11 +21,11 @@ require_relative 'site-inspector/checks/sniffer'
|
|
21
21
|
require_relative 'site-inspector/checks/cookies'
|
22
22
|
require_relative 'site-inspector/endpoint'
|
23
23
|
require_relative 'site-inspector/version'
|
24
|
+
require_relative 'cliver/dependency_ext'
|
24
25
|
|
25
26
|
class SiteInspector
|
26
27
|
class << self
|
27
|
-
|
28
|
-
attr_writer :timeout, :cache
|
28
|
+
attr_writer :timeout, :cache, :typhoeus_options
|
29
29
|
|
30
30
|
def cache
|
31
31
|
@cache ||= if ENV['CACHE']
|
@@ -46,15 +46,17 @@ class SiteInspector
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def typhoeus_defaults
|
49
|
-
{
|
50
|
-
:
|
51
|
-
:
|
52
|
-
:
|
53
|
-
|
54
|
-
:
|
55
|
-
|
49
|
+
defaults = {
|
50
|
+
followlocation: false,
|
51
|
+
timeout: SiteInspector.timeout,
|
52
|
+
accept_encoding: 'gzip',
|
53
|
+
method: :head,
|
54
|
+
headers: {
|
55
|
+
'User-Agent' => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
|
56
56
|
}
|
57
57
|
}
|
58
|
+
defaults.merge! @typhoeus_options if @typhoeus_options
|
59
|
+
defaults
|
58
60
|
end
|
59
61
|
|
60
62
|
# Returns a thread-safe, memoized hydra instance
|
@@ -64,8 +66,8 @@ class SiteInspector
|
|
64
66
|
end
|
65
67
|
end
|
66
68
|
|
67
|
-
if ENV[
|
68
|
-
Ethon.logger = Logger.new(STDOUT)
|
69
|
+
if ENV['DEBUG']
|
70
|
+
Ethon.logger = Logger.new(STDOUT)
|
69
71
|
Ethon.logger.level = Logger::DEBUG
|
70
72
|
Typhoeus::Config.verbose = true
|
71
73
|
end
|
@@ -15,19 +15,29 @@ class SiteInspector
|
|
15
15
|
|
16
16
|
DEFAULT_LEVEL = :error
|
17
17
|
|
18
|
+
REQUIRED_PA11Y_VERSION = '~> 2.1'
|
19
|
+
|
18
20
|
class << self
|
19
21
|
def pa11y_version
|
20
|
-
|
21
|
-
output.strip if status == 0
|
22
|
+
@pa11y_version ||= pa11y.version
|
22
23
|
end
|
23
24
|
|
24
25
|
def pa11y?
|
25
|
-
|
26
|
+
return @pa11y_detected if defined? @pa11y_detected
|
27
|
+
@pa11y_detected = !!(pa11y.detect)
|
26
28
|
end
|
27
29
|
|
28
30
|
def enabled?
|
29
31
|
@@enabled && pa11y?
|
30
32
|
end
|
33
|
+
|
34
|
+
def pa11y
|
35
|
+
@pa11y ||= begin
|
36
|
+
node_bin = File.expand_path('../../../node_modules/pa11y/bin', File.dirname(__FILE__))
|
37
|
+
path = ['*', node_bin].join(File::PATH_SEPARATOR)
|
38
|
+
Cliver::Dependency.new('pa11y', REQUIRED_PA11Y_VERSION, path: path)
|
39
|
+
end
|
40
|
+
end
|
31
41
|
end
|
32
42
|
|
33
43
|
def level
|
@@ -35,7 +45,7 @@ class SiteInspector
|
|
35
45
|
end
|
36
46
|
|
37
47
|
def level=(level)
|
38
|
-
|
48
|
+
fail ArgumentError, "Invalid level '#{level}'" unless [:error, :warning, :notice].include?(level)
|
39
49
|
@level = level
|
40
50
|
end
|
41
51
|
|
@@ -48,7 +58,7 @@ class SiteInspector
|
|
48
58
|
end
|
49
59
|
|
50
60
|
def standard=(standard)
|
51
|
-
|
61
|
+
fail ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
|
52
62
|
@standard = standard
|
53
63
|
end
|
54
64
|
|
@@ -57,11 +67,11 @@ class SiteInspector
|
|
57
67
|
end
|
58
68
|
|
59
69
|
def errors
|
60
|
-
check[:results].count { |r| r[
|
70
|
+
check[:results].count { |r| r['type'] == 'error' } if check
|
61
71
|
end
|
62
72
|
|
63
73
|
def check
|
64
|
-
@check ||=
|
74
|
+
@check ||= run_pa11y(standard)
|
65
75
|
rescue Pa11yError
|
66
76
|
nil
|
67
77
|
end
|
@@ -69,7 +79,7 @@ class SiteInspector
|
|
69
79
|
|
70
80
|
def method_missing(method_sym, *arguments, &block)
|
71
81
|
if standard?(method_sym)
|
72
|
-
|
82
|
+
run_pa11y(method_sym)
|
73
83
|
else
|
74
84
|
super
|
75
85
|
end
|
@@ -85,32 +95,32 @@ class SiteInspector
|
|
85
95
|
|
86
96
|
private
|
87
97
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
98
|
+
def run_pa11y(standard)
|
99
|
+
self.class.pa11y.detect! unless ENV['SKIP_PA11Y_CHECK']
|
100
|
+
fail ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
|
91
101
|
|
92
102
|
args = [
|
93
|
-
|
94
|
-
|
95
|
-
|
103
|
+
'--standard', STANDARDS[standard],
|
104
|
+
'--reporter', 'json',
|
105
|
+
'--level', level.to_s,
|
96
106
|
endpoint.uri.to_s
|
97
107
|
]
|
98
108
|
output, status = run_command(args)
|
99
109
|
|
100
110
|
# Pa11y exit codes: https://github.com/nature/pa11y#exit-codes
|
101
111
|
# 0: No errors, 1: Technical error within pa11y, 2: accessibility error (configurable via --level)
|
102
|
-
|
112
|
+
fail Pa11yError if status == 1
|
103
113
|
|
104
114
|
{
|
105
115
|
valid: status == 0,
|
106
116
|
results: JSON.parse(output)
|
107
117
|
}
|
108
118
|
rescue Pa11yError, JSON::ParserError
|
109
|
-
raise Pa11yError, "Command `pa11y #{args.join(
|
119
|
+
raise Pa11yError, "Command `pa11y #{args.join(' ')}` failed: #{output}"
|
110
120
|
end
|
111
121
|
|
112
122
|
def run_command(args)
|
113
|
-
Open3.capture2e(
|
123
|
+
Open3.capture2e(self.class.pa11y.path, *args)
|
114
124
|
end
|
115
125
|
end
|
116
126
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
class SiteInspector
|
2
2
|
class Endpoint
|
3
3
|
class Check
|
4
|
-
|
5
4
|
attr_reader :endpoint
|
6
5
|
|
7
6
|
# A check is an abstract class that takes an Endpoint object
|
@@ -34,11 +33,10 @@ class SiteInspector
|
|
34
33
|
end
|
35
34
|
|
36
35
|
class << self
|
37
|
-
|
38
36
|
@@enabled = true
|
39
37
|
|
40
38
|
def name
|
41
|
-
|
39
|
+
to_s.split('::').last.downcase.to_sym
|
42
40
|
end
|
43
41
|
|
44
42
|
def enabled?
|
@@ -9,7 +9,7 @@ class SiteInspector
|
|
9
9
|
# The default Check#response method is from a HEAD request
|
10
10
|
# The content check has a special response which includes the body from a GET request
|
11
11
|
def response
|
12
|
-
@response ||= endpoint.request(:
|
12
|
+
@response ||= endpoint.request(method: :get)
|
13
13
|
end
|
14
14
|
|
15
15
|
def document
|
@@ -19,19 +19,19 @@ class SiteInspector
|
|
19
19
|
alias_method :doc, :document
|
20
20
|
|
21
21
|
def body
|
22
|
-
@body ||= document.to_s.force_encoding(
|
22
|
+
@body ||= document.to_s.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, replace: '')
|
23
23
|
end
|
24
24
|
|
25
25
|
def robots_txt?
|
26
|
-
@bodts_txt ||= path_exists?(
|
26
|
+
@bodts_txt ||= path_exists?('robots.txt') if proper_404s?
|
27
27
|
end
|
28
28
|
|
29
29
|
def sitemap_xml?
|
30
|
-
@sitemap_xml ||= path_exists?(
|
30
|
+
@sitemap_xml ||= path_exists?('sitemap.xml') if proper_404s?
|
31
31
|
end
|
32
32
|
|
33
33
|
def humans_txt?
|
34
|
-
@humans_txt ||= path_exists?(
|
34
|
+
@humans_txt ||= path_exists?('humans.txt') if proper_404s?
|
35
35
|
end
|
36
36
|
|
37
37
|
def doctype
|
@@ -41,7 +41,7 @@ class SiteInspector
|
|
41
41
|
def prefetch
|
42
42
|
return unless endpoint.up?
|
43
43
|
options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
|
44
|
-
[
|
44
|
+
['robots.txt', 'sitemap.xml', 'humans.txt', random_path].each do |path|
|
45
45
|
request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)
|
46
46
|
SiteInspector.hydra.queue(request)
|
47
47
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
class SiteInspector
|
2
2
|
class Endpoint
|
3
3
|
class Cookies < Check
|
4
|
-
|
5
4
|
def any?(&block)
|
6
5
|
if cookie_header.nil? || cookie_header.empty?
|
7
6
|
false
|
@@ -14,7 +13,7 @@ class SiteInspector
|
|
14
13
|
alias_method :cookies?, :any?
|
15
14
|
|
16
15
|
def all
|
17
|
-
@cookies ||= cookie_header.map { |c| CGI::Cookie
|
16
|
+
@cookies ||= cookie_header.map { |c| CGI::Cookie.parse(c) } if cookies?
|
18
17
|
end
|
19
18
|
|
20
19
|
def [](key)
|
@@ -22,14 +21,14 @@ class SiteInspector
|
|
22
21
|
end
|
23
22
|
|
24
23
|
def secure?
|
25
|
-
pairs = cookie_header.join(
|
26
|
-
pairs.any? { |c| c.downcase ==
|
24
|
+
pairs = cookie_header.join('; ').split('; ') # CGI::Cookies#Parse doesn't seem to like secure headers
|
25
|
+
pairs.any? { |c| c.downcase == 'secure' } && pairs.any? { |c| c.downcase == 'httponly' }
|
27
26
|
end
|
28
27
|
|
29
28
|
def to_h
|
30
29
|
{
|
31
|
-
|
32
|
-
|
30
|
+
cookie?: any?,
|
31
|
+
secure?: secure?
|
33
32
|
}
|
34
33
|
end
|
35
34
|
|
@@ -37,9 +36,8 @@ class SiteInspector
|
|
37
36
|
|
38
37
|
def cookie_header
|
39
38
|
# Cookie header may be an array or string, always return an array
|
40
|
-
[endpoint.headers.all[
|
39
|
+
[endpoint.headers.all['set-cookie']].flatten.compact
|
41
40
|
end
|
42
|
-
|
43
41
|
end
|
44
42
|
end
|
45
43
|
end
|