site-inspector 3.1.1 → 4.0.0

data/lib/site-inspector/domain.rb

@@ -1,22 +1,20 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Domain
     attr_reader :host
 
     def initialize(host)
-      host = host.downcase
-      host = host.sub(/^https?\:/, '')
-      host = host.sub(%r{^/+}, '')
-      host = host.sub(/^www\./, '')
-      uri = Addressable::URI.parse "//#{host}"
-      @host = uri.host
+      @host = DomainParser.parse(host)
+      @host = strip_www(@host) if @host&.trd
     end
 
     def endpoints
       @endpoints ||= [
-        Endpoint.new("https://#{host}", domain: self),
-        Endpoint.new("https://www.#{host}", domain: self),
-        Endpoint.new("http://#{host}", domain: self),
-        Endpoint.new("http://www.#{host}", domain: self)
+        Endpoint.build(self, https: true, www: false),
+        Endpoint.build(self, https: true, www: true),
+        Endpoint.build(self, https: false, www: false),
+        Endpoint.build(self, https: false, www: true)
       ]
     end
 
@@ -82,7 +80,8 @@ class SiteInspector
     # TODO: don't need to require that the HTTPS cert is valid for this purpose.
     def enforces_https?
       return false unless https?
-      endpoints.select(&:http?).all? { |e| !e.up? || (e.redirect && e.redirect.https?) }
+
+      endpoints.select(&:http?).all? { |e| !e.up? || e.redirect&.https? }
     end
 
     # we can say that a canonical HTTPS site "defaults" to HTTPS,
@@ -91,18 +90,20 @@ class SiteInspector
     #
     # TODO: not implemented.
     def defaults_https?
-      fail 'Not implemented. Halp?'
+      raise 'Not implemented. Halp?'
     end
 
     # HTTPS is "downgraded" if both:
     #
     # * HTTPS is supported, and
-    # * The 'canonical' endpoint gets an immediate internal redirect to HTTP.
+    # * Any HTTPS endpoint redirects to HTTP.
     #
-    # TODO: the redirect must be internal.
+    # This correctly identifies domains where HTTPS is available but
+    # redirects to HTTP, regardless of which endpoint is canonical.
     def downgrades_https?
       return false unless https?
-      canonical_endpoint.redirect? && canonical_endpoint.redirect.http?
+
+      endpoints.select(&:https?).any? { |e| e.redirect&.http? }
     end
 
     # A domain is "canonically" at www if:
@@ -129,7 +130,7 @@ class SiteInspector
       return true if endpoints.select(&:root?).all? { |e| !e.up? }
 
       # Does either root endpoint redirect to a www endpoint?
-      endpoints.select(&:root?).any? { |e| e.redirect && e.redirect.www? }
+      endpoints.select(&:root?).any? { |e| e.redirect&.www? }
     end
 
     # A domain is "canonically" at https if:
@@ -160,7 +161,7 @@ class SiteInspector
       return true if endpoints.select(&:http?).all? { |e| !e.up? }
 
       # at least one http endpoint redirects immediately to https
-      endpoints.select(&:http?).any? { |e| e.redirect && e.redirect.https? }
+      endpoints.select(&:http?).any? { |e| e.redirect&.https? }
     end
 
     # A domain redirects if
@@ -168,6 +169,7 @@ class SiteInspector
     # 2. All endpoints are either down or an external redirect
     def redirect?
       return false unless redirect
+
       endpoints.all? { |e| !e.up? || e.external_redirect? }
     end
 
@@ -178,7 +180,7 @@ class SiteInspector
 
     # HSTS on the canonical domain?
     def hsts?
-      canonical_endpoint.hsts && canonical_endpoint.hsts.enabled?
+      canonical_endpoint.hsts&.enabled?
     end
 
     def hsts_subdomains?
@@ -187,11 +189,12 @@ class SiteInspector
 
     def hsts_preload_ready?
       return false unless hsts_subdomains?
+
       endpoints.find { |e| e.root? && e.https? }.hsts.preload_ready?
     end
 
     def to_s
-      host
+      host.to_s
     end
 
     def inspect
@@ -222,44 +225,59 @@ class SiteInspector
     #
     # Returns a complete hash of the domain's information
     def to_h(options = {})
+      return {} unless host
+
       prefetch
 
       hash = {
-        host:               host,
-        up:                 up?,
-        responds:           responds?,
-        www:                www?,
-        root:               root?,
-        https:              https?,
-        enforces_https:     enforces_https?,
-        downgrades_https:   downgrades_https?,
-        canonically_www:    canonically_www?,
-        canonically_https:  canonically_https?,
-        redirect:           redirect?,
-        hsts:               hsts?,
-        hsts_subdomains:    hsts_subdomains?,
+        host: canonical_endpoint.host.to_s,
+        tld: canonical_endpoint.host.tld,
+        sld: canonical_endpoint.host.sld,
+        trd: canonical_endpoint.host.trd,
+        up: up?,
+        responds: responds?,
+        www: www?,
+        root: root?,
+        https: https?,
+        enforces_https: enforces_https?,
+        downgrades_https: downgrades_https?,
+        canonically_www: canonically_www?,
+        canonically_https: canonically_https?,
+        redirect: redirect?,
+        hsts: hsts?,
+        hsts_subdomains: hsts_subdomains?,
         hsts_preload_ready: hsts_preload_ready?,
         canonical_endpoint: canonical_endpoint.to_h(options)
       }
 
       if options['all']
-        hash.merge!(endpoints: {
-                      https: {
-                        root: endpoints[0].to_h(options),
-                        www:  endpoints[1].to_h(options)
-                      },
-                      http:  {
-                        root: endpoints[2].to_h(options),
-                        www:  endpoints[3].to_h(options)
-                      }
-                    })
+        hash[:endpoints] = {
+          https: {
+            root: endpoints[0].to_h(options),
+            www: endpoints[1].to_h(options)
+          },
+          http: {
+            root: endpoints[2].to_h(options),
+            www: endpoints[3].to_h(options)
+          }
+        }
       end
 
       hash
     end
 
-    def to_json
+    def to_json(*_args)
       to_h.to_json
     end
+
+    private
+
+    # Rebuild the domain without a leading `www` label
+    def strip_www(domain)
+      return domain unless domain.trd == 'www' || domain.trd.start_with?('www.')
+
+      trd = domain.trd.delete_prefix('www').delete_prefix('.')
+      PublicSuffix::Domain.new(domain.tld, domain.sld, trd.empty? ? nil : trd)
+    end
   end
 end

data/lib/site-inspector/domain_parser.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class DomainParser
+    include NaughtyOrNice
+
+    def self.parse(domain)
+      new(domain).domain
+    end
+  end
+end

data/lib/site-inspector/{checks → endpoint}/accessibility.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'open3'
 
@@ -7,24 +9,28 @@ class SiteInspector
       class Pa11yError < RuntimeError; end
 
       STANDARDS = {
-        section508: 'Section508', # Default standard
-        wcag2a:     'WCAG2A',
-        wcag2aa:    'WCAG2AA',
-        wcag2aaa:   'WCAG2AAA'
-      }
+        wcag2a: 'WCAG2A', # Default standard
+        wcag2aa: 'WCAG2AA',
+        wcag2aaa: 'WCAG2AAA',
+        section508: 'Section508'
+      }.freeze
 
       DEFAULT_LEVEL = :error
 
-      REQUIRED_PA11Y_VERSION = '~> 2.1'
+      REQUIRED_PA11Y_VERSION = '~> 5.0'
 
       class << self
         def pa11y_version
-          @pa11y_version ||= pa11y.version
+          @pa11y_version ||= begin
+            output, status = run_command('--version')
+            output.strip if status.exitstatus.zero?
+          end
         end
 
         def pa11y?
           return @pa11y_detected if defined? @pa11y_detected
-          @pa11y_detected = !!(pa11y.detect)
+
+          @pa11y_detected = !!pa11y.detect
         end
 
         def enabled?
@@ -33,11 +39,14 @@ class SiteInspector
 
         def pa11y
           @pa11y ||= begin
-            node_bin = File.expand_path('../../../node_modules/pa11y/bin', File.dirname(__FILE__))
-            path = ['*', node_bin].join(File::PATH_SEPARATOR)
-            Cliver::Dependency.new('pa11y', REQUIRED_PA11Y_VERSION, path: path)
+            path = ['*', './bin', './node_modules/.bin'].join(File::PATH_SEPARATOR)
+            Cliver::Dependency.new('pa11y', REQUIRED_PA11Y_VERSION, path:)
           end
         end
+
+        def run_command(args)
+          Open3.capture2e(pa11y.detect, *args)
+        end
       end
 
       def level
@@ -45,12 +54,13 @@ class SiteInspector
       end
 
       def level=(level)
-        fail ArgumentError, "Invalid level '#{level}'" unless [:error, :warning, :notice].include?(level)
+        raise ArgumentError, "Invalid level '#{level}'" unless %i[error warning notice].include?(level)
+
         @level = level
       end
 
       def standard?(standard)
-        STANDARDS.keys.include?(standard)
+        STANDARDS.key?(standard)
       end
 
       def standard
@@ -58,7 +68,8 @@ class SiteInspector
       end
 
       def standard=(standard)
-        fail ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+        raise ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+
         @standard = standard
       end
 
@@ -71,13 +82,16 @@ class SiteInspector
       end
 
       def check
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+
         @check ||= run_pa11y(standard)
       rescue Pa11yError
         nil
       end
-      alias_method :to_h, :check
+      alias to_h check
 
-      def method_missing(method_sym, *arguments, &block)
+      def method_missing(method_sym, *arguments, &)
         if standard?(method_sym)
           run_pa11y(method_sym)
         else
@@ -85,7 +99,7 @@ class SiteInspector
         end
       end
 
-      def respond_to?(method_sym, include_private = false)
+      def respond_to_missing?(method_sym, include_private = false)
         if standard?(method_sym)
           true
         else
@@ -97,7 +111,7 @@ class SiteInspector
 
       def run_pa11y(standard)
         self.class.pa11y.detect! unless ENV['SKIP_PA11Y_CHECK']
-        fail ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+        raise ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
 
         args = [
           '--standard', STANDARDS[standard],
@@ -105,23 +119,19 @@ class SiteInspector
           '--level',    level.to_s,
           endpoint.uri.to_s
         ]
-        output, status = run_command(args)
+        output, status = self.class.run_command(args)
 
         # Pa11y exit codes: https://github.com/nature/pa11y#exit-codes
         # 0: No errors, 1: Technical error within pa11y, 2: accessibility error (configurable via --level)
-        fail Pa11yError if status == 1
+        raise Pa11yError if status.exitstatus == 1
 
         {
-          valid:   status == 0,
+          valid: status.exitstatus.zero?,
           results: JSON.parse(output)
         }
       rescue Pa11yError, JSON::ParserError
         raise Pa11yError, "Command `pa11y #{args.join(' ')}` failed: #{output}"
       end
-
-      def run_command(args)
-        Open3.capture2e(self.class.pa11y.path, *args)
-      end
     end
   end
 end

data/lib/site-inspector/{checks → endpoint}/check.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Check
@@ -21,7 +23,7 @@ class SiteInspector
       end
 
       def host
-        request.base_url.host
+        endpoint.host.to_s
       end
 
       def inspect
@@ -32,6 +34,10 @@ class SiteInspector
         self.class.name
       end
 
+      def uri_for(_key)
+        nil
+      end
+
       class << self
         @@enabled = true
 
@@ -40,11 +46,11 @@ class SiteInspector
         end
 
         def enabled?
-          !!(@@enabled)
+          !!@@enabled
         end
 
         def enabled=(value)
-          @@enabled = !!(value)
+          @@enabled = !!value
         end
       end
     end

data/lib/site-inspector/endpoint/content.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Content < Check
+      PATHS = [
+        'robots.txt', 'sitemap.xml', 'humans.txt', 'vulnerability-disclosure-policy', 'security.txt', '.well-known/security.txt', 'data.json'
+      ].freeze
+
+      class << self
+        def paths
+          @paths ||= PATHS.to_h { |p| [key_for(p), p] }
+        end
+
+        def path_for(key)
+          paths[key]
+        end
+
+        def key_for(path)
+          path.gsub(/(\.|-)/, '_').to_sym
+        end
+      end
+
+      # Given a path (e.g, "/data"), check if the given path exists on the canonical endpoint
+      def path_exists?(path)
+        return unless proper_404s?
+
+        @exists ||= {}
+        @exists[path] ||= endpoint.up? && endpoint.request(path:, followlocation: true).success?
+      rescue URI::InvalidURIError
+        false
+      end
+
+      # The default Check#response method is from a HEAD request
+      # The content check has a special response which includes the body from a GET request
+      def response
+        @response ||= endpoint.request(method: :get, followlocation: true)
+      end
+
+      def document
+        require 'nokogiri'
+        @doc ||= Nokogiri::HTML response.body if response
+      end
+      alias doc document
+
+      def body
+        @body ||= document.to_s.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, replace: '')
+      end
+
+      def method_missing(method_sym, *arguments, &)
+        key = method_sym.to_s.gsub(/\?$/, '').to_sym
+        if respond_to_missing?(key)
+          path = self.class.paths[key]
+          path_exists?(path)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method_sym, include_private = false)
+        if self.class.paths.key?(method_sym)
+          true
+        else
+          super
+        end
+      end
+
+      def security_txt?
+        @security_txt ||= if proper_404s?
+                            path_exists?('security.txt') || path_exists?('.well-known/security.txt')
+                          else
+                            false
+                          end
+      end
+
+      def uri_for(key)
+        return nil unless self.class.paths.key?(key)
+
+        endpoint.join(self.class.paths[key]) if proper_404s?
+      end
+
+      def doctype
+        document.internal_subset.external_id
+      end
+
+      def generator
+        @generator ||= begin
+          tag = document.at('meta[name="generator"]')
+          tag['content'] if tag
+        end
+      end
+
+      def prefetch
+        return unless endpoint.up?
+
+        paths = self.class.paths.values.concat(random_paths)
+        paths.each do |path|
+          request = endpoint.build_request(path:, followlocation: true)
+          SiteInspector.hydra.queue(request)
+        end
+
+        # Request for content is a GET, not a HEAD request
+        request = endpoint.build_request(method: :get, followlocation: true)
+        SiteInspector.hydra.queue(request)
+
+        SiteInspector.hydra.run
+      end
+
+      def proper_404s?
+        @proper_404s ||= begin
+          return unless endpoint.up?
+
+          random_paths.all? do |random_path|
+            endpoint.request(path: random_path, followlocation: true).code == 404
+          end
+        end
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+        return @hash if defined?(@hash)
+
+        prefetch
+        @hash = {
+          doctype:,
+          generator:,
+          proper_404s: proper_404s?
+        }
+
+        self.class.paths.each do |key, _path|
+          @hash[key] = send(key) unless key.to_s.start_with?('_')
+        end
+
+        @hash[:security_txt] = security_txt?
+        @hash
+      end
+
+      private
+
+      def random_paths
+        require 'securerandom'
+        @random_paths ||= [
+          SecureRandom.hex,
+          "#{SecureRandom.hex}.html",
+          "#{SecureRandom.hex}.json"
+        ]
+      end
+    end
+  end
+end

data/lib/site-inspector/endpoint/cookies.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'http-cookie'
+
+class SiteInspector
+  class Endpoint
+    class Cookies < Check
+      def any?(&block)
+        if cookie_header.nil? || cookie_header.empty?
+          false
+        elsif block
+          all.any?(&block)
+        else
+          true
+        end
+      end
+      alias cookies? any?
+
+      # Returns an array of HTTP::Cookie objects parsed from the Set-Cookie headers
+      def all
+        @cookies ||= cookie_header.flat_map { |c| HTTP::Cookie.parse(c, endpoint.uri.to_s) } if cookies?
+      end
+
+      def [](key)
+        all.find { |cookie| cookie.name == key } if cookies?
+      end
+
+      # Does at least one cookie set both the Secure and HttpOnly flags?
+      def secure?
+        return false unless cookies?
+
+        all.any? { |cookie| cookie.secure? && cookie.httponly? }
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+
+        {
+          cookie?: any?,
+          secure?: secure?
+        }
+      end
+
+      private
+
+      def cookie_header
+        # Cookie header may be an array or string, always return an array
+        [endpoint.headers.all['set-cookie']].flatten.compact
+      end
+    end
+  end
+end