site-inspector 3.2.0 → 4.0.0

data/lib/site-inspector/domain.rb

@@ -5,20 +5,16 @@ class SiteInspector
     attr_reader :host
 
     def initialize(host)
-      host = host.downcase
-      host = host.sub(/^https?:/, '')
-      host = host.sub(%r{^/+}, '')
-      host = host.sub(/^www\./, '')
-      uri = Addressable::URI.parse "//#{host}"
-      @host = uri.host
+      @host = DomainParser.parse(host)
+      @host = strip_www(@host) if @host&.trd
     end
 
     def endpoints
       @endpoints ||= [
-        Endpoint.new("https://#{host}", domain: self),
-        Endpoint.new("https://www.#{host}", domain: self),
-        Endpoint.new("http://#{host}", domain: self),
-        Endpoint.new("http://www.#{host}", domain: self)
+        Endpoint.build(self, https: true, www: false),
+        Endpoint.build(self, https: true, www: true),
+        Endpoint.build(self, https: false, www: false),
+        Endpoint.build(self, https: false, www: true)
       ]
     end
 
@@ -100,13 +96,14 @@ class SiteInspector
     # HTTPS is "downgraded" if both:
     #
     # * HTTPS is supported, and
-    # * The 'canonical' endpoint gets an immediate internal redirect to HTTP.
+    # * Any HTTPS endpoint redirects to HTTP.
     #
-    # TODO: the redirect must be internal.
+    # This correctly identifies domains where HTTPS is available but
+    # redirects to HTTP, regardless of which endpoint is canonical.
     def downgrades_https?
       return false unless https?
 
-      canonical_endpoint.redirect? && canonical_endpoint.redirect.http?
+      endpoints.select(&:https?).any? { |e| e.redirect&.http? }
     end
 
     # A domain is "canonically" at www if:
@@ -197,7 +194,7 @@ class SiteInspector
     end
 
     def to_s
-      host
+      host.to_s
     end
 
     def inspect
@@ -228,10 +225,15 @@ class SiteInspector
     #
     # Returns a complete hash of the domain's information
     def to_h(options = {})
+      return {} unless host
+
       prefetch
 
       hash = {
-        host: host,
+        host: canonical_endpoint.host.to_s,
+        tld: canonical_endpoint.host.tld,
+        sld: canonical_endpoint.host.sld,
+        trd: canonical_endpoint.host.trd,
         up: up?,
         responds: responds?,
         www: www?,
@@ -267,5 +269,15 @@ class SiteInspector
     def to_json(*_args)
       to_h.to_json
     end
+
+    private
+
+    # Rebuild the domain without a leading `www` label
+    def strip_www(domain)
+      return domain unless domain.trd == 'www' || domain.trd.start_with?('www.')
+
+      trd = domain.trd.delete_prefix('www').delete_prefix('.')
+      PublicSuffix::Domain.new(domain.tld, domain.sld, trd.empty? ? nil : trd)
+    end
   end
 end

data/lib/site-inspector/domain_parser.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class DomainParser
+    include NaughtyOrNice
+
+    def self.parse(domain)
+      new(domain).domain
+    end
+  end
+end

data/lib/site-inspector/{checks → endpoint}/accessibility.rb

@@ -39,9 +39,8 @@ class SiteInspector
 
         def pa11y
           @pa11y ||= begin
-            node_bin = File.expand_path('../../../node_modules/pa11y/bin', File.dirname(__FILE__))
-            path = ['*', node_bin].join(File::PATH_SEPARATOR)
-            Cliver::Dependency.new('pa11y.js', REQUIRED_PA11Y_VERSION, path: path)
+            path = ['*', './bin', './node_modules/.bin'].join(File::PATH_SEPARATOR)
+            Cliver::Dependency.new('pa11y', REQUIRED_PA11Y_VERSION, path:)
           end
         end
 
@@ -83,13 +82,16 @@ class SiteInspector
       end
 
       def check
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+
         @check ||= run_pa11y(standard)
       rescue Pa11yError
         nil
       end
       alias to_h check
 
-      def method_missing(method_sym, *arguments, &block)
+      def method_missing(method_sym, *arguments, &)
         if standard?(method_sym)
           run_pa11y(method_sym)
         else

data/lib/site-inspector/{checks → endpoint}/check.rb

@@ -23,7 +23,7 @@ class SiteInspector
       end
 
       def host
-        request.base_url.host
+        endpoint.host.to_s
       end
 
       def inspect
@@ -34,6 +34,10 @@ class SiteInspector
         self.class.name
       end
 
+      def uri_for(_key)
+        nil
+      end
+
       class << self
         @@enabled = true
 

data/lib/site-inspector/endpoint/content.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Content < Check
+      PATHS = [
+        'robots.txt', 'sitemap.xml', 'humans.txt', 'vulnerability-disclosure-policy', 'security.txt', '.well-known/security.txt', 'data.json'
+      ].freeze
+
+      class << self
+        def paths
+          @paths ||= PATHS.to_h { |p| [key_for(p), p] }
+        end
+
+        def path_for(key)
+          paths[key]
+        end
+
+        def key_for(path)
+          path.gsub(/(\.|-)/, '_').to_sym
+        end
+      end
+
+      # Given a path (e.g, "/data"), check if the given path exists on the canonical endpoint
+      def path_exists?(path)
+        return unless proper_404s?
+
+        @exists ||= {}
+        @exists[path] ||= endpoint.up? && endpoint.request(path:, followlocation: true).success?
+      rescue URI::InvalidURIError
+        false
+      end
+
+      # The default Check#response method is from a HEAD request
+      # The content check has a special response which includes the body from a GET request
+      def response
+        @response ||= endpoint.request(method: :get, followlocation: true)
+      end
+
+      def document
+        require 'nokogiri'
+        @doc ||= Nokogiri::HTML response.body if response
+      end
+      alias doc document
+
+      def body
+        @body ||= document.to_s.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, replace: '')
+      end
+
+      def method_missing(method_sym, *arguments, &)
+        key = method_sym.to_s.gsub(/\?$/, '').to_sym
+        if respond_to_missing?(key)
+          path = self.class.paths[key]
+          path_exists?(path)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method_sym, include_private = false)
+        if self.class.paths.key?(method_sym)
+          true
+        else
+          super
+        end
+      end
+
+      def security_txt?
+        @security_txt ||= if proper_404s?
+                            path_exists?('security.txt') || path_exists?('.well-known/security.txt')
+                          else
+                            false
+                          end
+      end
+
+      def uri_for(key)
+        return nil unless self.class.paths.key?(key)
+
+        endpoint.join(self.class.paths[key]) if proper_404s?
+      end
+
+      def doctype
+        document.internal_subset.external_id
+      end
+
+      def generator
+        @generator ||= begin
+          tag = document.at('meta[name="generator"]')
+          tag['content'] if tag
+        end
+      end
+
+      def prefetch
+        return unless endpoint.up?
+
+        paths = self.class.paths.values.concat(random_paths)
+        paths.each do |path|
+          request = endpoint.build_request(path:, followlocation: true)
+          SiteInspector.hydra.queue(request)
+        end
+
+        # Request for content is a GET, not a HEAD request
+        request = endpoint.build_request(method: :get, followlocation: true)
+        SiteInspector.hydra.queue(request)
+
+        SiteInspector.hydra.run
+      end
+
+      def proper_404s?
+        @proper_404s ||= begin
+          return unless endpoint.up?
+
+          random_paths.all? do |random_path|
+            endpoint.request(path: random_path, followlocation: true).code == 404
+          end
+        end
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+        return @hash if defined?(@hash)
+
+        prefetch
+        @hash = {
+          doctype:,
+          generator:,
+          proper_404s: proper_404s?
+        }
+
+        self.class.paths.each do |key, _path|
+          @hash[key] = send(key) unless key.to_s.start_with?('_')
+        end
+
+        @hash[:security_txt] = security_txt?
+        @hash
+      end
+
+      private
+
+      def random_paths
+        require 'securerandom'
+        @random_paths ||= [
+          SecureRandom.hex,
+          "#{SecureRandom.hex}.html",
+          "#{SecureRandom.hex}.json"
+        ]
+      end
+    end
+  end
+end

data/lib/site-inspector/{checks → endpoint}/cookies.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+require 'http-cookie'
+
 class SiteInspector
   class Endpoint
     class Cookies < Check
       def any?(&block)
         if cookie_header.nil? || cookie_header.empty?
           false
-        elsif block_given?
+        elsif block
           all.any?(&block)
         else
           true
@@ -14,20 +16,26 @@ class SiteInspector
       end
       alias cookies? any?
 
+      # Returns an array of HTTP::Cookie objects parsed from the Set-Cookie headers
       def all
-        @cookies ||= cookie_header.map { |c| CGI::Cookie.parse(c) } if cookies?
+        @cookies ||= cookie_header.flat_map { |c| HTTP::Cookie.parse(c, endpoint.uri.to_s) } if cookies?
       end
 
       def [](key)
-        all.find { |cookie| cookie.keys.first == key } if cookies?
+        all.find { |cookie| cookie.name == key } if cookies?
       end
 
+      # Does at least one cookie set both the Secure and HttpOnly flags?
       def secure?
-        pairs = cookie_header.join('; ').split('; ') # CGI::Cookies#Parse doesn't seem to like secure headers
-        pairs.any? { |c| c.casecmp('secure').zero? } && pairs.any? { |c| c.casecmp('httponly').zero? }
+        return false unless cookies?
+
+        all.any? { |cookie| cookie.secure? && cookie.httponly? }
       end
 
       def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+
         {
           cookie?: any?,
           secure?: secure?

data/lib/site-inspector/{checks → endpoint}/dns.rb

@@ -1,10 +1,16 @@
 # frozen_string_literal: true
 
+require 'ipaddr'
+
 class SiteInspector
   class Endpoint
     class Dns < Check
       class LocalhostError < StandardError; end
 
+      # Record types fetched for #records. ANY queries are deprecated
+      # (RFC 8482) and most resolvers answer them with a single HINFO record.
+      RECORD_TYPES = %w[A AAAA CNAME MX DNSKEY].freeze
+
       def self.resolver
         require 'dnsruby'
         @resolver ||= begin
@@ -14,18 +20,19 @@ class SiteInspector
         end
       end
 
-      def query(type = 'ANY')
-        SiteInspector::Endpoint::Dns.resolver.query(host.to_s, type).answer
-      rescue Dnsruby::ResolvTimeout, Dnsruby::ServFail, Dnsruby::NXDomain
-        []
+      def query(type = 'A')
+        lookup(host.to_s, type)
       end
 
       def records
-        @records ||= query
+        @records ||= RECORD_TYPES.flat_map { |type| query(type) }.uniq(&:to_s)
       end
 
       def record?(type)
-        records.any? { |record| record.type == type } || query(type).count != 0
+        return true if records.any? { |record| record.type == type }
+        return false if RECORD_TYPES.include?(type.to_s)
+
+        query(type).any?
       end
       alias has_record? record?
 
@@ -60,20 +67,30 @@ class SiteInspector
       end
 
       def localhost?
-        ip == '127.0.0.1'
+        return false unless ip
+
+        IPAddr.new(ip).loopback?
+      rescue IPAddr::InvalidAddressError
+        false
       end
 
+      # The host's first IPv4 address, falling back to IPv6
       def ip
-        @ip ||= Resolv.getaddress host
-      rescue Resolv::ResolvError
-        nil
+        @ip ||= begin
+          record = records.find { |r| r.type == 'A' } || records.find { |r| r.type == 'AAAA' }
+          record&.address&.to_s
+        end
       end
 
+      # The PTR hostname for #ip
       def hostname
-        require 'resolv'
-        @hostname ||= PublicSuffix.parse(Resolv.getname(ip))
-      rescue Resolv::ResolvError, PublicSuffix::DomainInvalid
-        nil
+        return @hostname if defined?(@hostname)
+        return @hostname = nil unless ip
+
+        ptr = lookup(ip, 'PTR').find { |record| record.type == 'PTR' }
+        @hostname = ptr ? PublicSuffix.parse(ptr.domainname.to_s) : nil
+      rescue PublicSuffix::DomainInvalid
+        @hostname = nil
       end
 
       def cnames
@@ -92,16 +109,23 @@ class SiteInspector
         {
           dnssec: dnssec?,
           ipv6: ipv6?,
-          cdn: cdn,
-          cloud_provider: cloud_provider,
+          cdn:,
+          cloud_provider:,
           google_apps: google_apps?,
-          hostname: hostname,
-          ip: ip
+          hostname: hostname.to_s,
+          ip:
         }
       end
 
       private
 
+      def lookup(name, type)
+        SiteInspector::Endpoint::Dns.resolver.query(name, type).answer
+      rescue Dnsruby::ResolvTimeout, Dnsruby::ResolvError => e
+        SiteInspector.logger.warn e.message
+        []
+      end
+
       def data
         @data ||= {}
       end

data/lib/site-inspector/endpoint/headers.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Headers < Check
+      COMMON = %w[
+        strict-transport-security
+        content-security-policy
+        x-frame-options
+        server
+        x-xss-protection
+      ].freeze
+
+      COMMON.each do |header|
+        name = header.gsub(/^x-/, '').tr('-', '_')
+
+        define_method name do
+          headers[header]
+        end
+
+        define_method "#{name}?" do
+          !!headers[header]
+        end
+      end
+
+      # more specific checks than presence of headers
+      def xss_protection?
+        xss_protection == '1; mode=block'
+      end
+
+      # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
+      def all
+        @all ||= response&.headers ? response.headers.transform_keys(&:downcase) : {}
+      end
+      alias headers all
+
+      def custom_headers
+        @custom_headers ||= all.select do |header|
+          header.start_with?('x-') && !COMMON.include?(header)
+        end
+      end
+
+      def [](header)
+        headers[header]
+      end
+
+      def to_h
+        return {} if endpoint.redirect?
+
+        hash = COMMON.to_h { |h| [h, headers[h] || false] }
+        hash.merge custom_headers
+      end
+    end
+  end
+end

data/lib/site-inspector/{checks → endpoint}/hsts.rb

@@ -5,14 +5,16 @@ class SiteInspector
     # Utility parser for HSTS headers.
     # RFC: http://tools.ietf.org/html/rfc6797
     class Hsts < Check
+      STATUS_ENDPOINT = 'https://hstspreload.org/api/v2/status'
+
       def valid?
-        return false unless header
+        return false if header.empty?
 
         pairs.none? { |key, value| "#{key}#{value}" =~ /[\s'"]/ }
       end
 
       def max_age
-        pairs[:"max-age"].to_i
+        pairs[:'max-age'].to_i
       end
 
       def include_subdomains?
@@ -37,11 +39,12 @@ class SiteInspector
       def to_h
         {
           valid: valid?,
-          max_age: max_age,
+          max_age:,
           include_subdomains: include_subdomains?,
           preload: preload?,
           enabled: enabled?,
-          preload_ready: preload_ready?
+          preload_ready: preload_ready?,
+          preload_list_status: preload_list['status']
         }
       end
 
@@ -52,7 +55,7 @@ class SiteInspector
       end
 
       def header
-        @header ||= headers['strict-transport-security']
+        @header ||= [headers['strict-transport-security']].flatten.join('; ').squeeze(';')
       end
 
       def directives
@@ -76,6 +79,34 @@ class SiteInspector
           pairs
         end
       end
+
+      def request
+        @request ||= begin
+          options = SiteInspector.typhoeus_defaults
+          options = options.merge(method: :get)
+          Typhoeus::Request.new(url, options)
+        end
+      end
+
+      def url
+        url = Addressable::URI.parse(STATUS_ENDPOINT)
+        url.query_values = { domain: endpoint.host.to_s }
+        url
+      end
+
+      def preload_list
+        @preload_list ||= begin
+          SiteInspector.hydra.queue(request)
+          SiteInspector.hydra.run
+
+          response = request.response
+          if response.success?
+            JSON.parse(response.body)
+          else
+            {}
+          end
+        end
+      end
     end
   end
 end

data/lib/site-inspector/endpoint/wappalyzer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Wappalyzer < Check
+      class WappalyzerError < RuntimeError; end
+
+      class << self
+        def wappalyzer?
+          return @wappalyzer_detected if defined? @wappalyzer_detected
+
+          @wappalyzer_detected = !!wappalyzer.detect
+        end
+
+        def enabled?
+          @@enabled && wappalyzer?
+        end
+
+        def wappalyzer
+          @wappalyzer ||= begin
+            path = ['*', './bin', './node_modules/.bin'].join(File::PATH_SEPARATOR)
+            Cliver::Dependency.new('wappalyzer', path:)
+          end
+        end
+
+        def run_command(args)
+          Open3.capture2e(wappalyzer.detect, *args)
+        end
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+
+        @to_h ||= begin
+          technologies = {}
+
+          data['technologies']&.each do |t|
+            category = t['categories'].first
+            category = category ? category['name'] : 'other'
+            category = category.downcase.tr(' ', '_').to_sym
+            technologies[category] ||= []
+            technologies[category].push t['name']
+          end
+
+          technologies
+        end
+      end
+
+      private
+
+      def data
+        @data ||= begin
+          args = [endpoint.uri.to_s]
+          output, _status = self.class.run_command(args)
+          JSON.parse(output)
+        rescue Timeout::Error, IOError
+          { technologies: {
+            categories: ['timeout'],
+            name: "https://www.wappalyzer.com/lookup/#{endpoint.host}"
+          } }
+        rescue JSON::ParserError
+          raise WappalyzerError, "Command `wappalyzer #{args.join(' ')}` failed: #{output}"
+        end
+      end
+    end
+  end
+end