site-inspector 3.1.1 → 4.0.0

data/lib/site-inspector/{checks → endpoint}/dns.rb

@@ -1,8 +1,16 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+
 class SiteInspector
   class Endpoint
     class Dns < Check
       class LocalhostError < StandardError; end
 
+      # Record types fetched for #records. ANY queries are deprecated
+      # (RFC 8482) and most resolvers answer them with a single HINFO record.
+      RECORD_TYPES = %w[A AAAA CNAME MX DNSKEY].freeze
+
       def self.resolver
         require 'dnsruby'
         @resolver ||= begin
@@ -12,20 +20,21 @@ class SiteInspector
         end
       end
 
-      def query(type = 'ANY')
-        SiteInspector::Endpoint::Dns.resolver.query(host.to_s, type).answer
-      rescue Dnsruby::ResolvTimeout, Dnsruby::ServFail, Dnsruby::NXDomain
-        []
+      def query(type = 'A')
+        lookup(host.to_s, type)
       end
 
       def records
-        @records ||= query
+        @records ||= RECORD_TYPES.flat_map { |type| query(type) }.uniq(&:to_s)
       end
 
       def record?(type)
-        records.any? { |record| record.type == type } || query(type).count != 0
+        return true if records.any? { |record| record.type == type }
+        return false if RECORD_TYPES.include?(type.to_s)
+
+        query(type).any?
       end
-      alias_method :has_record?, :record?
+      alias has_record? record?
 
       def dnssec?
         @dnssec ||= has_record? 'DNSKEY'
@@ -52,27 +61,36 @@ class SiteInspector
       end
 
       def google_apps?
-        @google ||= records.any? do |record|
+        @google_apps ||= records.any? do |record|
           record.type == 'MX' && record.exchange.to_s =~ /google(mail)?\.com\.?\z/i
         end
       end
 
       def localhost?
-        ip == '127.0.0.1'
+        return false unless ip
+
+        IPAddr.new(ip).loopback?
+      rescue IPAddr::InvalidAddressError
+        false
       end
 
+      # The host's first IPv4 address, falling back to IPv6
       def ip
-        require 'resolv'
-        @ip ||= Resolv.getaddress host
-      rescue Resolv::ResolvError
-        nil
+        @ip ||= begin
+          record = records.find { |r| r.type == 'A' } || records.find { |r| r.type == 'AAAA' }
+          record&.address&.to_s
+        end
       end
 
+      # The PTR hostname for #ip
       def hostname
-        require 'resolv'
-        @hostname ||= PublicSuffix.parse(Resolv.getname(ip))
-      rescue Resolv::ResolvError, PublicSuffix::DomainInvalid
-        nil
+        return @hostname if defined?(@hostname)
+        return @hostname = nil unless ip
+
+        ptr = lookup(ip, 'PTR').find { |record| record.type == 'PTR' }
+        @hostname = ptr ? PublicSuffix.parse(ptr.domainname.to_s) : nil
+      rescue PublicSuffix::DomainInvalid
+        @hostname = nil
       end
 
       def cnames
@@ -87,19 +105,27 @@ class SiteInspector
 
       def to_h
         return { error: LocalhostError } if localhost?
+
         {
-          dnssec:         dnssec?,
-          ipv6:           ipv6?,
-          cdn:            cdn,
-          cloud_provider: cloud_provider,
-          google_apps:    google_apps?,
-          hostname:       hostname,
-          ip:             ip
+          dnssec: dnssec?,
+          ipv6: ipv6?,
+          cdn:,
+          cloud_provider:,
+          google_apps: google_apps?,
+          hostname: hostname.to_s,
+          ip:
         }
       end
 
       private
 
+      def lookup(name, type)
+        SiteInspector::Endpoint::Dns.resolver.query(name, type).answer
+      rescue Dnsruby::ResolvTimeout, Dnsruby::ResolvError => e
+        SiteInspector.logger.warn e.message
+        []
+      end
+
       def data
         @data ||= {}
       end
@@ -118,7 +144,7 @@ class SiteInspector
         haystack = load_data(type)
         needle = haystack.find do |_name, domain|
           cnames.any? do |cname|
-            domain == cname.tld || domain == "#{cname.sld}.#{cname.tld}"
+            [cname.tld, "#{cname.sld}.#{cname.tld}"].include? domain
           end
         end
 
@@ -126,7 +152,7 @@ class SiteInspector
         return nil unless hostname
 
         needle = haystack.find do |_name, domain|
-          domain == hostname.tld || domain == "#{hostname.sld}.#{hostname.tld}"
+          [hostname.tld, "#{hostname.sld}.#{hostname.tld}"].include? domain
         end
 
         needle ? needle[0].to_sym : nil

data/lib/site-inspector/endpoint/headers.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Headers < Check
+      COMMON = %w[
+        strict-transport-security
+        content-security-policy
+        x-frame-options
+        server
+        x-xss-protection
+      ].freeze
+
+      COMMON.each do |header|
+        name = header.gsub(/^x-/, '').tr('-', '_')
+
+        define_method name do
+          headers[header]
+        end
+
+        define_method "#{name}?" do
+          !!headers[header]
+        end
+      end
+
+      # more specific checks than presence of headers
+      def xss_protection?
+        xss_protection == '1; mode=block'
+      end
+
+      # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
+      def all
+        @all ||= response&.headers ? response.headers.transform_keys(&:downcase) : {}
+      end
+      alias headers all
+
+      def custom_headers
+        @custom_headers ||= all.select do |header|
+          header.start_with?('x-') && !COMMON.include?(header)
+        end
+      end
+
+      def [](header)
+        headers[header]
+      end
+
+      def to_h
+        return {} if endpoint.redirect?
+
+        hash = COMMON.to_h { |h| [h, headers[h] || false] }
+        hash.merge custom_headers
+      end
+    end
+  end
+end

data/lib/site-inspector/endpoint/hsts.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    # Utility parser for HSTS headers.
+    # RFC: http://tools.ietf.org/html/rfc6797
+    class Hsts < Check
+      STATUS_ENDPOINT = 'https://hstspreload.org/api/v2/status'
+
+      def valid?
+        return false if header.empty?
+
+        pairs.none? { |key, value| "#{key}#{value}" =~ /[\s'"]/ }
+      end
+
+      def max_age
+        pairs[:'max-age'].to_i
+      end
+
+      def include_subdomains?
+        pairs.key?(:includesubdomains)
+      end
+
+      def preload?
+        pairs.key?(:preload)
+      end
+
+      def enabled?
+        return false unless max_age
+
+        max_age.positive?
+      end
+
+      # Google's minimum max-age for automatic preloading
+      def preload_ready?
+        include_subdomains? && preload? && max_age >= 10_886_400
+      end
+
+      def to_h
+        {
+          valid: valid?,
+          max_age:,
+          include_subdomains: include_subdomains?,
+          preload: preload?,
+          enabled: enabled?,
+          preload_ready: preload_ready?,
+          preload_list_status: preload_list['status']
+        }
+      end
+
+      private
+
+      def headers
+        endpoint.headers
+      end
+
+      def header
+        @header ||= [headers['strict-transport-security']].flatten.join('; ').squeeze(';')
+      end
+
+      def directives
+        @directives ||= header ? header.split(/\s*;\s*/) : []
+      end
+
+      def pairs
+        @pairs ||= begin
+          pairs = {}
+          directives.each do |directive|
+            key, value = directive.downcase.split('=')
+
+            if /".*"/.match?(value)
+              value = value.sub(/^"/, '')
+              value = value.sub(/"$/, '')
+            end
+
+            pairs[key.to_sym] = value
+          end
+
+          pairs
+        end
+      end
+
+      def request
+        @request ||= begin
+          options = SiteInspector.typhoeus_defaults
+          options = options.merge(method: :get)
+          Typhoeus::Request.new(url, options)
+        end
+      end
+
+      def url
+        url = Addressable::URI.parse(STATUS_ENDPOINT)
+        url.query_values = { domain: endpoint.host.to_s }
+        url
+      end
+
+      def preload_list
+        @preload_list ||= begin
+          SiteInspector.hydra.queue(request)
+          SiteInspector.hydra.run
+
+          response = request.response
+          if response.success?
+            JSON.parse(response.body)
+          else
+            {}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/site-inspector/{checks → endpoint}/https.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Https < Check
@@ -23,7 +25,7 @@ class SiteInspector
 
       def to_h
         {
-          valid:       valid?,
+          valid: valid?,
           return_code: response.return_code
         }
       end

data/lib/site-inspector/endpoint/wappalyzer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Wappalyzer < Check
+      class WappalyzerError < RuntimeError; end
+
+      class << self
+        def wappalyzer?
+          return @wappalyzer_detected if defined? @wappalyzer_detected
+
+          @wappalyzer_detected = !!wappalyzer.detect
+        end
+
+        def enabled?
+          @@enabled && wappalyzer?
+        end
+
+        def wappalyzer
+          @wappalyzer ||= begin
+            path = ['*', './bin', './node_modules/.bin'].join(File::PATH_SEPARATOR)
+            Cliver::Dependency.new('wappalyzer', path:)
+          end
+        end
+
+        def run_command(args)
+          Open3.capture2e(wappalyzer.detect, *args)
+        end
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+
+        @to_h ||= begin
+          technologies = {}
+
+          data['technologies']&.each do |t|
+            category = t['categories'].first
+            category = category ? category['name'] : 'other'
+            category = category.downcase.tr(' ', '_').to_sym
+            technologies[category] ||= []
+            technologies[category].push t['name']
+          end
+
+          technologies
+        end
+      end
+
+      private
+
+      def data
+        @data ||= begin
+          args = [endpoint.uri.to_s]
+          output, _status = self.class.run_command(args)
+          JSON.parse(output)
+        rescue Timeout::Error, IOError
+          { technologies: {
+            categories: ['timeout'],
+            name: "https://www.wappalyzer.com/lookup/#{endpoint.host}"
+          } }
+        rescue JSON::ParserError
+          raise WappalyzerError, "Command `wappalyzer #{args.join(' ')}` failed: #{output}"
+        end
+      end
+    end
+  end
+end

data/lib/site-inspector/endpoint/well_known.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+class SiteInspector
+  class Endpoint
+    class WellKnown < Check
+      BASE_PATH = './.well-known/'
+
+      class << self
+        def well_knowns
+          data = File.expand_path '../../data/well-known.yml', File.dirname(__FILE__)
+          wks = YAML.load_file(data)
+          wks.reject { |wk| wk.include?('deprecated') }
+        end
+
+        def keys
+          well_knowns.map { |wk| key_for(wk) }
+        end
+
+        def key_paths
+          @key_paths ||= begin
+            key_paths = well_knowns.map do |wk|
+              [key_for(wk), path_for(wk)]
+            end
+
+            key_paths.to_h
+          end
+        end
+
+        def path_for(wk)
+          return unless well_knowns.include?(wk)
+
+          "#{BASE_PATH}#{wk}"
+        end
+
+        def key_for(well_known)
+          well_known.gsub(/(\.|-)/, '_').to_sym
+        end
+      end
+
+      def exists?(path)
+        endpoint.content.path_exists?(path)
+      end
+
+      def uri_for(key)
+        path = self.class.key_paths[key]
+        endpoint.join(path) if path
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+        return @hash if defined?(@hash)
+
+        prefetch
+        @hash = {}
+
+        self.class.keys.each do |key|
+          @hash[key] = public_send(key)
+        end
+
+        @hash
+      end
+
+      def method_missing(method_sym, *arguments, &)
+        if respond_to_missing?(method_sym)
+          exists?(self.class.key_paths[method_sym])
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method_sym, include_private = false)
+        if self.class.keys.include?(method_sym)
+          true
+        else
+          super
+        end
+      end
+
+      private
+
+      def prefetch
+        options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
+        self.class.key_paths.each do |_key, path|
+          uri = endpoint.join(path)
+          request = Typhoeus::Request.new(uri, options)
+          SiteInspector.hydra.queue(request)
+        end
+        SiteInspector.hydra.run
+      end
+    end
+  end
+end

data/lib/site-inspector/endpoint/whois.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Whois < Check
+      def domain
+        return @domain if defined? @domain
+
+        @domain = begin
+          whois.lookup host
+        rescue Timeout::Error
+          nil
+        end
+      end
+
+      def ip
+        return @ip if defined? @ip
+
+        @ip = begin
+          whois.lookup ip_address if ip_address
+        rescue Timeout::Error
+          nil
+        end
+      end
+
+      def to_h
+        {
+          domain: record_to_h(domain),
+          ip: record_to_h(ip)
+        }
+      end
+
+      private
+
+      def record_to_h(record)
+        return unless record
+
+        record.content.scan(/^\s*(.*?):\s*(.*?)\r?\n/).to_h
+      end
+
+      def ip_address
+        return @ip_address if defined? @ip_address
+
+        @ip_address = begin
+          Resolv.getaddress host
+        rescue Resolv::ResolvError
+          nil
+        end
+      end
+
+      def whois
+        @whois ||= ::Whois::Client.new
+      end
+    end
+  end
+end