site-inspector 1.0.2 → 3.2.0

data/lib/site-inspector/checks/headers.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Headers < Check
+      # TODO: kill this
+      def strict_transport_security?
+        !!strict_transport_security
+      end
+
+      def content_security_policy?
+        !!content_security_policy
+      end
+
+      def click_jacking_protection?
+        !!click_jacking_protection
+      end
+
+      # return the found header value
+
+      # TODO: kill this
+      def strict_transport_security
+        headers['strict-transport-security']
+      end
+
+      def content_security_policy
+        headers['content-security-policy']
+      end
+
+      def click_jacking_protection
+        headers['x-frame-options']
+      end
+
+      def server
+        headers['server']
+      end
+
+      def xss_protection
+        headers['x-xss-protection']
+      end
+
+      # more specific checks than presence of headers
+      def xss_protection?
+        xss_protection == '1; mode=block'
+      end
+
+      # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
+      def all
+        @all ||= response&.headers ? response.headers.transform_keys(&:downcase) : {}
+      end
+      alias headers all
+
+      def [](header)
+        headers[header]
+      end
+
+      def to_h
+        {
+          strict_transport_security: strict_transport_security || false,
+          content_security_policy: content_security_policy || false,
+          click_jacking_protection: click_jacking_protection || false,
+          server: server,
+          xss_protection: xss_protection || false
+        }
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/hsts.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    # Utility parser for HSTS headers.
+    # RFC: http://tools.ietf.org/html/rfc6797
+    class Hsts < Check
+      def valid?
+        return false unless header
+
+        pairs.none? { |key, value| "#{key}#{value}" =~ /[\s'"]/ }
+      end
+
+      def max_age
+        pairs[:"max-age"].to_i
+      end
+
+      def include_subdomains?
+        pairs.key?(:includesubdomains)
+      end
+
+      def preload?
+        pairs.key?(:preload)
+      end
+
+      def enabled?
+        return false unless max_age
+
+        max_age.positive?
+      end
+
+      # Google's minimum max-age for automatic preloading
+      def preload_ready?
+        include_subdomains? && preload? && max_age >= 10_886_400
+      end
+
+      def to_h
+        {
+          valid: valid?,
+          max_age: max_age,
+          include_subdomains: include_subdomains?,
+          preload: preload?,
+          enabled: enabled?,
+          preload_ready: preload_ready?
+        }
+      end
+
+      private
+
+      def headers
+        endpoint.headers
+      end
+
+      def header
+        @header ||= headers['strict-transport-security']
+      end
+
+      def directives
+        @directives ||= header ? header.split(/\s*;\s*/) : []
+      end
+
+      def pairs
+        @pairs ||= begin
+          pairs = {}
+          directives.each do |directive|
+            key, value = directive.downcase.split('=')
+
+            if /".*"/.match?(value)
+              value = value.sub(/^"/, '')
+              value = value.sub(/"$/, '')
+            end
+
+            pairs[key.to_sym] = value
+          end
+
+          pairs
+        end
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/https.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Https < Check
+      def scheme?
+        scheme == 'https'
+      end
+
+      def valid?
+        scheme? && response && response.return_code == :ok
+      end
+
+      def bad_chain?
+        scheme? && response && response.return_code == :ssl_cacert
+      end
+
+      def bad_name?
+        scheme? && response && response.return_code == :peer_failed_verification
+      end
+
+      def inspect
+        "#<SiteInspector::Endpoint::Https valid=#{valid?}>"
+      end
+
+      def to_h
+        {
+          valid: valid?,
+          return_code: response.return_code
+        }
+      end
+
+      private
+
+      def scheme
+        @scheme ||= request.base_url.scheme
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/sniffer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Sniffer < Check
+      OPEN_SOURCE_FRAMEWORKS = [
+        # Sniffles
+        :drupal,
+        :joomla,
+        :movabletype,
+        :phpbb,
+        :wordpress,
+
+        # Internal
+        :php,
+        :expression_engine,
+        :cowboy
+      ].freeze
+
+      def framework
+        cms = sniff :cms
+        return cms unless cms.nil?
+        return :expression_engine if endpoint.cookies.any? { |c| c.keys.first =~ /^exp_/ }
+        return :php if endpoint.cookies['PHPSESSID']
+        return :coldfusion if endpoint.cookies['CFID'] && endpoint.cookies['CFTOKEN']
+        return :cowboy if endpoint.headers.server.to_s.casecmp('cowboy').zero?
+
+        nil
+      end
+
+      def open_source?
+        OPEN_SOURCE_FRAMEWORKS.include?(framework)
+      end
+
+      def analytics
+        sniff :analytics
+      end
+
+      def javascript
+        sniff :javascript
+      end
+
+      def advertising
+        sniff :advertising
+      end
+
+      def to_h
+        {
+          framework: framework,
+          analytics: analytics,
+          javascript: javascript,
+          advertising: advertising
+        }
+      end
+
+      private
+
+      def sniff(type)
+        require 'sniffles'
+        results = Sniffles.sniff(endpoint.content.body, type).select { |_name, meta| meta[:found] }
+        results&.keys&.first
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/wappalyzer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Wappalyzer < Check
+      ENDPOINT = 'https://api.wappalyzer.com/lookup/v2/'
+
+      def to_h
+        return {} unless data['technologies']
+
+        @to_h ||= begin
+          technologies = {}
+          data['technologies'].each do |t|
+            category = t['categories'].first
+            category = category ? category['name'] : 'Other'
+            technologies[category] ||= []
+            technologies[category].push t['name']
+          end
+
+          technologies
+        end
+      end
+
+      private
+
+      def request
+        @request ||= begin
+          options = SiteInspector.typhoeus_defaults
+          headers = options[:headers].merge({ "x-api-key": api_key })
+          options = options.merge(method: :get, headers: headers)
+          Typhoeus::Request.new(url, options)
+        end
+      end
+
+      def data
+        return {} unless api_key && api_key != ''
+
+        @data ||= begin
+          SiteInspector.hydra.queue(request)
+          SiteInspector.hydra.run
+
+          response = request.response
+          if response.success?
+            JSON.parse(response.body).first
+          else
+            {}
+          end
+        end
+      end
+
+      def url
+        url = Addressable::URI.parse(ENDPOINT)
+        url.query_values = { urls: endpoint.uri }
+        url
+      end
+
+      def api_key
+        @api_key ||= ENV['WAPPALYZER_API_KEY']
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/whois.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Whois < Check
+      def domain
+        @domain ||= whois.lookup host
+      end
+
+      def ip
+        @ip ||= whois.lookup ip_address
+      end
+
+      def to_h
+        {
+          domain: record_to_h(domain),
+          ip: record_to_h(ip)
+        }
+      end
+
+      private
+
+      def record_to_h(record)
+        record.content.scan(/^\s*(.*?):\s*(.*?)\r?\n/).to_h
+      end
+
+      def ip_address
+        @ip_address ||= Resolv.getaddress host
+      end
+
+      def whois
+        @whois ||= ::Whois::Client.new
+      end
+    end
+  end
+end

data/lib/site-inspector/disk_cache.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class DiskCache
+    def initialize(dir = nil, replace = nil)
+      @dir     = dir || ENV['CACHE']
+      @replace = replace || ENV['CACHE_REPLACE']
+      @memory  = {}
+    end
+
+    def get(request)
+      return unless File.exist?(path(request))
+      return @memory[request] if @memory[request]
+
+      if @replace
+        FileUtils.rm(path(request))
+        nil
+      else
+        begin
+          contents = File.read(path(request))
+          Marshal.load(contents)
+        rescue ArgumentError
+          FileUtils.rm(path(request))
+          nil
+        end
+      end
+    end
+
+    def set(request, response)
+      File.write(path(request), Marshal.dump(response))
+      @memory[request] = response
+    end
+
+    private
+
+    # The `request` is a Typhoeus::Request, which provides a
+    # unique `cache_key` string for exactly this sort of thing.
+    def path(request)
+      File.join(@dir, request.cache_key)
+    end
+  end
+end

data/lib/site-inspector/domain.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Domain
+    attr_reader :host
+
+    def initialize(host)
+      host = host.downcase
+      host = host.sub(/^https?:/, '')
+      host = host.sub(%r{^/+}, '')
+      host = host.sub(/^www\./, '')
+      uri = Addressable::URI.parse "//#{host}"
+      @host = uri.host
+    end
+
+    def endpoints
+      @endpoints ||= [
+        Endpoint.new("https://#{host}", domain: self),
+        Endpoint.new("https://www.#{host}", domain: self),
+        Endpoint.new("http://#{host}", domain: self),
+        Endpoint.new("http://www.#{host}", domain: self)
+      ]
+    end
+
+    def canonical_endpoint
+      @canonical_endpoint ||= begin
+        prefetch
+        endpoints.find do |e|
+          e.https? == canonically_https? && e.www? == canonically_www?
+        end
+      end
+    end
+
+    def government?
+      require 'gman'
+      Gman.valid? host
+    end
+
+    # Does *any* endpoint return a 200 or 300 response code?
+    def up?
+      endpoints.any?(&:up?)
+    end
+
+    # Does *any* endpoint respond to HTTP?
+    # TODO: needs to allow an invalid chain.
+    def responds?
+      endpoints.any?(&:responds?)
+    end
+
+    # TODO: These weren't present before, and may not be useful.
+    # Can you connect to www?
+    def www?
+      endpoints.any? { |e| e.www? && e.up? }
+    end
+
+    # Can you connect without www?
+    def root?
+      endpoints.any? { |e| e.root? && e.up? }
+    end
+
+    # HTTPS is "supported" (different than "canonical" or "enforced") if:
+    #
+    # * Either of the HTTPS endpoints is listening, and doesn't have
+    #   an invalid hostname.
+    #
+    # TODO: needs to allow an invalid chain.
+    def https?
+      endpoints.any? { |e| e.https? && e.up? && e.https.valid? }
+    end
+
+    # HTTPS is enforced if one of the HTTPS endpoints is "up",
+    # and if both *HTTP* endpoints are either:
+    #
+    #  * down, or
+    #  * redirect immediately to HTTPS.
+    #
+    # This is different than whether a domain is "canonically" HTTPS.
+    #
+    # * an HTTP redirect can go to HTTPS on another domain, as long
+    #   as it's immediate.
+    # * a domain with an invalid cert can still be enforcing HTTPS.
+    #
+    # TODO: need to ensure the redirect *immediately* goes to HTTPS.
+    # TODO: don't need to require that the HTTPS cert is valid for this purpose.
+    def enforces_https?
+      return false unless https?
+
+      endpoints.select(&:http?).all? { |e| !e.up? || e.redirect&.https? }
+    end
+
+    # we can say that a canonical HTTPS site "defaults" to HTTPS,
+    # even if it doesn't *strictly* enforce it (e.g. having a www
+    # subdomain first to go HTTP root before HTTPS root).
+    #
+    # TODO: not implemented.
+    def defaults_https?
+      raise 'Not implemented. Halp?'
+    end
+
+    # HTTPS is "downgraded" if both:
+    #
+    # * HTTPS is supported, and
+    # * The 'canonical' endpoint gets an immediate internal redirect to HTTP.
+    #
+    # TODO: the redirect must be internal.
+    def downgrades_https?
+      return false unless https?
+
+      canonical_endpoint.redirect? && canonical_endpoint.redirect.http?
+    end
+
+    # A domain is "canonically" at www if:
+    #  * at least one of its www endpoints responds
+    #  * both root endpoints are either down ~~or redirect *somewhere*~~, or
+    #  * at least one root endpoint redirect should immediately go to
+    #    an *internal* www endpoint
+    # This is meant to affirm situations like:
+    #   http:// -> https:// -> https://www
+    #   https:// -> http:// -> https://www
+    # and meant to avoid affirming situations like:
+    #   http:// -> http://non-www,
+    #   http://www -> http://non-www
+    # or like:
+    #   https:// -> 200, http:// -> http://www
+    def canonically_www?
+      # Does any endpoint respond?
+      return false unless up?
+
+      # Does at least one www endpoint respond?
+      return false unless www?
+
+      # Are both root endpoints down?
+      return true if endpoints.select(&:root?).all? { |e| !e.up? }
+
+      # Does either root endpoint redirect to a www endpoint?
+      endpoints.select(&:root?).any? { |e| e.redirect&.www? }
+    end
+
+    # A domain is "canonically" at https if:
+    #  * at least one of its https endpoints is live and
+    #    doesn't have an invalid hostname
+    #  * both http endpoints are either down or redirect *somewhere*
+    #  * at least one http endpoint redirects immediately to
+    #    an *internal* https endpoint
+    # This is meant to affirm situations like:
+    #   http:// -> http://www -> https://
+    #   https:// -> http:// -> https://www
+    # and meant to avoid affirming situations like:
+    #   http:// -> http://non-www
+    #   http://www -> http://non-www
+    # or:
+    #   http:// -> 200, http://www -> https://www
+    #
+    # It allows a site to be canonically HTTPS if the cert has
+    # a valid hostname but invalid chain issues.
+    def canonically_https?
+      # Does any endpoint respond?
+      return false unless up?
+
+      # At least one of its https endpoints is live and doesn't have an invalid hostname
+      return false unless https?
+
+      # Both http endpoints are down
+      return true if endpoints.select(&:http?).all? { |e| !e.up? }
+
+      # at least one http endpoint redirects immediately to https
+      endpoints.select(&:http?).any? { |e| e.redirect&.https? }
+    end
+
+    # A domain redirects if
+    # 1. At least one endpoint is an external redirect, and
+    # 2. All endpoints are either down or an external redirect
+    def redirect?
+      return false unless redirect
+
+      endpoints.all? { |e| !e.up? || e.external_redirect? }
+    end
+
+    # The first endpoint to respond with a redirect
+    def redirect
+      endpoints.find(&:external_redirect?)
+    end
+
+    # HSTS on the canonical domain?
+    def hsts?
+      canonical_endpoint.hsts&.enabled?
+    end
+
+    def hsts_subdomains?
+      endpoints.find { |e| e.root? && e.https? }.hsts.include_subdomains?
+    end
+
+    def hsts_preload_ready?
+      return false unless hsts_subdomains?
+
+      endpoints.find { |e| e.root? && e.https? }.hsts.preload_ready?
+    end
+
+    def to_s
+      host
+    end
+
+    def inspect
+      "#<SiteInspector::Domain host=\"#{host}\">"
+    end
+
+    # We know most API calls to the domain model are going to require
+    # That the root of all four endpoints are called. Rather than process them
+    # In serial, lets grab them in parallel and cache the results to speed
+    # up later calls.
+    def prefetch
+      endpoints.each do |endpoint|
+        request = Typhoeus::Request.new(endpoint.uri, SiteInspector.typhoeus_defaults)
+        SiteInspector.hydra.queue(request)
+      end
+      SiteInspector.hydra.run
+    end
+
+    # Converts the domain to a hash
+    #
+    # By default, it only returns domain-wide information and
+    # information about the canonical endpoint
+    #
+    # It will also pass options allong to each endpoint's to_h method
+    #
+    # options:
+    #  :all - return information about all endpoints
+    #
+    # Returns a complete hash of the domain's information
+    def to_h(options = {})
+      prefetch
+
+      hash = {
+        host: host,
+        up: up?,
+        responds: responds?,
+        www: www?,
+        root: root?,
+        https: https?,
+        enforces_https: enforces_https?,
+        downgrades_https: downgrades_https?,
+        canonically_www: canonically_www?,
+        canonically_https: canonically_https?,
+        redirect: redirect?,
+        hsts: hsts?,
+        hsts_subdomains: hsts_subdomains?,
+        hsts_preload_ready: hsts_preload_ready?,
+        canonical_endpoint: canonical_endpoint.to_h(options)
+      }
+
+      if options['all']
+        hash[:endpoints] = {
+          https: {
+            root: endpoints[0].to_h(options),
+            www: endpoints[1].to_h(options)
+          },
+          http: {
+            root: endpoints[2].to_h(options),
+            www: endpoints[3].to_h(options)
+          }
+        }
+      end
+
+      hash
+    end
+
+    def to_json(*_args)
+      to_h.to_json
+    end
+  end
+end