site-inspector 1.0.2 → 3.2.0

data/lib/cliver/dependency_ext.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Cliver
+  class Dependency
+    # Memoized shortcut for detect
+    # Returns the path to the detected dependency
+    # Raises an error if the dependency was not satisfied
+    def path
+      @path ||= detect!
+    end
+
+    # Returns the version of the resolved dependency
+    def version
+      return @version if defined? @version
+
+      version = installed_versions.find { |p, _v| p == path }
+      @detected_version = version.nil? ? nil : version[1]
+    end
+
+    def major_version
+      version&.split('.')&.first
+    end
+  end
+end

data/lib/site-inspector.rb

@@ -1,636 +1,83 @@
+# frozen_string_literal: true
 
-# needed for HTTP analysis
 require 'open-uri'
-require "addressable/uri"
+require 'addressable/uri'
 require 'public_suffix'
 require 'typhoeus'
+require 'parallel'
+require 'cliver'
+require 'whois'
+require 'cgi'
+require 'resolv'
+require 'dotenv/load'
 
 require_relative 'site-inspector/cache'
-require_relative 'site-inspector/headers'
-require_relative 'site-inspector/sniffer'
-require_relative 'site-inspector/dns'
-require_relative 'site-inspector/compliance'
-
-
-if ENV['CACHE']
-  Typhoeus::Config.cache = SiteInspectorDiskCache.new(ENV['CACHE'], ENV['CACHE_REPLACE'])
-else
-  Typhoeus::Config.cache = SiteInspectorCache.new
-end
+require_relative 'site-inspector/disk_cache'
+require_relative 'site-inspector/rails_cache'
+require_relative 'site-inspector/domain'
+require_relative 'site-inspector/checks/check'
+require_relative 'site-inspector/checks/accessibility'
+require_relative 'site-inspector/checks/content'
+require_relative 'site-inspector/checks/dns'
+require_relative 'site-inspector/checks/headers'
+require_relative 'site-inspector/checks/hsts'
+require_relative 'site-inspector/checks/https'
+require_relative 'site-inspector/checks/sniffer'
+require_relative 'site-inspector/checks/cookies'
+require_relative 'site-inspector/checks/whois'
+require_relative 'site-inspector/checks/wappalyzer'
+require_relative 'site-inspector/endpoint'
+require_relative 'site-inspector/version'
+require_relative 'cliver/dependency_ext'
 
 class SiteInspector
-
-  def self.load_data(name)
-    require 'yaml'
-    YAML.load_file File.expand_path "./data/#{name}.yml", File.dirname(__FILE__)
-  end
-
-  # Utility parser for HSTS headers.
-  # RFC: http://tools.ietf.org/html/rfc6797
-  def self.hsts_parse(header)
-    # no hsts for you
-    nothing = {
-      max_age: nil,
-      include_subdomains: false,
-      preload: false,
-      enabled: false,
-      preload_ready: false
-    }
-
-    return nothing unless header and header.is_a?(String)
-
-    directives = header.split(/\s*;\s*/)
-
-    pairs = []
-    directives.each do |directive|
-      name, value = directive.downcase.split("=")
-
-      if value and value.start_with?("\"") and value.end_with?("\"")
-        value = value.sub(/^\"/, '')
-        value = value.sub(/\"$/, '')
-      end
-
-      pairs.push([name, value])
+  class << self
+    attr_writer :timeout, :cache, :typhoeus_options
+
+    def cache
+      @cache ||= if ENV['CACHE']
+                   SiteInspector::DiskCache.new
+                 elsif Object.const_defined?('Rails')
+                   SiteInspector::RailsCache.new
+                 else
+                   SiteInspector::Cache.new
+                 end
     end
 
-    # reject invalid directives
-    fatal = pairs.any? do |name, value|
-      # TODO: more comprehensive rejection of characters
-      invalid_chars = /[\s\'\"]/
-      (name =~ invalid_chars) or (value =~ invalid_chars)
+    def timeout
+      @timeout || 10
     end
 
-    # good DAY, sir
-    return nothing if fatal
-
-    max_age_directive = pairs.find {|n, v| n == "max-age"}
-    max_age = max_age_directive ? max_age_directive[1].to_i : nil
-    include_subdomains = !!pairs.find {|n, v| n == "includesubdomains"}
-    preload = !!pairs.find {|n, v| n == "preload"}
-
-    enabled = !!(max_age and (max_age > 0))
-
-    # Google's minimum max-age for automatic preloading
-    eighteen_weeks = !!(max_age and (max_age >= 10886400))
-    preload_ready = !!(eighteen_weeks and include_subdomains and preload)
-
-    {
-      max_age: max_age,
-      include_subdomains: include_subdomains,
-      preload: preload,
-      enabled: enabled,
-      preload_ready: preload_ready
-    }
-  end
-
-  # makes no network requests
-  def initialize(domain, options = {})
-    domain = domain.downcase
-    domain = domain.sub /^https?\:/, ""
-    domain = domain.sub /^\/+/, ""
-    domain = domain.sub /^www\./, ""
-    @uri = Addressable::URI.parse "//#{domain}"
-    @domain = PublicSuffix.parse @uri.host
-    @timeout = options[:timeout] || 10
-  end
-
-  def inspect
-    "<SiteInspector domain=\"#{domain}\">"
-  end
-
-  def uri(ssl=enforce_https?,www=www?)
-    uri = @uri.clone
-    uri.host = www ? "www.#{uri.host}" : uri.host
-    uri.scheme = ssl ? "https" : "http"
-    uri
-  end
-
-  def domain
-    www? ? PublicSuffix.parse("www.#{@uri.host}") : @domain
-  end
-
-  def request(ssl=false, www=false, followlocation=true, ssl_verifypeer=true, ssl_verifyhost=true)
-    to_get = uri(ssl, www)
-
-    # debugging
-    # puts "fetching: #{to_get}, #{followlocation ? "follow" : "no follow"}, #{ssl_verifypeer ? "verify peer, " : ""}#{ssl_verifyhost ? "verify host" : ""}"
-
-    Typhoeus.get(to_get, followlocation: followlocation, ssl_verifypeer: ssl_verifypeer, ssl_verifyhost: (ssl_verifyhost ? 2 : 0), timeout: @timeout)
-  end
-
-  def response
-    @response ||= begin
-      if response = request(false, false) and response.success?
-        @non_www = true
-        response
-      elsif response = request(false, true) and response.success?
-        @non_www = false
-        response
-      else
-        false
-      end
-    end
-  end
-
-  def timed_out?
-    response && response.timed_out?
-  end
-
-  def doc
-    require 'nokogiri'
-    @doc ||= Nokogiri::HTML response.body if response
-  end
-
-  def body
-    doc.to_s.force_encoding("UTF-8").encode("UTF-8", :invalid => :replace, :replace => "")
-  end
-
-  def government?
-    require 'gman'
-    Gman.valid? domain.to_s
-  end
-
-  def https?
-    @https ||= request(true, www?).success?
-  end
-  alias_method :ssl?, :https?
-
-  def enforce_https?
-    return false unless https?
-    @enforce_https ||= begin
-      response = request(false, www?)
-      if response.effective_url
-        Addressable::URI.parse(response.effective_url).scheme == "https"
-      else
-        false
-      end
+    def inspect(domain)
+      Domain.new(domain)
     end
-  end
-
-  def www?
-    response && response.effective_url && !!response.effective_url.match(/^https?:\/\/www\./)
-  end
-
-  def non_www?
-    response && @non_www
-  end
-
-  def redirect?
-    !!redirect
-  end
-
-  def redirect
-    @redirect ||= begin
-      if location = request(https?, www?, false).headers["location"]
-        redirect_domain = SiteInspector.new(location).domain
-        redirect_domain.to_s if redirect_domain.to_s != domain.to_s
-      end
-    rescue
-      nil
-    end
-  end
-
-  def http
-    details = {
-      endpoints: endpoints
-    }
-
-    # convenient shorthand for the extensive statements to come
-    combos = details[:endpoints]
-
-    # A domain is "canonically" at www if:
-    #  * at least one of its www endpoints responds
-    #  * both root endpoints are either down or redirect *somewhere*
-    #  * either both root endpoints are down, *or* at least one
-    #    root endpoint redirect should immediately go to
-    #    an *internal* www endpoint
-    # This is meant to affirm situations like:
-    #   http:// -> https:// -> https://www
-    #   https:// -> http:// -> https://www
-    # and meant to avoid affirming situations like:
-    #   http:// -> http://non-www,
-    #   http://www -> http://non-www
-    # or like:
-    #   https:// -> 200, http:// -> http://www
 
-    www = !!(
-      (
-        combos[:https][:www][:up] or
-        combos[:http][:www][:up]
-      ) and (
-        (
-          combos[:https][:root][:redirect] or
-          !combos[:https][:root][:up] or
-          combos[:https][:root][:https_bad_name] or
-          !combos[:https][:root][:status].to_s.start_with?("2")
-        ) and (
-          combos[:http][:root][:redirect] or
-          !combos[:http][:root][:up] or
-          !combos[:http][:root][:status].to_s.start_with?("2")
-        )
-      ) and (
-        (
-          (
-            !combos[:https][:root][:up] or
-            combos[:https][:root][:https_bad_name] or
-            !combos[:https][:root][:status].to_s.start_with?("2")
-          ) and
-          (
-            !combos[:http][:root][:up] or
-            !combos[:http][:root][:status].to_s.start_with?("2")
-          )
-        ) or
-        (
-          combos[:https][:root][:redirect_immediately_to_www] and
-          !combos[:https][:root][:redirect_immediately_external]
-        ) or
-        (
-          combos[:http][:root][:redirect_immediately_to_www] and
-          !combos[:http][:root][:redirect_immediately_external]
-        )
-      )
-    )
-
-    # A domain is "canonically" at https if:
-    #  * at least one of its https endpoints is live and
-    #    doesn't have an invalid hostname
-    #  * both http endpoints are either down or redirect *somewhere*
-    #  * at least one http endpoint redirects immediately to
-    #    an *internal* https endpoint
-    # This is meant to affirm situations like:
-    #   http:// -> http://www -> https://
-    #   https:// -> http:// -> https://www
-    # and meant to avoid affirming situations like:
-    #   http:// -> http://non-www
-    #   http://www -> http://non-www
-    # or:
-    #   http:// -> 200, http://www -> https://www
-    #
-    # It allows a site to be canonically HTTPS if the cert has
-    # a valid hostname but invalid chain issues.
-
-    https = !!(
-      (
-        (
-          combos[:https][:root][:up] and
-          !combos[:https][:root][:https_bad_name]
-        ) or
-        (
-          combos[:https][:www][:up] and
-          !combos[:https][:www][:https_bad_name]
-        )
-      ) and (
-        (
-          combos[:http][:root][:redirect] or
-          !combos[:http][:root][:up] or
-          !combos[:http][:root][:status].to_s.start_with?("2")
-        ) and (
-          combos[:http][:www][:redirect] or
-          !combos[:http][:www][:up] or
-          !combos[:http][:www][:status].to_s.start_with?("2")
-        )
-      ) and (
-        (
-          combos[:http][:root][:redirect_immediately_to_https] and
-          !combos[:http][:root][:redirect_immediately_external]
-        ) or (
-          combos[:http][:www][:redirect_immediately_to_https] and
-          !combos[:http][:www][:redirect_immediately_external]
-        )
-      )
-    )
-
-    details[:canonical_endpoint] = www ? :www : :root
-    details[:canonical_protocol] = https ? :https : :http
-    details[:canonical] = uri(https, www).to_s
-
-    # If any endpoint is up, the domain is up.
-    details[:up] = !!(
-      combos[:https][:www][:up] or
-      combos[:https][:root][:up] or
-      combos[:http][:www][:up] or
-      combos[:http][:root][:up]
-    )
-
-    # A domain's root is broken if neither protocol can connect.
-    details[:broken_root] = !!(
-      !combos[:https][:root][:up] and
-      !combos[:http][:root][:up]
-    )
-
-    # A domain's www is broken if neither protocol can connect.
-    details[:broken_www] = !!(
-      !combos[:https][:www][:up] and
-      !combos[:http][:www][:up]
-    )
-
-    # HTTPS is "supported" (different than "canonical" or "enforced") if:
-    #
-    # * Either of the HTTPS endpoints is listening, and doesn't have
-    #   an invalid hostname.
-    details[:support_https] = !!(
-      (
-        (combos[:https][:root][:status] != 0) and
-        !combos[:https][:root][:https_bad_name]
-      ) or (
-        (combos[:https][:www][:status] != 0) and
-        !combos[:https][:www][:https_bad_name]
-      )
-    )
-
-    # we can say that a canonical HTTPS site "defaults" to HTTPS,
-    # even if it doesn't *strictly* enforce it (e.g. having a www
-    # subdomain first to go HTTP root before HTTPS root).
-    details[:default_https] = https
-
-    # HTTPS is "downgraded" if both:
-    #
-    # * HTTPS is supported, and
-    # * The 'canonical' endpoint gets an immediate internal redirect to HTTP.
-
-    details[:downgrade_https] = !!(
-      details[:support_https] and
-      (
-        combos[:https][details[:canonical_endpoint]][:redirect] and
-        !combos[:https][details[:canonical_endpoint]][:redirect_immediately_external] and
-        !combos[:https][details[:canonical_endpoint]][:redirect_immediately_to_https]
-      )
-    )
-
-    # HTTPS is enforced if one of the HTTPS endpoints is "live",
-    # and if both *HTTP* endpoints are either:
-    #
-    #  * down, or
-    #  * redirect immediately to HTTPS.
-    #
-    # This is different than whether a domain is "canonically" HTTPS.
-    #
-    # * an HTTP redirect can go to HTTPS on another domain, as long
-    #   as it's immediate.
-    # * a domain with an invalid cert can still be enforcing HTTPS.
-    details[:enforce_https] = !!(
-      (
-        !combos[:http][:www][:up] or
-        (combos[:http][:www][:redirect_immediately_to_https])
-      ) and
-      (
-        !combos[:http][:root][:up] or
-        (combos[:http][:root][:redirect_immediately_to_https])
-      ) and
-      (
-        combos[:https][:www][:up] or
-        combos[:https][:root][:up]
-      )
-    )
-
-    # The domain is a redirect if at least one endpoint is up,
-    # and each one is *either* an external redirect or down entirely.
-    details[:redirect] = !!(
-      details[:up] and
-      (
-        combos[:http][:www][:redirect_external] or
-        !combos[:http][:www][:up] or
-        combos[:http][:www][:status] >= 400
-      ) and
-      (
-        combos[:http][:root][:redirect_external] or
-        !combos[:http][:root][:up] or
-        combos[:http][:root][:status] >= 400
-      ) and
-      (
-        combos[:https][:www][:redirect_external] or
-        !combos[:https][:www][:up] or
-        combos[:https][:www][:https_bad_name] or
-        combos[:https][:www][:status] >= 400
-      ) and
-      (
-        combos[:https][:root][:redirect_external] or
-        !combos[:https][:root][:up] or
-        combos[:https][:root][:https_bad_name] or
-        combos[:https][:root][:status] >= 400
-      )
-    )
-
-    # OK, we've said a domain is a "redirect" domain.
-    # What does the domain redirect to?
-    if details[:redirect]
-      canon = combos[details[:canonical_protocol]][details[:canonical_endpoint]]
-      details[:redirect_to] = canon[:redirect_to]
-    else
-      details[:redirect_to] = nil
-    end
-
-    # HSTS on the canonical domain? (valid HTTPS checked in endpoint)
-    details[:hsts] = !!combos[:https][details[:canonical_endpoint]][:hsts]
-    details[:hsts_header] = combos[:https][details[:canonical_endpoint]][:hsts_header]
-
-    # HSTS on the entire domain?
-    details[:hsts_entire_domain] = !!(
-      combos[:https][:root][:hsts] and
-      combos[:https][:root][:hsts_details][:include_subdomains]
-    )
-
-    # HSTS preload-ready for the entire domain?
-    #
-    # Re-checks :hsts_entire_domain in case the :preload_ready
-    # flag ever changes its definition to not require include_subdomains.
-
-    details[:hsts_entire_domain_preload] = !!(
-      details[:hsts_entire_domain] and
-      combos[:https][:root][:hsts_details][:preload_ready]
-    )
-
-    details
-  end
-
-  def endpoints
-    https_www = http_endpoint(true, true)
-    http_www = http_endpoint(false, true)
-    https_root = http_endpoint(true, false)
-    http_root = http_endpoint(false, false)
-
-    {
-      https: {
-        www: https_www,
-        root: https_root
-      },
-      http: {
-        www: http_www,
-        root: http_root
+    def typhoeus_defaults
+      defaults = {
+        followlocation: false,
+        timeout: SiteInspector.timeout,
+        accept_encoding: 'gzip',
+        method: :head,
+        headers: {
+          'User-Agent' => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
+        }
       }
-    }
-  end
-
-  # State of affairs at a particular endpoint.
-  def http_endpoint(ssl, www)
-    details = {}
-
-    # Don't follow redirects for first ping.
-    response = request(ssl, www, false)
-
-
-    # For HTTPS: examine the full range of possibilities.
-    if ssl
-      if response.return_code == :ok
-        details[:https_valid] = true
-        details[:https_bad_chain] = false
-        details[:https_bad_name] = false
-
-      # Bad certificate chain.
-      elsif response.return_code == :ssl_cacert
-        details[:https_valid] = false
-        details[:https_bad_chain] = true
-        response = request(ssl, www, false, false, true)
-        # Bad everything.
-        if response.return_code == :peer_failed_verification
-          details[:https_bad_name] = true
-          response = request(ssl, www, false, false, false)
-        end
-
-      # Bad hostname.
-      elsif response.return_code == :peer_failed_verification
-        details[:https_valid] = false
-        details[:https_bad_name] = true
-        response = request(ssl, www, false, true, false)
-        # Bad everything.
-        if response.return_code == :ssl_cacert
-          details[:https_bad_chain] = true
-          response = request(ssl, www, false, false, false)
-        end
-
-      # not sure what else would happen
-      elsif response.response_code != 0
-        details[:https_valid] = false
-        details[:https_unknown_issue] = response.return_code
-      end
+      defaults.merge! @typhoeus_options if @typhoeus_options
+      defaults
     end
 
-    # If we ended up with a failure, return it.
-    details[:status] = response.response_code
-    details[:up] = (response.response_code != 0)
-    return details if !details[:up]
-
-    headers = Hash[response.headers.map{ |k,v| [k.downcase,v] }]
-    details[:headers] = headers
-
-
-    # HSTS only takes effect when delivered over valid HTTPS.
-    hsts = SiteInspector.hsts_parse(headers["strict-transport-security"])
-
-    details[:hsts] = !!(
-      ssl and
-      details[:https_valid] and
-      hsts[:enabled]
-    )
-
-    details[:hsts_header] = headers["strict-transport-security"]
-    details[:hsts_details] = hsts
-
-
-    # If it's a redirect, go find the ultimate response starting from this combo.
-    redirect_code = response.response_code.to_s.start_with?("3")
-    location_header = headers["location"]
-    if redirect_code and location_header
-      location_header = location_header.downcase
-      details[:redirect] = true
-
-      ultimate_response = request(ssl, www, true, !details[:https_bad_chain], !details[:https_bad_name])
-      uri_original = URI(ultimate_response.request.url)
-
-      # treat relative Location headers as having the original hostname
-      if location_header.start_with?("http:") or location_header.start_with?("https:")
-        uri_immediate = URI(URI.escape(location_header))
-      else
-        uri_immediate = URI.join(uri_original, URI.escape(location_header))
-      end
-
-      uri_eventual = URI(ultimate_response.effective_url.downcase)
-
-      # compare base domain names
-      base_original = PublicSuffix.parse(uri_original.hostname).domain
-
-      # if the redirects aren't to valid hostnames (e.g. IP addresses)
-      # then fine just compare them directly, they're not going to be
-      # identical anyway.
-      base_immediate = begin
-        PublicSuffix.parse(uri_immediate.hostname).domain
-      rescue PublicSuffix::DomainInvalid
-        uri_immediate.to_s
-      end
-
-      base_eventual = begin
-        PublicSuffix.parse(uri_eventual.hostname).domain
-      rescue PublicSuffix::DomainInvalid
-        uri_eventual.to_s
-      end
-
-      details[:redirect_immediately_to] = uri_immediate.to_s
-      details[:redirect_immediately_to_www] = !!uri_immediate.to_s.match(/^https?:\/\/www\./)
-      details[:redirect_immediately_to_https] = uri_immediate.to_s.start_with?("https://")
-      details[:redirect_immediately_external] = (base_original != base_immediate)
-
-      details[:redirect_to] = uri_eventual.to_s
-      details[:redirect_external] = (base_original != base_eventual)
-
-    # otherwise, mark all the redirect fields as false/null
-    else
-      details[:redirect] = false
-      details[:redirect_immediately_to] = nil
-      details[:redirect_immediately_to_www] = false
-      details[:redirect_immediately_to_https] = false
-      details[:redirect_immediately_external] = false
-
-      details[:redirect_to] = nil
-      details[:redirect_external] = false
+    # Returns a thread-safe, memoized hydra instance
+    def hydra
+      Typhoeus::Hydra.hydra
     end
-
-    details
   end
+end
 
-  def to_hash(http_only=false)
-    if http_only
-      {
-        :domain => domain.to_s,
-        :uri => uri.to_s,
-        :live => !!response,
-        :ssl => https?,
-        :enforce_https => enforce_https?,
-        :non_www => non_www?,
-        :redirect => redirect,
-        :headers => headers
-      }
-    else
-      {
-        :domain => domain.to_s,
-        :uri => uri.to_s,
-        :government => government?,
-        :live => !!response,
-        :ssl => https?,
-        :enforce_https => enforce_https?,
-        :non_www => non_www?,
-        :redirect => redirect,
-        :ip => ip,
-        :hostname => hostname.to_s,
-        :ipv6 => ipv6?,
-        :dnssec => dnssec?,
-        :cdn => cdn,
-        :google_apps => google_apps?,
-        :cloud_provider => cloud_provider,
-        :server => server,
-        :cms => cms,
-        :analytics => analytics,
-        :javascript => javascript,
-        :advertising => advertising,
-        :slash_data => slash_data?,
-        :slash_developer => slash_developer?,
-        :data_dot_json => data_dot_json?,
-        :click_jacking_protection => click_jacking_protection?,
-        :content_security_policy => content_security_policy?,
-        :xss_protection => xss_protection?,
-        :secure_cookies => secure_cookies?,
-        :strict_transport_security => strict_transport_security?,
-        :headers => headers
-      }
-    end
-  end
+if ENV['DEBUG']
+  Ethon.logger = Logger.new($stdout)
+  Ethon.logger.level = Logger::DEBUG
+  Typhoeus::Config.verbose = true
 end
+
+Typhoeus::Config.memoize = true
+Typhoeus::Config.cache = SiteInspector.cache