site-inspector 3.2.0 → 4.0.0

data/lib/site-inspector/endpoint/well_known.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+class SiteInspector
+  class Endpoint
+    class WellKnown < Check
+      BASE_PATH = './.well-known/'
+
+      class << self
+        def well_knowns
+          data = File.expand_path '../../data/well-known.yml', File.dirname(__FILE__)
+          wks = YAML.load_file(data)
+          wks.reject { |wk| wk.include?('deprecated') }
+        end
+
+        def keys
+          well_knowns.map { |wk| key_for(wk) }
+        end
+
+        def key_paths
+          @key_paths ||= begin
+            key_paths = well_knowns.map do |wk|
+              [key_for(wk), path_for(wk)]
+            end
+
+            key_paths.to_h
+          end
+        end
+
+        def path_for(wk)
+          return unless well_knowns.include?(wk)
+
+          "#{BASE_PATH}#{wk}"
+        end
+
+        def key_for(well_known)
+          well_known.gsub(/(\.|-)/, '_').to_sym
+        end
+      end
+
+      def exists?(path)
+        endpoint.content.path_exists?(path)
+      end
+
+      def uri_for(key)
+        path = self.class.key_paths[key]
+        endpoint.join(path) if path
+      end
+
+      def to_h
+        return {} unless endpoint.up?
+        return {} if endpoint.redirect?
+        return @hash if defined?(@hash)
+
+        prefetch
+        @hash = {}
+
+        self.class.keys.each do |key|
+          @hash[key] = public_send(key)
+        end
+
+        @hash
+      end
+
+      def method_missing(method_sym, *arguments, &)
+        if respond_to_missing?(method_sym)
+          exists?(self.class.key_paths[method_sym])
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method_sym, include_private = false)
+        if self.class.keys.include?(method_sym)
+          true
+        else
+          super
+        end
+      end
+
+      private
+
+      def prefetch
+        options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
+        self.class.key_paths.each do |_key, path|
+          uri = endpoint.join(path)
+          request = Typhoeus::Request.new(uri, options)
+          SiteInspector.hydra.queue(request)
+        end
+        SiteInspector.hydra.run
+      end
+    end
+  end
+end

data/lib/site-inspector/endpoint/whois.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Whois < Check
+      def domain
+        return @domain if defined? @domain
+
+        @domain = begin
+          whois.lookup host
+        rescue Timeout::Error
+          nil
+        end
+      end
+
+      def ip
+        return @ip if defined? @ip
+
+        @ip = begin
+          whois.lookup ip_address if ip_address
+        rescue Timeout::Error
+          nil
+        end
+      end
+
+      def to_h
+        {
+          domain: record_to_h(domain),
+          ip: record_to_h(ip)
+        }
+      end
+
+      private
+
+      def record_to_h(record)
+        return unless record
+
+        record.content.scan(/^\s*(.*?):\s*(.*?)\r?\n/).to_h
+      end
+
+      def ip_address
+        return @ip_address if defined? @ip_address
+
+        @ip_address = begin
+          Resolv.getaddress host
+        rescue Resolv::ResolvError
+          nil
+        end
+      end
+
+      def whois
+        @whois ||= ::Whois::Client.new
+      end
+    end
+  end
+end

data/lib/site-inspector/endpoint.rb

@@ -1,5 +1,17 @@
 # frozen_string_literal: true
 
+require 'site-inspector/endpoint/check'
+require 'site-inspector/endpoint/accessibility'
+require 'site-inspector/endpoint/content'
+require 'site-inspector/endpoint/dns'
+require 'site-inspector/endpoint/headers'
+require 'site-inspector/endpoint/hsts'
+require 'site-inspector/endpoint/https'
+require 'site-inspector/endpoint/cookies'
+require 'site-inspector/endpoint/well_known'
+require 'site-inspector/endpoint/whois'
+require 'site-inspector/endpoint/wappalyzer'
+
 class SiteInspector
   # Every domain has four possible "endpoints" to evaluate
   #
@@ -14,23 +26,32 @@ class SiteInspector
   class Endpoint
     attr_accessor :host, :uri, :domain
 
+    def self.build(domain, https: true, www: false)
+      uri = Addressable::URI.new
+      uri.host = www ? "www.#{domain.host}" : domain.host.to_s
+      uri.scheme = https ? 'https' : 'http'
+
+      new(uri.to_s, domain:)
+    end
+
     # Initatiate a new Endpoint object
     #
     # endpoint - (string) the endpoint to query (e.g., `https://example.com`)
     # options  - A hash of options
     #   domain - the parent domain object, if passed, facilitates caching of redirects
     def initialize(host, options = {})
-      @uri = Addressable::URI.parse(host.downcase)
+      host = host.downcase if host.is_a?(String)
+      @uri = Addressable::URI.parse(host)
       # The root URL always has an implict path of "/", even if not requested
       # Make it explicit to facilitate caching and prevent a potential redirect
       @uri.path = '/'
-      @host = uri.host.sub(/^www\./, '')
+      @host = DomainParser.parse(host)
       @checks = {}
       @domain = options[:domain]
     end
 
     def www?
-      !!(uri.host =~ /^www\./)
+      @host.trd == 'www'
     end
 
     def root?
@@ -50,13 +71,21 @@ class SiteInspector
     end
 
     def request(options = {})
-      target = options[:path] ? URI.join(uri, options.delete(:path)) : uri
-      request = Typhoeus::Request.new(target, SiteInspector.typhoeus_defaults.merge(options))
+      request = build_request(options)
       hydra.queue(request)
       hydra.run
       request.response
     end
 
+    def build_request(options)
+      target = options[:path] ? join(options.delete(:path)) : uri
+      Typhoeus::Request.new(target, SiteInspector.typhoeus_defaults.merge(options))
+    end
+
+    def join(*args)
+      URI.join(uri, *args)
+    end
+
     # Makes a GET request of the given host
     #
     # Retutns the Typhoeus::Response object
@@ -74,7 +103,9 @@ class SiteInspector
 
     # Does the endpoint return a 2xx or 3xx response code?
     def up?
-      response && response_code.start_with?('2') || response_code.start_with?('3')
+      return false unless response
+
+      response_code.start_with?('2', '3')
     end
 
     # Does the server respond at all?
@@ -86,18 +117,16 @@ class SiteInspector
     def redirect
       return unless response && response_code.start_with?('3')
 
-      @redirect ||= begin
-        redirect = Addressable::URI.parse(headers['location'])
+      location = headers['location']
+      return if location.nil? || location.empty?
 
-        # This is a relative redirect, but we still need the absolute URI
-        if redirect.relative?
-          redirect.path = "/#{redirect.path}" unless redirect.path[0] == '/'
-          redirect.host = host
-          redirect.scheme = scheme
-        end
+      @redirect ||= begin
+        # Resolve per RFC 3986, so protocol-relative (`//host`), query-only
+        # (`?q`), and dot-segment (`../x`) locations become absolute URIs
+        redirect = uri.join(location)
 
         # This was a redirect to a subpath or back to itself, which we don't care about
-        return if redirect.host == host && redirect.scheme == scheme
+        return if redirect.host == host.to_s && redirect.scheme == scheme
 
         # Init a new endpoint representing the redirect
         find_or_create_by_uri(redirect.to_s)
@@ -132,7 +161,7 @@ class SiteInspector
     end
 
     def external_redirect?
-      host != resolves_to.host
+      host.to_s != resolves_to.host.to_s
     end
 
     def to_s
@@ -153,16 +182,19 @@ class SiteInspector
     #
     # Returns the hash representing the endpoint and its checks
     def to_h(options = {})
-      hash = {
+      return @hash if defined?(@hash) && options == {}
+
+      @hash = {
         uri: uri.to_s,
-        host: host,
+        host: host.to_s,
         www: www?,
         https: https?,
-        scheme: scheme,
+        scheme:,
         up: up?,
         responds: responds?,
         timed_out: timed_out?,
         redirect: redirect?,
+        resolves_to: resolves_to.to_s,
         external_redirect: external_redirect?
       }
 
@@ -170,11 +202,11 @@ class SiteInspector
       checks = SiteInspector::Endpoint.checks.select { |c| options.key?(c.name) }
       checks = SiteInspector::Endpoint.checks if checks.empty?
 
-      Parallel.each(checks, in_threads: 4) do |check|
-        hash[check.name] = send(check.name).to_h
+      Parallel.each(checks, in_processes: 0) do |check|
+        @hash[check.name] = send(check.name).to_h
       end
 
-      hash
+      @hash
     end
 
     def self.checks
@@ -183,7 +215,7 @@ class SiteInspector
       @checks = ObjectSpace.each_object(Class).select { |klass| klass < Check }.select(&:enabled?).sort_by(&:name)
     end
 
-    def method_missing(method_sym, *arguments, &block)
+    def method_missing(method_sym, *arguments, &)
       check = SiteInspector::Endpoint.checks.find { |c| c.name == method_sym }
       if check
         @checks[method_sym] ||= check.new(self)
@@ -200,6 +232,11 @@ class SiteInspector
       end
     end
 
+    def eql?(other)
+      other.class == self.class && other.uri == uri
+    end
+    alias == eql?
+
     private
 
     def hydra

data/lib/site-inspector/formatter.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/string/inflections'
+require 'nokogiri'
+
+class SiteInspector
+  module Formatter
+    ACRONYMS = %w[
+      acme cdn cms crm dns dnssec dnt hsts http https id ip ipv6 json paas pki sld ssl tld tls trd txt ui uri url whois www xml xss
+    ].freeze
+
+    def format_table(hash)
+      build_fragment do |f|
+        f.table(class: 'table table-striped') do
+          hash.each do |key, value|
+            f << format_key_value(key, value)
+          end
+        end
+      end
+    end
+
+    def format_key(string)
+      capitalize_acronyms(string.to_s.gsub(/^x-/, '').tr('-', ' ').humanize)
+    end
+
+    def format_value(value)
+      case value
+      when Hash
+        format_hash(value)
+      when Array
+        format_array(value)
+      else
+        format_string(value)
+      end
+    end
+
+    def format_key_value(key, value, check = nil)
+      c = class_for_value(value)
+
+      build_fragment do |f|
+        f.tr do
+          f.th { f.text format_key(key) }
+          f.td(c) do
+            uri = check&.uri_for(key)
+
+            if value == true && uri
+              f.a({ href: uri }.merge(c)) { f << format_value(value) }
+            else
+              f << format_value(value)
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def class_for_value(value)
+      if value.instance_of?(TrueClass)
+        { class: 'text-success' }
+      elsif value.instance_of?(FalseClass)
+        { class: 'text-danger' }
+      else
+        {}
+      end
+    end
+
+    def build_fragment(&block)
+      fragment = Nokogiri::HTML::DocumentFragment.parse ''
+      Nokogiri::HTML::Builder.with(fragment, &block)
+      fragment.to_s
+    end
+
+    def capitalize_acronyms(string)
+      string.gsub(/\b(#{ACRONYMS.join("|")})\b/i) do
+        Regexp.last_match(1).to_s.upcase
+      end
+    end
+
+    def format_string(value)
+      value = CGI.escapeHTML(value.to_s)
+
+      if %r{^https?:/}.match?(value)
+        value = build_fragment do |f|
+          f.a(href: value) { f.text value }
+        end
+      end
+
+      value
+    end
+
+    def format_hash(hash)
+      build_fragment do |f|
+        f.ul do
+          hash.each do |key, value|
+            f.li do
+              f.span(class: 'font-weight-bold') { f.text format_key(key) }
+              f.text ': '
+              f << format_value(value)
+            end
+          end
+        end
+      end
+    end
+
+    def format_array(array)
+      return format_value(array[0]) if array.length == 1
+
+      build_fragment do |f|
+        f.ol do
+          array.each do |el|
+            f.li { f << format_value(el) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/site-inspector/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class SiteInspector
-  VERSION = '3.2.0'
+  VERSION = '4.0.0'
 end

data/lib/site-inspector.rb

@@ -4,43 +4,37 @@ require 'open-uri'
 require 'addressable/uri'
 require 'public_suffix'
 require 'typhoeus'
+require 'typhoeus/cache/rails'
+require 'active_support'
+require 'active_support/cache'
 require 'parallel'
 require 'cliver'
 require 'whois'
 require 'cgi'
 require 'resolv'
-require 'dotenv/load'
-
-require_relative 'site-inspector/cache'
-require_relative 'site-inspector/disk_cache'
-require_relative 'site-inspector/rails_cache'
-require_relative 'site-inspector/domain'
-require_relative 'site-inspector/checks/check'
-require_relative 'site-inspector/checks/accessibility'
-require_relative 'site-inspector/checks/content'
-require_relative 'site-inspector/checks/dns'
-require_relative 'site-inspector/checks/headers'
-require_relative 'site-inspector/checks/hsts'
-require_relative 'site-inspector/checks/https'
-require_relative 'site-inspector/checks/sniffer'
-require_relative 'site-inspector/checks/cookies'
-require_relative 'site-inspector/checks/whois'
-require_relative 'site-inspector/checks/wappalyzer'
-require_relative 'site-inspector/endpoint'
-require_relative 'site-inspector/version'
+require 'naughty_or_nice'
 require_relative 'cliver/dependency_ext'
 
 class SiteInspector
+  autoload :Formatter, 'site-inspector/formatter'
+  autoload :Domain, 'site-inspector/domain'
+  autoload :DomainParser, 'site-inspector/domain_parser'
+  autoload :Endpoint, 'site-inspector/endpoint'
+  autoload :VERSION, 'site-inspector/version'
+
   class << self
     attr_writer :timeout, :cache, :typhoeus_options
 
+    # The Typhoeus response cache. Set CACHE to a directory to persist
+    # responses to disk; inside Rails, Rails.cache is used; otherwise
+    # responses are kept in memory for the life of the process.
     def cache
       @cache ||= if ENV['CACHE']
-                   SiteInspector::DiskCache.new
-                 elsif Object.const_defined?('Rails')
-                   SiteInspector::RailsCache.new
+                   Typhoeus::Cache::Rails.new(ActiveSupport::Cache::FileStore.new(ENV.fetch('CACHE')))
+                 elsif Object.const_defined?(:Rails)
+                   Typhoeus::Cache::Rails.new
                  else
-                   SiteInspector::Cache.new
+                   Typhoeus::Cache::Rails.new(ActiveSupport::Cache::MemoryStore.new)
                  end
     end
 
@@ -57,10 +51,7 @@ class SiteInspector
         followlocation: false,
         timeout: SiteInspector.timeout,
         accept_encoding: 'gzip',
-        method: :head,
-        headers: {
-          'User-Agent' => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
-        }
+        method: :head
       }
       defaults.merge! @typhoeus_options if @typhoeus_options
       defaults
@@ -68,16 +59,29 @@ class SiteInspector
 
     # Returns a thread-safe, memoized hydra instance
     def hydra
-      Typhoeus::Hydra.hydra
+      @hydra ||= Typhoeus::Hydra.hydra
+    end
+
+    def logger
+      @logger ||= Logger.new($stdout)
     end
   end
 end
 
 if ENV['DEBUG']
-  Ethon.logger = Logger.new($stdout)
+  Ethon.logger = SiteInspector.logger
   Ethon.logger.level = Logger::DEBUG
   Typhoeus::Config.verbose = true
 end
 
-Typhoeus::Config.memoize = true
 Typhoeus::Config.cache = SiteInspector.cache
+Typhoeus::Config.memoize = true
+Typhoeus::Config.user_agent = "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
+
+if ENV['VERBOSE']
+  Typhoeus.before do |req|
+    hit = Typhoeus::Config.cache.get(req)
+    SiteInspector.logger.info "#{req.options[:method]} #{req.base_url} (CACHE #{hit ? 'HIT' : 'MISS'})"
+    req
+  end
+end