site-inspector 3.1.1 → 4.0.0

data/lib/site-inspector/endpoint.rb

@@ -1,3 +1,17 @@
+# frozen_string_literal: true
+
+require 'site-inspector/endpoint/check'
+require 'site-inspector/endpoint/accessibility'
+require 'site-inspector/endpoint/content'
+require 'site-inspector/endpoint/dns'
+require 'site-inspector/endpoint/headers'
+require 'site-inspector/endpoint/hsts'
+require 'site-inspector/endpoint/https'
+require 'site-inspector/endpoint/cookies'
+require 'site-inspector/endpoint/well_known'
+require 'site-inspector/endpoint/whois'
+require 'site-inspector/endpoint/wappalyzer'
+
 class SiteInspector
   # Every domain has four possible "endpoints" to evaluate
   #
@@ -12,23 +26,32 @@ class SiteInspector
   class Endpoint
     attr_accessor :host, :uri, :domain
 
+    def self.build(domain, https: true, www: false)
+      uri = Addressable::URI.new
+      uri.host = www ? "www.#{domain.host}" : domain.host.to_s
+      uri.scheme = https ? 'https' : 'http'
+
+      new(uri.to_s, domain:)
+    end
+
     # Initatiate a new Endpoint object
     #
     # endpoint - (string) the endpoint to query (e.g., `https://example.com`)
     # options  - A hash of options
     #   domain - the parent domain object, if passed, facilitates caching of redirects
     def initialize(host, options = {})
-      @uri = Addressable::URI.parse(host.downcase)
+      host = host.downcase if host.is_a?(String)
+      @uri = Addressable::URI.parse(host)
       # The root URL always has an implict path of "/", even if not requested
       # Make it explicit to facilitate caching and prevent a potential redirect
       @uri.path = '/'
-      @host = uri.host.sub(/^www\./, '')
+      @host = DomainParser.parse(host)
       @checks = {}
       @domain = options[:domain]
     end
 
     def www?
-      !!(uri.host =~ /^www\./)
+      @host.trd == 'www'
     end
 
     def root?
@@ -48,13 +71,21 @@ class SiteInspector
     end
 
     def request(options = {})
-      target = options[:path] ? URI.join(uri, options.delete(:path)) : uri
-      request = Typhoeus::Request.new(target, SiteInspector.typhoeus_defaults.merge(options))
+      request = build_request(options)
       hydra.queue(request)
       hydra.run
       request.response
     end
 
+    def build_request(options)
+      target = options[:path] ? join(options.delete(:path)) : uri
+      Typhoeus::Request.new(target, SiteInspector.typhoeus_defaults.merge(options))
+    end
+
+    def join(*args)
+      URI.join(uri, *args)
+    end
+
     # Makes a GET request of the given host
     #
     # Retutns the Typhoeus::Response object
@@ -67,12 +98,14 @@ class SiteInspector
     end
 
     def timed_out?
-      response && response.timed_out?
+      response&.timed_out?
     end
 
     # Does the endpoint return a 2xx or 3xx response code?
     def up?
-      response && response_code.start_with?('2') || response_code.start_with?('3')
+      return false unless response
+
+      response_code.start_with?('2', '3')
     end
 
     # Does the server respond at all?
@@ -84,18 +117,16 @@ class SiteInspector
     def redirect
       return unless response && response_code.start_with?('3')
 
-      @redirect ||= begin
-        redirect = Addressable::URI.parse(headers['location'])
+      location = headers['location']
+      return if location.nil? || location.empty?
 
-        # This is a relative redirect, but we still need the absolute URI
-        if redirect.relative?
-          redirect.path = "/#{redirect.path}" unless redirect.path[0] == '/'
-          redirect.host = host
-          redirect.scheme = scheme
-        end
+      @redirect ||= begin
+        # Resolve per RFC 3986, so protocol-relative (`//host`), query-only
+        # (`?q`), and dot-segment (`../x`) locations become absolute URIs
+        redirect = uri.join(location)
 
         # This was a redirect to a subpath or back to itself, which we don't care about
-        return if redirect.host == host && redirect.scheme == scheme
+        return if redirect.host == host.to_s && redirect.scheme == scheme
 
         # Init a new endpoint representing the redirect
         find_or_create_by_uri(redirect.to_s)
@@ -119,22 +150,18 @@ class SiteInspector
         response = request(followlocation: true)
 
         # Workaround for Webmock not playing nicely with Typhoeus redirects
-        if response.mock?
-          if response.headers['Location']
-            url = response.headers['Location']
-          else
-            url = response.request.url
-          end
-        else
-          url = response.effective_url
-        end
+        url = if response.mock?
+                response.headers['Location'] || response.request.url
+              else
+                response.effective_url
+              end
 
         find_or_create_by_uri(url)
       end
     end
 
     def external_redirect?
-      host != resolves_to.host
+      host.to_s != resolves_to.host.to_s
     end
 
     def to_s
@@ -155,36 +182,40 @@ class SiteInspector
     #
     # Returns the hash representing the endpoint and its checks
     def to_h(options = {})
-      hash = {
-        uri:               uri.to_s,
-        host:              host,
-        www:               www?,
-        https:             https?,
-        scheme:            scheme,
-        up:                up?,
-        responds:          responds?,
-        timed_out:         timed_out?,
-        redirect:          redirect?,
+      return @hash if defined?(@hash) && options == {}
+
+      @hash = {
+        uri: uri.to_s,
+        host: host.to_s,
+        www: www?,
+        https: https?,
+        scheme:,
+        up: up?,
+        responds: responds?,
+        timed_out: timed_out?,
+        redirect: redirect?,
+        resolves_to: resolves_to.to_s,
         external_redirect: external_redirect?
       }
 
       # Either they've specifically asked for a check, or we throw everything at them
-      checks = SiteInspector::Endpoint.checks.select { |c| options.keys.include?(c.name) }
+      checks = SiteInspector::Endpoint.checks.select { |c| options.key?(c.name) }
       checks = SiteInspector::Endpoint.checks if checks.empty?
 
-      Parallel.each(checks, in_threads: 4) do |check|
-        hash[check.name] = send(check.name).to_h
+      Parallel.each(checks, in_processes: 0) do |check|
+        @hash[check.name] = send(check.name).to_h
       end
 
-      hash
+      @hash
     end
 
     def self.checks
       return @checks if defined? @checks
-      @checks = ObjectSpace.each_object(Class).select { |klass| klass < Check }.select(&:enabled?)
+
+      @checks = ObjectSpace.each_object(Class).select { |klass| klass < Check }.select(&:enabled?).sort_by(&:name)
     end
 
-    def method_missing(method_sym, *arguments, &block)
+    def method_missing(method_sym, *arguments, &)
       check = SiteInspector::Endpoint.checks.find { |c| c.name == method_sym }
       if check
         @checks[method_sym] ||= check.new(self)
@@ -193,14 +224,19 @@ class SiteInspector
       end
     end
 
-    def respond_to?(method_sym, include_private = false)
-      if checks.keys.include?(method_sym)
+    def respond_to_missing?(method_sym, include_private = false)
+      if checks.key?(method_sym)
         true
       else
         super
       end
     end
 
+    def eql?(other)
+      other.class == self.class && other.uri == uri
+    end
+    alias == eql?
+
     private
 
     def hydra

data/lib/site-inspector/formatter.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/string/inflections'
+require 'nokogiri'
+
+class SiteInspector
+  module Formatter
+    ACRONYMS = %w[
+      acme cdn cms crm dns dnssec dnt hsts http https id ip ipv6 json paas pki sld ssl tld tls trd txt ui uri url whois www xml xss
+    ].freeze
+
+    def format_table(hash)
+      build_fragment do |f|
+        f.table(class: 'table table-striped') do
+          hash.each do |key, value|
+            f << format_key_value(key, value)
+          end
+        end
+      end
+    end
+
+    def format_key(string)
+      capitalize_acronyms(string.to_s.gsub(/^x-/, '').tr('-', ' ').humanize)
+    end
+
+    def format_value(value)
+      case value
+      when Hash
+        format_hash(value)
+      when Array
+        format_array(value)
+      else
+        format_string(value)
+      end
+    end
+
+    def format_key_value(key, value, check = nil)
+      c = class_for_value(value)
+
+      build_fragment do |f|
+        f.tr do
+          f.th { f.text format_key(key) }
+          f.td(c) do
+            uri = check&.uri_for(key)
+
+            if value == true && uri
+              f.a({ href: uri }.merge(c)) { f << format_value(value) }
+            else
+              f << format_value(value)
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def class_for_value(value)
+      if value.instance_of?(TrueClass)
+        { class: 'text-success' }
+      elsif value.instance_of?(FalseClass)
+        { class: 'text-danger' }
+      else
+        {}
+      end
+    end
+
+    def build_fragment(&block)
+      fragment = Nokogiri::HTML::DocumentFragment.parse ''
+      Nokogiri::HTML::Builder.with(fragment, &block)
+      fragment.to_s
+    end
+
+    def capitalize_acronyms(string)
+      string.gsub(/\b(#{ACRONYMS.join("|")})\b/i) do
+        Regexp.last_match(1).to_s.upcase
+      end
+    end
+
+    def format_string(value)
+      value = CGI.escapeHTML(value.to_s)
+
+      if %r{^https?:/}.match?(value)
+        value = build_fragment do |f|
+          f.a(href: value) { f.text value }
+        end
+      end
+
+      value
+    end
+
+    def format_hash(hash)
+      build_fragment do |f|
+        f.ul do
+          hash.each do |key, value|
+            f.li do
+              f.span(class: 'font-weight-bold') { f.text format_key(key) }
+              f.text ': '
+              f << format_value(value)
+            end
+          end
+        end
+      end
+    end
+
+    def format_array(array)
+      return format_value(array[0]) if array.length == 1
+
+      build_fragment do |f|
+        f.ol do
+          array.each do |el|
+            f.li { f << format_value(el) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/site-inspector/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
-  VERSION = '3.1.1'
+  VERSION = '4.0.0'
 end

data/lib/site-inspector.rb

@@ -1,40 +1,41 @@
+# frozen_string_literal: true
+
 require 'open-uri'
 require 'addressable/uri'
 require 'public_suffix'
 require 'typhoeus'
+require 'typhoeus/cache/rails'
+require 'active_support'
+require 'active_support/cache'
 require 'parallel'
 require 'cliver'
+require 'whois'
 require 'cgi'
-
-require_relative 'site-inspector/cache'
-require_relative 'site-inspector/disk_cache'
-require_relative 'site-inspector/rails_cache'
-require_relative 'site-inspector/domain'
-require_relative 'site-inspector/checks/check'
-require_relative 'site-inspector/checks/accessibility'
-require_relative 'site-inspector/checks/content'
-require_relative 'site-inspector/checks/dns'
-require_relative 'site-inspector/checks/headers'
-require_relative 'site-inspector/checks/hsts'
-require_relative 'site-inspector/checks/https'
-require_relative 'site-inspector/checks/sniffer'
-require_relative 'site-inspector/checks/cookies'
-require_relative 'site-inspector/endpoint'
-require_relative 'site-inspector/version'
+require 'resolv'
+require 'naughty_or_nice'
 require_relative 'cliver/dependency_ext'
 
 class SiteInspector
+  autoload :Formatter, 'site-inspector/formatter'
+  autoload :Domain, 'site-inspector/domain'
+  autoload :DomainParser, 'site-inspector/domain_parser'
+  autoload :Endpoint, 'site-inspector/endpoint'
+  autoload :VERSION, 'site-inspector/version'
+
   class << self
     attr_writer :timeout, :cache, :typhoeus_options
 
+    # The Typhoeus response cache. Set CACHE to a directory to persist
+    # responses to disk; inside Rails, Rails.cache is used; otherwise
+    # responses are kept in memory for the life of the process.
     def cache
       @cache ||= if ENV['CACHE']
-        SiteInspector::DiskCache.new
-      elsif Object.const_defined?('Rails')
-        SiteInspector::RailsCache.new
-      else
-        SiteInspector::Cache.new
-      end
+                   Typhoeus::Cache::Rails.new(ActiveSupport::Cache::FileStore.new(ENV.fetch('CACHE')))
+                 elsif Object.const_defined?(:Rails)
+                   Typhoeus::Cache::Rails.new
+                 else
+                   Typhoeus::Cache::Rails.new(ActiveSupport::Cache::MemoryStore.new)
+                 end
     end
 
     def timeout
@@ -47,13 +48,10 @@ class SiteInspector
 
     def typhoeus_defaults
       defaults = {
-        followlocation:  false,
-        timeout:         SiteInspector.timeout,
+        followlocation: false,
+        timeout: SiteInspector.timeout,
         accept_encoding: 'gzip',
-        method:          :head,
-        headers:         {
-          'User-Agent' => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
-        }
+        method: :head
       }
       defaults.merge! @typhoeus_options if @typhoeus_options
       defaults
@@ -61,16 +59,29 @@ class SiteInspector
 
     # Returns a thread-safe, memoized hydra instance
     def hydra
-      Typhoeus::Hydra.hydra
+      @hydra ||= Typhoeus::Hydra.hydra
+    end
+
+    def logger
+      @logger ||= Logger.new($stdout)
     end
   end
 end
 
 if ENV['DEBUG']
-  Ethon.logger = Logger.new(STDOUT)
+  Ethon.logger = SiteInspector.logger
   Ethon.logger.level = Logger::DEBUG
   Typhoeus::Config.verbose = true
 end
 
-Typhoeus::Config.memoize = true
 Typhoeus::Config.cache = SiteInspector.cache
+Typhoeus::Config.memoize = true
+Typhoeus::Config.user_agent = "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
+
+if ENV['VERBOSE']
+  Typhoeus.before do |req|
+    hit = Typhoeus::Config.cache.get(req)
+    SiteInspector.logger.info "#{req.options[:method]} #{req.base_url} (CACHE #{hit ? 'HIT' : 'MISS'})"
+    req
+  end
+end