site-inspector 1.0.2 → 2.0.0

data/lib/site-inspector/checks/https.rb

@@ -0,0 +1,40 @@
+class SiteInspector
+  class Endpoint
+    class Https < Check
+
+      def scheme?
+        scheme == "https"
+      end
+
+      def valid?
+        scheme? && response && response.return_code == :ok
+      end
+
+      def bad_chain?
+        scheme? && response && response.return_code == :ssl_cacert
+      end
+
+      def bad_name?
+        scheme? && response && response.return_code == :peer_failed_verification
+      end
+
+      def inspect
+        "#<SiteInspector::Endpoint::Https valid=#{valid?}>"
+      end
+
+      def to_h
+        {
+          valid: valid?,
+          return_code: response.return_code,
+        }
+      end
+
+      private
+
+      def scheme
+        @scheme ||= request.base_url.scheme
+      end
+
+    end
+  end
+end

data/lib/site-inspector/checks/sniffer.rb

@@ -0,0 +1,42 @@
+class SiteInspector
+  class Endpoint
+    class Sniffer < Check
+
+      def cms
+        sniff :cms
+      end
+
+      def analytics
+        sniff :analytics
+      end
+
+      def javascript
+        sniff :javascript
+      end
+
+      def advertising
+        sniff :advertising
+      end
+
+      def to_h
+        {
+          :cms         => cms,
+          :analytics   => analytics,
+          :javascript  => javascript,
+          :advertising => advertising
+        }
+      end
+
+      private
+
+      def sniff(type)
+        require 'sniffles'
+        results = Sniffles.sniff(endpoint.content.body, type).select { |name, meta| meta[:found] == true }
+        results.each { |name, result| result.delete :found} if results
+        results
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/site-inspector/disk_cache.rb

@@ -0,0 +1,38 @@
+class SiteInspector
+  class DiskCache
+    def initialize(dir = nil, replace = nil)
+      @dir     = dir || ENV['CACHE']
+      @replace = replace || ENV['CACHE_REPLACE']
+      @memory  = {}
+    end
+
+    def get(request)
+      return unless File.exist?(path(request))
+      return @memory[request] if @memory[request]
+
+      if @replace
+        FileUtils.rm(path(request))
+        nil
+      else
+        begin
+          contents = File.read(path(request))
+          Marshal.load(contents)
+        rescue ArgumentError
+          FileUtils.rm(path(request))
+          nil
+        end
+      end
+    end
+
+    def set(request, response)
+      File.write(path(request), Marshal.dump(response))
+      @memory[request] = response
+    end
+
+    private
+
+    def path(request)
+      File.join(@dir, request)
+    end
+  end
+end

data/lib/site-inspector/domain.rb

@@ -0,0 +1,248 @@
+class SiteInspector
+  class Domain
+
+    attr_reader :host
+
+    def initialize(host)
+      host = host.downcase
+      host = host.sub /^https?\:/, ""
+      host = host.sub /^\/+/, ""
+      host = host.sub /^www\./, ""
+      uri = Addressable::URI.parse "//#{host}"
+      @host = uri.host
+    end
+
+    def endpoints
+      @endpoints ||= [
+        Endpoint.new("https://#{host}"),
+        Endpoint.new("https://www.#{host}"),
+        Endpoint.new("http://#{host}"),
+        Endpoint.new("http://www.#{host}")
+      ]
+    end
+
+    def canonical_endpoint
+      @canonical_endpoint ||= endpoints.find do |e|
+        e.https? == canonically_https? && e.www? == canonically_www?
+      end
+    end
+
+    def government?
+      require 'gman'
+      Gman.valid? host
+    end
+
+    # Does *any* endpoint return a 200 response code?
+    def up?
+      endpoints.any? { |e| e.up? }
+    end
+
+    # Does any www endpoint return a 200 response code?
+    def www?
+      endpoints.any? { |e| e.www? && e.up? }
+    end
+
+    # Can you connect without www?
+    def root?
+      endpoints.any? { |e| e.root? && e.up? }
+    end
+
+    # HTTPS is "supported" (different than "canonical" or "enforced") if:
+    #
+    # * Either of the HTTPS endpoints is listening, and doesn't have
+    #   an invalid hostname.
+    def https?
+      endpoints.any? { |e| e.https? && e.up? && e.https.valid? }
+    end
+
+    # HTTPS is enforced if one of the HTTPS endpoints is "live",
+    # and if both *HTTP* endpoints are either:
+    #
+    #  * down, or
+    #  * redirect immediately to HTTPS.
+    #
+    # This is different than whether a domain is "canonically" HTTPS.
+    #
+    # * an HTTP redirect can go to HTTPS on another domain, as long
+    #   as it's immediate.
+    # * a domain with an invalid cert can still be enforcing HTTPS.
+    def enforces_https?
+      return false unless https?
+      endpoints.select { |e| e.http? }.all? { |e| e.down? || (e.redirect && e.redirect.https?) }
+    end
+
+    # we can say that a canonical HTTPS site "defaults" to HTTPS,
+    # even if it doesn't *strictly* enforce it (e.g. having a www
+    # subdomain first to go HTTP root before HTTPS root).
+    def defaults_https?
+      raise "Not implemented. Halp?"
+    end
+
+    # HTTPS is "downgraded" if both:
+    #
+    # * HTTPS is supported, and
+    # * The 'canonical' endpoint gets an immediate internal redirect to HTTP.
+    def downgrades_https?
+      return false unless https?
+      canonical_endpoint.redirect && canonical_endpoint.redirect.http?
+    end
+
+    # A domain is "canonically" at www if:
+    #  * at least one of its www endpoints responds
+    #  * both root endpoints are either down ~~or redirect *somewhere*~~, or
+    #  * at least one root endpoint redirect should immediately go to
+    #    an *internal* www endpoint
+    # This is meant to affirm situations like:
+    #   http:// -> https:// -> https://www
+    #   https:// -> http:// -> https://www
+    # and meant to avoid affirming situations like:
+    #   http:// -> http://non-www,
+    #   http://www -> http://non-www
+    # or like:
+    #   https:// -> 200, http:// -> http://www
+    def canonically_www?
+      # Does any endpoint respond?
+      return false unless up?
+
+      # Does at least one www endpoint respond?
+      return false unless www?
+
+      # Are both root endpoints down?
+      return true if endpoints.select { |e| e.root? }.all? { |e| e.down? }
+
+      # Does either root endpoint redirect to a www endpoint?
+      endpoints.select { |e| e.root? }.any? { |e| e.redirect && e.redirect.www? }
+    end
+
+    # A domain is "canonically" at https if:
+    #  * at least one of its https endpoints is live and
+    #    doesn't have an invalid hostname
+    #  * both http endpoints are either down or redirect *somewhere*
+    #  * at least one http endpoint redirects immediately to
+    #    an *internal* https endpoint
+    # This is meant to affirm situations like:
+    #   http:// -> http://www -> https://
+    #   https:// -> http:// -> https://www
+    # and meant to avoid affirming situations like:
+    #   http:// -> http://non-www
+    #   http://www -> http://non-www
+    # or:
+    #   http:// -> 200, http://www -> https://www
+    #
+    # It allows a site to be canonically HTTPS if the cert has
+    # a valid hostname but invalid chain issues.
+    def canonically_https?
+      # Does any endpoint respond?
+      return false unless up?
+
+      # At least one of its https endpoints is live and doesn't have an invalid hostname
+      return false unless https?
+
+      # Both http endpoints are down
+      return true if endpoints.select { |e| e.http? }.all? { |e| e.down? }
+
+      # at least one http endpoint redirects immediately to https
+      endpoints.select { |e| e.http? }.any? { |e| e.redirect && e.redirect.https? }
+    end
+
+    # A domain redirects if
+    # 1. At least one endpoint is an external redirect, and
+    # 2. All endpoints are either down or an external redirect
+    def redirect?
+      return false unless redirect
+      endpoints.all? { |e| e.down? || e.external_redirect? }
+    end
+
+    # The first endpoint to respond with a redirect
+    def redirect
+      endpoints.find { |e| e.external_redirect? }
+    end
+
+    # HSTS on the canonical domain?
+    def hsts?
+      canonical_endpoint.hsts && canonical_endpoint.hsts.enabled?
+    end
+
+    def hsts_subdomains?
+      endpoints.find { |e| e.root? && e.https? }.hsts.include_subdomains?
+    end
+
+    def hsts_preload_ready?
+      return false unless hsts_subdomains?
+      endpoints.find { |e| e.root? && e.https? }.hsts.preload_ready?
+    end
+
+    def to_s
+      host
+    end
+
+    def inspect
+      "#<SiteInspector::Domain host=\"#{host}\">"
+    end
+
+    # We know most API calls to the domain model are going to require
+    # That the root of all four endpoints are called. Rather than process them
+    # In serial, lets grab them in parallel and cache the results to speed
+    # up later calls.
+    def prefetch
+      endpoints.each do |endpoint|
+        request = Typhoeus::Request.new(endpoint.uri, SiteInspector.typhoeus_defaults)
+        SiteInspector.hydra.queue(request)
+      end
+      SiteInspector.hydra.run
+    end
+
+    # Converts the domain to a hash
+    #
+    # By default, it only returns domain-wide information and
+    # information about the canonical endpoint
+    #
+    # It will also pass options allong to each endpoint's to_h method
+    #
+    # options:
+    #  :all - return information about all endpoints
+    #
+    # Returns a complete hash of the domain's information
+    def to_h(options={})
+      prefetch
+      
+      hash = {
+        host:               host,
+        up:                 up?,
+        www:                www?,
+        root:               root?,
+        https:              https?,
+        enforces_https:     enforces_https?,
+        downgrades_https:   downgrades_https?,
+        canonically_www:    canonically_www?,
+        canonically_https:  canonically_https?,
+        redirect:           redirect?,
+        hsts:               hsts?,
+        hsts_subdomains:    hsts_subdomains?,
+        hsts_preload_ready: hsts_preload_ready?,
+        canoncial_endpoint: canonical_endpoint.to_h(options)
+      }
+
+      if options["all"]
+        hash.merge!({
+          endpoints: {
+            https: {
+              root: endpoints[0].to_h(options),
+              www:  endpoints[1].to_h(options)
+            },
+            http: {
+              root: endpoints[2].to_h(options),
+              www:  endpoints[3].to_h(options)
+            }
+          }
+        })
+      end
+
+      hash
+    end
+
+    def to_json
+      to_h.to_json
+    end
+  end
+end

data/lib/site-inspector/endpoint.rb

@@ -0,0 +1,200 @@
+class SiteInspector
+  # Every domain has four possible "endpoints" to evaluate
+  #
+  # For example, if you had `example.com` you'd have:
+  #   1. `http://example.com`
+  #   2. `http://www.example.com`
+  #   3. `https://example.com`
+  #   4. `https://www.example.com`
+  #
+  # Because each of the four endpoints could potentially respond differently
+  # We must evaluate all four to make certain determination
+  class Endpoint
+    attr_accessor :host, :uri
+
+    # Initatiate a new Endpoint object
+    #
+    # endpoint - (string) the endpoint to query (e.g., `https://example.com`)
+    def initialize(host)
+      @uri = Addressable::URI.parse(host.downcase)
+      @host = uri.host.sub(/^www\./, "")
+      @checks = {}
+    end
+
+    def www?
+      !!(uri.host =~ /^www\./)
+    end
+
+    def root?
+      !www?
+    end
+
+    def https?
+      https.scheme?
+    end
+
+    def http?
+      !https?
+    end
+
+    def scheme
+      @uri.scheme
+    end
+
+    def request(options = {})
+      target = options[:path] ? URI.join(uri, options.delete(:path)) : uri
+      request = Typhoeus::Request.new(target, SiteInspector.typhoeus_defaults.merge(options))
+      hydra.queue(request)
+      hydra.run
+      request.response
+    end
+
+    # Makes a GET request of the given host
+    #
+    # Retutns the Typhoeus::Response object
+    def response
+      @response ||= request
+    end
+
+    # Does the server return any response? (including 50x)
+    def response?
+       response.code != 0 && !timed_out?
+    end
+
+    def response_code
+      response.response_code.to_s if response
+    end
+
+    def timed_out?
+      response && response.timed_out?
+    end
+
+    # Does the endpoint return a 2xx or 3xx response code?
+    def up?
+      response && response_code.start_with?("2") || response_code.start_with?("3")
+    end
+
+    def down?
+      !up?
+    end
+
+    # If the domain is a redirect, what's the first endpoint we're redirected to?
+    def redirect
+      return unless response && response_code.start_with?("3")
+
+      @redirect ||= begin
+        redirect = Addressable::URI.parse(headers["location"])
+
+        # This is a relative redirect, but we still need the absolute URI
+        if redirect.relative?
+          redirect.path = "/#{redirect.path}" unless redirect.path[0] == "/"
+          redirect.host = host
+          redirect.scheme = scheme
+        end
+
+        # This was a redirect to a subpath or back to itself, which we don't care about
+        return if redirect.host == host && redirect.scheme == scheme
+
+        # Init a new endpoint representing the redirect
+        Endpoint.new(redirect.to_s)
+      end
+    end
+
+    # Does this endpoint return a redirect?
+    def redirect?
+      !!redirect
+    end
+
+    # What's the effective URL of a request to this domain?
+    def resolves_to
+      return self unless redirect?
+      @resolves_to ||= begin
+        response = request(:followlocation => true)
+
+        # Workaround for Webmock not playing nicely with Typhoeus redirects
+        if response.mock?
+          if response.headers["Location"]
+            url = response.headers["Location"]
+          else
+            url = response.request.url
+          end
+        else
+          url = response.effective_url
+        end
+
+        Endpoint.new(url)
+      end
+    end
+
+    def external_redirect?
+      host != resolves_to.host
+    end
+
+    def to_s
+      uri.to_s
+    end
+
+    def inspect
+      "#<SiteInspector::Endpoint uri=\"#{uri.to_s}\">"
+    end
+
+    # Returns information about the endpoint
+    #
+    # By default, all checks are run. If one or more check names are passed
+    # in the options hash, only those checks will be run.
+    #
+    # options:
+    #   a hash of check symbols and bools representing which checks should be run
+    #
+    # Returns the hash representing the endpoint and its checks
+    def to_h(options={})
+      hash = {
+        uri: uri.to_s,
+        host: host,
+        www: www?,
+        https: https?,
+        scheme: scheme,
+        up: up?,
+        timed_out: timed_out?,
+        redirect: redirect?,
+        external_redirect: external_redirect?,
+      }
+
+      # Either they've specifically asked for a check, or we throw everything at them
+      checks = SiteInspector::Endpoint.checks.select { |c| options.keys.include?(c.name) }
+      checks = SiteInspector::Endpoint.checks if checks.empty?
+
+      checks.each do |check|
+        hash[check.name] = self.send(check.name).to_h
+      end
+
+      hash
+    end
+
+    def self.checks
+      ObjectSpace.each_object(Class).select { |klass| klass < Check }
+    end
+
+    def method_missing(method_sym, *arguments, &block)
+      if check = SiteInspector::Endpoint.checks.find { |c| c.name == method_sym }
+        @checks[method_sym] ||= check.new(self)
+      else
+        super
+      end
+    end
+
+    def respond_to?(method_sym, include_private = false)
+      if checks.keys.include?(method_sym)
+        true
+      else
+        super
+      end
+    end
+
+    private
+
+    def hydra
+      SiteInspector.hydra
+    end
+  end
+end