site-inspector 1.0.2 → 2.0.0

data/lib/site-inspector/cache.rb

@@ -1,58 +1,15 @@
-class SiteInspectorCache
-  def initialize
-    @memory = {}
-  end
-
-  def get(request)
-    @memory[request]
-  end
-
-  def set(request, response)
-    @memory[request] = response
-  end
-end
-
-class SiteInspectorDiskCache
-  def initialize(dir = nil, replace = false)
-    @dir = dir
-    @memory = {}
-    @replace = replace
-  end
-
-  def path(request)
-    File.join(@dir, request.cache_key)
-  end
-
-  def fetch(request)
-    if File.exist?(path(request))
-
-      if @replace
-        FileUtils.rm(path(request))
-        nil
-      else
-        contents = File.read(path(request))
-        begin
-          Marshal.load(contents)
-        rescue ArgumentError
-          FileUtils.rm(path(request))
-          nil
-        end
-      end
+class SiteInspector
+  class Cache
+    def memory
+      @memory ||= {}
     end
-  end
 
-  def store(request, response)
-    File.open(File.join(@dir, request.cache_key), "w") do |f|
-      f.write Marshal.dump(response)
+    def get(request)
+      memory[request]
     end
-  end
-
-  def get(request)
-    @memory[request] || fetch(request)
-  end
 
-  def set(request, response)
-    store(request, response)
-    @memory[request] = response
+    def set(request, response)
+      memory[request] = response
+    end
   end
 end

data/lib/site-inspector/checks/check.rb

@@ -0,0 +1,41 @@
+class SiteInspector
+  class Endpoint
+    class Check
+
+      attr_reader :endpoint
+
+      # A check is an abstract class that takes an Endpoint object
+      # and is extended to preform the specific site inspector checks
+      #
+      # It is automatically accessable within the endpoint object
+      # by virtue of extending the Check class
+      def initialize(endpoint)
+        @endpoint = endpoint
+      end
+
+      def response
+        endpoint.response
+      end
+
+      def request
+        response.request
+      end
+
+      def host
+        request.base_url.host
+      end
+
+      def inspect
+        "#<#{self.class} endpoint=\"#{response.effective_url}\">"
+      end
+
+      def name
+        self.class.name
+      end
+
+      def self.name
+        self.to_s.split('::').last.downcase.to_sym
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/content.rb

@@ -0,0 +1,67 @@
+class SiteInspector
+  class Endpoint
+    class Content < Check
+      # Given a path (e.g, "/data"), check if the given path exists on the canonical endpoint
+      def path_exists?(path)
+        endpoint.request(path: path, followlocation: true).success?
+      end
+
+      def document
+        require 'nokogiri'
+        @doc ||= Nokogiri::HTML response.body if response
+      end
+      alias_method :doc, :document
+
+      def body
+        @body ||= document.to_s.force_encoding("UTF-8").encode("UTF-8", :invalid => :replace, :replace => "")
+      end
+
+      def robots_txt?
+        @bodts_txt ||= path_exists?("robots.txt")
+      end
+
+      def sitemap_xml?
+        @sitemap_xml ||= path_exists?("sitemap.xml")
+      end
+
+      def humans_txt?
+        @humans_txt ||= path_exists?("humans.txt")
+      end
+
+      def doctype
+        document.internal_subset.name
+      end
+
+      def prefetch
+        options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
+        ["robots.txt", "sitemap.xml", "humans.txt", random_path].each do |path|
+          request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)
+          SiteInspector.hydra.queue(request)
+        end
+        SiteInspector.hydra.run
+      end
+
+      def proper_404s?
+        @proper_404s ||= !path_exists?(random_path)
+      end
+
+      def to_h
+        prefetch
+        {
+          doctype:     doctype,
+          sitemap_xml: sitemap_xml?,
+          robots_txt:  robots_txt?,
+          humans_txt:  humans_txt?,
+          proper_404s: proper_404s?
+        }
+      end
+
+      private
+
+      def random_path
+        require 'securerandom'
+        @random_path ||= SecureRandom.hex
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/dns.rb

@@ -0,0 +1,129 @@
+class SiteInspector
+  class Endpoint
+    class Dns < Check
+
+      def self.resolver
+        require "dnsruby"
+        @resolver ||= begin
+          resolver = Dnsruby::Resolver.new
+          resolver.config.nameserver = ["8.8.8.8", "8.8.4.4"]
+          resolver
+        end
+      end
+
+      def query(type="ANY")
+        SiteInspector::Endpoint::Dns.resolver.query(host.to_s, type).answer
+      rescue Dnsruby::ResolvTimeout, Dnsruby::ServFail, Dnsruby::NXDomain
+        []
+      end
+
+      def records
+        @records ||= query
+      end
+
+      def has_record?(type)
+        records.any? { |record| record.type == type } || query(type).count != 0
+      end
+
+      def dnssec?
+        @dnssec ||= has_record? "DNSKEY"
+      end
+
+      def ipv6?
+        @ipv6 ||= has_record? "AAAA"
+      end
+
+      def cdn
+        detect_by_hostname "cdn"
+      end
+
+      def cdn?
+        !!cdn
+      end
+
+      def cloud_provider
+        detect_by_hostname "cloud"
+      end
+
+      def cloud?
+        !!cloud_provider
+      end
+
+      def google_apps?
+        @google ||= records.any? do |record|
+          record.type == "MX" && record.exchange.to_s =~ /google(mail)?\.com\.?$/
+        end
+      end
+
+      def ip
+        require 'resolv'
+        @ip ||= Resolv.getaddress host
+      rescue Resolv::ResolvError
+        nil
+      end
+
+      def hostname
+        require 'resolv'
+        @hostname ||= PublicSuffix.parse(Resolv.getname(ip))
+      rescue Resolv::ResolvError, PublicSuffix::DomainInvalid
+        nil
+      end
+
+      def cnames
+        @cnames ||= records.select { |record| record.type == "CNAME" }.map do |record|
+          PublicSuffix.parse(record.cname.to_s)
+        end
+      end
+
+      def inspect
+        "#<SiteInspector::Domain::Dns host=\"#{host}\">"
+      end
+
+      def to_h
+        {
+          :dnssec => dnssec?,
+          :ipv6   => ipv6?,
+          :cdn    => cdn,
+          :cloud_provider => cloud_provider,
+          :google_apps => google_apps?,
+          :hostname => hostname,
+          :ip => ip
+        }
+      end
+
+      private
+
+      def data
+        @data ||= {}
+      end
+
+      def data_path(name)
+        File.expand_path "../../data/#{name}.yml", File.dirname(__FILE__)
+      end
+
+      def load_data(name)
+        require 'yaml'
+        path = data_path(name)
+        data[name] ||= YAML.load_file(path)
+      end
+
+      def detect_by_hostname(type)
+        haystack = load_data(type)
+        needle = haystack.find do |name, domain|
+          cnames.any? do |cname|
+            domain == cname.tld || domain == "#{cname.sld}.#{cname.tld}"
+          end
+        end
+
+        return needle[0].to_sym if needle
+        return nil unless hostname
+
+        needle = haystack.find do |name, domain|
+          domain == hostname.tld || domain == "#{hostname.sld}.#{hostname.tld}"
+        end
+
+        needle ? needle[0].to_sym : nil
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/headers.rb

@@ -0,0 +1,83 @@
+class SiteInspector
+  class Endpoint
+    class Headers < Check
+
+      # cookies can have multiple set-cookie headers, so this detects
+      # whether cookies are set, but not all their values.
+      def cookies?
+        !!headers["set-cookie"]
+      end
+
+      # TODO: kill this
+      def strict_transport_security?
+        !!strict_transport_security
+      end
+
+      def content_security_policy?
+        !!content_security_policy
+      end
+
+      def click_jacking_protection?
+        !!click_jacking_protection
+      end
+
+      # return the found header value
+
+      # TODO: kill this
+      def strict_transport_security
+        headers["strict-transport-security"]
+      end
+
+      def content_security_policy
+        headers["content-security-policy"]
+      end
+
+      def click_jacking_protection
+        headers["x-frame-options"]
+      end
+
+      def server
+        headers["server"]
+      end
+
+      def xss_protection
+        headers["x-xss-protection"]
+      end
+
+      # more specific checks than presence of headers
+      def xss_protection?
+        xss_protection == "1; mode=block"
+      end
+
+      def secure_cookies?
+        return false if !cookies?
+        cookie = headers["set-cookie"]
+        cookie = cookie.first if cookie.is_a?(Array)
+        !!(cookie =~ /(; secure.*; httponly|; httponly.*; secure)/i)
+      end
+
+      # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
+      def all
+        @all ||= (response && response.headers) ? Hash[response.headers.map{ |k,v| [k.downcase,v] }] : {}
+      end
+      alias_method :headers, :all
+
+      def [](header)
+        headers[header]
+      end
+
+      def to_h
+        {
+          :cookies => cookies?,
+          :strict_transport_security => strict_transport_security || false,
+          :content_security_policy => content_security_policy || false,
+          :click_jacking_protection => click_jacking_protection || false,
+          :click_jacking_protection => click_jacking_protection || false,
+          :server => server,
+          :xss_protection => xss_protection || false,
+          :secure_cookies => secure_cookies?
+        }
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/hsts.rb

@@ -0,0 +1,78 @@
+class SiteInspector
+  class Endpoint
+    # Utility parser for HSTS headers.
+    # RFC: http://tools.ietf.org/html/rfc6797
+    class Hsts < Check
+
+      def valid?
+        return false unless header
+        pairs.none? { |key, value| "#{key}#{value}" =~ /[\s\'\"]/ }
+      end
+
+      def max_age
+        pairs[:"max-age"].to_i
+      end
+
+      def include_subdomains?
+        pairs.keys.include? :includesubdomains
+      end
+
+      def preload?
+        pairs.keys.include? :preload
+      end
+
+      def enabled?
+        return false unless max_age
+        max_age > 0
+      end
+
+      # Google's minimum max-age for automatic preloading
+      def preload_ready?
+        include_subdomains? and preload? and max_age >= 10886400
+      end
+
+      def to_h
+        {
+          valid: valid?,
+          max_age: max_age,
+          include_subdomains: include_subdomains?,
+          preload: preload?,
+          enabled: enabled?,
+          preload_ready: preload_ready?
+        }
+      end
+
+      private
+
+      def headers
+        endpoint.headers
+      end
+
+      def header
+        @header ||= headers["strict-transport-security"]
+      end
+
+      def directives
+        @directives ||= header ? header.split(/\s*;\s*/) : []
+      end
+
+      def pairs
+        @pairs ||= begin
+          pairs = {}
+          directives.each do |directive|
+            key, value = directive.downcase.split("=")
+
+            if value =~ /\".*\"/
+              value = value.sub(/^\"/, '')
+              value = value.sub(/\"$/, '')
+            end
+
+            pairs[key.to_sym] = value
+          end
+
+          pairs
+        end
+      end
+    end
+  end
+end