site-inspector 1.0.2 → 3.2.0

data/lib/site-inspector/cache.rb

@@ -1,58 +1,17 @@
-class SiteInspectorCache
-  def initialize
-    @memory = {}
-  end
-
-  def get(request)
-    @memory[request]
-  end
-
-  def set(request, response)
-    @memory[request] = response
-  end
-end
-
-class SiteInspectorDiskCache
-  def initialize(dir = nil, replace = false)
-    @dir = dir
-    @memory = {}
-    @replace = replace
-  end
+# frozen_string_literal: true
 
-  def path(request)
-    File.join(@dir, request.cache_key)
-  end
-
-  def fetch(request)
-    if File.exist?(path(request))
-
-      if @replace
-        FileUtils.rm(path(request))
-        nil
-      else
-        contents = File.read(path(request))
-        begin
-          Marshal.load(contents)
-        rescue ArgumentError
-          FileUtils.rm(path(request))
-          nil
-        end
-      end
+class SiteInspector
+  class Cache
+    def memory
+      @memory ||= {}
     end
-  end
 
-  def store(request, response)
-    File.open(File.join(@dir, request.cache_key), "w") do |f|
-      f.write Marshal.dump(response)
+    def get(request)
+      memory[request]
     end
-  end
-
-  def get(request)
-    @memory[request] || fetch(request)
-  end
 
-  def set(request, response)
-    store(request, response)
-    @memory[request] = response
+    def set(request, response)
+      memory[request] = response
+    end
   end
 end

data/lib/site-inspector/checks/accessibility.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+class SiteInspector
+  class Endpoint
+    class Accessibility < Check
+      class Pa11yError < RuntimeError; end
+
+      STANDARDS = {
+        wcag2a: 'WCAG2A', # Default standard
+        wcag2aa: 'WCAG2AA',
+        wcag2aaa: 'WCAG2AAA',
+        section508: 'Section508'
+      }.freeze
+
+      DEFAULT_LEVEL = :error
+
+      REQUIRED_PA11Y_VERSION = '~> 5.0'
+
+      class << self
+        def pa11y_version
+          @pa11y_version ||= begin
+            output, status = run_command('--version')
+            output.strip if status.exitstatus.zero?
+          end
+        end
+
+        def pa11y?
+          return @pa11y_detected if defined? @pa11y_detected
+
+          @pa11y_detected = !!pa11y.detect
+        end
+
+        def enabled?
+          @@enabled && pa11y?
+        end
+
+        def pa11y
+          @pa11y ||= begin
+            node_bin = File.expand_path('../../../node_modules/pa11y/bin', File.dirname(__FILE__))
+            path = ['*', node_bin].join(File::PATH_SEPARATOR)
+            Cliver::Dependency.new('pa11y.js', REQUIRED_PA11Y_VERSION, path: path)
+          end
+        end
+
+        def run_command(args)
+          Open3.capture2e(pa11y.detect, *args)
+        end
+      end
+
+      def level
+        @level ||= DEFAULT_LEVEL
+      end
+
+      def level=(level)
+        raise ArgumentError, "Invalid level '#{level}'" unless %i[error warning notice].include?(level)
+
+        @level = level
+      end
+
+      def standard?(standard)
+        STANDARDS.key?(standard)
+      end
+
+      def standard
+        @standard ||= STANDARDS.keys.first
+      end
+
+      def standard=(standard)
+        raise ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+
+        @standard = standard
+      end
+
+      def valid?
+        check[:valid] if check
+      end
+
+      def errors
+        check[:results].count { |r| r['type'] == 'error' } if check
+      end
+
+      def check
+        @check ||= run_pa11y(standard)
+      rescue Pa11yError
+        nil
+      end
+      alias to_h check
+
+      def method_missing(method_sym, *arguments, &block)
+        if standard?(method_sym)
+          run_pa11y(method_sym)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method_sym, include_private = false)
+        if standard?(method_sym)
+          true
+        else
+          super
+        end
+      end
+
+      private
+
+      def run_pa11y(standard)
+        self.class.pa11y.detect! unless ENV['SKIP_PA11Y_CHECK']
+        raise ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+
+        args = [
+          '--standard', STANDARDS[standard],
+          '--reporter', 'json',
+          '--level',    level.to_s,
+          endpoint.uri.to_s
+        ]
+        output, status = self.class.run_command(args)
+
+        # Pa11y exit codes: https://github.com/nature/pa11y#exit-codes
+        # 0: No errors, 1: Technical error within pa11y, 2: accessibility error (configurable via --level)
+        raise Pa11yError if status.exitstatus == 1
+
+        {
+          valid: status.exitstatus.zero?,
+          results: JSON.parse(output)
+        }
+      rescue Pa11yError, JSON::ParserError
+        raise Pa11yError, "Command `pa11y #{args.join(' ')}` failed: #{output}"
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/check.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Check
+      attr_reader :endpoint
+
+      # A check is an abstract class that takes an Endpoint object
+      # and is extended to preform the specific site inspector checks
+      #
+      # It is automatically accessable within the endpoint object
+      # by virtue of extending the Check class
+      def initialize(endpoint)
+        @endpoint = endpoint
+      end
+
+      def response
+        endpoint.response
+      end
+
+      def request
+        response.request
+      end
+
+      def host
+        request.base_url.host
+      end
+
+      def inspect
+        "#<#{self.class} endpoint=\"#{response.effective_url}\">"
+      end
+
+      def name
+        self.class.name
+      end
+
+      class << self
+        @@enabled = true
+
+        def name
+          to_s.split('::').last.downcase.to_sym
+        end
+
+        def enabled?
+          !!@@enabled
+        end
+
+        def enabled=(value)
+          @@enabled = !!value
+        end
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/content.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Content < Check
+      # Given a path (e.g, "/data"), check if the given path exists on the canonical endpoint
+      def path_exists?(path)
+        endpoint.up? && endpoint.request(path: path, followlocation: true).success?
+      end
+
+      # The default Check#response method is from a HEAD request
+      # The content check has a special response which includes the body from a GET request
+      def response
+        @response ||= endpoint.request(method: :get)
+      end
+
+      def document
+        require 'nokogiri'
+        @doc ||= Nokogiri::HTML response.body if response
+      end
+      alias doc document
+
+      def body
+        @body ||= document.to_s.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, replace: '')
+      end
+
+      def robots_txt?
+        @bodts_txt ||= path_exists?('robots.txt') if proper_404s?
+      end
+
+      def sitemap_xml?
+        @sitemap_xml ||= path_exists?('sitemap.xml') if proper_404s?
+      end
+
+      def humans_txt?
+        @humans_txt ||= path_exists?('humans.txt') if proper_404s?
+      end
+
+      def doctype
+        document.internal_subset.external_id
+      end
+
+      def generator
+        @generator ||= begin
+          tag = document.at('meta[name="generator"]')
+          tag['content'] if tag
+        end
+      end
+
+      def prefetch
+        return unless endpoint.up?
+
+        options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
+        ['robots.txt', 'sitemap.xml', 'humans.txt', random_path].each do |path|
+          request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)
+          SiteInspector.hydra.queue(request)
+        end
+        SiteInspector.hydra.run
+      end
+
+      def proper_404s?
+        @proper_404s ||= !path_exists?(random_path)
+      end
+
+      def to_h
+        prefetch
+        {
+          doctype: doctype,
+          generator: generator,
+          sitemap_xml: sitemap_xml?,
+          robots_txt: robots_txt?,
+          humans_txt: humans_txt?,
+          proper_404s: proper_404s?
+        }
+      end
+
+      private
+
+      def random_path
+        require 'securerandom'
+        @random_path ||= SecureRandom.hex
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/cookies.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Cookies < Check
+      def any?(&block)
+        if cookie_header.nil? || cookie_header.empty?
+          false
+        elsif block_given?
+          all.any?(&block)
+        else
+          true
+        end
+      end
+      alias cookies? any?
+
+      def all
+        @cookies ||= cookie_header.map { |c| CGI::Cookie.parse(c) } if cookies?
+      end
+
+      def [](key)
+        all.find { |cookie| cookie.keys.first == key } if cookies?
+      end
+
+      def secure?
+        pairs = cookie_header.join('; ').split('; ') # CGI::Cookies#Parse doesn't seem to like secure headers
+        pairs.any? { |c| c.casecmp('secure').zero? } && pairs.any? { |c| c.casecmp('httponly').zero? }
+      end
+
+      def to_h
+        {
+          cookie?: any?,
+          secure?: secure?
+        }
+      end
+
+      private
+
+      def cookie_header
+        # Cookie header may be an array or string, always return an array
+        [endpoint.headers.all['set-cookie']].flatten.compact
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/dns.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Dns < Check
+      class LocalhostError < StandardError; end
+
+      def self.resolver
+        require 'dnsruby'
+        @resolver ||= begin
+          resolver = Dnsruby::Resolver.new
+          resolver.config.nameserver = ['8.8.8.8', '8.8.4.4']
+          resolver
+        end
+      end
+
+      def query(type = 'ANY')
+        SiteInspector::Endpoint::Dns.resolver.query(host.to_s, type).answer
+      rescue Dnsruby::ResolvTimeout, Dnsruby::ServFail, Dnsruby::NXDomain
+        []
+      end
+
+      def records
+        @records ||= query
+      end
+
+      def record?(type)
+        records.any? { |record| record.type == type } || query(type).count != 0
+      end
+      alias has_record? record?
+
+      def dnssec?
+        @dnssec ||= has_record? 'DNSKEY'
+      end
+
+      def ipv6?
+        @ipv6 ||= has_record? 'AAAA'
+      end
+
+      def cdn
+        detect_by_hostname 'cdn'
+      end
+
+      def cdn?
+        !!cdn
+      end
+
+      def cloud_provider
+        detect_by_hostname 'cloud'
+      end
+
+      def cloud?
+        !!cloud_provider
+      end
+
+      def google_apps?
+        @google_apps ||= records.any? do |record|
+          record.type == 'MX' && record.exchange.to_s =~ /google(mail)?\.com\.?\z/i
+        end
+      end
+
+      def localhost?
+        ip == '127.0.0.1'
+      end
+
+      def ip
+        @ip ||= Resolv.getaddress host
+      rescue Resolv::ResolvError
+        nil
+      end
+
+      def hostname
+        require 'resolv'
+        @hostname ||= PublicSuffix.parse(Resolv.getname(ip))
+      rescue Resolv::ResolvError, PublicSuffix::DomainInvalid
+        nil
+      end
+
+      def cnames
+        @cnames ||= records.select { |record| record.type == 'CNAME' }.map do |record|
+          PublicSuffix.parse(record.cname.to_s)
+        end
+      end
+
+      def inspect
+        "#<SiteInspector::Domain::Dns host=\"#{host}\">"
+      end
+
+      def to_h
+        return { error: LocalhostError } if localhost?
+
+        {
+          dnssec: dnssec?,
+          ipv6: ipv6?,
+          cdn: cdn,
+          cloud_provider: cloud_provider,
+          google_apps: google_apps?,
+          hostname: hostname,
+          ip: ip
+        }
+      end
+
+      private
+
+      def data
+        @data ||= {}
+      end
+
+      def data_path(name)
+        File.expand_path "../../data/#{name}.yml", File.dirname(__FILE__)
+      end
+
+      def load_data(name)
+        require 'yaml'
+        path = data_path(name)
+        data[name] ||= YAML.load_file(path)
+      end
+
+      def detect_by_hostname(type)
+        haystack = load_data(type)
+        needle = haystack.find do |_name, domain|
+          cnames.any? do |cname|
+            [cname.tld, "#{cname.sld}.#{cname.tld}"].include? domain
+          end
+        end
+
+        return needle[0].to_sym if needle
+        return nil unless hostname
+
+        needle = haystack.find do |_name, domain|
+          [hostname.tld, "#{hostname.sld}.#{hostname.tld}"].include? domain
+        end
+
+        needle ? needle[0].to_sym : nil
+      end
+    end
+  end
+end