site-inspector 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5eab4885077637bf3251776f685d29b3cde53a3f
-  data.tar.gz: de588b40792c8fb27be03badd85df8c7b74c3d6b
+  metadata.gz: b994e74251bb7b1be968d5bbad80273ee3ffea24
+  data.tar.gz: e104a5856e1710233df4da02b234935e56910f3b
 SHA512:
-  metadata.gz: c23e39615a372abb0ce5958d1ed7abdbfb7d98c436b1cdfd21ec9072471f0eed3c0365641d3211945212309205b6501cb59b2f7b158017ef082f71b72c738d02
-  data.tar.gz: 0dc5713aa8301858af5e0c8941aa12c05140837e33151d650475aed15d39794d950ea90c1152799ec6ad04b9d9739f0cd14a634abc69ae9ff48145a17f0da7fd
+  metadata.gz: c68e92c9e4c7b82aa7f19fb56f625f0656104b026ef81ede02b8f9dff90b430fc1af16062bb0fa8d337ad53f410ee563c127de9c594a3b47ce4a93723b668b90
+  data.tar.gz: 2e07f88d11a2c116289bfeb3e9050ad59b7314523aedeb45cf7d4b4042912c31e2ac4e80828ed27a272cec8952726ed02735c2af4022821255d3685c38478c61

data/.gitignore

@@ -2,6 +2,7 @@
 /*.gem
 vendor
 Gemfile.lock
-
 /.env
 /tmp
+/cache
+node_modules

data/.travis.yml

@@ -3,4 +3,7 @@ rvm:
   - 2.1
 script: "script/cibuild"
 sudo: false
-cache: bundler
+cache:
+  - bundler
+  - node_modules
+before_script: npm install

data/README.md

@@ -2,7 +2,7 @@
 
 A Ruby Gem to sniff information about a domain's technology and capabilities.
 
-[![Gem Version](https://badge.fury.io/rb/site-inspector.svg)](http://badge.fury.io/rb/site-inspector) [![Build Status](https://travis-ci.org/benbalter/site-inspector-ruby.svg)](https://travis-ci.org/benbalter/site-inspector-ruby)
+[![Gem Version](https://badge.fury.io/rb/site-inspector.svg)](http://badge.fury.io/rb/site-inspector) [![Build Status](https://travis-ci.org/benbalter/site-inspector.svg)](https://travis-ci.org/benbalter/site-inspector)
 
 ## Demo
 
@@ -87,6 +87,15 @@ Options:
 
 Each endpoint also returns the following checks:
 
+#### Accessibility
+
+Uses the `pa11y` CLI to run automated accessibility tests. Requires `node`. To install `pally`: `[sudo] npm install -g pa11y`.
+
+* `section508` - Tests against the Section508 standard
+* `wcag2a` - Tests against the WCAG2A standard
+* `wcag2aa` - Tests against the WCAG2AA standard
+* `wcag2aaa` - Tests against the WCAG2AAA standard
+
 #### Content
 
 * `doctype` - The HTML doctype returned
@@ -136,7 +145,7 @@ Each endpoint also returns the following checks:
 
 ## Adding your own check
 
-[Checks](https://github.com/benbalter/site-inspector-ruby/tree/master/lib/site-inspector/checks) are special classes that are children of [`SiteInspector::Endpoint::Check`](https://github.com/benbalter/site-inspector-ruby/blob/master/lib/site-inspector/checks/check.rb). You can implement your own check like this:
+[Checks](https://github.com/benbalter/site-inspector/tree/master/lib/site-inspector/checks) are special classes that are children of [`SiteInspector::Endpoint::Check`](https://github.com/benbalter/site-inspector/blob/master/lib/site-inspector/checks/check.rb). You can implement your own check like this:
 
 ```ruby
 class SiteInspector

data/lib/site-inspector/checks/accessibility.rb

@@ -0,0 +1,114 @@
+require 'json'
+require 'open3'
+
+class SiteInspector
+  class Endpoint
+    class Accessibility < Check
+      class Pa11yError < RuntimeError; end
+
+      STANDARDS = {
+        section508: 'Section508', # Default standard
+        wcag2a:     'WCAG2A',
+        wcag2aa:    'WCAG2AA',
+        wcag2aaa:   'WCAG2AAA'
+      }
+
+      DEFAULT_LEVEL = :error
+
+      class << self
+        def pa11y_version
+          output, status = Open3.capture2e("pa11y", "--version")
+          output.strip if status == 0
+        end
+
+        def pa11y?
+          !!(Cliver.detect('pa11y'))
+        end
+        alias_method :enabled?, :pa11y?
+      end
+
+      def level
+        @level ||= DEFAULT_LEVEL
+      end
+
+      def level=(level)
+        raise ArgumentError, "Invalid level '#{level}'" unless [:error, :warning, :notice].include?(level)
+        @level = level
+      end
+
+      def standard?(standard)
+        STANDARDS.keys.include?(standard)
+      end
+
+      def standard
+        @standard ||= STANDARDS.keys.first
+      end
+
+      def standard=(standard)
+        raise ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+        @standard = standard
+      end
+
+      def valid?
+        check[:valid] if check
+      end
+
+      def errors
+        check[:results].count { |r| r["type"] == "error" } if check
+      end
+
+      def check
+        @check ||= pa11y(standard)
+      rescue Pa11yError
+        nil
+      end
+      alias_method :to_h, :check
+
+      def method_missing(method_sym, *arguments, &block)
+        if standard?(method_sym)
+          pa11y(method_sym)
+        else
+          super
+        end
+      end
+
+      def respond_to?(method_sym, include_private = false)
+        if standard?(method_sym)
+          true
+        else
+          super
+        end
+      end
+
+      private
+
+      def pa11y(standard)
+        Cliver.assert('pa11y')
+        raise ArgumentError, "Unknown standard '#{standard}'" unless standard?(standard)
+
+        args = [
+          "--standard", STANDARDS[standard],
+          "--reporter", "json",
+          "--level",    level.to_s,
+          endpoint.uri.to_s
+        ]
+        output, status = run_command(args)
+
+        # Pa11y exit codes: https://github.com/nature/pa11y#exit-codes
+        # 0: No errors, 1: Technical error within pa11y, 2: accessibility error (configurable via --level)
+        raise Pa11yError if status == 1
+
+        {
+          valid:   status == 0,
+          results: JSON.parse(output)
+        }
+      rescue Pa11yError, JSON::ParserError
+        raise Pa11yError, "Command `pa11y #{args.join(" ")}` failed: #{output}"
+      end
+
+      def run_command(args)
+        Open3.capture2e("pa11y", *args)
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/check.rb

@@ -36,6 +36,10 @@ class SiteInspector
       def self.name
         self.to_s.split('::').last.downcase.to_sym
       end
+
+      def self.enabled?
+        true
+      end
     end
   end
 end

data/lib/site-inspector/checks/content.rb

@@ -3,7 +3,7 @@ class SiteInspector
     class Content < Check
       # Given a path (e.g, "/data"), check if the given path exists on the canonical endpoint
       def path_exists?(path)
-        endpoint.request(path: path, followlocation: true).success?
+        endpoint.up? && endpoint.request(path: path, followlocation: true).success?
       end
 
       def document
@@ -17,22 +17,23 @@ class SiteInspector
       end
 
       def robots_txt?
-        @bodts_txt ||= path_exists?("robots.txt")
+        @bodts_txt ||= path_exists?("robots.txt") if proper_404s?
       end
 
       def sitemap_xml?
-        @sitemap_xml ||= path_exists?("sitemap.xml")
+        @sitemap_xml ||= path_exists?("sitemap.xml") if proper_404s?
       end
 
       def humans_txt?
-        @humans_txt ||= path_exists?("humans.txt")
+        @humans_txt ||= path_exists?("humans.txt") if proper_404s?
       end
 
       def doctype
-        document.internal_subset.name
+        document.internal_subset.external_id
       end
 
       def prefetch
+        return unless endpoint.up?
         options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
         ["robots.txt", "sitemap.xml", "humans.txt", random_path].each do |path|
           request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)

data/lib/site-inspector/checks/cookies.rb

@@ -0,0 +1,45 @@
+class SiteInspector
+  class Endpoint
+    class Cookies < Check
+
+      def any?(&block)
+        if cookie_header.nil? || cookie_header.empty?
+          false
+        elsif block_given?
+          all.any? { |cookie| block.call(cookie) }
+        else
+          true
+        end
+      end
+      alias_method :cookies?, :any?
+
+      def all
+        @cookies ||= cookie_header.map { |c| CGI::Cookie::parse(c) } if cookies?
+      end
+
+      def [](key)
+        all.find { |cookie| cookie.keys.first == key } if cookies?
+      end
+
+      def secure?
+        pairs = cookie_header.join("; ").split("; ") # CGI::Cookies#Parse doesn't seem to like secure headers
+        pairs.any? { |c| c.downcase == "secure" } && pairs.any? { |c| c.downcase == "httponly" }
+      end
+
+      def to_h
+        {
+          :cookie? => any?,
+          :secure? => secure?
+        }
+      end
+
+      private
+
+      def cookie_header
+        # Cookie header may be an array or string, always return an array
+        [endpoint.headers.all["set-cookie"]].flatten.compact
+      end
+
+    end
+  end
+end

data/lib/site-inspector/checks/headers.rb

@@ -2,12 +2,6 @@ class SiteInspector
   class Endpoint
     class Headers < Check
 
-      # cookies can have multiple set-cookie headers, so this detects
-      # whether cookies are set, but not all their values.
-      def cookies?
-        !!headers["set-cookie"]
-      end
-
       # TODO: kill this
       def strict_transport_security?
         !!strict_transport_security
@@ -49,13 +43,6 @@ class SiteInspector
         xss_protection == "1; mode=block"
       end
 
-      def secure_cookies?
-        return false if !cookies?
-        cookie = headers["set-cookie"]
-        cookie = cookie.first if cookie.is_a?(Array)
-        !!(cookie =~ /(; secure.*; httponly|; httponly.*; secure)/i)
-      end
-
       # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
       def all
         @all ||= (response && response.headers) ? Hash[response.headers.map{ |k,v| [k.downcase,v] }] : {}
@@ -68,14 +55,11 @@ class SiteInspector
 
       def to_h
         {
-          :cookies => cookies?,
           :strict_transport_security => strict_transport_security || false,
           :content_security_policy => content_security_policy || false,
           :click_jacking_protection => click_jacking_protection || false,
-          :click_jacking_protection => click_jacking_protection || false,
           :server => server,
           :xss_protection => xss_protection || false,
-          :secure_cookies => secure_cookies?
         }
       end
     end

data/lib/site-inspector/checks/sniffer.rb

@@ -2,8 +2,31 @@ class SiteInspector
   class Endpoint
     class Sniffer < Check
 
-      def cms
-        sniff :cms
+      OPEN_SOURCE_FRAMEWORKS = [
+        # Sniffles
+        :drupal,
+        :joomla,
+        :movabletype,
+        :phpbb,
+        :wordpress,
+
+        # Internal
+        :php,
+        :expression_engine,
+        :cowboy
+      ]
+
+      def framework
+        cms = sniff :cms
+        return cms unless cms.nil?
+        return :expression_engine if endpoint.cookies.any? { |c| c.keys.first =~ /^exp_/ }
+        return :php if endpoint.cookies["PHPSESSID"]
+        return :cowboy if endpoint.headers.server.to_s.downcase == "cowboy"
+        nil
+      end
+
+      def open_source?
+        OPEN_SOURCE_FRAMEWORKS.include?(framework)
       end
 
       def analytics
@@ -20,7 +43,7 @@ class SiteInspector
 
       def to_h
         {
-          :cms         => cms,
+          :framework   => framework,
           :analytics   => analytics,
           :javascript  => javascript,
           :advertising => advertising
@@ -31,9 +54,8 @@ class SiteInspector
 
       def sniff(type)
         require 'sniffles'
-        results = Sniffles.sniff(endpoint.content.body, type).select { |name, meta| meta[:found] == true }
-        results.each { |name, result| result.delete :found} if results
-        results
+        results = Sniffles.sniff(endpoint.content.body, type).select { |name, meta| meta[:found] }
+        results.keys.first if results
       rescue
         nil
       end

data/lib/site-inspector/disk_cache.rb

@@ -31,8 +31,10 @@ class SiteInspector
 
     private
 
+    # The `request` is a Typhoeus::Request, which provides a
+    # unique `cache_key` string for exactly this sort of thing.
     def path(request)
-      File.join(@dir, request)
+      File.join(@dir, request.cache_key)
     end
   end
 end

data/lib/site-inspector/domain.rb

@@ -32,12 +32,20 @@ class SiteInspector
       Gman.valid? host
     end
 
-    # Does *any* endpoint return a 200 response code?
+    # Does *any* endpoint return a 200 or 300 response code?
     def up?
       endpoints.any? { |e| e.up? }
     end
 
-    # Does any www endpoint return a 200 response code?
+    # Does *any* endpoint respond to HTTP?
+    # TODO: needs to allow an invalid chain.
+    def responds?
+      endpoints.any? { |e| e.responds? }
+    end
+
+
+    # TODO: These weren't present before, and may not be useful.
+    # Can you connect to www?
     def www?
       endpoints.any? { |e| e.www? && e.up? }
     end
@@ -51,11 +59,13 @@ class SiteInspector
     #
     # * Either of the HTTPS endpoints is listening, and doesn't have
     #   an invalid hostname.
+    #
+    # TODO: needs to allow an invalid chain.
     def https?
       endpoints.any? { |e| e.https? && e.up? && e.https.valid? }
     end
 
-    # HTTPS is enforced if one of the HTTPS endpoints is "live",
+    # HTTPS is enforced if one of the HTTPS endpoints is "up",
     # and if both *HTTP* endpoints are either:
     #
     #  * down, or
@@ -66,14 +76,19 @@ class SiteInspector
     # * an HTTP redirect can go to HTTPS on another domain, as long
     #   as it's immediate.
     # * a domain with an invalid cert can still be enforcing HTTPS.
+    #
+    # TODO: need to ensure the redirect *immediately* goes to HTTPS.
+    # TODO: don't need to require that the HTTPS cert is valid for this purpose.
     def enforces_https?
       return false unless https?
-      endpoints.select { |e| e.http? }.all? { |e| e.down? || (e.redirect && e.redirect.https?) }
+      endpoints.select { |e| e.http? }.all? { |e| !e.up? || (e.redirect && e.redirect.https?) }
     end
 
     # we can say that a canonical HTTPS site "defaults" to HTTPS,
     # even if it doesn't *strictly* enforce it (e.g. having a www
     # subdomain first to go HTTP root before HTTPS root).
+    #
+    # TODO: not implemented.
     def defaults_https?
       raise "Not implemented. Halp?"
     end
@@ -82,9 +97,11 @@ class SiteInspector
     #
     # * HTTPS is supported, and
     # * The 'canonical' endpoint gets an immediate internal redirect to HTTP.
+    #
+    # TODO: the redirect must be internal.
     def downgrades_https?
       return false unless https?
-      canonical_endpoint.redirect && canonical_endpoint.redirect.http?
+      canonical_endpoint.redirect? && canonical_endpoint.redirect.http?
     end
 
     # A domain is "canonically" at www if:
@@ -108,7 +125,7 @@ class SiteInspector
       return false unless www?
 
       # Are both root endpoints down?
-      return true if endpoints.select { |e| e.root? }.all? { |e| e.down? }
+      return true if endpoints.select { |e| e.root? }.all? { |e| !e.up? }
 
       # Does either root endpoint redirect to a www endpoint?
       endpoints.select { |e| e.root? }.any? { |e| e.redirect && e.redirect.www? }
@@ -139,7 +156,7 @@ class SiteInspector
       return false unless https?
 
       # Both http endpoints are down
-      return true if endpoints.select { |e| e.http? }.all? { |e| e.down? }
+      return true if endpoints.select { |e| e.http? }.all? { |e| !e.up? }
 
       # at least one http endpoint redirects immediately to https
       endpoints.select { |e| e.http? }.any? { |e| e.redirect && e.redirect.https? }
@@ -150,7 +167,7 @@ class SiteInspector
     # 2. All endpoints are either down or an external redirect
     def redirect?
       return false unless redirect
-      endpoints.all? { |e| e.down? || e.external_redirect? }
+      endpoints.all? { |e| !e.up? || e.external_redirect? }
     end
 
     # The first endpoint to respond with a redirect
@@ -205,10 +222,11 @@ class SiteInspector
     # Returns a complete hash of the domain's information
     def to_h(options={})
       prefetch
-      
+
       hash = {
         host:               host,
         up:                 up?,
+        responds:           responds?,
         www:                www?,
         root:               root?,
         https:              https?,
@@ -220,7 +238,7 @@ class SiteInspector
         hsts:               hsts?,
         hsts_subdomains:    hsts_subdomains?,
         hsts_preload_ready: hsts_preload_ready?,
-        canoncial_endpoint: canonical_endpoint.to_h(options)
+        canonical_endpoint: canonical_endpoint.to_h(options)
       }
 
       if options["all"]

data/lib/site-inspector/endpoint.rb

@@ -56,11 +56,6 @@ class SiteInspector
       @response ||= request
     end
 
-    # Does the server return any response? (including 50x)
-    def response?
-       response.code != 0 && !timed_out?
-    end
-
     def response_code
       response.response_code.to_s if response
     end
@@ -74,8 +69,9 @@ class SiteInspector
       response && response_code.start_with?("2") || response_code.start_with?("3")
     end
 
-    def down?
-      !up?
+    # Does the server respond at all?
+    def responds?
+       response.code != 0 && !timed_out?
     end
 
     # If the domain is a redirect, what's the first endpoint we're redirected to?
@@ -155,6 +151,7 @@ class SiteInspector
         https: https?,
         scheme: scheme,
         up: up?,
+        responds: responds?,
         timed_out: timed_out?,
         redirect: redirect?,
         external_redirect: external_redirect?,
@@ -172,7 +169,7 @@ class SiteInspector
     end
 
     def self.checks
-      ObjectSpace.each_object(Class).select { |klass| klass < Check }
+      ObjectSpace.each_object(Class).select { |klass| klass < Check }.select { |check| check.enabled? }
     end
 
     def method_missing(method_sym, *arguments, &block)

data/lib/site-inspector/version.rb

@@ -1,3 +1,3 @@
 class SiteInspector
-  VERSION = "2.0.0"
+  VERSION = "3.0.0"
 end

data/lib/site-inspector.rb

@@ -2,18 +2,22 @@ require 'open-uri'
 require 'addressable/uri'
 require 'public_suffix'
 require 'typhoeus'
+require 'cliver'
+require 'cgi'
 
 require_relative 'site-inspector/cache'
 require_relative 'site-inspector/disk_cache'
 require_relative 'site-inspector/rails_cache'
 require_relative 'site-inspector/domain'
 require_relative 'site-inspector/checks/check'
+require_relative 'site-inspector/checks/accessibility'
 require_relative 'site-inspector/checks/content'
 require_relative 'site-inspector/checks/dns'
 require_relative 'site-inspector/checks/headers'
 require_relative 'site-inspector/checks/hsts'
 require_relative 'site-inspector/checks/https'
 require_relative 'site-inspector/checks/sniffer'
+require_relative 'site-inspector/checks/cookies'
 require_relative 'site-inspector/endpoint'
 require_relative 'site-inspector/version'
 
@@ -46,7 +50,7 @@ class SiteInspector
         :timeout => SiteInspector.timeout,
         :accept_encoding => "gzip",
         :headers => {
-          "User-Agent" => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector-ruby)"
+          "User-Agent" => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
         }
       }
     end

data/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "site-inspector",
+  "version": "2.0.0",
+  "description": "Returns information about a domain's technology and capabilities",
+  "main": "site-inspector",
+  "dependencies": {
+    "pa11y": "^2.1.0"
+  },
+  "devDependencies": {},
+  "scripts": {
+    "test": "script/cibuild"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/benbalter/site-inspector.git"
+  },
+  "author": "",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/benbalter/site-inspector/issues"
+  },
+  "homepage": "https://github.com/benbalter/site-inspector#readme"
+}

data/script/bootstrap

@@ -1 +1,2 @@
 bundle install
+npm install

data/script/cibuild

@@ -2,6 +2,8 @@
 
 set -e
 
+echo "Pa11y version: $(pa11y --version)"
+
 bundle exec rake spec
 
 gem build site-inspector.gemspec

data/site-inspector.gemspec

@@ -4,11 +4,11 @@ Gem::Specification.new do |s|
 
   s.name                  = "site-inspector"
   s.version               = SiteInspector::VERSION
-  s.summary               = "A Ruby port and v2 of Site Inspector (http://github.com/benbalter/site-inspector)"
+  s.summary               = "A Ruby port and v2 of Site Inspector (https://github.com/benbalter/site-inspector)"
   s.description           = "Returns information about a domain's technology and capabilities"
   s.authors               = "Ben Balter"
   s.email                 = "ben@balter.com"
-  s.homepage              = "https://github.com/benbalter/site-inspector-ruby"
+  s.homepage              = "https://github.com/benbalter/site-inspector"
   s.license               = "MIT"
 
   s.files                 = `git ls-files -z`.split("\x0")
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_dependency("oj", "~> 2.11")
   s.add_dependency("mercenary", "~> 0.3")
   s.add_dependency("colorator", "~> 0.1")
+  s.add_dependency("cliver", "~> 0.3")
   s.add_development_dependency("pry", "~> 0.10")
   s.add_development_dependency( "rake", "~> 10.4" )
   s.add_development_dependency( "rspec", "~> 3.2")