site-inspector 1.0.2 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e4e41e2a1639e9f5f6e7f018ef58c664c834b2c2
-  data.tar.gz: 3205114fcaaaa11cf03ec1eb8fa2f5736b6a99f7
+SHA256:
+  metadata.gz: 9da6e03626519a29904ac4f663565104f6793e9fedae46f786f268ea7beaf870
+  data.tar.gz: 100c9e6f24fd2f5ded3cc7239d37f30852eb7c7cd4a0c69651fc1d260c0043c6
 SHA512:
-  metadata.gz: ffa69fcc3949abe434a476bf8bff0c65dbd4085b5c40f379c0f7b3bbb9eaf43ffc41527ea571fe211ac71d3911b58deaebf8446ee96a69d8a279c2877e87e3bb
-  data.tar.gz: b369d00e140c4b258b02f02e5b841f4ce26d84c0b7b32f10e6015a99673355b3a505194495fbf2bf5e9ecfabfed6e486db18572a56418bcfaf329b6acb746778
+  metadata.gz: 974095f3bc91b3a83c90ef9fa9879942e66e3a4927b0e1d83a0cba2727c9f7e301ab151a3f0bac1ff507d3a16549c64af4be23fdd245ede0ac0dc25401096ba2
+  data.tar.gz: 16378f1054392ef3aa472469860367bab3c4d276bd57eaec9131382109aa85218ad535d82477d215d7c0294f310e82eac987275fd1b7d5148c5d3a1e722c8ab5

data/.gitignore

@@ -0,0 +1,8 @@
+.bundle
+/*.gem
+vendor
+Gemfile.lock
+/.env
+/tmp
+/cache
+node_modules

data/.rubocop.yml

@@ -0,0 +1,42 @@
+inherit_from: .rubocop_todo.yml
+
+require:
+  - rubocop-performance
+  - rubocop-rspec
+
+AllCops:
+  NewCops: enable
+
+Layout/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Naming/FileName:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Style/DoubleNegation:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Style/ClassVars:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Security/MarshalLoad:
+  Exclude:
+    - lib/site-inspector/disk_cache.rb

data/.rubocop_todo.yml

@@ -0,0 +1,139 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2020-11-13 02:59:59 UTC using RuboCop version 1.3.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'site-inspector.gemspec'
+
+# Offense count: 3
+# Configuration parameters: AllowComments, AllowEmptyLambdas.
+Lint/EmptyBlock:
+  Exclude:
+    - 'spec/site_inspector_domain_spec.rb'
+
+# Offense count: 1
+Lint/NoReturnInBeginEndBlocks:
+  Exclude:
+    - 'lib/site-inspector/endpoint.rb'
+
+# Offense count: 2
+# Configuration parameters: IgnoredMethods.
+Metrics/PerceivedComplexity:
+  Max: 9
+
+# Offense count: 3
+# Configuration parameters: EnforcedStyleForLeadingUnderscores.
+# SupportedStylesForLeadingUnderscores: disallowed, required, optional
+Naming/MemoizedInstanceVariableName:
+  Exclude:
+    - 'lib/site-inspector/checks/accessibility.rb'
+
+# Offense count: 23
+# Configuration parameters: Prefixes.
+# Prefixes: when, with, without
+RSpec/ContextWording:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_content_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_dns_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_sniffer_spec.rb'
+    - 'spec/site_inspector_domain_spec.rb'
+    - 'spec/site_inspector_endpoint_spec.rb'
+
+# Offense count: 1
+RSpec/EmptyExampleGroup:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_accessibility_spec.rb'
+
+# Offense count: 24
+# Configuration parameters: Max.
+RSpec/ExampleLength:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_dns_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_hsts_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_https_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_sniffer_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_wappalyzer_spec.rb'
+    - 'spec/site_inspector_disk_cache_spec.rb'
+    - 'spec/site_inspector_domain_spec.rb'
+    - 'spec/site_inspector_endpoint_spec.rb'
+    - 'spec/site_inspector_spec.rb'
+
+# Offense count: 15
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
+# Include: **/*_spec*rb*, **/spec/**/*
+RSpec/FilePath:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_accessibility_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_check_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_content_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_cookies_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_dns_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_headers_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_hsts_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_https_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_sniffer_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_wappalyzer_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_whois_spec.rb'
+    - 'spec/site_inspector_cache_spec.rb'
+    - 'spec/site_inspector_disk_cache_spec.rb'
+    - 'spec/site_inspector_domain_spec.rb'
+    - 'spec/site_inspector_endpoint_spec.rb'
+
+# Offense count: 55
+RSpec/MultipleExpectations:
+  Max: 5
+
+# Offense count: 240
+# Configuration parameters: IgnoreSharedExamples.
+RSpec/NamedSubject:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_accessibility_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_check_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_content_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_cookies_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_dns_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_headers_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_hsts_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_https_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_sniffer_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_wappalyzer_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_whois_spec.rb'
+    - 'spec/site_inspector_cache_spec.rb'
+    - 'spec/site_inspector_disk_cache_spec.rb'
+    - 'spec/site_inspector_domain_spec.rb'
+    - 'spec/site_inspector_endpoint_spec.rb'
+
+# Offense count: 2
+RSpec/RepeatedDescription:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_content_spec.rb'
+
+# Offense count: 9
+RSpec/RepeatedExample:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_accessibility_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_content_spec.rb'
+    - 'spec/site_inspector_domain_spec.rb'
+
+# Offense count: 2
+RSpec/RepeatedExampleGroupDescription:
+  Exclude:
+    - 'spec/site_inspector_domain_spec.rb'
+
+# Offense count: 30
+RSpec/SubjectStub:
+  Exclude:
+    - 'spec/checks/site_inspector_endpoint_accessibility_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_dns_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_headers_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_hsts_spec.rb'
+    - 'spec/checks/site_inspector_endpoint_https_spec.rb'
+    - 'spec/site_inspector_endpoint_spec.rb'

data/.ruby-version

@@ -0,0 +1 @@
+2.6.6

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 2.1
+script: "script/cibuild"
+sudo: false
+cache:
+  - bundler
+  - node_modules
+before_script: npm install

data/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'sniffles', github: 'wa0x6e/sniffles'
+
+gemspec

data/Guardfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rake', task: 'test' do
+  watch(%r{^test/.+$})
+  watch(%r{^lib/.+$})
+  watch(%r{^views/.+$})
+end

data/README.md

@@ -0,0 +1,189 @@
+# Site Inspector
+
+A Ruby Gem to sniff information about a domain's technology and capabilities.
+
+[![Gem Version](https://badge.fury.io/rb/site-inspector.svg)](http://badge.fury.io/rb/site-inspector) [![Build Status](https://travis-ci.org/benbalter/site-inspector.svg)](https://travis-ci.org/benbalter/site-inspector)
+
+## Demo
+
+[site-inspector.herokuapp.com](https://site-inspector.herokuapp.com) ([source](https://github.com/benbalter/site-inspector-demo))
+
+## Concepts
+
+Site Inspector involves three primary concepts:
+
+* **Domain** - A domain has a host defined by it's TLD + SLD. A domain might be `example.com`. Domain's have certain domain-wide properties like whether it supports non-www requests, or if it enforces HTTPS.
+
+* **Endpoint** - Each domain has four endpoints based on whether you make your request with HTTPS or not, and whether you prefix the host with `www.` or not. So the domain `example.com` may have endpoints at `https://example.com`, `https://www.example.com`, `http://example.com`, and `https://www.example.com`. There may theoretically be a different server responding to each endpoint, so endpoints have certain endpoint-specific properties, like whether it responds or not, or whether it redirects. Each domain has one canonical (primary) endpoint.
+
+* **Checks** - A check is a set of tests performed on an endpoint. A check might look at what headers are returned, what CMS is used, or whether there is a valid HTTPS certificate. There are some built in checks, listed below, or you can define your own. While they're endpoint specific, checks often filter up and inform some of the domain-wide logic (such as if the domain supports HTTPS).
+
+## Usage
+
+### Ruby
+
+```ruby
+domain = SiteInspector.inspect "whitehouse.gov"
+domain.https?
+#  =>  true
+domain.www?
+#  =>  true
+domain.canonical_endpoint.to_s
+#  => "https://www.whitehouse.gov"
+domain.canonical_endpoint.sniffer.cms
+#  =>  { :drupal  =>  {}}
+```
+
+### Command line usage
+
+```
+site-inspector inspect -- inspects a domain
+
+Usage:
+
+  site-inspector inspect <domain> [options]
+
+Options:
+        -j, --json         JSON encode the output
+        -a, --all          return results for all endpoints (defaults to only the canonical endpoint)
+            --sniffer      return results for the sniffer check (defaults to all checks unless one or more checks are specified)
+            --https        return results for the https check (defaults to all checks unless one or more checks are specified)
+            --hsts         return results for the hsts check (defaults to all checks unless one or more checks are specified)
+            --headers      return results for the headers check (defaults to all checks unless one or more checks are specified)
+            --dns          return results for the dns check (defaults to all checks unless one or more checks are specified)
+            --content      return results for the content check (defaults to all checks unless one or more checks are specified)
+        -h, --help         Show this message
+        -v, --version      Print the name and version
+        -t, --trace        Show the full backtrace when an error occurs
+```
+
+## What's checked
+
+### Domain
+
+* `canonical_endpoint` - The domain's primary endpoint
+* `government` - whether the domain is a government domain
+* `up` - whether any endpoint responds
+* `www` - whether either `www` endpoint responds
+* `root` - whether you can access the domain with `www.`
+* `https` - whether HTTPS is supported
+* `enforces_https` - whether non-htttps endpoints are either down or redirects to https
+* `downgrades_https` - whether the canonical endpoint redirects to an http endpoint
+* `canonically_www` - whether non-www requests are redirected to www (or all non-www endpoints are down)
+* `canonically_https` - whether non-https request are redirected to https (or all http endpoints are down)
+* `redirect` - whether the domain redirects to an external domain
+* `hsts` - does the canonical endpoint have HSTS enabled
+* `hsts_subdomains` - are subdomains included in the HSTS list?
+* `hsts_preload_ready` - can this domain be added to the HSTS preload list?
+
+### Endpoint
+
+* `up` - whether the endpoint responds or not
+* `timed_out` - whether the endpoint times out
+* `redirect` - whether the endpoint redirects
+* `external_redirect` - whether the endpoint redirects to another domain
+
+### Checks
+
+Each endpoint also returns the following checks:
+
+#### Accessibility
+
+Uses the `pa11y` CLI to run automated accessibility tests. Requires `node`. To install `pally`: `[sudo] npm install -g pa11y`.
+
+* `section508` - Tests against the Section508 standard
+* `wcag2a` - Tests against the WCAG2A standard
+* `wcag2aa` - Tests against the WCAG2AA standard
+* `wcag2aaa` - Tests against the WCAG2AAA standard
+
+#### Content
+
+* `doctype` - The HTML doctype returned
+* `sitemap_xml` - Whether the endpoint has a sitemap
+* `robots_txt` - whether the endpoint has a `robots.txt` file
+
+#### DNS
+
+* `dnssec` - is DNSSEC supported
+* `ipv6` - is IPV6 supported
+* `cdn` - the endpoint's CDN, if any
+* `cloud_provider` - the endpoint's cloud provider, if any
+* `google_apps` - whether the domain is using google apps
+* `hostname` - the server hostname
+* `ip` - the server IP
+
+#### Headers
+
+* `cookies` - does the domain use cookies
+* `strict_transport_security` - whether STS is enabled
+* `content_security_policy` - the endpoint's CSP
+* `click_jacking_protection` - whether an `x-frame-options` header is sent
+* `xss_protection` - whether an `x-xss-protection` header is sent
+* `server` - the server header
+* `secure_cookies` - whether the cookies are secure, or not
+
+#### HSTS
+
+* `valid` - whether the HSTS header is valid
+* `max_age` - the HSTS max age
+* `include_subdomains` - whether subdomains are included
+* `preload` - whether its preloaded
+* `enabled` - whether HSTS is enabled
+* `preload_ready` - whether HSTS could be preloaded
+
+#### HTTPS
+
+* `valid` - if the HTTPS response is valid
+* `return_code` - the HTTPS error, if any
+
+#### Sniffer
+
+* `cms` - the CMS used, if any
+* `analytics` - the analytics providers used, if any
+* `javascript` - the javascript libraries used, if any
+* `advertising` - the advertising providers used, if any
+
+## Adding your own check
+
+[Checks](https://github.com/benbalter/site-inspector/tree/master/lib/site-inspector/checks) are special classes that are children of [`SiteInspector::Endpoint::Check`](https://github.com/benbalter/site-inspector/blob/master/lib/site-inspector/checks/check.rb). You can implement your own check like this:
+
+```ruby
+class SiteInspector
+  class Endpoint
+    class Mention < Check
+      def mentions_ben?
+        endpoint.content.body =~ /ben/i
+      end
+    end
+  end
+end
+```
+
+This check can then be used as follows:
+```
+domain.canonical_endpoint.mention.mentions_ben?
+```
+
+Checks can call the `endpoint` object, which, contains the request, response, and other checks. Custom checks are automatically exposed as endpoint methods.
+
+## Contributing
+
+### Bootstrapping locally
+
+1. Clone down the repo
+2. `script/bootstrap`
+
+### Running tests
+
+`script/cibuild`
+
+### Development console
+
+`script/console`
+
+### How to contribute
+
+1. Fork the project
+2. Create a new, descriptively named feature branch
+3. Make your changes
+4. Submit a pull request

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require './lib/site-inspector'
+require 'rspec/core/rake_task'
+
+desc 'Run specs'
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+  t.rspec_opts = ['--order', 'rand', '--color']
+end

data/bin/site-inspector

@@ -1,29 +1,57 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require_relative "../lib/site-inspector"
+require 'mercenary'
+require 'oj'
+require 'yaml'
+require 'colorator'
+require_relative '../lib/site-inspector'
 
-require "oj"
+def stringify_keys_deep!(hash)
+  hash.each_key do |k|
+    ks = k.respond_to?(:to_s) ? k.to_s : k
+    hash[ks] = hash.delete k # Preserve order even when k == ks
+    stringify_keys_deep! h[ks] if hash[ks].is_a? Hash
+  end
+end
 
-domain = ARGV[0]
-http_mode = (ARGV[1] == "--http")
+Mercenary.program(:"site-inspector") do |p|
+  p.version SiteInspector::VERSION
+  p.description "Returns information about a domain's technology and capabilities"
+  p.syntax 'site-inspector <command> <domain> [options]'
 
-if domain.to_s.empty?
-  puts "Usage: site-inspector [DOMAIN] [--http]"
-  exit 1
-end
+  p.command(:inspect) do |c|
+    c.syntax 'inspect <domain> [options]'
+    c.description 'inspects a domain'
+    c.option 'json', '-j', '--json', 'JSON encode the output'
+    c.option 'all', '-a', '--all', 'return results for all endpoints (defaults to only the canonical endpoint)'
 
-# HTTP mode:
-#   * all details for possible endpoints
-#   * don't follow redirects
-#   * shorter timeout
-if http_mode
-  site = SiteInspector.new(domain)
-  details = site.http
-
-# Normal mode: autodetect canonical domain, sweep every attribute.
-else
-  site = SiteInspector.new(domain)
-  details = site.to_hash
-end
+    SiteInspector::Endpoint.checks.each do |check|
+      c.option check.name, "--#{check.name}", "return results for the #{check.name} check (defaults to all checks unless one or more checks are specified)"
+    end
+
+    c.action do |args, options|
+      next c.logger.fatal 'Must specify a domain' if args.length != 1
+
+      # Build our domain hash as requested
+      domain = SiteInspector.inspect(args[0])
+      hash = domain.to_h(options)
+      json = Oj.dump(hash, indent: 2, mode: :compat)
 
-puts Oj.dump(details, indent: 2, mode: :compat)
+      # Dump the JSON and run
+      next puts json if options['json']
+
+      # This is a dirty, dirty hack, but it's a simple way to stringify keys recursively
+      # And format the output in a human-readable way
+      yaml = YAML.dump Oj.load(json)
+
+      # Colorize bools
+      yaml.gsub!(/: (true|ok)$/, ": #{'true'.green}")
+      yaml.gsub!(/: false$/, ": #{'false'.red}")
+
+      puts yaml
+    end
+  end
+
+  p.default_command(:inspect)
+end