site-inspector 3.2.0 → 4.0.0

data/spec/site_inspector/endpoint/well_known_spec.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe SiteInspector::Endpoint::WellKnown do
+  subject do
+    stub_request(:head, 'http://example.com/')
+      .to_return(status: 200, headers: { foo: 'bar' })
+    endpoint = SiteInspector::Endpoint.new('http://example.com')
+    described_class.new(endpoint)
+  end
+end

data/spec/{checks/site_inspector_endpoint_whois_spec.rb → site_inspector/endpoint/whois_spec.rb}

@@ -2,7 +2,8 @@
 
 require 'spec_helper'
 
-describe SiteInspector::Endpoint::Whois do
+# NOTE: these tests make external WHOIS calls; run them with LIVE=1
+describe SiteInspector::Endpoint::Whois, :live do
   subject do
     stub_request(:head, site).to_return(status: 200)
     endpoint = SiteInspector::Endpoint.new(site)

data/spec/{site_inspector_endpoint_spec.rb → site_inspector/endpoint_spec.rb}

@@ -7,12 +7,13 @@ describe SiteInspector::Endpoint do
 
   it 'downcases the host' do
     endpoint = described_class.new('http://EXAMPLE.com')
-    expect(endpoint.host).to eql('example.com')
+    expect(endpoint.host.to_s).to eql('example.com')
   end
 
   it 'strips www from the host' do
+    skip
     endpoint = described_class.new('http://www.example.com')
-    expect(endpoint.host).to eql('example.com')
+    expect(endpoint.host.to_s).to eql('example.com')
   end
 
   it 'returns the uri' do
@@ -114,6 +115,22 @@ describe SiteInspector::Endpoint do
       expect(subject.response_code).to eql('200')
     end
 
+    it 'is not up when there is no response' do
+      allow(subject).to receive(:response).and_return(nil)
+      expect(subject.up?).to be(false)
+    end
+
+    it 'is up for 2xx and 3xx responses and down otherwise' do
+      allow(subject).to receive(:response) { Typhoeus::Response.new(code: 200) }
+      expect(subject.up?).to be(true)
+
+      allow(subject).to receive(:response) { Typhoeus::Response.new(code: 301) }
+      expect(subject.up?).to be(true)
+
+      allow(subject).to receive(:response) { Typhoeus::Response.new(code: 500) }
+      expect(subject.up?).to be(false)
+    end
+
     it 'knows if a response has timed out' do
       allow(subject).to receive(:response) { Typhoeus::Response.new(return_code: :operation_timedout) }
       expect(subject.timed_out?).to be(true)
@@ -198,6 +215,44 @@ describe SiteInspector::Endpoint do
       expect(subject.redirect?).to be(false)
     end
 
+    it 'resolves protocol-relative redirects against the current scheme' do
+      stub_request(:head, 'http://example.com/')
+        .to_return(status: 301, headers: { location: '//www.example.com/' })
+
+      expect(subject.redirect?).to be(true)
+      expect(subject.redirect.uri.to_s).to eql('http://www.example.com/')
+    end
+
+    it 'keeps the https scheme for protocol-relative redirects to another host' do
+      endpoint = described_class.new('https://example.com')
+      stub_request(:head, 'https://example.com/')
+        .to_return(status: 301, headers: { location: '//other.com/' })
+
+      expect(endpoint.redirect.uri.to_s).to eql('https://other.com/')
+      expect(endpoint.redirect.scheme).to eql('https')
+    end
+
+    it 'treats query-only redirects as internal' do
+      stub_request(:head, 'http://example.com/')
+        .to_return(status: 301, headers: { location: '?q=1' })
+
+      expect(subject.redirect?).to be(false)
+    end
+
+    it 'treats dot-segment redirects as internal' do
+      stub_request(:head, 'http://example.com/')
+        .to_return(status: 301, headers: { location: '../x' })
+
+      expect(subject.redirect?).to be(false)
+    end
+
+    it 'handles a redirect without a location header' do
+      stub_request(:head, 'http://example.com/')
+        .to_return(status: 301)
+
+      expect(subject.redirect?).to be(false)
+    end
+
     it 'knows what it resolves to' do
       stub_request(:head, 'http://example.com/')
         .to_return(status: 301, headers: { location: 'http://www.example.com' })
@@ -233,9 +288,10 @@ describe SiteInspector::Endpoint do
 
   context 'checks' do
     it 'identifies checks' do
-      expected = 9
+      expected = 10
       pa11y = SiteInspector::Endpoint::Accessibility.pa11y?
       expected -= 1 unless pa11y
+      expected -= 1 unless SiteInspector::Endpoint::Wappalyzer.wappalyzer?
       expect(described_class.checks.count).to eql(expected)
     end
 

data/spec/site_inspector_spec.rb

@@ -4,22 +4,72 @@ require 'spec_helper'
 
 describe SiteInspector do
   before do
-    described_class.instance_variable_set('@cache', nil)
-    described_class.instance_variable_set('@timeout', nil)
+    described_class.instance_variable_set(:@cache, nil)
+    described_class.instance_variable_set(:@timeout, nil)
   end
 
-  it 'defaults to ephemeral cache' do
+  it 'defaults to an in-memory cache' do
     with_env 'CACHE', nil do
-      expect(described_class.cache.class).to be(SiteInspector::Cache)
+      expect(described_class.cache).to be_a(Typhoeus::Cache::Rails)
+      store = described_class.cache.instance_variable_get(:@cache)
+      expect(store).to be_a(ActiveSupport::Cache::MemoryStore)
     end
   end
 
-  it 'uses disk cache when requested' do
-    with_env 'CACHE', '/tmp' do
-      expect(described_class.cache.class).to be(SiteInspector::DiskCache)
+  it 'uses a file cache when requested' do
+    with_env 'CACHE', tmpdir do
+      expect(described_class.cache).to be_a(Typhoeus::Cache::Rails)
+      store = described_class.cache.instance_variable_get(:@cache)
+      expect(store).to be_a(ActiveSupport::Cache::FileStore)
+      expect(store.cache_path).to eql(tmpdir)
     end
   end
 
+  context 'with the response cache' do
+    let(:url) { 'https://example.com/' }
+    let(:response) do
+      Typhoeus::Response.new(code: 200, body: 'content', headers: { 'Server' => 'nginx' })
+    end
+
+    before { FileUtils.rm_rf(tmpdir) }
+
+    after { FileUtils.rm_rf(tmpdir) }
+
+    {
+      'memory' => -> { ActiveSupport::Cache::MemoryStore.new },
+      'file' => -> { ActiveSupport::Cache::FileStore.new(tmpdir) }
+    }.each do |name, store|
+      it "round-trips a response through the #{name} store using an equal request" do
+        cache = Typhoeus::Cache::Rails.new(store.call)
+        request = Typhoeus::Request.new(url, described_class.typhoeus_defaults)
+        request.finish(response)
+        cache.set(request, response)
+
+        cached = cache.get(Typhoeus::Request.new(url, described_class.typhoeus_defaults))
+        expect(cached).to be_a(Typhoeus::Response)
+        expect(cached.code).to be(200)
+        expect(cached.body).to eql('content')
+      end
+    end
+
+    it 'misses for a different request' do
+      cache = Typhoeus::Cache::Rails.new(ActiveSupport::Cache::MemoryStore.new)
+      cache.set(Typhoeus::Request.new(url, described_class.typhoeus_defaults), response)
+
+      expect(cache.get(Typhoeus::Request.new('https://example.org/', described_class.typhoeus_defaults))).to be_nil
+    end
+  end
+
+  it 'restores the environment when a with_env block raises' do
+    ENV['SITE_INSPECTOR_TEST_ENV'] = 'before'
+    expect do
+      with_env('SITE_INSPECTOR_TEST_ENV', 'during') { raise 'boom' }
+    end.to raise_error('boom')
+    expect(ENV.fetch('SITE_INSPECTOR_TEST_ENV')).to eql('before')
+  ensure
+    ENV.delete('SITE_INSPECTOR_TEST_ENV')
+  end
+
   it 'returns the default timeout' do
     expect(described_class.timeout).to be(10)
   end
@@ -38,10 +88,7 @@ describe SiteInspector do
       accept_encoding: 'gzip',
       followlocation: false,
       method: :head,
-      timeout: 10,
-      headers: {
-        'User-Agent' => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
-      }
+      timeout: 10
     }
     expect(described_class.typhoeus_defaults).to eql(expected)
   end

data/spec/spec_helper.rb

@@ -3,14 +3,21 @@
 require 'bundler/setup'
 require 'webmock/rspec'
 require 'fileutils'
-require_relative '../lib/site-inspector'
+require 'site-inspector'
 
 WebMock.disable_net_connect!
 
+# Specs tagged :live hit real DNS and WHOIS servers, whose answers change over
+# time. Skip them unless LIVE=1 is set.
+RSpec.configure do |config|
+  config.filter_run_excluding :live unless ENV['LIVE']
+end
+
 def with_env(key, value)
-  old_env = ENV[key]
+  old_env = ENV.fetch(key, nil)
   ENV[key] = value
   yield
+ensure
   ENV[key] = old_env
 end
 

metadata

@@ -1,15 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: site-inspector
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Ben Balter
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: cliver
   requirement: !ruby/object:Gem::Requirement
@@ -39,137 +52,143 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: dnsruby
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: dotenv
+  name: dnsruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: gman
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '7.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.0.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '7.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.0.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
-  name: mercenary
+  name: http-cookie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: mercenary
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: oj
+  name: naughty_or_nice
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: public_suffix
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: sniffles
+  name: public_suffix
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
@@ -288,14 +307,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -317,11 +336,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
 - ".ruby-version"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Guardfile
 - LICENSE
@@ -331,55 +351,61 @@ files:
 - lib/cliver/dependency_ext.rb
 - lib/data/cdn.yml
 - lib/data/cloud.yml
+- lib/data/well-known.yml
 - lib/site-inspector.rb
-- lib/site-inspector/cache.rb
-- lib/site-inspector/checks/accessibility.rb
-- lib/site-inspector/checks/check.rb
-- lib/site-inspector/checks/content.rb
-- lib/site-inspector/checks/cookies.rb
-- lib/site-inspector/checks/dns.rb
-- lib/site-inspector/checks/headers.rb
-- lib/site-inspector/checks/hsts.rb
-- lib/site-inspector/checks/https.rb
-- lib/site-inspector/checks/sniffer.rb
-- lib/site-inspector/checks/wappalyzer.rb
-- lib/site-inspector/checks/whois.rb
-- lib/site-inspector/disk_cache.rb
 - lib/site-inspector/domain.rb
+- lib/site-inspector/domain_parser.rb
 - lib/site-inspector/endpoint.rb
-- lib/site-inspector/rails_cache.rb
+- lib/site-inspector/endpoint/accessibility.rb
+- lib/site-inspector/endpoint/check.rb
+- lib/site-inspector/endpoint/content.rb
+- lib/site-inspector/endpoint/cookies.rb
+- lib/site-inspector/endpoint/dns.rb
+- lib/site-inspector/endpoint/headers.rb
+- lib/site-inspector/endpoint/hsts.rb
+- lib/site-inspector/endpoint/https.rb
+- lib/site-inspector/endpoint/wappalyzer.rb
+- lib/site-inspector/endpoint/well_known.rb
+- lib/site-inspector/endpoint/whois.rb
+- lib/site-inspector/formatter.rb
 - lib/site-inspector/version.rb
 - package-lock.json
 - package.json
+- renovate.json
+- script/benchmark
 - script/bootstrap
 - script/cibuild
 - script/console
 - script/pa11y-version
 - script/release
+- script/update-well-known
 - site-inspector.gemspec
-- spec/checks/site_inspector_endpoint_accessibility_spec.rb
-- spec/checks/site_inspector_endpoint_check_spec.rb
-- spec/checks/site_inspector_endpoint_content_spec.rb
-- spec/checks/site_inspector_endpoint_cookies_spec.rb
-- spec/checks/site_inspector_endpoint_dns_spec.rb
-- spec/checks/site_inspector_endpoint_headers_spec.rb
-- spec/checks/site_inspector_endpoint_hsts_spec.rb
-- spec/checks/site_inspector_endpoint_https_spec.rb
-- spec/checks/site_inspector_endpoint_sniffer_spec.rb
-- spec/checks/site_inspector_endpoint_wappalyzer_spec.rb
-- spec/checks/site_inspector_endpoint_whois_spec.rb
+- spec/cliver/dependency_ext_spec.rb
 - spec/fixtures/wappalyzer.json
-- spec/site_inspector_cache_spec.rb
-- spec/site_inspector_disk_cache_spec.rb
-- spec/site_inspector_domain_spec.rb
-- spec/site_inspector_endpoint_spec.rb
+- spec/site_inspector/domain_spec.rb
+- spec/site_inspector/endpoint/accessibility_spec.rb
+- spec/site_inspector/endpoint/check_spec.rb
+- spec/site_inspector/endpoint/content_spec.rb
+- spec/site_inspector/endpoint/cookies_spec.rb
+- spec/site_inspector/endpoint/dns_spec.rb
+- spec/site_inspector/endpoint/headers_spec.rb
+- spec/site_inspector/endpoint/hsts_spec.rb
+- spec/site_inspector/endpoint/https_spec.rb
+- spec/site_inspector/endpoint/wappalyzer_spec.rb
+- spec/site_inspector/endpoint/well_known_spec.rb
+- spec/site_inspector/endpoint/whois_spec.rb
+- spec/site_inspector/endpoint_spec.rb
 - spec/site_inspector_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/benbalter/site-inspector
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+metadata:
+  rubygems_mfa_required: 'true'
+  homepage_uri: https://github.com/benbalter/site-inspector
+  source_code_uri: https://github.com/benbalter/site-inspector
+  bug_tracker_uri: https://github.com/benbalter/site-inspector/issues
+  changelog_uri: https://github.com/benbalter/site-inspector/releases
 rdoc_options: []
 require_paths:
 - lib
@@ -387,33 +413,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
-summary: A Ruby port and v2 of Site Inspector (https://github.com/benbalter/site-inspector)
-test_files:
-- spec/checks/site_inspector_endpoint_accessibility_spec.rb
-- spec/checks/site_inspector_endpoint_check_spec.rb
-- spec/checks/site_inspector_endpoint_content_spec.rb
-- spec/checks/site_inspector_endpoint_cookies_spec.rb
-- spec/checks/site_inspector_endpoint_dns_spec.rb
-- spec/checks/site_inspector_endpoint_headers_spec.rb
-- spec/checks/site_inspector_endpoint_hsts_spec.rb
-- spec/checks/site_inspector_endpoint_https_spec.rb
-- spec/checks/site_inspector_endpoint_sniffer_spec.rb
-- spec/checks/site_inspector_endpoint_wappalyzer_spec.rb
-- spec/checks/site_inspector_endpoint_whois_spec.rb
-- spec/fixtures/wappalyzer.json
-- spec/site_inspector_cache_spec.rb
-- spec/site_inspector_disk_cache_spec.rb
-- spec/site_inspector_domain_spec.rb
-- spec/site_inspector_endpoint_spec.rb
-- spec/site_inspector_spec.rb
-- spec/spec_helper.rb
+summary: Ruby gem and CLI that checks a domain's HTTPS, HSTS, DNS, headers, cookies,
+  WHOIS, accessibility, and tech stack
+test_files: []

data/.travis.yml

@@ -1,9 +0,0 @@
-language: ruby
-rvm:
-  - 2.1
-script: "script/cibuild"
-sudo: false
-cache:
-  - bundler
-  - node_modules
-before_script: npm install

data/lib/site-inspector/cache.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class Cache
-    def memory
-      @memory ||= {}
-    end
-
-    def get(request)
-      memory[request]
-    end
-
-    def set(request, response)
-      memory[request] = response
-    end
-  end
-end