site-inspector 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b994e74251bb7b1be968d5bbad80273ee3ffea24
-  data.tar.gz: e104a5856e1710233df4da02b234935e56910f3b
+  metadata.gz: 56a9e64300435fa494e8ae17ca710b46238bcffd
+  data.tar.gz: 1e552e430d8f1fea10eba7d873e1ae17aab00415
 SHA512:
-  metadata.gz: c68e92c9e4c7b82aa7f19fb56f625f0656104b026ef81ede02b8f9dff90b430fc1af16062bb0fa8d337ad53f410ee563c127de9c594a3b47ce4a93723b668b90
-  data.tar.gz: 2e07f88d11a2c116289bfeb3e9050ad59b7314523aedeb45cf7d4b4042912c31e2ac4e80828ed27a272cec8952726ed02735c2af4022821255d3685c38478c61
+  metadata.gz: 96eaaa8846783ef2b9dfe5b76e84ebd3c160139c425c69edc25761828690b2d508770aa842e234cbb63e9f6ecaa8bb7ffc7b2121a9449d934928938a9a9ec637
+  data.tar.gz: b3b4d86a6b6ead26b17eb380b27a5da37afd645c582b94b53860eb1948e28c8cf6baec1c088ace2a53ebdb4e755a316bb635236b9b46c36cb60f1012581c5500

data/lib/site-inspector/checks/accessibility.rb

@@ -24,7 +24,10 @@ class SiteInspector
         def pa11y?
           !!(Cliver.detect('pa11y'))
         end
-        alias_method :enabled?, :pa11y?
+
+        def enabled?
+          @@enabled && pa11y?
+        end
       end
 
       def level

data/lib/site-inspector/checks/check.rb

@@ -33,12 +33,21 @@ class SiteInspector
         self.class.name
       end
 
-      def self.name
-        self.to_s.split('::').last.downcase.to_sym
-      end
+      class << self
+
+        @@enabled = true
+
+        def name
+          self.to_s.split('::').last.downcase.to_sym
+        end
+
+        def enabled?
+          !!(@@enabled)
+        end
 
-      def self.enabled?
-        true
+        def enabled=(value)
+          @@enabled = !!(value)
+        end
       end
     end
   end

data/lib/site-inspector/checks/content.rb

@@ -6,6 +6,12 @@ class SiteInspector
         endpoint.up? && endpoint.request(path: path, followlocation: true).success?
       end
 
+      # The default Check#response method is from a HEAD request
+      # The content check has a special response which includes the body from a GET request
+      def response
+        @response ||= endpoint.request(:method => :get)
+      end
+
       def document
         require 'nokogiri'
         @doc ||= Nokogiri::HTML response.body if response

data/lib/site-inspector/checks/dns.rb

@@ -1,6 +1,7 @@
 class SiteInspector
   class Endpoint
     class Dns < Check
+      class LocalhostError < StandardError; end
 
       def self.resolver
         require "dnsruby"
@@ -55,6 +56,10 @@ class SiteInspector
         end
       end
 
+      def localhost?
+        ip == '127.0.0.1'
+      end
+
       def ip
         require 'resolv'
         @ip ||= Resolv.getaddress host
@@ -80,6 +85,7 @@ class SiteInspector
       end
 
       def to_h
+        return { :error => LocalhostError } if localhost?
         {
           :dnssec => dnssec?,
           :ipv6   => ipv6?,

data/lib/site-inspector/checks/sniffer.rb

@@ -21,6 +21,7 @@ class SiteInspector
         return cms unless cms.nil?
         return :expression_engine if endpoint.cookies.any? { |c| c.keys.first =~ /^exp_/ }
         return :php if endpoint.cookies["PHPSESSID"]
+        return :coldfusion if endpoint.cookies["CFID"] && endpoint.cookies["CFTOKEN"]
         return :cowboy if endpoint.headers.server.to_s.downcase == "cowboy"
         nil
       end

data/lib/site-inspector/domain.rb

@@ -14,16 +14,19 @@ class SiteInspector
 
     def endpoints
       @endpoints ||= [
-        Endpoint.new("https://#{host}"),
-        Endpoint.new("https://www.#{host}"),
-        Endpoint.new("http://#{host}"),
-        Endpoint.new("http://www.#{host}")
+        Endpoint.new("https://#{host}", :domain => self),
+        Endpoint.new("https://www.#{host}", :domain => self),
+        Endpoint.new("http://#{host}", :domain => self),
+        Endpoint.new("http://www.#{host}", :domain => self)
       ]
     end
 
     def canonical_endpoint
-      @canonical_endpoint ||= endpoints.find do |e|
-        e.https? == canonically_https? && e.www? == canonically_www?
+      @canonical_endpoint ||= begin
+        prefetch
+        endpoints.find do |e|
+          e.https? == canonically_https? && e.www? == canonically_www?
+        end
       end
     end
 

data/lib/site-inspector/endpoint.rb

@@ -10,15 +10,21 @@ class SiteInspector
   # Because each of the four endpoints could potentially respond differently
   # We must evaluate all four to make certain determination
   class Endpoint
-    attr_accessor :host, :uri
+    attr_accessor :host, :uri, :domain
 
     # Initatiate a new Endpoint object
     #
     # endpoint - (string) the endpoint to query (e.g., `https://example.com`)
-    def initialize(host)
+    # options  - A hash of options
+    #   domain - the parent domain object, if passed, facilitates caching of redirects
+    def initialize(host, options={})
       @uri = Addressable::URI.parse(host.downcase)
+      # The root URL always has an implict path of "/", even if not requested
+      # Make it explicit to facilitate caching and prevent a potential redirect
+      @uri.path = "/"
       @host = uri.host.sub(/^www\./, "")
       @checks = {}
+      @domain = options[:domain]
     end
 
     def www?
@@ -92,7 +98,7 @@ class SiteInspector
         return if redirect.host == host && redirect.scheme == scheme
 
         # Init a new endpoint representing the redirect
-        Endpoint.new(redirect.to_s)
+        find_or_create_by_uri(redirect.to_s)
       end
     end
 
@@ -104,6 +110,11 @@ class SiteInspector
     # What's the effective URL of a request to this domain?
     def resolves_to
       return self unless redirect?
+
+      # If the redirect doesn't return a 30x response code, return the redirected endpoint
+      # Otherwise, we'll need to go down the rabbit hole and see how deep it goes
+      return redirect unless redirect.redirect?
+
       @resolves_to ||= begin
         response = request(:followlocation => true)
 
@@ -118,7 +129,7 @@ class SiteInspector
           url = response.effective_url
         end
 
-        Endpoint.new(url)
+        find_or_create_by_uri(url)
       end
     end
 
@@ -161,7 +172,7 @@ class SiteInspector
       checks = SiteInspector::Endpoint.checks.select { |c| options.keys.include?(c.name) }
       checks = SiteInspector::Endpoint.checks if checks.empty?
 
-      checks.each do |check|
+      Parallel.each(checks, :in_threads => 4) do |check|
         hash[check.name] = self.send(check.name).to_h
       end
 
@@ -193,5 +204,16 @@ class SiteInspector
     def hydra
       SiteInspector.hydra
     end
+
+    # In the event that a redirect is to one of the domain's four endpoints,
+    # Try to return the existing endpoint, rather than create a new one
+    def find_or_create_by_uri(uri)
+      uri = Addressable::URI.parse(uri.downcase)
+      if domain && cached_endpoint = domain.endpoints.find { |e| e.uri.to_s == uri.to_s }
+        cached_endpoint
+      else
+        Endpoint.new(uri.to_s)
+      end
+    end
   end
 end

data/lib/site-inspector/version.rb

@@ -1,3 +1,3 @@
 class SiteInspector
-  VERSION = "3.0.0"
+  VERSION = "3.1.0"
 end

data/lib/site-inspector.rb

@@ -2,6 +2,7 @@ require 'open-uri'
 require 'addressable/uri'
 require 'public_suffix'
 require 'typhoeus'
+require 'parallel'
 require 'cliver'
 require 'cgi'
 
@@ -49,17 +50,25 @@ class SiteInspector
         :followlocation => false,
         :timeout => SiteInspector.timeout,
         :accept_encoding => "gzip",
+        :method => :head,
         :headers => {
           "User-Agent" => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"
         }
       }
     end
 
+    # Returns a thread-safe, memoized hydra instance
     def hydra
-      @hydra ||= Typhoeus::Hydra.new(max_concurrency: 4)
+      Typhoeus::Hydra.hydra
     end
   end
 end
 
+if ENV["DEBUG"]
+  Ethon.logger = Logger.new(STDOUT);
+  Ethon.logger.level = Logger::DEBUG
+  Typhoeus::Config.verbose = true
+end
+
 Typhoeus::Config.memoize = true
 Typhoeus::Config.cache = SiteInspector.cache

data/script/console

@@ -1 +1,3 @@
-bundle exec pry -r ./lib/site-inspector.rb
+#! /bin/sh
+
+DEBUG=1 bundle exec pry -r './lib/site-inspector'

data/site-inspector.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_dependency("mercenary", "~> 0.3")
   s.add_dependency("colorator", "~> 0.1")
   s.add_dependency("cliver", "~> 0.3")
+  s.add_dependency("parallel", "~> 1.6")
   s.add_development_dependency("pry", "~> 0.10")
   s.add_development_dependency( "rake", "~> 10.4" )
   s.add_development_dependency( "rspec", "~> 3.2")

data/spec/checks/site_inspector_endpoint_accessibility_spec.rb

@@ -44,7 +44,7 @@ describe SiteInspector::Endpoint::Accessibility do
   context "with pa11y installed" do
 
     before do
-      stub_request(:get, "http://example.com/").to_return(:status => 200 )
+      stub_request(:head, "http://example.com/").to_return(:status => 200 )
     end
 
   end

data/spec/checks/site_inspector_endpoint_check_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe SiteInspector::Endpoint::Check do
 
   subject do
-    stub_request(:get, "http://example.com/").to_return(:status => 200)
+    stub_request(:head, "http://example.com/").to_return(:status => 200)
     endpoint = SiteInspector::Endpoint.new("http://example.com")
     SiteInspector::Endpoint::Check.new(endpoint)
   end
@@ -32,7 +32,10 @@ describe SiteInspector::Endpoint::Check do
     expect(SiteInspector::Endpoint::Check.name).to eql(:check)
   end
 
-  it "enables the check" do
+  it "enables and disables the check" do
     expect(SiteInspector::Endpoint::Check.enabled?).to eql(true)
+    SiteInspector::Endpoint::Check.enabled = false
+    expect(SiteInspector::Endpoint::Check.enabled?).to eql(false)
+    SiteInspector::Endpoint::Check.enabled = true
   end
 end

data/spec/checks/site_inspector_endpoint_content_spec.rb

@@ -14,6 +14,8 @@ describe SiteInspector::Endpoint::Content do
 
     stub_request(:get, "http://example.com/").
       to_return(:status => 200, :body => body )
+      stub_request(:head, "http://example.com/").
+        to_return(:status => 200 )
     endpoint = SiteInspector::Endpoint.new("http://example.com")
     SiteInspector::Endpoint::Content.new(endpoint)
   end
@@ -32,62 +34,62 @@ describe SiteInspector::Endpoint::Content do
   end
 
   it "knows when robots.txt exists" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
+    stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
 
-    stub_request(:get, "http://example.com/robots.txt").
+    stub_request(:head, "http://example.com/robots.txt").
       to_return(:status => 200)
     expect(subject.robots_txt?).to eql(true)
   end
 
   it "knows when robots.txt doesn't exist" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
+    stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
 
-    stub_request(:get, "http://example.com/robots.txt").
+    stub_request(:head, "http://example.com/robots.txt").
       to_return(:status => 404)
     expect(subject.robots_txt?).to eql(false)
   end
 
   it "knows when sitemap.xml exists" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
+    stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
 
-    stub_request(:get, "http://example.com/sitemap.xml").
+    stub_request(:head, "http://example.com/sitemap.xml").
       to_return(:status => 200)
     expect(subject.sitemap_xml?).to eql(true)
   end
 
   it "knows when sitemap.xml exists" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
+    stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
 
-    stub_request(:get, "http://example.com/sitemap.xml").
+    stub_request(:head, "http://example.com/sitemap.xml").
       to_return(:status => 404)
     expect(subject.sitemap_xml?).to eql(false)
   end
 
   it "knows when humans.txt exists" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
+    stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
 
-    stub_request(:get, "http://example.com/humans.txt").
+    stub_request(:head, "http://example.com/humans.txt").
       to_return(:status => 200)
     expect(subject.humans_txt?).to eql(true)
   end
 
   it "knows when humans.txt doesn't exist" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
+    stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 404)
 
-    stub_request(:get, "http://example.com/humans.txt").
+    stub_request(:head, "http://example.com/humans.txt").
       to_return(:status => 200)
     expect(subject.humans_txt?).to eql(true)
   end
 
   context "404s" do
     it "knows when an endpoint returns a proper 404" do
-      stub_request(:get, /http\:\/\/example.com\/.*/).
+      stub_request(:head, /http\:\/\/example.com\/.*/).
         to_return(:status => 404)
       expect(subject.proper_404s?).to eql(true)
     end
 
     it "knows when an endpoint doesn't return a proper 404" do
-      stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).
+      stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).
         to_return(:status => 200)
       expect(subject.proper_404s?).to eql(false)
     end
@@ -99,8 +101,8 @@ describe SiteInspector::Endpoint::Content do
     end
 
     it "doesn't say something exists when there are no 404s" do
-      stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 200)
-      stub_request(:get, "http://example.com/humans.txt").to_return(:status => 200)
+      stub_request(:head, /http\:\/\/example.com\/[a-z0-9]{32}/i).to_return(:status => 200)
+      stub_request(:head, "http://example.com/humans.txt").to_return(:status => 200)
       expect(subject.humans_txt?).to eql(nil)
     end
   end

data/spec/checks/site_inspector_endpoint_cookies_spec.rb

@@ -4,7 +4,7 @@ describe SiteInspector::Endpoint::Cookies do
 
   context "without cookies" do
     subject do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 200, :body => "" )
       endpoint = SiteInspector::Endpoint.new("http://example.com")
       SiteInspector::Endpoint::Cookies.new(endpoint)
@@ -33,7 +33,7 @@ describe SiteInspector::Endpoint::Cookies do
         )
       ].map { |c| c.to_s }
 
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 200, :body => "", :headers => { "set-cookie" => cookies })
       endpoint = SiteInspector::Endpoint.new("http://example.com")
       SiteInspector::Endpoint::Cookies.new(endpoint)
@@ -59,7 +59,7 @@ describe SiteInspector::Endpoint::Cookies do
         "foo=bar; domain=example.com; path=/; secure; HttpOnly",
         "foo2=bar2; domain=example.com; path=/"
       ]
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 200, :body => "", :headers => { "set-cookie" => cookies })
       endpoint = SiteInspector::Endpoint.new("http://example.com")
       SiteInspector::Endpoint::Cookies.new(endpoint)

data/spec/checks/site_inspector_endpoint_dns_spec.rb

@@ -4,7 +4,7 @@ require 'dnsruby'
 describe SiteInspector::Endpoint::Dns do
 
   subject do
-    stub_request(:get, "http://github.com/").to_return(:status => 200)
+    stub_request(:head, "http://github.com/").to_return(:status => 200)
     endpoint = SiteInspector::Endpoint.new("http://github.com")
     SiteInspector::Endpoint::Dns.new(endpoint)
   end
@@ -81,6 +81,10 @@ describe SiteInspector::Endpoint::Dns do
       expect(subject.ipv6?).to eql(true)
     end
 
+    it "knows it's not a localhost address" do
+      expect(subject.localhost?).to eql(false)
+    end
+
     context "hostname detection" do
       it "lists cnames" do
         records = []
@@ -164,4 +168,20 @@ describe SiteInspector::Endpoint::Dns do
       end
     end
   end
+
+  context "localhost" do
+
+    before do
+      allow(subject).to receive(:ip) { "127.0.0.1" }
+    end
+
+    it "knows it's a localhost address" do
+      expect(subject.localhost?).to eql(true)
+    end
+
+    it "returns a LocalhostError" do
+      expect(subject.to_h).to eql({:error => SiteInspector::Endpoint::Dns::LocalhostError})
+    end
+
+  end
 end

data/spec/checks/site_inspector_endpoint_headers_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe SiteInspector::Endpoint::Headers do
 
   subject do
-    stub_request(:get, "http://example.com/").
+    stub_request(:head, "http://example.com/").
       to_return(:status => 200, :headers => { :foo => "bar" } )
     endpoint = SiteInspector::Endpoint.new("http://example.com")
     SiteInspector::Endpoint::Headers.new(endpoint)

data/spec/checks/site_inspector_endpoint_hsts_spec.rb

@@ -4,7 +4,7 @@ describe SiteInspector::Endpoint::Hsts do
 
   subject do
     headers = { "strict-transport-security" => "max-age=31536000; includeSubDomains;" }
-    stub_request(:get, "http://example.com/").
+    stub_request(:head, "http://example.com/").
       to_return(:status => 200, :headers => headers )
     endpoint = SiteInspector::Endpoint.new("http://example.com")
     SiteInspector::Endpoint::Hsts.new(endpoint)

data/spec/checks/site_inspector_endpoint_https_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe SiteInspector::Endpoint::Https do
 
   subject do
-    stub_request(:get, "https://example.com/").
+    stub_request(:head, "https://example.com/").
       to_return(:status => 200 )
     endpoint = SiteInspector::Endpoint.new("https://example.com")
     allow(endpoint.response).to receive(:return_code) { :ok }

data/spec/checks/site_inspector_endpoint_sniffer_spec.rb

@@ -23,7 +23,10 @@ describe SiteInspector::Endpoint::Sniffer do
     ].map { |c| c.to_s }
 
     stub_request(:get, "http://example.com/").
-      to_return(:status => 200, :body => "", :headers => { "set-cookie" => cookies } )
+      to_return(:status => 200, :body => "" )
+
+    stub_request(:head, "http://example.com/").
+      to_return(:status => 200, :headers => { "set-cookie" => cookies } )
   end
 
   context "stubbed body" do
@@ -50,6 +53,9 @@ describe SiteInspector::Endpoint::Sniffer do
 
       stub_request(:get, "http://example.com/").
         to_return(:status => 200, :body => body )
+
+      stub_request(:head, "http://example.com/").
+          to_return(:status => 200)
       endpoint = SiteInspector::Endpoint.new("http://example.com")
       SiteInspector::Endpoint::Sniffer.new(endpoint)
     end
@@ -105,10 +111,39 @@ describe SiteInspector::Endpoint::Sniffer do
 
     it "detects cowboy" do
       stub_request(:get, "http://example.com/").
-        to_return(:status => 200, :body => "", :headers => { "server" => "Cowboy" } )
+        to_return(:status => 200, :body => "" )
+
+      stub_request(:head, "http://example.com/").
+        to_return(:status => 200, :headers => { "server" => "Cowboy" } )
 
       expect(subject.framework).to eql(:cowboy)
       expect(subject.open_source?).to eql(true)
     end
+
+    it "detects ColdFusion" do
+      cookies = [
+        CGI::Cookie::new(
+          "name" => "CFID",
+          "value" => "1234",
+          "domain" => "example.com",
+          "path" => "/"
+        ),
+        CGI::Cookie::new(
+          "name" => "CFTOKEN",
+          "value" => "5678",
+          "domain" => "example.com",
+          "path" => "/"
+        )
+      ].map { |c| c.to_s }
+
+      stub_request(:get, "http://example.com/").
+        to_return(:status => 200, :body => "" )
+        
+      stub_request(:head, "http://example.com/").
+        to_return(:status => 200, :headers => { "set-cookie" => cookies } )
+
+      expect(subject.framework).to eql(:coldfusion)
+      expect(subject.open_source?).to eql(false)
+    end
   end
 end

data/spec/site_inspector_domain_spec.rb

@@ -45,19 +45,19 @@ describe SiteInspector::Domain do
     it "generates the endpoints" do
       endpoints = subject.endpoints
       expect(endpoints.count).to eql(4)
-      expect(endpoints[0].to_s).to eql("https://example.com")
-      expect(endpoints[1].to_s).to eql("https://www.example.com")
-      expect(endpoints[2].to_s).to eql("http://example.com")
-      expect(endpoints[3].to_s).to eql("http://www.example.com")
+      expect(endpoints[0].to_s).to eql("https://example.com/")
+      expect(endpoints[1].to_s).to eql("https://www.example.com/")
+      expect(endpoints[2].to_s).to eql("http://example.com/")
+      expect(endpoints[3].to_s).to eql("http://www.example.com/")
     end
   end
 
   it "knows the canonical domain" do
-    stub_request(:get, "https://example.com/").to_return(:status => 500)
-    stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-    stub_request(:get, "http://www.example.com/").to_return(:status => 200)
-    stub_request(:get, "http://example.com/").to_return(:status => 200)
-    expect(subject.canonical_endpoint.to_s).to eql("http://example.com")
+    stub_request(:head, "https://example.com/").to_return(:status => 500)
+    stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+    stub_request(:head, "http://www.example.com/").to_return(:status => 200)
+    stub_request(:head, "http://example.com/").to_return(:status => 200)
+    expect(subject.canonical_endpoint.to_s).to eql("http://example.com/")
   end
 
   it "knows if a domain is a government domain" do
@@ -75,7 +75,7 @@ describe SiteInspector::Domain do
         end
       end
 
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
       expect(subject.up?).to eql(true)
     end
@@ -91,19 +91,19 @@ describe SiteInspector::Domain do
 
   context "up" do
     it "considers a domain up if at least one endpoint is up" do
-      stub_request(:get, "https://example.com/").to_return(:status => 500)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "https://example.com/").to_return(:status => 500)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
       expect(subject.up?).to eql(true)
     end
 
     it "doesn't consider a domain up if all endpoints are down" do
-      stub_request(:get, "https://example.com/").to_return(:status => 500)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "https://example.com/").to_return(:status => 500)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 500)
 
       expect(subject.up?).to eql(false)
     end
@@ -111,19 +111,19 @@ describe SiteInspector::Domain do
 
   context "www" do
     it "considers a site www when at least one endpoint is www" do
-      stub_request(:get, "https://example.com/").to_return(:status => 200)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "https://example.com/").to_return(:status => 200)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
       expect(subject.www?).to eql(true)
     end
 
     it "doesn't consider a site www when no endpoint is www" do
-      stub_request(:get, "https://example.com/").to_return(:status => 200)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 200)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "https://example.com/").to_return(:status => 200)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 200)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 500)
 
       expect(subject.www?).to eql(false)
     end
@@ -131,19 +131,19 @@ describe SiteInspector::Domain do
 
   context "root" do
     it "considers a domain root if you can connect without www" do
-      stub_request(:get, "https://example.com/").to_return(:status => 200)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "https://example.com/").to_return(:status => 200)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 500)
 
       expect(subject.root?).to eql(true)
     end
 
     it "doesn't call a www-only domain root" do
-      stub_request(:get, "https://example.com/").to_return(:status => 500)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 200)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "https://example.com/").to_return(:status => 500)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
       expect(subject.root?).to eql(false)
     end
@@ -151,38 +151,38 @@ describe SiteInspector::Domain do
 
   context "https" do
     it "knows when a domain supports https" do
-      stub_request(:get, "https://example.com/").to_return(:status => 200)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 200)
-      stub_request(:get, "http://example.com/").to_return(:status => 200)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "https://example.com/").to_return(:status => 200)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "http://example.com/").to_return(:status => 200)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
       allow(subject.endpoints.first.https).to receive(:valid?) { true }
 
       expect(subject.https?).to eql(true)
     end
 
     it "knows when a domain doesn't support https" do
-      stub_request(:get, "https://example.com/").to_return(:status => 500)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 200)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "https://example.com/").to_return(:status => 500)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 200)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
       expect(subject.https?).to eql(false)
     end
 
     it "considers HTTPS inforced when no http endpoint responds" do
-      stub_request(:get, "https://example.com/").to_return(:status => 200)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "https://example.com/").to_return(:status => 200)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 500)
 
       #expect(subject.enforces_https?).to eql(true)
     end
 
     it "doesn't consider HTTPS inforced when an http endpoint responds" do
-      stub_request(:get, "https://example.com/").to_return(:status => 200)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").to_return(:status => 500)
-      stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+      stub_request(:head, "https://example.com/").to_return(:status => 200)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
       expect(subject.enforces_https?).to eql(false)
     end
@@ -199,20 +199,20 @@ describe SiteInspector::Domain do
   context "canonical" do
     context "www" do
       it "detects a domain as canonically www when root is down" do
-        stub_request(:get, "https://example.com/").to_return(:status => 500)
-        stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-        stub_request(:get, "http://example.com/").to_return(:status => 500)
-        stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+        stub_request(:head, "https://example.com/").to_return(:status => 500)
+        stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+        stub_request(:head, "http://example.com/").to_return(:status => 500)
+        stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
         expect(subject.canonically_www?).to eql(true)
       end
 
       it "detects a domain as canonically www when root redirects" do
-        stub_request(:get, "https://example.com/").to_return(:status => 500)
-        stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-        stub_request(:get, "http://example.com/").
+        stub_request(:head, "https://example.com/").to_return(:status => 500)
+        stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+        stub_request(:head, "http://example.com/").
           to_return(:status => 301, :headers => { :location => "http://www.example.com" } )
-        stub_request(:get, "http://www.example.com/").to_return(:status => 200)
+        stub_request(:head, "http://www.example.com/").to_return(:status => 200)
 
         expect(subject.canonically_www?).to eql(true)
       end
@@ -220,21 +220,21 @@ describe SiteInspector::Domain do
 
     context "https" do
       it "detects a domain as canonically https when http is down" do
-        stub_request(:get, "https://example.com/").to_return(:status => 200)
-        stub_request(:get, "https://www.example.com/").to_return(:status => 200)
-        stub_request(:get, "http://example.com/").to_return(:status => 500)
-        stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+        stub_request(:head, "https://example.com/").to_return(:status => 200)
+        stub_request(:head, "https://www.example.com/").to_return(:status => 200)
+        stub_request(:head, "http://example.com/").to_return(:status => 500)
+        stub_request(:head, "http://www.example.com/").to_return(:status => 500)
         allow(subject.endpoints.first.https).to receive(:valid?) { true }
 
         expect(subject.canonically_https?).to eql(true)
       end
 
       it "detects a domain as canonically https when http redirect" do
-        stub_request(:get, "https://example.com/").to_return(:status => 200)
-        stub_request(:get, "https://www.example.com/").to_return(:status => 200)
-        stub_request(:get, "http://example.com/").
+        stub_request(:head, "https://example.com/").to_return(:status => 200)
+        stub_request(:head, "https://www.example.com/").to_return(:status => 200)
+        stub_request(:head, "http://example.com/").
           to_return(:status => 301, :headers => { :location => "https://example.com" } )
-        stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+        stub_request(:head, "http://www.example.com/").to_return(:status => 500)
         allow(subject.endpoints.first.https).to receive(:valid?) { true }
 
         expect(subject.canonically_https?).to eql(true)
@@ -244,11 +244,12 @@ describe SiteInspector::Domain do
 
   context "redirects" do
     it "knows when a domain redirects" do
-      stub_request(:get, "https://example.com/").to_return(:status => 500)
-      stub_request(:get, "https://www.example.com/").to_return(:status => 500)
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "https://example.com/").to_return(:status => 500)
+      stub_request(:head, "https://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://example.com/").
         to_return(:status => 301, :headers => { :location => "http://foo.example.com" } )
-      stub_request(:get, "http://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://www.example.com/").to_return(:status => 500)
+      stub_request(:head, "http://foo.example.com/").to_return(:status => 200)
 
       expect(subject.redirect?).to eql(true)
     end

data/spec/site_inspector_endpoint_spec.rb

@@ -15,7 +15,7 @@ describe SiteInspector::Endpoint do
   end
 
   it "returns the uri" do
-    expect(subject.uri.to_s).to eql("http://example.com")
+    expect(subject.uri.to_s).to eql("http://example.com/")
   end
 
   it "knows if an endpoint is www" do
@@ -28,10 +28,10 @@ describe SiteInspector::Endpoint do
   end
 
   it "knows if an endpoint is http" do
-    stub_request(:get, "http://example.com/").
+    stub_request(:head, "http://example.com/").
       to_return(:status => 200, :body => "content")
 
-    stub_request(:get, "https://example.com/").
+    stub_request(:head, "https://example.com/").
       to_return(:status => 500, :body => "content")
 
     expect(subject.https?).to eql(false)
@@ -39,10 +39,10 @@ describe SiteInspector::Endpoint do
   end
 
   it "knows if an endpoint is https" do
-    stub_request(:get, "http://example.com/").
+    stub_request(:head, "http://example.com/").
       to_return(:status => 200, :body => "content")
 
-    stub_request(:get, "https://example.com/").
+    stub_request(:head, "https://example.com/").
       to_return(:status => 200, :body => "content")
 
     endpoint = SiteInspector::Endpoint.new("https://example.com")
@@ -60,7 +60,7 @@ describe SiteInspector::Endpoint do
   context "requests" do
 
     it "requests a URL" do
-      stub = stub_request(:get, "http://example.com/").
+      stub = stub_request(:head, "http://example.com/").
            to_return(:status => 200, :body => "content")
 
       expect(subject.request.body).to eql("content")
@@ -68,7 +68,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "requests a requested path" do
-      stub = stub_request(:get, "http://example.com/foo").
+      stub = stub_request(:head, "http://example.com/foo").
            to_return(:status => 200, :body => "content")
 
       expect(subject.request(:path => "foo").body).to eql("content")
@@ -76,7 +76,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "requests with typhoeus options" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 301, :headers => { :location => "http://example.com/foo" } )
 
       response = subject.request(:followlocation => true)
@@ -84,7 +84,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "returns the response" do
-      stub = stub_request(:get, "http://example.com/").
+      stub = stub_request(:head, "http://example.com/").
            to_return(:status => 200, :body => "content")
 
       expect(subject.response.body).to eql("content")
@@ -93,7 +93,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "knows if there's a response" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 200, :body => "content")
 
       expect(subject.responds?).to eql(true)
@@ -108,7 +108,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "knows the response code" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 200)
 
       expect(subject.response_code).to eql("200")
@@ -120,7 +120,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "considers a 200 response code to be live and a response" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 200)
 
       expect(subject.up?).to eql(true)
@@ -128,7 +128,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "considers a 301 response code to be live and a response" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 301)
 
       expect(subject.up?).to eql(true)
@@ -136,7 +136,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "considers a 404 response code to be down but a response" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 404)
 
       expect(subject.up?).to eql(false)
@@ -144,7 +144,7 @@ describe SiteInspector::Endpoint do
     end
 
     it "considers a 500 response code to be down but a response" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 500)
 
       expect(subject.up?).to eql(false)
@@ -168,59 +168,65 @@ describe SiteInspector::Endpoint do
 
   context "redirects" do
     it "knows when there's a redirect" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 301, :headers => { :location => "http://www.example.com" } )
 
       expect(subject.redirect?).to eql(true)
     end
 
     it "returns the redirect" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 301, :headers => { :location => "http://www.example.com" } )
 
-      stub_request(:get, "http://www.example.com/").
+      stub_request(:head, "http://www.example.com/").
         to_return(:status => 200)
 
-      expect(subject.redirect.uri.to_s).to eql("http://www.example.com")
+      expect(subject.redirect.uri.to_s).to eql("http://www.example.com/")
     end
 
     it "handles relative redirects" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 301, :headers => { :location => "/foo" } )
 
       expect(subject.redirect?).to eql(false)
     end
 
     it "handles relative redirects without a leading slash" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
            to_return(:status => 301, :headers => { :location => "foo" } )
 
       expect(subject.redirect?).to eql(false)
     end
 
     it "knows what it resolves to" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 301, :headers => { :location => "http://www.example.com" } )
 
-      stub_request(:get, "http://www.example.com/").
+      stub_request(:head, "http://www.example.com/").
         to_return(:status => 200)
 
       expect(subject.redirect?).to eql(true)
-      expect(subject.resolves_to.uri.to_s).to eql("http://www.example.com")
+      expect(subject.resolves_to.uri.to_s).to eql("http://www.example.com/")
     end
 
     it "detects external redirects" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 301, :headers => { :location => "http://www.example.gov" } )
 
+      stub_request(:head, "http://www.example.gov").
+        to_return(:status => 200)
+
       expect(subject.redirect?).to eql(true)
       expect(subject.external_redirect?).to eql(true)
     end
 
     it "knows internal redirects are not external redirects" do
-      stub_request(:get, "http://example.com/").
+      stub_request(:head, "http://example.com/").
         to_return(:status => 301, :headers => { :location => "https://example.com" } )
 
+      stub_request(:head, "https://example.com/").
+        to_return(:status => 200)
+
       expect(subject.external_redirect?).to eql(false)
     end
   end
@@ -233,7 +239,7 @@ describe SiteInspector::Endpoint do
     SiteInspector::Endpoint.checks.each do |check|
       it "responds to the #{check} check" do
 
-        stub_request(:get, "http://example.com/").
+        stub_request(:head, "http://example.com/").
           to_return(:status => 200)
 
         expect(subject.send(check.name)).to_not be_nil

data/spec/site_inspector_spec.rb

@@ -36,6 +36,7 @@ describe SiteInspector do
     expected = {
       :accept_encoding => "gzip",
       :followlocation => false,
+      :method  => :head,
       :timeout => 10,
       :headers => {
         "User-Agent" => "Mozilla/5.0 (compatible; SiteInspector/#{SiteInspector::VERSION}; +https://github.com/benbalter/site-inspector)"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site-inspector
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Ben Balter
@@ -150,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement