site-inspector 1.0.2 → 3.2.0

data/spec/checks/site_inspector_endpoint_wappalyzer_spec.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe SiteInspector::Endpoint::Wappalyzer do
+  subject { described_class.new(endpoint) }
+
+  let(:domain) { 'http://ben.balter.com.com' }
+  let(:endpoint) { SiteInspector::Endpoint.new(domain) }
+  let(:url) { "https://api.wappalyzer.com/lookup/v2/?urls=#{domain}/" }
+
+  before do
+    path = File.expand_path '../fixtures/wappalyzer.json', __dir__
+    body = File.read path
+    stub_request(:get, url).to_return(status: 200, body: body)
+  end
+
+  it 'returns the API response' do
+    expected = {
+      'Analytics' => ['Google Analytics'],
+      'CDN' => %w[Cloudflare Fastly],
+      'Caching' => ['Varnish'],
+      'Other' => %w[Disqus Jekyll],
+      'PaaS' => ['GitHub Pages'],
+      'Web frameworks' => ['Ruby on Rails']
+    }
+    expect(subject.to_h).to eql(expected)
+  end
+
+  it 'fails gracefully' do
+    stub_request(:get, url).to_return(status: 400, body: '')
+    expect(subject.to_h).to eql({})
+  end
+end

data/spec/checks/site_inspector_endpoint_whois_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe SiteInspector::Endpoint::Whois do
+  subject do
+    stub_request(:head, site).to_return(status: 200)
+    endpoint = SiteInspector::Endpoint.new(site)
+    described_class.new(endpoint)
+  end
+
+  let(:site) { 'https://example.com' }
+
+  it 'returns the whois for the IP' do
+    expect(subject.ip).to match(/Derrick Sawyer/)
+  end
+
+  it 'returns the whois for the domain' do
+    expect(subject.domain).to match(/Domain Name: EXAMPLE\.COM/)
+  end
+
+  it 'returns the hash' do
+    expect(subject.to_h[:domain].keys.first).to eql('Domain Name')
+    expect(subject.to_h[:domain].values.first).to eql('EXAMPLE.COM')
+  end
+end

data/spec/fixtures/wappalyzer.json

@@ -0,0 +1,125 @@
+[
+   {
+      "url":"https://ben.balter.com",
+      "technologies":[
+         {
+            "slug":"cloudflare",
+            "name":"Cloudflare",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               {
+                  "id":31,
+                  "slug":"cdn",
+                  "name":"CDN"
+               }
+            ]
+         },
+         {
+            "slug":"varnish",
+            "name":"Varnish",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               {
+                  "id":23,
+                  "slug":"caching",
+                  "name":"Caching"
+               }
+            ]
+         },
+         {
+            "slug":"disqus",
+            "name":"Disqus",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               
+            ]
+         },
+         {
+            "slug":"google-analytics",
+            "name":"Google Analytics",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               {
+                  "id":10,
+                  "slug":"analytics",
+                  "name":"Analytics"
+               },
+               {
+                  "id":61,
+                  "slug":"saas",
+                  "name":"SaaS"
+               }
+            ]
+         },
+         {
+            "slug":"jekyll",
+            "name":"Jekyll",
+            "versions":[
+               "v3.9.0"
+            ],
+            "trafficRank":11,
+            "categories":[
+               
+            ]
+         },
+         {
+            "slug":"ruby-on-rails",
+            "name":"Ruby on Rails",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               {
+                  "id":18,
+                  "slug":"web-frameworks",
+                  "name":"Web frameworks"
+               }
+            ]
+         },
+         {
+            "slug":"fastly",
+            "name":"Fastly",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               {
+                  "id":31,
+                  "slug":"cdn",
+                  "name":"CDN"
+               }
+            ]
+         },
+         {
+            "slug":"github-pages",
+            "name":"GitHub Pages",
+            "versions":[
+               
+            ],
+            "trafficRank":11,
+            "categories":[
+               {
+                  "id":62,
+                  "slug":"paas",
+                  "name":"PaaS"
+               }
+            ]
+         }
+      ],
+      "crawl":true
+   }
+]

data/spec/site_inspector_cache_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe SiteInspector::Cache do
+  it 'stores a cache value' do
+    subject.set 'foo', 'bar'
+    expect(subject.instance_variable_get('@memory')['foo']).to eql('bar')
+  end
+
+  it 'retrieves values from the cache' do
+    subject.instance_variable_set('@memory', 'foo' => 'bar')
+    expect(subject.get('foo')).to eql('bar')
+  end
+end

data/spec/site_inspector_disk_cache_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe SiteInspector::DiskCache do
+  subject { described_class.new(tmpdir) }
+
+  before do
+    FileUtils.rm_rf(tmpdir)
+    Dir.mkdir(tmpdir)
+  end
+
+  it 'writes a value to disk' do
+    foo = Typhoeus::Request.new('foo')
+
+    path = File.expand_path foo.cache_key, tmpdir
+    expect(File.exist?(path)).to be(false)
+
+    subject.set foo, 'bar'
+
+    expect(File.exist?(path)).to be(true)
+    expect(File.open(path).read).to eql("I\"bar:ET")
+  end
+
+  it 'reads a value from disk' do
+    foo = Typhoeus::Request.new('foo')
+
+    path = File.expand_path foo.cache_key, tmpdir
+    File.write(path, "I\"bar:ET")
+    expect(subject.get(foo)).to eql('bar')
+  end
+
+  it "calculates a file's path" do
+    foo = Typhoeus::Request.new('foo')
+
+    path = File.expand_path foo.cache_key, tmpdir
+    expect(subject.send(:path, foo)).to eql(path)
+  end
+end

data/spec/site_inspector_domain_spec.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe SiteInspector::Domain do
+  subject { described_class.new('example.com') }
+
+  context 'domain parsing' do
+    it 'downcases the domain' do
+      domain = described_class.new('EXAMPLE.com')
+      expect(domain.host).to eql('example.com')
+    end
+
+    it 'strips http from the domain' do
+      domain = described_class.new('http://example.com')
+      expect(domain.host).to eql('example.com')
+    end
+
+    it 'strips https from the domain' do
+      domain = described_class.new('https://example.com')
+      expect(domain.host).to eql('example.com')
+    end
+
+    it 'strips www from the domain' do
+      domain = described_class.new('www.example.com')
+      expect(domain.host).to eql('example.com')
+    end
+
+    it 'strips http://www from the domain' do
+      domain = described_class.new('http://www.example.com')
+      expect(domain.host).to eql('example.com')
+    end
+
+    it 'strips paths from the domain' do
+      domain = described_class.new('http://www.example.com/foo')
+      expect(domain.host).to eql('example.com')
+    end
+
+    it 'strips trailing slashes from the domain' do
+      domain = described_class.new('http://www.example.com/')
+      expect(domain.host).to eql('example.com')
+    end
+  end
+
+  context 'endpoints' do
+    it 'generates the endpoints' do
+      endpoints = subject.endpoints
+      expect(endpoints.count).to be(4)
+      expect(endpoints[0].to_s).to eql('https://example.com/')
+      expect(endpoints[1].to_s).to eql('https://www.example.com/')
+      expect(endpoints[2].to_s).to eql('http://example.com/')
+      expect(endpoints[3].to_s).to eql('http://www.example.com/')
+    end
+  end
+
+  it 'knows the canonical domain' do
+    stub_request(:head, 'https://example.com/').to_return(status: 500)
+    stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+    stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+    stub_request(:head, 'http://example.com/').to_return(status: 200)
+    expect(subject.canonical_endpoint.to_s).to eql('http://example.com/')
+  end
+
+  it 'knows if a domain is a government domain' do
+    expect(subject.government?).to be(false)
+
+    domain = described_class.new('whitehouse.gov')
+    expect(domain.government?).to be(true)
+  end
+
+  context 'up' do
+    it 'considers a domain up if at least one endpoint is up' do
+      subject.endpoints.each do |endpoint|
+        allow(endpoint).to receive(:response) { Typhoeus::Response.new(code: 0) } unless endpoint.uri.to_s.start_with?('http://www')
+      end
+
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+      expect(subject.up?).to be(true)
+    end
+
+    it "doesn't consider a domain up when all endpoints are down" do
+      subject.endpoints.each do |endpoint|
+        allow(endpoint).to receive(:response) { Typhoeus::Response.new(code: 0) }
+      end
+
+      expect(subject.up?).to be(false)
+    end
+  end
+
+  context 'up' do
+    it 'considers a domain up if at least one endpoint is up' do
+      stub_request(:head, 'https://example.com/').to_return(status: 500)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+      expect(subject.up?).to be(true)
+    end
+
+    it "doesn't consider a domain up if all endpoints are down" do
+      stub_request(:head, 'https://example.com/').to_return(status: 500)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+
+      expect(subject.up?).to be(false)
+    end
+  end
+
+  context 'www' do
+    it 'considers a site www when at least one endpoint is www' do
+      stub_request(:head, 'https://example.com/').to_return(status: 200)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+      expect(subject.www?).to be(true)
+    end
+
+    it "doesn't consider a site www when no endpoint is www" do
+      stub_request(:head, 'https://example.com/').to_return(status: 200)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 200)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+
+      expect(subject.www?).to be(false)
+    end
+  end
+
+  context 'root' do
+    it 'considers a domain root if you can connect without www' do
+      stub_request(:head, 'https://example.com/').to_return(status: 200)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+
+      expect(subject.root?).to be(true)
+    end
+
+    it "doesn't call a www-only domain root" do
+      stub_request(:head, 'https://example.com/').to_return(status: 500)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 200)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+      expect(subject.root?).to be(false)
+    end
+  end
+
+  context 'https' do
+    it 'knows when a domain supports https' do
+      stub_request(:head, 'https://example.com/').to_return(status: 200)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 200)
+      stub_request(:head, 'http://example.com/').to_return(status: 200)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+      allow(subject.endpoints.first.https).to receive(:valid?).and_return(true)
+
+      expect(subject.https?).to be(true)
+    end
+
+    it "knows when a domain doesn't support https" do
+      stub_request(:head, 'https://example.com/').to_return(status: 500)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 200)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+      expect(subject.https?).to be(false)
+    end
+
+    it 'considers HTTPS inforced when no http endpoint responds' do
+      stub_request(:head, 'https://example.com/').to_return(status: 200)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+
+      # expect(subject.enforces_https?).to eql(true)
+    end
+
+    it "doesn't consider HTTPS inforced when an http endpoint responds" do
+      stub_request(:head, 'https://example.com/').to_return(status: 200)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/').to_return(status: 500)
+      stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+      expect(subject.enforces_https?).to be(false)
+    end
+
+    it 'detects when a domain downgrades to http' do
+      # TODO
+    end
+
+    it 'detects when a domain enforces https' do
+      # TODO
+    end
+  end
+
+  context 'canonical' do
+    context 'www' do
+      it 'detects a domain as canonically www when root is down' do
+        stub_request(:head, 'https://example.com/').to_return(status: 500)
+        stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+        stub_request(:head, 'http://example.com/').to_return(status: 500)
+        stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+        expect(subject.canonically_www?).to be(true)
+      end
+
+      it 'detects a domain as canonically www when root redirects' do
+        stub_request(:head, 'https://example.com/').to_return(status: 500)
+        stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+        stub_request(:head, 'http://example.com/')
+          .to_return(status: 301, headers: { location: 'http://www.example.com' })
+        stub_request(:head, 'http://www.example.com/').to_return(status: 200)
+
+        expect(subject.canonically_www?).to be(true)
+      end
+    end
+
+    context 'https' do
+      it 'detects a domain as canonically https when http is down' do
+        stub_request(:head, 'https://example.com/').to_return(status: 200)
+        stub_request(:head, 'https://www.example.com/').to_return(status: 200)
+        stub_request(:head, 'http://example.com/').to_return(status: 500)
+        stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+        allow(subject.endpoints.first.https).to receive(:valid?).and_return(true)
+
+        expect(subject.canonically_https?).to be(true)
+      end
+
+      it 'detects a domain as canonically https when http redirect' do
+        stub_request(:head, 'https://example.com/').to_return(status: 200)
+        stub_request(:head, 'https://www.example.com/').to_return(status: 200)
+        stub_request(:head, 'http://example.com/')
+          .to_return(status: 301, headers: { location: 'https://example.com' })
+        stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+        allow(subject.endpoints.first.https).to receive(:valid?).and_return(true)
+
+        expect(subject.canonically_https?).to be(true)
+      end
+    end
+  end
+
+  context 'redirects' do
+    it 'knows when a domain redirects' do
+      stub_request(:head, 'https://example.com/').to_return(status: 500)
+      stub_request(:head, 'https://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://example.com/')
+        .to_return(status: 301, headers: { location: 'http://foo.example.com' })
+      stub_request(:head, 'http://www.example.com/').to_return(status: 500)
+      stub_request(:head, 'http://foo.example.com/').to_return(status: 200)
+
+      expect(subject.redirect?).to be(true)
+    end
+  end
+
+  context 'hsts' do
+    it 'enabled' do
+    end
+
+    it 'subdomains' do
+    end
+
+    it 'preload ready' do
+    end
+  end
+
+  it 'returns the host as a string' do
+    expect(subject.to_s).to eql('example.com')
+  end
+end