site-inspector 1.0.2 → 2.0.0

data/lib/site-inspector/rails_cache.rb

@@ -0,0 +1,11 @@
+class SiteInspector
+  class RailsCache
+    def get(request)
+      Rails.cache.read(request)
+    end
+
+    def set(request, response)
+      Rails.cache.write(request, response)
+    end
+  end
+end

data/lib/site-inspector/version.rb

@@ -0,0 +1,3 @@
+class SiteInspector
+  VERSION = "2.0.0"
+end

data/script/bootstrap

@@ -0,0 +1 @@
+bundle install

data/script/cibuild

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+bundle exec rake spec
+
+gem build site-inspector.gemspec

data/script/console

@@ -0,0 +1 @@
+bundle exec pry -r ./lib/site-inspector.rb

data/script/release

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Tag and push a release.
+
+set -e
+
+# Make sure we're in the project root.
+
+cd $(dirname "$0")/..
+
+# Build a new gem archive.
+
+rm -rf site-inspector-*.gem
+gem build -q site-inspector.gemspec
+
+# Make sure we're on the master branch.
+
+(git branch | grep -q '* master') || {
+  echo "Only release from the master branch."
+  exit 1
+}
+
+# Figure out what version we're releasing.
+
+tag=v`ls site-inspector-*.gem | sed 's/^site-inspector-\(.*\)\.gem$/\1/'`
+
+# Make sure we haven't released this version before.
+
+git fetch -t origin
+
+(git tag -l | grep -q "$tag") && {
+  echo "Whoops, there's already a '${tag}' tag."
+  exit 1
+}
+
+# Tag it and bag it.
+
+gem push site-inspector-*.gem && git tag "$tag" &&
+  git push origin master && git push origin "$tag"

data/site-inspector.gemspec

@@ -0,0 +1,33 @@
+require File.expand_path "./lib/site-inspector/version", File.dirname(__FILE__)
+
+Gem::Specification.new do |s|
+
+  s.name                  = "site-inspector"
+  s.version               = SiteInspector::VERSION
+  s.summary               = "A Ruby port and v2 of Site Inspector (http://github.com/benbalter/site-inspector)"
+  s.description           = "Returns information about a domain's technology and capabilities"
+  s.authors               = "Ben Balter"
+  s.email                 = "ben@balter.com"
+  s.homepage              = "https://github.com/benbalter/site-inspector-ruby"
+  s.license               = "MIT"
+
+  s.files                 = `git ls-files -z`.split("\x0")
+  s.executables           = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files            = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths         = ["lib"]
+
+  s.add_dependency("nokogiri", "~> 1.6")
+  s.add_dependency("public_suffix", "~> 1.4")
+  s.add_dependency("gman", "~> 4.1")
+  s.add_dependency("dnsruby", "~> 1.56")
+  s.add_dependency("sniffles", "~> 0.2")
+  s.add_dependency("typhoeus", "~> 0.7")
+  s.add_dependency("oj", "~> 2.11")
+  s.add_dependency("mercenary", "~> 0.3")
+  s.add_dependency("colorator", "~> 0.1")
+  s.add_development_dependency("pry", "~> 0.10")
+  s.add_development_dependency( "rake", "~> 10.4" )
+  s.add_development_dependency( "rspec", "~> 3.2")
+  s.add_development_dependency( "bundler", "~> 1.6" )
+  s.add_development_dependency( "webmock", "~> 1.2" )
+end

data/spec/checks/site_inspector_endpoint_check_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe SiteInspector::Endpoint::Check do
+
+  subject do
+    stub_request(:get, "http://example.com/").to_return(:status => 200)
+    endpoint = SiteInspector::Endpoint.new("http://example.com")
+    SiteInspector::Endpoint::Check.new(endpoint)
+  end
+
+  it "returns the endpoint" do
+    expect(subject.endpoint.class).to eql(SiteInspector::Endpoint)
+  end
+
+  it "returns the response" do
+    expect(subject.response.class).to eql(Typhoeus::Response)
+  end
+
+  it "returns the request" do
+    expect(subject.request.class).to eql(Typhoeus::Request)
+  end
+
+  it "returns the host" do
+    expect(subject.host).to eql("example.com")
+  end
+
+  it "returns its name" do
+    expect(subject.name).to eql(:check)
+  end
+
+  it "returns the instance name" do
+    expect(SiteInspector::Endpoint::Check.name).to eql(:check)
+  end
+end

data/spec/checks/site_inspector_endpoint_content_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe SiteInspector::Endpoint::Content do
+
+  subject do
+    body = <<-eos
+      <!DOCTYPE html>
+      <html>
+        <body>
+          <h1>Some page</h1>
+        </body>
+      </html>
+    eos
+
+    stub_request(:get, "http://example.com/").
+      to_return(:status => 200, :body => body )
+    endpoint = SiteInspector::Endpoint.new("http://example.com")
+    SiteInspector::Endpoint::Content.new(endpoint)
+  end
+
+  it "returns the doc" do
+    expect(subject.document.class).to eql(Nokogiri::HTML::Document)
+    expect(subject.document.css("h1").text).to eql("Some page")
+  end
+
+  it "returns the body" do
+    expect(subject.body).to match("<h1>Some page</h1>")
+  end
+
+  it "returns the doctype" do
+    expect(subject.doctype).to eql("html")
+  end
+
+  it "knows when robots.txt exists" do
+    stub_request(:get, "http://example.com/robots.txt").
+      to_return(:status => 200)
+    expect(subject.robots_txt?).to eql(true)
+  end
+
+  it "knows when robots.txt doesn't exist" do
+    stub_request(:get, "http://example.com/robots.txt").
+      to_return(:status => 404)
+    expect(subject.robots_txt?).to eql(false)
+  end
+
+  it "knows when sitemap.xml exists" do
+    stub_request(:get, "http://example.com/sitemap.xml").
+      to_return(:status => 200)
+    expect(subject.sitemap_xml?).to eql(true)
+  end
+
+  it "knows when sitemap.xml exists" do
+    stub_request(:get, "http://example.com/sitemap.xml").
+      to_return(:status => 404)
+    expect(subject.sitemap_xml?).to eql(false)
+  end
+
+  it "knows when humans.txt exists" do
+    stub_request(:get, "http://example.com/humans.txt").
+      to_return(:status => 200)
+    expect(subject.humans_txt?).to eql(true)
+  end
+
+  it "knows when humans.txt doesn't exist" do
+    stub_request(:get, "http://example.com/humans.txt").
+      to_return(:status => 200)
+    expect(subject.humans_txt?).to eql(true)
+  end
+
+  context "404s" do
+    it "knows when an endpoint returns a proper 404" do
+      stub_request(:get, /http\:\/\/example.com\/.*/).
+        to_return(:status => 404)
+      expect(subject.proper_404s?).to eql(true)
+    end
+
+    it "knows when an endpoint doesn't return a proper 404" do
+      stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).
+        to_return(:status => 200)
+      expect(subject.proper_404s?).to eql(false)
+    end
+
+    it "generates a random path" do
+      path = subject.send(:random_path)
+      expect(path).to match /[a-z0-9]{32}/i
+      expect(subject.send(:random_path)).to eql(path)
+    end
+  end
+end

data/spec/checks/site_inspector_endpoint_dns_spec.rb

@@ -0,0 +1,167 @@
+require 'spec_helper'
+require 'dnsruby'
+
+describe SiteInspector::Endpoint::Dns do
+
+  subject do
+    stub_request(:get, "http://github.com/").to_return(:status => 200)
+    endpoint = SiteInspector::Endpoint.new("http://github.com")
+    SiteInspector::Endpoint::Dns.new(endpoint)
+  end
+
+  it "inits the resolver" do
+    expect(SiteInspector::Endpoint::Dns.resolver.class).to eql(Dnsruby::Resolver)
+  end
+
+  # Note: these tests makes external calls
+  context "live tests" do
+    it "it runs the query" do
+      expect(subject.query).not_to be_empty
+    end
+
+    context "resolv" do
+      it "returns the IP" do
+        expect(subject.ip).to include("192.30.252.")
+      end
+
+      it "returns the hostname" do
+        expect(subject.hostname.sld).to eql("github")
+
+      end
+    end
+  end
+
+  context "stubbed tests" do
+
+    before do
+      record = Dnsruby::RR.create :type => "A", :address => "1.2.3.4", :name => "test"
+      allow(subject).to receive(:records) { [record] }
+      allow(subject).to receive(:query) { [] }
+    end
+
+    it "returns the records" do
+      expect(subject.records.count).to eql(1)
+      expect(subject.records.first.class).to eql(Dnsruby::RR::IN::A)
+    end
+
+    it "knows if a record exists" do
+      expect(subject.has_record?("A")).to eql(true)
+      expect(subject.has_record?("CNAME")).to eql(false)
+    end
+
+    it "knows if a domain supports dnssec" do
+      expect(subject.dnssec?).to eql(false)
+
+      # via https://github.com/alexdalitz/dnsruby/blob/master/test/tc_dnskey.rb
+      input = "example.com. 86400 IN DNSKEY 256 3 5 ( AQPSKmynfzW4kyBv015MUG2DeIQ3" +
+        "Cbl+BBZH4b/0PY1kxkmvHjcZc8no" +
+        "kfzj31GajIQKY+5CptLr3buXA10h" +
+        "WqTkF7H6RfoRqXQeogmMHfpftf6z" +
+        "Mv1LyBUgia7za6ZEzOJBOztyvhjL" +
+        "742iU/TpPSEDhm2SNKLijfUppn1U" +
+        "aNvv4w==  )"
+
+      record = Dnsruby::RR.create input
+      allow(subject).to receive(:records) { [record] }
+
+      expect(subject.dnssec?).to eql(true)
+    end
+
+    it "knows if a domain supports ipv6" do
+      expect(subject.ipv6?).to eql(false)
+
+      input = {
+        :type => "AAAA",
+        :name => "test",
+        :address => '102:304:506:708:90a:b0c:d0e:ff10'
+      }
+      record = Dnsruby::RR.create input
+      allow(subject).to receive(:records) { [record] }
+
+      expect(subject.ipv6?).to eql(true)
+    end
+
+    context "hostname detection" do
+      it "lists cnames" do
+        records = []
+
+        records.push Dnsruby::RR.create({
+          :type => "CNAME",
+          :domainname => "example.com",
+          :name => "example"
+        })
+
+        records.push Dnsruby::RR.create({
+          :type => "CNAME",
+          :domainname => "github.com",
+          :name => "github"
+        })
+
+        allow(subject).to receive(:records) { records }
+
+        expect(subject.cnames.count).to eql(2)
+        expect(subject.cnames.first.sld).to eql("example")
+      end
+
+      it "knows when a domain doesn't have a cdn" do
+        expect(subject.cdn?).to eql(false)
+      end
+
+      it "detects CDNs" do
+        records = [Dnsruby::RR.create({
+          :type => "CNAME",
+          :domainname => "foo.cloudfront.net",
+          :name => "example"
+        })]
+        allow(subject).to receive(:records) { records }
+
+        expect(subject.send(:detect_by_hostname, "cdn")).to eql(:cloudfront)
+        expect(subject.cdn).to eql(:cloudfront)
+        expect(subject.cdn?).to eql(true)
+      end
+
+      it "builds that path to a data file" do
+        path = subject.send(:data_path, "foo")
+        expected = File.expand_path "../../lib/data/foo.yml", File.dirname(__FILE__)
+        expect(path).to eql(expected)
+      end
+
+      it "loads data files" do
+        data = subject.send(:load_data, "cdn")
+        expect(data.keys).to include("cloudfront")
+      end
+
+      it "knows when a domain isn't cloud" do
+        expect(subject.cloud?).to eql(false)
+      end
+
+      it "detects cloud providers" do
+        records = [Dnsruby::RR.create({
+          :type => "CNAME",
+          :domainname => "foo.herokuapp.com",
+          :name => "example"
+        })]
+        allow(subject).to receive(:records) { records }
+
+        expect(subject.send(:detect_by_hostname, "cloud")).to eql(:heroku)
+        expect(subject.cloud_provider).to eql(:heroku)
+        expect(subject.cloud?).to eql(true)
+      end
+
+      it "knows when a domain doesn't have google apps" do
+        expect(subject.google_apps?).to eql(false)
+      end
+
+      it "knows when a domain is using google apps" do
+        records = [Dnsruby::RR.create({
+          :type => "MX",
+          :exchange => "mx1.google.com",
+          :name => "example",
+          :preference => 10
+        })]
+        allow(subject).to receive(:records) { records }
+        expect(subject.google_apps?).to eql(true)
+      end
+    end
+  end
+end

data/spec/checks/site_inspector_endpoint_headers_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+
+describe SiteInspector::Endpoint::Headers do
+
+  subject do
+    stub_request(:get, "http://example.com/").
+      to_return(:status => 200, :headers => { :foo => "bar" } )
+    endpoint = SiteInspector::Endpoint.new("http://example.com")
+    SiteInspector::Endpoint::Headers.new(endpoint)
+  end
+
+  def stub_header(header, value)
+    allow(subject).to receive(:headers) { { header => value } }
+  end
+
+  it "parses the headers" do
+    expect(subject.headers.count).to eql(1)
+    expect(subject.headers.keys).to include("foo")
+  end
+
+  it "returns a header" do
+    expect(subject["foo"]).to eql("bar")
+    expect(subject.headers["foo"]).to eql("bar")
+  end
+
+  it "knows the server" do
+    stub_header "server", "foo"
+    expect(subject.server).to eql("foo")
+  end
+
+  it "knows if a server has an xss protection header" do
+    stub_header "x-xss-protection", "foo"
+    expect(subject.xss_protection).to eql("foo")
+  end
+
+  it "validates xss-protection" do
+    stub_header "x-xss-protection", "foo"
+    expect(subject.xss_protection?).to eql(false)
+
+    stub_header "x-xss-protection", "1; mode=block"
+    expect(subject.xss_protection?).to eql(true)
+  end
+
+  it "checks for clickjack proetection" do
+    expect(subject.click_jacking_protection?).to eql(false)
+    stub_header "x-frame-options", "foo"
+    expect(subject.click_jacking_protection).to eql("foo")
+    expect(subject.click_jacking_protection?).to eql(true)
+  end
+
+  it "checks for CSP" do
+    expect(subject.content_security_policy?).to eql(false)
+    stub_header "content-security-policy", "foo"
+    expect(subject.content_security_policy).to eql("foo")
+    expect(subject.content_security_policy?).to eql(true)
+  end
+
+  it "checks for strict-transport-security" do
+    expect(subject.strict_transport_security?).to eql(false)
+    stub_header "strict-transport-security", "foo"
+    expect(subject.strict_transport_security).to eql("foo")
+    expect(subject.strict_transport_security?).to eql(true)
+  end
+
+  it "knows if there are cookies" do
+    expect(subject.cookies?).to eql(false)
+    stub_header "set-cookie", "foo"
+    expect(subject.cookies?).to eql(true)
+  end
+
+  it "knows if the cookies are secure" do
+
+  end
+end