site-inspector 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 23a0a7c4e9875a9bec4ce8f904c75e9c29f292d4
+  data.tar.gz: 5fea15e6dc4a6b1b59243ae300a30a5f9cb2af5b
+SHA512:
+  metadata.gz: 37e8b7e3f3579f266c8c61cd2d712c203f693734edd58d4fa97f1881fc856499e72205baaed781d12662f263a61d05b824deb66387808999352f821045b483fd
+  data.tar.gz: b4b374743289936b256fdb949537f9095521dcb6c0720abf38d5df529c61c2b8e48b80a00ca4bf0f5ff8717dd884105767c67b6a30f12ff9f664732b53d263bd

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Ben Balter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/data/cdn.yml

@@ -0,0 +1,3 @@
+#fastly: ""
+cloudfront: "cloutfront.net"
+akamai: "akamaitechnologies.com"

data/lib/data/cloud.yml

@@ -0,0 +1,6 @@
+amazon: "compute-1.amazonaws.com"
+Amazon GovCloud: "us-gov-west-1.compute.amazonaws.com"
+linode: "lindoe.com"
+rackspace: ""
+heroku: "herokuapp.com"
+github: "github.io"

data/lib/site-inspector/cache.rb

@@ -0,0 +1,13 @@
+class SiteInspectorCache
+  def initialize
+    @memory = {}
+  end
+
+  def get(request)
+    @memory[request]
+  end
+
+  def set(request, response)
+    @memory[request] = response
+  end
+end

data/lib/site-inspector/compliance.rb

@@ -0,0 +1,19 @@
+class SiteInspector
+
+  def path_exists?(path)
+    url = URI.join uri, path
+    Typhoeus::Request.get(url, followlocation: true, timeout: 10).success?
+  end
+
+  def slash_data?
+    @slash_data ||= path_exists?("/data")
+  end
+
+  def slash_developer?
+    @slash_developer ||= (path_exists?("/developer") || path_exists?("/developers"))
+  end
+
+  def data_dot_json?
+    @data_dot_json ||= path_exists?("/data.json")
+  end
+end

data/lib/site-inspector/dns.rb

@@ -0,0 +1,81 @@
+class SiteInspector
+
+  def resolver
+    @resolver ||= Dnsruby::Resolver.new
+  end
+
+  def query(type="ANY")
+    resolver.query(domain.to_s, type).answer
+  rescue
+    []
+  end
+
+  def dns
+    @dns ||= query
+  end
+
+  def dnssec?
+    @dnssec ||= dns.any? { |record| record.type == "DNSKEY" }
+  end
+
+  def ipv6?
+    @ipv6 ||= dns.any? { |record| record.type == "AAAA" }
+  end
+
+  def detect_by_hostname(type)
+
+    haystack = load_data(type)
+    needle = haystack.find { |name, domain|
+      cnames.any? { |cname|
+        domain == cname.tld || domain == "#{cname.sld}.#{cname.tld}"
+      }
+    }
+
+    return needle[0] if needle
+    return false unless hostname
+
+    needle = haystack.find { |name, domain|
+      domain == hostname.tld || domain == "#{hostname.sld}.#{hostname.tld}"
+    }
+
+    needle ? needle[0] : false
+  end
+
+  def cdn
+    detect_by_hostname "cdn"
+  end
+
+  def cdn?
+    !!cdn
+  end
+
+  def cloud_provider
+    detect_by_hostname "cloud"
+  end
+
+  def cloud?
+    !!cloud_provider
+  end
+
+  def google_apps?
+    @google ||= dns.any? do |record|
+      record.type == "MX" && record.exchange =~ /google(mail)?\.com\.?$/
+    end
+  end
+
+  def ip
+    @ip ||= Resolv.getaddress domain.to_s
+  rescue Resolv::ResolvError
+    nil
+  end
+
+  def hostname
+    @hostname ||= PublicSuffix.parse(Resolv.getname(ip))
+  rescue Exception => e
+    nil
+  end
+
+  def cnames
+    @cnames ||= dns.select {|record| record.type == "CNAME" }.map { |record| PublicSuffix.parse(record.cname.to_s) }
+  end
+end

data/lib/site-inspector/headers.rb

@@ -0,0 +1,34 @@
+class SiteInspector
+  def server
+    response && response.headers["Server"]
+  end
+
+  def xss_protection?
+    response && response.headers["X-XSS-Protection"] == "1; mode=block"
+  end
+
+  def has_cookies?
+    response && response.headers.include?("Set-Cookie")
+  end
+
+  def secure_cookies?
+    return nil if !response || !has_cookies?
+    cookie = response.headers["Set-Cookie"]
+    cookie = cookie.first if cookie.is_a?(Array)
+    marked_secure = !!(cookie.downcase =~ /secure/)
+    marked_http_only = !!(cookie.downcase =~ /HttpOnly/)
+    marked_secure and marked_http_only
+  end
+
+  def strict_transport_security?
+    response && response.headers.include?("Strict-Transport-Security")
+  end
+
+  def content_security_policy?
+    response && response.headers.include?("Content-Security-Policy")
+  end
+
+  def click_jacking_protection?
+    response && response.headers.include?("X-Frame-Options")
+  end
+end

data/lib/site-inspector/sniffer.rb

@@ -0,0 +1,24 @@
+class SiteInspector
+  def sniff(type)
+    results = Sniffles.sniff(body, type).select { |name, meta| meta[:found] == true }
+    results.each { |name, result| result.delete :found} if results
+    results
+  end
+
+  def cms
+    sniff :cms
+  end
+
+  def analytics
+    sniff :analytics
+  end
+
+  def javascript
+    sniff :javascript
+  end
+
+  def advertising
+    sniff :advertising
+  end
+
+end

data/lib/site-inspector.rb

@@ -0,0 +1,163 @@
+require 'nokogiri'
+require 'open-uri'
+require 'public_suffix'
+require 'gman'
+require 'net/http'
+require "dnsruby"
+require 'yaml'
+require 'sniffles'
+require "addressable/uri"
+require 'typhoeus'
+require 'json'
+require 'resolv'
+
+require_relative 'site-inspector/cache'
+require_relative 'site-inspector/sniffer'
+require_relative 'site-inspector/dns'
+require_relative 'site-inspector/compliance'
+require_relative 'site-inspector/headers'
+
+Typhoeus::Config.cache = SiteInspectorCache.new
+
+class SiteInspector
+
+  def initialize(domain)
+    domain = domain.downcase
+    domain = domain.sub /^http\:/, ""
+    domain = domain.sub /^\/+/, ""
+    domain = domain.sub /^www\./, ""
+    @uri = Addressable::URI.parse "//#{domain}"
+    @domain = PublicSuffix.parse @uri.host
+  end
+
+  def inspect
+    "<SiteInspector domain=\"#{domain}\">"
+  end
+
+  def uri(ssl=https?,www=www?)
+    uri = @uri.clone
+    uri.host = "www.#{uri.host}" if www
+    uri.scheme = ssl ? "https" : "http"
+    uri
+  end
+
+  def domain
+    non_www? ? @domain : PublicSuffix.parse("www.#{@uri.host}")
+  end
+
+  def request(ssl=false, www=false, followlocation=true)
+    Typhoeus::Request.get(uri(ssl, www), followlocation: followlocation, timeout: 10)
+  end
+
+  def response
+    @response ||= begin
+      if response = request(false, false) and response.success?
+        @non_www = true
+        response
+      elsif response = request(false, true) and response.success?
+        @non_www = false
+        response
+      else
+        false
+      end
+    end
+  end
+
+  def timed_out?
+    response && response.timed_out?
+  end
+
+  def doc
+    @doc ||= Nokogiri::HTML response.body if response
+  end
+
+  def body
+    doc.to_s
+  end
+
+  def load_data(name)
+    YAML.load_file File.expand_path "./data/#{name}.yml", File.dirname(__FILE__)
+  end
+
+  def government?
+    Gman.valid? domain.to_s
+  end
+
+  def https?
+    @https ||= request(true, www?).success?
+  end
+  alias_method :ssl?, :https?
+
+  def enforce_https?
+    return false unless https?
+    @enforce_https ||= begin
+      response = request(false, www?)
+      if response.effective_url
+        Addressable::URI.parse(response.effective_url).scheme == "https"
+      else
+        puts response.inspect
+        false
+      end
+    end
+  end
+
+  def www?
+    response && response.effective_url && !!response.effective_url.match(/https?:\/\/www\./)
+  end
+
+  def non_www?
+    response && @non_www
+  end
+
+  def redirect?
+    !!redirect
+  end
+
+  def redirect
+    @redirect ||= begin
+      if location = request(https?, www?, false).headers["location"]
+        redirect_domain = SiteInspector.new(location).domain
+        redirect_domain.to_s if redirect_domain.to_s != domain.to_s
+      end
+    rescue
+      nil
+    end
+  end
+
+  def to_json
+    as_json.to_json
+  end
+
+  def as_json
+    {
+      :domain => domain.to_s,
+      :uri => uri.to_s,
+      :government => government?,
+      :live => !!response,
+      :ssl => https?,
+      :enforce_https => enforce_https?,
+      :non_www => non_www?,
+      :redirect => redirect,
+      :ip => ip,
+      :hostname => hostname.to_s,
+      :ipv6 => ipv6?,
+      :dnssec => dnssec?,
+      :cdn => cdn,
+      :google_apps => google_apps?,
+      :could_provider => cloud_provider,
+      :server => server,
+      :cms => cms,
+      :analytics => analytics,
+      :javascript => javascript,
+      :advertising => advertising,
+      :slash_data => slash_data?,
+      :slash_developer => slash_developer?,
+      :data_dot_json => data_dot_json?,
+      :click_jacking_protection => click_jacking_protection?,
+      :content_security_policy => content_security_policy?,
+      :xss_protection => xss_protection?,
+      :secure_cookies => secure_cookies?,
+      :strict_transport_security => strict_transport_security?
+    }
+  end
+end

metadata

@@ -0,0 +1,248 @@
+--- !ruby/object:Gem::Specification
+name: site-inspector
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ben Balter
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: public_suffix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gman
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dnsruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sniffles
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rerun
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Returns information about a domain's technology and capabilities
+email: ben@balter.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- lib/data/cdn.yml
+- lib/data/cloud.yml
+- lib/site-inspector.rb
+- lib/site-inspector/cache.rb
+- lib/site-inspector/compliance.rb
+- lib/site-inspector/dns.rb
+- lib/site-inspector/headers.rb
+- lib/site-inspector/sniffer.rb
+homepage: https://github.com/benbalter/site-inspector-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A Ruby port and v2 of Site Inspector (http://github.com/benbalter/site-inspector)
+test_files: []