site-inspector 3.2.0 → 4.0.0

data/lib/site-inspector/checks/content.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class Endpoint
-    class Content < Check
-      # Given a path (e.g, "/data"), check if the given path exists on the canonical endpoint
-      def path_exists?(path)
-        endpoint.up? && endpoint.request(path: path, followlocation: true).success?
-      end
-
-      # The default Check#response method is from a HEAD request
-      # The content check has a special response which includes the body from a GET request
-      def response
-        @response ||= endpoint.request(method: :get)
-      end
-
-      def document
-        require 'nokogiri'
-        @doc ||= Nokogiri::HTML response.body if response
-      end
-      alias doc document
-
-      def body
-        @body ||= document.to_s.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, replace: '')
-      end
-
-      def robots_txt?
-        @bodts_txt ||= path_exists?('robots.txt') if proper_404s?
-      end
-
-      def sitemap_xml?
-        @sitemap_xml ||= path_exists?('sitemap.xml') if proper_404s?
-      end
-
-      def humans_txt?
-        @humans_txt ||= path_exists?('humans.txt') if proper_404s?
-      end
-
-      def doctype
-        document.internal_subset.external_id
-      end
-
-      def generator
-        @generator ||= begin
-          tag = document.at('meta[name="generator"]')
-          tag['content'] if tag
-        end
-      end
-
-      def prefetch
-        return unless endpoint.up?
-
-        options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
-        ['robots.txt', 'sitemap.xml', 'humans.txt', random_path].each do |path|
-          request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)
-          SiteInspector.hydra.queue(request)
-        end
-        SiteInspector.hydra.run
-      end
-
-      def proper_404s?
-        @proper_404s ||= !path_exists?(random_path)
-      end
-
-      def to_h
-        prefetch
-        {
-          doctype: doctype,
-          generator: generator,
-          sitemap_xml: sitemap_xml?,
-          robots_txt: robots_txt?,
-          humans_txt: humans_txt?,
-          proper_404s: proper_404s?
-        }
-      end
-
-      private
-
-      def random_path
-        require 'securerandom'
-        @random_path ||= SecureRandom.hex
-      end
-    end
-  end
-end

data/lib/site-inspector/checks/headers.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class Endpoint
-    class Headers < Check
-      # TODO: kill this
-      def strict_transport_security?
-        !!strict_transport_security
-      end
-
-      def content_security_policy?
-        !!content_security_policy
-      end
-
-      def click_jacking_protection?
-        !!click_jacking_protection
-      end
-
-      # return the found header value
-
-      # TODO: kill this
-      def strict_transport_security
-        headers['strict-transport-security']
-      end
-
-      def content_security_policy
-        headers['content-security-policy']
-      end
-
-      def click_jacking_protection
-        headers['x-frame-options']
-      end
-
-      def server
-        headers['server']
-      end
-
-      def xss_protection
-        headers['x-xss-protection']
-      end
-
-      # more specific checks than presence of headers
-      def xss_protection?
-        xss_protection == '1; mode=block'
-      end
-
-      # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
-      def all
-        @all ||= response&.headers ? response.headers.transform_keys(&:downcase) : {}
-      end
-      alias headers all
-
-      def [](header)
-        headers[header]
-      end
-
-      def to_h
-        {
-          strict_transport_security: strict_transport_security || false,
-          content_security_policy: content_security_policy || false,
-          click_jacking_protection: click_jacking_protection || false,
-          server: server,
-          xss_protection: xss_protection || false
-        }
-      end
-    end
-  end
-end

data/lib/site-inspector/checks/sniffer.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class Endpoint
-    class Sniffer < Check
-      OPEN_SOURCE_FRAMEWORKS = [
-        # Sniffles
-        :drupal,
-        :joomla,
-        :movabletype,
-        :phpbb,
-        :wordpress,
-
-        # Internal
-        :php,
-        :expression_engine,
-        :cowboy
-      ].freeze
-
-      def framework
-        cms = sniff :cms
-        return cms unless cms.nil?
-        return :expression_engine if endpoint.cookies.any? { |c| c.keys.first =~ /^exp_/ }
-        return :php if endpoint.cookies['PHPSESSID']
-        return :coldfusion if endpoint.cookies['CFID'] && endpoint.cookies['CFTOKEN']
-        return :cowboy if endpoint.headers.server.to_s.casecmp('cowboy').zero?
-
-        nil
-      end
-
-      def open_source?
-        OPEN_SOURCE_FRAMEWORKS.include?(framework)
-      end
-
-      def analytics
-        sniff :analytics
-      end
-
-      def javascript
-        sniff :javascript
-      end
-
-      def advertising
-        sniff :advertising
-      end
-
-      def to_h
-        {
-          framework: framework,
-          analytics: analytics,
-          javascript: javascript,
-          advertising: advertising
-        }
-      end
-
-      private
-
-      def sniff(type)
-        require 'sniffles'
-        results = Sniffles.sniff(endpoint.content.body, type).select { |_name, meta| meta[:found] }
-        results&.keys&.first
-      rescue StandardError
-        nil
-      end
-    end
-  end
-end

data/lib/site-inspector/checks/wappalyzer.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class Endpoint
-    class Wappalyzer < Check
-      ENDPOINT = 'https://api.wappalyzer.com/lookup/v2/'
-
-      def to_h
-        return {} unless data['technologies']
-
-        @to_h ||= begin
-          technologies = {}
-          data['technologies'].each do |t|
-            category = t['categories'].first
-            category = category ? category['name'] : 'Other'
-            technologies[category] ||= []
-            technologies[category].push t['name']
-          end
-
-          technologies
-        end
-      end
-
-      private
-
-      def request
-        @request ||= begin
-          options = SiteInspector.typhoeus_defaults
-          headers = options[:headers].merge({ "x-api-key": api_key })
-          options = options.merge(method: :get, headers: headers)
-          Typhoeus::Request.new(url, options)
-        end
-      end
-
-      def data
-        return {} unless api_key && api_key != ''
-
-        @data ||= begin
-          SiteInspector.hydra.queue(request)
-          SiteInspector.hydra.run
-
-          response = request.response
-          if response.success?
-            JSON.parse(response.body).first
-          else
-            {}
-          end
-        end
-      end
-
-      def url
-        url = Addressable::URI.parse(ENDPOINT)
-        url.query_values = { urls: endpoint.uri }
-        url
-      end
-
-      def api_key
-        @api_key ||= ENV['WAPPALYZER_API_KEY']
-      end
-    end
-  end
-end

data/lib/site-inspector/checks/whois.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class Endpoint
-    class Whois < Check
-      def domain
-        @domain ||= whois.lookup host
-      end
-
-      def ip
-        @ip ||= whois.lookup ip_address
-      end
-
-      def to_h
-        {
-          domain: record_to_h(domain),
-          ip: record_to_h(ip)
-        }
-      end
-
-      private
-
-      def record_to_h(record)
-        record.content.scan(/^\s*(.*?):\s*(.*?)\r?\n/).to_h
-      end
-
-      def ip_address
-        @ip_address ||= Resolv.getaddress host
-      end
-
-      def whois
-        @whois ||= ::Whois::Client.new
-      end
-    end
-  end
-end

data/lib/site-inspector/disk_cache.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class DiskCache
-    def initialize(dir = nil, replace = nil)
-      @dir     = dir || ENV['CACHE']
-      @replace = replace || ENV['CACHE_REPLACE']
-      @memory  = {}
-    end
-
-    def get(request)
-      return unless File.exist?(path(request))
-      return @memory[request] if @memory[request]
-
-      if @replace
-        FileUtils.rm(path(request))
-        nil
-      else
-        begin
-          contents = File.read(path(request))
-          Marshal.load(contents)
-        rescue ArgumentError
-          FileUtils.rm(path(request))
-          nil
-        end
-      end
-    end
-
-    def set(request, response)
-      File.write(path(request), Marshal.dump(response))
-      @memory[request] = response
-    end
-
-    private
-
-    # The `request` is a Typhoeus::Request, which provides a
-    # unique `cache_key` string for exactly this sort of thing.
-    def path(request)
-      File.join(@dir, request.cache_key)
-    end
-  end
-end

data/lib/site-inspector/rails_cache.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-class SiteInspector
-  class RailsCache
-    def get(request)
-      Rails.cache.read(request)
-    end
-
-    def set(request, response)
-      Rails.cache.write(request, response)
-    end
-  end
-end

data/spec/checks/site_inspector_endpoint_sniffer_spec.rb

@@ -1,150 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe SiteInspector::Endpoint::Sniffer do
-  def stub_header(header, value)
-    allow(subject.endpoint.headers).to receive(:headers) { { header => value } }
-  end
-
-  def set_cookie(key, value)
-    cookies = [
-      CGI::Cookie.new(
-        'name' => 'foo',
-        'value' => 'bar',
-        'domain' => 'example.com',
-        'path' => '/'
-      ),
-      CGI::Cookie.new(
-        'name' => key,
-        'value' => value,
-        'domain' => 'example.com',
-        'path' => '/'
-      )
-    ].map(&:to_s)
-
-    stub_request(:get, 'http://example.com/')
-      .to_return(status: 200, body: '')
-
-    stub_request(:head, 'http://example.com/')
-      .to_return(status: 200, headers: { 'set-cookie' => cookies })
-  end
-
-  context 'stubbed body' do
-    subject do
-      body = <<-BODY
-        <html>
-          <head>
-            <link rel='stylesheet' href='/wp-content/themes/foo/style.css type='text/css' media='all' />
-          </head>
-          <body>
-            <h1>Some page</h1>
-            <script>
-              jQuery(); googletag.pubads();
-            </script>
-            <script>
-              var _gaq=[['_setAccount','UA-12345678-1'],['_trackPageview']];
-              (function(d,t){var g=d.createElement(t),s=d.getElementsByTagName(t)[0];
-              g.src=('https:'==location.protocol?'//ssl':'//www')+'.google-analytics.com/ga.js';
-              s.parentNode.insertBefore(g,s)}(document,'script'));
-            </script>
-          </body>
-        </html>
-      BODY
-
-      stub_request(:get, 'http://example.com/')
-        .to_return(status: 200, body: body)
-
-      stub_request(:head, 'http://example.com/')
-        .to_return(status: 200)
-      endpoint = SiteInspector::Endpoint.new('http://example.com')
-      described_class.new(endpoint)
-    end
-
-    it 'sniffs' do
-      sniff = subject.send(:sniff, :cms)
-      expect(sniff).to be(:wordpress)
-    end
-
-    it 'detects the CMS' do
-      expect(subject.framework).to be(:wordpress)
-    end
-
-    it 'detects the analytics' do
-      expect(subject.analytics).to be(:google_analytics)
-    end
-
-    it 'detects javascript' do
-      expect(subject.javascript).to be(:jquery)
-    end
-
-    it 'detects advertising' do
-      expect(subject.advertising).to be(:adsense)
-    end
-
-    it 'knows wordpress is open source' do
-      expect(subject.open_source?).to be(true)
-    end
-  end
-
-  context 'no body' do
-    subject do
-      endpoint = SiteInspector::Endpoint.new('http://example.com')
-      described_class.new(endpoint)
-    end
-
-    it "knows when something isn't open source" do
-      set_cookie('foo', 'bar')
-      expect(subject.open_source?).to be(false)
-    end
-
-    it 'detects PHP' do
-      set_cookie('PHPSESSID', '1234')
-      expect(subject.framework).to be(:php)
-      expect(subject.open_source?).to be(true)
-    end
-
-    it 'detects Expression Engine' do
-      set_cookie('exp_csrf_token', '1234')
-      expect(subject.framework).to be(:expression_engine)
-      expect(subject.open_source?).to be(true)
-    end
-
-    it 'detects cowboy' do
-      stub_request(:get, 'http://example.com/')
-        .to_return(status: 200, body: '')
-
-      stub_request(:head, 'http://example.com/')
-        .to_return(status: 200, headers: { 'server' => 'Cowboy' })
-
-      expect(subject.framework).to be(:cowboy)
-      expect(subject.open_source?).to be(true)
-    end
-
-    it 'detects ColdFusion' do
-      cookies = [
-        CGI::Cookie.new(
-          'name' => 'CFID',
-          'value' => '1234',
-          'domain' => 'example.com',
-          'path' => '/'
-        ),
-        CGI::Cookie.new(
-          'name' => 'CFTOKEN',
-          'value' => '5678',
-          'domain' => 'example.com',
-          'path' => '/'
-        )
-      ].map(&:to_s)
-
-      stub_request(:get, 'http://example.com/')
-        .to_return(status: 200, body: '')
-
-      stub_request(:head, 'http://example.com/')
-        .to_return(status: 200, headers: { 'set-cookie' => cookies })
-
-      expect(subject.framework).to be(:coldfusion)
-      expect(subject.open_source?).to be(false)
-    end
-  end
-end

data/spec/checks/site_inspector_endpoint_wappalyzer_spec.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe SiteInspector::Endpoint::Wappalyzer do
-  subject { described_class.new(endpoint) }
-
-  let(:domain) { 'http://ben.balter.com.com' }
-  let(:endpoint) { SiteInspector::Endpoint.new(domain) }
-  let(:url) { "https://api.wappalyzer.com/lookup/v2/?urls=#{domain}/" }
-
-  before do
-    path = File.expand_path '../fixtures/wappalyzer.json', __dir__
-    body = File.read path
-    stub_request(:get, url).to_return(status: 200, body: body)
-  end
-
-  it 'returns the API response' do
-    expected = {
-      'Analytics' => ['Google Analytics'],
-      'CDN' => %w[Cloudflare Fastly],
-      'Caching' => ['Varnish'],
-      'Other' => %w[Disqus Jekyll],
-      'PaaS' => ['GitHub Pages'],
-      'Web frameworks' => ['Ruby on Rails']
-    }
-    expect(subject.to_h).to eql(expected)
-  end
-
-  it 'fails gracefully' do
-    stub_request(:get, url).to_return(status: 400, body: '')
-    expect(subject.to_h).to eql({})
-  end
-end

data/spec/site_inspector_cache_spec.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe SiteInspector::Cache do
-  it 'stores a cache value' do
-    subject.set 'foo', 'bar'
-    expect(subject.instance_variable_get('@memory')['foo']).to eql('bar')
-  end
-
-  it 'retrieves values from the cache' do
-    subject.instance_variable_set('@memory', 'foo' => 'bar')
-    expect(subject.get('foo')).to eql('bar')
-  end
-end

data/spec/site_inspector_disk_cache_spec.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe SiteInspector::DiskCache do
-  subject { described_class.new(tmpdir) }
-
-  before do
-    FileUtils.rm_rf(tmpdir)
-    Dir.mkdir(tmpdir)
-  end
-
-  it 'writes a value to disk' do
-    foo = Typhoeus::Request.new('foo')
-
-    path = File.expand_path foo.cache_key, tmpdir
-    expect(File.exist?(path)).to be(false)
-
-    subject.set foo, 'bar'
-
-    expect(File.exist?(path)).to be(true)
-    expect(File.open(path).read).to eql("I\"bar:ET")
-  end
-
-  it 'reads a value from disk' do
-    foo = Typhoeus::Request.new('foo')
-
-    path = File.expand_path foo.cache_key, tmpdir
-    File.write(path, "I\"bar:ET")
-    expect(subject.get(foo)).to eql('bar')
-  end
-
-  it "calculates a file's path" do
-    foo = Typhoeus::Request.new('foo')
-
-    path = File.expand_path foo.cache_key, tmpdir
-    expect(subject.send(:path, foo)).to eql(path)
-  end
-end