site-inspector 3.1.1 → 3.2.0

data/lib/site-inspector/checks/check.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Check
@@ -40,11 +42,11 @@ class SiteInspector
         end
 
         def enabled?
-          !!(@@enabled)
+          !!@@enabled
         end
 
         def enabled=(value)
-          @@enabled = !!(value)
+          @@enabled = !!value
         end
       end
     end

data/lib/site-inspector/checks/content.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Content < Check
@@ -16,7 +18,7 @@ class SiteInspector
         require 'nokogiri'
         @doc ||= Nokogiri::HTML response.body if response
       end
-      alias_method :doc, :document
+      alias doc document
 
       def body
         @body ||= document.to_s.force_encoding('UTF-8').encode('UTF-8', invalid: :replace, replace: '')
@@ -38,8 +40,16 @@ class SiteInspector
         document.internal_subset.external_id
       end
 
+      def generator
+        @generator ||= begin
+          tag = document.at('meta[name="generator"]')
+          tag['content'] if tag
+        end
+      end
+
       def prefetch
         return unless endpoint.up?
+
         options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
         ['robots.txt', 'sitemap.xml', 'humans.txt', random_path].each do |path|
           request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)
@@ -55,10 +65,11 @@ class SiteInspector
       def to_h
         prefetch
         {
-          doctype:     doctype,
+          doctype: doctype,
+          generator: generator,
           sitemap_xml: sitemap_xml?,
-          robots_txt:  robots_txt?,
-          humans_txt:  humans_txt?,
+          robots_txt: robots_txt?,
+          humans_txt: humans_txt?,
           proper_404s: proper_404s?
         }
       end

data/lib/site-inspector/checks/cookies.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Cookies < Check
@@ -5,12 +7,12 @@ class SiteInspector
         if cookie_header.nil? || cookie_header.empty?
           false
         elsif block_given?
-          all.any? { |cookie| block.call(cookie) }
+          all.any?(&block)
         else
           true
         end
       end
-      alias_method :cookies?, :any?
+      alias cookies? any?
 
       def all
         @cookies ||= cookie_header.map { |c| CGI::Cookie.parse(c) } if cookies?
@@ -22,7 +24,7 @@ class SiteInspector
 
       def secure?
         pairs = cookie_header.join('; ').split('; ') # CGI::Cookies#Parse doesn't seem to like secure headers
-        pairs.any? { |c| c.downcase == 'secure' } && pairs.any? { |c| c.downcase == 'httponly' }
+        pairs.any? { |c| c.casecmp('secure').zero? } && pairs.any? { |c| c.casecmp('httponly').zero? }
       end
 
       def to_h

data/lib/site-inspector/checks/dns.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Dns < Check
@@ -25,7 +27,7 @@ class SiteInspector
       def record?(type)
         records.any? { |record| record.type == type } || query(type).count != 0
       end
-      alias_method :has_record?, :record?
+      alias has_record? record?
 
       def dnssec?
         @dnssec ||= has_record? 'DNSKEY'
@@ -52,7 +54,7 @@ class SiteInspector
       end
 
       def google_apps?
-        @google ||= records.any? do |record|
+        @google_apps ||= records.any? do |record|
           record.type == 'MX' && record.exchange.to_s =~ /google(mail)?\.com\.?\z/i
         end
       end
@@ -62,7 +64,6 @@ class SiteInspector
       end
 
       def ip
-        require 'resolv'
         @ip ||= Resolv.getaddress host
       rescue Resolv::ResolvError
         nil
@@ -87,14 +88,15 @@ class SiteInspector
 
       def to_h
         return { error: LocalhostError } if localhost?
+
         {
-          dnssec:         dnssec?,
-          ipv6:           ipv6?,
-          cdn:            cdn,
+          dnssec: dnssec?,
+          ipv6: ipv6?,
+          cdn: cdn,
           cloud_provider: cloud_provider,
-          google_apps:    google_apps?,
-          hostname:       hostname,
-          ip:             ip
+          google_apps: google_apps?,
+          hostname: hostname,
+          ip: ip
         }
       end
 
@@ -118,7 +120,7 @@ class SiteInspector
         haystack = load_data(type)
         needle = haystack.find do |_name, domain|
           cnames.any? do |cname|
-            domain == cname.tld || domain == "#{cname.sld}.#{cname.tld}"
+            [cname.tld, "#{cname.sld}.#{cname.tld}"].include? domain
           end
         end
 
@@ -126,7 +128,7 @@ class SiteInspector
         return nil unless hostname
 
         needle = haystack.find do |_name, domain|
-          domain == hostname.tld || domain == "#{hostname.sld}.#{hostname.tld}"
+          [hostname.tld, "#{hostname.sld}.#{hostname.tld}"].include? domain
         end
 
         needle ? needle[0].to_sym : nil

data/lib/site-inspector/checks/headers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Headers < Check
@@ -44,9 +46,9 @@ class SiteInspector
 
       # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
       def all
-        @all ||= (response && response.headers) ? Hash[response.headers.map { |k, v| [k.downcase, v] }] : {}
+        @all ||= response&.headers ? response.headers.transform_keys(&:downcase) : {}
       end
-      alias_method :headers, :all
+      alias headers all
 
       def [](header)
         headers[header]
@@ -55,10 +57,10 @@ class SiteInspector
       def to_h
         {
           strict_transport_security: strict_transport_security || false,
-          content_security_policy:   content_security_policy || false,
-          click_jacking_protection:  click_jacking_protection || false,
-          server:                    server,
-          xss_protection:            xss_protection || false
+          content_security_policy: content_security_policy || false,
+          click_jacking_protection: click_jacking_protection || false,
+          server: server,
+          xss_protection: xss_protection || false
         }
       end
     end

data/lib/site-inspector/checks/hsts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     # Utility parser for HSTS headers.
@@ -5,7 +7,8 @@ class SiteInspector
     class Hsts < Check
       def valid?
         return false unless header
-        pairs.none? { |key, value| "#{key}#{value}" =~ /[\s\'\"]/ }
+
+        pairs.none? { |key, value| "#{key}#{value}" =~ /[\s'"]/ }
       end
 
       def max_age
@@ -13,16 +16,17 @@ class SiteInspector
       end
 
       def include_subdomains?
-        pairs.keys.include? :includesubdomains
+        pairs.key?(:includesubdomains)
       end
 
       def preload?
-        pairs.keys.include? :preload
+        pairs.key?(:preload)
       end
 
       def enabled?
         return false unless max_age
-        max_age > 0
+
+        max_age.positive?
       end
 
       # Google's minimum max-age for automatic preloading
@@ -32,12 +36,12 @@ class SiteInspector
 
       def to_h
         {
-          valid:              valid?,
-          max_age:            max_age,
+          valid: valid?,
+          max_age: max_age,
           include_subdomains: include_subdomains?,
-          preload:            preload?,
-          enabled:            enabled?,
-          preload_ready:      preload_ready?
+          preload: preload?,
+          enabled: enabled?,
+          preload_ready: preload_ready?
         }
       end
 
@@ -61,9 +65,9 @@ class SiteInspector
           directives.each do |directive|
             key, value = directive.downcase.split('=')
 
-            if value =~ /\".*\"/
-              value = value.sub(/^\"/, '')
-              value = value.sub(/\"$/, '')
+            if /".*"/.match?(value)
+              value = value.sub(/^"/, '')
+              value = value.sub(/"$/, '')
             end
 
             pairs[key.to_sym] = value

data/lib/site-inspector/checks/https.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Https < Check
@@ -23,7 +25,7 @@ class SiteInspector
 
       def to_h
         {
-          valid:       valid?,
+          valid: valid?,
           return_code: response.return_code
         }
       end

data/lib/site-inspector/checks/sniffer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Endpoint
     class Sniffer < Check
@@ -13,7 +15,7 @@ class SiteInspector
         :php,
         :expression_engine,
         :cowboy
-      ]
+      ].freeze
 
       def framework
         cms = sniff :cms
@@ -21,7 +23,8 @@ class SiteInspector
         return :expression_engine if endpoint.cookies.any? { |c| c.keys.first =~ /^exp_/ }
         return :php if endpoint.cookies['PHPSESSID']
         return :coldfusion if endpoint.cookies['CFID'] && endpoint.cookies['CFTOKEN']
-        return :cowboy if endpoint.headers.server.to_s.downcase == 'cowboy'
+        return :cowboy if endpoint.headers.server.to_s.casecmp('cowboy').zero?
+
         nil
       end
 
@@ -43,9 +46,9 @@ class SiteInspector
 
       def to_h
         {
-          framework:   framework,
-          analytics:   analytics,
-          javascript:  javascript,
+          framework: framework,
+          analytics: analytics,
+          javascript: javascript,
           advertising: advertising
         }
       end
@@ -55,8 +58,8 @@ class SiteInspector
       def sniff(type)
         require 'sniffles'
         results = Sniffles.sniff(endpoint.content.body, type).select { |_name, meta| meta[:found] }
-        results.keys.first if results
-      rescue
+        results&.keys&.first
+      rescue StandardError
         nil
       end
     end

data/lib/site-inspector/checks/wappalyzer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Wappalyzer < Check
+      ENDPOINT = 'https://api.wappalyzer.com/lookup/v2/'
+
+      def to_h
+        return {} unless data['technologies']
+
+        @to_h ||= begin
+          technologies = {}
+          data['technologies'].each do |t|
+            category = t['categories'].first
+            category = category ? category['name'] : 'Other'
+            technologies[category] ||= []
+            technologies[category].push t['name']
+          end
+
+          technologies
+        end
+      end
+
+      private
+
+      def request
+        @request ||= begin
+          options = SiteInspector.typhoeus_defaults
+          headers = options[:headers].merge({ "x-api-key": api_key })
+          options = options.merge(method: :get, headers: headers)
+          Typhoeus::Request.new(url, options)
+        end
+      end
+
+      def data
+        return {} unless api_key && api_key != ''
+
+        @data ||= begin
+          SiteInspector.hydra.queue(request)
+          SiteInspector.hydra.run
+
+          response = request.response
+          if response.success?
+            JSON.parse(response.body).first
+          else
+            {}
+          end
+        end
+      end
+
+      def url
+        url = Addressable::URI.parse(ENDPOINT)
+        url.query_values = { urls: endpoint.uri }
+        url
+      end
+
+      def api_key
+        @api_key ||= ENV['WAPPALYZER_API_KEY']
+      end
+    end
+  end
+end

data/lib/site-inspector/checks/whois.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class SiteInspector
+  class Endpoint
+    class Whois < Check
+      def domain
+        @domain ||= whois.lookup host
+      end
+
+      def ip
+        @ip ||= whois.lookup ip_address
+      end
+
+      def to_h
+        {
+          domain: record_to_h(domain),
+          ip: record_to_h(ip)
+        }
+      end
+
+      private
+
+      def record_to_h(record)
+        record.content.scan(/^\s*(.*?):\s*(.*?)\r?\n/).to_h
+      end
+
+      def ip_address
+        @ip_address ||= Resolv.getaddress host
+      end
+
+      def whois
+        @whois ||= ::Whois::Client.new
+      end
+    end
+  end
+end

data/lib/site-inspector/disk_cache.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class DiskCache
     def initialize(dir = nil, replace = nil)

data/lib/site-inspector/domain.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 class SiteInspector
   class Domain
     attr_reader :host
 
     def initialize(host)
       host = host.downcase
-      host = host.sub(/^https?\:/, '')
+      host = host.sub(/^https?:/, '')
       host = host.sub(%r{^/+}, '')
       host = host.sub(/^www\./, '')
       uri = Addressable::URI.parse "//#{host}"
@@ -82,7 +84,8 @@ class SiteInspector
     # TODO: don't need to require that the HTTPS cert is valid for this purpose.
     def enforces_https?
       return false unless https?
-      endpoints.select(&:http?).all? { |e| !e.up? || (e.redirect && e.redirect.https?) }
+
+      endpoints.select(&:http?).all? { |e| !e.up? || e.redirect&.https? }
     end
 
     # we can say that a canonical HTTPS site "defaults" to HTTPS,
@@ -91,7 +94,7 @@ class SiteInspector
     #
     # TODO: not implemented.
     def defaults_https?
-      fail 'Not implemented. Halp?'
+      raise 'Not implemented. Halp?'
     end
 
     # HTTPS is "downgraded" if both:
@@ -102,6 +105,7 @@ class SiteInspector
     # TODO: the redirect must be internal.
     def downgrades_https?
       return false unless https?
+
       canonical_endpoint.redirect? && canonical_endpoint.redirect.http?
     end
 
@@ -129,7 +133,7 @@ class SiteInspector
       return true if endpoints.select(&:root?).all? { |e| !e.up? }
 
       # Does either root endpoint redirect to a www endpoint?
-      endpoints.select(&:root?).any? { |e| e.redirect && e.redirect.www? }
+      endpoints.select(&:root?).any? { |e| e.redirect&.www? }
     end
 
     # A domain is "canonically" at https if:
@@ -160,7 +164,7 @@ class SiteInspector
       return true if endpoints.select(&:http?).all? { |e| !e.up? }
 
       # at least one http endpoint redirects immediately to https
-      endpoints.select(&:http?).any? { |e| e.redirect && e.redirect.https? }
+      endpoints.select(&:http?).any? { |e| e.redirect&.https? }
     end
 
     # A domain redirects if
@@ -168,6 +172,7 @@ class SiteInspector
     # 2. All endpoints are either down or an external redirect
     def redirect?
       return false unless redirect
+
       endpoints.all? { |e| !e.up? || e.external_redirect? }
     end
 
@@ -178,7 +183,7 @@ class SiteInspector
 
     # HSTS on the canonical domain?
     def hsts?
-      canonical_endpoint.hsts && canonical_endpoint.hsts.enabled?
+      canonical_endpoint.hsts&.enabled?
     end
 
     def hsts_subdomains?
@@ -187,6 +192,7 @@ class SiteInspector
 
     def hsts_preload_ready?
       return false unless hsts_subdomains?
+
       endpoints.find { |e| e.root? && e.https? }.hsts.preload_ready?
     end
 
@@ -225,40 +231,40 @@ class SiteInspector
       prefetch
 
       hash = {
-        host:               host,
-        up:                 up?,
-        responds:           responds?,
-        www:                www?,
-        root:               root?,
-        https:              https?,
-        enforces_https:     enforces_https?,
-        downgrades_https:   downgrades_https?,
-        canonically_www:    canonically_www?,
-        canonically_https:  canonically_https?,
-        redirect:           redirect?,
-        hsts:               hsts?,
-        hsts_subdomains:    hsts_subdomains?,
+        host: host,
+        up: up?,
+        responds: responds?,
+        www: www?,
+        root: root?,
+        https: https?,
+        enforces_https: enforces_https?,
+        downgrades_https: downgrades_https?,
+        canonically_www: canonically_www?,
+        canonically_https: canonically_https?,
+        redirect: redirect?,
+        hsts: hsts?,
+        hsts_subdomains: hsts_subdomains?,
         hsts_preload_ready: hsts_preload_ready?,
         canonical_endpoint: canonical_endpoint.to_h(options)
       }
 
       if options['all']
-        hash.merge!(endpoints: {
-                      https: {
-                        root: endpoints[0].to_h(options),
-                        www:  endpoints[1].to_h(options)
-                      },
-                      http:  {
-                        root: endpoints[2].to_h(options),
-                        www:  endpoints[3].to_h(options)
-                      }
-                    })
+        hash[:endpoints] = {
+          https: {
+            root: endpoints[0].to_h(options),
+            www: endpoints[1].to_h(options)
+          },
+          http: {
+            root: endpoints[2].to_h(options),
+            www: endpoints[3].to_h(options)
+          }
+        }
       end
 
       hash
     end
 
-    def to_json
+    def to_json(*_args)
       to_h.to_json
     end
   end