new_cms_scanner 0.13.7

data/lib/cms_scanner/target/scope.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  # Scope system logic
+  class Target < WebSite
+    # @return [ Array<PublicSuffix::Domain, String> ]
+    def scope
+      @scope ||= Scope.new
+    end
+
+    # @param [ String, Addressable::URI ] url An absolute URL or URI
+    #
+    # @return [ Boolean ] true if the url given is in scope
+    def in_scope?(url_or_uri)
+      url_or_uri = Addressable::URI.parse(url_or_uri.strip) unless url_or_uri.is_a?(Addressable::URI)
+
+      scope.include?(url_or_uri.host)
+    rescue StandardError
+      false
+    end
+
+    # @param [ Typhoeus::Response ] res
+    # @param [ String ] xpath
+    #
+    # @yield [ Addressable::URI, Nokogiri::XML::Element ] The in scope url and its associated tag
+    #
+    # @return [ Array<Addressable::URI> ] The in scope absolute URIs detected in the response's body
+    #
+    # @note It is highly recommended to use the xpath parameter to focus on the uris needed, as this method can be quite
+    #       time consuming when there are a lof of uris to check
+    def in_scope_uris(res, xpath = '//@href|//@src|//@data-src')
+      found = []
+
+      uris_from_page(res, xpath) do |uri, tag|
+        next unless in_scope?(uri)
+
+        yield uri, tag if block_given?
+
+        found << uri
+      end
+
+      found
+    end
+
+    # Similar to Target#url_pattern but considering the in scope domains as well
+    #
+    # @return [ Regexp ] The pattern related to the target url and in scope domains,
+    #                    it also matches escaped /, such as in JSON JS data: http:\/\/t.com\/
+    # rubocop:disable Metrics/AbcSize
+    def scope_url_pattern
+      return @scope_url_pattern if @scope_url_pattern
+
+      domains = [uri.host + uri.path]
+
+      domains += if scope.domains.empty?
+                   Array(scope.invalid_domains[1..-1])
+                 else
+                   Array(scope.domains[1..-1]).map(&:to_s) + scope.invalid_domains
+                 end
+
+      domains.map! { |d| Regexp.escape(d.delete_suffix('/')).gsub('\*', '.*').gsub('/', '\\\\\?/') }
+
+      domains[0].gsub!(Regexp.escape(uri.host), "#{Regexp.escape(uri.host)}(?::\\d+)?") if uri.port
+
+      @scope_url_pattern = %r{https?:\\?/\\?/(?:#{domains.join('|')})\\?/?}i
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    # Scope Implementation
+    class Scope
+      # @return [ Array<PublicSuffix::Domain> ] The valid domains in scope
+      def domains
+        @domains ||= []
+      end
+
+      # @return [ Array<String> ] The invalid domains in scope (such as IP addresses etc)
+      def invalid_domains
+        @invalid_domains ||= []
+      end
+
+      def <<(element)
+        if PublicSuffix.valid?(element, ignore_private: true)
+          domains << PublicSuffix.parse(element, ignore_private: true)
+        else
+          invalid_domains << element
+        end
+      end
+
+      # @return [ Boolean ] Wether or not the host is in the scope
+      def include?(host)
+        if PublicSuffix.valid?(host, ignore_private: true)
+          domain = PublicSuffix.parse(host, ignore_private: true)
+
+          domains.each { |d| return true if domain.match(d) }
+        else
+          invalid_domains.each { |d| return true if host == d }
+        end
+
+        false
+      end
+    end
+  end
+end

data/lib/cms_scanner/target/server/apache.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  class Target < WebSite
+    module Server
+      # Some Apche specific implementation
+      module Apache
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Symbol ] :Apache
+        def server(_path = nil, _params = {})
+          :Apache
+        end
+
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Array<String> ] The first level of directories/files listed,
+        #                           or an empty array if none
+        def directory_listing_entries(path = nil, params = {})
+          super(path, params, 'td a')
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/target/server/generic.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  class Target < WebSite
+    module Server
+      # Generic Server methods
+      module Generic
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Symbol ] The detected remote server (:Apache, :IIS, :Nginx)
+        def server(path = nil, params = {})
+          headers = headers(path, params)
+
+          return unless headers
+
+          case headers[:server]
+          when /\Aapache/i
+            :Apache
+          when /\AMicrosoft-IIS/i
+            :IIS
+          when /\Anginx/
+            :Nginx
+          end
+        end
+
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Hash ] The headers
+        def headers(path = nil, params = {})
+          # The HEAD method might be rejected by some servers ... maybe switch to GET ?
+          NS::Browser.head(url(path), params).headers
+        end
+
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Boolean ] true if url(path) has the directory
+        #                          listing enabled, false otherwise
+        def directory_listing?(path = nil, params = {})
+          res = NS::Browser.get(url(path), params)
+
+          res.code == 200 && res.body.include?('<h1>Index of')
+        end
+
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        # @param [ String ] selector
+        # @param [ Regexp ] ignore
+        #
+        # @return [ Array<String> ] The first level of directories/files listed,
+        #                           or an empty array if none
+        def directory_listing_entries(path = nil, params = {}, selector = 'pre a', ignore = /parent directory/i)
+          return [] unless directory_listing?(path, params)
+
+          found = []
+
+          NS::Browser.get(url(path), params).html.css(selector).each do |node|
+            entry = node.text.to_s
+
+            next if entry&.match?(ignore)
+
+            found << entry
+          end
+
+          found
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/target/server/iis.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  class Target < WebSite
+    module Server
+      # Some IIS specific implementation
+      module IIS
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Symbol ] :IIS
+        def server(_path = nil, _params = {})
+          :IIS
+        end
+
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Boolean ] true if url(path) has the directory
+        #                          listing enabled, false otherwise
+        def directory_listing?(path = nil, params = {})
+          res = NS::Browser.get(url(path), params)
+
+          res.code == 200 && res.body =~ %r{<H1>#{uri.host} - /} ? true : false
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/target/server/nginx.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  class Target < WebSite
+    module Server
+      # Some Nginx specific implementation
+      module Nginx
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Symbol ] :Nginx
+        def server(_path = nil, _params = {})
+          :Nginx
+        end
+
+        # @param [ String ] path
+        # @param [ Hash ] params The request params
+        #
+        # @return [ Array<String> ] The first level of directories/files listed,
+        #                           or an empty array if none
+        def directory_listing_entries(path = nil, params = {})
+          super(path, params, 'pre a', /\A\.\./i)
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/target/server.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'cms_scanner/target/server/generic'
+require 'cms_scanner/target/server/apache'
+require 'cms_scanner/target/server/iis'
+require 'cms_scanner/target/server/nginx'

data/lib/cms_scanner/target.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'cms_scanner/web_site'
+require 'cms_scanner/target/platform'
+require 'cms_scanner/target/server'
+require 'cms_scanner/target/scope'
+require 'cms_scanner/target/hashes'
+
+module CMSScanner
+  # Target to Scan
+  class Target < WebSite
+    include Server::Generic
+
+    # @param [ String ] url
+    # @param [ Hash ] opts
+    # @option opts [ Array<PublicSuffix::Domain, String> ] :scope
+    def initialize(url, opts = {})
+      super(url, opts)
+
+      scope << uri.host
+      Array(opts[:scope]).each { |s| scope << s }
+    end
+
+    # @param [ Hash ] opts
+    #
+    # @return [ Findings ]
+    def interesting_findings(opts = {})
+      @interesting_findings ||= NS::Finders::InterestingFindings::Base.find(self, opts)
+    end
+
+    # Weteher or not vulnerabilities have been found.
+    # Used to set the exit code of the scanner
+    # and it should be overriden in the implementation
+    #
+    # @return [ Boolean ]
+    def vulnerable?
+      raise NotImplementedError
+    end
+
+    # @return [ Regexp ] The pattern related to the target url, also matches escaped /, such as
+    #                    in JSON JS data: http:\/\/t.com\/
+    def url_pattern
+      @url_pattern ||= Regexp.new(Regexp.escape(url).gsub(/https?/i, 'https?').gsub('/', '\\\\\?/'), Regexp::IGNORECASE)
+    end
+
+    # @param [ String ] xpath
+    # @param [ Regexp ] pattern
+    # @param [ Typhoeus::Response, String ] page
+    #
+    # @return [ Array<Array<MatchData, Nokogiri::XML::Element>> ]
+    # @yield  [ MatchData, Nokogiri::XML::Element ]
+    def xpath_pattern_from_page(xpath, pattern, page = nil)
+      page    = NS::Browser.get(url(page)) unless page.is_a?(Typhoeus::Response)
+      matches = []
+
+      page.html.xpath(xpath).each do |node|
+        next unless node.text.strip =~ pattern
+
+        yield Regexp.last_match, node if block_given?
+
+        matches << [Regexp.last_match, node]
+      end
+
+      matches
+    end
+
+    # @param [ Regexp ] pattern
+    # @param [ Typhoeus::Response, String ] page
+    #
+    # @return [ Array<Array<MatchData, Nokogiri::XML::Comment>> ]
+    # @yield  [ MatchData, Nokogiri::XML::Comment ]
+    def comments_from_page(pattern, page = nil)
+      xpath_pattern_from_page('//comment()', pattern, page) do |match, node|
+        yield match, node if block_given?
+      end
+    end
+
+    # @param [ Regexp ] pattern
+    # @param [ Typhoeus::Response, String ] page
+    #
+    # @return [ Array<Array<MatchData, Nokogiri::XML::Element>> ]
+    # @yield  [ MatchData, Nokogiri::XML::Element ]
+    def javascripts_from_page(pattern, page = nil)
+      xpath_pattern_from_page('//script', pattern, page) do |match, node|
+        yield match, node if block_given?
+      end
+    end
+
+    # @param [ Typhoeus::Response, String ] page
+    # @param [ String ] xpath
+    #
+    # @yield [ Addressable::URI, Nokogiri::XML::Element ] The url and its associated tag
+    #
+    # @return [ Array<Addressable::URI> ] The absolute URIs detected in the response's body from the HTML tags
+    #
+    # @note It is highly recommended to use the xpath parameter to focus on the uris needed, as this method can be quite
+    #       time consuming when there are a lof of uris to check
+    def uris_from_page(page = nil, xpath = '//@href|//@src|//@data-src')
+      page    = NS::Browser.get(url(page)) unless page.is_a?(Typhoeus::Response)
+      found   = []
+
+      page.html.xpath(xpath).each do |node|
+        attr_value = node.text.to_s
+
+        next unless attr_value && !attr_value.empty?
+
+        node_uri = begin
+          uri.join(attr_value.strip)
+        rescue StandardError
+          # Skip potential malformed URLs etc.
+          next
+        end
+
+        next unless node_uri.host
+
+        yield node_uri, node.parent if block_given? && !found.include?(node_uri)
+
+        found << node_uri
+      end
+
+      found.uniq
+    end
+  end
+end

data/lib/cms_scanner/typhoeus/hydra.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Typhoeus
+  # Ensure a clean abort of hydra
+  # See https://github.com/typhoeus/typhoeus/issues/439
+  class Hydra
+    def abort
+      super
+      run
+    end
+  end
+end

data/lib/cms_scanner/typhoeus/response.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Typhoeus
+  # Custom Response class
+  class Response
+    # @return [ Nokogiri::XML ] The response's body parsed by Nokogiri::HTML
+    def html
+      @html ||= Nokogiri::HTML(body.encode('UTF-8', invalid: :replace, undef: :replace))
+    end
+
+    # @return [ Nokogiri::XML ] The response's body parsed by Nokogiri::XML
+    def xml
+      @xml ||= Nokogiri::XML(body.encode('UTF-8', invalid: :replace, undef: :replace))
+    end
+
+    # Override of the original to ensure an integer is returned
+    # @return [ Integer ]
+    def request_size
+      super || 0
+    end
+
+    # @return [ Integer ]
+    def size
+      (body.nil? ? 0 : body.size) + (response_headers.nil? ? 0 : response_headers.size)
+    end
+  end
+end

data/lib/cms_scanner/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Version
+module CMSScanner
+  VERSION = '0.13.7'
+end

data/lib/cms_scanner/vulnerability.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  # Generic Vulnerability
+  class Vulnerability
+    include References
+
+    attr_reader :title, :type, :fixed_in, :introduced_in, :cvss
+
+    # @param [ String ] title
+    # @param [ Hash ] references
+    # @option references [ Array<String>, String ] :cve
+    # @option references [ Array<String>, String ] :secunia
+    # @option references [ Array<String>, String ] :osvdb
+    # @option references [ Array<String>, String ] :exploitdb
+    # @option references [ Array<String> ] :url URL(s) to related advisories etc
+    # @option references [ Array<String>, String ] :metasploit The related metasploit module(s)
+    # @option references [ Array<String> ] :youtube
+    # @param [ String ] type
+    # @param [ String ] fixed_in
+    # @param [ String ] introduced_in
+    # @param [ HashSymbol ] cvss
+    # @option cvss [ String ] :score
+    # @option cvss [ String ] :vector
+    def initialize(title, references: {}, type: nil, fixed_in: nil, introduced_in: nil, cvss: nil)
+      @title         = title
+      @type          = type
+      @fixed_in      = fixed_in
+      @introduced_in = introduced_in
+      @cvss          = { score: cvss[:score], vector: cvss[:vector] } if cvss
+
+      self.references = references
+    end
+
+    # param [ Vulnerability ] other
+    #
+    # @return [ Boolean ]
+    def ==(other)
+      title == other.title &&
+        type == other.type &&
+        references == other.references &&
+        fixed_in == other.fixed_in &&
+        cvss == other.cvss
+    end
+  end
+end

data/lib/cms_scanner/web_site.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  # WebSite Implementation
+  class WebSite
+    attr_reader :uri, :opts
+
+    # @param [ String ] site_url
+    # @param [ Hash ] opts
+    def initialize(site_url, opts = {})
+      self.url = site_url
+      @opts    = opts
+    end
+
+    def url=(site_url)
+      new_url = site_url.dup
+
+      # Add a trailing slash to the URL
+      new_url << '/' if new_url[-1, 1] != '/'
+
+      # Use the validator to ensure the URL has a correct format
+      OptParseValidator::OptURL.new([]).validate(new_url)
+
+      @uri = Addressable::URI.parse(new_url).normalize
+    end
+
+    # @param [ String ] path Optional path to merge with the uri
+    #
+    # @return [ String ]
+    def url(path = nil)
+      return @uri.to_s unless path
+
+      @uri.join(Addressable::URI.encode(path).gsub('#', '%23')).to_s
+    end
+
+    # @return [ String ] The IP address of the target
+    def ip
+      @ip ||= IPSocket.getaddress(uri.host)
+    rescue SocketError
+      'Unknown'
+    end
+
+    attr_writer :homepage_res
+
+    # @return [ Typhoeus::Response ]
+    #
+    # As webmock does not support redirects mocking, coverage is ignored
+    # :nocov:
+    def homepage_res
+      @homepage_res ||= NS::Browser.get_and_follow_location(url)
+    end
+    # :nocov:
+
+    # @return [ String ]
+    def homepage_url
+      @homepage_url ||= homepage_res.effective_url
+    end
+
+    # @return [ Typhoeus::Response ]
+    def error_404_res
+      @error_404_res ||= NS::Browser.get_and_follow_location(error_404_url)
+    end
+
+    # @return [ String ] The URL of an unlikely existant page
+    def error_404_url
+      @error_404_url ||= uri.join("#{Digest::MD5.hexdigest(rand(999_999).to_s)[0..6]}.html").to_s
+    end
+
+    # Checks if the remote website is up.
+    #
+    # @param [ String ] path
+    #
+    # @return [ Boolean ]
+    def online?(path = nil)
+      NS::Browser.get(url(path)).code.nonzero? ? true : false
+    end
+
+    # @param [ String ] path
+    #
+    # @return [ Boolean ]
+    def http_auth?(path = nil)
+      NS::Browser.get(url(path)).code == 401
+    end
+
+    # @param [ String ] path
+    #
+    # @return [ Boolean ]
+    def access_forbidden?(path = nil)
+      NS::Browser.get(url(path)).code == 403
+    end
+
+    # @param [ String ] path
+    #
+    # @return [ Boolean ]
+    def proxy_auth?(path = nil)
+      NS::Browser.get(url(path)).code == 407
+    end
+
+    # @param [ String ] url
+    #
+    # @return [ String ] The redirection url or nil
+    #
+    # As webmock does not support redirects mocking, coverage is ignored
+    # :nocov:
+    def redirection(url = nil)
+      url ||= @uri.to_s
+
+      return unless [301, 302].include?(NS::Browser.get(url).code)
+
+      res = NS::Browser.get(url, followlocation: true, maxredirs: 10)
+
+      res.effective_url == url ? nil : res.effective_url
+    end
+    # :nocov:
+
+    # @return [ Hash ] The Typhoeus params to use to perform head requests
+    def head_or_get_params
+      @head_or_get_params ||= if NS::Browser.head(homepage_url).code == 405
+                                { method: :get, maxfilesize: 1 }
+                              else
+                                { method: :head }
+                              end
+    end
+
+    # Perform a HEAD request to the path provided, then if its response code
+    # is in the array of codes given, a GET is done and the response returned. Otherwise the
+    # HEAD response is returned.
+    #
+    # @param [ String ] path
+    # @param [ Array<String> ] codes
+    # @param [ Hash ] params The requests params
+    # @option params [ Hash ] :head Request params for the HEAD
+    # @option params [ hash ] :get Request params for the GET
+    #
+    # @return [ Typhoeus::Response ]
+    def head_and_get(path, codes = [200], params = {})
+      url_to_get  = url(path)
+      head_params = (params[:head] || {}).merge(head_or_get_params)
+
+      head_res = NS::Browser.forge_request(url_to_get, head_params).run
+
+      codes.include?(head_res.code) ? NS::Browser.get(url_to_get, params[:get] || {}) : head_res
+    end
+  end
+end