new_cms_scanner 0.13.7

data/lib/cms_scanner/finders/finder/enumerator.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    class Finder
+      # Module to provide an easy way to enumerate items such as plugins, themes etc
+      module Enumerator
+        # @return [ Hash ]
+        def head_or_get_request_params
+          # Disabling the cache, as it causes a 'stack level too deep' exception
+          # with a large number of requests.
+          # See https://github.com/typhoeus/typhoeus/issues/408
+          @head_or_get_request_params ||= target.head_or_get_params.merge(cache_ttl: 0)
+        end
+
+        # @return [ Array<Integer> ]
+        def valid_response_codes
+          @valid_response_codes ||= [200]
+        end
+
+        # @param [ Hash ] The target urls
+        # @param [ Hash ] opts
+        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
+        # @option opts [ Regexp ] :exclude_content
+        # @option opts [ Boolean, Array, String ] :check_full_response
+        #
+        # @yield [ Typhoeus::Response, String ]
+        def enumerate(urls, opts = {})
+          create_progress_bar(opts.merge(total: urls.size))
+
+          urls.each do |url, id|
+            request = browser.forge_request(url, head_or_get_request_params)
+
+            request.on_complete do |head_res|
+              progress_bar.increment
+
+              next unless valid_response_codes.include?(head_res.code)
+
+              next if opts[:exclude_content] && head_res.response_headers&.match(opts[:exclude_content])
+
+              head_or_full_res = maybe_get_full_response(head_res, opts)
+
+              yield head_or_full_res, id if head_or_full_res
+            end
+
+            hydra.queue(request)
+          end
+
+          hydra.run
+        end
+
+        # @param [ Typhoeus::Response ] head_res
+        # @param [ Hash ] opts
+        #
+        # @return [ Typhoeus::Response, nil ]
+        def maybe_get_full_response(head_res, opts)
+          return head_res unless opts[:check_full_response] == true ||
+                                 Array(opts[:check_full_response]).include?(head_res.code)
+
+          full_res = NS::Browser.get(head_res.effective_url, full_request_params)
+
+          return unless valid_response_codes.include?(full_res.code)
+
+          return if target.homepage_or_404?(full_res) ||
+                    (opts[:exclude_content] && full_res.body&.match(opts[:exclude_content]))
+
+          full_res
+        end
+
+        # @return [ Hash ]
+        def full_request_params
+          @full_request_params ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/finder/fingerprinter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    class Finder
+      # Module to provide an easy way to fingerprint things such as versions
+      module Fingerprinter
+        include Enumerator
+
+        # @param [ Hash ] fingerprints The fingerprints
+        # Format should be like the following:
+        # {
+        #   file_path_1: {
+        #     md5_hash_1: version_1,
+        #     md5_hash_2: [version_2]
+        #   },
+        #   file_path_2: {
+        #     md5_hash_3: [version_1, version_2],
+        #     md5_hash_4: version_3
+        #   }
+        # }
+        # Note that the version can either be an array or a string
+        #
+        # @param [ Hash ] opts
+        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
+        #
+        # @yield [ Mixed, String, String ] version/s, url, hash The version associated to the
+        #                                                       fingerprint of the url
+        def fingerprint(fingerprints, opts = {})
+          enum_opts = opts.merge(check_full_response: 200)
+
+          enumerate(fingerprints.transform_keys { |k| target.url(k) }, enum_opts) do |res, fingerprint|
+            md5sum = hexdigest(res.body)
+
+            next unless fingerprint.key?(md5sum)
+
+            yield fingerprint[md5sum], res.effective_url, md5sum
+          end
+        end
+
+        # @return [ String ] The hashed value for the given body
+        def hexdigest(body)
+          Digest::MD5.hexdigest(body)
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/finder/smart_url_checker/findings.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    class Finder
+      module SmartURLChecker
+        # Findings
+        class Findings < Array
+          def <<(finding)
+            return self unless finding
+
+            each do |f|
+              next unless f == finding && f.found_by == finding.found_by
+
+              # This makes sure entries added are unique
+              # and prevent pages redirecting to the same one to be added twice
+              entries_to_add = finding.interesting_entries - f.interesting_entries
+              return self if entries_to_add.empty?
+
+              entries_to_add.each { |entry| f.interesting_entries << entry }
+
+              f.confidence += finding.confidence
+
+              return self
+            end
+
+            super(finding)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/finder/smart_url_checker.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'cms_scanner/finders/finder/smart_url_checker/findings'
+
+module CMSScanner
+  module Finders
+    class Finder
+      # Smart URL Checker
+      # Typically used when some URLs are potentially in the homepage. If they are found
+      # in it, they will be checked in the #passive (like a browser/client would do when loading the page).
+      # Otherwise they will be checked in the #aggressive
+      module SmartURLChecker
+        # @param [ Array<String> ] urls
+        # @param [ Hash ] opts
+        #
+        # @return []
+        def process_urls(_urls, _opts = {})
+          raise NotImplementedError
+        end
+
+        # @param [ Hash ] opts
+        #
+        # @return [ Array<Finding> ]
+        def passive(opts = {})
+          process_urls(passive_urls(opts), opts)
+        end
+
+        # @param [ Hash ] opts
+        #
+        # @return [ Array<String> ]
+        def passive_urls(_opts = {})
+          target.in_scope_uris(target.homepage_res, passive_urls_xpath).map(&:to_s)
+        end
+
+        # @return [ String ]
+        def passive_urls_xpath
+          raise NotImplementedError
+        end
+
+        # @param [ Hash ] opts
+        #
+        # @return [ Array<Finding> ]
+        def aggressive(opts = {})
+          # To avoid scanning the same twice
+          urls = aggressive_urls(opts)
+          urls -= passive_urls(opts) if opts[:mode] == :mixed
+
+          process_urls(urls, opts)
+        end
+
+        # @param [ Hash ] opts
+        #
+        # @return [ Array<String> ]
+        def aggressive_urls(_opts = {})
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/finder.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'cms_scanner/finders/finder/smart_url_checker'
+require 'cms_scanner/finders/finder/enumerator'
+require 'cms_scanner/finders/finder/fingerprinter'
+require 'cms_scanner/finders/finder/breadth_first_dictionary_attack'
+
+module CMSScanner
+  module Finders
+    # Finder
+    class Finder
+      # Constants for common found_by
+      DIRECT_ACCESS = 'Direct Access (Aggressive Detection)'
+
+      attr_accessor :target, :progress_bar
+
+      def initialize(target)
+        @target = target
+      end
+
+      # @return [ String ] The titleized name of the finder
+      def titleize
+        # Put a _ char before any digits except those at the end, which will be replaced by a space
+        # Otherwise, class such as Error404Page are returned as Error404 Page instead of Error 404 page
+        # The keep_id_suffix is to concevert classes such as CssId to Css Id instead of Css
+
+        @titleize ||= self.class.to_s.demodulize.gsub(/(\d+)[a-z]+/i, '_\0').titleize(keep_id_suffix: true)
+      end
+
+      # @param [ Hash ] _opts
+      def passive(_opts = {}); end
+
+      # @param [ Hash ] _opts
+      def aggressive(_opts = {}); end
+
+      # @param [ Hash ] opts See https://github.com/jfelchner/ruby-progressbar/wiki/Options
+      # @option opts [ Boolean ] :show_progression
+      #
+      # @return [ ProgressBar::Base ]
+      def create_progress_bar(opts = {})
+        bar_opts          = { format: '%t %a <%B> (%c / %C) %P%% %e' }
+        bar_opts[:output] = ProgressBarNullOutput unless opts[:show_progression]
+
+        @progress_bar = ::ProgressBar.create(bar_opts.merge(opts))
+      end
+
+      # @return [ Browser ]
+      def browser
+        @browser ||= NS::Browser.instance
+      end
+
+      # @return [ Typhoeus::Hydra ]
+      def hydra
+        @hydra ||= browser.hydra
+      end
+
+      # @param [String, Class ] klass
+      # @return [ String ]
+      def found_by(klass = self.class)
+        labels = %w[aggressive passive]
+
+        caller_locations.each do |call|
+          label = call.label
+
+          next unless labels.include? label
+
+          title = klass.to_s.demodulize.gsub(/(\d+)[a-z]+/i, '_\0').titleize(keep_id_suffix: true)
+
+          return "#{title} (#{label.capitalize} Detection)"
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/finding.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # Finding
+    module Finding
+      # Fix for "Double/Dynamic Inclusion Problem"
+      def self.included(base)
+        base.include References
+        super(base)
+      end
+
+      FINDING_OPTS = %i[confidence confirmed_by references found_by interesting_entries].freeze
+
+      attr_accessor(*FINDING_OPTS)
+
+      # @return [ Array ]
+      def confirmed_by
+        @confirmed_by ||= []
+      end
+
+      # Should be overriden in child classes
+      # @return [ Array ]
+      def interesting_entries
+        @interesting_entries ||= []
+      end
+
+      # @return [ Integer ]
+      def confidence
+        @confidence ||= 0
+      end
+
+      # @param [ Integer ] value
+      def confidence=(value)
+        @confidence = value >= 100 ? 100 : value
+      end
+
+      # @param [ Hash ] opts
+      def parse_finding_options(opts = {})
+        FINDING_OPTS.each { |opt| send("#{opt}=", opts[opt]) if opts.key?(opt) }
+      end
+
+      # TODO: maybe also check for interesting_entries and confirmed_by ?
+      # So far this is used in specs only
+      def eql?(other)
+        self == other && confidence == other.confidence && found_by == other.found_by
+      end
+
+      def <=>(other)
+        to_s.downcase <=> other.to_s.downcase
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/findings.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # Findings container
+    class Findings < Array
+      # Override to include the confirmed_by logic
+      #
+      # @param [ Finding ] finding
+      def <<(finding)
+        return self unless finding
+
+        each do |found|
+          next unless found == finding
+
+          found.confirmed_by << finding
+          found.confidence += finding.confidence
+
+          return self
+        end
+
+        super(finding)
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/independent_finder.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # Independent Finder
+    module IndependentFinder
+      extend ActiveSupport::Concern
+
+      # See ActiveSupport::Concern
+      module ClassMethods
+        def find(target, opts = {})
+          new(target).find(opts)
+        end
+      end
+
+      # @param [ Hash ] opts
+      # @option opts [ Symbol ] mode (:mixed, :passive, :aggressive)
+      #
+      # @return [ Findings ]
+      def find(opts = {})
+        finders.run(opts)
+      end
+
+      # @return [ Array ]
+      def finders
+        @finders ||= NS::Finders::IndependentFinders.new
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/independent_finders.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # This class is designed to handle independent results
+    # which are not related with each others
+    # e.g: interesting files
+    class IndependentFinders < BaseFinders
+      # @param [ Hash ] opts
+      # @option opts [ Symbol ] mode :mixed, :passive or :aggressive
+      #
+      # @return [ Findings ]
+      def run(opts = {})
+        methods = symbols_from_mode(opts[:mode])
+
+        each do |finder|
+          methods.each do |symbol|
+            run_finder(finder, symbol, opts)
+          end
+        end
+
+        filter_findings
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/same_type_finder.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # Same Type Finder
+    module SameTypeFinder
+      def self.included(klass)
+        klass.class_eval do
+          include IndependentFinder
+
+          # @return [ Array ]
+          def finders
+            @finders ||= NS::Finders::SameTypeFinders.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/same_type_finders.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # This class is designed to handle same type results, such as enumeration of plugins,
+    # themes etc.
+    class SameTypeFinders < BaseFinders
+      # @param [ Hash ] opts
+      # @option opts [ Symbol ] :mode :mixed, :passive or :aggressive
+      # @option opts [ Boolean ] :sort Wether or not to sort the findings
+      #
+      # @return [ Findings ]
+      def run(opts = {})
+        symbols_from_mode(opts[:mode]).each do |symbol|
+          each do |finder|
+            run_finder(finder, symbol, opts)
+          end
+        end
+
+        findings.sort! if opts[:sort]
+
+        filter_findings
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/unique_finder.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # Unique Finder
+    module UniqueFinder
+      def self.included(klass)
+        klass.class_eval do
+          include IndependentFinder
+
+          # @return [ Array ]
+          def finders
+            @finders ||= NS::Finders::UniqueFinders.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/unique_finders.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Finders
+    # This class is designed to return a unique result such as a version
+    # Note: Finders contained can return multiple results but the #run will only
+    # returned the best finding
+    class UniqueFinders < BaseFinders
+      # @param [ Hash ] opts
+      # @option opts [ Symbol ] :mode :mixed, :passive or :aggressive
+      # @option opts [ Int ] :confidence_threshold  If a finding's confidence reaches this value,
+      #                                             it will be returned as the best finding.
+      #                                             Default is 100.
+      #                                             If <= 0, all finders will be ran.
+      #
+      # @return [ Object, false ] The best finding or false if none
+      def run(opts = {})
+        opts[:confidence_threshold] ||= 100
+
+        symbols_from_mode(opts[:mode]).each do |symbol|
+          each do |finder|
+            run_finder(finder, symbol, opts)
+
+            next if opts[:confidence_threshold] <= 0
+
+            findings.each { |f| return f if f.confidence >= opts[:confidence_threshold] }
+          end
+        end
+
+        filter_findings
+      end
+
+      protected
+
+      # @return [ Object, false ] The best finding or false if none
+      def filter_findings
+        # results are sorted by confidence ASC
+        findings.sort_by!(&:confidence)
+
+        # If all findings have the same confidence, false is returned
+        return false if findings.size > 1 && findings.first.confidence == findings.last.confidence
+
+        findings.last || false
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'cms_scanner/finders/finder'
+require 'cms_scanner/finders/finding'
+require 'cms_scanner/finders/findings'
+require 'cms_scanner/finders/base_finders'
+require 'cms_scanner/finders/independent_finders'
+require 'cms_scanner/finders/independent_finder'
+require 'cms_scanner/finders/unique_finders'
+require 'cms_scanner/finders/unique_finder'
+require 'cms_scanner/finders/same_type_finders'
+require 'cms_scanner/finders/same_type_finder'

data/lib/cms_scanner/formatter/buffer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module CMSScanner
+  module Formatter
+    # Module used to output the rendered views into a buffer
+    # and beautify it a the end of the scan
+    module Buffer
+      def output(tpl, vars = {}, controller_name = nil)
+        buffer << render(tpl, vars, controller_name).encode('UTF-8', invalid: :replace, undef: :replace)
+      end
+
+      def buffer
+        @buffer ||= +''
+      end
+    end
+  end
+end