cms_scanner 0.0.18 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4967adc46d0b17b3762f4aa25608fbd5fa7372f4
-  data.tar.gz: d8b5abec16cff468b52d2596c8544b279dbfc347
+  metadata.gz: 0c16a64244ed7befaa7147416fb7f0602dd5e01a
+  data.tar.gz: 0eecb7cd60d586dcce20ceac600b65e48573581b
 SHA512:
-  metadata.gz: a807670ea5bdf871cd0f70779f939ddfb8c1dbd83262447e1427369109db103cc38192bbf6f95531264081e003200cf29be0c3af0849544b0d943824d6a0c734
-  data.tar.gz: d9c5c442e85d106dc960eee3db9a5277e2078816fd2dd331e73ac6206ffdcf08d6945ef071cdbf035d278ef6bf361ae11b22842be6190404f57e0b87d59025db
+  metadata.gz: 8fae913eb98950ceab827816b37f72ed9ce2d0dcad33677b9e12900b14275f242382155635273ec4e4d26d60e3c4ee507752ff47d0ece9aa0a3f8ea51a09ed70
+  data.tar.gz: bc44ad999e82fdf7744ea738939ee8a7da795681d563f0f494c0e581eaa31998b437be6239ca2e3bd8c9cf6496b2eab26e3ae4396667a9a242dfcfdf44e7adf1

data/app/controllers/core.rb

@@ -38,9 +38,10 @@ module CMSScanner
       end
 
       def after_scan
-        @stop_time   = Time.now
-        @elapsed     = @stop_time - @start_time
-        @used_memory = memory_usage - @start_memory
+        @stop_time     = Time.now
+        @elapsed       = @stop_time - @start_time
+        @used_memory   = memory_usage - @start_memory
+        @requests_done = CMSScanner.total_requests
 
         output('finished')
       end

data/app/views/cli/core/finished.erb

@@ -1,3 +1,4 @@
 <%= green('[+]') %> Finished: <%= @stop_time.asctime %>
+<%= green('[+]') %> Requests Done: <%= @requests_done %>
 <%= green('[+]') %> Memory used: <%= @used_memory.bytes_to_human %>
 <%= green('[+]') %> Elapsed time: <%= Time.at(@elapsed).utc.strftime('%H:%M:%S') %>

data/app/views/json/core/finished.erb

@@ -1,3 +1,4 @@
 "stop_time": <%= @stop_time.to_i %>,
 "elapsed": <%= @elapsed.to_i %>,
+"requests_done": <%= @requests_done.to_i %>,
 "used_memory": <%= @used_memory.to_i %>,

data/cms_scanner.gemspec

@@ -16,10 +16,20 @@ Gem::Specification.new do |s|
   s.homepage              = 'https://github.com/wpscanteam/CMSScanner'
   s.license               = 'MIT'
 
-  s.files                 = `git ls-files -z`.split("\x0")
+  s.files                 = `git ls-files -z`.split("\x0").reject do |file|
+    file =~ /^(?:
+      spec\/.*
+      |Gemfile
+      |Rakefile
+      |\.rspec
+      |\.gitignore
+      |\.rubocop.yml
+      |\.travis.yml
+      )$/x
+  end
+  s.test_files            = []
   s.executables           = s.files.grep(/^bin\//) { |f| File.basename(f) }
-  s.test_files            = s.files.grep(/^(test|spec|features)\//)
-  s.require_path         = 'lib'
+  s.require_path          = 'lib'
 
   s.add_dependency 'opt_parse_validator', '~> 0.0.9'
   s.add_dependency 'typhoeus', '~> 0.7'

data/lib/cms_scanner.rb

@@ -13,6 +13,7 @@ require 'fileutils'
 require 'pathname'
 # Monkey Patches
 require 'cms_scanner/typhoeus/response'
+require 'cms_scanner/typhoeus/hydra'
 require 'cms_scanner/public_suffix/domain'
 # Custom Libs
 require 'cms_scanner/helper'
@@ -38,6 +39,19 @@ module CMSScanner
     super(base)
   end
 
+  # Number of requests performed to display at the end of the scan
+  Typhoeus.on_complete do |response|
+    self.total_requests += 1 unless response.cached?
+  end
+
+  def self.total_requests
+    @@total_requests ||= 0
+  end
+
+  def self.total_requests=(value)
+    @@total_requests = value
+  end
+
   # Scan
   class Scan
     def initialize
@@ -59,9 +73,7 @@ module CMSScanner
                        trace: e.backtrace,
                        verbose: controllers.first.parsed_options[:verbose])
     ensure
-      # Ensures a clean abort of Hydra
       Browser.instance.hydra.abort
-      Browser.instance.hydra.run
 
       formatter.beautify
     end

data/lib/cms_scanner/finders/finder.rb

@@ -1,5 +1,6 @@
 require 'cms_scanner/finders/finder/smart_url_checker'
 require 'cms_scanner/finders/finder/enumerator'
+require 'cms_scanner/finders/finder/fingerprinter'
 
 module CMSScanner
   module Finders
@@ -14,7 +15,7 @@ module CMSScanner
         @target = target
       end
 
-      # @return [ String ] The titleize name of the finder
+      # @return [ String ] The titleized name of the finder
       def titleize
         self.class.to_s.demodulize.underscore.titleize
       end
@@ -27,13 +28,21 @@ module CMSScanner
       def aggressive(_opts = {})
       end
 
-      # @param [ Integer ] total
-      # @param [ String ] title
-      # @param [ String ] format See https://github.com/jfelchner/ruby-progressbar/wiki/Formatting
+      # @param [ Hash ] opts See https://github.com/jfelchner/ruby-progressbar/wiki/Options
       #
-      # @return [ ProgressBar ]
-      def progress_bar(total, title = '', format = '%t %a <%B> (%c / %C) %P%% %e')
-        ProgressBar.create(total: total, title: title, format: format)
+      # @return [ ProgressBar::Base ]
+      def progress_bar(opts = {})
+        ProgressBar.create({ format: '%t %a <%B> (%c / %C) %P%% %e' }.merge(opts))
+      end
+
+      # @return [ Browser ]
+      def browser
+        @browser ||= NS::Browser.instance
+      end
+
+      # @return [ Typhoeus::Hydra ]
+      def hydra
+        @hydra ||= browser.hydra
       end
 
       def found_by

data/lib/cms_scanner/finders/finder/enumerator.rb

@@ -3,16 +3,16 @@ module CMSScanner
     class Finder
       # Module to provide an easy way to enumerate items such as plugins, themes etc
       module Enumerator
+        # @param [ Hash ] The target urls
         # @param [ Hash ] opts
         # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
         # @option opts [ Regexp ] :exclude_content
         #
         # @yield [ Typhoeus::Response, String ]
-        def enumerate(opts = {})
-          targets = target_urls(opts)
-          bar     = progress_bar(targets.size, progress_bar_title) if opts[:show_progression]
+        def enumerate(target_urls, opts = {})
+          bar = progress_bar(total: target_urls.size) if opts[:show_progression]
 
-          targets.each do |url, id|
+          target_urls.each do |url, id|
             request = browser.forge_request(url, request_params)
 
             request.on_complete do |res|
@@ -30,37 +30,13 @@ module CMSScanner
           hydra.run
         end
 
-        # @param [ Hash ] opts
-        #
         # @return [ Hash ]
-        def target_urls(_opts = {})
-          fail NotImplementedError
-        end
-
-        # Progress Bar title to use, allow instance using this module
-        # to display a custom title if needed
-        #
-        # @return [ String ]
-        def progress_bar_title
-          ' ' # Used to create a left margin
-        end
-
-        # @return [ CMSScanner::Browser ]
-        def browser
-          @browser ||= NS::Browser.instance
-        end
-
         def request_params
           # disabling the cache, as it causes a 'stack level too deep' exception
           # with a large number of requests :/
           # See https://github.com/typhoeus/typhoeus/issues/408
           { cache_ttl: 0 }
         end
-
-        # @return [ Typhoeus::Hydra ]
-        def hydra
-          @hydra ||= browser.hydra
-        end
       end
     end
   end

data/lib/cms_scanner/finders/finder/fingerprinter.rb

@@ -0,0 +1,60 @@
+module CMSScanner
+  module Finders
+    class Finder
+      # Module to provide an easy way to fingerprint things such as versions
+      module Fingerprinter
+        # @param [ Hash ] fingerprints The fingerprints
+        # Format should be the following:
+        # {
+        #   file_path_1: {
+        #     md5_hash_1: version_1,
+        #     md5_hash_2: [version_2]
+        #   },
+        #   file_path_2: {
+        #     md5_hash_3: [version_1, version_2],
+        #     md5_hash_4: version_3
+        #   }
+        # }
+        # Note that the version can either be an array or a string
+        #
+        # @param [ Hash ] opts
+        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
+        #
+        # @yield [ Mixed, String, String ] version/s, url, hash The version associated to the
+        #                                                       fingerprint of the url
+        def fingerprint(fingerprints, opts = {})
+          bar = progress_bar(total: fingerprints.size) if opts[:show_progression]
+
+          fingerprints.each do |path, f|
+            url     = target.url(path.dup)
+            request = browser.forge_request(url, request_params)
+
+            request.on_complete do |res|
+              bar.progress += 1 if opts[:show_progression]
+
+              md5sum = hexdigest(res.body)
+
+              next unless f.key?(md5sum)
+
+              yield f[md5sum], url, md5sum
+            end
+
+            hydra.queue(request)
+          end
+
+          hydra.run
+        end
+
+        # @return [ Hash ]
+        def request_params
+          {}
+        end
+
+        # @return [ String ] The hashed value for the given body
+        def hexdigest(body)
+          Digest::MD5.hexdigest(body)
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/finders/finding.rb

@@ -45,7 +45,7 @@ module CMSScanner
       end
 
       def <=>(other)
-        to_s <=> other.to_s
+        to_s.downcase <=> other.to_s.downcase
       end
     end
   end

data/lib/cms_scanner/target/scope.rb

@@ -19,6 +19,8 @@ module CMSScanner
     # @param [ String ] xpath
     # @param [ Array<String> ] attributes
     #
+    # @yield [ String, Nokogiri::XML::Element ] The in scope url and its associated tag
+    #
     # @return [ Array<String> ] The in scope absolute URLs detected in the response's body
     def in_scope_urls(res, xpath = '//link|//script|//style|//img|//a', attributes = %w(href src))
       found = []
@@ -33,7 +35,8 @@ module CMSScanner
 
           next unless in_scope?(url)
 
-          yield url if block_given? && !found.include?(url)
+          yield url, tag if block_given? && !found.include?(url)
+
           found << url
         end
       end

data/lib/cms_scanner/target/server/generic.rb

@@ -10,7 +10,7 @@ module CMSScanner
         def server(path = nil, params = {})
           headers = headers(path, params)
 
-          return nil unless headers
+          return unless headers
 
           case headers[:server]
           when /\Aapache/i

data/lib/cms_scanner/typhoeus/hydra.rb

@@ -0,0 +1,10 @@
+module Typhoeus
+  # Ensure a clean abort of hydra
+  # See https://github.com/typhoeus/typhoeus/issues/439
+  class Hydra
+    def abort
+      super
+      run
+    end
+  end
+end

data/lib/cms_scanner/version.rb

@@ -1,4 +1,4 @@
 # Version
 module CMSScanner
-  VERSION = '0.0.18'
+  VERSION = '0.0.19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cms_scanner
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.0.19
 platform: ruby
 authors:
 - WPScanTeam - Erwan Le Rousseau
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-27 00:00:00.000000000 Z
+date: 2015-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opt_parse_validator
@@ -213,14 +213,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".rspec"
-- ".rubocop.yml"
-- ".travis.yml"
-- Gemfile
 - LICENSE
 - README.md
-- Rakefile
 - app/app.rb
 - app/controllers.rb
 - app/controllers/core.rb
@@ -268,6 +262,7 @@ files:
 - lib/cms_scanner/finders/confidence.rb
 - lib/cms_scanner/finders/finder.rb
 - lib/cms_scanner/finders/finder/enumerator.rb
+- lib/cms_scanner/finders/finder/fingerprinter.rb
 - lib/cms_scanner/finders/finder/smart_url_checker.rb
 - lib/cms_scanner/finders/finder/smart_url_checker/findings.rb
 - lib/cms_scanner/finders/finding.rb
@@ -291,108 +286,12 @@ files:
 - lib/cms_scanner/target/server/apache.rb
 - lib/cms_scanner/target/server/generic.rb
 - lib/cms_scanner/target/server/iis.rb
+- lib/cms_scanner/typhoeus/hydra.rb
 - lib/cms_scanner/typhoeus/response.rb
 - lib/cms_scanner/version.rb
 - lib/cms_scanner/vulnerability.rb
 - lib/cms_scanner/vulnerability/references.rb
 - lib/cms_scanner/web_site.rb
-- spec/app/controllers/core_spec.rb
-- spec/app/controllers/interesting_files_spec.rb
-- spec/app/finders/interesting_files/fantastico_fileslist_spec.rb
-- spec/app/finders/interesting_files/headers_spec.rb
-- spec/app/finders/interesting_files/robots_txt_spec.rb
-- spec/app/finders/interesting_files/search_replace_db_2_spec.rb
-- spec/app/finders/interesting_files/xml_rpc_spec.rb
-- spec/app/finders/interesting_files_spec.rb
-- spec/app/formatters/cli_no_colour_spec.rb
-- spec/app/formatters/cli_spec.rb
-- spec/app/formatters/json_spec.rb
-- spec/app/models/fantastico_fileslist_spec.rb
-- spec/app/models/headers_spec.rb
-- spec/app/models/interesting_file_spec.rb
-- spec/app/models/robots_txt_spec.rb
-- spec/app/models/version_spec.rb
-- spec/app/models/xml_rpc_spec.rb
-- spec/app/views_spec.rb
-- spec/cache/.gitignore
-- spec/dummy_finding.rb
-- spec/dummy_independent_finders.rb
-- spec/dummy_unique_finders.rb
-- spec/fixtures/finders/interesting_files/fantastico_fileslist/fantastico_fileslist.txt
-- spec/fixtures/finders/interesting_files/file.txt
-- spec/fixtures/finders/interesting_files/headers/interesting.txt
-- spec/fixtures/finders/interesting_files/headers/no_interesting.txt
-- spec/fixtures/finders/interesting_files/robots_txt/robots.txt
-- spec/fixtures/finders/interesting_files/search_replace_db_2/searchreplacedb2.php
-- spec/fixtures/finders/interesting_files/xml_rpc/homepage_in_scope_pingback.html
-- spec/fixtures/finders/interesting_files/xml_rpc/homepage_out_of_scope_pingback.html
-- spec/fixtures/finders/interesting_files/xml_rpc/xmlrpc.php
-- spec/fixtures/output.txt
-- spec/fixtures/target/comments.html
-- spec/fixtures/target/platform/php/debug_log/debug.log
-- spec/fixtures/target/platform/php/fpd/wp_rss_functions.php
-- spec/fixtures/target/scope/index.html
-- spec/fixtures/target/server/apache/directory_listing/2.2.16.html
-- spec/fixtures/target/server/generic/server/apache/basic.txt
-- spec/fixtures/target/server/generic/server/iis/basic.txt
-- spec/fixtures/target/server/generic/server/not_detected.txt
-- spec/fixtures/target/server/iis/directory_listing/no_parent.html
-- spec/fixtures/target/server/iis/directory_listing/with_parent.html
-- spec/fixtures/views/base/ctrl/local.erb
-- spec/fixtures/views/base/ctrl/test.erb
-- spec/fixtures/views/base/global.erb
-- spec/fixtures/views/base/test.erb
-- spec/fixtures/views/based_format/test.erb
-- spec/fixtures/views/json/render_me.erb
-- spec/lib/browser_spec.rb
-- spec/lib/cache/file_store_spec.rb
-- spec/lib/cache/typhoeus_spec.rb
-- spec/lib/cms_scanner_spec.rb
-- spec/lib/controller_spec.rb
-- spec/lib/controllers_spec.rb
-- spec/lib/finders/confidence_spec.rb
-- spec/lib/finders/finder/enumerator_spec.rb
-- spec/lib/finders/finder/smart_url_checker/findings_spec.rb
-- spec/lib/finders/finder/smart_url_checker_spec.rb
-- spec/lib/finders/finder_spec.rb
-- spec/lib/finders/findings_spec.rb
-- spec/lib/finders/independent_finders_spec.rb
-- spec/lib/finders/same_type_finder_spec.rb
-- spec/lib/finders/same_type_finders_spec.rb
-- spec/lib/finders/unique_finder_spec.rb
-- spec/lib/finders/unique_finders_spec.rb
-- spec/lib/formatter_spec.rb
-- spec/lib/public_suffix/domain_spec.rb
-- spec/lib/sub_scanner_spec.rb
-- spec/lib/target/hashes_spec.rb
-- spec/lib/target/platforms_spec.rb
-- spec/lib/target/scope_spec.rb
-- spec/lib/target/servers_spec.rb
-- spec/lib/target_spec.rb
-- spec/lib/vulnerability/references_spec.rb
-- spec/lib/vulnerability_spec.rb
-- spec/lib/web_site_spec.rb
-- spec/output/core/finished.cli_no_colour
-- spec/output/core/finished.json
-- spec/output/core/started.cli_no_colour
-- spec/output/core/started.json
-- spec/output/interesting_files/empty.cli_no_colour
-- spec/output/interesting_files/empty.json
-- spec/output/interesting_files/findings.cli_no_colour
-- spec/output/interesting_files/findings.json
-- spec/shared_examples.rb
-- spec/shared_examples/browser_actions.rb
-- spec/shared_examples/finding.rb
-- spec/shared_examples/formatter_buffer.rb
-- spec/shared_examples/formatter_class_methods.rb
-- spec/shared_examples/independent_finder.rb
-- spec/shared_examples/target/platform/php.rb
-- spec/shared_examples/target/server/apache.rb
-- spec/shared_examples/target/server/generic.rb
-- spec/shared_examples/target/server/iis.rb
-- spec/shared_examples/views/core.rb
-- spec/shared_examples/views/interesting_files.rb
-- spec/spec_helper.rb
 homepage: https://github.com/wpscanteam/CMSScanner
 licenses:
 - MIT
@@ -417,101 +316,4 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Experimental CMSScanner
-test_files:
-- spec/app/controllers/core_spec.rb
-- spec/app/controllers/interesting_files_spec.rb
-- spec/app/finders/interesting_files/fantastico_fileslist_spec.rb
-- spec/app/finders/interesting_files/headers_spec.rb
-- spec/app/finders/interesting_files/robots_txt_spec.rb
-- spec/app/finders/interesting_files/search_replace_db_2_spec.rb
-- spec/app/finders/interesting_files/xml_rpc_spec.rb
-- spec/app/finders/interesting_files_spec.rb
-- spec/app/formatters/cli_no_colour_spec.rb
-- spec/app/formatters/cli_spec.rb
-- spec/app/formatters/json_spec.rb
-- spec/app/models/fantastico_fileslist_spec.rb
-- spec/app/models/headers_spec.rb
-- spec/app/models/interesting_file_spec.rb
-- spec/app/models/robots_txt_spec.rb
-- spec/app/models/version_spec.rb
-- spec/app/models/xml_rpc_spec.rb
-- spec/app/views_spec.rb
-- spec/cache/.gitignore
-- spec/dummy_finding.rb
-- spec/dummy_independent_finders.rb
-- spec/dummy_unique_finders.rb
-- spec/fixtures/finders/interesting_files/fantastico_fileslist/fantastico_fileslist.txt
-- spec/fixtures/finders/interesting_files/file.txt
-- spec/fixtures/finders/interesting_files/headers/interesting.txt
-- spec/fixtures/finders/interesting_files/headers/no_interesting.txt
-- spec/fixtures/finders/interesting_files/robots_txt/robots.txt
-- spec/fixtures/finders/interesting_files/search_replace_db_2/searchreplacedb2.php
-- spec/fixtures/finders/interesting_files/xml_rpc/homepage_in_scope_pingback.html
-- spec/fixtures/finders/interesting_files/xml_rpc/homepage_out_of_scope_pingback.html
-- spec/fixtures/finders/interesting_files/xml_rpc/xmlrpc.php
-- spec/fixtures/output.txt
-- spec/fixtures/target/comments.html
-- spec/fixtures/target/platform/php/debug_log/debug.log
-- spec/fixtures/target/platform/php/fpd/wp_rss_functions.php
-- spec/fixtures/target/scope/index.html
-- spec/fixtures/target/server/apache/directory_listing/2.2.16.html
-- spec/fixtures/target/server/generic/server/apache/basic.txt
-- spec/fixtures/target/server/generic/server/iis/basic.txt
-- spec/fixtures/target/server/generic/server/not_detected.txt
-- spec/fixtures/target/server/iis/directory_listing/no_parent.html
-- spec/fixtures/target/server/iis/directory_listing/with_parent.html
-- spec/fixtures/views/base/ctrl/local.erb
-- spec/fixtures/views/base/ctrl/test.erb
-- spec/fixtures/views/base/global.erb
-- spec/fixtures/views/base/test.erb
-- spec/fixtures/views/based_format/test.erb
-- spec/fixtures/views/json/render_me.erb
-- spec/lib/browser_spec.rb
-- spec/lib/cache/file_store_spec.rb
-- spec/lib/cache/typhoeus_spec.rb
-- spec/lib/cms_scanner_spec.rb
-- spec/lib/controller_spec.rb
-- spec/lib/controllers_spec.rb
-- spec/lib/finders/confidence_spec.rb
-- spec/lib/finders/finder/enumerator_spec.rb
-- spec/lib/finders/finder/smart_url_checker/findings_spec.rb
-- spec/lib/finders/finder/smart_url_checker_spec.rb
-- spec/lib/finders/finder_spec.rb
-- spec/lib/finders/findings_spec.rb
-- spec/lib/finders/independent_finders_spec.rb
-- spec/lib/finders/same_type_finder_spec.rb
-- spec/lib/finders/same_type_finders_spec.rb
-- spec/lib/finders/unique_finder_spec.rb
-- spec/lib/finders/unique_finders_spec.rb
-- spec/lib/formatter_spec.rb
-- spec/lib/public_suffix/domain_spec.rb
-- spec/lib/sub_scanner_spec.rb
-- spec/lib/target/hashes_spec.rb
-- spec/lib/target/platforms_spec.rb
-- spec/lib/target/scope_spec.rb
-- spec/lib/target/servers_spec.rb
-- spec/lib/target_spec.rb
-- spec/lib/vulnerability/references_spec.rb
-- spec/lib/vulnerability_spec.rb
-- spec/lib/web_site_spec.rb
-- spec/output/core/finished.cli_no_colour
-- spec/output/core/finished.json
-- spec/output/core/started.cli_no_colour
-- spec/output/core/started.json
-- spec/output/interesting_files/empty.cli_no_colour
-- spec/output/interesting_files/empty.json
-- spec/output/interesting_files/findings.cli_no_colour
-- spec/output/interesting_files/findings.json
-- spec/shared_examples.rb
-- spec/shared_examples/browser_actions.rb
-- spec/shared_examples/finding.rb
-- spec/shared_examples/formatter_buffer.rb
-- spec/shared_examples/formatter_class_methods.rb
-- spec/shared_examples/independent_finder.rb
-- spec/shared_examples/target/platform/php.rb
-- spec/shared_examples/target/server/apache.rb
-- spec/shared_examples/target/server/generic.rb
-- spec/shared_examples/target/server/iis.rb
-- spec/shared_examples/views/core.rb
-- spec/shared_examples/views/interesting_files.rb
-- spec/spec_helper.rb
+test_files: []