cms_scanner 0.0.36 → 0.0.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bfb8bab5a963e2803fa4991564f4affe3e7415d
-  data.tar.gz: 50faf8b01e45138166d74d3b4d769dbf5bf828ea
+  metadata.gz: acd54c82dac47f135bd9a9eb739493617a114f8c
+  data.tar.gz: 0cd17341caa2bd782beb2d07b2666d2dbf132d06
 SHA512:
-  metadata.gz: a49537023c9c2ba55ab12641dc6fb9407e357e88ca358b1c9428d533bb984df8f1f6e07e7218c9195cd9459f20da24c4f4e268c63ae30bc75a083db5fae58309
-  data.tar.gz: 7ad19781bcda3f02d744eb0eceb0b3e43289cc3d323950df3b2060b051426053cf1ef3957c9ae5df417ce5f18006881e908b3cbcca6b9e63afc1981a07856d77
+  metadata.gz: d5d16f8d26a3873ea5a76c09c48e69f4c67f1dddca9e30a41f775060cccb4543b998845133be7ee937bb21bc0cb63dcf8523c12facc5e773fb8959d20ba5b752
+  data.tar.gz: 581b8ea2a409f4854fbbb214f09b3c3dc7cb00c308665a9545bef7e507475da3da3d56841ee4e6022adb903d66f37fa771610ae3c8d4879f24b15ca9e64c8fb4

data/app/controllers/core.rb

@@ -18,11 +18,25 @@ module CMSScanner
 
         setup_cache
 
-        fail "The url supplied '#{target.url}' seems to be down" unless target.online?
+        check_target_availability
+      end
 
-        fail AccessForbiddenError if target.access_forbidden?
-        fail HTTPAuthRequiredError if target.http_auth?
-        fail ProxyAuthRequiredError if target.proxy_auth?
+      # Checks that the target is accessible, raises related errors otherwise
+      #
+      # @return [ Void ]
+      def check_target_availability
+        res = NS::Browser.get(target.url)
+
+        case res.code
+        when 0
+          fail TargetDownError, res
+        when 401
+          fail HTTPAuthRequiredError
+        when 403
+          fail AccessForbiddenError
+        when 407
+          fail ProxyAuthRequiredError
+        end
 
         redirection = target.redirection
         fail HTTPRedirectError, redirection if redirection && !parsed_options[:ignore_main_redirect]

data/app/controllers/core/cli_options.rb

@@ -34,6 +34,8 @@ module CMSScanner
           OptCredentials.new(['--http-auth login:password']),
           OptPositiveInteger.new(['--max-threads VALUE', '-t', 'The max threads to use'],
                                  default: 5),
+          OptPositiveInteger.new(['--throttle MilliSeconds', 'Milliseconds to wait before doing another web request. ' \
+                                  'If used, the max threads will be set to 1.']),
           OptPositiveInteger.new(['--request-timeout SECONDS', 'The request timeout in seconds'],
                                  default: 60),
           OptPositiveInteger.new(['--connect-timeout SECONDS', 'The connection timeout in seconds'],

data/app/models/interesting_finding.rb

@@ -26,13 +26,14 @@ module CMSScanner
       res.body.split("\n").reject { |s| s.strip.empty? }
     end
 
+    # @return [ String ]
     def to_s
       @to_s || url
     end
 
+    # @return [ Bollean ]
     def ==(other)
-      return false unless self.class == other.class
-      to_s == other.to_s
+      self.class == other.class && to_s == other.to_s
     end
   end
 end

data/app/models/robots_txt.rb

@@ -14,7 +14,8 @@ module CMSScanner
 
         results << match
       end
-      results
+
+      results.uniq
     end
   end
 end

data/app/views/json/interesting_findings/findings.erb

@@ -1,9 +1,9 @@
-"interesting_findings": [
+"interesting_findings": {
 <% unless @findings.empty? -%>
-  {
   <% last_index = @findings.size - 1 %>
   <% @findings.each.with_index do |finding, index| -%>
-    <%= finding.url.to_s.to_json %>: {
+    <%= finding.to_s.to_json %>: {
+      "url": <%= finding.url.to_s.to_json %>,
       "found_by": <%= finding.found_by.to_s.to_json %>,
       "confidence": <%= finding.confidence.to_json %>,
       "confirmed_by": [
@@ -20,6 +20,5 @@
       "interesting_entries": <%= finding.interesting_entries.to_json %>
     }<% unless index == last_index %>,<% end %>
   <% end -%>
-  }
 <% end -%>
-],
+},

data/cms_scanner.gemspec

@@ -31,19 +31,20 @@ Gem::Specification.new do |s|
   s.executables           = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.require_path          = 'lib'
 
-  s.add_dependency 'opt_parse_validator', '~> 0.0.13'
-  s.add_dependency 'typhoeus', '~> 0.7'
+  s.add_dependency 'typhoeus', '~> 0.8'
   s.add_dependency 'nokogiri', '~> 1.6.6'
+  s.add_dependency 'yajl-ruby', '~> 1.2.1' # Better JSON parser regarding memory usage
   s.add_dependency 'addressable', '~> 2.3.8'
   s.add_dependency 'activesupport', '~> 4.2'
   s.add_dependency 'public_suffix', '~> 1.5'
   s.add_dependency 'ruby-progressbar', '~> 1.7.5'
+  s.add_dependency 'opt_parse_validator', '~> 0.0.13'
 
   s.add_development_dependency 'rake', '~> 10.4.2'
   s.add_development_dependency 'rspec', '~> 3.3'
   s.add_development_dependency 'rspec-its', '~> 1.2'
   s.add_development_dependency 'bundler', '~> 1.6'
-  s.add_development_dependency 'rubocop', '~> 0.32'
-  s.add_development_dependency 'webmock', '~> 1.21'
+  s.add_development_dependency 'rubocop', '~> 0.34'
+  s.add_development_dependency 'webmock', '~> 1.22'
   s.add_development_dependency 'simplecov', '~> 0.10'
 end

data/lib/cms_scanner.rb

@@ -1,12 +1,13 @@
 # Gems
-require 'opt_parse_validator'
 require 'typhoeus'
 require 'nokogiri'
-require 'active_support/inflector'
-require 'active_support/concern'
-require 'addressable/uri'
+require 'yajl/json_gem'
 require 'public_suffix'
+require 'addressable/uri'
 require 'ruby-progressbar'
+require 'opt_parse_validator'
+require 'active_support/concern'
+require 'active_support/inflector'
 # Standard Libs
 require 'erb'
 require 'uri'
@@ -38,7 +39,11 @@ module CMSScanner
 
   # Number of requests performed to display at the end of the scan
   Typhoeus.on_complete do |response|
-    self.total_requests += 1 unless response.cached?
+    next if response.cached?
+
+    self.total_requests += 1
+
+    NS::Browser.instance.trottle!
   end
 
   # Module to be able to use these class methods when the CMSScanner

data/lib/cms_scanner/browser.rb

@@ -10,6 +10,8 @@ module CMSScanner
     #
     # @return [ Void ]
     def initialize(parsed_options = {})
+      self.throttle = 0
+
       load_options(parsed_options)
     end
 

data/lib/cms_scanner/browser/options.rb

@@ -12,6 +12,7 @@ module CMSScanner
       :proxy_auth,
       :random_user_agent,
       :request_timeout,
+      :throttle,
       :user_agent,
       :user_agents_list,
       :vhost
@@ -19,6 +20,15 @@ module CMSScanner
 
     attr_accessor(*OPTIONS)
 
+    # @return [ String ]
+    def default_user_agent
+      "#{NS} v#{NS::VERSION}"
+    end
+
+    def hydra
+      @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
+    end
+
     # @param [ Hash ] options
     def load_options(options = {})
       OPTIONS.each do |sym|
@@ -26,22 +36,21 @@ module CMSScanner
       end
     end
 
-    def hydra
-      @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
-    end
-
     # Set the threads attribute and update
     # the max_concurrency of Typhoeus::Hydra
     #
+    # If the throttle attribute is > 0, max_threads will be forced to 1
+    #
     # @param [ Integer ] number
     def max_threads=(number)
-      @max_threads = number.to_i > 0 ? number.to_i : 1
+      @max_threads = number.to_i > 0 && throttle == 0 ? number.to_i : 1
+
       hydra.max_concurrency = @max_threads
     end
 
-    # @return [ String ] The path to the user agents list
-    def user_agents_list
-      @user_agents_list ||= File.join(APP_DIR, 'user_agents.txt')
+    # @return [ String ] The user agent
+    def user_agent
+      @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
     end
 
     # @return [ Array<String> ]
@@ -58,14 +67,22 @@ module CMSScanner
       @user_agents
     end
 
-    # @return [ String ]
-    def default_user_agent
-      "#{NS} v#{NS::VERSION}"
+    # @return [ String ] The path to the user agents list
+    def user_agents_list
+      @user_agents_list ||= File.join(APP_DIR, 'user_agents.txt')
     end
 
-    # @return [ String ] The user agent
-    def user_agent
-      @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
+    # @param [ value ] The throttle time in milliseconds
+    #
+    # if value > 0, the max_threads will be set to 1
+    def throttle=(value)
+      @throttle = value.to_i.abs / 1000.0
+
+      self.max_threads = 1 if @throttle > 0
+    end
+
+    def trottle!
+      sleep(throttle) if throttle > 0
     end
   end
 end

data/lib/cms_scanner/errors/http.rb

@@ -1,5 +1,18 @@
 module CMSScanner
-  class Error < RuntimeError
+  class Error < StandardError
+  end
+
+  # Target Down Error
+  class TargetDownError < Error
+    attr_reader :response
+
+    def initialize(response)
+      @response = response
+    end
+
+    def to_s
+      "The url supplied '#{response.request.url}' seems to be down (#{response.return_message})"
+    end
   end
 
   # HTTP Authentication Required Error

data/lib/cms_scanner/formatter.rb

@@ -14,7 +14,7 @@ module CMSScanner
         format ||= 'cli'
         custom_views ||= []
 
-        f = const_get(format.gsub(/-/, '_').camelize).new
+        f = const_get(format.tr('-', '_').camelize).new
         custom_views.each { |v| f.views_directories << v }
         f
       end

data/lib/cms_scanner/version.rb

@@ -1,4 +1,4 @@
 # Version
 module CMSScanner
-  VERSION = '0.0.36'
+  VERSION = '0.0.37'
 end

data/lib/cms_scanner/web_site.rb

@@ -77,7 +77,7 @@ module CMSScanner
 
       return unless [301, 302].include?(NS::Browser.get(url).code)
 
-      res   = NS::Browser.get(url, followlocation: true)
+      res = NS::Browser.get(url, followlocation: true)
 
       res.effective_url == url ? nil : res.effective_url
     end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: cms_scanner
 version: !ruby/object:Gem::Version
-  version: 0.0.36
+  version: 0.0.37
 platform: ruby
 authors:
 - WPScanTeam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-23 00:00:00.000000000 Z
+date: 2015-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: opt_parse_validator
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: 1.6.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: 1.6.6
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.7.5
+- !ruby/object:Gem::Dependency
+  name: opt_parse_validator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.13
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -170,28 +184,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.32'
+        version: '0.34'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.32'
+        version: '0.34'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.21'
+        version: '1.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.21'
+        version: '1.22'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement