cms_scanner 0.0.14 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b776de3125da0037eef520d8cf0f69caa17a9f22
-  data.tar.gz: 1f328394f7c5395b8b3191ee37e98627f0b90d0c
+  metadata.gz: e4c298cb52add0fd3b06836ec3d5f5acc7fcc06c
+  data.tar.gz: 32c3dec825b5d3641b86938ea2641bafb34cc0bd
 SHA512:
-  metadata.gz: f1b3aa71d953d878b8bbebdf70372ecaf7eb89352638008748d7b5cad16f4378c9ee26534042bf2c1ed32268c5f4b4e4a95ed1b88eab0186dfd0ef2bc4133d7b
-  data.tar.gz: 1cfa9bb846e83bdf8702295f62049c175d850a9e19eae1bdc73d8a882690b298ab127056866355792f37126bd0a4257756128c5e4878d46835096f77913dcc19
+  metadata.gz: 4444cb50bf62d3b2715bd49054a539ae058aa234b4f27d013b6dda5fa5bfcc4664fd270e85a00f82d3de99add49729223f4f5333146262f92ad67360ea4d6665
+  data.tar.gz: dbd2746ebc6b5912d4bf3c828dbc413499d0b1e306dc2826dc8572017a5025adfa9a1b0c4528a49bd44ae5a704a3cee0c04e719376907679e97926913d88166d

data/.rubocop.yml

@@ -5,6 +5,6 @@ ClassVars:
 MethodLength:
   Max: 15
 Metrics/AbcSize:
-  Max: 20
+  Max: 22
 Metrics/CyclomaticComplexity:
   Max: 10

data/app/controllers/core.rb

@@ -14,6 +14,8 @@ module CMSScanner
       end
 
       def before_scan
+        output('banner')
+
         setup_cache
 
         fail "The url supplied '#{target.url}' seems to be down" unless target.online?

data/app/views/cli/core/started.erb

@@ -1,4 +1,3 @@
-<%= render('banner') -%>
 <%= green('[+]') %> URL: <%= @url %>
 <%= green('[+]') %> Started: <%= @start_time.asctime %>
 

data/cms_scanner.gemspec

@@ -21,12 +21,13 @@ Gem::Specification.new do |s|
   s.test_files            = s.files.grep(/^(test|spec|features)\//)
   s.require_path         = 'lib'
 
-  s.add_dependency 'opt_parse_validator', '~> 0.0.8'
+  s.add_dependency 'opt_parse_validator', '~> 0.0.9'
   s.add_dependency 'typhoeus', '~> 0.7'
   s.add_dependency 'nokogiri', '~> 1.6'
   s.add_dependency 'addressable', '~> 2.3'
   s.add_dependency 'activesupport', '~> 4.2'
   s.add_dependency 'public_suffix', '~> 1.4'
+  s.add_dependency 'ruby-progressbar', '~> 1.7.1'
 
   s.add_development_dependency 'rake', '~> 10.4'
   s.add_development_dependency 'rspec', '~> 3.2'

data/lib/cms_scanner.rb

@@ -5,6 +5,7 @@ require 'nokogiri'
 require 'active_support/inflector'
 require 'addressable/uri'
 require 'public_suffix'
+require 'ruby-progressbar'
 # Standard Libs
 require 'erb'
 require 'fileutils'
@@ -57,6 +58,10 @@ module CMSScanner
                        trace: e.backtrace,
                        verbose: controllers.first.parsed_options[:verbose])
     ensure
+      # Ensures a clean abort of Hydra
+      Browser.instance.hydra.abort
+      Browser.instance.hydra.run
+
       formatter.beautify
     end
 

data/lib/cms_scanner/controller.rb

@@ -61,6 +61,11 @@ module CMSScanner
         formatter.render(*tpl_params(tpl, vars))
       end
 
+      # @return [ Boolean ]
+      def user_interaction?
+        formatter.user_interaction? && !parsed_options[:output]
+      end
+
       protected
 
       # @param [ String ] tpl

data/lib/cms_scanner/finders/finder.rb

@@ -1,4 +1,5 @@
 require 'cms_scanner/finders/finder/smart_url_checker'
+require 'cms_scanner/finders/finder/enumerator'
 
 module CMSScanner
   module Finders

data/lib/cms_scanner/finders/finder/enumerator.rb

@@ -0,0 +1,72 @@
+module CMSScanner
+  module Finders
+    class Finder
+      # Module to provide an easy way to enumerate items such as plugins, themes etc
+      module Enumerator
+        # @param [ Hash ] opts
+        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
+        # @option opts [ Regexp ] :exclude_content
+        #
+        # @yield [ Typhoeus::Response, String ]
+        def enumerate(opts = {})
+          targets = target_urls(opts)
+          bar     = progress_bar(targets.size) if opts[:show_progression]
+
+          targets.each do |url, id|
+            request = browser.forge_request(url, request_params)
+
+            request.on_complete do |res|
+              bar.progress += 1 if opts[:show_progression]
+
+              next if target.homepage_or_404?(res)
+              next if opts[:exclude_content] && res.body.match(opts[:exclude_content])
+
+              yield res, id
+            end
+
+            hydra.queue(request)
+          end
+
+          hydra.run
+        end
+
+        # @param [ Hash ] opts
+        #
+        # @return [ Hash ]
+        def target_urls(_opts = {})
+          fail NotImplementedError
+        end
+
+        # @param [ Integer ] total
+        #
+        # @return [ ProgressBar ]
+        # :nocov:
+        def progress_bar(total)
+          ProgressBar.create(
+            format: '%t %a <%B> (%c / %C) %P%% %e',
+            title: ' ', # Used to craete a left margin
+            total: total
+          )
+        end
+        # :nocov:
+
+        # @return [ CMSScanner::Browser ]
+        def browser
+          @browser ||= NS::Browser.instance
+        end
+
+        def request_params
+          # disabling the cache, as it causes a 'stack level too deep' exception
+          # with a large number of requests :/
+          # See https://github.com/typhoeus/typhoeus/issues/408
+          { cache_ttl: 0 }
+        end
+
+        # @return [ Typhoeus::Hydra ]
+        def hydra
+          @hydra ||= browser.hydra
+        end
+      end
+    end
+  end
+end

data/lib/cms_scanner/formatter.rb

@@ -58,6 +58,11 @@ module CMSScanner
         self.class.name.demodulize.underscore
       end
 
+      # @return [ Boolean ]
+      def user_interaction?
+        format == 'cli'
+      end
+
       # @return [ String ] The underscored format to use as a base
       def base_format; end
 

data/lib/cms_scanner/target/hashes.rb

@@ -14,13 +14,13 @@ module CMSScanner
 
     # @return [ String ] The hash of the homepage
     def homepage_hash
-      @homepage_hash ||= Target.page_hash(url)
+      @homepage_hash ||= self.class.page_hash(url)
     end
 
     # @note This is used to detect potential custom 404 responding with a 200
     # @return [ String ] The hash of a 404
     def error_404_hash
-      @error_404_hash ||= Target.page_hash(non_existant_page_url)
+      @error_404_hash ||= self.class.page_hash(non_existant_page_url)
     end
 
     # @return [ String ] The URL of an unlikely existant page
@@ -31,7 +31,7 @@ module CMSScanner
     # @param [ Typhoeus::Response, String ] page
     # @return [ Boolean ] Wether or not the page is a the homepage or a 404 based on its md5sum
     def homepage_or_404?(page)
-      md5sum = Target.page_hash(page)
+      md5sum = self.class.page_hash(page)
 
       md5sum == homepage_hash || md5sum == error_404_hash
     end

data/lib/cms_scanner/version.rb

@@ -1,4 +1,4 @@
 # Version
 module CMSScanner
-  VERSION = '0.0.14'
+  VERSION = '0.0.15'
 end

data/spec/app/controllers/core_spec.rb

@@ -36,6 +36,8 @@ describe CMSScanner::Controller::Core do
   end
 
   describe '#before_scan' do
+    before { expect(core.formatter).to receive(:output) }
+
     it 'does not raise an error when everything is fine' do
       stub_request(:get, target_url).to_return(status: 200)
 

data/spec/app/formatters/cli_no_colour_spec.rb

@@ -3,9 +3,8 @@ require 'spec_helper'
 describe CMSScanner::Formatter::CliNoColour do
   subject(:formatter) { described_class.new }
 
-  describe '#format' do
-    its(:format) { should eq 'cli' }
-  end
+  its(:format)            { should eq 'cli' }
+  its(:user_interaction?) { should be true }
 
   describe '#colorize' do
     it 'returns the text w/o any colour' do

data/spec/app/formatters/cli_spec.rb

@@ -3,9 +3,8 @@ require 'spec_helper'
 describe CMSScanner::Formatter::Cli do
   subject(:formatter) { described_class.new }
 
-  describe '#format' do
-    its(:format) { should eq 'cli' }
-  end
+  its(:format)            { should eq 'cli' }
+  its(:user_interaction?) { should be true }
 
   describe '#bold, #red, #green, #amber, #blue, #colorize' do
     it 'returns the correct bold string' do

data/spec/app/formatters/json_spec.rb

@@ -8,9 +8,8 @@ describe CMSScanner::Formatter::Json do
 
   before { formatter.views_directories << FIXTURES_VIEWS }
 
-  describe '#format' do
-    its(:format) { should eq 'json' }
-  end
+  its(:format)            { should eq 'json' }
+  its(:user_interaction?) { should be false }
 
   describe '#output' do
     it 'puts the rendered text in the buffer' do

data/spec/lib/cms_scanner_spec.rb

@@ -21,8 +21,13 @@ describe CMSScanner::Scan do
 
   describe '#run' do
     it 'runs the controlllers and calls the formatter#beautify' do
+      hydra = CMSScanner::Browser.instance.hydra
+
       expect(scanner.controllers).to receive(:run).ordered
+      expect(hydra).to receive(:abort).ordered
+      expect(hydra).to receive(:run).ordered
       expect(scanner.formatter).to receive(:beautify).ordered
+
       scanner.run
     end
 

data/spec/lib/controller_spec.rb

@@ -10,9 +10,16 @@ describe CMSScanner::Controller do
 
     its(:parsed_options)        { should eq(parsed_options) }
     its(:formatter)             { should be_a CMSScanner::Formatter::Cli }
+    its(:user_interaction?)     { should be true }
     its(:target)                { should be_a CMSScanner::Target }
     its('target.scope.domains') { should eq [PublicSuffix.parse('example.com')] }
 
+    context 'when output option' do
+      let(:parsed_options) { super().merge(output: '/tmp/spec.txt') }
+
+      its(:user_interaction?) { should be false }
+    end
+
     describe '#render' do
       it 'calls the formatter#render' do
         expect(controller.formatter).to receive(:render).with('test', { verbose: nil }, 'base')

data/spec/lib/finders/finder/enumerator_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe CMSScanner::Finders::Finder::Enumerator do
+  # Dummy class to test the module
+  class DummyFinder < CMSScanner::Finders::Finder
+    include CMSScanner::Finders::Finder::Enumerator
+  end
+
+  subject(:finder) { DummyFinder.new(target) }
+  let(:target)     { CMSScanner::Target.new('http://e.org') }
+
+  context 'when #target_urls not implemented' do
+    it 'raises errors' do
+      expect { finder.target_urls }.to raise_error NotImplementedError
+    end
+  end
+
+  describe '#progress_bar' do
+    it 'returns a ProgressBar' do
+      expect(finder.progress_bar(2)).to be_a ProgressBar::Base
+    end
+  end
+
+  its(:browser) { should be_a CMSScanner::Browser }
+
+  its(:request_params) { should eql(cache_ttl: 0) }
+
+  its(:hydra) { should be_a Typhoeus::Hydra }
+
+  describe '#aggressive' do
+    before do
+      expect(finder).to receive(:target_urls).and_return(target_urls)
+      target_urls.each { |url, _| stub_request(:get, url).to_return(status: 200, body: 'rspec') }
+    end
+
+    let(:target_urls) do
+      {
+        target.url('1') => 1,
+        target.url('2') => 2
+      }
+    end
+
+    context 'when no opts' do
+      let(:opts) { {} }
+
+      context 'when response are the homepage or custom 404' do
+        before { expect(finder.target).to receive(:homepage_or_404?).twice.and_return(true) }
+
+        it 'does not yield anything' do
+          expect { |b| finder.enumerate(opts, &b) }.to_not yield_control
+        end
+      end
+
+      context 'when not the hompage or 404' do
+        before { expect(finder.target).to receive(:homepage_or_404?).twice }
+
+        it 'yield the expected items' do
+          expect { |b| finder.enumerate(opts, &b) }.to yield_successive_args(
+            [Typhoeus::Response, 1], [Typhoeus::Response, 2]
+          )
+        end
+      end
+    end
+
+    context 'when opts' do
+      context 'when :exclude_content' do
+        before { expect(finder.target).to receive(:homepage_or_404?).twice }
+
+        context 'when it matches' do
+          let(:opts) { { exclude_content: /spec/i } }
+
+          it 'does not yield anything' do
+            expect { |b| finder.enumerate(opts, &b) }.to_not yield_control
+          end
+        end
+
+        context 'when it does not match' do
+          let(:opts) { { exclude_content: /not/i } }
+
+          it 'yield the expected items' do
+            expect { |b| finder.enumerate(opts, &b) }.to yield_successive_args(
+              [Typhoeus::Response, 1], [Typhoeus::Response, 2]
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/spec/lib/formatter_spec.rb

@@ -33,6 +33,18 @@ describe CMSScanner::Formatter::Base do
     its(:format) { should eq 'base' }
   end
 
+  describe '#user_interaction?' do
+    context 'when not a cli format' do
+      its(:user_interaction?) { should be false }
+    end
+
+    context 'when a cli format' do
+      before { expect(formatter).to receive(:format).and_return('cli') }
+
+      its(:user_interaction?) { should be true }
+    end
+  end
+
   describe '#render, output' do
     before { formatter.views_directories << FIXTURES_VIEWS }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cms_scanner
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - WPScanTeam - Erwan Le Rousseau
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-18 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opt_parse_validator
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.8
+        version: 0.0.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.8
+        version: 0.0.9
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -253,6 +267,7 @@ files:
 - lib/cms_scanner/finders.rb
 - lib/cms_scanner/finders/confidence.rb
 - lib/cms_scanner/finders/finder.rb
+- lib/cms_scanner/finders/finder/enumerator.rb
 - lib/cms_scanner/finders/finder/smart_url_checker.rb
 - lib/cms_scanner/finders/finder/smart_url_checker/findings.rb
 - lib/cms_scanner/finders/finding.rb
@@ -335,6 +350,7 @@ files:
 - spec/lib/controller_spec.rb
 - spec/lib/controllers_spec.rb
 - spec/lib/finders/confidence_spec.rb
+- spec/lib/finders/finder/enumerator_spec.rb
 - spec/lib/finders/finder/smart_url_checker/findings_spec.rb
 - spec/lib/finders/finder/smart_url_checker_spec.rb
 - spec/lib/finders/findings_spec.rb
@@ -454,6 +470,7 @@ test_files:
 - spec/lib/controller_spec.rb
 - spec/lib/controllers_spec.rb
 - spec/lib/finders/confidence_spec.rb
+- spec/lib/finders/finder/enumerator_spec.rb
 - spec/lib/finders/finder/smart_url_checker/findings_spec.rb
 - spec/lib/finders/finder/smart_url_checker_spec.rb
 - spec/lib/finders/findings_spec.rb