arachni 1.1 → 1.2

data/lib/arachni/issue.rb

@@ -15,7 +15,7 @@ require Options.paths.lib + 'issue/severity'
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class Issue
 
-    # Attributes removed from a parent issue (i.e. an issues with variations)
+    # Attributes removed from a parent issue (i.e. an issue with variations)
     # and solely populating variations.
     VARIATION_ATTRIBUTES = Set.new([
         :@page, :@referring_page, :@proof, :@signature, :@remarks, :@trusted
@@ -286,12 +286,23 @@ class Issue
         end
     end
 
-    [:name, :description, :remedy_guidance, :remedy_code, :proof, :signature].each do |m|
+    [:name, :description, :remedy_guidance, :remedy_code, :proof].each do |m|
         define_method "#{m}=" do |s|
             instance_variable_set( "@#{m}".to_sym, s ? s.to_s.freeze : nil )
         end
     end
 
+    def signature=( sig )
+        @signature = case sig
+                         when Regexp
+                             sig.source
+                         when String, nil
+                             sig
+                         else
+                             sig.to_s
+                     end
+    end
+
     # @return   [Hash]
     def to_h
         h = {}
@@ -495,7 +506,10 @@ class Issue
             data['variations'] = data['variations'].map(&:to_rpc_data)
         end
 
-        data['digest']   = digest
+        if !variation?
+            data['digest'] = digest
+        end
+
         data['severity'] = data['severity'].to_s
 
         data

data/lib/arachni/option_groups/browser_cluster.rb

@@ -13,6 +13,11 @@ module Arachni::OptionGroups
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class BrowserCluster < Arachni::OptionGroup
 
+    # @return   [Hash<Regexp,String>]
+    #   When the page URL matched the key `Regexp`, wait until the `String` CSS
+    #   selector in the value matches an element.
+    attr_accessor :wait_for_elements
+
     # @return   [Integer]
     #   Amount of {BrowserCluster::Worker} to keep in the pool and put to work.
     attr_accessor :pool_size
@@ -38,13 +43,41 @@ class BrowserCluster < Arachni::OptionGroup
     attr_accessor :screen_height
 
     set_defaults(
+        wait_for_elements:   {},
         pool_size:           6,
-        job_timeout:         15,
+        job_timeout:         25,
         worker_time_to_live: 100,
         ignore_images:       false,
         screen_width:        1600,
         screen_height:       1200
     )
 
+    def css_to_wait_for( url )
+        wait_for_elements.map do |pattern, css|
+            next if !(url =~ pattern)
+            css
+        end.compact
+    end
+
+    def wait_for_elements=( rules )
+        return @wait_for_elements = defaults[:wait_for_elements].dup if !rules
+
+        @wait_for_elements = rules.inject({}) do |h, (regexp, value)|
+            regexp = regexp.is_a?( Regexp ) ? regexp : Regexp.new( regexp.to_s )
+            h.merge!( regexp => value )
+            h
+        end
+    end
+
+    def to_rpc_data
+        d = super
+
+        %w(wait_for_elements).each do |k|
+            d[k] = d[k].my_stringify
+        end
+
+        d
+    end
+
 end
 end

data/lib/arachni/option_groups/http.rb

@@ -242,7 +242,7 @@ class HTTP < Arachni::OptionGroup
         request_timeout:        10_000,
         request_redirect_limit: 5,
         request_concurrency:    20,
-        request_queue_size:     500,
+        request_queue_size:     100,
         request_headers:        {},
         response_max_size:      500_000,
         cookies:                {}

data/lib/arachni/page.rb

@@ -379,31 +379,44 @@ class Page
     def has_script?
         return @has_javascript if !@has_javascript.nil?
 
-        if !response.headers.content_type.to_s.start_with?( 'text/html' ) ||
-            !text? || !document
+        if !response.headers.content_type.to_s.start_with?( 'text/html' ) || !text?
             return @has_javascript = false
         end
 
+        dbody = body.downcase
+
         # First check, quick and simple.
-        return @has_javascript = true if document.css( 'script' ).any?
+        if dbody.include?( '<script' ) || dbody.include?( 'javascript:' )
+            return @has_javascript = true
+        end
 
         # Check for event attributes, if there are any then there's JS to be
         # executed.
         Browser::Javascript.events.flatten.each do |event|
-            return @has_javascript = true if document.xpath( "//*[@#{event}]" ).any?
+            return @has_javascript = true if dbody.include?( "#{event}=" )
         end
 
-        # If there's 'javascript:' in 'href' and 'action' attributes then
-        # there's JS to be executed.
-        [:action, :href].each do |candidate|
-            document.xpath( "//*[@#{candidate}]" ).each do |attribute|
-                if attribute.attributes[candidate.to_s].to_s.start_with?( 'javascript:' )
-                    return @has_javascript = true
-                end
-            end
+        @has_javascript = false
+    end
+
+    # @param    [String, Symbol,Array<String, Symbol>]  tags
+    #   Element tag names.
+    #
+    # @return   [Boolean]
+    #   `true` if the page contains any of the given elements, `false` otherwise.
+    def has_elements?( *tags )
+        return if !text?
+
+        tags.flatten.each do |tag|
+            tag = tag.to_s
+
+            next if !(body =~ /<\s*#{tag}/i)
+
+            return false if !document
+            return true  if document.css( tag ).any?
         end
 
-        @has_javascript = false
+        false
     end
 
     # @return   [Boolean]

data/lib/arachni/page/dom/transition.rb

@@ -270,13 +270,13 @@ class Transition
 
                         when 'element'
                             if value.is_a? String
-                                value.to_sym
+                                data['event'].to_s == 'request' ? value : value.to_sym
                             else
                                 Browser::ElementLocator.from_rpc_data( value )
                             end
 
                         when 'options'
-                            value.my_symbolize_keys
+                            value.my_symbolize_keys(false)
 
                         else
                             value

data/lib/arachni/parser.rb

@@ -31,14 +31,26 @@ class Parser
         # @abstract
         class Base
 
+            attr_reader :html
+            attr_reader :document
+            attr_reader :downcased_html
+
+            def initialize( options = {} )
+                @html           = options[:html]
+                @downcased_html = @html.downcase
+                @document       = options[:document]
+            end
+
             # This method must be implemented by all checks and must return an
             # array of paths as plain strings
             #
-            # @param    [Nokogiri]  document   Nokogiri document
-            #
             # @return   [Array<String>]  paths
             # @abstract
-            def run( document )
+            def run
+            end
+
+            def includes?( string_or_regexp )
+                !!@downcased_html[string_or_regexp]
             end
 
         end
@@ -125,9 +137,8 @@ class Parser
         @document = Nokogiri::HTML( body ) if text? rescue nil
     end
 
-    # @note It's more of a placeholder method, it doesn't actually analyze anything.
-    #   It's a long shot that any of these will be vulnerable but better be safe
-    #   than sorry.
+    # @note It will include common request headers as well headers from the HTTP
+    #   request.
     #
     # @return    [Hash]
     #   List of valid auditable HTTP header fields.
@@ -141,14 +152,15 @@ class Parser
             'User-Agent'      => @options.http.user_agent || '',
             'Referer'         => @url,
             'Pragma'          => 'no-cache'
-        }.map { |k, v| Header.new( url: @url, inputs: { k => v } ) }.freeze
+        }.merge( response.request.headers ).
+            map { |k, v| Header.new( url: @url, inputs: { k => v } ) }.freeze
     end
 
     # @return [Array<Element::Form>]
     #   Forms from {#document}.
     def forms
         return @forms.freeze if @forms
-        return [] if !text?
+        return [] if !text? || !(body =~ /<\s*form/i)
 
         f = Form.from_document( @url, document )
         return f if !@secondary_responses
@@ -199,7 +211,7 @@ class Parser
     #   Links in {#document}.
     def links
         return @links.freeze if @links
-        return @links = [link].compact if !text?
+        return @links = [link].compact if !text? || !(body =~ /\?.*=/)
 
         @links = [link].compact | Link.from_document( @url, document )
     end
@@ -227,7 +239,7 @@ class Parser
     # @return   [Hash]
     #   Parameters found in {#url}.
     def link_vars
-        return {} if (!parsed = uri_parse( @url ))
+        return {} if !(parsed = uri_parse( @url ))
 
         @link_vars ||= parsed.rewrite.query_parameters.freeze
     end
@@ -309,7 +321,12 @@ class Parser
     def run_extractors
         begin
             self.class.extractors.available.map do |name|
-                exception_jail( false ){ self.class.extractors[name].new.run( document ) }
+                exception_jail false do
+                    self.class.extractors[name].new(
+                        document: document,
+                        html:     body
+                    ).run
+                end
             end.flatten.uniq.compact.
                 map { |path| to_absolute( path ) }.compact.uniq.
                 reject { |path| skip?( path ) }

data/lib/arachni/platform/fingerprinter.rb

@@ -36,6 +36,11 @@ class Fingerprinter
     def run
     end
 
+    def html?
+        @is_html ||= page.response.headers['content-type'].to_s.
+            downcase.include?( 'text/html' )
+    end
+
     # @param    [String]    string
     #
     # @return   [Boolean]

data/lib/arachni/platform/manager.rb

@@ -89,12 +89,13 @@ class Manager
         :nginx,
         :tomcat,
         :iis,
-        :jetty
+        :jetty,
+        :gunicorn
     ]
 
     LANGUAGES = [
         :php,
-        :jsp,
+        :java,
         :python,
         :ruby,
         :asp,
@@ -104,7 +105,13 @@ class Manager
 
     # WebApp frameworks.
     FRAMEWORKS = [
-        :rack
+        :rack,
+        :rails,
+        :cakephp,
+        :django,
+        :aspx_mvc,
+        :jsf,
+        :cherrypy
     ]
 
     PLATFORM_NAMES = {
@@ -143,10 +150,11 @@ class Manager
         tomcat:     'TomCat',
         iis:        'IIS',
         jetty:      'Jetty',
+        gunicorn:   'Gunicorn',
 
         # Programming languages
         php:    'PHP',
-        jsp:    'JSP',
+        java:   'Java',
         python: 'Python',
         ruby:   'Ruby',
         asp:    'ASP',
@@ -154,7 +162,13 @@ class Manager
         perl:   'Perl',
 
         # Web frameworks
-        rack:   'Rack'
+        rack:     'Rack',
+        django:   'Django',
+        cakephp:  'CakePHP',
+        rails:    'Ruby on Rails',
+        aspx_mvc: 'ASP.NET MVC',
+        jsf:      'JavaServer Faces',
+        cherrypy: 'CherryPy'
     }
 
     # Amount of
@@ -194,7 +208,7 @@ class Manager
     # Sets global platforms fingerprints
     # @private
     def self.set( platforms )
-        @platforms = Support::Cache::RandomReplacement.new( PLATFORM_CACHE_SIZE )
+        @platforms = Support::Cache::LeastRecentlyPushed.new( PLATFORM_CACHE_SIZE )
         platforms.each { |k, v| @platforms[k] = v }
         @platforms
     end
@@ -228,8 +242,8 @@ class Manager
     # @return   [Bool]
     #   `true` if the resource should be fingerprinted, `false` otherwise.
     def self.fingerprint?( resource )
-        !(!Options.fingerprint? || !resource.text? ||
-        include?( resource.url ) || resource.scope.out?)
+        !(!Options.fingerprint? || resource.code != 200 || !resource.text? ||
+            include?( resource.url ) || resource.scope.out?)
     end
 
     # Runs all fingerprinters against the given `page`.
@@ -240,14 +254,40 @@ class Manager
     # @return   [Manager]
     #   Updated `self`.
     def self.fingerprint( page )
-        return page if !fingerprint? page
+        synchronize do
+            return page if !fingerprint? page
 
-        fingerprinters.available.each do |name|
-            exception_jail( false ) do
-                fingerprinters[name].new( page ).run
+            fingerprinters.available.each do |name|
+                exception_jail( false ) do
+                    fingerprinters[name].new( page ).run
+                end
             end
+
+            # We do this to flag the resource as checked even if no platforms
+            # were identified. We don't want to keep checking a resource that
+            # yields nothing over and over.
+            update( page.url, [] )
         end
-        page
+
+        # Fingerprinting will have resulted in element parsing, clear the element
+        # caches to keep RAM consumption down.
+        page.clear_cache
+    end
+
+    # @param    [String, URI]   uri
+    #
+    # @return   [Manager]
+    #   Platform for the given `uri`
+    def self.[]( uri )
+        # If fingerprinting is disabled there's no point in filling the cache
+        # with the same object over and over, create an identical one for all
+        # URLs and return that always.
+        if !Options.fingerprint?
+            return @default ||= new_from_options
+        end
+
+        return new_from_options if !(key = make_key( uri ))
+        synchronize { @platforms[key] ||= new_from_options }
     end
 
     # Sets platform manager for the given `uri`.
@@ -260,11 +300,16 @@ class Manager
     # @raise    [Error::Invalid]
     #   On {#invalid?} platforms.
     def self.[]=( uri, platforms )
-        return new( platforms ) if !(key = make_key( uri ))
+        # For some reason we failed to make a key, try to salvage the situation.
+        if !(key = make_key( uri ))
+            return new_from_options( platforms )
+        end
 
         synchronize do
             @platforms[key] =
-                platforms.is_a?( self ) ? platforms : new( platforms )
+                platforms.is_a?( self ) ?
+                    platforms :
+                    new_from_options( platforms )
         end
     end
 
@@ -293,22 +338,6 @@ class Manager
         end
     end
 
-    # @param    [String, URI]   uri
-    #
-    # @return   [Manager]
-    #   Platform for the given `uri`
-    def self.[]( uri )
-        # If fingerprinting is disabled there's no point in filling the cache
-        # with the same object over and over, create an identical one for all
-        # URLs and return that always.
-        if !Options.fingerprint?
-            return @default ||= new( Options.platforms )
-        end
-
-        return new if !(key = make_key( uri ))
-        synchronize { @platforms[key] ||= new }
-    end
-
     # @return   [Boolean]
     #   `true` if there are no platforms fingerprints, `false` otherwise.
     def self.empty?
@@ -326,6 +355,10 @@ class Manager
         parsed.without_query
     end
 
+    def self.new_from_options( platforms = [] )
+        new( platforms | Options.platforms )
+    end
+
     # @param    [Array<String, Symbol>] platforms
     #   Platforms with which to initialize the lists.
     def initialize( platforms = [] )
@@ -335,7 +368,7 @@ class Manager
                 List.new( self.class.const_get( type.to_s.upcase.to_sym ) )
         end
 
-        update [platforms | Options.platforms].flatten.compact
+        update platforms
     end
 
     # @!method os