arachni 0.3 → 0.4

data/lib/{mixins → arachni/mixins}/observable.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License

data/lib/arachni/mixins/progress_bar.rb

@@ -0,0 +1,81 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+module Arachni
+
+module Mixins
+
+#
+# Progress bar and ETA methods
+#
+module ProgressBar
+
+    #
+    # Formats elapsed time to hour:min:sec
+    #
+    def format_time( t )
+        t = t.to_i
+        sec = t % 60
+        min = ( t / 60 ) % 60
+        hour = t / 3600
+        sprintf( "%02d:%02d:%02d", hour, min, sec )
+    end
+
+    #
+    # Calculates ETA (Estimated Time of Arrival) based on current progress
+    # and the start time of the operation.
+    #
+    # @param    [Float]   prog         current progress percentage
+    # @param    [Time]   start_time    start time of the operation
+    #
+    # @return   [String]    ETA: hour:min:sec
+    #
+    def eta( prog, start_time )
+        @last_prog ||= prog
+        @last_eta  ||= "--:--:--"
+
+        if @last_prog != prog
+            elapsed = Time.now - start_time
+            eta     = elapsed * 100 / prog - elapsed
+            eta_str = format_time( eta )
+        else
+            eta_str = @last_eta
+            prog = @last_prog
+        end
+
+        @last_prog = prog
+        @last_eta  = eta_str
+    end
+
+    #
+    # Returns an ASCII progress bar based on the current progress percentage
+    #
+    # @param    [Float]     progress     progress percentage
+    # @param    [Integer]   width        width of the progressbar in characters
+    #
+    # @return   [String]    70% [=======>  ] 100%
+    #
+    def progress_bar( progress, width = 100 )
+        progress = 100.0 if progress > 100
+
+        bar_prog = progress * width / 100
+
+        bar  = '=' * ( bar_prog.ceil - 1 ).abs + '>'
+        pad = ( width - bar_prog )
+        bar += ' ' * pad if pad > 0
+
+        "#{progress}% [#{bar}] 100% "
+    end
+
+    extend self
+end
+
+end
+end

data/lib/arachni/mixins/terminal.rb

@@ -0,0 +1,106 @@
+=begin
+                  Arachni
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+
+  This is free software; you can copy and distribute and modify
+  this program under the term of the GPL v2.0 License
+  (See LICENSE file for details)
+
+=end
+
+module Arachni
+
+module Mixins
+
+
+#
+# Terminal manipulation methods
+#
+#
+# Driver/demo code
+#
+#
+#        require_relative 'terminal'
+#        require_relative 'progress_bar'
+#
+#        include Terminal
+#        include ProgressBar
+#
+#        # clear the screen
+#        clear_screen!
+#
+#        start_time = Time.now
+#
+#        MAX = 5000
+#        (1..MAX).each {
+#            |i|
+#
+#            # move the cursor to its home, top-left of the screen.
+#            move_to_home!
+#
+#            prog =  i / Float( MAX ) * 100
+#
+#            reputs "Counting to #{MAX}..."
+#            reputs "Progress:   #{prog}%"
+#            reputs "Current:    #{i}"
+#
+#            reputs
+#            reprint eta( prog, start_time ) + "    "
+#            reputs progress_bar( prog.ceil )
+#
+#
+#            # make sure that everything is sent out on time
+#            flush!
+#            sleep 0.003
+#        }
+#
+module Terminal
+
+    #
+    # Clears the line before printing using 'puts'
+    #
+    # @param    [String]    str  string to output
+    #
+    def reputs( str = '' )
+        reprint str + "\n"
+    end
+
+    #
+    # Clears the line before printing
+    #
+    # @param    [String]    str  string to output
+    #
+    def reprint( str = '' )
+        print restr( str )
+    end
+
+    def restr( str = '' )
+        "\e[0K" + str
+    end
+
+    #
+    # Clear the bottom of the screen
+    #
+    def clear_screen!
+        print "\e[2J"
+    end
+
+    #
+    # Moves cursor top left to its home
+    #
+    def move_to_home!
+        print "\e[H"
+    end
+
+    #
+    # Flushes the STDOUT buffer
+    #
+    def flush!
+        $stdout.flush
+    end
+
+    extend self
+end
+
+end
+end

data/lib/{module → arachni/module}/auditor.rb

@@ -1,6 +1,6 @@
 =begin
                   Arachni
-  Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+  Copyright (c) 2010-2012 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 
   This is free software; you can copy and distribute and modify
   this program under the term of the GPL v2.0 License
@@ -25,7 +25,7 @@ module Module
 # @author: Tasos "Zapotek" Laskos
 #                                      <tasos.laskos@gmail.com>
 #                                      <zapotek@segfault.gr>
-# @version: 0.3
+# @version: 0.3.1
 #
 module Auditor
 
@@ -33,23 +33,86 @@ module Auditor
         # @@__timeout_audited      ||= Set.new
 
         # holds timing-attack performing Procs to be run after all
-        # non-tming-attack modules have finished.
+        # non-timing-attack modules have finished.
         @@__timeout_audit_blocks   ||= Queue.new
 
+        @@__timeout_audit_operations_cnt ||= 0
+
         # populated by timing attack phase 1 with
         # candidate elements to be verified by phase 2
         @@__timeout_candidates     ||= Queue.new
 
-        # modules which have called the timing attack audit mthod (audit_timeout)
+        # modules which have called the timing attack audit method (audit_timeout)
         # we're interested in the amount, not the names, and is used to
         # determine scan progress
         @@__timeout_loaded_modules ||= Set.new
 
+        @@__on_timing_attacks      ||= []
+
+        @@__running_timeout_attacks ||= false
+
         # the rdiff attack performs it own redundancy checks so we need this to
         # keep track audited elements
         @@__rdiff_audited ||= Set.new
     end
 
+    #
+    # Returns the names of all loaded modules that use timing attacks.
+    #
+    # @return   [Set]
+    #
+    def self.timeout_loaded_modules
+        @@__timeout_loaded_modules
+    end
+
+    #
+    # Holds timing-attack performing Procs to be run after all
+    # non-timing-attack modules have finished.
+    #
+    # @return   [Queue]
+    #
+    def self.timeout_audit_blocks
+        @@__timeout_audit_blocks
+    end
+
+    def self.current_timeout_audit_operations_cnt
+        @@__timeout_audit_blocks.size + @@__timeout_candidates.size
+    end
+
+    def self.add_timeout_audit_block( &block )
+        @@__timeout_audit_operations_cnt += 1
+        @@__timeout_audit_blocks << block
+    end
+
+    def Auditor.add_timeout_candidate( elem )
+        @@__timeout_audit_operations_cnt += 1
+        @@__timeout_candidates << elem
+    end
+
+
+    def self.running_timeout_attacks?
+        @@__running_timeout_attacks
+    end
+
+    def self.on_timing_attacks( &block )
+        @@__on_timing_attacks << block
+    end
+
+    def self.timeout_audit_operations_cnt
+        @@__timeout_audit_operations_cnt
+    end
+
+    def Auditor.call_on_timing_blocks( res, elem )
+        @@__on_timing_attacks.each {
+            |block|
+            block.call( res, elem )
+        }
+    end
+
+    def call_on_timing_blocks( res, elem )
+        Auditor.call_on_timing_blocks( res, elem )
+    end
+
     #
     # Holds constant bitfields that describe the preferred formatting
     # of injection strings.
@@ -62,7 +125,7 @@ module Auditor
       STRAIGHT = 1 << 0
 
       #
-      # Apends the injection string to the default value of the input vector.<br/>
+      # Appends the injection string to the default value of the input vector.<br/>
       # (If no default value exists Arachni will choose one.)
       #
       APPEND   = 1 << 1
@@ -91,6 +154,17 @@ module Auditor
         SERVER  = Issue::Element::SERVER
     end
 
+    RDIFF_OPTIONS =  {
+        # append our seeds to the default values
+        :format      => [ Format::APPEND ],
+
+        # allow duplicate requests
+        :redundant   => true,
+
+        # amount of rdiff iterations
+        :precision   => 2
+    }
+
     #
     # Default audit options.
     #
@@ -134,10 +208,10 @@ module Auditor
         #
         # If 'train' is set to true the HTTP response will be
         # analyzed for new elements. <br/>
-        # Be carefull when enabling it, there'll be a performance penalty.
+        # Be careful when enabling it, there'll be a performance penalty.
         #
         # When the Auditor submits a form with original or sample values
-        # this option will be overriden to true.
+        # this option will be overridden to true.
         #
         :train     => false,
 
@@ -152,6 +226,130 @@ module Auditor
         :async     => true
     }
 
+    #
+    # ABSTRACT - OPTIONAL
+    #
+    # Prevents auditing elements that have been previously
+    # logged by any of the modules returned by this method.
+    #
+    # @return   [Array]     module names
+    #
+    def redundant
+        # [ 'sqli', 'sqli_blind_rdiff' ]
+        []
+    end
+
+    #
+    # ABSTRACT - OPTIONAL
+    #
+    # Allows modules to ignore HPG scope restrictions
+    #
+    # This way they can audit elements that are not on the Grid sanctioned whitlist.
+    #
+    # @return   [Bool]
+    #
+    def override_instance_scope?
+        false
+    end
+
+    #
+    # Just a delegator logs an array of issues.
+    #
+    # @param    [Array<Arachni::Issue>]     issues
+    #
+    # @see Arachni::Module::Manager.register_results
+    #
+    def register_results( issues )
+        Arachni::Module::Manager.register_results( issues )
+    end
+
+    #
+    # Logs a remote file or directory if it exists.
+    #
+    # @param    [String]    url
+    # @param    [Bool]      silent  if false, a message will be sent to stdout
+    #                               containing the status of the operation.
+    # @param    [Proc]      &block  called if the file exists, just before logging
+    #
+    # @return   [Object]    - nil if no URL was provided
+    #                       - false if the request couldn't be fired
+    #                       - true if everything went fine
+    #
+    # @see #remote_file_exist?
+    #
+    def log_remote_file_if_exists( url, silent = false, &block )
+        return nil if !url
+
+        req  = @http.get( url )
+        return false if !req
+        req.on_complete {
+            |res|
+
+            print_status( 'Analyzing response for: ' + url ) if !silent
+
+            if remote_file_exist?( res )
+                block.call( res ) if block_given?
+                log_remote_file( res )
+
+                # if the file exists let the trainer parse it since it may
+                # contain brand new data to audit
+                @http.trainer.add_response( res )
+            end
+        }
+
+        return true
+    end
+    alias :log_remote_directory_if_exists :log_remote_file_if_exists
+
+    #
+    # Checks that the response points to an existing file/page and not
+    # an error or custom 404 response.
+    #
+    # @param    [Typhoeus::Response]    res
+    #
+    def remote_file_exist?( res )
+        res.code == 200 && !@http.custom_404?( res )
+    end
+
+    #
+    # Logs the existence of a remote file as an issue.
+    #
+    # @param    [Typhoeus::Response]    res
+    #
+    # @see #log_issue
+    #
+    def log_remote_file( res )
+        url = res.effective_url
+        filename = File.basename( URI( res.effective_url ).path )
+
+        log_issue(
+            :url          => url,
+            :injected     => filename,
+            :id           => filename,
+            :elem         => Issue::Element::PATH,
+            :response     => res.body,
+            :headers      => {
+                :request    => res.request.headers,
+                :response   => res.headers,
+            }
+        )
+
+    end
+    alias :log_remote_directory :log_remote_file
+
+    #
+    # Helper method for issue logging.
+    #
+    # @param    [Hash]  opts    issue options ({Issue})
+    # @param    [Bool]  include_class_info    merge opts with module.info?
+    #
+    # @see Arachni::Module::Base#register_results
+    #
+    def log_issue( opts )
+        # register the issue
+        register_results( [ Issue.new( opts.merge( self.class.info ) ) ] )
+    end
+
     #
     # Matches the "string" (default string is the HTML code in @page.html) to
     # an array of regular expressions and logs the results.
@@ -231,7 +429,7 @@ module Auditor
             request_headers  = res.request.headers
             response_headers = res.headers
             response         = res.body
-            url              = res.effective_url
+            url              = opts[:action]
             method           = res.request.method.to_s.upcase
         end
 
@@ -251,9 +449,8 @@ module Auditor
         print_debug( 'Request ID: ' + res.request.id.to_s ) if res
         print_verbose( '---------' ) if only_positives?
 
-        # Instantiate a new Vulnerability class and
-        # append it to the results array
-        vuln = Issue.new( {
+        # Instantiate a new Issue class and append it to the results array
+        log_issue(
             :var          => opts[:altered],
             :url          => url,
             :injected     => opts[:injected],
@@ -269,8 +466,7 @@ module Auditor
                 :request    => request_headers,
                 :response   => response_headers,
             }
-        }.merge( self.class.info ) )
-        register_results( [vuln] )
+        )
     end
 
     #
@@ -323,9 +519,9 @@ module Auditor
 
                 when  Element::HEADER
                     audit_headers( injection_str, opts, &block )
+                when  Element::BODY
                 else
                     raise( 'Unknown element to audit:  ' + elem.to_s )
-
             end
 
         }
@@ -333,7 +529,7 @@ module Auditor
 
     #
     # This is called right before an [Arachni::Parser::Element]
-    # is submitted/auditted and is used to determine whether to skip it or not.
+    # is submitted/audited and is used to determine whether to skip it or not.
     #
     # Running modules can override this as they wish *but* at their own peril.
     #
@@ -347,17 +543,11 @@ module Auditor
 
             set_id = @framework.modules.class.issue_set_id_from_elem( mod_name, elem )
             return true if @framework.modules.issue_set.include?( set_id )
-        }
-
-
-        # if !@@__timeout_audited.empty?
-            # return @@__timeout_audited.include?( __rdiff_audit_id( elem ) )
-        # end
+        } if @framework
 
         return false
     end
 
-
     #
     # Audits elements using timing attacks and automatically logs results.
     #
@@ -365,7 +555,7 @@ module Auditor
     # * Loop 1 -- Populates the candidate queue. We're picking the low hanging
     #   fruit here so we can run this in larger concurrent bursts which cause *lots* of noise.
     #   - Initial probing for candidates -- Any element that times out is added to a queue.
-    #   - Stabilization -- The candidate is submited with its default values in
+    #   - Stabilization -- The candidate is submitted with its default values in
     #     order to wait until the effects of the timing attack have worn off.
     # * Loop 2 -- Verifies the candidates. This is much more delicate so the
     #   concurrent requests are lowered to pairs.
@@ -387,7 +577,7 @@ module Auditor
     #
     #
     # @param   [Array]     strings     injection strings
-    #                                       __TIME__ will be substituded with (timeout / timeout_divider)
+    #                                       __TIME__ will be substituted with (timeout / timeout_divider)
     # @param  [Hash]        opts        options as described in {OPTIONS} with the following extra:
     #                                   * :timeout -- milliseconds to wait for the request to complete
     #                                   * :timeout_divider -- __TIME__ = timeout / timeout_divider
@@ -395,54 +585,31 @@ module Auditor
     def audit_timeout( strings, opts )
         @@__timeout_loaded_modules << self.class.info[:name]
 
-        @@__timeout_audit_blocks << Proc.new {
+        Auditor.add_timeout_audit_block {
             delay = opts[:timeout]
 
             audit_timeout_debug_msg( 1, delay )
             timing_attack( strings, opts ) {
-                |res, opts, elem|
+                |res, c_opts, elem|
 
-                # maybe this should be removed to take care of accidental timeouts
-                # and let phase 2 clean up the mess
-                # if !@@__timeout_audited.include?( __rdiff_audit_id( elem ) )
+                elem.auditor( self )
 
-                    elem.auditor( self )
-                    # @@__timeout_audited << __rdiff_audit_id( elem )
+                print_info( "Found a candidate -- #{elem.type.capitalize} input '#{elem.altered}' at #{elem.action}" )
 
-                    print_info( "Found a candidate -- #{elem.type.capitalize} input '#{elem.altered}' at #{elem.action}" )
+                Arachni::Module::Auditor.audit_timeout_stabilize( elem )
 
-                    Arachni::Module::Auditor.audit_timeout_stabilize( elem )
-
-                    @@__timeout_candidates << elem
-                # end
+                Auditor.add_timeout_candidate( elem )
             }
         }
     end
 
-    #
-    # Returns the names of all loaded modules that use timing attacks.
-    #
-    # @return   [Set]
-    #
-    def self.timeout_loaded_modules
-        @@__timeout_loaded_modules
-    end
-
-    #
-    # Holds timing-attack performing Procs to be run after all
-    # non-tming-attack modules have finished.
-    #
-    # @return   [Queue]
-    #
-    def self.timeout_audit_blocks
-        @@__timeout_audit_blocks
-    end
-
     #
     # Runs all blocks in {timeout_audit_blocks} and verifies
     # and logs the candidate elements.
     #
     def self.timeout_audit_run
+        @@__running_timeout_attacks = true
+
         while( !@@__timeout_audit_blocks.empty? )
             @@__timeout_audit_blocks.pop.call
         end
@@ -453,7 +620,7 @@ module Auditor
     end
 
     #
-    # Runs phase 2 of the timing attack auditng an individual element
+    # Runs phase 2 of the timing attack auditing an individual element
     # (which passed phase 1) with a higher delay and timeout
     #
     def self.audit_timeout_phase_2( elem )
@@ -478,19 +645,21 @@ module Auditor
         elem.get_auditor.http.get( elem.action ).on_complete {
             |res|
 
+            self.call_on_timing_blocks( res, elem )
+
             if !res.timed_out?
 
                 elem.get_auditor.print_info( 'Liveness check was successful, progressing to verification...' )
 
                 elem.audit( str, opts ) {
-                    |res, opts|
+                    |c_res, c_opts|
 
-                    if res.timed_out?
+                    if c_res.timed_out?
 
                         # all issues logged by timing attacks need manual verification.
                         # end of story.
-                        opts[:verification] = true
-                        elem.get_auditor.log( opts, res)
+                        # c_opts[:verification] = true
+                        elem.get_auditor.log( c_opts, c_res )
 
                         self.audit_timeout_stabilize( elem )
 
@@ -554,7 +723,7 @@ module Auditor
     # Optionally, you can add a :timeout_divider.
     #
     # @param   [Array]     strings     injection strings
-    #                                       '__TIME__' will be substituded with (timeout / timeout_divider)
+    #                                       '__TIME__' will be substituted with (timeout / timeout_divider)
     # @param    [Hash]      opts        options as described in {OPTIONS}
     # @param    [Block]     &block      block to call if a timeout occurs,
     #                                       it will be passed the response and opts
@@ -562,7 +731,6 @@ module Auditor
     def timing_attack( strings, opts, &block )
 
         opts[:timeout_divider] ||= 1
-        # opts[:async] = false
 
         [strings].flatten.each {
             |str|
@@ -572,8 +740,10 @@ module Auditor
             opts[:skip_orig] = true
 
             audit( str, opts ) {
-                |res, opts, elem|
-                block.call( res, opts, elem ) if block && res.timed_out?
+                |res, c_opts, elem|
+
+                call_on_timing_blocks( res, elem )
+                block.call( res, c_opts, elem ) if block && res.timed_out?
             }
         }
 
@@ -627,15 +797,6 @@ module Auditor
             opts[:elements] = OPTIONS[:elements]
         end
 
-        opts = {
-            # append our seeds to the default values
-            :format      => [ Format::APPEND ],
-            # allow duplicate requests
-            :redundant   => true,
-            # amound of rdiff iterations
-            :precision   => 2
-        }.merge( opts )
-
         opts[:elements].each {
             |elem|
 
@@ -644,30 +805,31 @@ module Auditor
                 when  Element::LINK
                     next if !Options.instance.audit_links
                     @page.links.each {
-                        |elem|
-                        audit_rdiff_elem( elem, opts, &block )
+                        |c_elem|
+                        audit_rdiff_elem( c_elem, opts, &block )
                     }
 
                 when  Element::FORM
                     next if !Options.instance.audit_forms
                     @page.forms.each {
-                        |elem|
-                        audit_rdiff_elem( elem, opts, &block )
+                        |c_elem|
+                        audit_rdiff_elem( c_elem, opts, &block )
                     }
 
                 when  Element::COOKIE
                     next if !Options.instance.audit_cookies
                     @page.cookies.each {
-                        |elem|
-                        audit_rdiff_elem( elem, opts, &block )
+                        |c_elem|
+                        audit_rdiff_elem( c_elem, opts, &block )
                     }
 
                 when  Element::HEADER
                     next if !Options.instance.audit_headers
                     @page.headers.each {
-                        |elem|
-                        audit_rdiff_elem( elem, opts, &block )
+                        |c_elem|
+                        audit_rdiff_elem( c_elem, opts, &block )
                     }
+                when  Element::BODY
                 else
                     raise( 'Unknown element to audit:  ' + elem.to_s )
 
@@ -685,6 +847,8 @@ module Auditor
     #
     def audit_rdiff_elem( elem, opts = {}, &block )
 
+        opts = RDIFF_OPTIONS.merge( opts )
+
         # don't continue if there's a missing value
         elem.auditable.values.each {
             |val|
@@ -814,7 +978,7 @@ module Auditor
                                 :regexp_match => 'n/a',
                                 :elem         => res['elem'].type,
                                 :response     => res['res'].body,
-                                :verification => true,
+                                # :verification => true,
                                 :headers      => {
                                     :request    => res['res'].request.headers,
                                     :response   => res['res'].headers,
@@ -874,7 +1038,7 @@ module Auditor
     end
 
     #
-    # Audits Auditalble HTML/HTTP elements
+    # Audits Auditable HTML/HTTP elements
     #
     # @param  [Array<Arachni::Element::Auditable>]  elements    elements to audit
     # @param  [String]  injection_str  same as for {#audit}
@@ -885,12 +1049,12 @@ module Auditor
     #
     def audit_elems( elements, injection_str, opts = { }, &block )
 
-        opts            = OPTIONS.merge( opts )
-        url             = @page.url
+        opts = OPTIONS.merge( opts )
+        url  = @page.url
 
         opts[:injected_orig] = injection_str
 
-        elements.each{
+        elements.deep_clone.each {
             |elem|
             elem.auditor( self )
             elem.audit( injection_str, opts, &block )