arachni 0.4.5.2 → 0.4.6

data/lib/arachni/trainer.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -44,20 +44,18 @@ class Trainer
         # get us setup using the page that is being audited as a seed page
         framework.on_audit_page { |page| self.page = page }
 
-        HTTP.add_on_queue do |req, _|
-            next if !req.train?
-
-            req.on_complete( true ) do |res|
-                # handle redirections
-                if res.redirection? && res.location.is_a?( String )
-                    reference_url = @page ? @page.url : framework.opts.url
-                    HTTP.get( to_absolute( res.location, reference_url ) ) do |res2|
-                        push( res2 )
-                    end
-                else
-                    push( res )
+        HTTP.add_on_complete do |response|
+            next if !response.request.train?
+
+            if response.redirection? && response.location.is_a?( String )
+                reference_url = @page ? @page.url : @framework.opts.url
+                HTTP.get( to_absolute( response.location, reference_url ) ) do |res|
+                    push res
                 end
+                next
             end
+
+            push response
         end
     end
 
@@ -78,15 +76,27 @@ class Trainer
         end
 
         if @framework.link_count_limit_reached?
-            print_info 'Link count limit reached, skipping analysis.'
-            return
+            print_verbose 'Link count limit reached, skipping analysis.'
+            return false
         end
 
         @parser = Parser.new( res )
 
-        return false if !@parser.text? ||
-            @trainings_per_url[@parser.url] >= MAX_TRAININGS_PER_URL ||
-            redundant_path?( @parser.url ) || skip_resource?( res )
+        return false if !@parser.text?
+
+        skip_message = nil
+        if @trainings_per_url[@parser.url] >= MAX_TRAININGS_PER_URL
+            skip_message = "Reached maximum trainings (#{MAX_TRAININGS_PER_URL})"
+        elsif redundant_path?( @parser.url )
+            skip_message = 'Matched redundancy filters'
+        elsif skip_resource?( res )
+            skip_message = 'Matched exclusion criteria'
+        end
+
+        if skip_message
+            print_verbose "#{skip_message}, skipping: #{@parser.url}"
+            return false
+        end
 
         analyze( res )
         true
@@ -138,12 +148,13 @@ class Trainer
             page_data[:code]             = res.code
             page_data[:method]           = res.request.method.to_s.upcase
             page_data[:body]             = res.body
-            page_data[:doc]              = @parser.doc
+            page_data[:document]         = @parser.doc
             page_data[:response_headers] = res.headers_hash
 
             @trainings_per_url[@parser.url] += 1
 
             page = Page.new( page_data )
+            Platform::Manager.fingerprint( page ) if Options.fingerprint?
 
             @on_new_page_blocks.each { |block| block.call page }
 

data/lib/arachni/typhoeus/hydra.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/typhoeus/request.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/typhoeus/response.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ class Response
     def text?
         return if !@body
 
-        if type = content_type
+        if (type = content_type)
             return true if type.start_with?( 'text/' )
 
             # Non "application/" content types will surely not be text-based
@@ -65,6 +65,12 @@ class Response
         (300..399).include?( @code ) || !location.nil?
     end
 
+    # @return   [Float]
+    #   Approximated time the web application took to process the request.
+    def app_time
+        timed_out? ? time : start_transfer_time - pretransfer_time
+    end
+
     # @return    [Hash]   converts self to hash
     def to_hash
         hash = {}

data/lib/arachni/typhoeus/utils.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/ui/cli/cli.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -84,8 +84,9 @@ class CLI
         # trap Ctrl+C interrupts
         trap( 'INT' ) { handle_interrupt }
 
-        # trap SIGUSR1 interrupts
+        # trap user signals
         trap ( 'USR1' ) { handle_usr1_interrupt }
+        trap ( 'USR2' ) { handle_usr2_interrupt }
     end
 
     #
@@ -179,8 +180,8 @@ class CLI
         print_line( restr, unmute )
     end
 
-    def print_issues( audit_store, unmute = false )
-        super( audit_store.issues, unmute, &method( :restr ) )
+    def print_issues( unmute = false )
+        super( AuditStore.sort( @arachni.modules.issues ), unmute, &method( :restr ) )
     end
 
     def kill_interrupt_handler
@@ -235,7 +236,7 @@ class CLI
                 print_info( restr( 'Results thus far:' ), true )
 
                 begin
-                    print_issues( @arachni.audit_store, true )
+                    print_issues( true )
                     print_stats( true, true )
                 rescue Exception => e
                     exception_jail{ raise e }
@@ -254,15 +255,25 @@ class CLI
     end
 
     #
-    # Handles SIGUSR1 system calls
+    # Handles SIGUSR1 signals
     #
     # It will cause Arachni to create a report and shut down afterwards
     #
     def handle_usr1_interrupt
-        print_status 'Received SIGUSR1!'
+        print_info 'Received SIGUSR1!'
         shutdown
     end
 
+    #
+    # Handles SIGUSR2 signals
+    #
+    # It will cause Arachni to create a report.
+    #
+    def handle_usr2_interrupt
+        print_info 'Received SIGUSR2!'
+        @arachni.reports.run( @arachni.audit_store )
+    end
+
     def shutdown
         print_status 'Exiting...'
         print_info 'Please wait while the system cleans up.'

data/lib/arachni/ui/cli/output.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -345,6 +345,8 @@ module Output
         return if muted? && !unmute
 
         str = intercept_print_message( string )
+        str = add_resource_usage_statistics( str ) if Arachni.profile?
+
         # we may get IO errors...freaky stuff...
         begin
             if out.tty?
@@ -356,6 +358,31 @@ module Output
         end
     end
 
+    def add_resource_usage_statistics( message )
+        require 'sys/proctable'
+
+        procinfo = ::Sys::ProcTable.ps( Process.pid )
+        pctcpu   = procinfo[:pctcpu]
+        pctmem   = procinfo[:pctmem]
+        rss      = procinfo[:rss]
+        @rss   ||= rss
+
+        # Change in RAM consumption in MB | Total RAM consumption in MB (RAM %) |
+        # CPU usage % | Amount of Typhoeus::Request - Typhoeus::Response objects in RAM |
+        # Proc objects in RAM
+        sprintf( '%7.4f | %8.4f (%5.1f%%) | %5.1f%% | %3i - %3i | %5i | ',
+                rss_to_mb(rss - @rss), rss_to_mb(rss), pctmem, pctcpu,
+                ::ObjectSpace.each_object( ::Typhoeus::Request ){},
+                ::ObjectSpace.each_object( ::Typhoeus::Response ){},
+                ::ObjectSpace.each_object( ::Proc ){}) + message
+    ensure
+        @rss = rss
+    end
+
+    def rss_to_mb( rss )
+        rss * 4096.0 / 1024.0 / 1024.0
+    end
+
     extend self
 
 end

data/lib/arachni/ui/cli/rpc/dispatcher_monitor.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/ui/cli/rpc/instance.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/ui/cli/rpc/local.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/ui/cli/rpc/remote.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/ui/cli/utilities.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -21,13 +21,6 @@ require Options.dir['mixins'] + 'terminal'
 require Options.dir['mixins'] + 'progress_bar'
 
 module UI
-
-#
-#
-# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-#
-# @see Arachni::Framework::CLI
-#
 class CLI
 
 module Utilities
@@ -62,14 +55,13 @@ module Utilities
     #
     # Outputs all available modules and their info.
     #
-    def lsplat( platforms )
+    def lsplat( platform_info )
         print_line
         print_line
         print_info 'Available platforms:'
         print_line
 
-        i = 0
-        platforms.each do |type, platforms|
+        platform_info.each do |type, platforms|
             print_status "#{type}"
 
             platforms.each do |shortname, fullname|
@@ -90,7 +82,6 @@ module Utilities
         print_info 'Available modules:'
         print_line
 
-        i = 0
         modules.each do |info|
             print_status "#{info[:mod_name]}:"
             print_line '--------------------'
@@ -98,6 +89,10 @@ module Utilities
             print_line "Name:\t\t#{info[:name]}"
             print_line "Description:\t#{info[:description]}"
 
+            if info[:issue] && (severity = info[:issue][:severity])
+                print_line "Severity:\t#{severity}"
+            end
+
             if info[:elements] && info[:elements].size > 0
                 print_line "Elements:\t#{info[:elements].join( ', ' ).downcase}"
             end
@@ -130,21 +125,6 @@ module Utilities
 
             print_line "Path:\t#{info[:path]}"
 
-            i += 1
-
-            # pause every 3 modules to give the user time to read
-            # (cheers to aungkhant@yehg.net for suggesting it)
-            if i % 3 == 0 && i != modules.size
-                print_line
-                print_line 'Hit <space> <enter> to continue, any other key to exit. '
-
-                if gets[0] != ' '
-                    print_line
-                    return
-                end
-
-            end
-
             print_line
         end
 
@@ -175,7 +155,7 @@ module Utilities
                     print_info "\t#{option['name']} - #{option['desc']}"
                     print_info "\tType:        #{option['type']}"
                     print_info "\tDefault:     #{option['default']}"
-                    print_info "\tRequired?:   #{option['required?']}"
+                    print_info "\tRequired?:   #{option['required']}"
 
                     print_line
                 end
@@ -214,7 +194,7 @@ module Utilities
                     print_info "\t#{option['name']} - #{option['desc']}"
                     print_info "\tType:        #{option['type']}"
                     print_info "\tDefault:     #{option['default']}"
-                    print_info "\tRequired?:   #{option['required?']}"
+                    print_info "\tRequired?:   #{option['required']}"
 
                     print_line
                 end
@@ -315,6 +295,11 @@ module Utilities
                                   (Be careful not to kill your server.)
                                   (*NOTE*: If your scan seems unresponsive try lowering the limit.)
 
+    --http-queue-size=<integer> Maximum amount of requests to keep in the queue.
+                                  Bigger size means better scheduling and better performance,
+                                  smaller means less RAM consumption.
+                                  (Default: #{@opts.http_queue_size})
+
     --http-timeout=<integer>    HTTP request timeout in milliseconds.
 
     --cookie-jar=<filepath>     Netscape HTTP cookie file, use curl to create it.

data/lib/arachni/ui/foo/output.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/uri.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -291,8 +291,7 @@ class URI
                 if c_url.include?( '?' ) && !(query = dupped_url.split( '?', 2 ).last).empty?
                     components[:query] = (query.split( '&', -1 ).map do |pair|
                         Addressable::URI.normalize_component( pair,
-                            Addressable::URI::CharacterClasses::QUERY.sub( '\\&', '' ),
-                            '+'
+                            Addressable::URI::CharacterClasses::QUERY.sub( '\\&', '' )
                         )
                     end).join( '&' )
                 end

data/lib/arachni/utilities.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/arachni/version.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/lib/version

@@ -1 +1 @@
-0.4.5.2
+0.4.6

data/modules/audit/code_injection.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

data/modules/audit/code_injection_php_input_wrapper.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2013 Tasos Laskos <tasos.laskos@gmail.com>
+    Copyright 2010-2014 Tasos Laskos <tasos.laskos@gmail.com>
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.