arachni 1.4 → 1.6.0

data/components/plugins/email_notify.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -12,9 +12,10 @@ require 'pony'
 # of the scan over SMTP.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-# @version 0.1.5
 class Arachni::Plugins::EmailNotify < Arachni::Plugin::Base
 
+    TRIES = 5
+
     def run
         wait_while_framework_running
     end
@@ -34,8 +35,8 @@ class Arachni::Plugins::EmailNotify < Arachni::Plugin::Base
                 address:              options[:server_address],
                 port:                 options[:server_port],
                 enable_starttls_auto: options[:tls],
-                user_name:            options[:username],
-                password:             options[:password],
+                user_name:            !options[:username].empty? ? options[:username] : nil,
+                password:             !options[:password].empty? ? options[:password] : nil,
                 authentication:       !options[:authentication].empty? ? options[:authentication].to_sym : nil,
                 domain:               options[:domain]
             }
@@ -57,9 +58,25 @@ class Arachni::Plugins::EmailNotify < Arachni::Plugin::Base
 
         print_status 'Sending the notification...'
 
-        Pony.mail( opts )
+        sent      = false
+        exception = nil
+        TRIES.times do |i|
+            begin
+                Pony.mail( opts )
+                sent = true
+                break
+            rescue => e
+                exception = e
+                print_bad "Attempt ##{i} failed, retrying..."
+            end
+        end
 
-        print_status 'Done.'
+        if sent
+            print_status 'Done.'
+        else
+            print_error "Failed after #{TRIES} tries."
+            print_exception exception
+        end
     end
 
     def self.info
@@ -67,7 +84,7 @@ class Arachni::Plugins::EmailNotify < Arachni::Plugin::Base
             name:        'E-mail notify',
             description: %q{Sends a notification (and optionally a report) over SMTP at the end of the scan.},
             author:      'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
-            version:     '0.1.6',
+            version:     '0.1.7',
             options:     [
                 Options::String.new( :to,
                     required:    true,
@@ -111,8 +128,8 @@ class Arachni::Plugins::EmailNotify < Arachni::Plugin::Base
                 ),
                 Options::MultipleChoice.new( :report,
                     description: 'Report format to send as an attachment.',
-                    default:     'txt',
-                    choices:     ['txt', 'xml', 'html', 'json', 'yaml', 'marshal', 'afr', 'none']
+                    default:     'afr',
+                    choices:     ['xml', 'html', 'json', 'yaml', 'marshal', 'afr', 'none']
                 )
             ]
 

data/components/plugins/exec.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/form_dicattack.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -22,12 +22,11 @@ class Arachni::Plugins::FormDicattack < Arachni::Plugin::Base
         @verifier = Regexp.new( options[:login_verifier] )
 
         @found = false
-    end
 
-    def run
         framework_pause
-        print_info 'System paused.'
+    end
 
+    def run
         form = session.find_login_form(
             url:    @url,
             inputs: [ @user_field, @passwd_field ]

data/components/plugins/headers_collector.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/http_dicattack.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -17,18 +17,17 @@ class Arachni::Plugins::HTTPDicattack < Arachni::Plugin::Base
         @passwds = File.read( options[:password_list] ).split( "\n" )
 
         @found = false
+
+        framework_pause
     end
 
     def run
         if !protected?( @url )
             print_info "The URL you provided doesn't seem to be protected."
             print_info 'Aborting...'
-            return
+            return framework_resume
         end
 
-        framework_pause
-        print_info 'System paused.'
-
         url = uri_parse( @url )
 
         print_status 'Building the request queue...'

data/components/plugins/login_script.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -119,7 +119,7 @@ that process and, for example, load and set cookies from a shared cookie-jar.
 
 ## With browser (slow)
 
-If a [browser](http://watirwebdriver.com/) is available, it will be exposed to
+If a [browser](http://watir.github.io/) is available, it will be exposed to
 the script via the `browser` variable. Otherwise, that variable will have a
 value of `nil`.
 

data/components/plugins/metrics.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -7,7 +7,6 @@
 =end
 
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-# @version 0.1
 class Arachni::Plugins::Metrics < Arachni::Plugin::Base
 
     def prepare
@@ -26,18 +25,26 @@ class Arachni::Plugins::Metrics < Arachni::Plugin::Base
             },
             'http'      => {
                 'requests'              => 0,
-                'response_time_min'     => 0,
-                'response_time_max'     => 0,
-                'response_time_average' => 0,
+                'request_time_outs'     => 0,
                 'request_size_min'      => 0,
                 'request_size_max'      => 0,
                 'request_size_average'  => 0,
+                'responses_per_second'  => 0.0,
+                'response_time_min'     => 0,
+                'response_time_max'     => 0,
+                'response_time_average' => 0,
                 'response_size_min'     => 0,
                 'response_size_max'     => 0,
                 'response_size_average' => 0
             },
+            'browser_cluster' => {
+                'seconds_per_job' => 0.0,
+                'total_job_time'  => 0.0,
+                'job_time_outs'   => 0.0,
+                'job_count'       => 0
+            },
             'resource'  => {
-                'binary'            => Arachni::Support::LookUp::HashSet.new,
+                'binary'             => Arachni::Support::LookUp::HashSet.new,
                 'without_parameters' => Arachni::Support::LookUp::HashSet.new,
                 'with_parameters'    => Arachni::Support::LookUp::HashSet.new
             },
@@ -178,23 +185,42 @@ class Arachni::Plugins::Metrics < Arachni::Plugin::Base
 
         wait_while_framework_running
 
-        @metrics = process( @metrics )
+        metrics = process( @metrics )
 
-        @metrics['http']['requests'] = framework.statistics[:http][:response_count]
+        statistics = framework.statistics
 
-        @metrics['http']['response_time_average'] =
-            http_response_time_total / @metrics['http']['requests']
+        metrics['browser_cluster']['job_time_outs'] =
+            statistics[:browser_cluster][:time_out_count]
 
-        @metrics['http']['response_size_average'] =
-            @metrics['general']['ingress_traffic'] / @metrics['http']['requests']
+        metrics['browser_cluster']['seconds_per_job'] =
+            statistics[:browser_cluster][:seconds_per_job]
 
-        @metrics['http']['request_size_average'] =
-            @metrics['general']['egress_traffic'] / @metrics['http']['requests']
+        metrics['browser_cluster']['total_job_time'] =
+            statistics[:browser_cluster][:total_job_time]
+
+        metrics['browser_cluster']['job_count'] =
+            statistics[:browser_cluster][:queued_job_count]
+
+        metrics['http']['requests'] = statistics[:http][:response_count]
+
+        metrics['http']['request_time_outs']    = statistics[:http][:time_out_count]
+        metrics['http']['responses_per_second'] = statistics[:http][:total_responses_per_second]
+
+        if metrics['http']['requests'] > 0
+            metrics['http']['response_time_average'] =
+                http_response_time_total / metrics['http']['requests']
+
+            metrics['http']['response_size_average'] =
+                metrics['general']['ingress_traffic'] / metrics['http']['requests']
+
+            metrics['http']['request_size_average'] =
+                metrics['general']['egress_traffic'] / metrics['http']['requests']
+        end
 
-        @metrics['scan']['duration']      = framework.statistics[:runtime]
-        @metrics['scan']['authenticated'] = !!Arachni::Options.session.check_url
+        metrics['scan']['duration']      = statistics[:runtime]
+        metrics['scan']['authenticated'] = !!Arachni::Options.session.check_url
 
-        register_results @metrics
+        register_results metrics
     end
 
     def find_swf( page )
@@ -228,7 +254,7 @@ class Arachni::Plugins::Metrics < Arachni::Plugin::Base
 Captures metrics about multiple aspects of the scan and the web application.
 },
             author:      'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
-            version:     '0.1'
+            version:     '0.1.2'
         }
     end
 

data/components/plugins/page_dump.rb

@@ -0,0 +1,60 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
+# @version 0.1
+class Arachni::Plugins::PageDump < Arachni::Plugin::Base
+
+    def run
+        cnt  = 0
+        seen = Support::LookUp::HashSet.new
+
+        framework.on_page_audit do |page|
+            data = {
+                url:         page.dom.url,
+                response:    page.response.to_s,
+                request:     page.request.to_s,
+                source:      page.body,
+                transitions: page.dom.transitions.map(&:to_h)
+            }
+
+            next if seen.include? data
+            seen << data
+
+            cnt += 1
+            IO.write( "#{@options[:directory]}/#{cnt}.yaml", data.to_yaml )
+        end
+
+        wait_while_framework_running
+    end
+
+    def self.info
+        {
+            name:        'Page dump',
+            description: %q{
+Dumps the following page data to disk as YAML:
+
+* URL
+* Raw HTTP response
+* Raw HTTP request
+* Page source -- can differ due to JS.
+* DOM transitions
+
+The plugin will create one file for each unique page.
+},
+            author:      'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
+            version:     '0.1',
+            options:     [
+                Options::Path.new( :directory,
+                    description: 'Directory where which to save page data.'
+                )
+            ]
+        }
+    end
+
+end

data/components/plugins/proxy/panel/verify_login_sequence.html.erb

@@ -15,7 +15,7 @@
             <a class="close" data-dismiss="alert" href="#">×</a>
             Couldn't find a login form, this can happen for a number of reasons:
             <ul>
-                <li>The login form was generated using JavaScript -- Arachni can't interpret JS yet</li>
+                <li>The login form was generated using JavaScript</li>
                 <li>The login form was generated using a client-side applet (like Flash)</li>
             </ul>
         </div>

data/components/plugins/proxy/template_scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -15,6 +15,7 @@ class Arachni::Plugins::Proxy
 #
 class TemplateScope
     include Arachni::Utilities
+    include Arachni::UI::Output
 
     PANEL_BASEDIR  = "#{Arachni::Plugins::Proxy::BASEDIR}panel/"
     PANEL_TEMPLATE = "#{PANEL_BASEDIR}panel.html.erb"
@@ -108,6 +109,10 @@ class TemplateScope
 
     def render( tpl, opts )
         erb tpl, opts.merge( layout: false )
+    rescue => e
+        print_error "Error when rendering: #{tpl}"
+        print_exception e
+        nil
     end
 
     def layout

data/components/plugins/proxy.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -45,14 +45,14 @@ class Arachni::Plugins::Proxy < Arachni::Plugin::Base
              timeout:          options[:timeout]
         )
 
-        @pages = Set.new
+        @pages       = Set.new
+        @pages_mutex = Mutex.new
         @login_sequence = []
-    end
 
-    def run
         framework_pause
-        print_info 'System paused.'
+    end
 
+    def run
         print_status "Listening on: #{@server.url}"
 
         print_info "Control panel URL: #{url_for( :panel )}"
@@ -84,7 +84,10 @@ class Arachni::Plugins::Proxy < Arachni::Plugin::Base
 
         @server.shutdown
 
-        @pages.each { |p| framework.push_to_page_queue( p, true ) }
+        @pages_mutex.synchronize do
+            @pages.each { |p| framework.push_to_page_queue( p, true ) }
+        end
+
         framework_resume
     end
 
@@ -196,9 +199,10 @@ class Arachni::Plugins::Proxy < Arachni::Plugin::Base
         res.code = 200
 
         if url.start_with? url_for( :panel )
+            TemplateScope.get.set :pages, prepare_pages_for_inspection
+
             body =  case '/' + res.parsed_url.path.split( '/' )[2..-1].join( '/' )
                         when '/'
-                            TemplateScope.get.set :pages, prepare_pages_for_inspection
                             erb :panel
 
                         when '/vectors.yml'
@@ -326,11 +330,13 @@ class Arachni::Plugins::Proxy < Arachni::Plugin::Base
     def find_login_form_from_request( request )
         return if (params = request_parse_body( request.body )).empty?
 
-        f = session.find_login_form(
-            pages:  @pages.to_a,
-            action: normalize_url( request.url ),
-            inputs: params.keys
-        )
+        f = @pages_mutex.synchronize do
+            session.find_login_form(
+                pages:  @pages.to_a,
+                action: normalize_url( request.url ),
+                inputs: params.keys
+            )
+        end
 
         return if !f
         f.update( params )
@@ -341,28 +347,32 @@ class Arachni::Plugins::Proxy < Arachni::Plugin::Base
     #
     # @return   [Array<Arachni::Element::Form>]
     def forms_with_password
-        @pages.map { |p| p.forms.select { |f| f.requires_password? } }.flatten
+        @pages_mutex.synchronize do
+            @pages.map { |p| p.forms.select { |f| f.requires_password? } }.flatten
+        end
     end
 
     def prepare_pages_for_inspection
-        (@pages.map do |p|
-            next if !p.text?
-            p = p.dup
-
-            %w(links forms cookies jsons xmls).each do |type|
-                p.send(
-                    "#{type}=",
-                    p.send(type).reject { |e| e.inputs.empty? }
-                )
-            end
+        @pages_mutex.synchronize do
+            (@pages.map do |p|
+                next if !p.text?
+                p = p.dup
+
+                %w(links forms cookies jsons xmls).each do |type|
+                    p.send(
+                        "#{type}=",
+                        p.send(type).reject { |e| e.inputs.empty? }
+                    )
+                end
 
-            if !(p.forms.any? || p.links.any? || p.cookies.any? || p.jsons.any? ||
-                p.xmls.any?)
-                next
-            end
+                if !(p.forms.any? || p.links.any? || p.cookies.any? || p.jsons.any? ||
+                    p.xmls.any?)
+                    next
+                end
 
-            p
-        end).compact
+                p
+            end).compact
+        end
     end
 
     # Called by the proxy for each response.
@@ -387,7 +397,9 @@ class Arachni::Plugins::Proxy < Arachni::Plugin::Base
         print_info " *  #{page.jsons.size} JSON"
         print_info " *  #{page.xmls.size} XML"
 
-        @pages << page.dup
+        @pages_mutex.synchronize do
+            @pages << page.dup
+        end
     end
 
     def update_forms( page, request, response )
@@ -531,7 +543,8 @@ others' sessions.
                 ),
                 Options::Address.new( :bind_address,
                     description: 'IP address to bind to.',
-                    default:     '0.0.0.0'
+                    # Don't use 0.0.0.0, it breaks SSL interception on MS Windows.
+                    default:     '127.0.0.1'
                 ),
                 Options::Bool.new( :ignore_responses,
                     description: 'Forces the proxy to only extract vector '+

data/components/plugins/rate_limiter.rb

@@ -0,0 +1,80 @@
+=begin
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
+
+    This file is part of the Arachni Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Arachni Framework
+    web site for more information on licensing and terms of use.
+=end
+
+# Rate limiter for HTTP requests
+#
+# @author Bert Hekman <bert@pbwebmedia.com>
+class Arachni::Plugins::RateLimiter < Arachni::Plugin::Base
+
+    @@times_slept      = 0
+    @@total_time_slept = 0.0
+
+    is_distributable
+
+    def prepare
+        http = framework.http
+
+        last_count             = 0
+        last_response_time_sum = 0.0
+
+        http.on_queue do
+            last_count             = 0
+            last_response_time_sum = 0.0
+        end
+
+        http.on_complete do
+            # Sleep once per burst.
+            next if http.burst_response_count == 0 ||
+                http.burst_response_count % http.max_concurrency != 0
+
+            burst_response_count = http.burst_response_count - last_count
+            burst_response_time  = http.burst_response_time_sum - last_response_time_sum
+
+            last_count             = http.burst_response_count
+            last_response_time_sum = http.burst_response_time_sum
+
+            sleep_time = (
+                1.0 / options[:requests_per_second] * burst_response_count
+            ) - burst_response_time
+
+            next if sleep_time <= 0
+
+            print_info "Sleeping for #{sleep_time.round( 3 )} seconds"
+            sleep sleep_time
+
+            @@times_slept      += 1
+            @@total_time_slept += sleep_time
+        end
+    end
+
+    def self.info
+        {
+            name:        'RateLimiter',
+            description: %q{
+Rate limits HTTP requests
+},
+            author:      'Bert Hekman <bert@pbwebmedia.com>',
+            tags:        %w(http rate limit),
+            version:     '0.1',
+            options: [
+                Options::Int.new( :requests_per_second,
+                     description: 'Requests per second.',
+                     default: 20
+                )
+            ]
+        }
+    end
+
+    def self.times_slept
+        @@times_slept
+    end
+
+    def self.total_time_slept
+        @@total_time_slept
+    end
+end

data/components/plugins/restrict_to_dom_state.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/script.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/uncommon_headers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/vector_collector.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/vector_feed.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/components/plugins/waf_detector.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -55,11 +55,11 @@ class Arachni::Plugins::WAFDetector < Arachni::Plugin::Base
             vanilla:  nil,
             spicy:    nil
         }
-    end
 
-    def run
         framework_pause
+    end
 
+    def run
         print_status "Starting detection with a precision of #{@precision}."
 
         print_status 'Stage #1: Requesting original page.'