arachni 1.4 → 1.6.0

data/spec/support/shared/element/capabilities/auditable.rb

@@ -218,42 +218,12 @@ shared_examples_for 'auditable' do
     end
 
     describe '#audit' do
-        context 'when no block is given' do
-            it 'raises ArgumentError' do
-                expect { auditable.audit( 'stuff' ) }.to raise_error ArgumentError
-            end
-        end
-
-        context ':submit' do
-            it 'forwards :raw_parameters',
-               if: !described_class.ancestors.include?( Arachni::Element::DOM ) do
-
-                param           = auditable.inputs.keys.first
-                raw_parameters  = nil
-
-                auditable.audit(
-                    'stuff',
-                    format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
-                    submit: {
-                        raw_parameters: [ param ]
-                    },
-                    skip_original: true
-                ) do |response, _|
-                    raw_parameters = response.request.raw_parameters
-                end
-
-                run
-
-                expect(raw_parameters).to eq [param]
-            end
-        end
-
         context 'when the response is out of scope' do
             it 'ignores it' do
                 called = nil
 
                 allow_any_instance_of(Arachni::HTTP::Response::Scope).to receive(:out?).and_return(true)
-                allow_any_instance_of(Arachni::Page::Scope).to receive(:out?).and_return(true)
+                # allow_any_instance_of(Arachni::Page::Scope).to receive(:out?).and_return(true)
 
                 auditable.audit( 'stuff',
                                  format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
@@ -267,11 +237,11 @@ shared_examples_for 'auditable' do
             end
 
             context 'but the host includes the seed' do
-                it 'does not log the issue' do
+                it 'is considered in scope' do
                     called = nil
 
                     allow_any_instance_of(Arachni::HTTP::Response::Scope).to receive(:out?).and_return(true)
-                    allow_any_instance_of(Arachni::Page::Scope).to receive(:out?).and_return(true)
+                    # allow_any_instance_of(Arachni::Page::Scope).to receive(:out?).and_return(true)
                     allow_any_instance_of(Arachni::URI).to receive(:seed_in_host?).and_return(true)
 
                     auditable.audit( 'stuff',
@@ -472,6 +442,28 @@ shared_examples_for 'auditable' do
 
                     expect(called).to be_truthy
                 end
+
+                it 'forwards :raw_parameters',
+                   if: !described_class.ancestors.include?( Arachni::Element::DOM ) do
+
+                    param           = auditable.inputs.keys.first
+                    raw_parameters  = nil
+
+                    auditable.audit(
+                        'stuff',
+                        format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
+                        submit: {
+                            raw_parameters: [ param ]
+                        },
+                        skip_original: true
+                    ) do |response, _|
+                        raw_parameters = response.request.raw_parameters
+                    end
+
+                    run
+
+                    expect(raw_parameters).to eq [param]
+                end
             end
 
             describe ':each_mutation' do
@@ -626,7 +618,9 @@ shared_examples_for 'auditable' do
                 describe 'Arachni::Check::Auditor::Format::NULL' do
                     it 'terminates the seed with a null character',
                        if: described_class != Arachni::Element::Header &&
-                               described_class.is_a?( Arachni::Element::DOM ) do
+                            described_class != Arachni::Element::XML &&
+                               !described_class.ancestors.include?( Arachni::Element::DOM ) do
+
                         skip if !has_parameter_extractor?
 
                         injected = nil

data/spec/support/shared/element/capabilities/inputtable.rb

@@ -415,6 +415,32 @@ shared_examples_for 'inputtable' do |options = {}|
         end
     end
 
+    describe '#updated?' do
+        context 'when the inputs have been updated' do
+            it 'returns true' do
+                [
+                    { valid_key => 'val12345' },
+                    { valid_key => 'val2456' }
+                ].each do |updates|
+                    d = subject.dup
+                    d.update( updates )
+                    expect(d).to be_updated
+                end
+            end
+        end
+
+        context 'when the inputs have not been updated' do
+            it 'returns false' do
+                d = subject.dup
+
+                expect(d).to_not be_updated
+
+                d.update( subject.inputs )
+                expect(d).to_not be_updated
+            end
+        end
+    end
+
     describe '#changes' do
         it 'returns the changes the inputs have sustained' do
             [

data/spec/support/shared/element/capabilities/with_node.rb

@@ -8,8 +8,8 @@ shared_examples_for 'with_node' do |html|
     describe '#node' do
         it 'returns the set node' do
             node = with_node.node
-            expect(node.is_a?( Nokogiri::XML::Element )).to be_truthy
-            expect(node.to_s).to eq(Nokogiri::HTML.fragment( with_node.source.dup ).to_s)
+            expect(node.is_a?( Arachni::Parser::Nodes::Element )).to be_truthy
+            expect(node.to_s).to eq(Arachni::Parser.parse_fragment( with_node.source ).to_s)
         end
     end
 

data/spec/support/shared/element/dom/submittable.rb

@@ -12,7 +12,7 @@ shared_examples_for 'submittable_dom' do
                 called = true
             end
 
-            subject.auditor.browser_cluster.wait
+            auditor.browser_cluster.wait
             expect(called).to be_truthy
         end
 
@@ -23,7 +23,7 @@ shared_examples_for 'submittable_dom' do
                 called = true
             end
 
-            subject.auditor.browser_cluster.wait
+            auditor.browser_cluster.wait
             expect(called).to be_truthy
         end
 
@@ -34,7 +34,7 @@ shared_examples_for 'submittable_dom' do
                 called = true
             end
 
-            subject.auditor.browser_cluster.wait
+            auditor.browser_cluster.wait
             expect(called).to be_truthy
         end
 
@@ -45,13 +45,13 @@ shared_examples_for 'submittable_dom' do
                 browser.load subject.page
                 transitions = subject.trigger
             end
-            subject.auditor.browser_cluster.wait
+            auditor.browser_cluster.wait
 
             submitted_page = nil
             subject.dup.submit do |page|
                 submitted_page = page
             end
-            subject.auditor.browser_cluster.wait
+            auditor.browser_cluster.wait
 
             transitions.each do |transition|
                 expect(subject.page.dom.transitions).not_to include transition
@@ -61,13 +61,13 @@ shared_examples_for 'submittable_dom' do
 
         context 'when the element could not be submitted' do
             it 'does not call the block' do
-                allow(subject).to receive( :trigger ) { [nil] }
+                allow_any_instance_of(subject.class).to receive( :trigger ) { [nil] }
 
                 called = false
                 subject.submit do
                     called = true
                 end
-                subject.auditor.browser_cluster.wait
+                auditor.browser_cluster.wait
                 expect(called).to be_falsey
             end
         end
@@ -79,11 +79,11 @@ shared_examples_for 'submittable_dom' do
                     title = 'Injected title'
 
                     subject.submit custom_code: "document.title = #{title.inspect}" do |page|
-                        expect(page.document.css('title').text).to eq(title)
+                        expect(Nokogiri::HTML(page.body).css('title').text).to eq(title)
                         called = true
                     end
 
-                    subject.auditor.browser_cluster.wait
+                    auditor.browser_cluster.wait
                     expect(called).to be_truthy
                 end
             end
@@ -97,7 +97,7 @@ shared_examples_for 'submittable_dom' do
                         set_taint = page.performer.browser.javascript.taint
                     end
 
-                    subject.auditor.browser_cluster.wait
+                    auditor.browser_cluster.wait
                     expect(set_taint).to eq(taint)
                 end
             end

data/spec/support/shared/path_extractor.rb

@@ -24,8 +24,16 @@ shared_examples_for "path_extractor" do
         end
     end
 
-    def doc
-        Nokogiri::HTML( text )
+    def parser
+        Arachni::Parser.new(
+            Arachni::HTTP::Response.new(
+                url:     'http://localhost',
+                body:    text,
+                headers: {
+                    'Content-Type' => 'text/html'
+                }
+            )
+        )
     end
 
     def actual_results
@@ -33,15 +41,19 @@ shared_examples_for "path_extractor" do
     end
 
     def results_for( name )
-        paths = extractors[name].new( document: doc, html: text ).run || []
+        paths = extractors[name].new( parser: parser, html: text ).run || []
         paths.delete( 'http://www.w3.org/TR/REC-html40/loose.dtd' )
         paths.compact.flatten
     end
 
-    module Arachni::Parser::Extractors;end
+    module Arachni::Parser::Extractors
+    end
     def extractors
         @path_extractors ||=
-            ::Arachni::Component::Manager.new( options.paths.path_extractors, Arachni::Parser::Extractors )
+            ::Arachni::Component::Manager.new(
+                options.paths.path_extractors,
+                Arachni::Parser::Extractors
+            )
     end
 
 end

data/ui/cli/framework/option_parser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -35,6 +35,18 @@ class OptionParser < UI::CLI::OptionParser
         end
     end
 
+    def daemon_friendly
+        on( '--daemon-friendly',
+            'Enable this option when running the process in the background.'
+        ) do |b|
+            @daemon_friendly = b
+        end
+    end
+
+    def daemon_friendly?
+        !!@daemon_friendly
+    end
+
     def output
         separator ''
         separator 'Output'
@@ -152,18 +164,32 @@ class OptionParser < UI::CLI::OptionParser
                 substitution
         end
 
+        on( '--scope-https-only', 'Forces the system to only follow HTTPS URLs.',
+            "(Default: #{!!options.scope.https_only})"
+        ) do
+            options.scope.https_only = true
+        end
+
         on( '--scope-dom-depth-limit LIMIT', Integer,
-               'How deep to go into the DOM tree of each page, for pages with JavaScript code.',
-               "(Default: #{options.scope.dom_depth_limit})",
-               "(Setting it to '0' will disable browser analysis.)"
+            'How deep to go into the DOM tree of each page, for pages with JavaScript code.',
+            "(Default: #{options.scope.dom_depth_limit})",
+            "(Setting it to '0' will disable browser analysis.)"
         ) do |limit|
             options.scope.dom_depth_limit = limit
         end
 
-        on( '--scope-https-only', 'Forces the system to only follow HTTPS URLs.',
-            "(Default: #{!!options.scope.https_only})"
-        ) do
-            options.scope.https_only = true
+        on( '--scope-dom-event-limit LIMIT', Integer,
+            'How many DOM events to trigger for each DOM depth, for pages with JavaScript code.',
+            "(Default: #{options.scope.dom_event_limit.nil? ? 'inf' : options.scope.dom_event_limit })",
+        ) do |limit|
+            options.scope.dom_event_limit = limit
+        end
+
+        on( '--scope-dom-event-inheritance-limit LIMIT', Integer,
+            'How many elements should inherit the DOM events of their parents.',
+            "(Default: #{options.scope.dom_event_inheritance_limit.nil? ? 'inf' : options.scope.dom_event_inheritance_limit })",
+        ) do |limit|
+            options.scope.dom_event_inheritance_limit = limit
         end
     end
 
@@ -183,6 +209,10 @@ class OptionParser < UI::CLI::OptionParser
             options.audit.cookies = true
         end
 
+        on( '--audit-nested-cookies', 'Audit nested cookies.' ) do
+            options.audit.nested_cookies = true
+        end
+
         on( '--audit-cookies-extensively',
                'Submit all links and forms of the page along with the cookie permutations.',
                '(*WARNING*: This will severely increase the scan-time.)'
@@ -214,7 +244,7 @@ class OptionParser < UI::CLI::OptionParser
             options.audit.xmls = true
         end
 
-        on( '--audit-ui-inputs', 'Audit orphan Input elements with events.' ) do
+        on( '--audit-ui-inputs', 'Audit orphan <input> elements with events.' ) do
             options.audit.ui_inputs = true
         end
 
@@ -249,7 +279,7 @@ class OptionParser < UI::CLI::OptionParser
         end
 
         on( '--audit-exclude-vector PATTERN', Regexp,
-               'Exclude input vectorS whose name matches PATTERN.',
+               'Exclude input vectors whose name matches PATTERN.',
                '(Can be used multiple times.)' ) do |name|
             options.audit.exclude_vector_patterns << name
         end
@@ -326,7 +356,8 @@ class OptionParser < UI::CLI::OptionParser
         end
 
         on( '--http-cookie-string COOKIE',
-               "Cookie representation as an 'Cookie' HTTP request header."
+               "Cookie representation as a 'Set-Cookie' HTTP response header.",
+               'Example: my_cookie=my_value; Path=/, other_cookie=other_value; Path=/test'
         ) do |cookie|
             options.http.cookie_string = cookie
         end
@@ -341,6 +372,11 @@ class OptionParser < UI::CLI::OptionParser
             options.http.authentication_password = password
         end
 
+        on( "--http-authentication-type #{OptionGroups::HTTP::AUTHENTICATION_TYPES.join(',')}",
+            'HTTP authentication type.', '(Default: auto)' ) do |type|
+            options.http.authentication_type = type
+        end
+
         on( '--http-proxy ADDRESS:PORT', 'Proxy to use.' ) do |url|
             options.http.proxy = url
             options.http.proxy_host, options.http.proxy_port = url.split( ':', 2 )
@@ -548,6 +584,12 @@ class OptionParser < UI::CLI::OptionParser
             options.browser_cluster.local_storage = ::JSON.load( IO.read( file ) )
         end
 
+        on( '--browser-cluster-session-storage FILE',
+            "Sets the browsers' session storage using the JSON data in FILE."
+        ) do |file|
+            options.browser_cluster.session_storage = ::JSON.load( IO.read( file ) )
+        end
+
         on( '--browser-cluster-wait-for-element PATTERN:CSS',
             'Wait for element matching CSS to appear when visiting a page whose' <<
             ' URL matches the PATTERN.'
@@ -557,6 +599,13 @@ class OptionParser < UI::CLI::OptionParser
                 css
         end
 
+        on( '--browser-cluster-wait-for-timers',
+            'Wait for the maximum setTimeout() on each page.',
+            "(Default: #{options.browser_cluster.wait_for_timers})"
+        ) do
+            options.browser_cluster.wait_for_timers = true
+        end
+
         on( '--browser-cluster-pool-size SIZE', Integer,
             'Amount of browser workers to keep in the pool and put to work.',
             "(Default: #{options.browser_cluster.pool_size})"
@@ -604,8 +653,7 @@ class OptionParser < UI::CLI::OptionParser
         on( '--profile-save-filepath FILEPATH', String,
                'Save the current configuration profile/options to FILEPATH.'
         ) do |filepath|
-            save_profile( filepath )
-            exit 0
+            @save_profile_path = filepath
         end
 
         on( '--profile-load-filepath FILEPATH', String,
@@ -669,6 +717,11 @@ class OptionParser < UI::CLI::OptionParser
     end
 
     def after_parse
+        if @save_profile_path
+            save_profile( @save_profile_path )
+            exit 0
+        end
+
         options.url = ARGV.shift
     rescue Options::Error::InvalidURL => e
         print_bad e
@@ -680,9 +733,21 @@ class OptionParser < UI::CLI::OptionParser
         validate_report_path
         validate_snapshot_save_path
         validate_session
+        validate_browser
         validate_url
     end
 
+    def validate_browser
+        if options.browser_cluster.pool_size > 0 &&
+          options.scope.dom_depth_limit > 0 &&
+          !Browser.has_executable?
+
+            print_bad "Missing chromedriver but configured for a DOM scan."
+            print_info "Please install a compatible chromedriver and Chrome version."
+            exit 1
+        end
+    end
+
     def validate_url
         return if options.url
 

data/ui/cli/framework.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -39,6 +39,23 @@ class Framework
         @show_command_screen = nil
         @cleanup_handler     = nil
 
+        if Signal.list.include?( 'USR1' )
+            # Step into a pry session for debugging.
+            trap( 'USR1' ) do
+                Thread.new do
+                    require 'pry'
+
+                    mute
+                    clear_screen
+
+                    pry
+
+                    clear_screen
+                    unmute
+                end
+            end
+        end
+
         trap( 'INT' ) do
             hide_command_screen
             clear_screen
@@ -54,8 +71,8 @@ class Framework
     def run
         print_status 'Initializing...'
 
-        # Won't work properly on MS Windows.
-        get_user_command if !Arachni.windows?
+        # Won't work properly on MS Windows or when running in background.
+        get_user_command if !Arachni.windows? && !@daemon_friendly
 
         begin
             # We may need to kill the audit so put it in a thread.
@@ -121,7 +138,7 @@ class Framework
         browser_cluster = statistics[:browser_cluster]
 
         refresh_line nil, unmute
-        refresh_info( "Audited #{statistics[:audited_pages]} pages.", unmute )
+        refresh_info( "Audited #{statistics[:audited_pages]} page snapshots.", unmute )
 
         if @framework.options.scope.page_limit
             refresh_info( 'Audit limited to a max of ' <<
@@ -241,7 +258,7 @@ class Framework
 
             # Only accept the empty/toggle-screen command when the command
             # screen is not shown.
-            return if !command_screen_shown? && !command.empty?
+            next if !command_screen_shown? && !command.empty?
 
             case command
 
@@ -389,6 +406,7 @@ class Framework
     def parse_options
         parser = OptionParser.new
 
+        parser.daemon_friendly
         parser.authorized_by
         parser.output
         parser.scope
@@ -410,6 +428,8 @@ class Framework
         @timeout         = parser.get_timeout
         @timeout_suspend = parser.timeout_suspend?
 
+        @daemon_friendly = parser.daemon_friendly?
+
         if options.checks.any?
             begin
                 options.checks = @framework.checks.load( options.checks )
@@ -461,14 +481,15 @@ class Framework
             !options.audit.cookies? && !options.audit.headers? &&
             !options.audit.link_templates? && !options.audit.jsons? &&
             !options.audit.xmls? && !options.audit.ui_inputs? &&
-            !options.audit.ui_forms?
+            !options.audit.ui_forms? && !options.audit.nested_cookies?
 
             print_info 'No element audit options were specified, will audit ' <<
-                           'links, forms, cookies, UI inputs, UI forms, JSONs and XMLs.'
+                           'links, forms, cookies, nested_cookies, UI inputs,' <<
+                           ' UI forms, JSONs and XMLs.'
             print_line
 
             options.audit.elements :links, :forms, :cookies, :ui_inputs,
-                                   :ui_forms, :jsons, :xmls
+                                   :ui_forms, :jsons, :xmls, :nested_cookies
         end
     end
 

data/ui/cli/option_parser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/ui/cli/output.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -99,7 +99,7 @@ module Output
     # @param    [String]    str
     def print_error( str = '' )
         print_color( "[-] #{caller_location}", 31, str, $stderr )
-        log_error( "#{caller_location}#{str}" )
+        log_error( "#{caller_location} #{str}" )
     end
 
     # Prints the backtrace of an exception as error messages.
@@ -204,7 +204,14 @@ module Output
     def print_debug( str = '', level = 1 )
         return if !debug?( level )
 
-        print_color( "[#{'!' * level}] #{caller_location}", 36, str, $stderr )
+        # Let's keep track of how much time went by from the last debug call
+        # of the same level.
+        @level_time ||= {}
+        diff = @level_time[level] ? Time.now - @level_time[level] : 0.0
+        diff = diff.round(1)
+        @level_time[level] = Time.now
+
+        print_color( "[#{@level_time[level]} - #{diff}] [#{'!' * level}] #{caller_location}", 36, str, $stderr )
     end
 
     def caller_location

data/ui/cli/reporter/option_parser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/ui/cli/reporter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework