arachni 1.3.2 → 1.4

data/spec/support/servers/checks/passive/backup_files.rb

@@ -7,6 +7,8 @@ get '/' do
     HTML
 end
 
+get( '/some_filename.php' ){}
+
 current_check.formats.each do |format|
     path = format.gsub( '[name]', 'some_filename' ).gsub( '[extension]', 'php' )
     get '/' + path do

data/spec/support/servers/checks/passive/grep/emails.rb

@@ -2,12 +2,12 @@ require 'sinatra'
 require 'sinatra/contrib'
 
 ADDRESSES = [
-    'tasos@blah.com',
-    'john@foo.blah.com',
-    'john32.21d@foo.blah.com',
-    'a.little.more.unusual@dept.example.com',
-    'a.little.more.unusual[at]dept[dot]example[dot]com',
-    'a.little.more.unusual [at] dept [dot] example [dot] com'
+    'tasos@does.not.exist.com',
+    'tasos@example.com',
+    'john@www.example.com',
+    'john32.21d@example.com',
+    'a.little.more.unusual@example.com',
+    'a.little.more.unusual[at]example[dot]com'
 ]
 
 ADDRESSES.each.with_index  do |address, i|

data/spec/support/shared/check.rb

@@ -26,8 +26,10 @@ shared_examples_for 'check' do
         framework.http.headers['User-Agent'] = 'arachni_user'
 
         options.audit.parameter_names      = true
+        options.audit.with_raw_payloads    = true
         options.audit.with_extra_parameter = true
 
+        framework.checks.clear
         framework.checks.load @name
 
         # Do not deduplicate, the check tests need to see everything.
@@ -61,6 +63,32 @@ shared_examples_for 'check' do
         it 'holds the right elements' do
             expect(current_check.info[:elements].map(&:to_s).sort).to eq(self.class.elements.map(&:to_s).sort)
         end
+
+        context 'when it has references' do
+            it 'they are still available' do
+                if !(current_check.info[:issue] && current_check.info[:issue][:references])
+                    next
+                end
+
+                hydra = Typhoeus::Hydra.new
+
+                current_check.info[:issue][:references].each do |title, url|
+                    r = Typhoeus::Request.new(
+                        url,
+                        followlocation: true,
+                        headers:        {
+                            'User-Agent' => 'Mozilla/5.0 (Windows NT x.y; rv:10.0) Gecko/20100101 Firefox/10.0'
+                        }
+                    )
+                    r.on_complete do |response|
+                        expect(response.code).to eq(200), "#{response.code} -- #{title} => #{url}"
+                    end
+                    hydra.queue r
+                end
+
+                hydra.run
+            end
+        end
     end
 
     def self.easy_test( run_checks = true, &block )

data/spec/support/shared/element/capabilities/auditable.rb

@@ -224,8 +224,71 @@ shared_examples_for 'auditable' do
             end
         end
 
+        context ':submit' do
+            it 'forwards :raw_parameters',
+               if: !described_class.ancestors.include?( Arachni::Element::DOM ) do
+
+                param           = auditable.inputs.keys.first
+                raw_parameters  = nil
+
+                auditable.audit(
+                    'stuff',
+                    format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
+                    submit: {
+                        raw_parameters: [ param ]
+                    },
+                    skip_original: true
+                ) do |response, _|
+                    raw_parameters = response.request.raw_parameters
+                end
+
+                run
+
+                expect(raw_parameters).to eq [param]
+            end
+        end
+
+        context 'when the response is out of scope' do
+            it 'ignores it' do
+                called = nil
+
+                allow_any_instance_of(Arachni::HTTP::Response::Scope).to receive(:out?).and_return(true)
+                allow_any_instance_of(Arachni::Page::Scope).to receive(:out?).and_return(true)
+
+                auditable.audit( 'stuff',
+                                 format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
+                                 skip_original: true
+                ) do |_, element|
+                    called = true
+                end
+
+                run
+                expect(called).to be_falsey
+            end
+
+            context 'but the host includes the seed' do
+                it 'does not log the issue' do
+                    called = nil
+
+                    allow_any_instance_of(Arachni::HTTP::Response::Scope).to receive(:out?).and_return(true)
+                    allow_any_instance_of(Arachni::Page::Scope).to receive(:out?).and_return(true)
+                    allow_any_instance_of(Arachni::URI).to receive(:seed_in_host?).and_return(true)
+
+                    auditable.audit( 'stuff',
+                                     format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
+                                     skip_original: true
+                    ) do |_, element|
+                        called = true
+                    end
+
+                    run
+                    expect(called).to be_truthy
+                end
+            end
+        end
+
         context 'when the payloads is' do
-            context String do
+            context 'String' do
                 it 'injects the given payload' do
                     payload = 'stuff-here'
                     injected = nil
@@ -258,7 +321,7 @@ shared_examples_for 'auditable' do
                     end
                 end
             end
-            context Array do
+            context 'Array' do
                 it 'injects all supplied payload' do
                     payloads = [ 'stuff-here', 'stuff-here-2' ]
                     injected = []
@@ -290,7 +353,7 @@ shared_examples_for 'auditable' do
                 end
             end
 
-            context Hash do
+            context 'Hash' do
                 it 'picks payloads applicable to the resource\'s platforms' do
                     payloads = {
                         linux:   [ 'linux-payload-1', 'linux-payload-2' ],
@@ -396,7 +459,7 @@ shared_examples_for 'auditable' do
         end
 
         context 'when called with option' do
-            describe :submit do
+            describe ':submit' do
                 it 'uses them for the #submit call' do
                     options = { cookies: { stuff: 'blah' }}
 
@@ -411,7 +474,7 @@ shared_examples_for 'auditable' do
                 end
             end
 
-            describe :each_mutation do
+            describe ':each_mutation' do
                 it 'is passed each generated mutation' do
                     skip if !has_parameter_extractor?
 
@@ -482,8 +545,8 @@ shared_examples_for 'auditable' do
                 end
             end
 
-            describe :skip_like do
-                describe Proc do
+            describe ':skip_like' do
+                describe 'Proc' do
                     it 'skips mutations based on the block\'s return value' do
                         audited   = []
                         skip_like = proc { |m| m.affected_input_name != auditable.inputs.keys.first }
@@ -500,7 +563,7 @@ shared_examples_for 'auditable' do
                     end
                 end
 
-                describe Array do
+                describe 'Array' do
                     it 'skips mutations based on the blocks\' return value' do
                         audited   = []
                         skip_like = []
@@ -519,7 +582,7 @@ shared_examples_for 'auditable' do
                 end
             end
 
-            describe :format do
+            describe ':format' do
                 describe 'Arachni::Check::Auditor::Format::STRAIGHT' do
                     it 'injects the seed as is' do
                         skip if !has_parameter_extractor?
@@ -601,7 +664,7 @@ shared_examples_for 'auditable' do
                 end
             end
 
-            describe :redundant do
+            describe ':redundant' do
                 before do
                     @audit_opts = {
                         format: [ Arachni::Check::Auditor::Format::STRAIGHT ],
@@ -609,7 +672,7 @@ shared_examples_for 'auditable' do
                     }
                 end
 
-                context true do
+                context 'true' do
                     it 'allows redundant audits' do
                         cnt = 0
                         5.times do |i|
@@ -620,7 +683,7 @@ shared_examples_for 'auditable' do
                     end
                 end
 
-                context false do
+                context 'false' do
                     it 'does not allow redundant requests/audits' do
                         cnt = 0
                         5.times do |i|
@@ -689,7 +752,7 @@ shared_examples_for 'auditable' do
         end
 
         context "when #{described_class::Scope}#out?" do
-            context true do
+            context 'true' do
                 it 'returns immediately' do
                     allow_any_instance_of(described_class::Scope).to receive(:out?) { true }
 

data/spec/support/shared/element/capabilities/dom_only.rb

@@ -38,21 +38,20 @@ shared_examples_for 'dom_only' do |source|
     describe '#coverage_id' do
         it 'delegates to #dom' do
             allow(subject.dom).to receive(:coverage_id).and_return( 'stuff' )
-            expect(subject.coverage_id).to eq 'stuff'
+            expect(subject.coverage_id).to eq "#{described_class.type}:stuff"
         end
     end
 
     describe '#coverage_hash' do
-        it 'delegates to #dom' do
-            allow(subject.dom).to receive(:coverage_hash).and_return( 123 )
-            expect(subject.coverage_hash).to eq 123
+        it 'hashes #coverage_id' do
+            expect(subject.coverage_hash).to eq subject.coverage_id.persistent_hash
         end
     end
 
     describe '#id' do
         it 'delegates to #dom' do
-            allow(subject.dom).to receive(:id).and_return( '123' )
-            expect(subject.id).to eq '123'
+            allow(subject.dom).to receive(:id).and_return( 'stuff' )
+            expect(subject.id).to eq "#{described_class.type}:stuff"
         end
     end
 

data/spec/support/shared/element/capabilities/inputtable.rb

@@ -52,7 +52,7 @@ shared_examples_for 'inputtable' do |options = {}|
     describe '#to_rpc_data' do
         let(:data) { subject.to_rpc_data }
 
-        %w(inputs default_inputs).each do |attribute|
+        %w(inputs default_inputs raw_inputs).each do |attribute|
             it "includes '#{attribute}'" do
                 expect(data[attribute]).to eq(subject.send( attribute ))
             end
@@ -63,7 +63,7 @@ shared_examples_for 'inputtable' do |options = {}|
         let(:restored) { subject.class.from_rpc_data data }
         let(:data) { Arachni::RPC::Serializer.rpc_data( subject ) }
 
-        %w(inputs default_inputs).each do |attribute|
+        %w(inputs default_inputs raw_inputs).each do |attribute|
             it "restores '#{attribute}'" do
                 expect(restored.send( attribute )).to eq(subject.send( attribute ))
             end
@@ -76,6 +76,7 @@ shared_examples_for 'inputtable' do |options = {}|
 
             k, v = orig.inputs.keys.first, 'value'
 
+            subject.raw_inputs << k
             subject.update( k => v )
 
             expect(subject.inputs).not_to eq(orig.inputs)
@@ -83,6 +84,7 @@ shared_examples_for 'inputtable' do |options = {}|
             subject.reset
 
             expect(subject.inputs).to eq(orig.inputs)
+            expect(subject.raw_inputs).to be_empty
         end
     end
 
@@ -110,7 +112,7 @@ shared_examples_for 'inputtable' do |options = {}|
                 end
             end
 
-            context Array do
+            context 'Array' do
                 context 'when it has the given inputs' do
                     it 'returns true' do
                         expect(subject.has_inputs?( sym_keys )).to be_truthy
@@ -125,7 +127,7 @@ shared_examples_for 'inputtable' do |options = {}|
                 end
             end
 
-            context Hash do
+            context 'Hash' do
                 context 'when it has the given inputs (names and values)' do
                     it 'returns true' do
                         expect(subject.has_inputs?( subject.inputs )).to be_truthy
@@ -173,6 +175,28 @@ shared_examples_for 'inputtable' do |options = {}|
             expect(e.inputtable_id).not_to eq(c.inputtable_id)
         end
 
+        it 'takes into account raw inputs' do
+            e = subject.dup
+            e.inputs = { 1 => 2, 3 => 4 }
+            e.raw_inputs = [1]
+
+            c = subject.dup
+            c.inputs = { 1 => 2, 3 => 4 }
+            c.raw_inputs = [1]
+
+            expect(e.inputtable_id).to eq(c.inputtable_id)
+
+            e = subject.dup
+            e.inputs = { 1 => 2, 3 => 4 }
+            e.raw_inputs = [1]
+
+            c = subject.dup
+            e.inputs = { 1 => 2, 3 => 4 }
+            e.raw_inputs = [2]
+
+            expect(e.inputtable_id).not_to eq(c.inputtable_id)
+        end
+
         it 'takes into account input values' do
             e = subject.dup
             e.inputs = { 1 => 2 }
@@ -202,6 +226,49 @@ shared_examples_for 'inputtable' do |options = {}|
         end
     end
 
+    describe '#raw_inputs=' do
+        it 'converts all inputs to strings' do
+            subject.raw_inputs = [valid_key.to_sym]
+            expect(subject.raw_inputs).to eq [valid_key.to_s]
+        end
+
+        context 'when a name contains invalid data' do
+            it "raises #{Arachni::Element::Capabilities::Inputtable::Error::InvalidData::Name}" do
+                allow(subject).to receive(:valid_input_data?) { |data| data != valid_key }
+
+                expect do
+                    subject.raw_inputs = [ valid_key ]
+                end.to raise_error Arachni::Element::Capabilities::Inputtable::Error::InvalidData::Name
+            end
+        end
+
+        context 'when a name is invalid' do
+            it "raises #{Arachni::Element::Capabilities::Inputtable::Error::InvalidData::Name}" do
+                allow(subject).to receive(:valid_input_name?) { false }
+
+                expect do
+                    subject.raw_inputs = [ valid_key ]
+                end.to raise_error Arachni::Element::Capabilities::Inputtable::Error::InvalidData::Name
+            end
+        end
+    end
+
+    describe '#raw_input?' do
+        context 'if the name is in #raw_inputs' do
+            it 'returns true' do
+                subject.raw_inputs = [valid_key]
+                expect(subject.raw_input?( valid_key )).to be_truthy
+            end
+        end
+
+        context 'if the name is not in #raw_inputs' do
+            it 'returns false' do
+                subject.raw_inputs = []
+                expect(subject.raw_input?( valid_key )).to be_falsey
+            end
+        end
+    end
+
     describe '#inputs=' do
         it 'assigns a hash of auditable inputs' do
             a = subject.dup
@@ -465,9 +532,12 @@ shared_examples_for 'inputtable' do |options = {}|
 
     describe '#to_h' do
         it 'returns a hash representation of self' do
+            subject.raw_inputs = [ subject.inputs.keys.first ]
+
             hash = subject.to_h
             expect(hash[:inputs]).to         eq(subject.inputs)
             expect(hash[:default_inputs]).to eq(subject.default_inputs)
+            expect(hash[:raw_inputs]).to eq(subject.raw_inputs)
         end
     end
 

data/spec/support/shared/element/capabilities/mutable.rb

@@ -85,6 +85,26 @@ shared_examples_for 'mutable' do |options = {}|
         end
     end
 
+    describe '#with_raw_payload?' do
+        let(:mutation) do
+            mutable.mutations( seed ).first
+        end
+
+        context 'when #affected_input_name is in #raw_inputs' do
+            it 'returns true' do
+                mutation.raw_inputs << mutation.affected_input_name
+                expect(mutation).to be_with_raw_payload
+            end
+        end
+
+        context 'when #affected_input_name is not in #raw_inputs' do
+            it 'returns true' do
+                mutation.raw_inputs = []
+                expect(mutation).to_not be_with_raw_payload
+            end
+        end
+    end
+
     describe '#affected_input_value' do
         it 'returns the value of the affected_input_name input' do
             elem = mutable.mutations( seed ).first
@@ -155,15 +175,15 @@ shared_examples_for 'mutable' do |options = {}|
         end
 
         context 'with option' do
-            describe :parameter_values do
-                describe true do
+            describe ':parameter_values' do
+                describe 'true' do
                     it 'injects the payload into parameter values' do
                         expect(mutable.mutations( seed, parameter_values: true ).
                             find { |m| m.affected_input_value.include? seed }).
                             to be_truthy
                     end
                 end
-                describe false do
+                describe 'false' do
                     it 'does not inject the payload into parameter values' do
                         expect(mutable.mutations( seed, parameter_values: false ).
                             find { |m| m.affected_input_value.include? seed }).
@@ -193,7 +213,59 @@ shared_examples_for 'mutable' do |options = {}|
                 end
             end
 
-            describe :with_extra_parameter,
+            describe ':with_raw_payloads',
+                     if: !described_class.ancestors.include?(
+                         Arachni::Element::DOM
+                     ) && described_class != Arachni::Element::JSON &&
+                             described_class != Arachni::Element::XML &&
+                             described_class != Arachni::Element::Header &&
+                             described_class != Arachni::Element::Cookie do
+
+                describe 'true' do
+                    it 'adds an unencoded payload' do
+                        expect(
+                            mutable.mutations( seed, with_raw_payloads: true ).
+                                find(&:with_raw_payload?)
+                        ).to be_truthy
+                    end
+                end
+                describe 'false' do
+                    it 'does not add an unencoded payload' do
+                        expect(mutable.mutations( seed, with_raw_payloads: false ).find do |m|
+                            next if !m.audit_options[:submit]
+
+                            m.audit_options[:submit][:raw_parameters] &&
+                                m.audit_options[:submit][:raw_parameters].include?( m.affected_input_name )
+                        end).to be_falsey
+                    end
+                end
+                describe 'nil' do
+                    it 'does not add an unencoded payload' do
+                        expect(mutable.mutations( seed ).find do |m|
+                            next if !m.audit_options[:submit]
+
+                            m.audit_options[:submit][:raw_parameters] &&
+                                m.audit_options[:submit][:raw_parameters].include?( m.affected_input_name )
+                        end).to be_falsey
+                    end
+                end
+
+                describe "#{Arachni::OptionGroups::Audit}#with_raw_payloads" do
+                    it 'serves as the default value of :with_raw_payloads' do
+                        Arachni::Options.audit.with_raw_payloads = true
+                        expect(
+                            mutable.mutations( seed ).find(&:with_raw_payload?)
+                        ).to be_truthy
+
+                        Arachni::Options.audit.with_raw_payloads = false
+                        expect(
+                            mutable.mutations( seed ).find(&:with_raw_payload?)
+                        ).to be_falsey
+                    end
+                end
+            end
+
+            describe 'with_extra_parameter',
                      if: !described_class.ancestors.include?(
                          Arachni::Element::DOM
                      ) && described_class != Arachni::Element::LinkTemplate &&
@@ -201,13 +273,13 @@ shared_examples_for 'mutable' do |options = {}|
 
                 let(:extra_name) { described_class::EXTRA_NAME }
 
-                describe true do
+                describe 'true' do
                     it 'injects the payload into an extra parameter' do
                         expect(mutable.mutations( seed, with_extra_parameter: true ).
                             find { |m| m[extra_name].to_s.include? seed }).to be_truthy
                     end
                 end
-                describe false do
+                describe 'false' do
                     it 'does not inject the payload into an extra parameter' do
                         expect(mutable.mutations( seed, with_extra_parameter: false ).
                             find { |m| m[extra_name].to_s.include? seed }).to be_falsey
@@ -233,19 +305,19 @@ shared_examples_for 'mutable' do |options = {}|
                 end
             end
 
-            describe :with_both_http_methods,
+            describe 'with_both_http_methods',
                      if: !described_class.ancestors.include?(
                          Arachni::Element::DOM
                      ) && described_class != Arachni::Element::JSON &&
                              described_class != Arachni::Element::XML do
 
-                describe false do
+                describe 'false' do
                     it 'does not fuzz methods' do
                         expect(mutable.mutations( seed, with_both_http_methods: false ).
                             map(&:method).uniq).to eq [mutable.method]
                     end
                 end
-                describe true do
+                describe 'true' do
                     it 'fuzzes methods' do
                         expect(mutable.mutations( seed, with_both_http_methods: true ).
                             map(&:method).uniq).to eq [:get, :post]
@@ -271,19 +343,19 @@ shared_examples_for 'mutable' do |options = {}|
                 end
             end
 
-            describe :parameter_names,
+            describe 'parameter_names',
                      if: !described_class.ancestors.include?( Arachni::Element::DOM) &&
                              described_class != Arachni::Element::LinkTemplate &&
                              described_class != Arachni::Element::XML do
 
-                describe true do
+                describe 'true' do
                     it 'uses the seed as a parameter name' do
                         expect(mutable.mutations( seed, parameter_names: true ).
                             find { |m| m.inputs.keys.include? seed }).
                             to be_truthy
                     end
                 end
-                describe false do
+                describe 'false' do
                     it 'does not use the seed as a parameter name' do
                         allow_any_instance_of(mutable.class).
                             to receive(:valid_input_name_data?) { |instance, name| name != seed }
@@ -319,13 +391,13 @@ shared_examples_for 'mutable' do |options = {}|
                 end
             end
 
-            describe :skip do
+            describe ':skip' do
                 it 'skips mutation of parameters with these names' do
                     mutable.mutations( seed, skip: [ 'input_one' ] )
                 end
             end
 
-            describe :format do
+            describe ':format' do
                 describe 'Format::STRAIGHT' do
                     it 'injects the seed as is' do
                         m = mutable.mutations( seed,