arachni 1.4 → 1.6.0

data/spec/arachni/element/server_spec.rb

@@ -83,6 +83,12 @@ describe Arachni::Element::Server do
                 expect(logged_issue.trusted).to be_truthy
             end
 
+            it 'assigns the extra Issue options' do
+                auditable.log_remote_file_if_exists( @base_url + 'true', false, trusted: false )
+                @framework.http.run
+                expect(Arachni::Data.issues.first).to_not be_trusted
+            end
+
             context 'when one issue is logged' do
                 it "does not push the response to the #{Arachni::Trainer}" do
                     auditable.log_remote_file_if_exists( @base_url + 'true' )
@@ -222,11 +228,22 @@ describe Arachni::Element::Server do
             end
 
             context 'when the response is a redirect' do
-                it 'yields false' do
-                    exists = true
-                    auditable.remote_file_exist?( @base_url + 'redirect' ) { |bool| exists = bool }
-                    @framework.http.run
-                    expect(exists).to be_falsey
+                context 'and the final page is found' do
+                    it 'yields true' do
+                        exists = true
+                        auditable.remote_file_exist?( @base_url + 'redirect' ) { |bool| exists = bool }
+                        @framework.http.run
+                        expect(exists).to be_truthy
+                    end
+                end
+
+                context 'and the final page is not found' do
+                    it 'yields false' do
+                        exists = true
+                        auditable.remote_file_exist?( @base_url + 'redirect/not_found' ) { |bool| exists = bool }
+                        @framework.http.run
+                        expect(exists).to be_falsey
+                    end
                 end
             end
         end

data/spec/arachni/element/ui_form/dom_spec.rb

@@ -20,7 +20,7 @@ describe Arachni::Element::UIForm::DOM do
 
     def auditable_extract_parameters( page )
         {
-            'my-input' => page.document.css('#container').text.strip
+            'my-input' => Nokogiri::HTML(page.body).css('#container').text.strip
         }
     end
 

data/spec/arachni/element/ui_form_spec.rb

@@ -173,7 +173,7 @@ describe Arachni::Element::UIForm do
 
             context 'as <input type="button">' do
                 let(:url) { "#{super()}/input-button" }
-                let(:source) { '<input type="button" id="insert" value="Insert into DOM">' }
+                let(:source) { '<input id="insert" type="button" value="Insert into DOM">' }
 
                 context 'without inputs' do
                     let(:url) { "#{super()}/without-inputs" }
@@ -240,7 +240,7 @@ describe Arachni::Element::UIForm do
 
             context 'as <input type="submit">' do
                 let(:url) { "#{super()}/input-submit" }
-                let(:source) { '<input type="submit" id="insert" value="Insert into DOM">' }
+                let(:source) { '<input id="insert" type="submit" value="Insert into DOM">' }
 
                 context 'without inputs' do
                     let(:url) { "#{super()}/without-inputs" }

data/spec/arachni/element/ui_input/dom_spec.rb

@@ -18,7 +18,7 @@ describe Arachni::Element::UIInput::DOM do
     end
 
     def auditable_extract_parameters( page )
-        { 'my-input' => page.document.css('#container').text.strip }
+        { 'my-input' => Nokogiri::HTML(page.body).css('#container').text.strip }
     end
 
     def element

data/spec/arachni/element/ui_input_spec.rb

@@ -57,7 +57,7 @@ describe Arachni::Element::UIInput do
 
                 context 'with events' do
                     let(:url) { "#{super()}/with_events" }
-                    let(:source) { '<input type="text" id="my-input" value="stuff">' }
+                    let(:source) { '<input id="my-input" type="text" value="stuff">' }
 
                     it 'returns array of elements' do
                         input = described_class.from_browser( @browser, page ).first

data/spec/arachni/element/xml_spec.rb

@@ -13,7 +13,9 @@ describe Arachni::Element::XML do
     it_should_behave_like 'mutable',
                           supports_nulls: false,
                           inputs: described_class.parse_inputs( inputtable_source )
-    it_should_behave_like 'auditable'
+    it_should_behave_like 'auditable', supports_nulls: false
+    it_should_behave_like 'buffered_auditable'
+    it_should_behave_like 'line_buffered_auditable'
 
     before :each do
         @framework ||= Arachni::Framework.new
@@ -143,7 +145,7 @@ EOXML
 
     describe '#to_rpc_data' do
         it "includes 'source'" do
-            expect(subject.to_rpc_data['source']).to eq(source)
+            expect(subject.to_rpc_data['source']).to eq(source.strip)
         end
     end
 
@@ -162,7 +164,7 @@ EOXML
             it 'parses a request into an element' do
                 expect(subject.url).to    eq(url)
                 expect(subject.action).to eq(request.url)
-                expect(subject.source).to eq(request.body)
+                expect(subject.source).to eq(request.body.strip)
                 expect(subject.method).to eq(request.method)
             end
         end

data/spec/arachni/framework/parts/audit_spec.rb

@@ -247,24 +247,12 @@ describe Arachni::Framework::Parts::Audit do
                         f.options.audit.elements :links, :forms, :cookies
                         f.checks.load :signature
                         f.options.scope.dom_depth_limit = 1
-                        expect(f.url_queue_total_size).to eq(0)
-                        expect(f.audit_page( Arachni::Page.from_url( @url + '/with_javascript' ) )).to be_truthy
-                        f.run
-                        expect(f.url_queue_total_size).to eq(3)
-
-                        f.reset
-
-                        f.options.audit.elements :links, :forms, :cookies
-                        f.checks.load :signature
-                        f.options.scope.dom_depth_limit = 1
-                        expect(f.url_queue_total_size).to eq(0)
 
                         page = Arachni::Page.from_url( @url + '/with_javascript' )
                         page.dom.push_transition Arachni::Page::DOM::Transition.new( :page, :load )
+                        page.dom.push_transition Arachni::Page::DOM::Transition.new( :page, :load )
 
-                        expect(f.audit_page( page )).to be_truthy
-                        f.run
-                        expect(f.url_queue_total_size).to eq(1)
+                        expect(f.audit_page( page )).to be_falsey
                     end
                 end
 

data/spec/arachni/framework/parts/data_spec.rb

@@ -26,14 +26,8 @@ describe Arachni::Framework::Parts::Data do
         let(:page) { Arachni::Page.from_url( @url + '/train/true' ) }
 
         it 'pushes it to the page audit queue and returns true' do
-            subject.options.audit.elements :links, :forms, :cookies
-            subject.checks.load :signature
-
             expect(subject.page_queue_total_size).to eq(0)
             expect(subject.push_to_page_queue( page )).to be_truthy
-            subject.run
-
-            expect(subject.report.issues.size).to eq(1)
             expect(subject.page_queue_total_size).to be > 0
         end
 

data/spec/arachni/http/client/dynamic_404_handlers_spec.rb

@@ -14,6 +14,38 @@ describe Arachni::HTTP::Client::Dynamic404Handler do
     let(:url) { "#{@url}/" }
 
     describe '#_404?' do
+        context 'when not dealing with a redirect' do
+            context 'to an outside custom 404' do
+                it 'returns true' do
+                    @dynamic_404_handler_redirect_1 =
+                        web_server_url_for( :dynamic_404_handler_redirect_1 )
+
+                    @dynamic_404_handler_redirect_2 =
+                        web_server_url_for( :dynamic_404_handler_redirect_2 )
+
+                    Arachni::HTTP::Client.get(
+                        "#{@dynamic_404_handler_redirect_1}/set-redirect",
+                        parameters: {
+                            url: @dynamic_404_handler_redirect_2
+                        },
+                        mode: :sync
+                    )
+
+                    response = client.get(
+                        @dynamic_404_handler_redirect_1 + '/test/stuff.php',
+                        follow_location: true,
+                        mode:            :sync
+                    )
+
+                    bool = false
+                    subject._404?( response ) { |c_bool| bool = c_bool }
+                    client.run
+
+                    expect(bool).to be_true
+                end
+            end
+        end
+
         context 'when not dealing with a not-found response' do
             it 'returns false' do
                 res = nil
@@ -120,6 +152,100 @@ describe Arachni::HTTP::Client::Dynamic404Handler do
                     it 'returns true'
                 end
             end
+
+            context 'which ignores anything past the resource name' do
+                context 'with a non existent resource' do
+                    it 'returns true' do
+                        res = nil
+                        client.get( url + '/ignore-after-filename/123dd/' ) { |c_res| res = c_res }
+                        client.run
+
+                        bool = nil
+                        subject._404?( res ) { |c_bool| bool = c_bool }
+                        client.run
+
+                        expect(bool).to be_truthy
+                    end
+                end
+            end
+
+            context 'which ignores anything ahead of the resource name' do
+                context 'with a non existent resource' do
+                    it 'returns true' do
+                        res = nil
+                        client.get( url + '/ignore-before-filename/fff123/' ) { |c_res| res = c_res }
+                        client.run
+
+                        bool = nil
+                        subject._404?( res ) { |c_bool| bool = c_bool }
+                        client.run
+
+                        expect(bool).to be_truthy
+                    end
+                end
+            end
+
+            context 'when checking for a resource with a name that routes based on dash' do
+                context 'and the handler is pre-dash sensitive' do
+                    context 'and is found' do
+                        it 'returns false' do
+                            res = nil
+                            client.get( url + 'advanced/sensitive-dash/pre/blah-html' ) { |c_res| res = c_res }
+                            client.run
+
+                            bool = nil
+                            subject._404?( res ) { |c_bool| bool = c_bool }
+                            client.run
+
+                            expect(bool).to be_falsey
+                        end
+                    end
+
+                    context 'and is not found' do
+                        it 'returns true' do
+                            res = nil
+                            client.get( url + 'advanced/sensitive-dash/pre/blah2-html' ) { |c_res| res = c_res }
+                            client.run
+
+                            bool = nil
+                            subject._404?( res ) { |c_bool| bool = c_bool }
+                            client.run
+
+                            expect(bool).to be_truthy
+                        end
+                    end
+                end
+
+                context 'and the handler is post-dash sensitive' do
+                    context 'and is found' do
+                        it 'returns false' do
+                            res = nil
+                            client.get( url + 'advanced/sensitive-dash/post/blah-html' ) { |c_res| res = c_res }
+                            client.run
+
+                            bool = nil
+                            subject._404?( res ) { |c_bool| bool = c_bool }
+                            client.run
+
+                            expect(bool).to be_falsey
+                        end
+                    end
+
+                    context 'and is not found' do
+                        it 'returns true' do
+                            res = nil
+                            client.get( url + 'advanced/sensitive-dash/post/blah-html2' ) { |c_res| res = c_res }
+                            client.run
+
+                            bool = nil
+                            subject._404?( res ) { |c_bool| bool = c_bool }
+                            client.run
+
+                            expect(bool).to be_truthy
+                        end
+                    end
+                end
+            end
         end
 
         context 'when checking for an already checked URL' do

data/spec/arachni/http/client_spec.rb

@@ -137,19 +137,10 @@ describe Arachni::HTTP::Client do
 
     describe Arachni::OptionGroups::HTTP do
         describe '#request_concurrency' do
-            context 'Integer' do
-                it 'uses it as a max_concurrency' do
-                    @opts.http.request_concurrency = 34
-                    subject.reset
-                    expect(subject.max_concurrency).to eq(34)
-                end
-            end
-            context 'nil' do
-                it 'uses a default max concurrency setting' do
-                    @opts.http.request_concurrency = nil
-                    subject.reset
-                    expect(subject.max_concurrency).to eq(Arachni::HTTP::Client::MAX_CONCURRENCY)
-                end
+            it 'uses it as a max_concurrency' do
+                @opts.http.request_concurrency = 34
+                subject.reset
+                expect(subject.max_concurrency).to eq(34)
             end
         end
 
@@ -329,7 +320,7 @@ describe Arachni::HTTP::Client do
         it 'provides access to default headers' do
             headers = subject.headers
             expect(headers['Accept']).to eq('text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
-            expect(headers['User-Agent']).to eq('Arachni/v' + Arachni::VERSION)
+            expect(headers['User-Agent']).to include 'Arachni/v' + Arachni::VERSION
         end
 
         context "when #{Arachni::OptionGroups::HTTP}#request_headers is set" do
@@ -398,13 +389,14 @@ describe Arachni::HTTP::Client do
 
         context "when #{Arachni::OptionGroups::HTTP}#cookie_string is set" do
             it 'parses the string and add those cookies to the CookieJar' do
-                @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2; stuff=%25blah; another_name=another_val'
+                @opts.http.cookie_string = 'my_cookie_name=val1;path=/my/path,blah_name=val2, stuff=%25blah, another_name=another_val'
                 expect(subject.cookie_jar.cookies).to be_empty
                 subject.reset
                 cookies = subject.cookie_jar.cookies
                 expect(cookies.size).to eq(4)
                 expect(cookies.first.name).to eq('my_cookie_name')
                 expect(cookies.first.value).to eq('val1')
+                expect(cookies.first.path).to eq('/my/path')
                 expect(cookies[1].name).to eq('blah_name')
                 expect(cookies[1].value).to eq('val2')
                 expect(cookies[2].name).to eq('stuff')
@@ -417,7 +409,7 @@ describe Arachni::HTTP::Client do
 
     describe '#cookies' do
         it 'returns the current cookies' do
-            @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2; another_name=another_val'
+            @opts.http.cookie_string = 'my_cookie_name=val1,blah_name=val2, another_name=another_val'
             expect(subject.cookie_jar.cookies).to be_empty
             subject.reset
             expect(subject.cookies.size).to eq(3)
@@ -567,18 +559,15 @@ describe Arachni::HTTP::Client do
 
     describe '#original_max_concurrency' do
         it 'returns the original max concurrency' do
-            expect(subject.original_max_concurrency).to eq(20)
+            expect(subject.original_max_concurrency).to eq(10)
             expect(subject.original_max_concurrency).to eq(subject.max_concurrency)
 
-            subject.max_concurrency = 10
-            expect(subject.original_max_concurrency).to eq(20)
+            subject.max_concurrency = 5
+            expect(subject.original_max_concurrency).to eq(10)
         end
     end
 
     describe '#max_concurrency' do
-        it 'defaults to 20' do
-            expect(subject.max_concurrency).to eq(20)
-        end
         it 'respects the http_request_concurrency option' do
             @opts.http.request_concurrency = 50
             subject.reset
@@ -611,10 +600,11 @@ describe Arachni::HTTP::Client do
         it "fills in #{Arachni::HTTP::Request}#headers_string" do
             host = "#{Arachni::URI(@url).host}:#{Arachni::URI(@url).port}"
             expect(subject.request( @url, mode: :sync ).request.headers_string).to eq(
-                "GET / HTTP/1.1\r\nHost: #{host}\r\nAccept-Encoding: gzip, " +
-                    "deflate\r\nUser-Agent: Arachni/v#{Arachni::VERSION}\r\nAccept: text/html," +
+                "GET / HTTP/1.1\r\nHost: #{host}\r\nAuthorization: Basic Og==\r\nAccept-Encoding: gzip, " +
+                    "deflate\r\nUser-Agent: #{Arachni::Options.http.user_agent}\r\nAccept: text/html," +
                     "application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" +
-                    "Accept-Language: en-US,en;q=0.8,he;q=0.6\r\n\r\n"
+                    "Accept-Language: en-US,en;q=0.8,he;q=0.6\r\n" +
+                    "X-Arachni-Scan-Seed: #{Arachni::Utilities.random_seed}\r\n\r\n"
             )
         end
 
@@ -629,6 +619,73 @@ describe Arachni::HTTP::Client do
             ).request.effective_body).to eq("1=%202&%203=4")
         end
 
+        describe ':on_headers' do
+            it 'gets called when headers are available' do
+                h = nil
+
+                subject.request(
+                    "#{@url}/fast_stream",
+                    mode: :sync,
+                    on_headers: proc do |response|
+                        h = response.to_h
+                    end
+                )
+
+                expect(h[:code]).to eq 200
+                expect(h[:body]).to eq ''
+                expect(h[:headers]).to be_any
+            end
+        end
+
+        describe ':on_body' do
+            it 'gets called with body chunks' do
+                chunks = []
+
+                subject.request(
+                    "#{@url}/fast_stream",
+                    mode:    :sync,
+                    on_body: proc do |chunk|
+                        chunks << chunk
+                    end
+                )
+
+                expect(chunks.size).to be == 5
+            end
+        end
+
+        describe ':on_body_line' do
+            it 'gets called with body lines' do
+                lines = []
+
+                subject.request(
+                    "#{@url}/fast_stream",
+                    mode:         :sync,
+                    on_body_line: proc do |line|
+                        lines << line
+                    end
+                )
+
+                expect(lines.size).to be == 5
+            end
+        end
+
+        describe ':on_body_lines' do
+            it 'gets called with chunks of body lines' do
+                lines = []
+
+                subject.request(
+                    "#{@url}/lines/non-stream",
+                    mode:         :sync,
+                    on_body_lines: proc do |line|
+                        lines << line
+                    end
+                )
+
+                expect(lines.size).to be > 1
+                expect(lines.size).to be < 500
+            end
+        end
+
         describe ':fingerprint' do
             before do
                 Arachni::Platform::Manager.clear
@@ -734,21 +791,21 @@ describe Arachni::HTTP::Client do
                                          response_max_size: 0
                         )
 
-                        expect(r.headers).not_to include 'Content-Type'
+                        expect(r.headers['Content-Type']).to be_empty
                         expect(r.body).to be_empty
 
                         r = subject.request( @url + '/http_response_max_size/without_content_length',
                                          mode: :sync,
                                          response_max_size: 1
                         )
-                        expect(r.headers).not_to include 'Content-Type'
+                        expect(r.headers['Content-Type']).to be_empty
                         expect(r.body).to be_empty
 
                         r = subject.request( @url + '/http_response_max_size/without_content_length',
                                          mode: :sync,
                                          response_max_size: 999999
                         )
-                        expect(r.headers).not_to include 'Content-Type'
+                        expect(r.headers['Content-Type']).to be_empty
                         expect(r.body).to be_empty
 
                         r = subject.request( @url + '/http_response_max_size/without_content_length',
@@ -756,7 +813,7 @@ describe Arachni::HTTP::Client do
                                          response_max_size: 1000000
                         )
 
-                        expect(r.headers).not_to include 'Content-Type'
+                        expect(r.headers['Content-Type']).to be_empty
                         expect(r.body).not_to be_empty
                     end
                 end
@@ -820,7 +877,7 @@ describe Arachni::HTTP::Client do
             end
             context 'false' do
                 it 'uses the raw data from the cookie jar' do
-                    @opts.http.cookie_string = 'my_cookie_name="val1";"blah_name"=val2;another_name=another_val'
+                    @opts.http.cookie_string = 'my_cookie_name="val1","blah_name"=val2,another_name=another_val'
                     expect(subject.cookie_jar.cookies).to be_empty
                     subject.reset
 
@@ -836,7 +893,7 @@ describe Arachni::HTTP::Client do
                 end
                 context 'when custom cookies are provided' do
                     it 'merges them with the cookie_jar and override it' do
-                        @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                        @opts.http.cookie_string = 'my_cookie_name=val1,blah_name=val2,another_name=another_val'
                         expect(subject.cookie_jar.cookies).to be_empty
                         subject.reset
 
@@ -857,7 +914,7 @@ describe Arachni::HTTP::Client do
             end
             context 'nil' do
                 it 'defaults to false' do
-                    @opts.http.cookie_string = 'my_cookie_name="val1";"blah_name"=val2;another_name=another_val'
+                    @opts.http.cookie_string = 'my_cookie_name="val1","blah_name"=val2,another_name=another_val'
                     expect(subject.cookie_jar.cookies).to be_empty
                     subject.reset
 
@@ -1039,7 +1096,7 @@ describe Arachni::HTTP::Client do
 
             describe 'nil' do
                 it 'uses te cookies in the CookieJar' do
-                    @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                    @opts.http.cookie_string = 'my_cookie_name=val1,blah_name=val2,another_name=another_val'
                     expect(subject.cookie_jar.cookies).to be_empty
                     subject.reset
 
@@ -1082,7 +1139,7 @@ describe Arachni::HTTP::Client do
                 end
 
                 it 'merges them with the cookie-jar' do
-                    @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                    @opts.http.cookie_string = 'my_cookie_name=val1,blah_name=val2,another_name=another_val'
                     expect(subject.cookie_jar.cookies).to be_empty
                     subject.reset
 
@@ -1223,10 +1280,13 @@ describe Arachni::HTTP::Client do
                         value:  'val2',
                         domain: Arachni::URI( @url ).domain
                     )
+
                     subject.update_cookies( cookies )
                     subject.request( @url + '/update_cookies', update_cookies: true )
                     subject.run
-                    expect(subject.cookies.first.value).to eq(cookies.first.value + ' [UPDATED!]')
+
+                    cookie = subject.cookies.find { |c| c.value == 'val2 [UPDATED!]'}
+                    expect(cookie).to be_truthy
                 end
             end
         end
@@ -1263,7 +1323,7 @@ describe Arachni::HTTP::Client do
 
         context 'when cookie-jar lookup fails' do
             it 'only uses the given cookies' do
-                @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                @opts.http.cookie_string = 'my_cookie_name=val1,blah_name=val2,another_name=another_val'
                 expect(subject.cookie_jar.cookies).to be_empty
                 subject.reset
                 expect(subject.cookie_jar.cookies).to be_any

data/spec/arachni/http/cookie_jar_spec.rb

@@ -142,7 +142,7 @@ describe Arachni::HTTP::CookieJar do
                         expect(subject).to be_empty
 
                         Arachni::Options.url = 'http://test.com'
-                        subject.update( 'some_param=9e4ca2cc0f18a49f7c1881f78bebf7df; path=/; expires=Wed, 02-Oct-2020 23:53:46 GMT; HttpOnly' )
+                        subject.update( 'some_param=9e4ca2cc0f18a49f7c1881f78bebf7df; path=/; expires=Wed, 02-Oct-2030 23:53:46 GMT; HttpOnly' )
                         expect(subject.cookies.first.name).to eq('some_param')
                         expect(subject.cookies.first.value).to eq('9e4ca2cc0f18a49f7c1881f78bebf7df')
                     end
@@ -153,7 +153,7 @@ describe Arachni::HTTP::CookieJar do
                         expect(subject).to be_empty
 
                         Arachni::Options.url = 'http://test.com'
-                        subject.update( 'some_param=9e4ca2cc0f18a49f7c1881f78bebf7df; path=/; expires=Wed, 02-Oct-2020 23:53:46 GMT; HttpOnly' )
+                        subject.update( 'some_param=9e4ca2cc0f18a49f7c1881f78bebf7df; path=/; expires=Wed, 02-Oct-2030 23:53:46 GMT; HttpOnly' )
                         expect(subject.cookies.first.name).to eq('some_param')
                         expect(subject.cookies.first.value).to eq('9e4ca2cc0f18a49f7c1881f78bebf7df')
                     end