arachni 1.4 → 1.6.0

data/spec/arachni/http/headers_spec.rb

@@ -2,20 +2,60 @@ require 'spec_helper'
 
 describe Arachni::HTTP::Headers do
 
-    context 'when it includes multiple same names that differ in case' do
-        subject do
-            described_class.new( cookies )
+    subject do
+        described_class.new
+    end
+
+    describe '#merge!' do
+        context 'by default' do
+            context 'when it includes multiple same names that differ in case' do
+                let(:cookies) do
+                    {
+                        'set-cookie' => 'mycookie1=myvalue1',
+                        'Set-Cookie' => 'mycookie2=myvalue2',
+                        'SET-COOKIE' => 'mycookie3=myvalue3'
+                    }
+                end
+
+                it 'merges them into an array' do
+                    subject.merge!( cookies )
+                    expect(subject['set-cookie']).to eq(cookies.values)
+                end
+            end
         end
-        let(:cookies) do
-            {
-                'set-cookie' => 'mycookie1=myvalue1',
-                'Set-Cookie' => 'mycookie2=myvalue2',
-                'SET-COOKIE' => 'mycookie3=myvalue3'
-            }
+
+        context 'when convert to array is false' do
+            context 'when it includes multiple same names that differ in case' do
+                let(:cookies) do
+                    {
+                        'set-cookie' => 'mycookie1=myvalue1',
+                        'Set-Cookie' => 'mycookie2=myvalue2',
+                        'SET-COOKIE' => 'mycookie3=myvalue3'
+                    }
+                end
+
+                it 'does not merge them into an array' do
+                    subject.merge!( cookies, false )
+                    expect(subject['set-cookie']).to eq(cookies.values.last)
+                end
+            end
         end
 
-        it 'merges them into an array' do
-            expect(subject['set-cookie']).to eq(cookies.values)
+        context 'when convert to array is true' do
+            context 'when it includes multiple same names that differ in case' do
+                let(:cookies) do
+                    {
+                        'set-cookie' => 'mycookie1=myvalue1',
+                        'Set-Cookie' => 'mycookie2=myvalue2',
+                        'SET-COOKIE' => 'mycookie3=myvalue3'
+                    }
+                end
+
+                it 'does not merge them into an array' do
+                    subject.merge!( cookies )
+                    expect(subject['set-cookie']).to eq(cookies.values)
+                end
+            end
         end
     end
 
@@ -106,7 +146,7 @@ describe Arachni::HTTP::Headers do
     end
 
     describe '#location' do
-        it 'returns the content-type' do
+        it 'returns the Location' do
             ct = 'http://test.com'
             h = { 'location' => ct }
             expect(described_class.new( h ).location).to eq(ct)
@@ -125,5 +165,12 @@ describe Arachni::HTTP::Headers do
             h = { 'Content-Type' => ct }
             expect(described_class.new( h ).content_type).to eq(ct)
         end
+
+        context 'when there are multiple content-types' do
+            it 'returns the first one' do
+                h = { 'Content-Type' =>  ["application/x-javascript", "text/javascript"] }
+                expect(described_class.new( h ).content_type).to eq("application/x-javascript")
+            end
+        end
     end
 end

data/spec/arachni/http/proxy_server_spec.rb

@@ -46,6 +46,8 @@ describe Arachni::HTTP::ProxyServer do
         expect(response.body).to eq 'HTTPS GET'
     end
 
+    it 'supports WebSockets'
+
     it 'removes any size limits on the HTTP responses' do
         Arachni::Options.http.response_max_size = 1
 
@@ -54,6 +56,35 @@ describe Arachni::HTTP::ProxyServer do
         test_proxy( proxy )
     end
 
+    context 'when the response is text-based' do
+        let(:url) { "#{@url}/text" }
+
+        it 'sets charset encoding to UTF8' do
+            proxy = described_class.new
+            proxy.start_async
+
+            expect(via_proxy( proxy, url ).headers['Content-Type']).to eq 'text/html; charset=utf-8'
+        end
+
+        it 'transcodes to UTF-8' do
+            proxy = described_class.new
+            proxy.start_async
+
+            expect(via_proxy( proxy, url ).body.encoding.name).to eq 'UTF-8'
+        end
+    end
+
+    context 'when the response is binary' do
+        let(:url) { "#{@url}/binary" }
+
+        it 'does not add a charset' do
+            proxy = described_class.new
+            proxy.start_async
+
+            expect(via_proxy( proxy, url ).headers['Content-Type']).to eq 'application/binary'
+        end
+    end
+
     describe '#initialize' do
         describe ':address' do
             it 'sets the bind address' do
@@ -89,31 +120,31 @@ describe Arachni::HTTP::ProxyServer do
             end
         end
 
-        # describe ':concurrency' do
-        #     it 'sets the HTTP request concurrency' do
-        #         sleep_url = @url + 'sleep'
-        #
-        #         proxy = described_class.new( concurrency: 2 )
-        #         proxy.start_async
-        #         time = Time.now
-        #         threads = []
-        #         2.times do
-        #             threads << Thread.new { via_proxy( proxy, sleep_url ) }
-        #         end
-        #         threads.each(&:join)
-        #         expect((Time.now - time).to_i).to eq(5)
-        #
-        #         proxy = described_class.new( concurrency: 1 )
-        #         proxy.start_async
-        #         time = Time.now
-        #         threads = []
-        #         2.times do
-        #             threads << Thread.new { via_proxy( proxy, sleep_url ) }
-        #         end
-        #         threads.each(&:join)
-        #         expect((Time.now - time).to_i).to eq(10)
-        #     end
-        # end
+        describe ':concurrency' do
+            it 'sets the HTTP request concurrency' do
+                sleep_url = @url + 'sleep'
+
+                proxy = described_class.new( concurrency: 2 )
+                proxy.start_async
+                time = Time.now
+                threads = []
+                2.times do
+                    threads << Thread.new { via_proxy( proxy, sleep_url ) }
+                end
+                threads.each(&:join)
+                expect((Time.now - time).to_i).to eq(5)
+
+                proxy = described_class.new( concurrency: 1 )
+                proxy.start_async
+                time = Time.now
+                threads = []
+                2.times do
+                    threads << Thread.new { via_proxy( proxy, sleep_url ) }
+                end
+                threads.each(&:join)
+                expect((Time.now - time).to_i).to eq(10)
+            end
+        end
 
         describe ':request_handler' do
             it 'sets a block to handle each HTTP request before the request is forwarded to the origin server' do
@@ -171,12 +202,14 @@ describe Arachni::HTTP::ProxyServer do
 
                 expect(request.headers_string.split( "\r\n" ).sort).to eq(
                     ("POST / HTTP/1.1\r\n" <<
+                    "Authorization: Basic Og==\r\n" <<
                     "Host: #{request.parsed_url.host}:#{request.parsed_url.port}\r\n" <<
                     "Accept-Encoding: gzip, deflate\r\n" <<
                     "User-Agent: Typhoeus - https://github.com/typhoeus/typhoeus\r\n" <<
                     "Accept: */*\r\n" <<
                     "Content-Length: 7\r\n" <<
                     "Accept-Language: en-US,en;q=0.8,he;q=0.6\r\n" <<
+                    "#{Arachni::HTTP::Client::SEED_HEADER_NAME}: #{Arachni::Utilities.random_seed}\r\n" <<
                     "Content-Type: application/x-www-form-urlencoded\r\n\r\n").split( "\r\n" ).sort
                 )
 

data/spec/arachni/http/request_spec.rb

@@ -282,11 +282,50 @@ describe Arachni::HTTP::Request do
             expect(called.first.request).to eq(request)
         end
 
+        it 'calls #on_body callbacks' do
+            request  = described_class.new( url: @url )
+
+            s = ''
+            request.on_body do |chunk|
+                s << chunk
+            end
+
+            response = request.run
+
+            expect(s).to eq 'GET'
+        end
+
+        it 'calls #on_headers callbacks' do
+            request  = described_class.new( url: @url )
+
+            called = nil
+            request.on_headers do |r|
+                called = r
+            end
+
+            response = request.run
+            expect(response).to be_kind_of Arachni::HTTP::Response
+            expect(response.request).to eq(request)
+
+            h = response.to_h
+            h[:body] = ''
+            h.delete :time
+            h.delete :total_time
+
+            c = called.to_h
+            c.delete :time
+            c.delete :total_time
+
+            expect(c).to eq(h)
+            expect(called.request).to eq(request)
+        end
+
         it "fills in #{Arachni::HTTP::Request}#headers_string" do
             host = "#{Arachni::URI(@url).host}:#{Arachni::URI(@url).port}"
             expect(described_class.new( url: @url ).run.request.headers_string).to eq(
-                "GET / HTTP/1.1\r\nHost: #{host}\r\nAccept-Encoding: gzip, " +
-                    "deflate\r\nUser-Agent: Arachni/v#{Arachni::VERSION}\r\nAccept: text/html," +
+                "GET / HTTP/1.1\r\nHost: #{host}\r\nAuthorization: Basic Og==\r\n" +
+                    "Accept-Encoding: gzip, deflate\r\n" +
+                    "User-Agent: #{Arachni::Options.http.user_agent}\r\nAccept: text/html," +
                     "application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" +
                     "Accept-Language: en-US,en;q=0.8,he;q=0.6\r\n\r\n"
             )
@@ -332,18 +371,52 @@ describe Arachni::HTTP::Request do
         end
     end
 
+    describe '#buffered?' do
+        context 'when there are no callbacks' do
+            subject do
+                described_class.new( url: url )
+            end
+
+            expect_it { to_not be_buffered }
+        end
+
+        context 'when there are #on_body callbacks' do
+            subject do
+                described_class.new( url: url ).on_body {}
+            end
+
+            expect_it { to be_buffered }
+        end
+
+        context 'when there are #on_body_line callbacks' do
+            subject do
+                described_class.new( url: url ).on_body_line {}
+            end
+
+            expect_it { to be_buffered }
+        end
+
+        context 'when there are #on_body_lines callbacks' do
+            subject do
+                described_class.new( url: url ).on_body_lines {}
+            end
+
+            expect_it { to be_buffered }
+        end
+    end
+
     describe '#on_complete' do
         context 'when passed a block' do
             it 'adds it as a callback to be passed the response' do
-                request = described_class.new( url: url )
-
-                passed_response = nil
-                request.on_complete { |res| passed_response = res }
+                response = nil
 
-                response = Arachni::HTTP::Response.new( url: url )
-                request.handle_response( response )
+                request = described_class.new( url: url )
+                request.on_complete { |r| response = r }
+                request.run
 
-                expect(passed_response).to eq(response)
+                expect(response.code).to eq 200
+                expect(response.body).to eq 'GET'
+                expect(response.headers).to be_any
             end
 
             it 'can add multiple callbacks' do
@@ -354,55 +427,307 @@ describe Arachni::HTTP::Request do
                 2.times do
                     request.on_complete { |res| passed_responses << res }
                 end
-
-                response = Arachni::HTTP::Response.new( url: url )
-                request.handle_response( response )
+                request.run
 
                 expect(passed_responses.size).to eq(2)
-                expect(passed_responses.uniq.size).to eq(1)
-                expect(passed_responses.uniq.first).to eq(response)
+
+                passed_responses.each do |response|
+                    expect(response.code).to eq 200
+                    expect(response.body).to eq 'GET'
+                    expect(response.headers).to be_any
+                end
+            end
+
+            context 'when an exception is raised' do
+                it 'does not affect other callbacks' do
+                    response = nil
+
+                    request = described_class.new( url: url )
+                    request.on_complete { |r| fail }
+                    request.on_complete { |r| response = r }
+                    request.on_complete { |r| fail }
+                    request.run
+
+                    expect(response.code).to eq 200
+                    expect(response.body).to eq 'GET'
+                    expect(response.headers).to be_any
+                end
             end
         end
     end
 
-    describe '#clear_callbacks' do
-        it 'clears #on_complete callbacks' do
-            request = described_class.new( url: url )
+    describe '#on_headers' do
+        context 'when passed a block' do
+            it 'adds it as a callback to be passed the response with headers' do
+                request = described_class.new( url: "#{url}/stream" )
 
-            passed_response = nil
-            request.on_complete { |res| passed_response = res }
+                response_h = {}
+                request.on_headers do |response|
+                    response_h = response.to_h
+                end
+                request.run
 
-            response = Arachni::HTTP::Response.new( url: url )
-            request.clear_callbacks
-            request.handle_response( response )
+                expect(response_h[:code]).to eq 200
+                expect(response_h[:body]).to eq ''
+                expect(response_h[:headers]).to be_any
+            end
 
-            expect(passed_response).to be_nil
+            context 'when the block returns :abort' do
+                it 'aborts the request' do
+                    request = described_class.new( url: "#{url}/stream" )
+
+                    request.on_headers do |response|
+                        :abort
+                    end
+                    response = request.run
+
+                    expect(response.time).to be < 2
+                end
+            end
+
+            context 'when an exception is raised' do
+                it 'does not affect other callbacks' do
+                    response_h = nil
+
+                    request = described_class.new( url: url )
+                    request.on_headers { fail }
+                    request.on_headers { |r| response_h = r.to_h }
+                    request.on_headers { fail }
+                    request.run
+
+                    expect(response_h[:code]).to eq 200
+                    expect(response_h[:body]).to eq ''
+                    expect(response_h[:headers]).to be_any
+                end
+            end
         end
     end
 
+    describe '#on_body' do
+        context 'when passed a block' do
+            it 'adds it as a callback to be passed the each body chunk' do
+                request = described_class.new( url: "#{url}/lines" )
 
-    describe '#handle_response' do
-        it 'assigns self as the #request attribute of the response' do
-            request = described_class.new( url: url )
+                s = ''
+                500.times do |i|
+                    s << "#{i}: test\n"
+                end
 
-            passed_response = nil
-            request.on_complete { |res| passed_response = res }
+                lines = ''
+                request.on_body do |line|
+                    lines << line
+                end
+                request.run
 
-            response = Arachni::HTTP::Response.new( url: url )
-            request.handle_response( response )
+                expect(lines).to eq s
+            end
+
+            context 'when the block returns :abort' do
+                it 'aborts the request' do
+                    request = described_class.new( url: "#{url}/lines" )
+
+                    i = 0
+                    request.on_body do
+                        if i == 100
+                            next :abort
+                        end
+
+                        i += 1
+                    end
+                    request.run
+
+                    expect(i).to eq 100
+                end
+            end
+
+            context 'when an exception is raised' do
+                it 'does not affect other callbacks' do
+                    response_h = nil
+
+                    request = described_class.new( url: url )
+                    request.on_body { fail }
+                    request.on_body { |_, r| response_h = r.to_h }
+                    request.on_body { fail }
+                    request.run
+
+                    expect(response_h[:code]).to eq 200
+                    expect(response_h[:body]).to eq ''
+                    expect(response_h[:headers]).to be_any
+                end
+            end
+        end
+    end
+
+    describe '#on_body_line' do
+        context 'when passed a block' do
+            it 'adds it as a callback to be passed the each body line' do
+                request = described_class.new( url: "#{url}/lines" )
+
+                s = ''
+                500.times do |i|
+                    s << "#{i}: test\n"
+                end
+
+                lines = ''
+                request.on_body_line do |line|
+                    lines << line
+                end
+                request.run
+
+                expect(lines).to eq s
+            end
+
+            context 'when the body does not end in EOL' do
+                it 'provides the last line via #on_complete' do
+                    request = described_class.new( url: "#{url}/lines/incomplete" )
+
+                    lines = ''
+                    request.on_body_line do |line|
+                        lines << line
+                    end
+
+                    last_line = ''
+                    request.on_complete do |response|
+                        last_line = response.body
+                    end
+                    request.run
+
+                    expect(lines).to eq "Blah\n"
+                    expect(last_line).to eq 'Hello!'
+                end
+            end
+
+            context 'when the block returns :abort' do
+                it 'aborts the request' do
+                    request = described_class.new( url: "#{url}/lines" )
+
+                    i = 0
+                    request.on_body_line do
+                        if i == 100
+                            next :abort
+                        end
 
-            expect(passed_response.request).to eq(request)
+                        i += 1
+                    end
+                    request.run
+
+                    expect(i).to eq 100
+                end
+            end
+
+            context 'when an exception is raised' do
+                it 'does not affect other callbacks' do
+                    response_h = nil
+
+                    request = described_class.new( url: "#{url}/fast_stream" )
+                    request.on_body_line { fail }
+                    request.on_body_line { |_, r| response_h = r.to_h }
+                    request.on_body_line { fail }
+                    request.run
+
+                    expect(response_h[:code]).to eq 200
+                    expect(response_h[:body]).to eq ''
+                    expect(response_h[:headers]).to be_any
+                end
+            end
         end
+    end
 
-        it 'calls #on_complete callbacks' do
-            response = Arachni::HTTP::Response.new( url: url, code: 200 )
+    describe '#on_body_lines' do
+        context 'when passed a block' do
+            it 'adds it as a callback to be passed chunks of body lines' do
+                request = described_class.new( url: "#{url}/lines/non-stream" )
+
+                s = ''
+                # 1_575 is the magic number where packets are split but set it
+                # to 2_000 for good measure.
+                2_000.times do |i|
+                    s << "#{i}: test\n"
+                end
+
+                lines = []
+                request.on_body_lines do |line|
+                    lines << line
+                end
+                request.run
+
+                expect(lines.join).to eq s
+                expect(lines.size).to be > 1
+                expect(lines.size).to be < 500
+
+                lines.each do |line|
+                    expect(line).to end_with"\n"
+                end
+            end
+
+            context 'when the body does not end in EOL' do
+                it 'provides the last line via #on_complete' do
+                    request = described_class.new( url: "#{url}/lines/incomplete" )
+
+                    lines = ''
+                    request.on_body_lines do |line|
+                        lines << line
+                    end
+
+                    last_line = ''
+                    request.on_complete do |response|
+                        last_line = response.body
+                    end
+                    request.run
+
+                    expect(lines).to eq "Blah\n"
+                    expect(last_line).to eq 'Hello!'
+                end
+            end
+
+            context 'when the block returns :abort' do
+                it 'aborts the request' do
+                    request = described_class.new( url: "#{url}/lines" )
+
+                    i = 0
+                    request.on_body_lines do
+                        if i == 100
+                            next :abort
+                        end
+
+                        i += 1
+                    end
+                    request.run
+
+                    expect(i).to eq 100
+                end
+            end
+
+            context 'when an exception is raised' do
+                it 'does not affect other callbacks' do
+                    response_h = nil
+
+                    request = described_class.new( url: "#{url}/fast_stream" )
+                    request.on_body_lines { fail }
+                    request.on_body_lines { |_, r| response_h = r.to_h }
+                    request.on_body_lines { fail }
+                    request.run
+
+                    expect(response_h[:code]).to eq 200
+                    expect(response_h[:body]).to eq ''
+                    expect(response_h[:headers]).to be_any
+                end
+            end
+        end
+    end
+
+    describe '#clear_callbacks' do
+        it 'clears #on_complete callbacks' do
             request = described_class.new( url: url )
 
             passed_response = nil
             request.on_complete { |res| passed_response = res }
-            request.handle_response( response )
 
-            expect(passed_response).to eq(response)
+            response = Arachni::HTTP::Response.new( url: url )
+            request.clear_callbacks
+            request.run
+
+            expect(passed_response).to be_nil
         end
     end
 
@@ -676,6 +1001,28 @@ describe Arachni::HTTP::Request do
             end
         end
 
+        context "#{Arachni::OptionGroups::HTTP}#authentication_type" do
+            before :each do
+                Arachni::Options.http.authentication_username = 'name'
+                Arachni::Options.http.authentication_password = 'secret'
+                Arachni::Options.http.authentication_type     = 'ntlm'
+            end
+
+            let(:request) do
+                described_class.new( url: url )
+            end
+
+            it 'forwards it' do
+                expect(subject.options[:httpauth]).to eq(:ntlm)
+            end
+
+            it 'defaults to :auto' do
+                Arachni::Options.http.authentication_type = nil
+
+                expect(subject.options[:httpauth]).to eq(:auto)
+            end
+        end
+
         context "#{Arachni::OptionGroups::HTTP}#response_max_size" do
             before :each do
                 Arachni::Options.http.response_max_size = 10