arachni 0.4.7 → 1.0

data/spec/arachni/http/client_spec.rb

@@ -0,0 +1,1470 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Arachni::HTTP::Client do
+
+    before( :all ) do
+        @opts = Arachni::Options.instance
+        @url  = web_server_url_for( :client )
+    end
+    before( :each ) do
+        @opts.reset
+        @opts.audit.links = true
+        @opts.url = @url
+        subject.reset
+    end
+
+    subject { Arachni::HTTP::Client }
+    let(:custom_404_url) { @url + '/custom_404/' }
+
+    it 'supports gzip content-encoding' do
+        body = nil
+        subject.get( @opts.url + 'gzip' ) { |res| body = res.body }
+        subject.run
+        body.should == 'success'
+    end
+
+    it 'preserves set-cookies' do
+        body = nil
+        subject.get( @opts.url + 'set_and_preserve_cookies', update_cookies: true )
+        subject.run
+        subject.cookies.first.value.should == "=stuf \00 here=="
+
+        subject.get( @opts.url + 'cookies' ) { |res| body = res.body }
+        subject.run
+        YAML.load( body ).should == { 'stuff' => "=stuf \00 here==" }
+    end
+
+    describe '#statistics' do
+        let(:statistics) { subject.statistics }
+
+        [:request_count, :response_count, :time_out_count,
+         :total_responses_per_second, :burst_response_time_sum,
+         :burst_response_count, :burst_responses_per_second,
+         :burst_average_response_time, :total_average_response_time,
+         :max_concurrency].each do |k|
+            it "includes #{k}" do
+                statistics[k].should == subject.send(k)
+            end
+        end
+
+    end
+
+    describe Arachni::Options do
+        describe '#url' do
+            context 'when the target URL includes auth credentials' do
+                it 'uses them globally' do
+                    # first fail to make sure that our test server is actually working properly
+                    code = 0
+                    subject.get( "#{@opts.url}auth/simple-chars" ) { |res| code = res.code }
+                    subject.run
+                    code.should == 401
+
+                    url = Arachni::Utilities.uri_parse( "#{@opts.url}auth/simple-chars" )
+                    url.user = 'username'
+                    url.password = 'password'
+                    @opts.url = url.to_s
+
+                    body = nil
+                    subject.get( @opts.url ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'authenticated!'
+                end
+            end
+        end
+
+        describe '#fingerprint?' do
+            context true do
+                it 'performs platform fingerprinting on the response' do
+                    Arachni::Options.fingerprint
+
+                    res = nil
+                    subject.request( @url + '/fingerprint.php' ) { |c_res| res = c_res }
+                    subject.run
+
+                    res.platforms.to_a.should == [:php]
+                end
+            end
+
+            context false do
+                it 'does not fingerprint the response' do
+                    Arachni::Platform::Manager.clear
+                    Arachni::Options.do_not_fingerprint
+
+                    res = nil
+                    subject.request( @url + '/fingerprint.php' ) { |c_res| res = c_res }
+                    subject.run
+
+                    res.platforms.should be_empty
+                end
+            end
+        end
+    end
+
+    describe Arachni::OptionGroups::HTTP do
+        describe '#request_concurrency' do
+            context Integer do
+                it 'uses it as a max_concurrency' do
+                    @opts.http.request_concurrency = 34
+                    subject.reset
+                    subject.max_concurrency.should == 34
+                end
+            end
+            context 'nil' do
+                it 'uses a default max concurrency setting' do
+                    @opts.http.request_concurrency = nil
+                    subject.reset
+                    subject.max_concurrency.should == Arachni::HTTP::Client::MAX_CONCURRENCY
+                end
+            end
+        end
+
+        describe '#request_queue_size' do
+            context 'when reached' do
+                it 'performs an emergency run' do
+                    Arachni::Options.http.request_queue_size = 10
+
+                    responses = []
+                    11.times do
+                        subject.request do |response|
+                            responses << response
+                        end
+                    end
+
+                    responses.size.should == 10
+
+                    subject.run
+                    responses.size.should == 11
+                end
+            end
+        end
+
+        describe '#request_timeout' do
+            context Integer do
+                it 'uses it as an HTTP timeout' do
+                    @opts.http.request_timeout = 10000000000
+                    timed_out = false
+                    subject.request( @url + '/sleep' ) { |res| timed_out = res.timed_out? }
+                    subject.run
+                    timed_out.should be_false
+
+                    @opts.http.request_timeout = 1
+                    subject.reset
+                    timed_out = false
+                    subject.request( @url + '/sleep' ) { |res| timed_out = res.timed_out? }
+                    subject.run
+                    timed_out.should be_true
+                end
+            end
+            context 'nil' do
+                it 'uses a default timeout setting' do
+                    timed_out = false
+                    subject.request( @url + '/sleep' ) { |res| timed_out = res.timed_out? }
+                    subject.run
+                    timed_out.should be_false
+                end
+            end
+        end
+
+        describe '#authentication_username and #authentication_password' do
+            it 'uses them globally' do
+                Arachni::Options.http.authentication_username = 'username1'
+                Arachni::Options.http.authentication_password = 'password1'
+
+                # first fail to make sure that our test server is actually working properly
+                code = 0
+                subject.get( @opts.url + 'auth/weird-chars' ) { |res| code = res.code }
+                subject.run
+                code.should == 401
+
+                Arachni::Options.http.authentication_username,
+                    Arachni::Options.http.authentication_password =
+                        ['u se rname$@#@#%$3#@%@#', 'p a  :wo\'rd$@#@#%$3#@%@#' ]
+
+                response = nil
+                subject.get( @opts.url + 'auth/weird-chars' ) { |res| response = res }
+                subject.run
+                response.code.should == 200
+                response.body.should == 'authenticated!'
+            end
+        end
+
+        describe 'user_agent' do
+            it 'uses the default user-agent setting' do
+                body = nil
+                subject.get( @opts.url + 'user-agent' ) { |res| body = res.body }
+                subject.run
+
+                body.should == @opts.http.user_agent
+                @opts.http.user_agent.should == Arachni::OptionGroups::HTTP.defaults[:user_agent]
+            end
+            context String do
+                it 'uses it as a user-agent' do
+                    ua = 'my user agent'
+                    @opts.http.user_agent = ua.dup
+                    subject.reset
+
+                    body = nil
+                    subject.get( @opts.url + 'user-agent' ) { |res| body = res.body }
+                    subject.run
+                    body.should == ua
+                end
+            end
+        end
+
+        describe '#request_redirect_limit' do
+            context Integer do
+                it 'should not exceed that amount of redirects' do
+                    @opts.http.request_redirect_limit = 2
+                    code = nil
+                    subject.get( @opts.url + 'redirect', follow_location: true ) { |res| code = res.code }
+                    subject.run
+                    code.should == 302
+
+                    @opts.http.request_redirect_limit = 10
+                    subject.reset
+
+                    body = nil
+                    subject.get( @opts.url + 'redirect', follow_location: true ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'This is the end.'
+                end
+            end
+            context 'nil' do
+                it 'uses a default setting' do
+                    subject.reset
+
+                    body = nil
+                    subject.get( @opts.url + 'redirect', follow_location: true ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'This is the end.'
+                end
+            end
+        end
+    end
+
+    describe '#sandbox' do
+        it 'preserves state, runs the block and then restores state' do
+            subject.cookies.should be_empty
+            subject.get( @opts.url + 'set_and_preserve_cookies', update_cookies: true )
+            subject.run
+            subject.cookies.should be_any
+
+            headers = subject.headers.dup
+
+            signals = []
+            subject.on_complete do |r|
+                signals << :out
+            end
+
+            subject.get( @opts.url + 'out', mode: :sync )
+
+            subject.sandbox do
+                subject.cookies.should be_any
+                subject.cookie_jar.clear
+                subject.cookies.should be_empty
+
+                subject.headers.should == headers
+                subject.headers['X-Custom'] = 'stuff'
+                subject.headers.include?( 'X-Custom' ).should be_true
+
+                subject.on_complete do |r|
+                    signals << :in
+                end
+
+                subject.get( @opts.url + 'in', mode: :sync )
+            end
+
+            subject.get( @opts.url + 'out', mode: :sync )
+
+            signals.delete( :out )
+            signals.size.should == 1
+
+            subject.headers.include?( 'X-Custom' ).should be_false
+            subject.cookies.should be_any
+        end
+    end
+
+    describe '#url' do
+        it 'returns the URL in opts' do
+            subject.url.should == @opts.url.to_s
+        end
+    end
+
+    describe '#headers' do
+        it 'provides access to default headers' do
+            headers = subject.headers
+            headers['Accept'].should == 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+            headers['User-Agent'].should == 'Arachni/v' + Arachni::VERSION
+        end
+
+        context "when #{Arachni::OptionGroups::HTTP}#request_headers is set" do
+            it 'includes them' do
+                @opts.http.request_headers = {
+                    'User-Agent' => 'My UA',
+                    'From'       => 'Some dude',
+                }
+                subject.reset
+                headers = subject.headers
+                headers['From'].should == @opts.http.request_headers['From']
+                headers['Accept'].should == 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+                headers['User-Agent'].should == @opts.http.request_headers['User-Agent']
+            end
+        end
+
+        context 'when the authorized_by option is set' do
+            it 'includes it in the From field' do
+                @opts.authorized_by = 'The Dude'
+                subject.reset
+                subject.headers['From'].should == @opts.authorized_by
+            end
+        end
+    end
+
+    describe '#cookie_jar' do
+        it 'provides access to the Cookie-jar' do
+            subject.cookie_jar.is_a?( Arachni::HTTP::CookieJar ).should be_true
+        end
+
+        context "when #{Arachni::OptionGroups::HTTP}#cookie_jar_filepath is set" do
+            it 'adds the contained cookies to the CookieJar' do
+                @opts.http.cookie_jar_filepath = fixtures_path + 'cookies.txt'
+                subject.cookie_jar.cookies.should be_empty
+                subject.reset
+                cookies = subject.cookie_jar.cookies
+                cookies.size.should == 4
+                cookies.should == Arachni::Utilities.cookies_from_file( '', @opts.http.cookie_jar_filepath )
+            end
+            context 'but the path is invalid' do
+                it 'raises Arachni::HTTP::CookieJar::Error::CookieJarFileNotFound' do
+                    @opts.http.cookie_jar_filepath = fixtures_path + 'cookies.does_not_exist.txt'
+                    expect{ subject.reset }.to raise_error Arachni::HTTP::CookieJar::Error::CookieJarFileNotFound
+                end
+            end
+        end
+
+        context 'when the cookies option is set' do
+            it 'adds those cookies to the CookieJar' do
+                cookie_jar_file = fixtures_path + 'cookies.txt'
+                @opts.http.cookies = Arachni::Utilities.cookies_from_file( '', cookie_jar_file )
+                subject.cookie_jar.cookies.should be_empty
+                subject.reset
+                cookies = subject.cookie_jar.cookies
+                cookies.size.should == 4
+                cookies.should == @opts.http.cookies
+            end
+        end
+
+        context "when #{Arachni::OptionGroups::HTTP}#cookie_string is set" do
+            it 'parses the string and add those cookies to the CookieJar' do
+                @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2; stuff=%25blah; another_name=another_val'
+                subject.cookie_jar.cookies.should be_empty
+                subject.reset
+                cookies = subject.cookie_jar.cookies
+                cookies.size.should == 4
+                cookies.first.name.should == 'my_cookie_name'
+                cookies.first.value.should == 'val1'
+                cookies[1].name.should == 'blah_name'
+                cookies[1].value.should == 'val2'
+                cookies[2].name.should == 'stuff'
+                cookies[2].value.should == '%blah'
+                cookies.last.name.should == 'another_name'
+                cookies.last.value.should == 'another_val'
+            end
+        end
+    end
+
+    describe '#cookies' do
+        it 'returns the current cookies' do
+            @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2; another_name=another_val'
+            subject.cookie_jar.cookies.should be_empty
+            subject.reset
+            subject.cookies.size.should == 3
+            subject.cookies.should == subject.cookie_jar.cookies
+        end
+    end
+
+    describe '#after_run' do
+        it 'sets blocks to be called after #run' do
+            called = false
+            subject.after_run { called = true }
+            subject.run
+            called.should be_true
+
+            called = false
+            subject.run
+            called.should be_false
+        end
+
+        context 'when the callback creates new requests and nested callbacks' do
+            it 'run these too' do
+                called = false
+                subject.after_run do
+                    subject.after_run { called = true }
+                end
+                subject.run
+                called.should be_false
+
+                called = false
+                subject.after_run do
+                    subject.get
+                    subject.after_run { called = true }
+                end
+                subject.run
+                called.should be_true
+
+                called = false
+                subject.run
+                called.should be_false
+            end
+        end
+    end
+
+    describe '#run' do
+        it 'performs the queued requests' do
+            response = nil
+            subject.request { |r| response = r }
+
+            subject.run
+
+            response.should be_kind_of Arachni::HTTP::Response
+        end
+
+        it 'returns true' do
+            subject.run.should be_true
+        end
+
+        it 'calls the after_each_run callbacks EVERY TIME' do
+            called = false
+            subject.after_each_run { called = true }
+            subject.run
+            called.should be_true
+            called = false
+            subject.run
+            called.should be_true
+        end
+
+        it 'calculates the burst average response time' do
+            subject.run
+            subject.burst_runtime.should > 0
+        end
+
+        it 'updates burst_response_time_sum, burst_response_count,' +
+               ' burst_average_response_time and burst_responses_per_second' +
+               ' during runtime and resets them afterwards' do
+            subject.total_runtime.to_i.should          == 0
+            subject.total_average_response_time.should == 0
+            subject.total_responses_per_second.should  == 0
+
+            subject.burst_response_time_sum.should     == 0
+            subject.burst_response_count.should        == 0
+            subject.burst_average_response_time.should == 0
+            subject.burst_responses_per_second.should  == 0
+
+            total_runtime               = 0
+            total_average_response_time = 0
+            total_responses_per_second  = 0
+
+            burst_response_time_sum     = 0
+            burst_response_count        = 0
+            burst_average_response_time = 0
+            burst_responses_per_second  = 0
+
+            20.times do
+                subject.get do
+                    total_runtime               = subject.total_runtime
+                    total_average_response_time = subject.total_average_response_time
+                    total_responses_per_second  = subject.total_responses_per_second
+
+                    burst_response_time_sum     = subject.burst_response_time_sum
+                    burst_response_count        = subject.burst_response_count
+                    burst_average_response_time = subject.burst_average_response_time
+                    burst_responses_per_second  = subject.burst_responses_per_second
+                end
+            end
+
+            subject.run
+
+            total_runtime.should               > 0
+            total_average_response_time.should > 0
+            total_responses_per_second.should  > 0
+
+            burst_response_time_sum.should     > 0
+            burst_response_count.should        > 0
+            burst_average_response_time.should > 0
+            burst_responses_per_second.should  > 0
+        end
+
+        context "when a #{RuntimeError} occurs" do
+            it 'returns nil' do
+                subject.instance.stub(:hydra_run){ raise }
+
+                subject.run.should be_nil
+            end
+        end
+    end
+
+    describe '#abort' do
+        it 'aborts the running requests on a best effort basis' do
+            cnt = 0
+            n = 50
+            n.times do |i|
+                subject.get do
+                    cnt += 1
+                    subject.abort
+                end
+            end
+            subject.run
+            cnt.should < n
+        end
+    end
+
+    describe '#max_concurrency' do
+        it 'defaults to 20' do
+            subject.max_concurrency.should == 20
+        end
+        it 'respects the http_request_concurrency option' do
+            @opts.http.request_concurrency = 50
+            subject.reset
+            subject.max_concurrency.should == 50
+        end
+    end
+
+    describe '#max_concurrency=' do
+        it 'sets the max_concurrency setting' do
+            subject.max_concurrency.should_not == 30
+            subject.max_concurrency = 30
+            subject.max_concurrency.should == 30
+        end
+    end
+
+    describe '#request' do
+        it "uses the URL in #{Arachni::Options}#url as a default" do
+            url = nil
+            subject.request{ |res| url = res.url }
+            subject.run
+            url.start_with?( @opts.url.to_s ).should be_true
+        end
+
+        it 'raises exception when no URL is available' do
+            @opts.reset
+            subject.reset
+            expect { subject.request }.to raise_error
+        end
+
+        it "fills in #{Arachni::HTTP::Request}#headers_string" do
+            host = "#{Arachni::URI(@url).host}:#{Arachni::URI(@url).port}"
+            subject.request( @url, mode: :sync ).request.headers_string.should ==
+                "GET / HTTP/1.1\r\nHost: #{host}\r\nAccept-Encoding: gzip, " +
+                    "deflate\r\nUser-Agent: Arachni/v#{Arachni::VERSION}\r\nAccept: text/html," +
+                    "application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n\r\n"
+        end
+
+        it "fills in #{Arachni::HTTP::Request}#effective_body" do
+            subject.request( @url,
+               body: {
+                   '1' => ' 2',
+                   ' 3' => '4'
+               },
+               mode:   :sync,
+               method: :post
+            ).request.effective_body.should == "1=%202&%203=4"
+        end
+
+        describe :response_max_size do
+            context "when #{Arachni::OptionGroups::HTTP}#response_max_size is specified" do
+                it 'ignores bodies of responses which are larger than specified' do
+                    @opts.http.response_max_size = 0
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync
+                    ).body.should be_empty
+
+                    @opts.http.response_max_size = 1
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync
+                    ).body.should be_empty
+
+                    @opts.http.response_max_size = 999999
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync
+                    ).body.should be_empty
+
+                    @opts.http.response_max_size = 1000000
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync
+                    ).body.should_not be_empty
+                end
+            end
+
+            context 'when specified' do
+                it 'ignores bodies of responses which are larger than specified' do
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync,
+                                   response_max_size: 0
+                    ).body.should be_empty
+
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync,
+                                   response_max_size: 1
+                    ).body.should be_empty
+
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync,
+                                   response_max_size: 999999
+                    ).body.should be_empty
+
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync,
+                                   response_max_size: 1000000
+                    ).body.should_not be_empty
+                end
+            end
+
+            context 'by default' do
+                it 'does not enforce a limit' do
+                    subject.request( @url + '/http_response_max_size',
+                                   mode: :sync
+                    ).body.should_not be_empty
+                end
+            end
+        end
+
+        describe :no_cookie_jar do
+            context true do
+                it 'skips the cookie-jar' do
+                    body = nil
+                    subject.request( @url + '/cookies', no_cookie_jar: true ) { |res| body = res.body }
+                    subject.run
+                    YAML.load( body ).should == {}
+                end
+            end
+            context false do
+                it 'uses the cookie_jar' do
+                    @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                    subject.cookie_jar.cookies.should be_empty
+                    subject.reset
+
+                    body = nil
+
+                    subject.request( @url + '/cookies', no_cookie_jar: false ) { |res| body = res.body }
+                    subject.run
+                    YAML.load( body ).should == {
+                        'my_cookie_name' => 'val1',
+                        'blah_name' => 'val2',
+                        'another_name' => 'another_val'
+                    }
+                end
+                context 'when custom cookies are provided' do
+                    it 'merges them with the cookie_jar and override it' do
+                        @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                        subject.cookie_jar.cookies.should be_empty
+                        subject.reset
+
+                        body = nil
+
+                        custom_cookies = { 'newcookie' => 'newval', 'blah_name' => 'val3' }
+                        subject.request( @url + '/cookies', cookies: custom_cookies,
+                                       no_cookie_jar: false ) { |res| body = res.body }
+                        subject.run
+                        YAML.load( body ).should == {
+                            'my_cookie_name' => 'val1',
+                            'blah_name' => 'val3',
+                            'another_name' => 'another_val',
+                            'newcookie' => 'newval'
+                        }
+                    end
+                end
+            end
+            context 'nil' do
+                it 'defaults to false' do
+                    @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                    subject.cookie_jar.cookies.should be_empty
+                    subject.reset
+
+                    body = nil
+
+                    subject.request( @url + '/cookies' ) { |res| body = res.body }
+                    subject.run
+                    YAML.load( body ).should == {
+                        'my_cookie_name' => 'val1',
+                        'blah_name' => 'val2',
+                        'another_name' => 'another_val'
+                    }
+                end
+            end
+        end
+
+        describe :body do
+            it 'uses its value as a request body' do
+                req_body = 'heyaya'
+                body = nil
+                subject.request( @url + '/body', method: :post, body: req_body ) { |res| body = res.body }
+                subject.run
+                body.should == req_body
+            end
+        end
+
+        describe :method do
+            describe 'nil' do
+                it 'performs a GET HTTP request' do
+                    body = nil
+                    subject.request( @url ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'GET'
+                end
+            end
+            describe :get do
+                it 'performs a GET HTTP request' do
+                    body = nil
+                    subject.request( @url, method: :get ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'GET'
+                end
+
+                context 'when there are both query string and hash params' do
+                    it 'merges them while giving priority to the hash params' do
+                        body = nil
+                        params = {
+                            'param1' => 'value1_updated',
+                            'param2' => 'value 2'
+                        }
+                        url = @url + '/echo?param1=value1&param3=value3'
+                        subject.request( url, parameters: params, method: :get ){ |res| body = res.body }
+                        subject.run
+                        YAML.load( body ).should eq params.merge( 'param3' => 'value3' )
+                    end
+                end
+            end
+            describe :post do
+                it 'performs a POST HTTP request' do
+                    body = nil
+                    subject.request( @url, method: :post ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'POST'
+                end
+            end
+            describe :put do
+                it 'performs a PUT HTTP request' do
+                    body = nil
+                    subject.request( @url, method: :put ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'PUT'
+                end
+            end
+            describe :options do
+                it 'performs a OPTIONS HTTP request' do
+                    body = nil
+                    subject.request( @url, method: :options ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'OPTIONS'
+                end
+            end
+            describe :delete do
+                it 'performs a POST HTTP request' do
+                    body = nil
+                    subject.request( @url, method: :delete ) { |res| body = res.body }
+                    subject.run
+                    body.should == 'DELETE'
+                end
+            end
+        end
+
+        describe :parameters do
+            it 'specifies the query params as a hash' do
+                body = nil
+                params = { 'param' => 'value' }
+                subject.request( @url + '/echo', parameters: params ) { |res| body = res.body }
+                subject.run
+                params.should eq YAML.load( body )
+            end
+
+            it 'preserves nullbytes' do
+                body = nil
+                params = { "pa\0ram" => "v\0alue" }
+                subject.request( @url + '/echo', parameters: params ) { |res| body = res.body }
+                subject.run
+                params.should eq YAML.load( body )
+            end
+        end
+
+        describe :body do
+            it 'properly encodes special characters' do
+                body = nil
+                params = { '% param\ +=&;' => '% value\ +=&;', 'nil' => nil }
+                subject.request( @url + '/echo', method: :post, body: params ) { |res| body = res.body }
+                subject.run
+                YAML.load( body ).should == { '% param\ +=&;' => '% value\ +=&;', 'nil' => '' }
+            end
+
+            it 'preserves nullbytes' do
+                body = nil
+                params = { "st\0uff" => "test\0" }
+                subject.request( @url + '/echo', method: :post, body: params, ) { |res| body = res.body }
+                subject.run
+                YAML.load( body ).should == params
+            end
+        end
+
+        describe :timeout do
+            describe 'nil' do
+                it 'runs without a timeout' do
+                    timed_out = false
+                    subject.request( @url + '/sleep' ) { |res| timed_out = res.timed_out? }
+                    subject.run
+                    timed_out.should be_false
+                end
+            end
+            describe Numeric do
+                it 'sets a timeout value in milliseconds' do
+                    timed_out = false
+                    subject.request( @url + '/sleep', timeout: 4_000 ) { |res| timed_out = res.timed_out? }
+                    subject.run
+                    timed_out.should be_true
+
+                    timed_out = false
+                    subject.request( @url + '/sleep', timeout: 6_000 ) { |res| timed_out = res.timed_out? }
+                    subject.run
+                    timed_out.should be_false
+                end
+            end
+        end
+
+        describe ':username/:password' do
+            it 'uses them to authenticate' do
+                # first fail to make sure that our test server is actually working properly
+                code = 0
+                subject.get( @opts.url + 'auth/weird-chars' ) { |res| code = res.code }
+                subject.run
+                code.should == 401
+
+                response = nil
+                subject.get(
+                    @opts.url + 'auth/weird-chars',
+                    username: 'u se rname$@#@#%$3#@%@#',
+                    password: 'p a  :wo\'rd$@#@#%$3#@%@#' ) { |res| response = res }
+                subject.run
+                response.code.should == 200
+                response.body.should == 'authenticated!'
+            end
+        end
+
+        describe :cookies do
+            it 'preserves nullbytess' do
+                cookies = { "name\0" => "val\0" }
+                body = nil
+                subject.request( @url + '/cookies', cookies: cookies ) { |res| body = res.body }
+                subject.run
+                YAML.load( body ).should == cookies
+            end
+
+            describe 'nil' do
+                it 'uses te cookies in the CookieJar' do
+                    @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                    subject.cookie_jar.cookies.should be_empty
+                    subject.reset
+
+                    body = nil
+                    subject.request( @url + '/cookies' ) { |res| body = res.body }
+                    subject.run
+                    YAML.load( body ).should == {
+                        'my_cookie_name' => 'val1',
+                        'blah_name' => 'val2',
+                        'another_name' => 'another_val'
+                    }
+                end
+
+                it 'only sends the appropriate cookies for the domain' do
+                    cookies = []
+                    cookies << Arachni::Element::Cookie.new(
+                        url:    'http://test.com',
+                        inputs: { 'key1' => 'val1' }
+                    )
+                    cookies << Arachni::Element::Cookie.new(
+                        url:    @url,
+                        inputs: { 'key2' => 'val2' }
+                    )
+
+                    subject.cookie_jar.update( cookies )
+                    body = nil
+                    subject.request( @url + '/cookies' ) { |res| body = res.body }
+                    subject.run
+                    YAML.load( body ).should == { 'key2' => 'val2' }
+                end
+            end
+
+            describe Hash do
+                it 'uses the key-value pairs as cookies' do
+                    cookies = { 'name' => 'val' }
+                    body = nil
+                    subject.request( @url + '/cookies', cookies: cookies ) { |res| body = res.body }
+                    subject.run
+                    YAML.load( body ).should == cookies
+                end
+
+                it 'merges them with the cookie-jar' do
+                    @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                    subject.cookie_jar.cookies.should be_empty
+                    subject.reset
+
+                    body = nil
+                    subject.request(
+                        @url + '/cookies',
+                        cookies: {
+                            'my_cookie_name' => 'updated_val'
+                        }
+                    ) { |res| body = res.body }
+                    subject.run
+
+                    YAML.load( body ).should == {
+                        'my_cookie_name' => 'updated_val',
+                        'blah_name' => 'val2',
+                        'another_name' => 'another_val'
+                    }
+                end
+
+                context 'when also given a Cookie header' do
+                    it 'merges them, giving priority to the Hash' do
+                        cookies = { 'name' => 'val' }
+                        options = {
+                            cookies: cookies,
+                            headers: {
+                                'Cookie' => 'test=1;name=2'
+                            }
+                        }
+
+                        body = nil
+                        subject.request( @url + '/cookies', options ) { |res| body = res.body }
+                        subject.run
+
+                        YAML.load( body ).should == { 'test' => '1', 'name' => 'val' }
+                    end
+                end
+            end
+        end
+
+        describe :mode do
+            describe 'nil' do
+                it 'performs the request asynchronously' do
+                    performed = false
+                    subject.request( @url ) { performed = true }
+                    subject.run
+                    performed.should be_true
+                end
+            end
+            describe :async do
+                it 'performs the request asynchronously' do
+                    performed = false
+                    subject.request( @url, mode: :sync ) { performed = true }
+                    subject.run
+                    performed.should be_true
+                end
+            end
+            describe :sync do
+                it 'performs the request synchronously and returns the response' do
+                    subject.request( @url, mode: :sync ).should be_kind_of Arachni::HTTP::Response
+                end
+
+                it 'assigns a #request to the returned response' do
+                    subject.request( @url, mode: :sync ).request.should be_kind_of Arachni::HTTP::Request
+                end
+
+                context 'when a block is given' do
+                    it 'passes the response to it as well' do
+                        called = []
+                        response = subject.request( @url, mode: :sync ) do |r|
+                            called << r
+                        end
+
+                        response.should be_kind_of Arachni::HTTP::Response
+                        called.should == [response]
+                    end
+                end
+            end
+        end
+
+        describe :headers do
+            describe 'nil' do
+                it 'uses the default headers' do
+                    body = nil
+                    subject.request( @url + '/headers' ) { |res| body = res.body }
+                    subject.run
+                    sent_headers = YAML.load( body )
+                    subject.headers.each { |k, v| sent_headers[k].should == v }
+                end
+            end
+
+            describe Hash do
+                it 'merges them with the default headers' do
+                    headers = { 'My-Header' => 'my value'}
+                    body = nil
+                    subject.request( @url + '/headers', headers: headers ) { |res| body = res.body }
+                    subject.run
+                    sent_headers = YAML.load( body )
+                    subject.headers.merge( headers ).each { |k, v| sent_headers[k].should == v }
+                end
+            end
+        end
+
+        describe :update_cookies do
+            describe 'nil' do
+                it 'skips the cookie_jar' do
+                    cookies = []
+                    cookies << Arachni::Element::Cookie.new(
+                        url: @url,
+                        inputs: { 'key2' => 'val2' }
+                    )
+                    subject.update_cookies( cookies )
+                    subject.request( @url + '/update_cookies' )
+                    subject.run
+                    subject.cookies.should == cookies
+                end
+            end
+
+            describe false do
+                it 'skips the cookie_jar' do
+                    cookies = []
+                    cookies << Arachni::Element::Cookie.new(
+                        url: @url,
+                        inputs: { 'key2' => 'val2' }
+                    )
+                    subject.update_cookies( cookies )
+                    subject.request( @url + '/update_cookies', update_cookies: false )
+                    subject.run
+                    subject.cookies.should == cookies
+                end
+            end
+
+            describe true do
+                it 'updates the cookie_jar' do
+                    cookies = []
+                    cookies << Arachni::Element::Cookie.new(
+                        url:    @url,
+                        name:   'key2',
+                        value:  'val2',
+                        domain: Arachni::URI( @url ).domain
+                    )
+                    subject.update_cookies( cookies )
+                    subject.request( @url + '/update_cookies', update_cookies: true )
+                    subject.run
+                    subject.cookies.first.value.should == cookies.first.value + ' [UPDATED!]'
+                end
+            end
+        end
+
+        describe :follow_location do
+            describe 'nil' do
+                it 'ignores redirects' do
+                    res = nil
+                    subject.request( @url + '/follow_location' ) { |c_res| res = c_res }
+                    subject.run
+                    res.url.start_with?( @url + '/follow_location' ).should be_true
+                    res.body.should == ''
+                end
+            end
+            describe false do
+                it 'ignores redirects' do
+                    res = nil
+                    subject.request( @url + '/follow_location', follow_location: false ) { |c_res| res = c_res }
+                    subject.run
+                    res.url.start_with?( @url + '/follow_location' ).should be_true
+                    res.body.should == ''
+                end
+            end
+            describe true do
+                it 'follows redirects' do
+                    res = nil
+                    subject.request( @url + '/follow_location', follow_location: true ) { |c_res| res = c_res }
+                    subject.run
+                    res.url.should == @url + '/redir_2'
+                    res.body.should == "Welcome to redir_2!"
+                end
+            end
+        end
+
+        context 'when cookie-jar lookup fails' do
+            it 'only uses the given cookies' do
+                @opts.http.cookie_string = 'my_cookie_name=val1;blah_name=val2;another_name=another_val'
+                subject.cookie_jar.cookies.should be_empty
+                subject.reset
+                subject.cookie_jar.cookies.should be_any
+
+                subject.cookie_jar.stub(:for_url) { raise }
+
+                body = nil
+                subject.request(
+                    @url + '/cookies',
+                    cookies: { 'blah' => 'val' }
+                ) { |res| body = res.body }
+                subject.run
+
+                YAML.load( body ).should == { 'blah' => 'val' }
+            end
+        end
+    end
+
+    describe '#get' do
+        it 'queues a GET request' do
+            body = nil
+            subject.get { |res| body = res.body }
+            subject.run
+            body.should == 'GET'
+        end
+    end
+
+    describe '#trace' do
+        it 'queues a TRACE request' do
+            subject.trace.method.should == :trace
+        end
+    end
+
+    describe '#post' do
+        it 'queues a POST request' do
+            body = nil
+            subject.post { |res| body = res.body }
+            subject.run
+            body.should == 'POST'
+        end
+
+        it 'passes :parameters as a #request :body' do
+            body = nil
+            params = { '% param\ +=&;' => '% value\ +=&;', 'nil' => nil }
+            subject.post( @url + '/echo', parameters: params ) { |res| body = res.body }
+            subject.run
+            YAML.load( body ).should == { '% param\ +=&;' => '% value\ +=&;', 'nil' => '' }
+        end
+    end
+
+    describe '#cookie' do
+        it 'queues a GET request' do
+            body = nil
+            cookies = { 'name' => "v%+;al\00=" }
+            subject.cookie( @url + '/cookies', parameters: cookies ) { |res| body = res.body }
+            subject.run
+            YAML.load( body ).should == cookies
+        end
+    end
+
+    describe '#headers' do
+        it 'queues a GET request' do
+            body = nil
+            headers = { 'name' => 'val' }
+            subject.header( @url + '/headers', parameters: headers ) { |res| body = res.body }
+            subject.run
+            YAML.load( body )['Name'].should == headers.values.first
+        end
+    end
+
+    describe '#queue' do
+        it 'queues a request' do
+            r = nil
+
+            request = Arachni::HTTP::Request.new( url: @url )
+            request.on_complete do |response|
+                r = response
+            end
+
+            subject.queue request
+            subject.run
+
+            r.should be_kind_of Arachni::HTTP::Response
+        end
+    end
+
+    describe '#update_cookies' do
+        it 'updates the cookies' do
+            cookies = []
+            cookies << Arachni::Element::Cookie.new(
+                url: @url,
+                inputs: { 'key2' => 'val2' }
+            )
+
+            subject.cookies.should be_empty
+            subject.update_cookies( cookies )
+            subject.cookies.should == cookies
+        end
+    end
+
+    describe '#on_new_cookies' do
+        it 'adds blocks to be called when new cookies arrive' do
+            cookies = []
+            cookies << Arachni::Element::Cookie.new(
+                url:    @url,
+                inputs: { 'name' => 'value' }
+            )
+            res = Arachni::HTTP::Response.new( url: @url, headers: { 'Set-Cookie' => 'name=value' } )
+
+            callback_cookies  = nil
+            callback_response = nil
+            subject.on_new_cookies do |cookies, res|
+                callback_cookies  = cookies
+                callback_response = res
+            end
+            subject.parse_and_set_cookies( res )
+
+            callback_cookies.should == cookies
+            callback_response.should == res
+        end
+    end
+
+    describe '#parse_and_set_cookies' do
+        it 'updates the cookies from a response and call on_new_cookies blocks' do
+            cookies = []
+            cookies << Arachni::Element::Cookie.new(
+                url:    @url,
+                inputs: { 'name' => 'value' }
+            )
+            res = Arachni::HTTP::Response.new( url: @url, headers: { 'Set-Cookie' => 'name=value' } )
+
+            @opts.http.cookies.should be_empty
+            subject.cookies.should be_empty
+            subject.parse_and_set_cookies( res )
+            subject.cookies.should == cookies
+        end
+    end
+
+    describe '#custom_404?' do
+        before { @custom_404 = @url + '/custom_404/' }
+
+        context 'when not dealing with a custom 404 handler' do
+            it 'returns false' do
+                res = nil
+                subject.get( @custom_404 + 'not' ) { |c_res| res = c_res }
+                subject.run
+                bool = false
+                subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                subject.run
+                bool.should be_false
+            end
+        end
+
+        context 'when dealing with a static handler' do
+            it 'returns true' do
+                res = nil
+                subject.get( @custom_404 + 'static/crap' ) { |c_res| res = c_res }
+                subject.run
+                bool = false
+                subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                subject.run
+                bool.should be_true
+            end
+        end
+
+        context 'when dealing with a dynamic handler' do
+            context 'which includes the requested resource in the response' do
+                it 'returns true' do
+                    res = nil
+                    subject.get( @custom_404 + 'dynamic/crap' ) { |c_res| res = c_res }
+                    subject.run
+                    bool = false
+                    subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                    subject.run
+                    bool.should be_true
+                end
+            end
+            context 'which includes constantly changing text in the response' do
+                it 'returns true' do
+                    res = nil
+                    subject.get( @custom_404 + 'random/crap' ) { |c_res| res = c_res }
+                    subject.run
+                    bool = false
+                    subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                    subject.run
+                    bool.should be_true
+                end
+            end
+            context 'which returns a combination of the above' do
+                it 'returns true' do
+                    res = nil
+                    subject.get( @custom_404 + 'combo/crap' ) { |c_res| res = c_res }
+                    subject.run
+                    bool = false
+                    subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                    subject.run
+                    bool.should be_true
+                end
+            end
+        end
+
+        context 'when checking for an already checked URL' do
+            it 'returns the cached result' do
+                res = nil
+                subject.get( @custom_404 + 'static/crap' ) { |c_res| res = c_res }
+                subject.run
+
+                bool = false
+                subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                subject.run
+                bool.should be_true
+
+                fingerprints = 0
+                subject.on_complete do
+                    fingerprints += 1
+                end
+
+                res = nil
+                subject.get( @custom_404 + 'static/crap' ) { |c_res| res = c_res }
+                subject.run
+                fingerprints.should > 0
+
+                overhead = 0
+                subject.on_complete do
+                    overhead += 1
+                end
+
+                bool = false
+                subject.custom_404?( res ) { |c_bool| bool = c_bool }
+                subject.run
+                bool.should be_true
+
+                overhead.should == 0
+            end
+        end
+
+        context "when the 404 cache exceeds #{described_class::CUSTOM_404_CACHE_SIZE} entries" do
+            it 'it is pruned as soon as possible' do
+                subject._404_cache.should be_empty
+
+                (2 * described_class::CUSTOM_404_CACHE_SIZE).times do |i|
+                    subject.get( @custom_404 + "static/#{i}/test" ) do |response|
+                        subject.custom_404?( response ) {}
+                    end
+                end
+                subject.run
+
+                subject._404_cache.size.should == described_class::CUSTOM_404_CACHE_SIZE
+            end
+        end
+    end
+
+    describe '#checked_but_not_custom_404?' do
+        let(:url) { custom_404_url + 'combo/crap' }
+        let(:path) { subject.get_path( url ) }
+
+        context 'when the page has been fingerprinted for a custom 404 handler' do
+            context 'and it has a custom handler' do
+                it 'returns false' do
+                    subject.get( url ) do |response|
+                        subject.custom_404?( response ) {}
+                    end
+                    subject.run
+
+                    subject.checked_but_not_custom_404?( path ).should be_false
+                end
+            end
+
+            context 'and it does not have a custom handler' do
+                it 'returns true' do
+                    subject.get( @url ) do |response|
+                        subject.custom_404?( response ) {}
+                    end
+                    subject.run
+
+                    subject.checked_but_not_custom_404?( subject.get_path( @url ) ).should be_true
+                end
+            end
+        end
+
+        context 'when the page has not been fingerprinted for a custom 404 handler' do
+            it 'returns false' do
+                subject.checked_but_not_custom_404?( path ).should be_false
+            end
+        end
+    end
+
+    describe '#checked_for_custom_404?' do
+        let(:url) { custom_404_url + 'combo/crap' }
+
+        context 'when the page has been fingerprinted for a custom 404 handler' do
+            context 'and it has a custom handler' do
+                it 'returns true' do
+                    subject.get( url ) do |response|
+                        subject.custom_404?( response ) {}
+                    end
+                    subject.run
+
+                    subject.checked_for_custom_404?( url ).should be_true
+                end
+            end
+
+            context 'and it does not have a custom handler' do
+                it 'returns true' do
+                    subject.get( @url ) do |response|
+                        subject.custom_404?( response ) {}
+                    end
+                    subject.run
+
+                    subject.checked_for_custom_404?( @url ).should be_true
+                end
+            end
+        end
+
+        context 'when the page has not been fingerprinted for a custom 404 handler' do
+            it 'returns false' do
+                subject.checked_for_custom_404?( url ).should be_false
+            end
+        end
+    end
+
+    describe 'needs_custom_404_check?' do
+        context 'when #checked_for_custom_404?' do
+            context false do
+                before(:each) { subject.instance.stub(:checked_for_custom_404?) { false } }
+
+                it 'returns true' do
+                    subject.needs_custom_404_check?( @url ).should be_true
+                end
+
+                context 'and #checked_but_not_custom_404?' do
+                    context false do
+                        before(:each) { subject.instance.stub(:checked_but_not_custom_404?) { false } }
+
+                        it 'returns true' do
+                            subject.needs_custom_404_check?( @url ).should be_true
+                        end
+                    end
+
+                    context true do
+                        before(:each) { subject.instance.stub(:checked_but_not_custom_404?) { true } }
+
+                        it 'returns true' do
+                            subject.needs_custom_404_check?( @url ).should be_true
+                        end
+                    end
+                end
+            end
+
+            context true do
+                before(:each) { subject.instance.stub(:checked_for_custom_404?) { true } }
+
+                it 'returns true' do
+                    subject.needs_custom_404_check?( @url ).should be_true
+                end
+
+                context 'and #checked_but_not_custom_404?' do
+                    context true do
+                        before(:each) { subject.instance.stub(:checked_but_not_custom_404?) { true } }
+
+                        it 'returns false' do
+                            subject.needs_custom_404_check?( @url ).should be_false
+                        end
+                    end
+
+                    context false do
+                        before(:each) { subject.instance.stub(:checked_but_not_custom_404?) { false } }
+
+                        it 'returns true' do
+                            subject.needs_custom_404_check?( @url ).should be_true
+                        end
+                    end
+                end
+            end
+        end
+    end
+
+    describe '.info' do
+        it 'returns a hash with an output name' do
+            described_class.info[:name].should == 'HTTP'
+        end
+    end
+
+end