arachni 0.4.7 → 1.0

data/spec/arachni/uri/scope_spec.rb

@@ -0,0 +1,416 @@
+require 'spec_helper'
+
+describe Arachni::URI::Scope do
+
+    before :each do
+        Arachni::Options.reset
+    end
+
+    let(:scope){ Arachni::Options.scope }
+
+    describe '#too_deep?' do
+        subject { Arachni::URI.parse( '/very/very/very/very/deep' ).scope }
+
+        context 'when the directory depth of the URL is' do
+            context "less than #{Arachni::OptionGroups::Scope}#directory_depth_limit" do
+                it 'returns false' do
+                    scope.directory_depth_limit = 100
+                    subject.too_deep?.should be_false
+                end
+            end
+
+            context "less than #{Arachni::OptionGroups::Scope}#directory_depth_limit" do
+                it 'returns true' do
+                    scope.directory_depth_limit = 2
+                    subject.too_deep?.should be_true
+                end
+            end
+        end
+
+        context "when #{Arachni::OptionGroups::Scope}#directory_depth_limit has not been configured" do
+            it 'returns false' do
+                subject.too_deep?.should be_false
+            end
+        end
+    end
+
+    describe '#redundant?' do
+        subject { Arachni::URI.parse( 'http://stuff.com/match_this' ).scope }
+
+        context "when a URL's counter reaches 0" do
+            it 'returns true' do
+                scope.redundant_path_patterns = { /match_this/ => 10 }
+
+                10.times do
+                    subject.redundant?.should be_false
+                end
+
+                subject.redundant?.should be_true
+            end
+        end
+        context "when a URL's counter has not reached 0" do
+            it 'returns false' do
+                scope.redundant_path_patterns = { /match_this/ => 11 }
+
+                10.times do
+                    subject.redundant?.should be_false
+                end
+
+                subject.redundant?.should be_false
+            end
+        end
+
+        context 'when #auto_redundant returns true' do
+            it 'returns true' do
+                subject.stub(:auto_redundant?) { true }
+                subject.should be_redundant
+            end
+        end
+    end
+
+    describe '#auto_redundant?' do
+        subject { Arachni::URI( 'http://test.com/?test=2&test2=2').scope }
+
+        context 'when #auto_redundant_paths limit has been reached' do
+            it 'returns true' do
+                scope.auto_redundant_paths = 10
+
+                subject.auto_redundant?.should be_false
+                9.times do
+                    subject.auto_redundant?.should be_false
+                end
+
+                subject.auto_redundant?.should be_true
+            end
+        end
+
+        describe 'by default' do
+            it 'returns false' do
+                subject.auto_redundant?.should be_false
+            end
+        end
+    end
+
+    describe '#exclude?' do
+        subject { Arachni::URI.parse( 'http://test.com/exclude/' ).scope }
+
+        context 'when self matches the provided exclude rules' do
+            it 'returns true' do
+                scope.exclude_path_patterns = [ /exclude/ ]
+
+                subject.exclude?.should be_true
+            end
+        end
+
+        context 'when self does not match the provided exclude rules' do
+            it 'returns false' do
+                scope.exclude_path_patterns = [ /boo/ ]
+
+                subject.exclude?.should be_false
+            end
+        end
+    end
+
+    describe '#include?' do
+        subject { Arachni::URI.parse( 'http://test.com/include/' ).scope }
+
+        context 'when self matches the provided include rules in' do
+            it 'returns true' do
+                scope.include_path_patterns = [ /include/ ]
+                subject.include?.should be_true
+            end
+        end
+
+        context 'when self does not match the provided scope_include_path_patterns rules in' do
+            it 'returns false' do
+                scope.include_path_patterns = [ /boo/ ]
+                subject.include?.should be_false
+            end
+        end
+    end
+
+    describe '#in_domain?' do
+        let(:url_without_subdomain) { 'http://test.com' }
+        let(:url_with_subdomain) { 'http://boo.test.com' }
+        let(:url_with_same_subdomain) { 'http://boo.test.com/stuff' }
+        let(:url_with_different_subdomain) { 'http://boo2.test.com/stuff' }
+
+        let(:with_subdomain) { Arachni::URI.parse( url_with_subdomain ).scope }
+        let(:with_same_subdomain) { Arachni::URI.parse( url_with_same_subdomain ).scope }
+        let(:with_different_subdomain) { Arachni::URI.parse( url_with_different_subdomain ).scope }
+        let(:without_subdomain) { Arachni::URI.parse( url_without_subdomain ).scope }
+
+        context "when #{Arachni::OptionGroups::Scope}#include_subdomains is" do
+            context true do
+                before :each do
+                    scope.include_subdomains = true
+                end
+
+                context "when #{Arachni::Options}#url" do
+                    context 'has a subdomain' do
+                        before :each do
+                            Arachni::Options.url = url_with_subdomain
+                        end
+
+                        context 'and the url has a different subdomain' do
+                            it 'return true' do
+                                with_different_subdomain.in_domain?.should be_true
+                            end
+                        end
+
+                        context 'and the url has the same subdomain' do
+                            it 'return true' do
+                                with_same_subdomain.in_domain?.should be_true
+                            end
+                        end
+
+                        context 'and the url has no subdomain' do
+                            it 'return true' do
+                                without_subdomain.in_domain?.should be_true
+                            end
+                        end
+                    end
+
+                    context 'has no a subdomain' do
+                        before :each do
+                            Arachni::Options.url = url_without_subdomain
+                        end
+
+                        context 'and the url has a subdomain' do
+                            it 'return true' do
+                                with_subdomain.in_domain?.should be_true
+                            end
+                        end
+
+                        context 'and the url has no subdomain' do
+                            it 'return true' do
+                                without_subdomain.in_domain?.should be_true
+                            end
+                        end
+                    end
+                end
+            end
+
+            context false do
+                before :each do
+                    scope.include_subdomains = false
+                end
+
+                context "when #{Arachni::Options}#url" do
+                    context 'has a subdomain' do
+                        before :each do
+                            Arachni::Options.url = url_with_subdomain
+                        end
+
+                        context 'and the url has a different subdomain' do
+                            it 'return false' do
+                                with_different_subdomain.in_domain?.should be_false
+                            end
+                        end
+
+                        context 'and the url has the same subdomain' do
+                            it 'return true' do
+                                with_same_subdomain.in_domain?.should be_true
+                            end
+                        end
+
+                        context 'and the url has no subdomain' do
+                            it 'return false' do
+                                without_subdomain.in_domain?.should be_false
+                            end
+                        end
+                    end
+
+                    context 'has no a subdomain' do
+                        before :each do
+                            Arachni::Options.url = url_without_subdomain
+                        end
+
+                        context 'and the url has a subdomain' do
+                            it 'return false' do
+                                with_subdomain.in_domain?.should be_false
+                            end
+                        end
+
+                        context 'and the url has no subdomain' do
+                            it 'return true' do
+                                without_subdomain.in_domain?.should be_true
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+
+    describe '#follow_protocol?' do
+        let(:http) { Arachni::URI.parse( 'http://test2.com/blah/ha' ).scope }
+        let(:https) { Arachni::URI.parse( 'https://test2.com/blah/ha' ).scope }
+        let(:other) { Arachni::URI.parse( 'stuff://test2.com/blah/ha' ).scope }
+
+        context 'when the reference URL uses' do
+            context 'HTTPS' do
+                before :each do
+                    Arachni::Options.url = 'https://test2.com/blah/ha'
+                end
+
+                context 'and the checked URL uses' do
+                    context 'HTTPS' do
+                        context 'and Options#scope_https_only is' do
+                            context true do
+                                it 'returns true' do
+                                    scope.https_only = true
+                                    https.follow_protocol?.should be_true
+                                end
+                            end
+
+                            context false do
+                                it 'returns true' do
+                                    scope.https_only = false
+                                    https.follow_protocol?.should be_true
+                                end
+                            end
+                        end
+                    end
+
+                    context 'HTTP' do
+                        context 'and Options#scope_https_only is' do
+                            context true do
+                                it 'returns false' do
+                                    scope.https_only = true
+                                    http.follow_protocol?.should be_false
+                                end
+                            end
+
+                            context false do
+                                it 'returns true' do
+                                    scope.https_only = false
+                                    http.follow_protocol?.should be_true
+                                end
+                            end
+                        end
+                    end
+                end
+            end
+
+            context 'HTTP' do
+                before :each do
+                    Arachni::Options.url = 'http://test2.com/blah/ha'
+                end
+
+                context 'and the checked URL uses' do
+                    context 'HTTPS' do
+                        context 'and Options#scope_https_only is' do
+                            context true do
+                                it 'returns true' do
+                                    scope.https_only = true
+                                    https.follow_protocol?.should be_true
+                                end
+                            end
+
+                            context false do
+                                it 'returns true' do
+                                    scope.https_only = false
+                                    https.follow_protocol?.should be_true
+                                end
+                            end
+                        end
+                    end
+                    context 'HTTP' do
+                        context 'and Options#scope_https_only is' do
+                            context true do
+                                it 'returns true' do
+                                    scope.https_only = true
+                                    http.follow_protocol?.should be_true
+                                end
+                            end
+
+                            context false do
+                                it 'returns true' do
+                                    scope.https_only = false
+                                    http.follow_protocol?.should be_true
+                                end
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+
+    describe '#in?' do
+        subject { Arachni::URI.parse( 'http://stuff/' ).scope }
+
+        it 'returns true' do
+            subject.should be_in
+        end
+
+        context 'when #out?' do
+            context 'is true' do
+                it 'returns false' do
+                    subject.stub(:out?) { true }
+                    subject.should_not be_in
+                end
+            end
+        end
+    end
+
+    describe '#out?' do
+        subject { Arachni::URI.parse( 'http://stuff/' ).scope }
+
+        it 'returns false' do
+            subject.should_not be_out
+        end
+
+        it 'does not call #redundant?' do
+            subject.should_not receive(:redundant?)
+            subject.out?
+        end
+
+        context 'when #follow_protocol?' do
+            context 'is false' do
+                it 'returns true' do
+                    subject.stub(:follow_protocol?) { false }
+                    subject.should be_out
+                end
+            end
+        end
+
+        context 'when #in_domain?' do
+            context 'is false' do
+                it 'returns true' do
+                    subject.stub(:in_domain?) { false }
+                    subject.should be_out
+                end
+            end
+        end
+
+        context 'when #too_deep?' do
+            context 'is true' do
+                it 'returns true' do
+                    subject.stub(:too_deep?) { true }
+                    subject.should be_out
+                end
+            end
+        end
+
+        context 'when #include?' do
+            context 'is false' do
+                it 'returns true' do
+                    subject.stub(:include?) { false }
+                    subject.should be_out
+                end
+            end
+        end
+
+        context 'when #exclude?' do
+            context 'is true' do
+                it 'returns true' do
+                    subject.stub(:exclude?) { true }
+                    subject.should be_out
+                end
+            end
+        end
+    end
+
+end

data/spec/arachni/uri_spec.rb

@@ -90,11 +90,13 @@ describe Arachni::URI do
         @ref_normalizer = proc do |p|
             @normalized[p]
         end
-
-        @uri = Arachni::URI
     end
 
-    before { @opts = Arachni::Options.instance.reset }
+    before(:each) { @opts = Arachni::Options.instance.reset }
+
+    let(:rewrite_rules) do
+        { /articles\/[\w-]+\/(\d+)/ => 'articles.php?id=\1' }
+    end
 
     describe '.URI' do
         it 'parses and normalize the give string' do
@@ -106,23 +108,72 @@ describe Arachni::URI do
         end
     end
 
+    describe '.rewrite' do
+        let(:url) { 'http://test.com/articles/some-stuff/23' }
+
+        it 'rewrites a URL based on the given rules' do
+            described_class.rewrite( url, rewrite_rules ).should ==
+                'http://test.com/articles.php?id=23'
+        end
+
+        context 'when no rules are provided' do
+            it "uses the ones in #{Arachni::OptionGroups::Scope}#url_rewrites" do
+                Arachni::Options.scope.url_rewrites = rewrite_rules
+
+                described_class.rewrite( url ).should ==
+                    'http://test.com/articles.php?id=23'
+            end
+        end
+    end
+
+    describe '.parse_query' do
+        it 'returns the query parameters as a Hash' do
+            url = 'http://test/?param_one=value_one&param_two=value_two'
+            described_class.parse_query( url ).should == {
+                'param_one' => 'value_one',
+                'param_two' => 'value_two'
+            }
+        end
+
+        it 'decodes the parameters' do
+            url = 'http://test/?stuff%20here=bl%20ah'
+            described_class.parse_query( url ).should == {
+                'stuff here' => 'bl ah'
+            }
+        end
+
+        context 'when passed' do
+            describe 'nil' do
+                it 'returns an empty Hash' do
+                    described_class.parse_query( nil ).should == {}
+                end
+            end
+            describe 'an unparsable URL' do
+                it 'returns an empty Hash' do
+                    url = '$#%^$6#5436#$%^'
+                    described_class.parse_query( url ).should == {}
+                end
+            end
+        end
+    end
+
     describe '.encode' do
         it 'decodes a URI' do
             uri = "my test.asp?name=ståle&car=saab"
-            @uri.encode( uri ).should == 'my%20test.asp?name=st%C3%A5le&car=saab'
+            described_class.encode( uri ).should == 'my%20test.asp?name=st%C3%A5le&car=saab'
         end
     end
 
     describe '.decode' do
         it 'decodes a URI' do
             uri = 'my%20test.asp?name=st%C3%A5le&car=saab'
-            @uri.decode( uri ).should == "my test.asp?name=ståle&car=saab"
+            described_class.decode( uri ).should == "my test.asp?name=ståle&car=saab"
         end
     end
 
     describe '.parser' do
         it 'returns a URI::Parser' do
-            @uri.parser.class.should == ::URI::Parser
+            described_class.parser.class.should == ::URI::Parser
         end
     end
 
@@ -137,7 +188,7 @@ describe Arachni::URI do
 
             uri = "#{scheme}://#{user}:#{password}@#{host}#{path}?#{query}"
 
-            parsed_uri = @uri.parse( uri )
+            parsed_uri = described_class.parse( uri )
 
             parsed_uri.to_s.should == uri
 
@@ -148,17 +199,36 @@ describe Arachni::URI do
             parsed_uri.path.should == path
             parsed_uri.query.should == query
         end
+
+        it 'ignores javascript: URLs' do
+            described_class.parse( 'javascript:stuff()' ).should be_nil
+            described_class.parse( 'jAvaScRipT:stuff()' ).should be_nil
+        end
     end
 
     describe '.ruby_parse' do
         it 'cleans the URL' do
             @urls.each do |url|
-                @uri.ruby_parse( url ).to_s.should == @ref_normalizer.call( url )
+                described_class.ruby_parse( url ).to_s.should == @ref_normalizer.call( url )
+            end
+        end
+
+        it 'ignores javascript: URLs' do
+            described_class.ruby_parse( 'javascript:stuff()' ).should be_nil
+            described_class.ruby_parse( 'jAvaScRipT:stuff()' ).should be_nil
+        end
+
+        context 'when an error occurs' do
+            it 'returns nil' do
+                described_class.stub(:fast_parse){ raise }
+                described_class.stub(:normalize){ raise }
+
+                described_class.ruby_parse( 'http://test.com/222' ).should be_nil
             end
         end
     end
 
-    describe '.cheap_parse' do
+    describe '.fast_parse' do
         it 'parses a URI and return its components as a hash' do
             scheme   = 'http'
             user     = 'user'
@@ -169,7 +239,7 @@ describe Arachni::URI do
 
             uri = "#{scheme}://#{user}:#{password}@#{host}/#{path}?#{query}"
 
-            parsed_uri = @uri.cheap_parse( uri )
+            parsed_uri = described_class.fast_parse( uri )
 
             parsed_uri[:scheme].should == scheme
             parsed_uri[:userinfo].should == user + ':' + password
@@ -177,7 +247,7 @@ describe Arachni::URI do
             parsed_uri[:path].should == path
             parsed_uri[:query].should == query
 
-            parsed_uri = @uri.cheap_parse( "//#{user}:#{password}@#{host}/#{path}?#{query}" )
+            parsed_uri = described_class.fast_parse( "//#{user}:#{password}@#{host}/#{path}?#{query}" )
 
             parsed_uri[:scheme].should be_nil
             parsed_uri[:userinfo].should == user + ':' + password
@@ -187,11 +257,15 @@ describe Arachni::URI do
         end
 
         it 'returns a frozen hash (with frozen values)' do
-            h = @uri.cheap_parse( 'http://test.com/stuff/' )
+            h = described_class.fast_parse( 'http://test.com/stuff/' )
 
             expect { h[:stuff] = 0 }.to raise_error
             expect { h[:path] << '/' }.to raise_error
         end
+
+        it 'ignores javascript: URLs' do
+            described_class.fast_parse( 'javascript:stuff()' ).should be_nil
+        end
     end
 
     describe '.to_absolute' do
@@ -199,22 +273,22 @@ describe Arachni::URI do
             abs  = 'http://test.com/blah/ha'
             rel  = '/test'
             rel2 = 'test2'
-            @uri.to_absolute( rel, abs ).should == "http://test.com" + rel
-            @uri.to_absolute( rel2, abs ).should == "http://test.com/blah/" + rel2
-            @uri.to_absolute( rel2, abs + '/' ).should == "http://test.com/blah/ha/" + rel2
+            described_class.to_absolute( rel, abs ).should == "http://test.com" + rel
+            described_class.to_absolute( rel2, abs ).should == "http://test.com/blah/" + rel2
+            described_class.to_absolute( rel2, abs + '/' ).should == "http://test.com/blah/ha/" + rel2
 
             rel = '//domain-name.com/stuff'
-            @uri.to_absolute( rel, abs ).should == "http:" + rel
+            described_class.to_absolute( rel, abs ).should == "http:" + rel
         end
     end
 
     describe '.normalize' do
         it 'cleans the URL' do
             @urls.each do |url|
-                @uri.normalize( url ).should == @ref_normalizer.call( url )
+                described_class.normalize( url ).should == @ref_normalizer.call( url )
             end
             with_whitespace = 'http://test.com/stuff '
-            @uri.normalize( with_whitespace ).to_s.should == with_whitespace.strip
+            described_class.normalize( with_whitespace ).to_s.should == with_whitespace.strip
         end
     end
 
@@ -222,7 +296,7 @@ describe Arachni::URI do
         context String do
             it 'normalizes and parse the string' do
                 @urls.each do |url|
-                    uri = @uri.new( url )
+                    uri = described_class.new( url )
                     uri.is_a?( Arachni::URI ).should be_true
                     uri.to_s.should == @ref_normalizer.call( url )
                 end
@@ -232,7 +306,7 @@ describe Arachni::URI do
         context Hash do
             it 'normalizes and construct a URI from a Hash of components' do
                 @urls.each do |url|
-                    uri = @uri.new( @uri.cheap_parse( url ) )
+                    uri = described_class.new( described_class.fast_parse( url ) )
                     uri.is_a?( Arachni::URI ).should be_true
                     uri.to_s.should == @ref_normalizer.call( url )
                 end
@@ -242,10 +316,10 @@ describe Arachni::URI do
         context URI do
             it 'normalizes and construct a URI from a Hash of components' do
                 @urls.each do |url|
-                    uri = ::URI.parse( @uri.normalize( url ) )
+                    uri = ::URI.parse( described_class.normalize( url ) )
                     uri.is_a?( ::URI ).should be_true
 
-                    a_uri = @uri.new( url )
+                    a_uri = described_class.new( url )
                     a_uri.is_a?( Arachni::URI ).should be_true
                     a_uri.to_s.should == @ref_normalizer.call( url )
                 end
@@ -255,8 +329,8 @@ describe Arachni::URI do
         context Arachni::URI do
             it 'normalizes and construct a URI from a Hash of components' do
                 @urls.each do |url|
-                    uri = @uri.new( url )
-                    a_uri = @uri.new( uri )
+                    uri = described_class.new( url )
+                    a_uri = described_class.new( uri )
                     a_uri.is_a?( Arachni::URI ).should be_true
                     a_uri.should == uri
                 end
@@ -264,8 +338,8 @@ describe Arachni::URI do
         end
 
         context 'else' do
-            it 'raises a TypeError' do
-                expect { @uri.new( [] ) }.to raise_error TypeError
+            it 'raises a ArgumentError' do
+                expect { described_class.new( [] ) }.to raise_error ArgumentError
             end
         end
     end
@@ -273,11 +347,11 @@ describe Arachni::URI do
     describe '#==' do
         it 'converts both objects to strings and compare them' do
             @urls.each do |url|
-                normalized_str = @uri.normalize( url )
+                normalized_str = described_class.normalize( url )
                 uri = ::URI.parse( normalized_str )
                 uri.is_a?( ::URI ).should be_true
 
-                a_uri = @uri.new( url )
+                a_uri = described_class.new( url )
                 a_uri.is_a?( Arachni::URI ).should be_true
 
                 a_uri.should == uri
@@ -287,49 +361,99 @@ describe Arachni::URI do
         end
     end
 
+    describe '#query=' do
+        subject { described_class.new( 'http://test.com/?my=val' ) }
+
+        it 'sets the URL query' do
+            subject.query = 'my2=val2'
+            subject.query.should == 'my2=val2'
+        end
+
+        context 'when given an empty string' do
+            it 'removes the query' do
+                subject.query = ''
+                subject.query.should be_nil
+            end
+        end
+
+        context 'when given nil' do
+            it 'removes the query' do
+                subject.query = ''
+                subject.query.should be_nil
+            end
+        end
+    end
+
+    describe '#dup' do
+        subject { described_class.new( 'http://test.com/?my=val' ) }
+
+        it 'return a duplicate object' do
+            dupped = subject.dup
+
+            subject.should == dupped
+            subject.object_id.should_not == dupped.object_id
+        end
+    end
+
+    describe '#_dump' do
+        it 'returns the URL as a string' do
+            uri = 'http://test.com/?my=val'
+            described_class.new( uri )._dump(nil).should == uri
+        end
+    end
+
+    describe '._load' do
+        it 'restores the original object from #_dump' do
+            uri    = 'http://test.com/?my=val'
+            parsed = described_class.new( uri )
+
+            described_class._load( parsed._dump(nil) ).should == parsed
+        end
+    end
+
     describe '#to_absolute' do
         it 'converts a self to absolute using the reference URL' do
             abs  = 'http://test.com/blah/ha'
             rel  = '/test'
             rel2 = 'test2'
-            @uri.parse( rel ).to_absolute( abs ).should == "http://test.com" + rel
-            @uri.parse( rel2 ).to_absolute( abs ).should == "http://test.com/blah/" + rel2
-            @uri.parse( rel2 ).to_absolute( abs + '/' ).should == "http://test.com/blah/ha/" + rel2
+            described_class.parse( rel ).to_absolute( abs ).should == "http://test.com" + rel
+            described_class.parse( rel2 ).to_absolute( abs ).should == "http://test.com/blah/" + rel2
+            described_class.parse( rel2 ).to_absolute( abs + '/' ).should == "http://test.com/blah/ha/" + rel2
         end
     end
 
-    describe '#up_p_to_path' do
+    describe '#up_to_path' do
         it 'returns the URL up to its path component (no resource name, query, fragment, etc)' do
             url = 'http://test.com/path/goes/here.php?query=goes&here=.!#frag'
-            @uri.parse( url ).up_to_path.should == 'http://test.com/path/goes/'
+            described_class.parse( url ).up_to_path.should == 'http://test.com/path/goes/'
 
             url = 'http://test.com/path/goes/here/?query=goes&here=.!#frag'
-            @uri.parse( url ).up_to_path.should == 'http://test.com/path/goes/here/'
+            described_class.parse( url ).up_to_path.should == 'http://test.com/path/goes/here/'
 
             url = 'http://test.com/path/goes/here?query=goes&here=.!#frag'
-            @uri.parse( url ).up_to_path.should == 'http://test.com/path/goes/here/'
+            described_class.parse( url ).up_to_path.should == 'http://test.com/path/goes/here/'
 
             url = 'http://test.com'
-            @uri.parse( url ).up_to_path.should == 'http://test.com/'
+            described_class.parse( url ).up_to_path.should == 'http://test.com/'
 
             url = 'http://test.com/'
-            @uri.parse( url ).up_to_path.should == 'http://test.com/'
+            described_class.parse( url ).up_to_path.should == 'http://test.com/'
         end
     end
 
     describe '#domain' do
         it 'removes the deepest subdomain from the host' do
             url = 'http://test.com/'
-            @uri.parse( url ).domain.should == 'test.com'
+            described_class.parse( url ).domain.should == 'test.com'
 
             url = 'http://test/'
-            @uri.parse( url ).domain.should == 'test'
+            described_class.parse( url ).domain.should == 'test'
 
             url = 'http://subdomain.test.com/'
-            @uri.parse( url ).domain.should == 'test.com'
+            described_class.parse( url ).domain.should == 'test.com'
 
             url = 'http://deep.subdomain.test.com/'
-            @uri.parse( url ).domain.should == 'subdomain.test.com'
+            described_class.parse( url ).domain.should == 'subdomain.test.com'
         end
     end
 
@@ -337,183 +461,51 @@ describe Arachni::URI do
         context 'when passed a URL with' do
             context 'a domain name' do
                 it 'returns false' do
-                    @uri.parse( 'http://stuff.com/blah' ).ip_address?.should be_false
+                    described_class.parse( 'http://stuff.com/blah' ).ip_address?.should be_false
                 end
             end
 
             context 'an IP address' do
                 it 'returns the IP address' do
-                    @uri.parse( 'http://127.0.0.1/blah/' ).ip_address?.should be_true
-                end
-            end
-        end
-    end
-
-    describe '#too_deep?' do
-        before { @deep_url = @uri.parse( '/very/very/very/very/deep' ) }
-
-        context 'when the directory depth of the URL\'s path is' do
-            context 'not greater than the provided depth' do
-                it 'returns false' do
-                    @deep_url.too_deep?( -1 ).should be_false
-
-                    @opts.depth_limit = 100
-                    @deep_url.too_deep?( 100 ).should be_false
+                    described_class.parse( 'http://127.0.0.1/blah/' ).ip_address?.should be_true
                 end
             end
-
-            context 'greater than the provided depth' do
-                it 'returns true' do
-                    @deep_url.too_deep?( 2 ).should be_true
-                end
-            end
-        end
-    end
-
-    describe '#exclude?' do
-        before { @exclude_url = @uri.parse( 'http://test.com/exclude/' ) }
-
-        context 'when self matches the provided exclude rules' do
-            it 'returns true' do
-                rules = [ /exclude/ ]
-                @exclude_url.exclude?( rules ).should be_true
-
-                @exclude_url.exclude?( rules.first ).should be_true
-            end
         end
-
-        context 'when self does not match the provided exclude rules' do
-            it 'returns false' do
-                rules = [ /boo/ ]
-                @exclude_url.exclude?( rules ).should be_false
-
-                @exclude_url.exclude?( rules.first ).should be_false
-            end
-        end
-
-        context 'when the provided rules are nil' do
-            it 'raises a TypeError' do
-                expect { @exclude_url.exclude?( nil ) }.to raise_error TypeError
-            end
-        end
-
     end
 
-    describe '#include?' do
-        before { @include_url = @uri.parse( 'http://test.com/include/' ) }
-
-        context 'when self matches the provided include rules in' do
-            it 'returns true' do
-                rules = [ /include/ ]
-                @include_url.include?( rules ).should be_true
-
-                @include_url.include?( rules.first ).should be_true
-            end
-        end
-
-        context 'when self does not match the provided include rules in' do
-            it 'returns false' do
-                rules = [ /boo/ ]
-                @include_url.include?( rules ).should be_false
-
-                @include_url.include?( rules.first ).should be_false
-            end
-        end
-
-        context 'when the provided rules are empty' do
-            it 'returns true' do
-                @include_url.include?( [] ).should be_true
-            end
-        end
-
-        context 'when the provided rules are nil' do
-            it 'raises a TypeError' do
-                expect { @include_url.include?( nil ) }.to raise_error TypeError
-            end
+    describe '#without_query' do
+        it 'returns the URI up to its resource component without the query' do
+            expected = 'http://test.com/directory/resource.php'
+            described_class.new( "#{expected}?param=1&param2=2" ).without_query.should == expected
         end
     end
 
-    describe '#in_domain?' do
-        before { @in_domain_url = @uri.parse( 'http://test.com' ) }
-
-        context Arachni::URI do
-            context true do
-                it 'includes subdomains in the comparison' do
-                    u = @uri.parse( 'http://boo.test.com' )
-                    @in_domain_url.in_domain?( true, u ).should be_false
+    describe '#rewrite' do
+        let(:url) { described_class.new( 'http://test.com/articles/some-stuff/23' ) }
+        let(:rewritten) { described_class.new( 'http://test.com/articles.php?id=23' ) }
 
-                    u = @uri.parse( 'http://test.com' )
-                    @in_domain_url.in_domain?( true, u ).should be_true
-                end
-            end
-            context false do
-                it 'does not include subdomains in the comparison' do
-                    u = @uri.parse( 'http://boo.test.com' )
-                    @in_domain_url.in_domain?( false, u ).should be_true
-
-                    u = @uri.parse( 'http://test.com' )
-                    @in_domain_url.in_domain?( true, u ).should be_true
-                end
-            end
+        it 'rewrites a URL based on the given rules' do
+            url.rewrite( rewrite_rules ).should == rewritten
         end
 
-        context URI do
-            context true do
-                it 'includes subdomains in the comparison' do
-                    u = URI( 'http://boo.test.com' )
-                    @in_domain_url.in_domain?( true, u ).should be_false
-                end
-            end
-            context false do
-                it 'does not include subdomains in the comparison' do
-                    u = URI( 'http://boo.test.com' )
-                    @in_domain_url.in_domain?( false, u ).should be_true
-                end
-            end
-        end
+        context 'when no rules are provided' do
+            it "uses the ones in #{Arachni::OptionGroups::Scope}#url_rewrites" do
+                Arachni::Options.scope.url_rewrites = rewrite_rules
 
-        context Hash do
-            context true do
-                it 'includes subdomains in the comparison' do
-                    h = @uri.cheap_parse( 'http://boo.test.com' )
-                    @in_domain_url.in_domain?( true, h ).should be_false
-                end
-            end
-            context false do
-                it 'does not include subdomains in the comparison' do
-                    h = @uri.cheap_parse( 'http://boo.test.com' )
-                    @in_domain_url.in_domain?( false, h ).should be_true
-                end
+                url.rewrite.should == rewritten
             end
         end
 
-        context String do
-            context true do
-                it 'includes subdomains in the comparison' do
-                    @in_domain_url.in_domain?( true, 'http://boo.test.com' ).should be_false
-                end
-            end
-            context false do
-                it 'does not include subdomains in the comparison' do
-                    @in_domain_url.in_domain?( false, 'http://boo.test.com' ).should be_true
-                end
-            end
-        end
+        context 'when no rules match' do
+            let(:url) { described_class.new( 'http://blahblah/more.blah' ) }
 
-        context 'else' do
-            it 'returns false' do
-                @in_domain_url.in_domain?( false, [] ).should be_false
+            it 'returns a copy of self' do
+                url.rewrite.should == url
+                url.rewrite.object_id.should_not == url.object_id
             end
         end
     end
 
-    describe '#without_query' do
-        it 'returns the URI up to its resource component without the query' do
-            expected = 'http://test.com/directory/resource.php'
-            described_class.new( "#{expected}?param=1&param2=2" ).without_query.should == expected
-        end
-    end
-
     describe '#resource_extension' do
         context 'when there is no extension' do
             it 'returns nil' do
@@ -529,12 +521,12 @@ describe Arachni::URI do
     describe '#mailto?' do
         context 'when the URI has a mailto scheme' do
             it 'returns true' do
-                @uri.new( 'mailto:stuff@blah.com' ).mailto?.should be_true
+                described_class.new( 'mailto:stuff@blah.com' ).mailto?.should be_true
             end
         end
         context 'when the URI does not have a mailto scheme' do
             it 'returns false' do
-                @uri.new( 'blah.com' ).mailto?.should be_false
+                described_class.new( 'blah.com' ).mailto?.should be_false
             end
         end
     end