arachni 0.4.2 → 0.4.3

data/spec/arachni/rpc/server/{distributor_spec.rb → framework/distributor_spec.rb}

@@ -1,10 +1,6 @@
-require_relative '../../../spec_helper'
-
-require 'timeout'
-require Arachni::Options.dir['lib'] + 'rpc/client/instance'
-require Arachni::Options.dir['lib'] + 'rpc/server/instance'
-
-require Arachni::Options.dir['lib'] + 'rpc/server/distributor'
+require 'spec_helper'
+require Arachni::Options.dir['lib'] + 'rpc/server/base'
+require Arachni::Options.dir['lib'] + 'rpc/server/framework'
 
 class Distributor
     include Arachni::RPC::Server::Framework::Distributor
@@ -20,16 +16,12 @@ class Distributor
     end
 
     def initialize( token )
-        @opts = Arachni::Options.instance
-        @local_token = token
-        @instances = []
+        @opts           = Arachni::Options.instance
+        @local_token    = token
+        @instances      = []
         @running_slaves = Set.new
     end
 
-    def self_url
-        @master_url
-    end
-
     def dispatcher_url=( url )
         @opts.datastore[:dispatcher_url] = url
     end
@@ -42,7 +34,6 @@ end
 class FakeMaster
 
     attr_reader :issues
-    attr_reader :issue_summaries
 
     def initialize( opts, token )
         @opts  = opts
@@ -51,10 +42,9 @@ class FakeMaster
 
         @pages  = []
         @issues = []
-        @issue_summaries = []
         @element_ids     = []
 
-        @server.add_handler( "framework", self )
+        @server.add_handler( 'framework', self )
         @server.start
     end
 
@@ -63,29 +53,18 @@ class FakeMaster
                                                        instance_hash['url'],
                                                        instance_hash['token'])
 
-        instance.framework.set_master( "#{@server.opts[:host]}:#{@server.opts[:port]}",
-                                       @token )
-    end
-
-    def update_element_ids_per_page( *args )
-    end
-
-    def register_issue_summaries( issues, token = nil )
-        return false if !valid_token?( token )
-        @issue_summaries |= issues
-        true
-    end
-
-    def slave_done( *args )
+        instance.framework.
+            set_master( "#{@server.opts[:host]}:#{@server.opts[:port]}", @token )
     end
 
-    def register_issues( issues, token = nil )
+    def slave_sitrep( data, url, token = nil )
         return false if !valid_token?( token )
-        @issues |= issues
+        @issues |= data[:issues] || []
         true
     end
 
     private
+
     def valid_token?( token )
         @token == token
     end
@@ -94,53 +73,47 @@ end
 
 describe Arachni::RPC::Server::Framework::Distributor do
     before( :all ) do
-        @opts = Arachni::Options.instance
+        @opts             = Arachni::Options.instance
         @opts.audit_links = true
-        @token = 'secret'
-
-        @get_instance = proc do |opts|
-            opts ||= @opts
-            opts.rpc_port = random_port
-            fork_em { Arachni::RPC::Server::Instance.new( opts, @token ) }
-            sleep 1
-            Arachni::RPC::Client::Instance.new( opts,
-                "#{opts.rpc_address}:#{opts.rpc_port}", @token
-            )
-        end
+        @token            = 'secret'
 
         @distributor = Distributor.new( @token )
         2.times {
-            @distributor <<  { 'url' => @get_instance.call.url, 'token' => @token }
+            instance = instance_spawn
+            @distributor <<  {
+                'url' => instance.url,
+                'token' => instance_token_for( instance.url )
+            }
         }
 
-        @url = url  = 'http://test.com/'
-        @url2 = url2 = 'http://test.com/test/'
+        @url  = 'http://test.com/'
+        @url2 = 'http://test.com/test/'
         @urls = []
 
         url_gen = proc { |u, i| "#{u}?input_#{i}=val_#{i}" }
 
         10.times do |i|
-            @urls << url_gen.call( url, i )
+            @urls << url_gen.call( @url, i )
         end
 
         4.times do |i|
-            @urls << url_gen.call( url2, i )
+            @urls << url_gen.call( @url2, i )
         end
 
         5.times do |i|
-            @urls << url_gen.call( url, i )
+            @urls << url_gen.call( @url, i )
         end
 
         14.times do |i|
-            @urls << url_gen.call( url2, i )
+            @urls << url_gen.call( @url2, i )
         end
 
         20.times do |i|
-            @urls << url_gen.call( url, i )
+            @urls << url_gen.call( @url, i )
         end
 
         5.times do |i|
-            @urls << url_gen.call( url2, i )
+            @urls << url_gen.call( @url2, i )
         end
     end
 
@@ -280,9 +253,9 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
     describe '#build_elem_list' do
         it 'evenly distributes elements across instances' do
-            @opts.url = server_url_for( :parser )
-            @opts.audit_links = true
-            @opts.audit_forms = true
+            @opts.url = web_server_url_for( :parser )
+            @opts.audit_links   = true
+            @opts.audit_forms   = true
             @opts.audit_cookies = true
             @opts.audit_headers = true
 
@@ -292,8 +265,9 @@ describe Arachni::RPC::Server::Framework::Distributor do
                 async: false,
                 remove_id: true
             ).response
-            page = Arachni::Parser.new( @response, @opts ).run
-            @distributor.build_elem_list( page ).size.should == 7
+
+            @distributor.build_elem_list( Arachni::Parser.new( @response, @opts ).page ).
+                size.should == 7
         end
     end
 
@@ -426,57 +400,64 @@ describe Arachni::RPC::Server::Framework::Distributor do
             r = @distributor.distribute_elements( chunks, elem_ids_per_page )
             r.should == [ %w(elem elem_4), %w(elem_2), %w(elem_1 elem_3 elem_5)]
         end
+
+        it 'handles large data sets' do
+            elements = {}
+
+            30.times do
+                list = (elements[rand( 9999 )] ||= [])
+
+                1_000_000.times do
+                    list << rand( 9999 )
+                end
+            end
+
+            @distributor.distribute_elements( elements.keys.map { |i| [i] }, elements )
+        end
     end
 
     describe '#preferred_dispatchers' do
         it 'returns a sorted list of dispatchers for HPG use taking into account their pipe IDs and load balancing metrics' do
-            @opts.pool_size = 1
-            opts = @opts
-            port = random_port
-
             dispatchers = []
 
-            opts.rpc_port = port
-            exec_dispatcher( opts )
 
-            opts.rpc_port = random_port
-            dispatchers[0] = "#{opts.rpc_address}:#{opts.rpc_port}"
-            exec_dispatcher( opts ) do |conf|
-                conf.neighbour = "#{opts.rpc_address}:#{port}"
-                conf.pipe_id = '1'
-            end
+            d1 = dispatcher_light_spawn
 
-            opts.rpc_port = random_port
-            dispatchers[2] = "#{opts.rpc_address}:#{opts.rpc_port}"
-            exec_dispatcher( opts ) do |conf|
-                conf.neighbour = "#{opts.rpc_address}:#{port}"
-                conf.pipe_id = '2'
-                conf.weight = 3
-            end
+            dispatchers << dispatcher_light_spawn(
+                pipe_id: '1',
+                neighbour: d1.url
+            ).url
 
-            opts.rpc_port = random_port
-            exec_dispatcher( opts ) do |conf|
-                conf.neighbour = "#{opts.rpc_address}:#{port}"
-                conf.pipe_id = '3'
-                conf.weight = 2
-            end
+            dispatchers << dispatcher_light_spawn(
+                pipe_id:   '3',
+                neighbour: d1.url
+            ).url
 
-            opts.rpc_port = random_port
-            dispatchers[3] = "#{opts.rpc_address}:#{opts.rpc_port}"
-            exec_dispatcher( opts ) do |conf|
-                conf.neighbour = "#{opts.rpc_address}:#{port}"
-                conf.pipe_id = '4'
-                conf.weight = 4
-            end
+            dispatcher_light_spawn(
+                weight:  3,
+                pipe_id: '1',
+                neighbour: d1.url
+            )
 
-            opts.rpc_port = random_port
-            dispatchers[1] = "#{opts.rpc_address}:#{opts.rpc_port}"
-            exec_dispatcher( opts ) do |conf|
-                conf.neighbour = "#{opts.rpc_address}:#{port}"
-                conf.pipe_id = '3'
-            end
+            dispatchers << dispatcher_light_spawn(
+                weight:    3,
+                pipe_id:   '2',
+                neighbour: d1.url
+            ).url
+
+            dispatcher_light_spawn(
+                weight:    2,
+                pipe_id:   '3',
+                neighbour: d1.url
+            )
 
-            @distributor.dispatcher_url = "#{opts.rpc_address}:#{port}"
+            dispatchers << dispatcher_light_spawn(
+                weight:    4,
+                pipe_id:   '4',
+                neighbour: d1.url
+            ).url
+
+            @distributor.dispatcher_url = d1.url
 
             q = Queue.new
             @distributor.preferred_dispatchers { |d| q << d }
@@ -517,27 +498,26 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
     describe '#distribute_and_run' do
         before( :all ) do
-            @opts.rpc_port = random_port
-            @opts.dir['modules'] = spec_path + 'fixtures/taint_module/'
-            @master = FakeMaster.new( @opts, @token )
-            @distributor.master_url = "#{@opts.rpc_address}:#{@opts.rpc_port}"
+            @opts.dir['modules'] = fixtures_path + 'taint_module/'
 
-            port = random_port
-            @opts.pool_size = 1
-            @opts.rpc_port = port
-            exec_dispatcher( @opts )
+            @dispatcher_url = dispatcher_light_spawn.url
+
+            @opts.rpc_port          = available_port
+            @master                 = FakeMaster.new( @opts, @token )
+            @distributor.master_url = "#{@opts.rpc_address}:#{@opts.rpc_port}"
 
             # master's token
             @opts.datastore[:token] = @token
-            @opts.url = server_url_for( :framework_hpg )
-            @url = @opts.url.to_s
-            @opts.mods = %w(taint)
-
-            @dispatcher_url = "#{@opts.rpc_address}:#{port}"
+            @opts.url               = web_server_url_for( :framework_hpg )
+            @url                    = @opts.url
+            @opts.modules           = %w(taint)
 
             @get_instance_info = proc do
-                instance = @get_instance.call
-                info = { 'url' => instance.url, 'token' => @token }
+                instance = instance_spawn( token: @token, port: nil )
+                info = {
+                    'url'   => instance.url,
+                    'token' => instance_token_for( instance )
+                }
                 @master.enslave( info )
                 info
             end
@@ -545,7 +525,6 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
         after do
             @master.issues.clear
-            @master.issue_summaries.clear
         end
 
         context 'when called without auditable restrictions' do
@@ -558,10 +537,8 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
                 slave = @distributor.connect_to_instance( slave_info )
                 sleep 0.1 while slave.framework.busy?
-                sleep 1
 
                 @master.issues.size.should == 500
-                @master.issue_summaries.size.should == 500
             end
         end
         context 'when called with auditable URL restrictions' do
@@ -578,10 +555,8 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
                 slave.opts.restrict_paths.should == absolute_urls
                 sleep 0.1 while slave.framework.busy?
-                sleep 1
 
                 @master.issues.size.should == 2
-                @master.issue_summaries.size.should == 2
 
                 vuln_urls = @master.issues.map { |i| i.url }.sort.uniq
                 vuln_urls.should == absolute_urls.sort.uniq
@@ -605,10 +580,8 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
                 slave = @distributor.connect_to_instance( slave_info )
                 sleep 0.1 while slave.framework.busy?
-                sleep 1
 
                 @master.issues.size.should == 2
-                @master.issue_summaries.size.should == 2
 
                 vuln_urls = @master.issues.map { |i| i.url }.sort.uniq
                 exp_urls = %w(/vulnerable?0_vulnerable_20=stuff20 /vulnerable?9_vulnerable_30=stuff30)
@@ -616,9 +589,9 @@ describe Arachni::RPC::Server::Framework::Distributor do
                     sort.uniq
             end
             context 'and new elements appear via the trainer' do
-                it 'soverride the restrictions' do
+                it 'overrides the restrictions' do
                     @opts.audit_forms = true
-                    @opts.url = server_url_for( :auditor ) + '/train/default'
+                    @opts.url = web_server_url_for( :auditor ) + '/train/default'
                     url = @opts.url.to_s
 
                     id = Arachni::Element::Form.new( url + '?',
@@ -632,14 +605,8 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
                     slave = @distributor.connect_to_instance( slave_info )
                     sleep 0.1 while slave.framework.busy?
-                    sleep 1
 
                     @master.issues.size.should == 8
-                    @master.issue_summaries.size.should == 8
-
-                    #@master.issues.size.should == 1
-                    #@master.issues.first.url.should ==
-                    #    url + "?you_made_it=to+the+end+of+the+training"
                 end
             end
         end
@@ -687,10 +654,8 @@ describe Arachni::RPC::Server::Framework::Distributor do
 
                 slave = @distributor.connect_to_instance( slave_info )
                 sleep 0.1 while slave.framework.busy?
-                sleep 1
 
                 @master.issues.size.should == 4
-                @master.issue_summaries.size.should == 4
 
                 vuln_urls = @master.issues.map { |i| i.url }.sort.uniq
                 vuln_urls.should == exp_urls.sort

data/spec/arachni/rpc/server/framework_hpg_spec.rb

@@ -1,67 +1,24 @@
-require_relative '../../../spec_helper'
+require 'spec_helper'
 
-require Arachni::Options.instance.dir['lib'] + 'rpc/client/dispatcher'
-require Arachni::Options.instance.dir['lib'] + 'rpc/server/dispatcher'
-
-describe Arachni::RPC::Server::Framework do
+describe 'Arachni::RPC::Server::Framework' do
     before( :all ) do
         @opts = Arachni::Options.instance
         @opts.dir['modules'] = fixtures_path + '/taint_module/'
         @opts.audit_links = true
 
-        @dispatchers = []
-
-        @opts.pool_size = 1
-        @get_instance = proc do |opts|
-            opts ||= @opts
-            port = random_port
-            opts.rpc_port = port
-            exec_dispatcher( opts )
-
-            port2 =  random_port
-            opts.rpc_port = port2
-            opts.neighbour = "#{opts.rpc_address}:#{port}"
-            opts.pipe_id = 'blah'
-            exec_dispatcher( opts )
-
-            dispatcher = Arachni::RPC::Client::Dispatcher.new( opts,
-                "#{opts.rpc_address}:#{port}" )
-
-            inst_info = dispatcher.dispatch
-            inst = Arachni::RPC::Client::Instance.new( opts,
-                inst_info['url'], inst_info['token']
-            )
-            inst.opts.grid_mode = 'high_performance'
-            inst
-        end
-
-        @token = 'secret'
-        @get_simple_instance = proc do |opts|
-            opts ||= @opts
-            port = random_port
-            opts.rpc_port = port
-            fork_em { Arachni::RPC::Server::Instance.new( opts, @token ) }
-            sleep 1
-            Arachni::RPC::Client::Instance.new( opts,
-                "#{opts.rpc_address}:#{port}", @token
-            )
-        end
-
-        @instance = @get_instance.call
+        @instance  = instance_grid_spawn
         @framework = @instance.framework
-        @modules = @instance.modules
-        @plugins = @instance.plugins
+        @modules   = @instance.modules
+        @plugins   = @instance.plugins
 
-        @instance_clean = @get_instance.call
+        @instance_clean  = instance_grid_spawn
         @framework_clean = @instance_clean.framework
 
         @stat_keys = [
-            :requests, :responses, :time_out_count,
-            :time, :avg, :sitemap_size, :auditmap_size, :progress, :curr_res_time,
-            :curr_res_cnt, :curr_avg, :average_res_time, :max_concurrency,
-            :current_page, :eta,
+            :requests, :responses, :time_out_count, :time, :avg, :sitemap_size,
+            :auditmap_size, :progress, :curr_res_time, :curr_res_cnt, :curr_avg,
+            :average_res_time, :max_concurrency, :current_page, :eta
         ]
-
     end
 
     describe '#errors' do
@@ -90,7 +47,7 @@ describe Arachni::RPC::Server::Framework do
         end
         context 'when the scan is running' do
             it 'returns true' do
-                @instance.opts.url = server_url_for( :auditor )
+                @instance.opts.url = web_server_url_for( :auditor )
                 @modules.load( 'taint' )
                 @framework.run.should be_true
                 @framework.busy?.should be_true
@@ -107,14 +64,9 @@ describe Arachni::RPC::Server::Framework do
             @framework_clean.revision.should == Arachni::Framework::REVISION
         end
     end
-    describe '#high_performance?' do
-        it 'returns true' do
-            @framework_clean.high_performance?.should be_true
-        end
-    end
     describe '#master?' do
         it 'returns false' do
-            @framework_clean.high_performance?.should be_true
+            @framework_clean.master?.should be_true
         end
     end
     describe '#slave?' do
@@ -129,7 +81,7 @@ describe Arachni::RPC::Server::Framework do
     end
     describe '#set_as_master' do
         it 'sets the instance as the master' do
-            instance = @get_simple_instance.call
+            instance = instance_spawn
             instance.framework.master?.should be_false
             instance.framework.set_as_master
             instance.framework.master?.should be_true
@@ -137,26 +89,22 @@ describe Arachni::RPC::Server::Framework do
     end
     describe '#enslave' do
         it 'enslaves another instance and set itself as its master' do
-            master = @get_simple_instance.call
-            slave  = @get_simple_instance.call
+            master = instance_spawn
+            slave  = instance_spawn
 
             master.framework.master?.should be_false
-            master.framework.enslave( 'url' => slave.url, 'token' => @token )
+            master.framework.enslave(
+                'url'   => slave.url,
+                'token' => instance_token_for( slave )
+            )
             master.framework.master?.should be_true
         end
     end
-    describe '#output' do
-        it 'returns the instance\'s output messages' do
-            output = @framework_clean.output.first
-            output.keys.first.is_a?( Symbol ).should be_true
-            output.values.first.is_a?( String ).should be_true
-        end
-    end
     describe '#run' do
         context 'when Options#restrict_to_paths is set' do
             it 'fails with exception' do
-                instance = @get_instance.call
-                instance.opts.url = server_url_for( :framework_hpg )
+                instance = instance_grid_spawn
+                instance.opts.url = web_server_url_for( :framework_hpg )
                 instance.opts.restrict_paths = [instance.opts.url]
                 instance.modules.load( 'taint' )
 
@@ -172,7 +120,7 @@ describe Arachni::RPC::Server::Framework do
 
         it 'performs a scan' do
             instance = @instance_clean
-            instance.opts.url = server_url_for( :framework_hpg )
+            instance.opts.url = web_server_url_for( :framework_hpg )
             instance.modules.load( 'taint' )
             instance.framework.run.should be_true
             sleep( 1 ) while instance.framework.busy?
@@ -222,24 +170,19 @@ describe Arachni::RPC::Server::Framework do
         end
     end
     describe '#clean_up' do
-        it 'sets the framework state to finished, wait for plugins to finish and merge their results' do
-            instance = @get_instance.call
-            instance.opts.url = server_url_for( :framework_hpg )
+        it 'sets the framework state to finished, waits for plugins to finish and merges their results' do
+            instance = instance_grid_spawn
+            instance.opts.url = web_server_url_for( :framework_hpg )
             instance.modules.load( 'taint' )
             instance.plugins.load( { 'wait' => {}, 'distributable' => {} } )
             instance.framework.run.should be_true
             instance.framework.auditstore.plugins.should be_empty
             instance.framework.busy?.should be_true
-
-            sleep 1 while instance.framework.busy?
+            instance.framework.clean_up.should be_true
 
             instance_count = instance.framework.progress['instances'].size
-
-            instance.framework.clean_up
-
-            auditstore = instance.framework.auditstore
-
-            auditstore.issues.size.should == 500
+            auditstore     = instance.framework.auditstore
+            instance.service.shutdown
 
             results = auditstore.plugins
             results.should be_any
@@ -273,7 +216,7 @@ describe Arachni::RPC::Server::Framework do
                 data['busy'].nil?.should be_false
                 data['messages'].is_a?( Array ).should be_true
                 data['issues'].should be_any
-                data['instances'].size.should == 2
+                data['instances'].size.should == 3
                 data.should_not include 'errors'
 
                 keys = (keys | %w(current_page)).flatten.sort
@@ -286,27 +229,23 @@ describe Arachni::RPC::Server::Framework do
             describe :errors do
                 context 'when set to true' do
                     it 'includes all error messages' do
-                        @instance_clean.framework.
-                            progress( errors: true )['errors'].should be_empty
+                        instance = instance_grid_spawn
+                        instance.framework.progress( errors: true )['errors'].should be_empty
 
                         test = 'Test'
-                        @instance_clean.framework.error_test test
+                        instance.framework.error_test test
 
-                        @instance_clean.framework.
-                            progress( errors: true )['errors'].last.
-                            should end_with test
+                        instance.framework.progress( errors: true )['errors'].last.should end_with test
                     end
                 end
                 context 'when set to an Integer' do
                     it 'returns all logged errors after that line per Instance' do
-                        initial_errors = @instance_clean.framework.
-                            progress( errors: true )['errors']
+                        instance = instance_grid_spawn
 
-                        errors = @instance_clean.framework.
-                            progress( errors: 10 )['errors']
+                        100.times { instance.framework.error_test 'test' }
 
-                        # errors are per instance
-                        initial_errors.size.should == errors.size + 9
+                        (instance.framework.progress( errors: true )['errors'].size -
+                            instance.framework.progress( errors: 10 )['errors'].size).should == 10
                     end
                 end
             end
@@ -429,9 +368,9 @@ describe Arachni::RPC::Server::Framework do
             @instance_clean.framework.update_page_queue( [] ).should be_false
         end
     end
-    describe '#register_issues' do
+    describe '#update_issues' do
         it 'returns false' do
-            @instance_clean.framework.register_issues( [] ).should be_false
+            @instance_clean.framework.update_issues( [] ).should be_false
         end
     end
 end