arachni 1.0 → 1.0.1

data/lib/arachni/element/base.rb

@@ -48,7 +48,7 @@ class Base
     attr_reader   :initialization_options
 
     def initialize( options )
-        options = options.symbolize_keys( false )
+        options = options.my_symbolize_keys( false )
 
         if !(options[:url] || options[:action])
             fail 'Needs :url or :action option.'
@@ -167,7 +167,7 @@ class Base
 
                         when 'initialization_options'
                             value.is_a?( Hash ) ?
-                                value.symbolize_keys( false ) : value
+                                value.my_symbolize_keys( false ) : value
 
                         when 'method'
                             value.to_sym

data/lib/arachni/element/cookie.rb

@@ -77,7 +77,7 @@ class Cookie < Base
             @data[:expires] = Time.parse( @data[:expires] ) rescue nil
         end
 
-        @data[:domain] ||= ".#{parsed_uri.host}"
+        @data[:domain] ||= parsed_uri.host
 
         @default_inputs = self.inputs.dup.freeze
     end
@@ -321,7 +321,7 @@ class Cookie < Base
                     c['expires'] = nil
                 end
                 c['secure'] = (c['secure'] == 'TRUE') ? true : false
-                new( { url: url }.merge( c.symbolize_keys ) )
+                new( { url: url }.merge( c.my_symbolize_keys ) )
             end.flatten.compact
         end
 
@@ -428,7 +428,7 @@ class Cookie < Base
                 cookie_hash['name']  = decode( cookie.name )
                 cookie_hash['value'] = decode( cookie.value )
 
-                new( { url: url }.merge( cookie_hash.symbolize_keys ) )
+                new( { url: url }.merge( cookie_hash.my_symbolize_keys ) )
             end.flatten.compact
         end
         alias :parse_set_cookie :from_set_cookie
@@ -461,7 +461,7 @@ class Cookie < Base
         #
         # @return   [String]
         def encode( str, type = :value )
-            reserved = "+;%\0"
+            reserved = "+;%\0\'\""
             reserved << '=' if type == :name
 
             URI.encode( str, reserved ).recode.gsub( ' ', '+' )

data/lib/arachni/element/form.rb

@@ -78,7 +78,7 @@ class Form < Base
 
         cinputs = (options[:inputs] || {}).inject({}) do |h, (name, value_or_info)|
              if value_or_info.is_a? Hash
-                 value_or_info             = value_or_info.symbolize_keys
+                 value_or_info             = value_or_info.my_symbolize_keys
                  h[name]                   = value_or_info[:value]
                  @input_details[name.to_s] = value_or_info
              else

data/lib/arachni/element/generic_dom.rb

@@ -113,7 +113,7 @@ class GenericDOM < Base
                                 Arachni::Page::DOM::Transition.from_rpc_data( value )
 
                             when 'initialization_options'
-                                value = value.is_a?( Hash ) ? value.symbolize_keys(false) : value
+                                value = value.is_a?( Hash ) ? value.my_symbolize_keys(false) : value
                                 value[:transition] =
                                     Arachni::Page::DOM::Transition.from_rpc_data( value[:transition] )
                                 value

data/lib/arachni/framework.rb

@@ -198,7 +198,15 @@ class Framework
 
         # Initialization may take a while so since we lazy load this make sure
         # that only one thread gets to this code at a time.
-        synchronize { @browser_cluster ||= BrowserCluster.new }
+        synchronize do
+            if !@browser_cluster
+                state.set_status_message :browser_cluster_startup
+            end
+
+            @browser_cluster ||= BrowserCluster.new
+            state.clear_status_messages
+            @browser_cluster
+        end
     end
 
     # Starts the scan.

data/lib/arachni/http/client.rb

@@ -719,6 +719,7 @@ class Client
             print_debug_level_3 '------------'
             print_debug_level_3 'Queued request.'
             print_debug_level_3 "ID#: #{request.id}"
+            print_debug_level_3 "Performer: #{request.performer}"
             print_debug_level_3 "URL: #{request.url}"
             print_debug_level_3 "Method: #{request.method}"
             print_debug_level_3 "Params: #{request.parameters}"
@@ -747,6 +748,7 @@ class Client
                 if debug_level_3?
                     print_debug_level_3 '------------'
                     print_debug_level_3 "Got response for request ID#: #{response.request.id}"
+                    print_debug_level_3 "Performer: #{response.request.performer}"
                     print_debug_level_3 "Status: #{response.code}"
                     print_debug_level_3 "Code: #{response.return_code}"
                     print_debug_level_3 "Message: #{response.return_message}"

data/lib/arachni/http/request.rb

@@ -343,7 +343,7 @@ class Request < Message
         if proxy
             options.merge!(
                 proxy:     proxy,
-                proxytype: proxy_type
+                proxytype: (proxy_type || :http).to_sym
             )
 
             if proxy_user_password
@@ -353,7 +353,7 @@ class Request < Message
         elsif Arachni::Options.http.proxy_host && Arachni::Options.http.proxy_port
             options.merge!(
                 proxy:     "#{Arachni::Options.http.proxy_host}:#{Arachni::Options.http.proxy_port}",
-                proxytype: Arachni::Options.http.proxy_type
+                proxytype: (Arachni::Options.http.proxy_type || :http).to_sym
             )
 
             if Arachni::Options.http.proxy_username && Arachni::Options.http.proxy_password

data/lib/arachni/http/response.rb

@@ -176,7 +176,7 @@ class Response < Message
     def to_rpc_data
         data = to_h
         data[:request] = request.to_rpc_data
-        data.stringify_keys(false)
+        data.my_stringify_keys(false)
     end
 
     # @param    [Hash]  data    {#to_rpc_data}

data/lib/arachni/issue.rb

@@ -462,13 +462,13 @@ class Issue
                                 end)
                             end
 
-                            value.symbolize_keys(false)
+                            value.my_symbolize_keys(false)
 
                         when 'variations'
                             value.map { |i| from_rpc_data i }
 
                         when 'remarks'
-                            value.symbolize_keys
+                            value.my_symbolize_keys
 
                         when 'platform_name', 'platform_type'
                             next if !value

data/lib/arachni/option_group.rb

@@ -54,7 +54,7 @@ class OptionGroup
     end
 
     def to_rpc_data
-        to_h.stringify_keys(false)
+        to_h.my_stringify_keys(false)
     end
 
     # @return   [Hash]

data/lib/arachni/option_groups/input.rb

@@ -147,7 +147,7 @@ class Input < Arachni::OptionGroup
         h = super
         [:values, :default_values].each do |k|
             # We can't have blocks in there...
-            h[k] = h[k].select{ |_, v| v.is_a? String }.stringify
+            h[k] = h[k].select{ |_, v| v.is_a? String }.my_stringify
         end
         h
     end

data/lib/arachni/option_groups/scope.rb

@@ -219,7 +219,7 @@ class Scope < Arachni::OptionGroup
         d = super
 
         %w(redundant_path_patterns url_rewrites).each do |k|
-            d[k] = d[k].stringify
+            d[k] = d[k].my_stringify
         end
 
         %w(exclude_path_patterns exclude_content_patterns include_path_patterns).each do |k|

data/lib/arachni/page.rb

@@ -478,7 +478,7 @@ class Page
     # @return   [Hash]
     #   Data representing this instance that are suitable the RPC transmission.
     def to_rpc_data
-        data        = to_initialization_options.stringify_keys(false)
+        data        = to_initialization_options.my_stringify_keys(false)
         data['dom'] = dom.to_rpc_data
         data['element_audit_whitelist'] = element_audit_whitelist.to_a
         data['response'] = data['response'].to_rpc_data

data/lib/arachni/page/dom/transition.rb

@@ -143,7 +143,7 @@ class Transition
         self.event = event
         @element   = element
 
-        @options = options.symbolize_keys(false)
+        @options = options.my_symbolize_keys(false)
         @clock   = Time.now
 
         return self if !block_given?
@@ -253,7 +253,7 @@ class Transition
     # @return   [Hash]
     #   Data representing this instance that are suitable the RPC transmission.
     def to_rpc_data
-        h = to_hash.stringify_keys(false)
+        h = to_hash.my_stringify_keys(false)
         h['element'] = element.to_rpc_data_or_self
         h
     end
@@ -276,7 +276,7 @@ class Transition
                             end
 
                         when 'options'
-                            value.symbolize_keys
+                            value.my_symbolize_keys
 
                         else
                             value

data/lib/arachni/parser.rb

@@ -217,7 +217,9 @@ class Parser
     # @return   [Hash]
     #   Parameters found in {#url}.
     def link_vars
-        @link_vars ||= uri_parse( @url ).rewrite.query_parameters.freeze
+        return {} if (!parsed = uri_parse( @url ))
+
+        @link_vars ||= parsed.rewrite.query_parameters.freeze
     end
 
     # @return   [Array<Element::Cookie>]

data/lib/arachni/platform/list.rb

@@ -221,7 +221,7 @@ class List
             when String
                 platforms.to_sym
             when Hash
-                platforms.symbolize_keys
+                platforms.my_symbolize_keys
             when Enumerable, Array
                 platforms.to_a.flatten.map( &:to_sym ).uniq.sort
         end

data/lib/arachni/report.rb

@@ -245,7 +245,7 @@ class Report
 
         data['plugins'] = data['plugins'].inject({}) do |h, (k, v)|
             k    = k.to_sym
-            h[k] = v.symbolize_keys(false)
+            h[k] = v.my_symbolize_keys(false)
             next h if !h[k][:options]
 
             h[k][:options] = v['options'].map do |option|

data/lib/arachni/rpc/client/instance/framework.rb

@@ -26,9 +26,9 @@ class Framework < Proxy
     %w(list_reporters list_plugins).each do |m|
         translate m do |data|
             data.map do |c|
-                c = c.symbolize_keys
+                c = c.my_symbolize_keys
                 c[:options] = c[:options].map do |o|
-                    o = o.symbolize_keys
+                    o = o.my_symbolize_keys
                     o[:name] = o[:name].to_sym
                     o[:type] = o[:type].to_sym
                     o
@@ -39,18 +39,18 @@ class Framework < Proxy
     end
 
     translate :list_platforms do |platforms|
-        platforms.inject({}) { |h, (k, v)| h[k] = v.symbolize_keys; h }
+        platforms.inject({}) { |h, (k, v)| h[k] = v.my_symbolize_keys; h }
     end
 
     translate :statistics do |stats|
-        stats.symbolize_keys
+        stats.my_symbolize_keys
     end
 
     translate :progress do |data, options = {}|
         sitemap = data.delete('sitemap')
         issues  = data.delete('issues')
 
-        data = data.symbolize_keys
+        data = data.my_symbolize_keys
         data[:status] = data[:status].to_sym
 
         if issues
@@ -62,7 +62,7 @@ class Framework < Proxy
         end
 
         if data[:instances]
-            data[:instances] = data[:instances].map(&:symbolize_keys)
+            data[:instances] = data[:instances].map(&:my_symbolize_keys)
         end
 
         if sitemap

data/lib/arachni/rpc/client/instance/service.rb

@@ -26,9 +26,9 @@ class Service < Proxy
     %w(list_reporters list_plugins).each do |m|
         translate m do |data|
             data.map do |c|
-                c = c.symbolize_keys
+                c = c.my_symbolize_keys
                 c[:options] = c[:options].map do |o|
-                    o = o.symbolize_keys
+                    o = o.my_symbolize_keys
                     o[:name] = o[:name].to_sym
                     o[:type] = o[:type].to_sym
                     o
@@ -39,18 +39,18 @@ class Service < Proxy
     end
 
     translate :list_platforms do |platforms|
-        platforms.inject({}) { |h, (k, v)| h[k] = v.symbolize_keys; h }
+        platforms.inject({}) { |h, (k, v)| h[k] = v.my_symbolize_keys; h }
     end
 
     translate :progress do |data|
         sitemap = data.delete('sitemap')
         issues  = data.delete('issues')
 
-        data = data.symbolize_keys
+        data = data.my_symbolize_keys
         data[:status] = data[:status].to_sym
 
         if data[:instances]
-            data[:instances] = data[:instances].map(&:symbolize_keys)
+            data[:instances] = data[:instances].map(&:my_symbolize_keys)
         end
 
         if issues
@@ -68,7 +68,7 @@ class Service < Proxy
         sitemap = data.delete('sitemap')
         issues  = data.delete('issues')
 
-        data = data.symbolize_keys
+        data = data.my_symbolize_keys
         data[:status] = data[:status].to_sym
 
         if issues
@@ -76,7 +76,7 @@ class Service < Proxy
         end
 
         if data[:instances]
-            data[:instances] = data[:instances].map(&:symbolize_keys)
+            data[:instances] = data[:instances].map(&:my_symbolize_keys)
         end
 
         if sitemap

data/lib/arachni/rpc/server/dispatcher.rb

@@ -76,15 +76,21 @@ class Dispatcher
         @consumed_pids = []
         @pool          = Reactor.global.create_queue
 
+        print_status "Populating the pool with #{@options.dispatcher.pool_size}  Instances."
         if @options.dispatcher.pool_size > 0
             @options.dispatcher.pool_size.times { add_instance_to_pool( false ) }
         end
 
+        print_status 'Waiting for Instances to come on-line.'
+
         # Check up on the pool and start the server once it has been filled.
         Reactor.global.at_interval( 0.1 ) do |task|
+            print_debug "Instances: #{@pool.size}/#{@options.dispatcher.pool_size}"
             next if @options.dispatcher.pool_size != @pool.size
             task.done
 
+            print_status 'Instances are on-line.'
+
             _services.each do |name, service|
                 @server.add_handler( name, service.new( @options, self ) )
             end
@@ -284,6 +290,14 @@ class Dispatcher
 
     # Starts the dispatcher's server
     def run
+        Reactor.global.on_error do |_, e|
+            print_error "Arachni::Reactor: #{e}"
+
+            e.backtrace.each do |l|
+                print_error "Arachni::Reactor: #{l}"
+            end
+        end
+
         print_status 'Ready'
         @server.start
     rescue => e
@@ -326,17 +340,15 @@ class Dispatcher
         print_status "Instance added to pool -- PID: #{pid} - " +
             "Port: #{port} - Owner: #{owner}"
 
-        url = "#{@options.dispatcher.external_address}:#{port}"
-
         # Wait until the Instance has booted before adding it to the pool.
-        Client::Instance.when_ready( url, token ) do
+        Client::Instance.when_ready( "#{@options.rpc.server_address}:#{port}", token ) do
             @operation_in_progress = false
 
             @pool << {
                 'token'     => token,
                 'pid'       => pid,
                 'port'      => port,
-                'url'       => url,
+                'url'       => "#{@options.dispatcher.external_address}:#{port}",
                 'owner'     => owner,
                 'birthdate' => Time.now.to_s
             }
@@ -350,7 +362,8 @@ class Dispatcher
     end
 
     def connect_to_peer( url )
-        Client::Dispatcher.new( @options, url )
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Dispatcher.new( @options, url )
     end
 
     def struct_to_h( struct )

data/lib/arachni/rpc/server/dispatcher/node.rb

@@ -52,12 +52,18 @@ class Server::Dispatcher::Node
         @nodes_info_cache = []
 
         if (neighbour = @options.dispatcher.neighbour)
-            # add neighbour and announce him to everyone
-            add_neighbour( neighbour, true )
-
-            # grab the neighbour's neighbours
+            # Grab the neighbour's neighbours.
             connect_to_peer( neighbour ).neighbours do |urls|
-                fail "Neighbour '#{neighbour}' is unreachable." if urls.rpc_exception?
+                if urls.rpc_exception?
+                    add_dead_neighbour( neighbour )
+                    print_info "Neighbour seems dead: #{neighbour}"
+                    add_dead_neighbour( neighbour )
+                    next
+                end
+
+                # Add neighbour and announce it to everyone.
+                add_neighbour( neighbour, true )
+
                 urls.each { |url| @neighbours << url if url != @url }
             end
         end
@@ -229,7 +235,8 @@ class Server::Dispatcher::Node
     end
 
     def connect_to_peer( url )
-        Client::Dispatcher.new( @options, url ).node
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Dispatcher.new( @options, url ).node
     end
 
 end

data/lib/arachni/rpc/server/framework/distributor.rb

@@ -27,7 +27,7 @@ module Distributor
     #   The hash must hold the `'url'` and the `'token'`. In subsequent calls
     #   the `'token'` can be omitted.
     def connect_to_instance( instance )
-        instance = instance.symbolize_keys
+        instance = instance.my_symbolize_keys
         @instance_connections ||= {}
 
         if @instance_connections[instance[:url]]