cuboid 0.0.5 → 0.1.3

data/lib/cuboid/rest/server.rb

@@ -23,7 +23,7 @@ class Server < Sinatra::Base
     end
 
     register Routes::Instances
-    register Routes::Dispatcher
+    register Routes::Agent
     register Routes::Grid
     register Routes::Scheduler
 
@@ -81,9 +81,9 @@ class Server < Sinatra::Base
             halt 404, json( "Scan not found for id: #{h id}." )
         end
 
-        def ensure_dispatcher!
-            return if dispatcher
-            halt 501, json( 'No Dispatcher has been set.' )
+        def ensure_agent!
+            return if agent
+            halt 501, json( 'No Agent has been set.' )
         end
 
         def ensure_scheduler!
@@ -114,7 +114,7 @@ class Server < Sinatra::Base
 
         def reset
             @@instances.clear
-            @@dispatchers.clear
+            @@agents.clear
         end
 
         def run!( options )

data/lib/cuboid/rpc/client/{dispatcher.rb → agent.rb}

@@ -5,10 +5,10 @@ require Options.paths.lib + 'rpc/client/base'
 module RPC
 class Client
 
-# RPC Dispatcher client
+# RPC Agent client
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-class Dispatcher
+class Agent
     # Not always available, set by the parent.
     attr_accessor :pid
 
@@ -18,7 +18,7 @@ class Dispatcher
         @client = Base.new( url, nil, options )
         @node   = Arachni::RPC::Proxy.new( @client, 'node' )
 
-        Cuboid::Application.application.dispatcher_services.keys.each do |name|
+        Cuboid::Application.application.agent_services.keys.each do |name|
             self.class.send( :attr_reader, name.to_sym )
 
             instance_variable_set(
@@ -48,7 +48,7 @@ class Dispatcher
 
     # Used to provide the illusion of locality for remote methods
     def method_missing( sym, *args, &block )
-        @client.call( "dispatcher.#{sym.to_s}", *args, &block )
+        @client.call( "agent.#{sym.to_s}", *args, &block )
     end
 
 end

data/lib/cuboid/rpc/client/instance.rb

@@ -5,7 +5,7 @@ require Options.paths.lib + 'rpc/client/base'
 module RPC
 class Client
 
-# RPC client for remote instances spawned by a remote dispatcher
+# RPC client for remote instances spawned by a remote agent
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
 class Instance
@@ -47,7 +47,7 @@ class Instance
         @instance = Proxy.new( @client )
         @options  = Arachni::RPC::Proxy.new( @client, 'options' )
 
-        # map Dispatcher handlers
+        # map Agent handlers
         Cuboid::Application.application.instance_services.keys.each do |name|
             self.class.send( :attr_reader, name.to_sym )
 

data/lib/cuboid/rpc/client.rb

@@ -1,3 +1,3 @@
 require_relative 'client/instance'
-require_relative 'client/dispatcher'
+require_relative 'client/agent'
 require_relative 'client/scheduler'

data/lib/cuboid/rpc/server/agent/node.rb

@@ -0,0 +1,247 @@
+module Cuboid
+
+require Options.paths.lib + 'rpc/server/output'
+
+module RPC
+
+# Agent node class, helps maintain a list of all available Agents in
+# the grid and announce itself to peering Agents.
+#
+# As soon as a new Node is fired up it checks-in with its peer and grabs
+# a list of all available peers.
+#
+# As soon as it receives the peer list it then announces itself to them.
+#
+# Upon convergence there will be a grid of Agents each one with its own
+# copy of all available Agent URLs.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Server::Agent::Node
+    include UI::Output
+
+    # Initializes the node by:
+    #
+    #   * Adding the peer (if the user has supplied one) to the peer list.
+    #   * Getting the peer's peer list and appending them to its own.
+    #   * Announces itself to the peer and instructs it to propagate our URL
+    #     to the others.
+    #
+    # @param    [Cuboid::Options]    options
+    # @param    [Server::Base]    server
+    #   Agent's RPC server.
+    # @param    [String]              logfile
+    #   Where to send the output.
+    def initialize( options, server, logfile = nil )
+        @options = options
+        @server  = server
+        @url     = @server.url
+
+        reroute_to_file( logfile ) if logfile
+
+        print_status 'Initializing grid node...'
+
+        @dead_nodes = Set.new
+        @peers = Set.new
+        @nodes_info_cache = []
+
+        if (peer = @options.agent.peer)
+            # Grab the peer's peers.
+            connect_to_peer( peer ).peers do |urls|
+                if urls.rpc_exception?
+                    add_dead_peer( peer )
+                    print_info "Neighbour seems dead: #{peer}"
+                    add_dead_peer( peer )
+                    next
+                end
+
+                # Add peer and announce it to everyone.
+                add_peer( peer, true )
+
+                urls.each { |url| @peers << url if url != @url }
+            end
+        end
+
+        print_status 'Node ready.'
+
+        log_updated_peers
+
+        Arachni::Reactor.global.at_interval( @options.agent.ping_interval ) do
+            ping
+            check_for_comebacks
+        end
+    end
+
+    # @return   [Boolean]
+    #   `true` if grid member, `false` otherwise.
+    def grid_member?
+        @peers.any?
+    end
+
+    def unplug
+        @peers.each do |peer|
+            connect_to_peer( peer ).remove_peer( @url ) {}
+        end
+
+        @peers.clear
+        @dead_nodes.clear
+
+        nil
+    end
+
+    # Adds a peer to the peer list.
+    #
+    # @param    [String]    node_url
+    #   URL of a peering node.
+    # @param    [Boolean]   propagate
+    #   Whether or not to announce the new node to the peers.
+    def add_peer( node_url, propagate = false )
+        # we don't want ourselves in the Set
+        return false if node_url == @url
+        return false if @peers.include?( node_url )
+
+        print_status "Adding peer: #{node_url}"
+
+        @peers << node_url
+        log_updated_peers
+        announce( node_url ) if propagate
+
+        connect_to_peer( node_url ).add_peer( @url, propagate ) do |res|
+            next if !res.rpc_exception?
+            add_dead_peer( node_url )
+            print_status "Neighbour seems dead: #{node_url}"
+        end
+        true
+    end
+
+    def remove_peer( url )
+        @peers.delete url
+        @dead_nodes.delete url
+        nil
+    end
+
+    # @return   [Array]
+    #   Neighbour/node/peer URLs.
+    def peers
+        @peers.to_a
+    end
+
+    def peers_with_info( &block )
+        fail 'This method requires a block!' if !block_given?
+
+        @peers_cmp = ''
+
+        if @nodes_info_cache.empty? || @peers_cmp != peers.to_s
+            @peers_cmp = peers.to_s
+
+            each = proc do |peer, iter|
+                connect_to_peer( peer ).info do |info|
+                    if info.rpc_exception?
+                        print_info "Neighbour seems dead: #{peer}"
+                        add_dead_peer( peer )
+                        log_updated_peers
+
+                        iter.return( nil )
+                    else
+                        iter.return( info )
+                    end
+                end
+            end
+
+            after = proc do |nodes|
+                @nodes_info_cache = nodes.compact
+                block.call( @nodes_info_cache )
+            end
+
+            Arachni::Reactor.global.create_iterator( peers ).map( each, after )
+        else
+            block.call( @nodes_info_cache )
+        end
+    end
+
+    # @return    [Hash]
+    #
+    #   * `url` -- This node's URL.
+    #   * `name` -- Nickname
+    #   * `peers` -- Array of peers.
+    def info
+        {
+            'url'                    => @url,
+            'name'                   => @options.agent.name,
+            'peers'             => @peers.to_a,
+            'unreachable_peers' => @dead_nodes.to_a
+        }
+    end
+
+    def alive?
+        true
+    end
+
+    private
+
+    def add_dead_peer( url )
+        remove_peer( url )
+        @dead_nodes << url
+    end
+
+    def log_updated_peers
+        print_info 'Updated peers:'
+
+        if !peers.empty?
+            peers.each { |node| print_info( '---- ' + node ) }
+        else
+            print_info '<empty>'
+        end
+    end
+
+    def ping
+        peers.each do |peer|
+            connect_to_peer( peer ).alive? do |res|
+                next if !res.rpc_exception?
+                add_dead_peer( peer )
+                print_status "Found dead peer: #{peer} "
+            end
+        end
+    end
+
+    def check_for_comebacks
+        @dead_nodes.dup.each do |url|
+            peer = connect_to_peer( url )
+            peer.alive? do |res|
+                next if res.rpc_exception?
+
+                print_status "Agent came back to life: #{url}"
+                ([@url] | peers).each do |node|
+                    peer.add_peer( node ){}
+                end
+
+                add_peer( url )
+                @dead_nodes.delete url
+            end
+        end
+    end
+
+    # Announces the node to the ones in the peer list
+    #
+    # @param    [String]    node
+    #   URL
+    def announce( node )
+        print_status "Advertising: #{node}"
+
+        peers.each do |peer|
+            next if peer == node
+
+            print_info '---- to: ' + peer
+            connect_to_peer( peer ).add_peer( node ) do |res|
+                add_dead_peer( peer ) if res.rpc_exception?
+            end
+        end
+    end
+
+    def connect_to_peer( url )
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Agent.new( url ).node
+    end
+
+end
+end
+end

data/lib/cuboid/rpc/server/{dispatcher → agent}/service.rb

@@ -1,7 +1,7 @@
 module Cuboid
 module RPC
 
-# Base class and namespace for all Dispatcher services.
+# Base class and namespace for all Agent services.
 #
 # # RPC accessibility
 #
@@ -26,20 +26,20 @@ module RPC
 # results to that block instead of returning a value.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-class Server::Dispatcher::Service
+class Server::Agent::Service
 
     attr_reader :options
-    attr_reader :dispatcher
+    attr_reader :agent
 
-    def initialize( options, dispatcher )
+    def initialize( options, agent )
         @options    = options
-        @dispatcher = dispatcher
+        @agent = agent
     end
 
-    # @return   [Server::Dispatcher::Node]
+    # @return   [Server::Agent::Node]
     #   Local node.
     def node
-        dispatcher.instance_eval { @node }
+        agent.instance_eval { @node }
     end
 
     # Performs an asynchronous map operation over all running instances.
@@ -102,17 +102,17 @@ class Server::Dispatcher::Service
     # @return   [Array<Hash>]
     #   Alive instances.
     def instances
-        dispatcher.running_instances
+        agent.running_instances
     end
 
-    # Connects to a Dispatcher by `url`
+    # Connects to a Agent by `url`
     #
     # @param    [String]    url
     #
-    # @return   [Client::Dispatcher]
-    def connect_to_dispatcher( url )
-        @dispatcher_connections ||= {}
-        @dispatcher_connections[url] ||= Client::Dispatcher.new( url )
+    # @return   [Client::Agent]
+    def connect_to_agent( url )
+        @agent_connections ||= {}
+        @agent_connections[url] ||= Client::Agent.new( url )
     end
 
     # Connects to an Instance by `url`.

data/lib/cuboid/rpc/server/{dispatcher.rb → agent.rb}

@@ -14,24 +14,24 @@ class Server
 #
 # The process goes something like this:
 #
-# * A client issues a {#dispatch} call.
-# * The Dispatcher spawns and returns Instance info to the client (url, auth token, etc.).
+# * A client issues a {#spawn} call.
+# * The Agent spawns and returns Instance info to the client (url, auth token, etc.).
 # * The client connects to the Instance using that info.
 #
 # Once the client finishes using the RPC Instance it *must* shut it down
 # otherwise the system will be eaten away by zombie processes.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
-class Dispatcher
-    require Options.paths.lib + 'rpc/server/dispatcher/node'
-    require Options.paths.lib + 'rpc/server/dispatcher/service'
+class Agent
+    require Options.paths.lib + 'rpc/server/agent/node'
+    require Options.paths.lib + 'rpc/server/agent/service'
 
     include Utilities
     include UI::Output
 
     SERVICE_NAMESPACE = Service
 
-    PREFERENCE_STRATEGIES = Cuboid::OptionGroups::Dispatcher::STRATEGIES
+    PREFERENCE_STRATEGIES = Cuboid::OptionGroups::Agent::STRATEGIES
 
     def initialize( options = Options.instance )
         @options = options
@@ -46,20 +46,20 @@ class Dispatcher
             method.parameters.flatten.include? :block
         end
 
-        Options.dispatcher.url = @url = @server.url
+        Options.agent.url = @url = @server.url
 
         prep_logging
 
         print_status 'Starting the RPC Server...'
 
-        @server.add_handler( 'dispatcher', self )
+        @server.add_handler( 'agent', self )
 
         # trap interrupts and exit cleanly when required
         trap_interrupts { shutdown }
 
         @instances = []
 
-        Cuboid::Application.application.dispatcher_services.each do |name, service|
+        Cuboid::Application.application.agent_services.each do |name, service|
             @server.add_handler( name.to_s, service.new( @options, self ) )
         end
 
@@ -70,7 +70,7 @@ class Dispatcher
     end
 
     def services
-        Cuboid::Application.application.dispatcher_services.keys
+        Cuboid::Application.application.agent_services.keys
     end
 
     # @return   [TrueClass]
@@ -80,24 +80,25 @@ class Dispatcher
     end
 
     # @param    [Symbol]    strategy
-    #   `:horizontal` -- Pick the Dispatcher with the least amount of workload.
-    #   `:vertical` -- Pick the Dispatcher with the most amount of workload.
+    #   `:horizontal` -- Pick the Agent with the least amount of workload.
+    #   `:vertical` -- Pick the Agent with the most amount of workload.
+    #   `:direct` -- Bypass the grid and get an Instance directly from this agent.
     #
     # @return   [String, nil]
     #   Depending on strategy and availability:
     #
-    #   * URL of the preferred Dispatcher. If not a grid member it will return
-    #       this Dispatcher's URL.
+    #   * URL of the preferred Agent. If not a grid member it will return
+    #       this Agent's URL.
     #   * `nil` if all nodes are at max utilization or on error.
     #   * `ArgumentError` -- On invalid `strategy`.
-    def preferred( strategy = Cuboid::Options.dispatcher.strategy, &block )
+    def preferred( strategy = Cuboid::Options.agent.strategy, &block )
         strategy = strategy.to_sym
         if !PREFERENCE_STRATEGIES.include? strategy
             block.call :error_unknown_strategy
             raise ArgumentError, "Unknown strategy: #{strategy}"
         end
 
-        if !@node.grid_member?
+        if strategy == :direct || !@node.grid_member?
             block.call( self.utilization == 1 ? nil : @url )
             return
         end
@@ -117,9 +118,9 @@ class Dispatcher
             end
         end
 
-        each = proc do |neighbour, iter|
-            connect_to_peer( neighbour ).utilization do |utilization|
-                iter.return pick_utilization.call( neighbour, utilization )
+        each = proc do |peer, iter|
+            connect_to_peer( peer ).utilization do |utilization|
+                iter.return pick_utilization.call( peer, utilization )
             end
         end
 
@@ -136,32 +137,35 @@ class Dispatcher
             block.call nodes.sort_by { |_, score| adjust_score_by_strategy.call score }[0][0]
         end
 
-        Arachni::Reactor.global.create_iterator( @node.neighbours ).map( each, after )
+        Arachni::Reactor.global.create_iterator( @node.peers ).map( each, after )
     end
 
-    # Dispatches an {Instance}.
+    # Spawns an {Instance}.
     #
-    # @param    [String]  owner
+    # @param    [String]  options
+    # @option    [String]  strategy
+    # @option    [String]  owner
     #   An owner to assign to the {Instance}.
-    # @param    [Hash]    helpers
+    # @option    [Hash]    helpers
     #   Hash of helper data to be added to the instance info.
-    # @param    [Boolean]    load_balance
-    #   Return an {Instance} from the least burdened {Dispatcher} (when in Grid mode)
-    #   or from this one directly?
     #
     # @return   [Hash, nil]
     #   Depending on availability:
     #
     #   * `Hash`: Connection and proc info.
-    #   * `nil`: Max utilization, wait for one of the instances to finish and retry.
-    def dispatch( options = {}, &block )
+    #   * `nil`: Max utilization or currently spawning, wait and retry.
+    def spawn( options = {}, &block )
+        if @spawning
+            block.call nil
+            return
+        end
+
         options      = options.my_symbolize_keys
         strategy     = options.delete(:strategy)
         owner        = options[:owner]
         helpers      = options[:helpers] || {}
-        load_balance = options[:load_balance].nil? ? true : options[:load_balance]
 
-        if load_balance && @node.grid_member?
+        if strategy != 'direct' && @node.grid_member?
             preferred *[strategy].compact do |url|
                 if !url
                     block.call
@@ -173,9 +177,9 @@ class Dispatcher
                     next
                 end
 
-                connect_to_peer( url ).dispatch( options.merge(
-                      helpers: helpers.merge( via: @url ),
-                      load_balance: false
+                connect_to_peer( url ).spawn( options.merge(
+                      helpers:  helpers.merge( via: @url ),
+                      strategy: :direct
                     ),
                     &block
                 )
@@ -188,6 +192,7 @@ class Dispatcher
             return
         end
 
+        @spawning = true
         spawn_instance do |info|
             info['owner']   = owner
             info['helpers'] = helpers
@@ -195,6 +200,8 @@ class Dispatcher
             @instances << info
 
             block.call info
+
+            @spawning = false
         end
     end
 
@@ -243,7 +250,7 @@ class Dispatcher
     end
 
     # @return   [Float]
-    #   Workload score for this Dispatcher, calculated using {System#utilization}.
+    #   Workload score for this Agent, calculated using {System#utilization}.
     #
     #   * `0.0` => No utilization.
     #   * `1.0` => Max utilization.
@@ -285,7 +292,7 @@ class Dispatcher
         end
     end
 
-    # Starts the dispatcher's server
+    # Starts the agent's server
     def run
         Arachni::Reactor.global.on_error do |_, e|
             print_error "Reactor: #{e}"
@@ -314,7 +321,7 @@ class Dispatcher
     def spawn_instance( options = {}, &block )
         Processes::Instances.spawn( options.merge(
             address:     @server.address,
-            port_range:  Options.dispatcher.instance_port_range,
+            port_range:  Options.agent.instance_port_range,
             token:       Utilities.generate_token,
             application: Options.paths.application
         )) do |client|
@@ -331,12 +338,12 @@ class Dispatcher
     def prep_logging
         # reroute all output to a logfile
         @logfile ||= reroute_to_file( @options.paths.logs +
-            "/Dispatcher - #{Process.pid}-#{@options.rpc.server_port}.log" )
+            "/Agent - #{Process.pid}-#{@options.rpc.server_port}.log" )
     end
 
     def connect_to_peer( url )
         @rpc_clients ||= {}
-        @rpc_clients[url] ||= Client::Dispatcher.new( url )
+        @rpc_clients[url] ||= Client::Agent.new( url )
     end
 
 end

data/lib/cuboid/rpc/server/application_wrapper.rb

@@ -1,4 +1,5 @@
 require 'tempfile'
+require 'forwardable'
 
 module Cuboid
 
@@ -111,7 +112,7 @@ class ApplicationWrapper
             busy:           running?,
             application:    @application.class.to_s,
             seed:           Utilities.random_seed,
-            dispatcher_url: Cuboid::Options.dispatcher.url,
+            agent_url: Cuboid::Options.agent.url,
             scheduler_url:  Cuboid::Options.scheduler.url
         }
 

data/lib/cuboid/rpc/server/instance.rb

@@ -90,10 +90,10 @@ class Instance
     end
 
     # @return   [String, nil]
-    #   Dispatcher URL that provided this Instance, `nil` if not provided by a
-    #   Dispatcher.
-    def dispatcher_url
-        @options.dispatcher.url
+    #   Agent URL that provided this Instance, `nil` if not provided by a
+    #   Agent.
+    def agent_url
+        @options.agent.url
     end
 
     # @param (see Cuboid::Application#restore)