cuboid 0.0.4 → 0.1.2

data/lib/cuboid/rpc/server/agent.rb

@@ -0,0 +1,347 @@
+module Cuboid
+
+lib = Options.paths.lib
+require lib + 'processes/instances'
+require lib + 'rpc/client'
+require lib + 'rpc/server/base'
+require lib + 'rpc/server/instance'
+require lib + 'rpc/server/output'
+
+module RPC
+class Server
+
+# Dispatches RPC Instances on demand and allows for extensive process monitoring.
+#
+# The process goes something like this:
+#
+# * A client issues a {#spawn} call.
+# * The Agent spawns and returns Instance info to the client (url, auth token, etc.).
+# * The client connects to the Instance using that info.
+#
+# Once the client finishes using the RPC Instance it *must* shut it down
+# otherwise the system will be eaten away by zombie processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Agent
+    require Options.paths.lib + 'rpc/server/agent/node'
+    require Options.paths.lib + 'rpc/server/agent/service'
+
+    include Utilities
+    include UI::Output
+
+    SERVICE_NAMESPACE = Service
+
+    PREFERENCE_STRATEGIES = Cuboid::OptionGroups::Agent::STRATEGIES
+
+    def initialize( options = Options.instance )
+        @options = options
+
+        @options.snapshot.path ||= @options.paths.snapshots
+
+        @server = Base.new( @options.rpc.to_server_options )
+        @server.logger.level = @options.datastore.log_level if @options.datastore.log_level
+
+        @server.add_async_check do |method|
+            # methods that expect a block are async
+            method.parameters.flatten.include? :block
+        end
+
+        Options.agent.url = @url = @server.url
+
+        prep_logging
+
+        print_status 'Starting the RPC Server...'
+
+        @server.add_handler( 'agent', self )
+
+        # trap interrupts and exit cleanly when required
+        trap_interrupts { shutdown }
+
+        @instances = []
+
+        Cuboid::Application.application.agent_services.each do |name, service|
+            @server.add_handler( name.to_s, service.new( @options, self ) )
+        end
+
+        @node = Node.new( @options, @server, @logfile )
+        @server.add_handler( 'node', @node )
+
+        run
+    end
+
+    def services
+        Cuboid::Application.application.agent_services.keys
+    end
+
+    # @return   [TrueClass]
+    #   true
+    def alive?
+        @server.alive?
+    end
+
+    # @param    [Symbol]    strategy
+    #   `:horizontal` -- Pick the Agent with the least amount of workload.
+    #   `:vertical` -- Pick the Agent with the most amount of workload.
+    #
+    # @return   [String, nil]
+    #   Depending on strategy and availability:
+    #
+    #   * URL of the preferred Agent. If not a grid member it will return
+    #       this Agent's URL.
+    #   * `nil` if all nodes are at max utilization or on error.
+    #   * `ArgumentError` -- On invalid `strategy`.
+    def preferred( strategy = Cuboid::Options.agent.strategy, &block )
+        strategy = strategy.to_sym
+        if !PREFERENCE_STRATEGIES.include? strategy
+            block.call :error_unknown_strategy
+            raise ArgumentError, "Unknown strategy: #{strategy}"
+        end
+
+        if !@node.grid_member?
+            block.call( self.utilization == 1 ? nil : @url )
+            return
+        end
+
+        pick_utilization = proc do |url, utilization|
+            (utilization == 1 || utilization.rpc_exception?) ?
+                nil : [url, utilization]
+        end
+
+        adjust_score_by_strategy = proc do |score|
+            case strategy
+                when :horizontal
+                    score
+
+                when :vertical
+                    -score
+            end
+        end
+
+        each = proc do |peer, iter|
+            connect_to_peer( peer ).utilization do |utilization|
+                iter.return pick_utilization.call( peer, utilization )
+            end
+        end
+
+        after = proc do |nodes|
+            nodes << pick_utilization.call( @url, self.utilization )
+            nodes.compact!
+
+            # All nodes are at max utilization, pass.
+            if nodes.empty?
+                block.call
+                next
+            end
+
+            block.call nodes.sort_by { |_, score| adjust_score_by_strategy.call score }[0][0]
+        end
+
+        Arachni::Reactor.global.create_iterator( @node.peers ).map( each, after )
+    end
+
+    # Dispatches an {Instance}.
+    #
+    # @param    [String]  options
+    # @option    [String]  strategy
+    # @option    [String]  owner
+    #   An owner to assign to the {Instance}.
+    # @option    [Hash]    helpers
+    #   Hash of helper data to be added to the instance info.
+    # @option    [Boolean]    load_balance
+    #   Use the Grid (when available) or this one directly?
+    #
+    # @return   [Hash, nil]
+    #   Depending on availability:
+    #
+    #   * `Hash`: Connection and proc info.
+    #   * `nil`: Max utilization, wait for one of the instances to finish and retry.
+    def spawn( options = {}, &block )
+        options      = options.my_symbolize_keys
+        strategy     = options.delete(:strategy)
+        owner        = options[:owner]
+        helpers      = options[:helpers] || {}
+        load_balance = options[:load_balance].nil? ? true : options[:load_balance]
+
+        if load_balance && @node.grid_member?
+            preferred *[strategy].compact do |url|
+                if !url
+                    block.call
+                    next
+                end
+
+                if url == :error_unknown_strategy
+                    block.call :error_unknown_strategy
+                    next
+                end
+
+                connect_to_peer( url ).spawn( options.merge(
+                      helpers: helpers.merge( via: @url ),
+                      load_balance: false
+                    ),
+                    &block
+                )
+            end
+            return
+        end
+
+        if System.max_utilization?
+            block.call
+            return
+        end
+
+        spawn_instance do |info|
+            info['owner']   = owner
+            info['helpers'] = helpers
+
+            @instances << info
+
+            block.call info
+        end
+    end
+
+    # Returns proc info for a given pid
+    #
+    # @param    [Fixnum]      pid
+    #
+    # @return   [Hash]
+    def instance( pid )
+        @instances.each do |i|
+            next if i['pid'] != pid
+            i = i.dup
+
+            now = Time.now
+
+            i['now']   = now.to_s
+            i['age']   = now - Time.parse( i['birthdate'] )
+            i['alive'] = Cuboid::Processes::Manager.alive?( pid )
+
+            return i
+        end
+
+        nil
+    end
+
+    # @return   [Array<Hash>]
+    #   Returns info for all instances.
+    def instances
+        @instances.map { |i| instance( i['pid'] ) }.compact
+    end
+
+    # @return   [Array<Hash>]
+    #   Returns info for all running (alive) instances.
+    #
+    # @see #instances
+    def running_instances
+        instances.select { |i| i['alive'] }
+    end
+
+    # @return   [Array<Hash>]
+    #   Returns info for all finished (dead) instances.
+    #
+    # @see #instances
+    def finished_instances
+        instances.reject { |i| i['alive'] }
+    end
+
+    # @return   [Float]
+    #   Workload score for this Agent, calculated using {System#utilization}.
+    #
+    #   * `0.0` => No utilization.
+    #   * `1.0` => Max utilization.
+    #
+    #   Lower is better.
+    def utilization
+        System.utilization
+    end
+
+    # @return   [Hash]
+    #   Returns server stats regarding the instances and pool.
+    def statistics
+        {
+            'utilization'         => utilization,
+            'running_instances'   => running_instances,
+            'finished_instances'  => finished_instances,
+            'consumed_pids'       => @instances.map { |i| i['pid'] }.compact,
+            'snapshots'           => Dir.glob( "#{@options.snapshot.path}*.#{Snapshot::EXTENSION}" ),
+            'node'                => @node.info
+        }
+    end
+
+    # @return   [String]
+    #   Contents of the log file
+    def log
+        IO.read prep_logging
+    end
+
+    # @private
+    def pid
+        Process.pid
+    end
+
+    private
+
+    def trap_interrupts( &block )
+        %w(QUIT INT).each do |signal|
+            trap( signal, &block || Proc.new{ } ) if Signal.list.has_key?( signal )
+        end
+    end
+
+    # Starts the agent's server
+    def run
+        Arachni::Reactor.global.on_error do |_, e|
+            print_error "Reactor: #{e}"
+
+            e.backtrace.each do |l|
+                print_error "Reactor: #{l}"
+            end
+        end
+
+        print_status 'Ready'
+        @server.start
+    rescue => e
+        print_exception e
+
+        $stderr.puts "Could not start server, for details see: #{@logfile}"
+        exit 1
+    end
+
+    def shutdown
+        Thread.new do
+            print_status 'Shutting down...'
+            Arachni::Reactor.global.stop
+        end
+    end
+
+    def spawn_instance( options = {}, &block )
+        Processes::Instances.spawn( options.merge(
+            address:     @server.address,
+            port_range:  Options.agent.instance_port_range,
+            token:       Utilities.generate_token,
+            application: Options.paths.application
+        )) do |client|
+            block.call(
+                'token'       => client.token,
+                'url'         => client.url,
+                'pid'         => client.pid,
+                'birthdate'   => Time.now.to_s,
+                'application' => Options.paths.application
+            )
+        end
+    end
+
+    def prep_logging
+        # reroute all output to a logfile
+        @logfile ||= reroute_to_file( @options.paths.logs +
+            "/Agent - #{Process.pid}-#{@options.rpc.server_port}.log" )
+    end
+
+    def connect_to_peer( url )
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Agent.new( url )
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/server/application_wrapper.rb

@@ -1,4 +1,5 @@
 require 'tempfile'
+require 'forwardable'
 
 module Cuboid
 
@@ -111,7 +112,7 @@ class ApplicationWrapper
             busy:           running?,
             application:    @application.class.to_s,
             seed:           Utilities.random_seed,
-            dispatcher_url: Cuboid::Options.dispatcher.url,
+            agent_url: Cuboid::Options.agent.url,
             scheduler_url:  Cuboid::Options.scheduler.url
         }
 

data/lib/cuboid/rpc/server/instance.rb

@@ -90,10 +90,10 @@ class Instance
     end
 
     # @return   [String, nil]
-    #   Dispatcher URL that provided this Instance, `nil` if not provided by a
-    #   Dispatcher.
-    def dispatcher_url
-        @options.dispatcher.url
+    #   Agent URL that provided this Instance, `nil` if not provided by a
+    #   Agent.
+    def agent_url
+        @options.agent.url
     end
 
     # @param (see Cuboid::Application#restore)

data/lib/cuboid/rpc/server/scheduler.rb

@@ -6,7 +6,7 @@ require lib + 'processes/manager'
 require lib + 'processes/instances'
 
 require lib + 'rpc/client/instance'
-require lib + 'rpc/client/dispatcher'
+require lib + 'rpc/client/agent'
 
 require lib + 'rpc/server/base'
 require lib + 'rpc/server/output'
@@ -33,9 +33,9 @@ class Server
 # * {#attach Attached} to the queue monitor and transfer the management
 #   responsibility to the queue.
 #
-# If a {Dispatcher} has been provided, {Instance instances} will be
-# {Dispatcher#dispatch provided} by it.
-# If no {Dispatcher} has been given, {Instance instances} will be spawned on the
+# If a {Agent} has been provided, {Instance instances} will be
+# {Agent#spawn provided} by it.
+# If no {Agent} has been given, {Instance instances} will be spawned on the
 # Scheduler machine.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
@@ -379,7 +379,7 @@ class Scheduler
         end
     end
 
-    # Starts the dispatcher's server
+    # Starts the agent's server
     def run
         reactor.on_error do |_, e|
             print_error "Reactor: #{e}"
@@ -415,15 +415,16 @@ class Scheduler
         )
     end
 
-    def dispatcher
-        return if !Options.dispatcher.url
-        @dispatcher ||= RPC::Client::Dispatcher.new( Options.dispatcher.url )
+    def agent
+        return if !Options.agent.url
+        @agent ||= RPC::Client::Agent.new( Options.agent.url )
     end
 
     def spawn_instance( &block )
-        if dispatcher
+        if agent
             options = {
-              owner:   self.class.to_s,
+              owner:    self.class.to_s,
+              strategy: Cuboid::Options.agent.strategy,
               helpers: {
                     owner: {
                         url: @url
@@ -431,9 +432,9 @@ class Scheduler
                 }
             }
 
-            dispatcher.dispatch options do |info|
+            agent.spawn options do |info|
                 if info.rpc_exception?
-                    print_error "Failed to contact Dispatcher at: #{dispatcher.url}"
+                    print_error "Failed to contact Agent at: #{agent.url}"
                     print_error "[#{info.class}] #{info.to_s}"
                     block.call :error
                     next

data/lib/cuboid.rb

@@ -2,9 +2,6 @@ require 'rubygems'
 require 'bundler/setup'
 require 'tmpdir'
 
-require 'oj'
-require 'oj_mimic_json'
-
 require_relative 'cuboid/version'
 
 require 'concurrent'

data/lib/version

@@ -1 +1 @@
-0.0.4
+0.1.2

data/spec/cuboid/option_groups/dispatcher_spec.rb

@@ -1,10 +1,10 @@
 require 'spec_helper'
 
-describe Cuboid::OptionGroups::Dispatcher do
+describe Cuboid::OptionGroups::Agent do
     include_examples 'option_group'
     subject { described_class.new }
 
-    %w(url instance_port_range neighbour ping_interval name).each do |method|
+    %w(url instance_port_range peer ping_interval name).each do |method|
         it { is_expected.to respond_to method }
         it { is_expected.to respond_to "#{method}=" }
     end

data/spec/cuboid/rest/server_spec.rb

@@ -15,7 +15,7 @@ describe Cuboid::Rest::Server do
     let(:id) { @id }
     let(:non_existent_id) { 'stuff' }
 
-    let(:dispatcher) { Cuboid::Processes::Dispatchers.spawn }
+    let(:agent) { Cuboid::Processes::Agents.spawn }
     let(:scheduler) { Cuboid::Processes::Schedulers.spawn }
 
     def create_instance
@@ -182,7 +182,7 @@ describe Cuboid::Rest::Server do
             context 'when a running instance completes' do
                 it 'is removed' do
                     scheduler.push( options )
-                    sleep 0.1 while scheduler.completed.empty?
+                    sleep 0.1 while !scheduler.completed.empty?
 
                     get url
                     expect(response_data).to be_empty
@@ -235,17 +235,17 @@ describe Cuboid::Rest::Server do
             end
         end
 
-        context 'when a Dispatcher has been set' do
+        context 'when a Agent has been set' do
 
             it 'uses it' do
-                put '/dispatcher/url', dispatcher.url
+                put '/agent/url', agent.url
 
-                get "/grid/#{dispatcher.url}"
+                get "/grid/#{agent.url}"
                 expect(response_data['running_instances']).to be_empty
 
                 create_instance
 
-                get "/grid/#{dispatcher.url}"
+                get "/grid/#{agent.url}"
                 expect(response_data['running_instances'].size).to eq 1
             end
         end
@@ -623,32 +623,32 @@ describe Cuboid::Rest::Server do
         end
     end
 
-    describe 'GET /dispatcher/url' do
-        let(:tpl_url) { '/dispatcher/url' }
+    describe 'GET /agent/url' do
+        let(:tpl_url) { '/agent/url' }
 
-        it 'returns the Dispatcher' do
-            put url, dispatcher.url
+        it 'returns the Agent' do
+            put url, agent.url
             expect(response_code).to eq 200
 
             get url
             expect(response_code).to eq 200
-            expect(response_data).to eq dispatcher.url
+            expect(response_data).to eq agent.url
         end
 
-        context 'when no Dispatcher has been set' do
+        context 'when no Agent has been set' do
             it 'returns 501' do
                 get url
                 expect(response_code).to eq 501
-                expect(response_data).to eq 'No Dispatcher has been set.'
+                expect(response_data).to eq 'No Agent has been set.'
             end
         end
     end
 
-    describe 'PUT /dispatcher/url' do
-        let(:tpl_url) { '/dispatcher/url' }
+    describe 'PUT /agent/url' do
+        let(:tpl_url) { '/agent/url' }
 
-        it 'sets the Dispatcher' do
-            put url, dispatcher.url
+        it 'sets the Agent' do
+            put url, agent.url
             expect(response_code).to eq 200
         end
 
@@ -663,101 +663,101 @@ describe Cuboid::Rest::Server do
         end
     end
 
-    describe 'DELETE /dispatcher/url' do
-        let(:tpl_url) { '/dispatcher/url' }
+    describe 'DELETE /agent/url' do
+        let(:tpl_url) { '/agent/url' }
 
-        it 'removes the the Dispatcher' do
-            put url, dispatcher.url
+        it 'removes the the Agent' do
+            put url, agent.url
             expect(response_code).to eq 200
 
             delete url
             expect(response_code).to eq 200
 
-            get url, dispatcher.url
+            get url, agent.url
             expect(response_code).to eq 501
         end
 
-        context 'when no Dispatcher has been set' do
+        context 'when no Agent has been set' do
             it 'returns 501' do
                 delete url
                 expect(response_code).to eq 501
-                expect(response_data).to eq 'No Dispatcher has been set.'
+                expect(response_data).to eq 'No Agent has been set.'
             end
         end
     end
 
     describe 'GET /grid' do
-        let(:dispatcher) { Cuboid::Processes::Dispatchers.grid_spawn }
+        let(:agent) { Cuboid::Processes::Agents.grid_spawn }
         let(:tpl_url) { '/grid' }
 
         it 'returns Grid info' do
-            put '/dispatcher/url', dispatcher.url
+            put '/agent/url', agent.url
             expect(response_code).to eq 200
 
             get url
             expect(response_code).to eq 200
-            expect(response_data.sort).to eq ([dispatcher.url] + dispatcher.node.neighbours).sort
+            expect(response_data.sort).to eq ([agent.url] + agent.node.peers).sort
         end
 
-        context 'when no Dispatcher has been set' do
+        context 'when no Agent has been set' do
             it 'returns 501' do
                 get url
                 expect(response_code).to eq 501
-                expect(response_data).to eq 'No Dispatcher has been set.'
+                expect(response_data).to eq 'No Agent has been set.'
             end
         end
     end
 
-    describe 'GET /grid/:dispatcher' do
-        let(:dispatcher) { Cuboid::Processes::Dispatchers.grid_spawn }
+    describe 'GET /grid/:agent' do
+        let(:agent) { Cuboid::Processes::Agents.grid_spawn }
         let(:tpl_url) { '/grid/%s' }
 
-        it 'returns Dispatcher info' do
-            put '/dispatcher/url', dispatcher.url
+        it 'returns Agent info' do
+            put '/agent/url', agent.url
             expect(response_code).to eq 200
 
-            @id = dispatcher.url
+            @id = agent.url
 
             get url
             expect(response_code).to eq 200
-            expect(response_data).to eq dispatcher.statistics
+            expect(response_data).to eq agent.statistics
         end
 
-        context 'when no Dispatcher has been set' do
+        context 'when no Agent has been set' do
             it 'returns 501' do
                 @id = 'localhost:2222'
 
                 get url
                 expect(response_code).to eq 501
-                expect(response_data).to eq 'No Dispatcher has been set.'
+                expect(response_data).to eq 'No Agent has been set.'
             end
         end
     end
 
-    describe 'DELETE /grid/:dispatcher' do
-        let(:dispatcher) { Cuboid::Processes::Dispatchers.grid_spawn }
+    describe 'DELETE /grid/:agent' do
+        let(:agent) { Cuboid::Processes::Agents.grid_spawn }
         let(:tpl_url) { '/grid/%s' }
 
-        it 'unplugs the Dispatcher from the Grid' do
-            put '/dispatcher/url', dispatcher.url
+        it 'unplugs the Agent from the Grid' do
+            put '/agent/url', agent.url
             expect(response_code).to eq 200
 
-            @id = dispatcher.url
+            @id = agent.url
 
-            expect(dispatcher.node.grid_member?).to be_truthy
+            expect(agent.node.grid_member?).to be_truthy
 
             delete url
             expect(response_code).to eq 200
-            expect(dispatcher.node.grid_member?).to be_falsey
+            expect(agent.node.grid_member?).to be_falsey
         end
 
-        context 'when no Dispatcher has been set' do
+        context 'when no Agent has been set' do
             it 'returns 501' do
                 @id = 'localhost:2222'
 
                 delete url
                 expect(response_code).to eq 501
-                expect(response_data).to eq 'No Dispatcher has been set.'
+                expect(response_data).to eq 'No Agent has been set.'
             end
         end
     end