cuboid 0.0.0 → 0.0.1alpha

data/lib/cuboid/processes.rb

@@ -0,0 +1,13 @@
+require 'singleton'
+require 'ostruct'
+
+lib = Cuboid::Options.paths.lib
+require lib + 'rpc/client/instance'
+require lib + 'rpc/client/dispatcher'
+require lib + 'rpc/client/scheduler'
+
+lib = Cuboid::Options.paths.lib + 'processes/'
+require lib + 'manager'
+require lib + 'dispatchers'
+require lib + 'instances'
+require lib + 'schedulers'

data/lib/cuboid/processes/dispatchers.rb

@@ -0,0 +1,140 @@
+module Cuboid
+module Processes
+
+#
+# Helper for managing {RPC::Server::Dispatcher} processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Dispatchers
+    include Singleton
+    include Utilities
+
+    # @return   [Array<String>] URLs of all running Dispatchers.
+    attr_reader :list
+
+    def initialize
+        @list = []
+        @dispatcher_connections = {}
+    end
+
+    # Connects to a Dispatcher by URL.
+    #
+    # @param    [String]    url URL of the Dispatcher.
+    # @param    [Hash]    options Options for the RPC client.
+    #
+    # @return   [RPC::Client::Dispatcher]
+    def connect( url, options = nil )
+        Arachni::Reactor.global.run_in_thread if !Arachni::Reactor.global.running?
+
+        fresh = false
+        if options
+            fresh = options.delete( :fresh )
+        end
+
+        if fresh
+            @dispatcher_connections[url] = RPC::Client::Dispatcher.new( url, options )
+        else
+            @dispatcher_connections[url] ||= RPC::Client::Dispatcher.new( url, options )
+        end
+    end
+
+    # @param    [Block] block   Block to pass an RPC client for each Dispatcher.
+    def each( &block )
+        @list.each do |url|
+            block.call connect( url )
+        end
+    end
+
+    # Spawns a {RPC::Server::Dispatcher} process.
+    #
+    # @param    [Hash]  options
+    #   To be passed to {Cuboid::Options#set}. Allows `address` instead of
+    #   `rpc_server_address` and `port` instead of `rpc_port`.
+    #
+    # @return   [RPC::Client::Dispatcher]
+    def spawn( options = {} )
+        fork = options.delete(:fork)
+
+        options = {
+            dispatcher: {
+                neighbour: options[:neighbour],
+            },
+            rpc:        {
+                server_port:             options[:port]    || Utilities.available_port,
+                server_address:          options[:address] || '127.0.0.1',
+                server_external_address: options[:external_address]
+            },
+            paths: {
+                application: options[:application] || Options.paths.application
+            }
+        }
+
+        if options[:rpc][:server_external_address].nil?
+            options[:rpc].delete :server_external_address
+        end
+
+        if options[:dispatcher][:neighbour].nil?
+            options[:dispatcher].delete :neighbour
+        end
+
+        pid = Manager.spawn( :dispatcher, options: options, fork: fork )
+
+        url = "#{options[:rpc][:server_address]}:#{options[:rpc][:server_port]}"
+        while sleep( 0.1 )
+            begin
+                connect( url, connection_pool_size: 1, max_retries: 1 ).alive?
+                break
+            rescue => e
+                # ap e
+            end
+        end
+
+        @list << url
+        connect( url, fresh: true ).tap { |c| c.pid = pid }
+    end
+
+    def grid_spawn( options = {} )
+        d = spawn( options )
+        spawn( options.merge neighbour: d.url )
+    end
+
+    # @note Will also kill all Instances started by the Dispatcher.
+    #
+    # @param    [String]    url URL of the Dispatcher to kill.
+    def kill( url )
+        dispatcher = connect( url )
+        Manager.kill_many dispatcher.statistics['consumed_pids']
+        Manager.kill dispatcher.pid
+    rescue => e
+        #ap e
+        #ap e.backtrace
+        nil
+    ensure
+        @list.delete( url )
+        @dispatcher_connections.delete( url )
+    end
+
+    # Kills all {Dispatchers #list}.
+    def killall
+        @list.dup.each do |url|
+            kill url
+        end
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+
+end
+
+end
+end

data/lib/cuboid/processes/executables/base.rb

@@ -0,0 +1,54 @@
+require 'base64'
+
+$options = Marshal.load( Base64.strict_decode64( ARGV.pop ) )
+
+if !$options[:without_cuboid]
+    require 'cuboid'
+
+    include Cuboid
+
+    cuboid_options = Marshal.load( Base64.strict_decode64( ENV['CUBOID_SPAWN_OPTIONS'] ) )
+    Options.update cuboid_options
+
+    require( Options.paths.application ) if Options.paths.application
+else
+    if Gem.win_platform?
+        require 'Win32API'
+        require 'win32ole'
+    end
+end
+
+def ppid
+    $options[:ppid]
+end
+
+def parent_alive?
+    # Windows is not big on POSIX so try it its own way if possible.
+    if Gem.win_platform?
+        begin
+            alive = false
+            wmi = WIN32OLE.connect( 'winmgmts://' )
+            processes = wmi.ExecQuery( "select ProcessId from win32_process where ProcessID='#{ppid}'")
+            processes.each do |proc|
+                proc.ole_free
+                alive = true
+            end
+            processes.ole_free
+            wmi.ole_free
+
+            return alive
+        rescue WIN32OLERuntimeError
+        end
+    end
+
+    !!(Process.kill( 0, ppid ) rescue false)
+end
+
+def puts_stderr( str )
+    return if $stderr.closed?
+
+    $stderr.puts str
+rescue
+end
+
+load ARGV.pop

data/lib/cuboid/processes/executables/dispatcher.rb

@@ -0,0 +1,5 @@
+require Options.paths.lib  + 'rpc/server/dispatcher'
+
+Arachni::Reactor.global.run do
+    RPC::Server::Dispatcher.new
+end

data/lib/cuboid/processes/executables/instance.rb

@@ -0,0 +1,12 @@
+require Options.paths.lib  + 'rpc/server/instance'
+
+if (socket = $options[:socket])
+    Options.rpc.server_address          = nil
+    Options.rpc.server_external_address = nil
+    Options.rpc.server_port             = nil
+    Options.rpc.server_socket           = socket
+elsif (port = $options[:port])
+    Options.rpc.server_port = port
+end
+
+RPC::Server::Instance.new( Options, $options[:token] )

data/lib/cuboid/processes/executables/rest_service.rb

@@ -0,0 +1,13 @@
+require Options.paths.lib  + 'rest/server'
+
+Rest::Server.run!(
+    port:     Options.rpc.server_port,
+    bind:     Options.rpc.server_address,
+
+    username: Options.datastore['username'],
+    password: Options.datastore['password'],
+
+    ssl_ca:          Options.rpc.ssl_ca,
+    ssl_key:         Options.rpc.server_ssl_private_key,
+    ssl_certificate: Options.rpc.server_ssl_certificate
+)

data/lib/cuboid/processes/executables/scheduler.rb

@@ -0,0 +1,5 @@
+require Options.paths.lib  + 'rpc/server/scheduler'
+
+Arachni::Reactor.global.run do
+    RPC::Server::Scheduler.new
+end

data/lib/cuboid/processes/helpers.rb

@@ -0,0 +1,4 @@
+require_relative 'helpers/processes'
+require_relative 'helpers/dispatchers'
+require_relative 'helpers/instances'
+require_relative 'helpers/schedulers'

data/lib/cuboid/processes/helpers/dispatchers.rb

@@ -0,0 +1,23 @@
+# @param (see Cuboid::Processes::Dispatchers#spawn)
+# @return (see Cuboid::Processes::Dispatchers#spawn)
+def dispatcher_spawn( *args )
+    Cuboid::Processes::Dispatchers.spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Dispatchers#kill)
+# @return (see Cuboid::Processes::Dispatchers#kill)
+def dispatcher_kill( *args )
+    Cuboid::Processes::Dispatchers.kill( *args )
+end
+
+# @param (see Cuboid::Processes::Dispatchers#killall)
+# @return (see Cuboid::Processes::Dispatchers#killall)
+def dispatcher_killall
+    Cuboid::Processes::Dispatchers.killall
+end
+
+# @param (see Cuboid::Processes::Dispatchers#connect)
+# @return (see Cuboid::Processes::Dispatchers#connect)
+def dispatcher_connect( *args )
+    Cuboid::Processes::Dispatchers.connect( *args )
+end

data/lib/cuboid/processes/helpers/instances.rb

@@ -0,0 +1,39 @@
+# @param (see Cuboid::Processes::Instances#spawn)
+# @return (see Cuboid::Processes::Instances#spawn)
+def instance_spawn( *args )
+    Cuboid::Processes::Instances.spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Instances#grid_spawn)
+# @return (see Cuboid::Processes::Instances#grid_spawn)
+def instance_grid_spawn( *args )
+    Cuboid::Processes::Instances.grid_spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Instances#dispatcher_spawn)
+# @return (see Cuboid::Processes::Instances#dispatcher_spawn)
+def instance_dispatcher_spawn( *args )
+    Cuboid::Processes::Instances.dispatcher.spawn( *args )
+end
+
+def instance_kill( url )
+    Cuboid::Processes::Instances.kill url
+end
+
+# @param (see Cuboid::Processes::Instances#killall)
+# @return (see Cuboid::Processes::Instances#killall)
+def instance_killall
+    Cuboid::Processes::Instances.killall
+end
+
+# @param (see Cuboid::Processes::Instances#connect)
+# @return (see Cuboid::Processes::Instances#connect)
+def instance_connect( *args )
+    Cuboid::Processes::Instances.connect( *args )
+end
+
+# @param (see Cuboid::Processes::Instances#token_for)
+# @return (see Cuboid::Processes::Instances#token_for)
+def instance_token_for( *args )
+    Cuboid::Processes::Instances.token_for( *args )
+end

data/lib/cuboid/processes/helpers/processes.rb

@@ -0,0 +1,23 @@
+# @param (see Cuboid::Processes::Manager#kill_reactor)
+# @return (see Cuboid::Processes::Manager#kill_reactor)
+def process_kill_reactor( *args )
+    Cuboid::Processes::Manager.kill_reactor( *args )
+end
+
+# @param (see Cuboid::Processes::Manager#kill)
+# @return (see Cuboid::Processes::Manager#kill)
+def process_kill( *args )
+    Cuboid::Processes::Manager.kill( *args )
+end
+
+# @param (see Cuboid::Processes::Manager#killall)
+# @return (see Cuboid::Processes::Manager#killall)
+def process_killall( *args )
+    Cuboid::Processes::Manager.killall( *args )
+end
+
+# @param (see Cuboid::Processes::Manager#kill_many)
+# @return (see Cuboid::Processes::Manager#kill_many)
+def process_kill_many( *args )
+    Cuboid::Processes::Manager.kill_many( *args )
+end

data/lib/cuboid/processes/helpers/schedulers.rb

@@ -0,0 +1,23 @@
+# @param (see Cuboid::Processes::Schedulers#spawn)
+# @return (see Cuboid::Processes::Schedulers#spawn)
+def scheduler_spawn( *args )
+    Cuboid::Processes::Schedulers.spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Schedulers#kill)
+# @return (see Cuboid::Processes::Schedulers#kill)
+def scheduler_kill( *args )
+    Cuboid::Processes::Schedulers.kill( *args )
+end
+
+# @param (see Cuboid::Processes::Schedulers#killall)
+# @return (see Cuboid::Processes::Schedulers#killall)
+def scheduler_killall
+    Cuboid::Processes::Schedulers.killall
+end
+
+# @param (see Cuboid::Processes::Schedulers#connect)
+# @return (see Cuboid::Processes::Schedulers#connect)
+def scheduler_connect( *args )
+    Cuboid::Processes::Schedulers.connect( *args )
+end

data/lib/cuboid/processes/instances.rb

@@ -0,0 +1,203 @@
+module Cuboid
+module Processes
+
+#
+# Helper for managing {RPC::Server::Instance} processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Instances
+    include Singleton
+    include Utilities
+
+    # @return   [Array<String>] URLs and tokens of all running Instances.
+    attr_reader :list
+
+    def initialize
+        @list = {}
+        @instance_connections = {}
+    end
+
+    #
+    # Connects to a Instance by URL.
+    #
+    # @param    [String]    url URL of the Dispatcher.
+    # @param    [String]    token
+    #   Authentication token -- only need be provided once.
+    #
+    # @return   [RPC::Client::Instance]
+    #
+    def connect( url, token = nil )
+        Arachni::Reactor.global.run_in_thread if !Arachni::Reactor.global.running?
+
+        token ||= @list[url]
+        @list[url] ||= token
+
+        @instance_connections[url] ||= RPC::Client::Instance.new( url, token )
+    end
+
+    # @param    [Block] block   Block to pass an RPC client for each Instance.
+    def each( &block )
+        @list.keys.each do |url|
+            block.call connect( url )
+        end
+    end
+
+    #
+    # @param    [String, RPC::Client::Instance] client_or_url
+    #
+    # @return   [String]    Cached authentication token for the given Instance.
+    #
+    def token_for( client_or_url )
+        @list[client_or_url.is_a?( String ) ? client_or_url : client_or_url.url ]
+    end
+
+    # Spawns an {RPC::Server::Instance} process.
+    #
+    # @param    [Hash]  options
+    #   To be passed to {Cuboid::Options#set}. Allows `address` instead of
+    #   `rpc_server_address` and `port` instead of `rpc_port`.
+    #
+    # @return   [RPC::Client::Instance, Integer]
+    #   RPC client and PID.
+    def spawn( options = {}, &block )
+        token = options.delete(:token) || Utilities.generate_token
+        fork  = options.delete(:fork)
+
+        port_range = options.delete( :port_range )
+
+        options = {
+            rpc:    {
+                server_socket:  options[:socket],
+                server_port:    options[:port]    || Utilities.available_port( port_range ),
+                server_address: options[:address] || '127.0.0.1'
+            },
+            paths: {
+                application: options[:application]  || Options.paths.application
+            }
+        }
+
+        url = nil
+        if options[:rpc][:server_socket]
+            url = options[:rpc][:server_socket]
+
+            options[:rpc].delete :server_address
+            options[:rpc].delete :server_port
+        else
+            url = "#{options[:rpc][:server_address]}:#{options[:rpc][:server_port]}"
+        end
+
+        pid = Manager.spawn( :instance, options: options, token: token, fork: fork )
+
+        System.slots.use pid
+
+        client = connect( url, token )
+        client.pid = pid
+
+        if block_given?
+            client.when_ready do
+                block.call client
+            end
+        else
+            while sleep( 0.1 )
+                begin
+                    client.alive?
+                    break
+                rescue => e
+                    # ap "#{e.class}: #{e}"
+                    # ap e.backtrace
+                end
+            end
+
+            client
+        end
+    end
+
+    # Starts {RPC::Server::Dispatcher} grid and returns a high-performance Instance.
+    #
+    # @param    [Hash]  options
+    # @option options [Integer] :grid_size (3)  Amount of Dispatchers to spawn.
+    #
+    # @return   [RPC::Client::Instance]
+    def grid_spawn(options = {} )
+        options[:grid_size] ||= 3
+
+        last_member = nil
+        options[:grid_size].times do |i|
+            last_member = Dispatchers.spawn(
+                neighbour: last_member ? last_member.url : last_member,
+                pipe_id:   Utilities.available_port.to_s + Utilities.available_port.to_s
+            )
+        end
+
+        info = nil
+        info = last_member.dispatch while !info && sleep( 0.1 )
+
+        connect( info['url'], info['token'] )
+    end
+
+    # Starts {RPC::Server::Dispatcher} and returns an Instance.
+    #
+    # @return   [RPC::Client::Instance]
+    def dispatcher_spawn
+        info = Dispatchers.spawn.dispatch
+        connect( info['url'], info['token'] )
+    end
+
+    def kill( url )
+        service = connect( url )
+        service.consumed_pids do |pids|
+            service.shutdown do
+                # Make sure....
+                Manager.kill_many pids
+            end
+        end
+
+        @list.delete url
+    end
+
+    # Kills all {Instances #list}.
+    def killall
+        pids = []
+        each do |instance|
+            begin
+                Timeout.timeout 5 do
+                    pids |= instance.consumed_pids
+                end
+            rescue => e
+                #ap e
+                #ap e.backtrace
+            end
+        end
+
+        each do |instance|
+            begin
+                Timeout.timeout 5 do
+                    instance.shutdown
+                end
+            rescue => e
+                #ap e
+                #ap e.backtrace
+            end
+        end
+
+        @list.clear
+        @instance_connections.clear
+        Manager.kill_many pids
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+end
+
+end
+end