cuboid 0.0.0 → 0.0.3

data/lib/cuboid/processes/instances.rb

@@ -0,0 +1,203 @@
+module Cuboid
+module Processes
+
+#
+# Helper for managing {RPC::Server::Instance} processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Instances
+    include Singleton
+    include Utilities
+
+    # @return   [Array<String>] URLs and tokens of all running Instances.
+    attr_reader :list
+
+    def initialize
+        @list = {}
+        @instance_connections = {}
+    end
+
+    #
+    # Connects to a Instance by URL.
+    #
+    # @param    [String]    url URL of the Dispatcher.
+    # @param    [String]    token
+    #   Authentication token -- only need be provided once.
+    #
+    # @return   [RPC::Client::Instance]
+    #
+    def connect( url, token = nil )
+        Arachni::Reactor.global.run_in_thread if !Arachni::Reactor.global.running?
+
+        token ||= @list[url]
+        @list[url] ||= token
+
+        @instance_connections[url] ||= RPC::Client::Instance.new( url, token )
+    end
+
+    # @param    [Block] block   Block to pass an RPC client for each Instance.
+    def each( &block )
+        @list.keys.each do |url|
+            block.call connect( url )
+        end
+    end
+
+    #
+    # @param    [String, RPC::Client::Instance] client_or_url
+    #
+    # @return   [String]    Cached authentication token for the given Instance.
+    #
+    def token_for( client_or_url )
+        @list[client_or_url.is_a?( String ) ? client_or_url : client_or_url.url ]
+    end
+
+    # Spawns an {RPC::Server::Instance} process.
+    #
+    # @param    [Hash]  options
+    #   To be passed to {Cuboid::Options#set}. Allows `address` instead of
+    #   `rpc_server_address` and `port` instead of `rpc_port`.
+    #
+    # @return   [RPC::Client::Instance, Integer]
+    #   RPC client and PID.
+    def spawn( options = {}, &block )
+        token = options.delete(:token) || Utilities.generate_token
+        fork  = options.delete(:fork)
+
+        port_range = options.delete( :port_range )
+
+        options = {
+            rpc:    {
+                server_socket:  options[:socket],
+                server_port:    options[:port]    || Utilities.available_port( port_range ),
+                server_address: options[:address] || '127.0.0.1'
+            },
+            paths: {
+                application: options[:application]  || Options.paths.application
+            }
+        }
+
+        url = nil
+        if options[:rpc][:server_socket]
+            url = options[:rpc][:server_socket]
+
+            options[:rpc].delete :server_address
+            options[:rpc].delete :server_port
+        else
+            url = "#{options[:rpc][:server_address]}:#{options[:rpc][:server_port]}"
+        end
+
+        pid = Manager.spawn( :instance, options: options, token: token, fork: fork )
+
+        System.slots.use pid
+
+        client = connect( url, token )
+        client.pid = pid
+
+        if block_given?
+            client.when_ready do
+                block.call client
+            end
+        else
+            while sleep( 0.1 )
+                begin
+                    client.alive?
+                    break
+                rescue => e
+                    # ap "#{e.class}: #{e}"
+                    # ap e.backtrace
+                end
+            end
+
+            client
+        end
+    end
+
+    # Starts {RPC::Server::Dispatcher} grid and returns a high-performance Instance.
+    #
+    # @param    [Hash]  options
+    # @option options [Integer] :grid_size (3)  Amount of Dispatchers to spawn.
+    #
+    # @return   [RPC::Client::Instance]
+    def grid_spawn(options = {} )
+        options[:grid_size] ||= 3
+
+        last_member = nil
+        options[:grid_size].times do |i|
+            last_member = Dispatchers.spawn(
+                neighbour: last_member ? last_member.url : last_member,
+                pipe_id:   Utilities.available_port.to_s + Utilities.available_port.to_s
+            )
+        end
+
+        info = nil
+        info = last_member.dispatch while !info && sleep( 0.1 )
+
+        connect( info['url'], info['token'] )
+    end
+
+    # Starts {RPC::Server::Dispatcher} and returns an Instance.
+    #
+    # @return   [RPC::Client::Instance]
+    def dispatcher_spawn
+        info = Dispatchers.spawn.dispatch
+        connect( info['url'], info['token'] )
+    end
+
+    def kill( url )
+        service = connect( url )
+        service.consumed_pids do |pids|
+            service.shutdown do
+                # Make sure....
+                Manager.kill_many pids
+            end
+        end
+
+        @list.delete url
+    end
+
+    # Kills all {Instances #list}.
+    def killall
+        pids = []
+        each do |instance|
+            begin
+                Timeout.timeout 5 do
+                    pids |= instance.consumed_pids
+                end
+            rescue => e
+                #ap e
+                #ap e.backtrace
+            end
+        end
+
+        each do |instance|
+            begin
+                Timeout.timeout 5 do
+                    instance.shutdown
+                end
+            rescue => e
+                #ap e
+                #ap e.backtrace
+            end
+        end
+
+        @list.clear
+        @instance_connections.clear
+        Manager.kill_many pids
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+end
+
+end
+end

data/lib/cuboid/processes/manager.rb

@@ -0,0 +1,262 @@
+require 'singleton'
+require 'arachni/reactor'
+
+module Cuboid
+module Processes
+
+# Helper for managing processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Manager
+    include Singleton
+
+    RUNNER = "#{File.dirname( __FILE__ )}/executables/base.rb"
+
+    # @return   [Array<Integer>] PIDs of all running processes.
+    attr_reader :pids
+
+    def initialize
+        reset
+    end
+
+    def reset
+        @pids           = []
+        @discard_output = true
+    end
+
+    # @param    [Integer]   pid
+    #   Adds a PID to the {#pids} and detaches the process.
+    #
+    # @return   [Integer]   `pid`
+    def <<( pid )
+        @pids << pid
+        Process.detach pid
+        pid
+    end
+
+    # @param    [Integer]   pid
+    #   PID of the process to kill.
+    def kill( pid )
+        fail 'Cannot kill self.' if pid == Process.pid
+
+        Timeout.timeout 10 do
+            while sleep 0.1 do
+                begin
+                    Process.kill( Cuboid.windows? ? 'KILL' : 'TERM', pid )
+
+                # Either kill was successful or we don't have enough perms or
+                # we hit a reused PID for someone else's process, either way,
+                # consider the process gone.
+                rescue Errno::ESRCH, Errno::EPERM,
+                    # Don't kill ourselves.
+                    SignalException
+
+                    @pids.delete pid
+                    return
+                end
+            end
+        end
+    rescue Timeout::Error
+    end
+
+    def find( bin )
+        find_in_path( bin ) || find_in_applications( bin ) ||
+            find_in_program_files( bin )
+    end
+
+    def find_in_path( bin )
+        @find_in_path ||= {}
+        return @find_in_path[bin] if @find_in_path.include?( bin )
+
+        if Cuboid.windows?
+            bin = "#{bin}.exe"
+        end
+
+        ENV['PATH'].split( File::PATH_SEPARATOR ).each do |path|
+            f = File.join( path, bin )
+            return @find_in_path[bin] = f if File.exist?( f )
+        end
+
+        @find_in_path[bin] = nil
+    end
+
+    def find_in_applications( bin )
+        return if !Cuboid.mac?
+
+        @find_in_applications ||= {}
+        return @find_in_applications[bin] if @find_in_applications.include?( bin )
+
+        [
+            '/Applications/*/Contents/MacOS'
+        ].each do |root|
+            glob = File.join( root, '**', bin )
+
+            exe = Dir.glob( glob ).find { |f| File.executable?( f ) }
+            return @find_in_applications[bin] = exe if exe
+        end
+
+        @find_in_applications[bin] = nil
+    end
+
+    def find_in_program_files( bin )
+        return if !Cuboid.windows?
+
+        @find_in_program_files ||= {}
+        return @find_in_program_files[bin] if @find_in_program_files.include?( bin )
+
+        [
+            ENV['PROGRAMFILES']      || '\\Program Files',
+            ENV['ProgramFiles(x86)'] || '\\Program Files (x86)',
+            ENV['ProgramW6432']      || '\\Program Files'
+        ].each do |root|
+            glob = File.join( root, '**', "#{bin}.exe" )
+            glob.tr!( '\\', '/' )
+
+            exe = Dir.glob( glob ).find { |f| File.executable?( f ) }
+            return @find_in_program_files[bin] = exe if exe
+        end
+
+        @find_in_program_files[bin] = nil
+    end
+
+    # @param    [Integer]   pid
+    # @return   [Boolean]
+    #   `true` if the process is alive, `false` otherwise.
+    def alive?( pid )
+        # Windows is not big on POSIX so try it its own way if possible.
+        if Cuboid.windows?
+            begin
+                alive = false
+                processes = wmi.ExecQuery( "select ProcessId from win32_process where ProcessID='#{pid}'" )
+                processes.each do |proc|
+                    proc.ole_free
+                    alive = true
+                end
+                processes.ole_free
+
+                return alive
+            rescue WIN32OLERuntimeError
+            end
+        end
+
+        !!(Process.kill( 0, pid ) rescue false)
+    end
+
+    # @param    [Array<Integer>]   pids
+    #   PIDs of the process to {Cuboid::Processes::Manager#kill}.
+    def kill_many( pids )
+        pids.each { |pid| kill pid }
+    end
+
+    # Kills all {#pids processes}.
+    def killall
+        kill_many @pids.dup
+        @pids.clear
+    end
+
+    # Stops the Reactor.
+    def kill_reactor
+        Arachni::Reactor.stop
+    rescue
+        nil
+    end
+
+    # Overrides the default setting of discarding process outputs.
+    def preserve_output
+        @discard_output = false
+    end
+
+    def preserve_output?
+        !discard_output?
+    end
+
+    def discard_output
+        @discard_output = true
+    end
+
+    def discard_output?
+        @discard_output
+    end
+
+    # @param    [String]    executable
+    #   Name of the executable Ruby script found in {OptionGroups::Paths#executables}
+    #   without the '.rb' extension.
+    # @param    [Hash]  options
+    #   Options to pass to the script -- can be retrieved from `$options`.
+    #
+    # @return   [Integer]
+    #   PID of the process.
+    def spawn( executable, options = {} )
+        fail ArgumentError, 'Fork not supported.' if options.delete(:fork)
+
+        stdin      = options.delete(:stdin)
+        stdout     = options.delete(:stdout)
+        stderr     = options.delete(:stderr)
+        new_pgroup = options.delete(:new_pgroup)
+
+        spawn_options = {}
+
+        if new_pgroup
+            if Cuboid.windows?
+                spawn_options[:new_pgroup] = new_pgroup
+            else
+                spawn_options[:pgroup] = new_pgroup
+            end
+        end
+
+        spawn_options[:in]  = stdin  if stdin
+        spawn_options[:out] = stdout if stdout
+        spawn_options[:err] = stderr if stderr
+
+        options[:ppid]   = Process.pid
+        options[:tmpdir] = Options.paths.tmpdir
+
+        cuboid_options = Options.dup.update( options.delete(:options) || {} ).to_h
+        encoded_cuboid_options = Base64.strict_encode64( Marshal.dump( cuboid_options ) )
+
+        if executable.is_a? Symbol
+            executable = "#{Options.paths.executables}/#{executable}.rb"
+        elsif !File.exist?( executable )
+            raise ArgumentError, "Executable does not exist: #{executable}"
+        end
+
+        encoded_options = Base64.strict_encode64( Marshal.dump( options ) )
+        argv            = [executable, encoded_options]
+
+        # It's very, **VERY** important that we use this argument format as
+        # it bypasses the OS shell and we can thus count on a 1-to-1 process
+        # creation and that the PID we get will be for the actual process.
+        pid = Process.spawn(
+            {
+                'CUBOID_SPAWN_OPTIONS' => encoded_cuboid_options
+            },
+            RbConfig.ruby,
+            RUNNER,
+            *(argv + [spawn_options])
+        )
+
+        self << pid
+        pid
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+
+    private
+
+    def wmi
+        @wmi ||= WIN32OLE.connect( 'winmgmts://' )
+    end
+end
+
+end
+end

data/lib/cuboid/processes/schedulers.rb

@@ -0,0 +1,128 @@
+module Cuboid
+module Processes
+
+# Helper for managing {RPC::Server::Scheduler} processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Schedulers
+    include Singleton
+    include Utilities
+
+    # @return   [Array<String>] URLs of all running Queues.
+    attr_reader :list
+
+    def initialize
+        @list    = []
+        @clients = {}
+    end
+
+    # Connects to a Scheduler by URL.
+    #
+    # @param    [String]    url URL of the Scheduler.
+    # @param    [Hash]    options Options for the RPC client.
+    #
+    # @return   [RPC::Client::Scheduler]
+    def connect( url, options = nil )
+        Arachni::Reactor.global.run_in_thread if !Arachni::Reactor.global.running?
+
+        fresh = false
+        if options
+            fresh = options.delete( :fresh )
+        end
+
+        if fresh
+            @clients[url] = RPC::Client::Scheduler.new( url, options )
+        else
+            @clients[url] ||= RPC::Client::Scheduler.new( url, options )
+        end
+    end
+
+    # @param    [Block] block   Block to pass an RPC client for each Scheduler.
+    def each( &block )
+        @list.each do |url|
+            block.call connect( url )
+        end
+    end
+
+    # Spawns a {RPC::Server::Scheduler} process.
+    #
+    # @param    [Hash]  options
+    #   To be passed to {Cuboid::Options#set}. Allows `address` instead of
+    #   `rpc_server_address` and `port` instead of `rpc_port`.
+    #
+    # @return   [RPC::Client::Queue]
+    def spawn( options = {} )
+        fork = options.delete(:fork)
+
+        options = {
+            dispatcher: {
+                url: options[:dispatcher],
+            },
+            rpc:        {
+                server_port:             options[:port]    || Utilities.available_port,
+                server_address:          options[:address] || '127.0.0.1',
+                server_external_address: options[:external_address]
+            },
+            paths: {
+              application: options[:application]  || Options.paths.application
+            }
+        }
+
+        pid = Manager.spawn( :scheduler, options: options, fork: fork )
+
+        url = "#{options[:rpc][:server_address]}:#{options[:rpc][:server_port]}"
+        while sleep( 0.1 )
+            begin
+                connect( url, connection_pool_size: 1, max_retries: 1 ).alive?
+                break
+            rescue => e
+                # ap e
+            end
+        end
+
+        @list << url
+        connect( url, fresh: true ).tap { |c| c.pid = pid }
+    end
+
+    # @note Will also kill all Instances started by the Scheduler.
+    #
+    # @param    [String]    url URL of the Scheduler to kill.
+    def kill( url )
+        scheduler = connect( url )
+        scheduler.clear
+        scheduler.running.each do |id, instance|
+            Manager.kill instance['pid']
+        end
+        Manager.kill scheduler.pid
+    rescue => e
+        #ap e
+        #ap e.backtrace
+        nil
+    ensure
+        @list.delete( url )
+        @clients.delete( url ).close
+    end
+
+    # Kills all {Queues #list}.
+    def killall
+        @list.dup.each do |url|
+            kill url
+        end
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+
+end
+
+end
+end

data/lib/cuboid/processes.rb

@@ -0,0 +1,13 @@
+require 'singleton'
+require 'ostruct'
+
+lib = Cuboid::Options.paths.lib
+require lib + 'rpc/client/instance'
+require lib + 'rpc/client/dispatcher'
+require lib + 'rpc/client/scheduler'
+
+lib = Cuboid::Options.paths.lib + 'processes/'
+require lib + 'manager'
+require lib + 'dispatchers'
+require lib + 'instances'
+require lib + 'schedulers'