cuboid 0.0.0 → 0.0.3

data/lib/cuboid/options.rb

@@ -0,0 +1,254 @@
+require 'yaml'
+require 'singleton'
+
+require_relative 'error'
+require_relative 'utilities'
+
+module Cuboid
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+# @see OptionGroups
+class Options
+    include Singleton
+
+    def self.attr_accessor(*vars)
+        @attr_accessors ||= []
+        @attr_accessors |= vars
+        super( *vars )
+    end
+
+    def self.attr_accessors
+        @attr_accessors
+    end
+
+    def attr_accessors
+        self.class.attr_accessors
+    end
+
+    # {Options} error namespace.
+    #
+    # All {Options} errors inherit from and live under it.
+    #
+    # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+    class Error < Cuboid::Error
+    end
+
+    class <<self
+
+        def method_missing( sym, *args, &block )
+            if instance.respond_to?( sym )
+                instance.send( sym, *args, &block )
+            else
+                super( sym, *args, &block )
+            end
+        end
+
+        def respond_to?( *args )
+            super || instance.respond_to?( *args )
+        end
+
+        # Ruby 2.0 doesn't like my class-level method_missing for some reason.
+        # @private
+        public :allocate
+
+        # @return   [Hash<Symbol,OptionGroup>]
+        #   {OptionGroups Option group} classes by name.
+        def group_classes
+            @group_classes ||= {}
+        end
+
+        # Should be called by {OptionGroup.inherited}.
+        # @private
+        def register_group( group )
+            name = Utilities.caller_name
+
+            # Prepare an attribute reader for this group...
+            attr_reader name
+
+            # ... and initialize it.
+            instance_variable_set "@#{name}".to_sym, group.new
+
+            group_classes[name.to_sym] = group
+        end
+    end
+
+    # Load all {OptionGroups}.
+    require_relative 'option_groups'
+
+    TO_RPC_IGNORE = Set.new([
+        :instance, :rpc, :dispatcher, :queue, :paths,
+        :snapshot, :report, :output, :system
+    ])
+
+    TO_HASH_IGNORE = Set.new([ :instance ])
+
+
+    # @return    [String]
+    #   E-mail address of the person that authorized the run.
+    #
+    # @see HTTP::Client#headers
+    attr_accessor :authorized_by
+
+    attr_accessor :application
+
+    def initialize
+        reset
+    end
+
+    # Restores everything to their default values.
+    #
+    # @return [Options] `self`
+    def reset
+        # nil everything out.
+        instance_variables.each { |var| instance_variable_set( var.to_s, nil ) }
+
+        # Set fresh option groups.
+        group_classes.each do |name, klass|
+            instance_variable_set "@#{name}".to_sym, klass.new
+        end
+
+        @authorized_by  = nil
+
+        self
+    end
+
+
+    # Configures options via a Hash object.
+    #
+    # @param    [Hash]  options
+    #   If the key refers to a class attribute, the attribute will be assigned
+    #   the given value, if it refers to one of the {OptionGroups} the value
+    #   should be a hash with data to update that {OptionGroup group} using
+    #   {OptionGroup#update}.
+    #
+    # @return   [Options]
+    #
+    # @see OptionGroups
+    def update( options )
+        options.each do |k, v|
+            k = k.to_sym
+            if group_classes.include? k
+                send( k ).update v
+            else
+                send( "#{k.to_s}=", v )
+            end
+        end
+
+        self
+    end
+    alias :set :update
+
+    # @return   [Hash]
+    #   Hash of errors with the name of the invalid options/groups as the keys.
+    def validate
+        errors = {}
+        group_classes.keys.each do |name|
+            next if (group_errors = send(name).validate).empty?
+            errors[name] = group_errors
+        end
+        errors
+    end
+
+    # @param    [String]    file
+    #   Saves `self` to `file` using YAML.
+    def save( file )
+        File.open( file, 'w' ) do |f|
+            f.write to_save_data
+            f.path
+        end
+    end
+
+    def to_save_data
+        to_rpc_data.to_yaml
+    end
+
+    def to_save_data_without_defaults
+        to_rpc_data_without_defaults.to_yaml
+    end
+
+    # Loads a file created by {#save}.
+    #
+    # @param    [String]    filepath
+    #   Path to the file created by {#save}.
+    #
+    # @return   [Cuboid::Options]
+    def load( filepath )
+        update( YAML.load_file( filepath ) )
+    end
+
+    # @return    [Hash]
+    #   `self` converted to a Hash suitable for RPC transmission.
+    def to_rpc_data
+        hash = {}
+        instance_variables.each do |var|
+            val = instance_variable_get( var )
+            var = normalize_name( var )
+
+            next if TO_RPC_IGNORE.include?( var )
+
+            hash[var.to_s] = (val.is_a? OptionGroup) ? val.to_rpc_data : val
+        end
+        hash.deep_clone
+    end
+
+    def to_rpc_data_without_defaults
+        defaults = self.class.allocate.reset.to_rpc_data
+        to_rpc_data.reject { |k, v| defaults[k] == v }
+    end
+
+    # @return    [Hash]
+    #   `self` converted to a Hash.
+    def to_hash
+        hash = {}
+        instance_variables.each do |var|
+            val = instance_variable_get( var )
+            var = normalize_name( var )
+
+            next if TO_HASH_IGNORE.include?( var )
+
+            hash[var] = (val.is_a? OptionGroup) ? val.to_h : val
+        end
+
+        hash.deep_clone
+    end
+    alias :to_h :to_hash
+
+    # @param    [Hash]  hash
+    #   Hash to convert into {#to_hash} format.
+    #
+    # @return   [Hash]
+    #   `hash` in {#to_hash} format.
+    def rpc_data_to_hash( hash )
+        self.class.allocate.reset.update( hash ).to_hash.
+            reject { |k| TO_RPC_IGNORE.include? k }
+    end
+
+    # @param    [Hash]  hash
+    #   Hash to convert into {#to_rpc_data} format.
+    #
+    # @return   [Hash]
+    #   `hash` in {#to_rpc_data} format.
+    def hash_to_rpc_data( hash )
+        self.class.allocate.reset.update( hash ).to_rpc_data
+    end
+
+    def hash_to_save_data( hash )
+        self.class.allocate.reset.update( hash ).to_save_data
+    end
+
+    def dup
+        self.class.allocate.reset.update( self.to_h )
+    end
+
+    private
+
+    def group_classes
+        self.class.group_classes
+    end
+
+    def normalize_name( name )
+        name.to_s.gsub( '@', '' ).to_sym
+    end
+
+end
+end

data/lib/cuboid/processes/dispatchers.rb

@@ -0,0 +1,140 @@
+module Cuboid
+module Processes
+
+#
+# Helper for managing {RPC::Server::Dispatcher} processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+class Dispatchers
+    include Singleton
+    include Utilities
+
+    # @return   [Array<String>] URLs of all running Dispatchers.
+    attr_reader :list
+
+    def initialize
+        @list = []
+        @dispatcher_connections = {}
+    end
+
+    # Connects to a Dispatcher by URL.
+    #
+    # @param    [String]    url URL of the Dispatcher.
+    # @param    [Hash]    options Options for the RPC client.
+    #
+    # @return   [RPC::Client::Dispatcher]
+    def connect( url, options = nil )
+        Arachni::Reactor.global.run_in_thread if !Arachni::Reactor.global.running?
+
+        fresh = false
+        if options
+            fresh = options.delete( :fresh )
+        end
+
+        if fresh
+            @dispatcher_connections[url] = RPC::Client::Dispatcher.new( url, options )
+        else
+            @dispatcher_connections[url] ||= RPC::Client::Dispatcher.new( url, options )
+        end
+    end
+
+    # @param    [Block] block   Block to pass an RPC client for each Dispatcher.
+    def each( &block )
+        @list.each do |url|
+            block.call connect( url )
+        end
+    end
+
+    # Spawns a {RPC::Server::Dispatcher} process.
+    #
+    # @param    [Hash]  options
+    #   To be passed to {Cuboid::Options#set}. Allows `address` instead of
+    #   `rpc_server_address` and `port` instead of `rpc_port`.
+    #
+    # @return   [RPC::Client::Dispatcher]
+    def spawn( options = {} )
+        fork = options.delete(:fork)
+
+        options = {
+            dispatcher: {
+                neighbour: options[:neighbour],
+            },
+            rpc:        {
+                server_port:             options[:port]    || Utilities.available_port,
+                server_address:          options[:address] || '127.0.0.1',
+                server_external_address: options[:external_address]
+            },
+            paths: {
+                application: options[:application] || Options.paths.application
+            }
+        }
+
+        if options[:rpc][:server_external_address].nil?
+            options[:rpc].delete :server_external_address
+        end
+
+        if options[:dispatcher][:neighbour].nil?
+            options[:dispatcher].delete :neighbour
+        end
+
+        pid = Manager.spawn( :dispatcher, options: options, fork: fork )
+
+        url = "#{options[:rpc][:server_address]}:#{options[:rpc][:server_port]}"
+        while sleep( 0.1 )
+            begin
+                connect( url, connection_pool_size: 1, max_retries: 1 ).alive?
+                break
+            rescue => e
+                # ap e
+            end
+        end
+
+        @list << url
+        connect( url, fresh: true ).tap { |c| c.pid = pid }
+    end
+
+    def grid_spawn( options = {} )
+        d = spawn( options )
+        spawn( options.merge neighbour: d.url )
+    end
+
+    # @note Will also kill all Instances started by the Dispatcher.
+    #
+    # @param    [String]    url URL of the Dispatcher to kill.
+    def kill( url )
+        dispatcher = connect( url )
+        Manager.kill_many dispatcher.statistics['consumed_pids']
+        Manager.kill dispatcher.pid
+    rescue => e
+        #ap e
+        #ap e.backtrace
+        nil
+    ensure
+        @list.delete( url )
+        @dispatcher_connections.delete( url )
+    end
+
+    # Kills all {Dispatchers #list}.
+    def killall
+        @list.dup.each do |url|
+            kill url
+        end
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+
+end
+
+end
+end

data/lib/cuboid/processes/executables/base.rb

@@ -0,0 +1,54 @@
+require 'base64'
+
+$options = Marshal.load( Base64.strict_decode64( ARGV.pop ) )
+
+if !$options[:without_cuboid]
+    require 'cuboid'
+
+    include Cuboid
+
+    cuboid_options = Marshal.load( Base64.strict_decode64( ENV['CUBOID_SPAWN_OPTIONS'] ) )
+    Options.update cuboid_options
+
+    require( Options.paths.application ) if Options.paths.application
+else
+    if Gem.win_platform?
+        require 'Win32API'
+        require 'win32ole'
+    end
+end
+
+def ppid
+    $options[:ppid]
+end
+
+def parent_alive?
+    # Windows is not big on POSIX so try it its own way if possible.
+    if Gem.win_platform?
+        begin
+            alive = false
+            wmi = WIN32OLE.connect( 'winmgmts://' )
+            processes = wmi.ExecQuery( "select ProcessId from win32_process where ProcessID='#{ppid}'")
+            processes.each do |proc|
+                proc.ole_free
+                alive = true
+            end
+            processes.ole_free
+            wmi.ole_free
+
+            return alive
+        rescue WIN32OLERuntimeError
+        end
+    end
+
+    !!(Process.kill( 0, ppid ) rescue false)
+end
+
+def puts_stderr( str )
+    return if $stderr.closed?
+
+    $stderr.puts str
+rescue
+end
+
+load ARGV.pop

data/lib/cuboid/processes/executables/dispatcher.rb

@@ -0,0 +1,5 @@
+require Options.paths.lib  + 'rpc/server/dispatcher'
+
+Arachni::Reactor.global.run do
+    RPC::Server::Dispatcher.new
+end

data/lib/cuboid/processes/executables/instance.rb

@@ -0,0 +1,12 @@
+require Options.paths.lib  + 'rpc/server/instance'
+
+if (socket = $options[:socket])
+    Options.rpc.server_address          = nil
+    Options.rpc.server_external_address = nil
+    Options.rpc.server_port             = nil
+    Options.rpc.server_socket           = socket
+elsif (port = $options[:port])
+    Options.rpc.server_port = port
+end
+
+RPC::Server::Instance.new( Options, $options[:token] )

data/lib/cuboid/processes/executables/rest_service.rb

@@ -0,0 +1,13 @@
+require Options.paths.lib  + 'rest/server'
+
+Rest::Server.run!(
+    port:     Options.rpc.server_port,
+    bind:     Options.rpc.server_address,
+
+    username: Options.datastore['username'],
+    password: Options.datastore['password'],
+
+    ssl_ca:          Options.rpc.ssl_ca,
+    ssl_key:         Options.rpc.server_ssl_private_key,
+    ssl_certificate: Options.rpc.server_ssl_certificate
+)

data/lib/cuboid/processes/executables/scheduler.rb

@@ -0,0 +1,5 @@
+require Options.paths.lib  + 'rpc/server/scheduler'
+
+Arachni::Reactor.global.run do
+    RPC::Server::Scheduler.new
+end

data/lib/cuboid/processes/helpers/dispatchers.rb

@@ -0,0 +1,23 @@
+# @param (see Cuboid::Processes::Dispatchers#spawn)
+# @return (see Cuboid::Processes::Dispatchers#spawn)
+def dispatcher_spawn( *args )
+    Cuboid::Processes::Dispatchers.spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Dispatchers#kill)
+# @return (see Cuboid::Processes::Dispatchers#kill)
+def dispatcher_kill( *args )
+    Cuboid::Processes::Dispatchers.kill( *args )
+end
+
+# @param (see Cuboid::Processes::Dispatchers#killall)
+# @return (see Cuboid::Processes::Dispatchers#killall)
+def dispatcher_killall
+    Cuboid::Processes::Dispatchers.killall
+end
+
+# @param (see Cuboid::Processes::Dispatchers#connect)
+# @return (see Cuboid::Processes::Dispatchers#connect)
+def dispatcher_connect( *args )
+    Cuboid::Processes::Dispatchers.connect( *args )
+end

data/lib/cuboid/processes/helpers/instances.rb

@@ -0,0 +1,39 @@
+# @param (see Cuboid::Processes::Instances#spawn)
+# @return (see Cuboid::Processes::Instances#spawn)
+def instance_spawn( *args )
+    Cuboid::Processes::Instances.spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Instances#grid_spawn)
+# @return (see Cuboid::Processes::Instances#grid_spawn)
+def instance_grid_spawn( *args )
+    Cuboid::Processes::Instances.grid_spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Instances#dispatcher_spawn)
+# @return (see Cuboid::Processes::Instances#dispatcher_spawn)
+def instance_dispatcher_spawn( *args )
+    Cuboid::Processes::Instances.dispatcher.spawn( *args )
+end
+
+def instance_kill( url )
+    Cuboid::Processes::Instances.kill url
+end
+
+# @param (see Cuboid::Processes::Instances#killall)
+# @return (see Cuboid::Processes::Instances#killall)
+def instance_killall
+    Cuboid::Processes::Instances.killall
+end
+
+# @param (see Cuboid::Processes::Instances#connect)
+# @return (see Cuboid::Processes::Instances#connect)
+def instance_connect( *args )
+    Cuboid::Processes::Instances.connect( *args )
+end
+
+# @param (see Cuboid::Processes::Instances#token_for)
+# @return (see Cuboid::Processes::Instances#token_for)
+def instance_token_for( *args )
+    Cuboid::Processes::Instances.token_for( *args )
+end

data/lib/cuboid/processes/helpers/processes.rb

@@ -0,0 +1,23 @@
+# @param (see Cuboid::Processes::Manager#kill_reactor)
+# @return (see Cuboid::Processes::Manager#kill_reactor)
+def process_kill_reactor( *args )
+    Cuboid::Processes::Manager.kill_reactor( *args )
+end
+
+# @param (see Cuboid::Processes::Manager#kill)
+# @return (see Cuboid::Processes::Manager#kill)
+def process_kill( *args )
+    Cuboid::Processes::Manager.kill( *args )
+end
+
+# @param (see Cuboid::Processes::Manager#killall)
+# @return (see Cuboid::Processes::Manager#killall)
+def process_killall( *args )
+    Cuboid::Processes::Manager.killall( *args )
+end
+
+# @param (see Cuboid::Processes::Manager#kill_many)
+# @return (see Cuboid::Processes::Manager#kill_many)
+def process_kill_many( *args )
+    Cuboid::Processes::Manager.kill_many( *args )
+end

data/lib/cuboid/processes/helpers/schedulers.rb

@@ -0,0 +1,23 @@
+# @param (see Cuboid::Processes::Schedulers#spawn)
+# @return (see Cuboid::Processes::Schedulers#spawn)
+def scheduler_spawn( *args )
+    Cuboid::Processes::Schedulers.spawn( *args )
+end
+
+# @param (see Cuboid::Processes::Schedulers#kill)
+# @return (see Cuboid::Processes::Schedulers#kill)
+def scheduler_kill( *args )
+    Cuboid::Processes::Schedulers.kill( *args )
+end
+
+# @param (see Cuboid::Processes::Schedulers#killall)
+# @return (see Cuboid::Processes::Schedulers#killall)
+def scheduler_killall
+    Cuboid::Processes::Schedulers.killall
+end
+
+# @param (see Cuboid::Processes::Schedulers#connect)
+# @return (see Cuboid::Processes::Schedulers#connect)
+def scheduler_connect( *args )
+    Cuboid::Processes::Schedulers.connect( *args )
+end

data/lib/cuboid/processes/helpers.rb

@@ -0,0 +1,4 @@
+require_relative 'helpers/processes'
+require_relative 'helpers/dispatchers'
+require_relative 'helpers/instances'
+require_relative 'helpers/schedulers'