cuboid 0.0.0 → 0.0.1alpha

data/lib/cuboid/rpc/serializer.rb

@@ -0,0 +1,92 @@
+require 'msgpack'
+
+module Cuboid
+module RPC
+
+# Used for serialization of {RPC} messages.
+#
+# It's simply a delegator for `MessagePack` with `Zlib` compression for messages
+# that are larger than {COMPRESS_LARGER_THAN}.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+module Serializer
+
+    # Compress object dumps larger than 1KB.
+    COMPRESS_LARGER_THAN = 1_000
+
+    # @param    [#to_rpc_data]   object
+    #
+    # @return   [String]
+    #   {#compress Compressed} `object` dump.
+    def dump( object )
+        # ap object
+        compress( serializer.dump( object.to_rpc_data_or_self ) )
+    end
+
+    # @param    [String]   dump
+    #   {#dump Dumped} object.
+    #
+    # @return   [Object]
+    def load( dump )
+        serializer.load( decompress( dump ) )
+    end
+
+    # Simulates an object's over-the-wire transmission by {#dump dumping}
+    # and then {#load loading}.
+    #
+    # @param    [#to_rpc_data,.from_rpc_data]   object
+    #
+    # @return   [Object]
+    #   Data that the peer would receive.
+    def rpc_data( object )
+        load( dump( object ) )
+    end
+
+    # @param    [#to_rpc_data,.from_rpc_data]   object
+    #
+    # @return   [Object]
+    def deep_clone( object )
+        object.class.from_rpc_data rpc_data( object )
+    end
+
+    # @note Ignores strings smaller than #{COMPRESS_LARGER_THAN}.
+    #
+    # @param    [String]    string
+    #   String to compress.
+    #
+    # @return   [String]
+    #   Compressed (or not) `string`.
+    def compress( string )
+        return string if string.size < COMPRESS_LARGER_THAN
+        Zlib::Deflate.deflate string
+    end
+
+    # @note Will return the `string` as is if it was not compressed.
+    #
+    # @param    [String]    string
+    #   String to decompress.
+    #
+    # @return   [String]
+    #   Decompressed string.
+    def decompress( string )
+        return '' if string.to_s.empty?
+
+        # Just an ID representing a serialized, empty data structure.
+        return string if string.size == 1
+
+        begin
+            Zlib::Inflate.inflate string
+        rescue Zlib::DataError
+            string
+        end
+    end
+
+    def serializer
+        MessagePack
+    end
+
+    extend self
+end
+
+end
+end

data/lib/cuboid/rpc/server/active_options.rb

@@ -0,0 +1,38 @@
+module Cuboid
+module RPC
+class Server
+
+# It, for the most part, forwards calls to {Cuboid::Options} and intercepts
+# a few that need to be updated at other places throughout the framework.
+#
+# @private
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class ActiveOptions
+
+    def initialize
+        @options = Cuboid::Options.instance
+
+        (@options.public_methods( false ) - public_methods( false ) ).each do |m|
+            self.class.class_eval do
+                define_method m do |*args|
+                    @options.send( m, *args )
+                end
+            end
+        end
+    end
+
+    # @see Cuboid::Options#set
+    def set( options )
+        @options.set( options )
+        true
+    end
+
+    def to_h
+        @options.to_rpc_data
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/server/application_wrapper.rb

@@ -0,0 +1,138 @@
+require 'tempfile'
+
+module Cuboid
+
+lib = Options.paths.lib
+require lib + 'application'
+
+module RPC
+class Server
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class ApplicationWrapper
+    include Utilities
+
+    attr_reader :application
+
+    extend Forwardable
+    def_delegators :@application, :suspended?, :suspend!, :status,
+                   :pause!, :running?, :status_messages, :paused?,
+                   :snapshot_path, :restore!, :resume!, :generate_report
+
+    # {RPC::Server::Application} error namespace.
+    #
+    # All {RPC::Server::Application} errors inherit from and live under it.
+    #
+    # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+    class Error < Cuboid::Application::Error
+    end
+
+    def initialize( application )
+        super()
+
+        @application = application.instance
+        @extended_running = false
+    end
+
+    def valid_options?( options )
+        @application.class.valid_options?( options )
+    end
+
+    # @return   [Bool]
+    #   `true` If the system is scanning, `false` if {#run} hasn't been called
+    #   yet or if the scan has finished.
+    def busy?
+        ![:ready, :done, :suspended].include?( @application.status ) &&
+          !!@extended_running
+    end
+
+    # @return   [Bool]
+    #   `false` if already running, `true` otherwise.
+    def run
+        # Return if we're already running.
+        return false if busy?
+        @extended_running = true
+
+        # Start the scan  -- we can't block the RPC server so we're using a Thread.
+        Thread.new do
+            @application.run
+        end
+
+        true
+    end
+
+    def clean_up
+        return false if @rpc_cleaned_up
+
+        @rpc_cleaned_up   = true
+        @extended_running = false
+
+        @application.clean_up
+    end
+
+    # @param    [Integer]   starting_line
+    #   Sets the starting line for the range of errors to return.
+    #
+    # @return   [Array<String>]
+    def errors( starting_line = 0 )
+        return [] if UI::Output.error_buffer.empty?
+
+        error_strings = UI::Output.error_buffer
+
+        if starting_line != 0
+            error_strings = error_strings[starting_line..-1]
+        end
+
+        error_strings
+    end
+
+    # Provides aggregated progress data.
+    #
+    # @param    [Hash]  opts
+    #   Options about what data to include:
+    # @option opts [Bool] :statistics   (true)
+    #   Master/merged statistics.
+    # @option opts [Bool, Integer] :errors   (false)
+    #   Logged errors. If an integer is provided it will return errors past that
+    #   index.
+    #
+    # @return    [Hash]
+    #   Progress data.
+    def progress( opts = {} )
+        opts = opts.my_symbolize_keys
+
+        include_statistics = opts[:statistics].nil? ? true : opts[:statistics]
+        include_errors     = opts.include?( :errors ) ?
+            (opts[:errors] || 0) : false
+
+        data = {
+            status:         status,
+            busy:           running?,
+            application:    @application.class.to_s,
+            seed:           Utilities.random_seed,
+            dispatcher_url: Cuboid::Options.dispatcher.url,
+            scheduler_url:  Cuboid::Options.scheduler.url
+        }
+
+        if include_statistics
+            data[:statistics] = @application.statistics
+        end
+
+        if include_errors
+            data[:errors] =
+                errors( include_errors.is_a?( Integer ) ? include_errors : 0 )
+        end
+
+        data.merge( messages: status_messages )
+    end
+
+    # @private
+    def error_test( str )
+        @application.print_error str.to_s
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/server/base.rb

@@ -0,0 +1,63 @@
+require 'ostruct'
+require 'arachni/rpc'
+require_relative '../serializer'
+
+module Cuboid
+module RPC
+class Server
+
+# RPC server class
+#
+# @private
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Base < Arachni::RPC::Server
+
+    # @param    [Hash]   options
+    # @option options [Integer]  :host
+    # @option options [Integer]  :port
+    # @option options [Integer]  :socket
+    # @option options [Integer]  :ssl_ca
+    # @option options [Integer]  :ssl_pkey
+    # @option options [Integer]  :ssl_cert
+    # @param    [String]    token
+    #   Optional authentication token.
+    def initialize( options = nil, token = nil )
+
+        # If given nil use the global defaults.
+        options ||= Options.rpc.to_server_options
+        @options = options
+
+        super(options.merge(
+            serializer: Serializer,
+            token:      token
+        ))
+    end
+
+    def address
+        @options[:external_address] || @options[:host]
+    end
+
+    def port
+        @options[:port]
+    end
+
+    def url
+        return @options[:socket] if @options[:socket]
+
+        "#{address}:#{port}"
+    end
+
+    def start
+        super
+        @ready = true
+    end
+
+    def ready?
+        @ready ||= false
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/server/dispatcher.rb

@@ -0,0 +1,317 @@
+module Cuboid
+
+lib = Options.paths.lib
+require lib + 'processes/instances'
+require lib + 'rpc/client'
+require lib + 'rpc/server/base'
+require lib + 'rpc/server/instance'
+require lib + 'rpc/server/output'
+
+module RPC
+class Server
+
+# Dispatches RPC Instances on demand and allows for extensive process monitoring.
+#
+# The process goes something like this:
+#
+# * A client issues a {#dispatch} call.
+# * The Dispatcher spawns and returns Instance info to the client (url, auth token, etc.).
+# * The client connects to the Instance using that info.
+#
+# Once the client finishes using the RPC Instance it *must* shut it down
+# otherwise the system will be eaten away by zombie processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Dispatcher
+    require Options.paths.lib + 'rpc/server/dispatcher/node'
+    require Options.paths.lib + 'rpc/server/dispatcher/service'
+
+    include Utilities
+    include UI::Output
+
+    SERVICE_NAMESPACE = Service
+
+    def initialize( options = Options.instance )
+        @options = options
+
+        @options.snapshot.path ||= @options.paths.snapshots
+
+        @server = Base.new( @options.rpc.to_server_options )
+        @server.logger.level = @options.datastore.log_level if @options.datastore.log_level
+
+        @server.add_async_check do |method|
+            # methods that expect a block are async
+            method.parameters.flatten.include? :block
+        end
+
+        Options.dispatcher.url = @url = @server.url
+
+        prep_logging
+
+        print_status 'Starting the RPC Server...'
+
+        @server.add_handler( 'dispatcher', self )
+
+        # trap interrupts and exit cleanly when required
+        trap_interrupts { shutdown }
+
+        @instances = []
+
+        Cuboid::Application.application.dispatcher_services.each do |name, service|
+            @server.add_handler( name.to_s, service.new( @options, self ) )
+        end
+
+        @node = Node.new( @options, @server, @logfile )
+        @server.add_handler( 'node', @node )
+
+        run
+    end
+
+    def services
+        Cuboid::Application.application.dispatcher_services.keys
+    end
+
+    # @return   [TrueClass]
+    #   true
+    def alive?
+        @server.alive?
+    end
+
+    # @return   [String, nil]
+    #   Depending on availability:
+    #
+    #   * URL of the least burdened Dispatcher. If not a grid member it will
+    #       return this Dispatcher's URL.
+    #   * `nil` if all nodes are at max utilization.
+    def preferred( &block )
+        if !@node.grid_member?
+            block.call( self.utilization == 1 ? nil : @url )
+            return
+        end
+
+        pick_utilization = proc do |url, utilization|
+            (utilization == 1 || utilization.rpc_exception?) ?
+                nil : [url, utilization]
+        end
+
+        each = proc do |neighbour, iter|
+            connect_to_peer( neighbour ).utilization do |utilization|
+                iter.return pick_utilization.call( neighbour, utilization )
+            end
+        end
+
+        after = proc do |nodes|
+            nodes << pick_utilization.call( @url, self.utilization )
+            nodes.compact!
+
+            # All nodes are at max utilization, pass.
+            if nodes.empty?
+                block.call
+                next
+            end
+
+            block.call nodes.sort_by { |_, score| score }[0][0]
+        end
+
+        Arachni::Reactor.global.create_iterator( @node.neighbours ).map( each, after )
+    end
+
+    # Dispatches an {Instance}.
+    #
+    # @param    [String]  owner
+    #   An owner to assign to the {Instance}.
+    # @param    [Hash]    helpers
+    #   Hash of helper data to be added to the instance info.
+    # @param    [Boolean]    load_balance
+    #   Return an {Instance} from the least burdened {Dispatcher} (when in Grid mode)
+    #   or from this one directly?
+    #
+    # @return   [Hash, nil]
+    #   Depending on availability:
+    #
+    #   * `Hash`: Connection and proc info.
+    #   * `nil`: Max utilization, wait for one of the instances to finish and retry.
+    def dispatch( options = {}, &block )
+        options      = options.my_symbolize_keys
+        owner        = options[:owner]
+        helpers      = options[:helpers] || {}
+        load_balance = options[:load_balance].nil? ? true : options[:load_balance]
+
+        if load_balance && @node.grid_member?
+            preferred do |url|
+                if !url
+                    block.call
+                    next
+                end
+
+                connect_to_peer( url ).dispatch( options.merge(
+                      helpers: helpers.merge( via: @url ),
+                      load_balance: false
+                    ),
+                    &block
+                )
+            end
+            return
+        end
+
+        if System.max_utilization?
+            block.call
+            return
+        end
+
+        spawn_instance do |info|
+            info['owner']   = owner
+            info['helpers'] = helpers
+
+            @instances << info
+
+            block.call info
+        end
+    end
+
+    # Returns proc info for a given pid
+    #
+    # @param    [Fixnum]      pid
+    #
+    # @return   [Hash]
+    def instance( pid )
+        @instances.each do |i|
+            next if i['pid'] != pid
+            i = i.dup
+
+            now = Time.now
+
+            i['now']   = now.to_s
+            i['age']   = now - Time.parse( i['birthdate'] )
+            i['alive'] = Cuboid::Processes::Manager.alive?( pid )
+
+            return i
+        end
+
+        nil
+    end
+
+    # @return   [Array<Hash>]
+    #   Returns info for all instances.
+    def instances
+        @instances.map { |i| instance( i['pid'] ) }.compact
+    end
+
+    # @return   [Array<Hash>]
+    #   Returns info for all running (alive) instances.
+    #
+    # @see #instances
+    def running_instances
+        instances.select { |i| i['alive'] }
+    end
+
+    # @return   [Array<Hash>]
+    #   Returns info for all finished (dead) instances.
+    #
+    # @see #instances
+    def finished_instances
+        instances.reject { |i| i['alive'] }
+    end
+
+    # @return   [Float]
+    #   Workload score for this Dispatcher, calculated using {System#utilization}.
+    #
+    #   * `0.0` => No utilization.
+    #   * `1.0` => Max utilization.
+    #
+    #   Lower is better.
+    def utilization
+        System.utilization
+    end
+
+    # @return   [Hash]
+    #   Returns server stats regarding the instances and pool.
+    def statistics
+        {
+            'utilization'         => utilization,
+            'running_instances'   => running_instances,
+            'finished_instances'  => finished_instances,
+            'consumed_pids'       => @instances.map { |i| i['pid'] }.compact,
+            'snapshots'           => Dir.glob( "#{@options.snapshot.path}*.#{Snapshot::EXTENSION}" ),
+            'node'                => @node.info
+        }
+    end
+
+    # @return   [String]
+    #   Contents of the log file
+    def log
+        IO.read prep_logging
+    end
+
+    # @private
+    def pid
+        Process.pid
+    end
+
+    private
+
+    def trap_interrupts( &block )
+        %w(QUIT INT).each do |signal|
+            trap( signal, &block || Proc.new{ } ) if Signal.list.has_key?( signal )
+        end
+    end
+
+    # Starts the dispatcher's server
+    def run
+        Arachni::Reactor.global.on_error do |_, e|
+            print_error "Reactor: #{e}"
+
+            e.backtrace.each do |l|
+                print_error "Reactor: #{l}"
+            end
+        end
+
+        print_status 'Ready'
+        @server.start
+    rescue => e
+        print_exception e
+
+        $stderr.puts "Could not start server, for details see: #{@logfile}"
+        exit 1
+    end
+
+    def shutdown
+        Thread.new do
+            print_status 'Shutting down...'
+            Arachni::Reactor.global.stop
+        end
+    end
+
+    def spawn_instance( options = {}, &block )
+        Processes::Instances.spawn( options.merge(
+            address:     @server.address,
+            port_range:  Options.dispatcher.instance_port_range,
+            token:       Utilities.generate_token,
+            application: Options.paths.application
+        )) do |client|
+            block.call(
+                'token'       => client.token,
+                'url'         => client.url,
+                'pid'         => client.pid,
+                'birthdate'   => Time.now.to_s,
+                'application' => Options.paths.application
+            )
+        end
+    end
+
+    def prep_logging
+        # reroute all output to a logfile
+        @logfile ||= reroute_to_file( @options.paths.logs +
+            "/Dispatcher - #{Process.pid}-#{@options.rpc.server_port}.log" )
+    end
+
+    def connect_to_peer( url )
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Dispatcher.new( url )
+    end
+
+end
+
+end
+end
+end