cuboid 0.0.0 → 0.0.1alpha

data/lib/cuboid/rpc/server/dispatcher/node.rb

@@ -0,0 +1,247 @@
+module Cuboid
+
+require Options.paths.lib + 'rpc/server/output'
+
+module RPC
+
+# Dispatcher node class, helps maintain a list of all available Dispatchers in
+# the grid and announce itself to neighbouring Dispatchers.
+#
+# As soon as a new Node is fired up it checks-in with its neighbour and grabs
+# a list of all available peers.
+#
+# As soon as it receives the peer list it then announces itself to them.
+#
+# Upon convergence there will be a grid of Dispatchers each one with its own
+# copy of all available Dispatcher URLs.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Server::Dispatcher::Node
+    include UI::Output
+
+    # Initializes the node by:
+    #
+    #   * Adding the neighbour (if the user has supplied one) to the peer list.
+    #   * Getting the neighbour's peer list and appending them to its own.
+    #   * Announces itself to the neighbour and instructs it to propagate our URL
+    #     to the others.
+    #
+    # @param    [Cuboid::Options]    options
+    # @param    [Server::Base]    server
+    #   Dispatcher's RPC server.
+    # @param    [String]              logfile
+    #   Where to send the output.
+    def initialize( options, server, logfile = nil )
+        @options = options
+        @server  = server
+        @url     = @server.url
+
+        reroute_to_file( logfile ) if logfile
+
+        print_status 'Initializing grid node...'
+
+        @dead_nodes = Set.new
+        @neighbours = Set.new
+        @nodes_info_cache = []
+
+        if (neighbour = @options.dispatcher.neighbour)
+            # Grab the neighbour's neighbours.
+            connect_to_peer( neighbour ).neighbours do |urls|
+                if urls.rpc_exception?
+                    add_dead_neighbour( neighbour )
+                    print_info "Neighbour seems dead: #{neighbour}"
+                    add_dead_neighbour( neighbour )
+                    next
+                end
+
+                # Add neighbour and announce it to everyone.
+                add_neighbour( neighbour, true )
+
+                urls.each { |url| @neighbours << url if url != @url }
+            end
+        end
+
+        print_status 'Node ready.'
+
+        log_updated_neighbours
+
+        Arachni::Reactor.global.at_interval( @options.dispatcher.ping_interval ) do
+            ping
+            check_for_comebacks
+        end
+    end
+
+    # @return   [Boolean]
+    #   `true` if grid member, `false` otherwise.
+    def grid_member?
+        @neighbours.any?
+    end
+
+    def unplug
+        @neighbours.each do |peer|
+            connect_to_peer( peer ).remove_neighbour( @url ) {}
+        end
+
+        @neighbours.clear
+        @dead_nodes.clear
+
+        nil
+    end
+
+    # Adds a neighbour to the peer list.
+    #
+    # @param    [String]    node_url
+    #   URL of a neighbouring node.
+    # @param    [Boolean]   propagate
+    #   Whether or not to announce the new node to the peers.
+    def add_neighbour( node_url, propagate = false )
+        # we don't want ourselves in the Set
+        return false if node_url == @url
+        return false if @neighbours.include?( node_url )
+
+        print_status "Adding neighbour: #{node_url}"
+
+        @neighbours << node_url
+        log_updated_neighbours
+        announce( node_url ) if propagate
+
+        connect_to_peer( node_url ).add_neighbour( @url, propagate ) do |res|
+            next if !res.rpc_exception?
+            add_dead_neighbour( node_url )
+            print_status "Neighbour seems dead: #{node_url}"
+        end
+        true
+    end
+
+    def remove_neighbour( url )
+        @neighbours.delete url
+        @dead_nodes.delete url
+        nil
+    end
+
+    # @return   [Array]
+    #   Neighbour/node/peer URLs.
+    def neighbours
+        @neighbours.to_a
+    end
+
+    def neighbours_with_info( &block )
+        fail 'This method requires a block!' if !block_given?
+
+        @neighbours_cmp = ''
+
+        if @nodes_info_cache.empty? || @neighbours_cmp != neighbours.to_s
+            @neighbours_cmp = neighbours.to_s
+
+            each = proc do |neighbour, iter|
+                connect_to_peer( neighbour ).info do |info|
+                    if info.rpc_exception?
+                        print_info "Neighbour seems dead: #{neighbour}"
+                        add_dead_neighbour( neighbour )
+                        log_updated_neighbours
+
+                        iter.return( nil )
+                    else
+                        iter.return( info )
+                    end
+                end
+            end
+
+            after = proc do |nodes|
+                @nodes_info_cache = nodes.compact
+                block.call( @nodes_info_cache )
+            end
+
+            Arachni::Reactor.global.create_iterator( neighbours ).map( each, after )
+        else
+            block.call( @nodes_info_cache )
+        end
+    end
+
+    # @return    [Hash]
+    #
+    #   * `url` -- This node's URL.
+    #   * `name` -- Nickname
+    #   * `neighbours` -- Array of neighbours.
+    def info
+        {
+            'url'                    => @url,
+            'name'                   => @options.dispatcher.name,
+            'neighbours'             => @neighbours.to_a,
+            'unreachable_neighbours' => @dead_nodes.to_a
+        }
+    end
+
+    def alive?
+        true
+    end
+
+    private
+
+    def add_dead_neighbour( url )
+        remove_neighbour( url )
+        @dead_nodes << url
+    end
+
+    def log_updated_neighbours
+        print_info 'Updated neighbours:'
+
+        if !neighbours.empty?
+            neighbours.each { |node| print_info( '---- ' + node ) }
+        else
+            print_info '<empty>'
+        end
+    end
+
+    def ping
+        neighbours.each do |neighbour|
+            connect_to_peer( neighbour ).alive? do |res|
+                next if !res.rpc_exception?
+                add_dead_neighbour( neighbour )
+                print_status "Found dead neighbour: #{neighbour} "
+            end
+        end
+    end
+
+    def check_for_comebacks
+        @dead_nodes.dup.each do |url|
+            neighbour = connect_to_peer( url )
+            neighbour.alive? do |res|
+                next if res.rpc_exception?
+
+                print_status "Dispatcher came back to life: #{url}"
+                ([@url] | neighbours).each do |node|
+                    neighbour.add_neighbour( node ){}
+                end
+
+                add_neighbour( url )
+                @dead_nodes.delete url
+            end
+        end
+    end
+
+    # Announces the node to the ones in the peer list
+    #
+    # @param    [String]    node
+    #   URL
+    def announce( node )
+        print_status "Advertising: #{node}"
+
+        neighbours.each do |peer|
+            next if peer == node
+
+            print_info '---- to: ' + peer
+            connect_to_peer( peer ).add_neighbour( node ) do |res|
+                add_dead_neighbour( peer ) if res.rpc_exception?
+            end
+        end
+    end
+
+    def connect_to_peer( url )
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Dispatcher.new( url ).node
+    end
+
+end
+end
+end

data/lib/cuboid/rpc/server/dispatcher/service.rb

@@ -0,0 +1,145 @@
+module Cuboid
+module RPC
+
+# Base class and namespace for all Dispatcher services.
+#
+# # RPC accessibility
+#
+# Only PUBLIC methods YOU have defined will be accessible over RPC.
+#
+# # Blocking operations
+#
+# Please try to avoid blocking operations as they will block the main Reactor loop.
+#
+# However, if you really need to perform such operations, you can update the
+# relevant methods to expect a block and then pass the desired return value to
+# that block instead of returning it the usual way.
+#
+# This will result in the method's payload to be deferred into a Thread of its own.
+#
+# In addition, you can use the {#defer} and {#run_asap} methods is you need more
+# control over what gets deferred and general scheduling.
+#
+# # Asynchronous operations
+#
+# Methods which perform async operations should expect a block and pass their
+# results to that block instead of returning a value.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Server::Dispatcher::Service
+
+    attr_reader :options
+    attr_reader :dispatcher
+
+    def initialize( options, dispatcher )
+        @options    = options
+        @dispatcher = dispatcher
+    end
+
+    # @return   [Server::Dispatcher::Node]
+    #   Local node.
+    def node
+        dispatcher.instance_eval { @node }
+    end
+
+    # Performs an asynchronous map operation over all running instances.
+    #
+    # @param [Proc]  each
+    #   Block to be passed {Client::Instance} and `Arachni::Reactor::Iterator`.
+    # @param [Proc]  after
+    #   Block to be passed the Array of results.
+    def map_instances( each, after )
+        wrap_each = proc do |instance, iterator|
+            each.call( connect_to_instance( instance ), iterator )
+        end
+        iterator_for( instances ).map( wrap_each, after )
+    end
+
+    # Performs an asynchronous iteration over all running instances.
+    #
+    # @param [Proc]  block
+    #   Block to be passed {Client::Instance} and `Arachni::Reactor::Iterator`.
+    def each_instance( &block )
+        wrap = proc do |instance, iterator|
+            block.call( connect_to_instance( instance ), iterator )
+        end
+        iterator_for( instances ).each( &wrap )
+    end
+
+    # Defers a blocking operation in order to avoid blocking the main Reactor loop.
+    #
+    # The operation will be run in its own Thread - DO NOT block forever.
+    #
+    # Accepts either 2 parameters (an `operation` and a `callback` or an operation
+    # as a block.
+    #
+    # @param    [Proc]  operation
+    #   Operation to defer.
+    # @param    [Proc]  callback
+    #   Block to call with the results of the operation.
+    #
+    # @param    [Block]  block
+    #   Operation to defer.
+    def defer( operation = nil, callback = nil, &block )
+        Thread.new( *[operation, callback].compact, &block )
+    end
+
+    # Runs a block as soon as possible in the Reactor loop.
+    #
+    # @param    [Block] block
+    def run_asap( &block )
+        Arachni::Reactor.global.next_tick( &block )
+    end
+
+    # @param    [Array]    list
+    #
+    # @return   [Arachni::Reactor::Iterator]
+    #   Iterator for the provided array.
+    def iterator_for( list, max_concurrency = 10 )
+        Arachni::Reactor.global.create_iterator( list, max_concurrency )
+    end
+
+    # @return   [Array<Hash>]
+    #   Alive instances.
+    def instances
+        dispatcher.running_instances
+    end
+
+    # Connects to a Dispatcher by `url`
+    #
+    # @param    [String]    url
+    #
+    # @return   [Client::Dispatcher]
+    def connect_to_dispatcher( url )
+        @dispatcher_connections ||= {}
+        @dispatcher_connections[url] ||= Client::Dispatcher.new( url )
+    end
+
+    # Connects to an Instance by `url`.
+    #
+    # @example
+    #   connect_to_instance( url, token )
+    #   connect_to_instance( url: url, token: token )
+    #   connect_to_instance( 'url' => url, 'token' => token )
+    #
+    # @param    [Vararg]    args
+    #
+    # @return   [Client::Instance]
+    def connect_to_instance( *args )
+        url = token = nil
+
+        if args.size == 2
+            url, token = *args
+        elsif args.first.is_a? Hash
+            connection_options = args.first
+            url     = connection_options['url']   || connection_options[:url]
+            token   = connection_options['token'] || connection_options[:token]
+        end
+
+        @instance_connections ||= {}
+        @instance_connections[url] ||= Client::Instance.new( url, token )
+    end
+
+end
+end
+end

data/lib/cuboid/rpc/server/instance.rb

@@ -0,0 +1,338 @@
+require 'ostruct'
+
+module Cuboid
+lib = Options.paths.lib
+
+require lib + 'processes/manager'
+
+require lib + 'rpc/client/instance'
+
+require lib + 'rpc/server/base'
+require lib + 'rpc/server/active_options'
+require lib + 'rpc/server/output'
+require lib + 'rpc/server/application_wrapper'
+
+module RPC
+class Server
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Instance
+    include UI::Output
+    include Utilities
+
+    [
+        :suspend!, :suspended?, :snapshot_path,
+        :pause!,   :paused?,
+        :abort!,   :resume!,
+        :running?, :status
+    ].each do |m|
+        define_method m do
+            @application.send m
+        end
+    end
+
+    private :error_logfile
+    public  :error_logfile
+
+    # Initializes the RPC interface and the framework.
+    #
+    # @param    [Options]    options
+    # @param    [String]    token
+    #   Authentication token.
+    def initialize( options, token )
+        @options = options
+        @token   = token
+
+        @application    = Server::ApplicationWrapper.new(
+          Cuboid::Application.application
+        )
+        @active_options = Server::ActiveOptions.new
+
+        @server = Base.new( @options.rpc.to_server_options, token )
+
+        if @options.datastore.log_level
+            @server.logger.level = @options.datastore.log_level
+        end
+
+        @options.datastore.token = token
+
+        if @options.output.reroute_to_logfile
+            reroute_to_file "#{@options.paths.logs}/Instance - #{Process.pid}" <<
+                                "-#{@options.rpc.server_port}.log"
+        else
+            reroute_to_file false
+        end
+
+        set_error_logfile "#{@options.paths.logs}/Instance - #{Process.pid}" <<
+                              "-#{@options.rpc.server_port}.error.log"
+
+        set_handlers( @server )
+
+        # trap interrupts and exit cleanly when required
+        %w(QUIT INT).each do |signal|
+            next if !Signal.list.has_key?( signal )
+            trap( signal ){ shutdown if !@options.datastore.do_not_trap }
+        end
+
+        Arachni::Reactor.global.run do
+            _run
+        end
+    end
+
+    def application
+        Application.application.to_s
+    end
+
+    # @return   [String, nil]
+    #   Scheduler URL to which this Instance is attached, `nil` if not attached.
+    def scheduler_url
+        @options.scheduler.url
+    end
+
+    # @return   [String, nil]
+    #   Dispatcher URL that provided this Instance, `nil` if not provided by a
+    #   Dispatcher.
+    def dispatcher_url
+        @options.dispatcher.url
+    end
+
+    # @param (see Cuboid::Application#restore)
+    # @return (see Cuboid::Application#restore)
+    #
+    # @see #suspend
+    # @see #snapshot_path
+    def restore!( snapshot )
+        @application.restore!( snapshot ).run
+        true
+    end
+
+    # @return   [true]
+    def alive?
+        @server.alive?
+    end
+
+    # @return   [Bool]
+    #   `true` if the scan is initializing or running, `false` otherwise.
+    def busy?
+        @run_initializing || @application.busy?
+    end
+
+    # Cleans up and returns the report.
+    #
+    # @return  [Hash]
+    #
+    # @see #report
+    def abort_and_generate_report
+        @application.clean_up
+        generate_report
+    end
+
+    # @return [Hash]
+    #   {Report#to_rpc_data}
+    def generate_report
+        @application.generate_report.to_rpc_data
+    end
+
+    # # Recommended usage
+    #
+    #   Please request from the method only the things you are going to actually
+    #   use, otherwise you'll just be wasting bandwidth.
+    #   In addition, ask to **not** be served data you already have, like
+    #   error messages.
+    #
+    #   To be kept completely up to date on the progress of a scan (i.e. receive
+    #   new issues and error messages asap) in an efficient manner, you will need
+    #   to keep track of the error messages you already have and explicitly tell
+    #   the method to not send the same data back to you on subsequent calls.
+    #
+    # ## Retrieving errors (`:errors` option) without duplicate data
+    #
+    #   This is done by telling the method how many error messages you already
+    #   have and you will be served the errors from the error-log that are past
+    #   that line.
+    #   So, if you were to use a loop to get fresh progress data it would look
+    #   like so:
+    #
+    #     error_cnt = 0
+    #     i = 0
+    #     while sleep 1
+    #         # Test method, triggers an error log...
+    #         instance.error_test "BOOM! #{i+=1}"
+    #
+    #         # Only request errors we don't already have
+    #         errors = instance.progress( with: { errors: error_cnt } )[:errors]
+    #         error_cnt += errors.size
+    #
+    #         # You will only see new errors
+    #         puts errors.join("\n")
+    #     end
+    #
+    # @param  [Hash]  options
+    #   Options about what progress data to retrieve and return.
+    # @option options [Array<Symbol, Hash>]  :with
+    #   Specify data to include:
+    #
+    #   * :errors -- Errors and the line offset to use for {#errors}.
+    #     Pass as a hash, like: `{ errors: 10 }`
+    # @option options [Array<Symbol, Hash>]  :without
+    #   Specify data to exclude:
+    #
+    #   * :statistics -- Don't include runtime statistics.
+    #
+    # @return [Hash]
+    #   * `statistics` -- General runtime statistics (merged when part of Grid)
+    #       (enabled by default)
+    #   * `status` -- {#status}
+    #   * `busy` -- {#busy?}
+    #   * `errors` -- {#errors} (disabled by default)
+    def progress( options = {} )
+        progress_handler( options.merge( as_hash: true ) )
+    end
+
+    # Configures and runs a job.
+    def run( options = nil )
+        # If the instance isn't clean bail out now.
+        return false if busy? || @called
+
+        if !@application.valid_options?( options )
+            fail ArgumentError, 'Invalid options!'
+        end
+
+        # There may be follow-up/retry calls by the client in cases of network
+        # errors (after the request has reached us) so we need to keep minimal
+        # track of state in order to bail out on subsequent calls.
+        @called = @run_initializing = true
+
+        @active_options.set( application: options )
+
+        @application.run
+        @run_initializing = false
+
+        true
+    end
+
+    # Makes the server go bye-bye...Lights out!
+    def shutdown( &block )
+        if @shutdown
+            block.call if block_given?
+            return
+        end
+        @shutdown = true
+
+        print_status 'Shutting down...'
+
+        # We're shutting down services so we need to use a concurrent way but
+        # without going through the Reactor.
+        Thread.new do
+            @server.shutdown
+            block.call true if block_given?
+        end
+
+        true
+    end
+
+    def errors( starting_line = 0 )
+        @application.errors( starting_line )
+    end
+
+    # @private
+    def error_test( str )
+        @application.error_test( str )
+    end
+
+    # @private
+    def consumed_pids
+        [Process.pid]
+    end
+
+    def self.parse_progress_opts( options, key )
+        parsed = {}
+        [options.delete( key ) || options.delete( key.to_s )].compact.each do |w|
+            case w
+                when Array
+                    w.compact.flatten.each do |q|
+                        case q
+                            when String, Symbol
+                                parsed[q.to_sym] = nil
+
+                            when Hash
+                                parsed.merge!( q.my_symbolize_keys )
+                        end
+                    end
+
+                when String, Symbol
+                    parsed[w.to_sym] = nil
+
+                when Hash
+                    parsed.merge!( w.my_symbolize_keys )
+            end
+        end
+
+        parsed
+    end
+
+    private
+
+    def progress_handler( options = {}, &block )
+        with    = self.class.parse_progress_opts( options, :with )
+        without = self.class.parse_progress_opts( options, :without )
+
+        options = {
+            as_hash:    options[:as_hash],
+            statistics: !without.include?( :statistics )
+        }
+
+        if with[:errors]
+            options[:errors] = with[:errors]
+        end
+
+        @application.progress( options ) do |data|
+            data[:busy] = busy?
+            block.call( data )
+        end
+    end
+
+    # Starts  RPC service.
+    def _run
+        Arachni::Reactor.global.on_error do |_, e|
+            print_error "Reactor: #{e}"
+
+            e.backtrace.each do |l|
+                print_error "Reactor: #{l}"
+            end
+        end
+
+        print_status 'Starting the server...'
+        @server.start
+    end
+
+    # Outputs the Engine banner.
+    #
+    # Displays version number, author details etc.
+    def banner
+        puts BANNER
+        puts
+        puts
+    end
+
+    # @param    [Base]  server
+    #   Prepares all the RPC handlers for the given `server`.
+    def set_handlers( server )
+        server.add_async_check do |method|
+            # methods that expect a block are async
+            method.parameters.flatten.include? :block
+        end
+
+        server.add_handler( 'instance', self )
+        server.add_handler( 'options',  @active_options )
+
+        Cuboid::Application.application.instance_services.each do |name, service|
+            server.add_handler( name.to_s, service.new )
+        end
+    end
+
+end
+
+end
+end
+end