cuboid 0.0.0 → 0.0.3

data/lib/cuboid/rpc/server/application_wrapper.rb

@@ -0,0 +1,138 @@
+require 'tempfile'
+
+module Cuboid
+
+lib = Options.paths.lib
+require lib + 'application'
+
+module RPC
+class Server
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class ApplicationWrapper
+    include Utilities
+
+    attr_reader :application
+
+    extend Forwardable
+    def_delegators :@application, :suspended?, :suspend!, :status,
+                   :pause!, :running?, :status_messages, :paused?,
+                   :snapshot_path, :restore!, :resume!, :generate_report
+
+    # {RPC::Server::Application} error namespace.
+    #
+    # All {RPC::Server::Application} errors inherit from and live under it.
+    #
+    # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+    class Error < Cuboid::Application::Error
+    end
+
+    def initialize( application )
+        super()
+
+        @application = application.instance
+        @extended_running = false
+    end
+
+    def valid_options?( options )
+        @application.class.valid_options?( options )
+    end
+
+    # @return   [Bool]
+    #   `true` If the system is scanning, `false` if {#run} hasn't been called
+    #   yet or if the scan has finished.
+    def busy?
+        ![:ready, :done, :suspended].include?( @application.status ) &&
+          !!@extended_running
+    end
+
+    # @return   [Bool]
+    #   `false` if already running, `true` otherwise.
+    def run
+        # Return if we're already running.
+        return false if busy?
+        @extended_running = true
+
+        # Start the scan  -- we can't block the RPC server so we're using a Thread.
+        Thread.new do
+            @application.run
+        end
+
+        true
+    end
+
+    def clean_up
+        return false if @rpc_cleaned_up
+
+        @rpc_cleaned_up   = true
+        @extended_running = false
+
+        @application.clean_up
+    end
+
+    # @param    [Integer]   starting_line
+    #   Sets the starting line for the range of errors to return.
+    #
+    # @return   [Array<String>]
+    def errors( starting_line = 0 )
+        return [] if UI::Output.error_buffer.empty?
+
+        error_strings = UI::Output.error_buffer
+
+        if starting_line != 0
+            error_strings = error_strings[starting_line..-1]
+        end
+
+        error_strings
+    end
+
+    # Provides aggregated progress data.
+    #
+    # @param    [Hash]  opts
+    #   Options about what data to include:
+    # @option opts [Bool] :statistics   (true)
+    #   Master/merged statistics.
+    # @option opts [Bool, Integer] :errors   (false)
+    #   Logged errors. If an integer is provided it will return errors past that
+    #   index.
+    #
+    # @return    [Hash]
+    #   Progress data.
+    def progress( opts = {} )
+        opts = opts.my_symbolize_keys
+
+        include_statistics = opts[:statistics].nil? ? true : opts[:statistics]
+        include_errors     = opts.include?( :errors ) ?
+            (opts[:errors] || 0) : false
+
+        data = {
+            status:         status,
+            busy:           running?,
+            application:    @application.class.to_s,
+            seed:           Utilities.random_seed,
+            dispatcher_url: Cuboid::Options.dispatcher.url,
+            scheduler_url:  Cuboid::Options.scheduler.url
+        }
+
+        if include_statistics
+            data[:statistics] = @application.statistics
+        end
+
+        if include_errors
+            data[:errors] =
+                errors( include_errors.is_a?( Integer ) ? include_errors : 0 )
+        end
+
+        data.merge( messages: status_messages )
+    end
+
+    # @private
+    def error_test( str )
+        @application.print_error str.to_s
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/server/base.rb

@@ -0,0 +1,63 @@
+require 'ostruct'
+require 'arachni/rpc'
+require_relative '../serializer'
+
+module Cuboid
+module RPC
+class Server
+
+# RPC server class
+#
+# @private
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Base < Arachni::RPC::Server
+
+    # @param    [Hash]   options
+    # @option options [Integer]  :host
+    # @option options [Integer]  :port
+    # @option options [Integer]  :socket
+    # @option options [Integer]  :ssl_ca
+    # @option options [Integer]  :ssl_pkey
+    # @option options [Integer]  :ssl_cert
+    # @param    [String]    token
+    #   Optional authentication token.
+    def initialize( options = nil, token = nil )
+
+        # If given nil use the global defaults.
+        options ||= Options.rpc.to_server_options
+        @options = options
+
+        super(options.merge(
+            serializer: Serializer,
+            token:      token
+        ))
+    end
+
+    def address
+        @options[:external_address] || @options[:host]
+    end
+
+    def port
+        @options[:port]
+    end
+
+    def url
+        return @options[:socket] if @options[:socket]
+
+        "#{address}:#{port}"
+    end
+
+    def start
+        super
+        @ready = true
+    end
+
+    def ready?
+        @ready ||= false
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/server/dispatcher/node.rb

@@ -0,0 +1,247 @@
+module Cuboid
+
+require Options.paths.lib + 'rpc/server/output'
+
+module RPC
+
+# Dispatcher node class, helps maintain a list of all available Dispatchers in
+# the grid and announce itself to neighbouring Dispatchers.
+#
+# As soon as a new Node is fired up it checks-in with its neighbour and grabs
+# a list of all available peers.
+#
+# As soon as it receives the peer list it then announces itself to them.
+#
+# Upon convergence there will be a grid of Dispatchers each one with its own
+# copy of all available Dispatcher URLs.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Server::Dispatcher::Node
+    include UI::Output
+
+    # Initializes the node by:
+    #
+    #   * Adding the neighbour (if the user has supplied one) to the peer list.
+    #   * Getting the neighbour's peer list and appending them to its own.
+    #   * Announces itself to the neighbour and instructs it to propagate our URL
+    #     to the others.
+    #
+    # @param    [Cuboid::Options]    options
+    # @param    [Server::Base]    server
+    #   Dispatcher's RPC server.
+    # @param    [String]              logfile
+    #   Where to send the output.
+    def initialize( options, server, logfile = nil )
+        @options = options
+        @server  = server
+        @url     = @server.url
+
+        reroute_to_file( logfile ) if logfile
+
+        print_status 'Initializing grid node...'
+
+        @dead_nodes = Set.new
+        @neighbours = Set.new
+        @nodes_info_cache = []
+
+        if (neighbour = @options.dispatcher.neighbour)
+            # Grab the neighbour's neighbours.
+            connect_to_peer( neighbour ).neighbours do |urls|
+                if urls.rpc_exception?
+                    add_dead_neighbour( neighbour )
+                    print_info "Neighbour seems dead: #{neighbour}"
+                    add_dead_neighbour( neighbour )
+                    next
+                end
+
+                # Add neighbour and announce it to everyone.
+                add_neighbour( neighbour, true )
+
+                urls.each { |url| @neighbours << url if url != @url }
+            end
+        end
+
+        print_status 'Node ready.'
+
+        log_updated_neighbours
+
+        Arachni::Reactor.global.at_interval( @options.dispatcher.ping_interval ) do
+            ping
+            check_for_comebacks
+        end
+    end
+
+    # @return   [Boolean]
+    #   `true` if grid member, `false` otherwise.
+    def grid_member?
+        @neighbours.any?
+    end
+
+    def unplug
+        @neighbours.each do |peer|
+            connect_to_peer( peer ).remove_neighbour( @url ) {}
+        end
+
+        @neighbours.clear
+        @dead_nodes.clear
+
+        nil
+    end
+
+    # Adds a neighbour to the peer list.
+    #
+    # @param    [String]    node_url
+    #   URL of a neighbouring node.
+    # @param    [Boolean]   propagate
+    #   Whether or not to announce the new node to the peers.
+    def add_neighbour( node_url, propagate = false )
+        # we don't want ourselves in the Set
+        return false if node_url == @url
+        return false if @neighbours.include?( node_url )
+
+        print_status "Adding neighbour: #{node_url}"
+
+        @neighbours << node_url
+        log_updated_neighbours
+        announce( node_url ) if propagate
+
+        connect_to_peer( node_url ).add_neighbour( @url, propagate ) do |res|
+            next if !res.rpc_exception?
+            add_dead_neighbour( node_url )
+            print_status "Neighbour seems dead: #{node_url}"
+        end
+        true
+    end
+
+    def remove_neighbour( url )
+        @neighbours.delete url
+        @dead_nodes.delete url
+        nil
+    end
+
+    # @return   [Array]
+    #   Neighbour/node/peer URLs.
+    def neighbours
+        @neighbours.to_a
+    end
+
+    def neighbours_with_info( &block )
+        fail 'This method requires a block!' if !block_given?
+
+        @neighbours_cmp = ''
+
+        if @nodes_info_cache.empty? || @neighbours_cmp != neighbours.to_s
+            @neighbours_cmp = neighbours.to_s
+
+            each = proc do |neighbour, iter|
+                connect_to_peer( neighbour ).info do |info|
+                    if info.rpc_exception?
+                        print_info "Neighbour seems dead: #{neighbour}"
+                        add_dead_neighbour( neighbour )
+                        log_updated_neighbours
+
+                        iter.return( nil )
+                    else
+                        iter.return( info )
+                    end
+                end
+            end
+
+            after = proc do |nodes|
+                @nodes_info_cache = nodes.compact
+                block.call( @nodes_info_cache )
+            end
+
+            Arachni::Reactor.global.create_iterator( neighbours ).map( each, after )
+        else
+            block.call( @nodes_info_cache )
+        end
+    end
+
+    # @return    [Hash]
+    #
+    #   * `url` -- This node's URL.
+    #   * `name` -- Nickname
+    #   * `neighbours` -- Array of neighbours.
+    def info
+        {
+            'url'                    => @url,
+            'name'                   => @options.dispatcher.name,
+            'neighbours'             => @neighbours.to_a,
+            'unreachable_neighbours' => @dead_nodes.to_a
+        }
+    end
+
+    def alive?
+        true
+    end
+
+    private
+
+    def add_dead_neighbour( url )
+        remove_neighbour( url )
+        @dead_nodes << url
+    end
+
+    def log_updated_neighbours
+        print_info 'Updated neighbours:'
+
+        if !neighbours.empty?
+            neighbours.each { |node| print_info( '---- ' + node ) }
+        else
+            print_info '<empty>'
+        end
+    end
+
+    def ping
+        neighbours.each do |neighbour|
+            connect_to_peer( neighbour ).alive? do |res|
+                next if !res.rpc_exception?
+                add_dead_neighbour( neighbour )
+                print_status "Found dead neighbour: #{neighbour} "
+            end
+        end
+    end
+
+    def check_for_comebacks
+        @dead_nodes.dup.each do |url|
+            neighbour = connect_to_peer( url )
+            neighbour.alive? do |res|
+                next if res.rpc_exception?
+
+                print_status "Dispatcher came back to life: #{url}"
+                ([@url] | neighbours).each do |node|
+                    neighbour.add_neighbour( node ){}
+                end
+
+                add_neighbour( url )
+                @dead_nodes.delete url
+            end
+        end
+    end
+
+    # Announces the node to the ones in the peer list
+    #
+    # @param    [String]    node
+    #   URL
+    def announce( node )
+        print_status "Advertising: #{node}"
+
+        neighbours.each do |peer|
+            next if peer == node
+
+            print_info '---- to: ' + peer
+            connect_to_peer( peer ).add_neighbour( node ) do |res|
+                add_dead_neighbour( peer ) if res.rpc_exception?
+            end
+        end
+    end
+
+    def connect_to_peer( url )
+        @rpc_clients ||= {}
+        @rpc_clients[url] ||= Client::Dispatcher.new( url ).node
+    end
+
+end
+end
+end

data/lib/cuboid/rpc/server/dispatcher/service.rb

@@ -0,0 +1,145 @@
+module Cuboid
+module RPC
+
+# Base class and namespace for all Dispatcher services.
+#
+# # RPC accessibility
+#
+# Only PUBLIC methods YOU have defined will be accessible over RPC.
+#
+# # Blocking operations
+#
+# Please try to avoid blocking operations as they will block the main Reactor loop.
+#
+# However, if you really need to perform such operations, you can update the
+# relevant methods to expect a block and then pass the desired return value to
+# that block instead of returning it the usual way.
+#
+# This will result in the method's payload to be deferred into a Thread of its own.
+#
+# In addition, you can use the {#defer} and {#run_asap} methods is you need more
+# control over what gets deferred and general scheduling.
+#
+# # Asynchronous operations
+#
+# Methods which perform async operations should expect a block and pass their
+# results to that block instead of returning a value.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Server::Dispatcher::Service
+
+    attr_reader :options
+    attr_reader :dispatcher
+
+    def initialize( options, dispatcher )
+        @options    = options
+        @dispatcher = dispatcher
+    end
+
+    # @return   [Server::Dispatcher::Node]
+    #   Local node.
+    def node
+        dispatcher.instance_eval { @node }
+    end
+
+    # Performs an asynchronous map operation over all running instances.
+    #
+    # @param [Proc]  each
+    #   Block to be passed {Client::Instance} and `Arachni::Reactor::Iterator`.
+    # @param [Proc]  after
+    #   Block to be passed the Array of results.
+    def map_instances( each, after )
+        wrap_each = proc do |instance, iterator|
+            each.call( connect_to_instance( instance ), iterator )
+        end
+        iterator_for( instances ).map( wrap_each, after )
+    end
+
+    # Performs an asynchronous iteration over all running instances.
+    #
+    # @param [Proc]  block
+    #   Block to be passed {Client::Instance} and `Arachni::Reactor::Iterator`.
+    def each_instance( &block )
+        wrap = proc do |instance, iterator|
+            block.call( connect_to_instance( instance ), iterator )
+        end
+        iterator_for( instances ).each( &wrap )
+    end
+
+    # Defers a blocking operation in order to avoid blocking the main Reactor loop.
+    #
+    # The operation will be run in its own Thread - DO NOT block forever.
+    #
+    # Accepts either 2 parameters (an `operation` and a `callback` or an operation
+    # as a block.
+    #
+    # @param    [Proc]  operation
+    #   Operation to defer.
+    # @param    [Proc]  callback
+    #   Block to call with the results of the operation.
+    #
+    # @param    [Block]  block
+    #   Operation to defer.
+    def defer( operation = nil, callback = nil, &block )
+        Thread.new( *[operation, callback].compact, &block )
+    end
+
+    # Runs a block as soon as possible in the Reactor loop.
+    #
+    # @param    [Block] block
+    def run_asap( &block )
+        Arachni::Reactor.global.next_tick( &block )
+    end
+
+    # @param    [Array]    list
+    #
+    # @return   [Arachni::Reactor::Iterator]
+    #   Iterator for the provided array.
+    def iterator_for( list, max_concurrency = 10 )
+        Arachni::Reactor.global.create_iterator( list, max_concurrency )
+    end
+
+    # @return   [Array<Hash>]
+    #   Alive instances.
+    def instances
+        dispatcher.running_instances
+    end
+
+    # Connects to a Dispatcher by `url`
+    #
+    # @param    [String]    url
+    #
+    # @return   [Client::Dispatcher]
+    def connect_to_dispatcher( url )
+        @dispatcher_connections ||= {}
+        @dispatcher_connections[url] ||= Client::Dispatcher.new( url )
+    end
+
+    # Connects to an Instance by `url`.
+    #
+    # @example
+    #   connect_to_instance( url, token )
+    #   connect_to_instance( url: url, token: token )
+    #   connect_to_instance( 'url' => url, 'token' => token )
+    #
+    # @param    [Vararg]    args
+    #
+    # @return   [Client::Instance]
+    def connect_to_instance( *args )
+        url = token = nil
+
+        if args.size == 2
+            url, token = *args
+        elsif args.first.is_a? Hash
+            connection_options = args.first
+            url     = connection_options['url']   || connection_options[:url]
+            token   = connection_options['token'] || connection_options[:token]
+        end
+
+        @instance_connections ||= {}
+        @instance_connections[url] ||= Client::Instance.new( url, token )
+    end
+
+end
+end
+end