cuboid 0.0.0 → 0.0.3

data/lib/cuboid/rest/server.rb

@@ -0,0 +1,165 @@
+require 'puma'
+require 'puma/minissl'
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Cuboid
+module Rest
+
+class Server < Sinatra::Base
+    lib = Options.paths.lib
+    require lib + 'processes'
+    require lib + 'rest/server/instance_helpers'
+
+    Dir.glob( "#{File.dirname( __FILE__ )}/server/routes/*.rb" ).each { |f| require f }
+
+    helpers InstanceHelpers
+
+    register Sinatra::Namespace
+    Cuboid::Application.application.rest_services.each do |name, service|
+        namespace "/instances/:instance/#{name}" do
+            register service
+        end
+    end
+
+    register Routes::Instances
+    register Routes::Dispatcher
+    register Routes::Grid
+    register Routes::Scheduler
+
+    use Rack::Deflater
+    use Rack::Session::Pool
+
+    set :environment, :production
+
+    enable :logging
+
+    VALID_REPORT_FORMATS = %w(json xml yaml html.zip)
+
+    before do
+        protected!
+        content_type :json
+    end
+
+    helpers do
+        def max_utilization!
+            halt 503,
+                 json( error: 'Service unavailable: System is at maximum ' +
+                                  'utilization, slot limit reached.' )
+        end
+
+        def protected!
+            if !settings.respond_to?( :username )
+                settings.set :username, nil
+            end
+
+            if !settings.respond_to?( :password )
+                settings.set :password, nil
+            end
+
+            return if !settings.username && !settings.password
+            return if authorized?
+
+            headers['WWW-Authenticate'] = 'Basic realm="Restricted Area"'
+            halt 401, "Not authorized\n"
+        end
+
+        def authorized?
+            @auth ||= Rack::Auth::Basic::Request.new( request.env )
+            @auth.provided? && @auth.basic? && @auth.credentials == [
+                settings.username.to_s, settings.password.to_s
+            ]
+        end
+
+        def ensure_instance!
+            update_from_scheduler
+
+            id = params[:instance]
+
+            return if exists? id
+
+            halt 404, json( "Scan not found for id: #{h id}." )
+        end
+
+        def ensure_dispatcher!
+            return if dispatcher
+            halt 501, json( 'No Dispatcher has been set.' )
+        end
+
+        def ensure_scheduler!
+            return if scheduler
+            halt 501, json( 'No Scheduler has been set.' )
+        end
+
+        def handle_error( cleanup = nil, &block )
+            block.call
+        rescue => e
+            cleanup.call if cleanup
+
+            halt 500,
+                 json(
+                     error:       e.class,
+                     description: e.to_s,
+                     backtrace:   e.backtrace
+                 )
+        end
+
+        def h( text )
+            Rack::Utils.escape_html( text )
+        end
+    end
+
+    class <<self
+        include Cuboid::UI::Output
+
+        def reset
+            @@instances.clear
+            @@dispatchers.clear
+        end
+
+        def run!( options )
+            set :username, options[:username]
+            set :password, options[:password]
+
+            server = Puma::Server.new( self )
+            server.min_threads = 0
+            server.max_threads = 16
+
+            ssl = false
+            if options[:ssl_key] && options[:ssl_certificate]
+                ctx = Puma::MiniSSL::Context.new
+
+                ctx.key  = options[:ssl_key]
+                ctx.cert = options[:ssl_certificate]
+
+                if options[:ssl_ca]
+                    print_info 'CA provided, peer verification has been enabled.'
+
+                    ctx.ca          = options[:ssl_ca]
+                    ctx.verify_mode = Puma::MiniSSL::VERIFY_PEER |
+                        Puma::MiniSSL::VERIFY_FAIL_IF_NO_PEER_CERT
+                else
+                    print_info 'CA missing, peer verification has been disabled.'
+                end
+
+                ssl = true
+                server.binder.add_ssl_listener( options[:bind], options[:port], ctx )
+            else
+                ssl = false
+                server.add_tcp_listener( options[:bind], options[:port] )
+            end
+
+            print_status "Listening on http#{'s' if ssl}://#{options[:bind]}:#{options[:port]}"
+
+            begin
+                server.run.join
+            rescue Interrupt
+                server.stop( true )
+            end
+        end
+    end
+
+end
+
+end
+end

data/lib/cuboid/rpc/client/base.rb

@@ -0,0 +1,58 @@
+require 'arachni/rpc'
+require_relative '../serializer'
+
+module Cuboid
+module RPC
+class Client
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Base < Arachni::RPC::Client
+    attr_reader :url
+
+    # @param    [String]    url
+    #   Server URL in `address:port` format.
+    # @param    [String]    token
+    #   Optional authentication token.
+    # @param    [Hash]   options
+    # @option options [Integer]  :connection_pool_size
+    # @option options [Integer]  :max_retries
+    # @option options [Integer]  :ssl_ca
+    # @option options [Integer]  :ssl_pkey
+    # @option options [Integer]  :ssl_cert
+    def initialize( url, token = nil, options = nil )
+        @url = url
+
+        socket, host, port = nil
+        if url.include? ':'
+            host, port = url.split( ':' )
+        else
+            socket = url
+        end
+
+        @address = host
+        @port    = port
+
+        # If given nil use the global defaults.
+        options ||= Options.rpc.to_client_options
+
+        super( options.merge(
+            serializer: Serializer,
+            host:       host,
+            port:       port.to_i,
+            socket:     socket,
+            token:      token
+        ))
+    end
+
+    def address
+        @address
+    end
+
+    def port
+        @port
+    end
+
+end
+end
+end
+end

data/lib/cuboid/rpc/client/dispatcher.rb

@@ -0,0 +1,58 @@
+module Cuboid
+
+require Options.paths.lib + 'rpc/client/base'
+
+module RPC
+class Client
+
+# RPC Dispatcher client
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Dispatcher
+    # Not always available, set by the parent.
+    attr_accessor :pid
+
+    attr_reader :node
+
+    def initialize( url, options = nil )
+        @client = Base.new( url, nil, options )
+        @node   = Arachni::RPC::Proxy.new( @client, 'node' )
+
+        Cuboid::Application.application.dispatcher_services.keys.each do |name|
+            self.class.send( :attr_reader, name.to_sym )
+
+            instance_variable_set(
+              "@#{name}".to_sym,
+              Arachni::RPC::Proxy.new( @client, name )
+            )
+        end
+    end
+
+    def url
+        @client.url
+    end
+
+    def address
+        @client.address
+    end
+
+    def port
+        @client.port
+    end
+
+    def close
+        @client.close
+    end
+
+    private
+
+    # Used to provide the illusion of locality for remote methods
+    def method_missing( sym, *args, &block )
+        @client.call( "dispatcher.#{sym.to_s}", *args, &block )
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/client/instance/service.rb

@@ -0,0 +1,37 @@
+module Cuboid
+
+module RPC
+class Client
+class Instance
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Proxy < Arachni::RPC::Proxy
+
+    def initialize( client )
+        super client, 'instance'
+    end
+
+    translate :status do |status|
+        status.to_sym if status
+    end
+
+    translate :progress do |data|
+        data = data.my_symbolize_keys
+        data[:status] = data[:status].to_sym
+        data
+    end
+
+    translate :abort_and_generate_report do |data|
+        Report.from_rpc_data data
+    end
+
+    translate :generate_report do |data|
+        Report.from_rpc_data data
+    end
+
+end
+
+end
+end
+end
+end

data/lib/cuboid/rpc/client/instance.rb

@@ -0,0 +1,100 @@
+module Cuboid
+
+require Options.paths.lib + 'rpc/client/base'
+
+module RPC
+class Client
+
+# RPC client for remote instances spawned by a remote dispatcher
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Instance
+    # Not always available, set by the parent.
+    attr_accessor :pid
+    attr_reader :options
+
+    require_relative 'instance/service'
+
+    class <<self
+
+        def when_ready( url, token, &block )
+            options = Cuboid::Options.rpc.to_client_options.merge(
+                client_max_retries:   0,
+                connection_pool_size: 1
+            )
+
+            client = new( url, token, options )
+            Arachni::Reactor.global.delay( 0.1 ) do |task|
+                client.alive? do |r|
+                    if r.rpc_exception?
+                        Arachni::Reactor.global.delay( 0.1, &task )
+                        next
+                    end
+
+                    client.close
+
+                    block.call
+                end
+            end
+        end
+
+    end
+
+    def initialize( url, token = nil, options = nil )
+        @token    = token
+        @client   = Base.new( url, token, options )
+
+        @instance = Proxy.new( @client )
+        @options  = Arachni::RPC::Proxy.new( @client, 'options' )
+
+        # map Dispatcher handlers
+        Cuboid::Application.application.instance_services.keys.each do |name|
+            self.class.send( :attr_reader, name.to_sym )
+
+            instance_variable_set(
+              "@#{name}".to_sym,
+              Arachni::RPC::Proxy.new( @client, name )
+            )
+        end
+    end
+
+    def when_ready( &block )
+        self.class.when_ready( url, token, &block )
+    end
+
+    def token
+        @token
+    end
+
+    def client
+        @client
+    end
+
+    def close
+        @client.close
+    end
+
+    def url
+        @client.url
+    end
+
+    def address
+        @client.address
+    end
+
+    def port
+        @client.port
+    end
+
+    private
+
+    # Used to provide the illusion of locality for remote methods
+    def method_missing( sym, *args, &block )
+        @instance.send( sym, *args, &block )
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/client/scheduler.rb

@@ -0,0 +1,46 @@
+module Cuboid
+
+require Options.paths.lib + 'rpc/client/base'
+
+module RPC
+class Client
+
+# RPC Scheduler client
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Scheduler
+    # Not always available, set by the parent.
+    attr_accessor :pid
+
+    def initialize( url, options = nil )
+        @client = Base.new( url, nil, options )
+    end
+
+    def url
+        @client.url
+    end
+
+    def address
+        @client.address
+    end
+
+    def port
+        @client.port
+    end
+
+    def close
+        @client.close
+    end
+
+    private
+
+    # Used to provide the illusion of locality for remote methods
+    def method_missing( sym, *args, &block )
+        @client.call( "scheduler.#{sym.to_s}", *args, &block )
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rpc/client.rb

@@ -0,0 +1,3 @@
+require_relative 'client/instance'
+require_relative 'client/dispatcher'
+require_relative 'client/scheduler'

data/lib/cuboid/rpc/serializer.rb

@@ -0,0 +1,92 @@
+require 'msgpack'
+
+module Cuboid
+module RPC
+
+# Used for serialization of {RPC} messages.
+#
+# It's simply a delegator for `MessagePack` with `Zlib` compression for messages
+# that are larger than {COMPRESS_LARGER_THAN}.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+module Serializer
+
+    # Compress object dumps larger than 1KB.
+    COMPRESS_LARGER_THAN = 1_000
+
+    # @param    [#to_rpc_data]   object
+    #
+    # @return   [String]
+    #   {#compress Compressed} `object` dump.
+    def dump( object )
+        # ap object
+        compress( serializer.dump( object.to_rpc_data_or_self ) )
+    end
+
+    # @param    [String]   dump
+    #   {#dump Dumped} object.
+    #
+    # @return   [Object]
+    def load( dump )
+        serializer.load( decompress( dump ) )
+    end
+
+    # Simulates an object's over-the-wire transmission by {#dump dumping}
+    # and then {#load loading}.
+    #
+    # @param    [#to_rpc_data,.from_rpc_data]   object
+    #
+    # @return   [Object]
+    #   Data that the peer would receive.
+    def rpc_data( object )
+        load( dump( object ) )
+    end
+
+    # @param    [#to_rpc_data,.from_rpc_data]   object
+    #
+    # @return   [Object]
+    def deep_clone( object )
+        object.class.from_rpc_data rpc_data( object )
+    end
+
+    # @note Ignores strings smaller than #{COMPRESS_LARGER_THAN}.
+    #
+    # @param    [String]    string
+    #   String to compress.
+    #
+    # @return   [String]
+    #   Compressed (or not) `string`.
+    def compress( string )
+        return string if string.size < COMPRESS_LARGER_THAN
+        Zlib::Deflate.deflate string
+    end
+
+    # @note Will return the `string` as is if it was not compressed.
+    #
+    # @param    [String]    string
+    #   String to decompress.
+    #
+    # @return   [String]
+    #   Decompressed string.
+    def decompress( string )
+        return '' if string.to_s.empty?
+
+        # Just an ID representing a serialized, empty data structure.
+        return string if string.size == 1
+
+        begin
+            Zlib::Inflate.inflate string
+        rescue Zlib::DataError
+            string
+        end
+    end
+
+    def serializer
+        MessagePack
+    end
+
+    extend self
+end
+
+end
+end

data/lib/cuboid/rpc/server/active_options.rb

@@ -0,0 +1,38 @@
+module Cuboid
+module RPC
+class Server
+
+# It, for the most part, forwards calls to {Cuboid::Options} and intercepts
+# a few that need to be updated at other places throughout the framework.
+#
+# @private
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class ActiveOptions
+
+    def initialize
+        @options = Cuboid::Options.instance
+
+        (@options.public_methods( false ) - public_methods( false ) ).each do |m|
+            self.class.class_eval do
+                define_method m do |*args|
+                    @options.send( m, *args )
+                end
+            end
+        end
+    end
+
+    # @see Cuboid::Options#set
+    def set( options )
+        @options.set( options )
+        true
+    end
+
+    def to_h
+        @options.to_rpc_data
+    end
+
+end
+
+end
+end
+end