cuboid 0.0.0 → 0.0.3

data/lib/cuboid/report.rb

@@ -0,0 +1,220 @@
+require_relative 'rpc/serializer'
+
+module Cuboid
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Report
+    include Utilities
+
+    EXTENSION = 'crf'
+    
+    INTEGER_SIZE = 4
+
+    UNPACK = 'N'
+    
+    # @return    [String]
+    #   {Cuboid::VERSION}
+    attr_accessor :version
+
+    attr_accessor :application
+
+    # @return    [Symbol]
+    attr_accessor :status
+
+    # @return    [String]
+    #   Scan seed.
+    attr_accessor :seed
+
+    # @return    [Hash]
+    #   {Options#to_h}
+    attr_accessor :options
+
+    # Arbitrary data from the Application.
+    attr_accessor :data
+
+    # @return    [Time]
+    #   The date and time when the scan started.
+    attr_accessor :start_datetime
+
+    # @return    [Time]
+    #   The date and time when the scan finished.
+    attr_accessor :finish_datetime
+
+    def initialize( options = {} )
+        options.each { |k, v| send( "#{k}=", v ) }
+
+        @version ||= Cuboid::VERSION
+
+        @start_datetime  ||= Time.now
+        @finish_datetime ||= Time.now
+    end
+
+    # @note If no {#finish_datetime} has been provided, it will use `Time.now`.
+    #
+    # @return   [String]
+    #   `{#start_datetime} - {#finish_datetime}` in `00:00:00`
+    #   (`hours:minutes:seconds`) format.
+    def delta_time
+        seconds_to_hms( (@finish_datetime || Time.now) - @start_datetime )
+    end
+
+    # @param    [String]    report
+    #   Location of the report.
+    #
+    # @return   [Hash]
+    #   {#summary} associated with the given report.
+    def self.read_summary( report )
+        File.open( report ) do |f|
+            f.seek -INTEGER_SIZE, IO::SEEK_END
+            summary_size = f.read( INTEGER_SIZE ).unpack( 'N' ).first
+
+            f.seek -summary_size-INTEGER_SIZE, IO::SEEK_END
+            summary = RPC::Serializer.load( f.read( summary_size ) ).my_symbolize_keys
+            summary[:application] = ObjectSpace.const_get( summary[:application].to_sym )
+            summary
+        end
+    end
+
+    # Loads and a {#save saved} {Report} object from file.
+    #
+    # @param    [String]    file
+    #   File created by {#save}.
+    #
+    # @return    [Report]
+    #   Loaded instance.
+    def self.load( file )
+        File.open( file, 'rb' ) do |f|
+            from_rpc_data RPC::Serializer.load( self.crf_without_summary( f ) )
+        end
+    end
+
+    # @param    [String]    location
+    #   Location for the {#to_crf dumped} report file.
+    #
+    # @return   [String]
+    #   Absolute location of the report.
+    def save( location = nil )
+        if !location
+            location = default_filename
+        elsif File.directory? location
+            location += "/#{default_filename}"
+        end
+
+        # We do it this way to prevent FS watchers from grabbing files that
+        # are in the process of being written and thus partial.
+        tmp = "#{Options.paths.tmpdir}/#{Utilities.generate_token}"
+        IO.binwrite( tmp, to_crf )
+        FileUtils.mv tmp, location
+
+        File.expand_path( location )
+    end
+
+    # @return   [String]
+    #   Report serialized in the Cuboid Report format.
+    def to_crf
+        crf = RPC::Serializer.dump( self )
+
+        sum = summary
+        sum[:application] = sum[:application].to_s
+        # Append metadata to the end of the dump.
+        metadata = RPC::Serializer.dump( sum )
+        crf << [metadata, metadata.size].pack( "a*#{UNPACK}" )
+
+        crf
+    end
+
+    def self.from_crf( data )
+        from_rpc_data RPC::Serializer.load(
+          self.crf_without_summary( StringIO.new( data ) )
+        )
+    end
+
+    # @return   [Hash]
+    #   Hash representation of `self`.
+    def to_h
+        h = {
+            application:     @application,
+            version:         @version,
+            status:          @status,
+            seed:            @seed,
+            data:            @data,
+            options:         Cuboid::Options.hash_to_rpc_data( @options ),
+            start_datetime:  @start_datetime.to_s,
+            finish_datetime: @finish_datetime.to_s,
+            delta_time:      delta_time
+        }
+    end
+    alias :to_hash :to_h
+
+    # @return   [Hash]
+    #   Summary data of the report.
+    def summary
+        {
+            application:     @application,
+            version:         @version,
+            status:          @status,
+            seed:            @seed,
+            start_datetime:  @start_datetime.to_s,
+            finish_datetime: @finish_datetime.to_s,
+            delta_time:      delta_time
+        }
+    end
+
+    # @return   [Hash]
+    #   Data representing this instance that are suitable the RPC transmission.
+    def to_rpc_data
+        data = {}
+        instance_variables.each do |ivar|
+            data[ivar.to_s.gsub('@','')] = instance_variable_get( ivar )
+        end
+
+        data['application'] = data['application'].to_s
+        data['data']        = @application.serializer.dump( data['data'] )
+        data['options']     = @application.serializer.dump( data['options'] )
+
+        data['start_datetime']  = data['start_datetime'].to_s
+        data['finish_datetime'] = data['finish_datetime'].to_s
+        data
+    end
+
+    # @param    [Hash]  data    {#to_rpc_data}
+    # @return   [DOM]
+    def self.from_rpc_data( data )
+        data['start_datetime']  = Time.parse( data['start_datetime'] )
+        data['finish_datetime'] = Time.parse( data['finish_datetime'] )
+
+        data['application'] = ObjectSpace.const_get( data['application'] )
+        data['data']        = data['application'].serializer.load( data['data'] )
+        data['options']     = data['application'].serializer.load( data['options'] )
+
+        new data
+    end
+
+    def ==( other )
+        hash == other.hash
+    end
+
+    def hash
+        h = to_hash
+        [:start_datetime, :finish_datetime, :delta_datetime].each do |k|
+            h.delete k
+        end
+        h.hash
+    end
+
+    private
+
+    def self.crf_without_summary( io )
+        io.seek -INTEGER_SIZE, IO::SEEK_END
+        summary_size = io.read( INTEGER_SIZE ).unpack( UNPACK ).first
+
+        io.rewind
+        io.read( io.size - summary_size - INTEGER_SIZE )
+    end
+
+    def default_filename
+        "Cuboid #{@finish_datetime.to_s.gsub( ':', '_' )}.#{EXTENSION}"
+    end
+
+end
+end

data/lib/cuboid/rest/server/instance_helpers.rb

@@ -0,0 +1,99 @@
+module Cuboid
+module Rest
+class Server
+
+module InstanceHelpers
+
+    @@instances   = {}
+    @@dispatchers = {}
+
+    def get_instance
+        if dispatcher
+            options = {
+              owner:   self.class.to_s,
+              helpers: {
+                    owner: {
+                        url: env['HTTP_HOST']
+                    }
+                }
+            }
+
+            if (info = dispatcher.dispatch( options ))
+                connect_to_instance( info['url'], info['token'] )
+            end
+        else
+            Processes::Instances.spawn( application: Options.paths.application )
+        end
+    end
+
+    def dispatchers
+        @@dispatchers.keys
+    end
+
+    def dispatcher
+        return if !Options.dispatcher.url
+        @dispatcher ||= connect_to_dispatcher( Options.dispatcher.url )
+    end
+
+    def unplug_dispatcher( url )
+        connect_to_dispatcher( url ).node.unplug
+
+        c = @@dispatchers.delete( url )
+        c.close if c
+    end
+
+    def connect_to_dispatcher( url )
+        @@dispatchers[url] ||= RPC::Client::Dispatcher.new( url )
+    end
+
+    def connect_to_instance( url, token )
+        RPC::Client::Instance.new( url, token )
+    end
+
+    def update_from_scheduler
+        return if !scheduler
+
+        scheduler.running.each do |id, info|
+            instances[id] ||= connect_to_instance( info['url'], info['token'] )
+        end
+
+        (scheduler.failed.keys | scheduler.completed.keys).each do |id|
+            session.delete id
+            client = instances.delete( id )
+            client.close if client
+        end
+    end
+
+    def scheduler
+        return if !Options.scheduler.url
+        @scheduler ||= connect_to_scheduler( Options.scheduler.url )
+    end
+
+    def connect_to_scheduler( url )
+        RPC::Client::Scheduler.new( url )
+    end
+
+    def instances
+        @@instances
+    end
+
+    def instance_for( id, &block )
+        cleanup = proc do
+            instances.delete( id ).close
+            session.delete id
+        end
+
+        handle_error cleanup do
+            block.call @@instances[id]
+        end
+    end
+
+    def exists?( id )
+        instances.include? id
+    end
+
+end
+
+end
+end
+end

data/lib/cuboid/rest/server/routes/dispatcher.rb

@@ -0,0 +1,41 @@
+module Cuboid
+module Rest
+class Server
+module Routes
+
+module Dispatcher
+
+    def self.registered( app )
+
+        app.get '/dispatcher/url' do
+            ensure_dispatcher!
+
+            json Options.dispatcher.url
+        end
+
+        app.put '/dispatcher/url' do
+            url = ::JSON.load( request.body.read ) || {}
+
+            handle_error do
+                connect_to_dispatcher( url ).alive?
+
+                @dispatcher = nil
+                Options.dispatcher.url = url
+                json nil
+            end
+        end
+
+        app.delete '/dispatcher/url' do
+            ensure_dispatcher!
+
+            json @dispatcher = Options.dispatcher.url = nil
+        end
+
+    end
+
+end
+
+end
+end
+end
+end

data/lib/cuboid/rest/server/routes/grid.rb

@@ -0,0 +1,41 @@
+module Cuboid
+module Rest
+class Server
+module Routes
+
+module Grid
+
+    def self.registered( app )
+
+        app.get '/grid' do
+            ensure_dispatcher!
+
+            handle_error do
+                json [Options.dispatcher.url] + dispatcher.statistics['node']['neighbours']
+            end
+        end
+
+        app.get '/grid/:dispatcher' do |url|
+            ensure_dispatcher!
+
+            handle_error { json connect_to_dispatcher( url ).statistics }
+        end
+
+        app.delete '/grid/:dispatcher' do |url|
+            ensure_dispatcher!
+
+            handle_error do
+                unplug_dispatcher( url )
+            end
+
+            json nil
+        end
+
+    end
+
+end
+
+end
+end
+end
+end

data/lib/cuboid/rest/server/routes/instances.rb

@@ -0,0 +1,131 @@
+module Cuboid
+module Rest
+class Server
+module Routes
+
+module Instances
+
+    def self.registered( app )
+
+        # List instances.
+        app.get '/instances' do
+            update_from_scheduler
+
+            json instances.keys.inject({}){ |h, k| h.merge! k => {} }
+        end
+
+        # Create
+        app.post '/instances' do
+            max_utilization! if !dispatcher && System.max_utilization?
+
+            options = ::JSON.load( request.body.read ) || {}
+
+            instance = get_instance
+            max_utilization! if !instance
+
+            handle_error proc { instance.shutdown } do
+                instance.run( options )
+            end
+
+            instances[instance.token] = instance
+
+            json id: instance.token
+        end
+
+        # Progress
+        app.get '/instances/:instance' do
+            ensure_instance!
+
+            session[params[:instance]] ||= {
+                seen_errors:  0,
+            }
+
+            data = instance_for( params[:instance] ) do |instance|
+                instance.progress(
+                    with:    [
+                                 errors:  session[params[:instance]][:seen_errors],
+                             ]
+                )
+            end
+
+            session[params[:instance]][:seen_errors] += data[:errors].size
+
+            json data
+        end
+
+        app.put '/instances/:instance/scheduler' do |instance|
+            ensure_scheduler!
+            ensure_instance!
+
+            handle_error do
+                instance = instances.delete( instance )
+                instance.close
+
+                json scheduler.attach( instance.url, instance.token )
+            end
+        end
+
+        app.get '/instances/:instance/summary' do
+            ensure_instance!
+
+            instance_for( params[:instance] ) do |instance|
+                json instance.progress
+            end
+        end
+
+        app.get '/instances/:instance/report.crf' do
+            ensure_instance!
+            content_type 'application/octet-stream'
+
+            instance_for( params[:instance] ) do |instance|
+                instance.generate_report.to_crf
+            end
+        end
+
+        app.get '/instances/:instance/report.json' do
+            ensure_instance!
+
+            instance_for( params[:instance] ) do |instance|
+                instance.generate_report.to_rpc_data.to_json
+            end
+        end
+
+        app.put '/instances/:instance/pause' do
+            ensure_instance!
+
+            instance_for( params[:instance] ) do |instance|
+                json instance.pause!
+            end
+        end
+
+        app.put '/instances/:instance/resume' do
+            ensure_instance!
+
+            instance_for( params[:instance] ) do |instance|
+                json instance.resume!
+            end
+        end
+
+        # Abort/shutdown
+        app.delete '/instances/:instance' do
+            ensure_instance!
+            id = params[:instance]
+
+            instance = instances[id]
+            handle_error { instance.shutdown }
+
+            instances.delete( id ).close
+
+            session.delete params[:instance]
+
+            json nil
+        end
+
+    end
+
+end
+
+end
+end
+end
+end

data/lib/cuboid/rest/server/routes/scheduler.rb

@@ -0,0 +1,140 @@
+module Cuboid
+module Rest
+class Server
+module Routes
+
+module Scheduler
+
+    def self.registered( app )
+
+        app.get '/scheduler' do
+            ensure_scheduler!
+
+            handle_error do
+                json scheduler.list
+            end
+        end
+
+        app.get '/scheduler/url' do
+            ensure_scheduler!
+
+            handle_error do
+                json Options.scheduler.url
+            end
+        end
+
+        app.put '/scheduler/url' do
+            url = ::JSON.load( request.body.read ) || {}
+
+            handle_error do
+                connect_to_scheduler( url ).alive?
+
+                @scheduler = nil
+                Options.scheduler.url = url
+                json nil
+            end
+        end
+
+        app.delete '/scheduler/url' do
+            ensure_scheduler!
+
+            json @scheduler = Options.scheduler.url = nil
+        end
+
+        app.get '/scheduler/running' do
+            ensure_scheduler!
+
+            handle_error do
+                json scheduler.running
+            end
+        end
+
+        app.get '/scheduler/completed' do
+            ensure_scheduler!
+
+            handle_error do
+                json scheduler.completed
+            end
+        end
+
+        app.get '/scheduler/failed' do
+            ensure_scheduler!
+
+            handle_error do
+                json scheduler.failed
+            end
+        end
+
+        app.get '/scheduler/size' do
+            ensure_scheduler!
+
+            handle_error do
+                json scheduler.size
+            end
+        end
+
+        app.delete '/scheduler' do
+            ensure_scheduler!
+
+            handle_error do
+                json scheduler.clear
+            end
+        end
+
+        app.post '/scheduler' do
+            ensure_scheduler!
+
+            handle_error do
+                json id: scheduler.push( *[::JSON.load( request.body.read )].flatten )
+            end
+        end
+
+        app.get '/scheduler/:instance' do |instance|
+            ensure_scheduler!
+
+            handle_error do
+                instance = scheduler.get( instance )
+                if !instance
+                    halt 404, json( 'Instance not in Scheduler.' )
+                end
+
+                json instance
+            end
+        end
+
+        app.put '/scheduler/:instance/detach' do |instance|
+            ensure_scheduler!
+
+            handle_error do
+                info = scheduler.detach( instance )
+
+                if !info
+                    halt 404, json( 'Instance not in Scheduler.' )
+                end
+
+                instances[instance] ||= connect_to_instance( info['url'], info['token'] )
+            end
+
+            json nil
+        end
+
+        app.delete '/scheduler/:instance' do |instance|
+            ensure_scheduler!
+
+            handle_error do
+                if scheduler.remove( instance )
+                    json nil
+                else
+                    halt 404, json( 'Instance not in Scheduler.' )
+                end
+            end
+        end
+
+    end
+
+end
+
+end
+end
+end
+end