cuboid 0.0.0 → 0.0.1alpha

data/lib/cuboid/processes/manager.rb

@@ -0,0 +1,262 @@
+require 'singleton'
+require 'arachni/reactor'
+
+module Cuboid
+module Processes
+
+# Helper for managing processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Manager
+    include Singleton
+
+    RUNNER = "#{File.dirname( __FILE__ )}/executables/base.rb"
+
+    # @return   [Array<Integer>] PIDs of all running processes.
+    attr_reader :pids
+
+    def initialize
+        reset
+    end
+
+    def reset
+        @pids           = []
+        @discard_output = true
+    end
+
+    # @param    [Integer]   pid
+    #   Adds a PID to the {#pids} and detaches the process.
+    #
+    # @return   [Integer]   `pid`
+    def <<( pid )
+        @pids << pid
+        Process.detach pid
+        pid
+    end
+
+    # @param    [Integer]   pid
+    #   PID of the process to kill.
+    def kill( pid )
+        fail 'Cannot kill self.' if pid == Process.pid
+
+        Timeout.timeout 10 do
+            while sleep 0.1 do
+                begin
+                    Process.kill( Cuboid.windows? ? 'KILL' : 'TERM', pid )
+
+                # Either kill was successful or we don't have enough perms or
+                # we hit a reused PID for someone else's process, either way,
+                # consider the process gone.
+                rescue Errno::ESRCH, Errno::EPERM,
+                    # Don't kill ourselves.
+                    SignalException
+
+                    @pids.delete pid
+                    return
+                end
+            end
+        end
+    rescue Timeout::Error
+    end
+
+    def find( bin )
+        find_in_path( bin ) || find_in_applications( bin ) ||
+            find_in_program_files( bin )
+    end
+
+    def find_in_path( bin )
+        @find_in_path ||= {}
+        return @find_in_path[bin] if @find_in_path.include?( bin )
+
+        if Cuboid.windows?
+            bin = "#{bin}.exe"
+        end
+
+        ENV['PATH'].split( File::PATH_SEPARATOR ).each do |path|
+            f = File.join( path, bin )
+            return @find_in_path[bin] = f if File.exist?( f )
+        end
+
+        @find_in_path[bin] = nil
+    end
+
+    def find_in_applications( bin )
+        return if !Cuboid.mac?
+
+        @find_in_applications ||= {}
+        return @find_in_applications[bin] if @find_in_applications.include?( bin )
+
+        [
+            '/Applications/*/Contents/MacOS'
+        ].each do |root|
+            glob = File.join( root, '**', bin )
+
+            exe = Dir.glob( glob ).find { |f| File.executable?( f ) }
+            return @find_in_applications[bin] = exe if exe
+        end
+
+        @find_in_applications[bin] = nil
+    end
+
+    def find_in_program_files( bin )
+        return if !Cuboid.windows?
+
+        @find_in_program_files ||= {}
+        return @find_in_program_files[bin] if @find_in_program_files.include?( bin )
+
+        [
+            ENV['PROGRAMFILES']      || '\\Program Files',
+            ENV['ProgramFiles(x86)'] || '\\Program Files (x86)',
+            ENV['ProgramW6432']      || '\\Program Files'
+        ].each do |root|
+            glob = File.join( root, '**', "#{bin}.exe" )
+            glob.tr!( '\\', '/' )
+
+            exe = Dir.glob( glob ).find { |f| File.executable?( f ) }
+            return @find_in_program_files[bin] = exe if exe
+        end
+
+        @find_in_program_files[bin] = nil
+    end
+
+    # @param    [Integer]   pid
+    # @return   [Boolean]
+    #   `true` if the process is alive, `false` otherwise.
+    def alive?( pid )
+        # Windows is not big on POSIX so try it its own way if possible.
+        if Cuboid.windows?
+            begin
+                alive = false
+                processes = wmi.ExecQuery( "select ProcessId from win32_process where ProcessID='#{pid}'" )
+                processes.each do |proc|
+                    proc.ole_free
+                    alive = true
+                end
+                processes.ole_free
+
+                return alive
+            rescue WIN32OLERuntimeError
+            end
+        end
+
+        !!(Process.kill( 0, pid ) rescue false)
+    end
+
+    # @param    [Array<Integer>]   pids
+    #   PIDs of the process to {Cuboid::Processes::Manager#kill}.
+    def kill_many( pids )
+        pids.each { |pid| kill pid }
+    end
+
+    # Kills all {#pids processes}.
+    def killall
+        kill_many @pids.dup
+        @pids.clear
+    end
+
+    # Stops the Reactor.
+    def kill_reactor
+        Arachni::Reactor.stop
+    rescue
+        nil
+    end
+
+    # Overrides the default setting of discarding process outputs.
+    def preserve_output
+        @discard_output = false
+    end
+
+    def preserve_output?
+        !discard_output?
+    end
+
+    def discard_output
+        @discard_output = true
+    end
+
+    def discard_output?
+        @discard_output
+    end
+
+    # @param    [String]    executable
+    #   Name of the executable Ruby script found in {OptionGroups::Paths#executables}
+    #   without the '.rb' extension.
+    # @param    [Hash]  options
+    #   Options to pass to the script -- can be retrieved from `$options`.
+    #
+    # @return   [Integer]
+    #   PID of the process.
+    def spawn( executable, options = {} )
+        fail ArgumentError, 'Fork not supported.' if options.delete(:fork)
+
+        stdin      = options.delete(:stdin)
+        stdout     = options.delete(:stdout)
+        stderr     = options.delete(:stderr)
+        new_pgroup = options.delete(:new_pgroup)
+
+        spawn_options = {}
+
+        if new_pgroup
+            if Cuboid.windows?
+                spawn_options[:new_pgroup] = new_pgroup
+            else
+                spawn_options[:pgroup] = new_pgroup
+            end
+        end
+
+        spawn_options[:in]  = stdin  if stdin
+        spawn_options[:out] = stdout if stdout
+        spawn_options[:err] = stderr if stderr
+
+        options[:ppid]   = Process.pid
+        options[:tmpdir] = Options.paths.tmpdir
+
+        cuboid_options = Options.dup.update( options.delete(:options) || {} ).to_h
+        encoded_cuboid_options = Base64.strict_encode64( Marshal.dump( cuboid_options ) )
+
+        if executable.is_a? Symbol
+            executable = "#{Options.paths.executables}/#{executable}.rb"
+        elsif !File.exist?( executable )
+            raise ArgumentError, "Executable does not exist: #{executable}"
+        end
+
+        encoded_options = Base64.strict_encode64( Marshal.dump( options ) )
+        argv            = [executable, encoded_options]
+
+        # It's very, **VERY** important that we use this argument format as
+        # it bypasses the OS shell and we can thus count on a 1-to-1 process
+        # creation and that the PID we get will be for the actual process.
+        pid = Process.spawn(
+            {
+                'CUBOID_SPAWN_OPTIONS' => encoded_cuboid_options
+            },
+            RbConfig.ruby,
+            RUNNER,
+            *(argv + [spawn_options])
+        )
+
+        self << pid
+        pid
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+
+    private
+
+    def wmi
+        @wmi ||= WIN32OLE.connect( 'winmgmts://' )
+    end
+end
+
+end
+end

data/lib/cuboid/processes/schedulers.rb

@@ -0,0 +1,128 @@
+module Cuboid
+module Processes
+
+# Helper for managing {RPC::Server::Scheduler} processes.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Schedulers
+    include Singleton
+    include Utilities
+
+    # @return   [Array<String>] URLs of all running Queues.
+    attr_reader :list
+
+    def initialize
+        @list    = []
+        @clients = {}
+    end
+
+    # Connects to a Scheduler by URL.
+    #
+    # @param    [String]    url URL of the Scheduler.
+    # @param    [Hash]    options Options for the RPC client.
+    #
+    # @return   [RPC::Client::Scheduler]
+    def connect( url, options = nil )
+        Arachni::Reactor.global.run_in_thread if !Arachni::Reactor.global.running?
+
+        fresh = false
+        if options
+            fresh = options.delete( :fresh )
+        end
+
+        if fresh
+            @clients[url] = RPC::Client::Scheduler.new( url, options )
+        else
+            @clients[url] ||= RPC::Client::Scheduler.new( url, options )
+        end
+    end
+
+    # @param    [Block] block   Block to pass an RPC client for each Scheduler.
+    def each( &block )
+        @list.each do |url|
+            block.call connect( url )
+        end
+    end
+
+    # Spawns a {RPC::Server::Scheduler} process.
+    #
+    # @param    [Hash]  options
+    #   To be passed to {Cuboid::Options#set}. Allows `address` instead of
+    #   `rpc_server_address` and `port` instead of `rpc_port`.
+    #
+    # @return   [RPC::Client::Queue]
+    def spawn( options = {} )
+        fork = options.delete(:fork)
+
+        options = {
+            dispatcher: {
+                url: options[:dispatcher],
+            },
+            rpc:        {
+                server_port:             options[:port]    || Utilities.available_port,
+                server_address:          options[:address] || '127.0.0.1',
+                server_external_address: options[:external_address]
+            },
+            paths: {
+              application: options[:application]  || Options.paths.application
+            }
+        }
+
+        pid = Manager.spawn( :scheduler, options: options, fork: fork )
+
+        url = "#{options[:rpc][:server_address]}:#{options[:rpc][:server_port]}"
+        while sleep( 0.1 )
+            begin
+                connect( url, connection_pool_size: 1, max_retries: 1 ).alive?
+                break
+            rescue => e
+                # ap e
+            end
+        end
+
+        @list << url
+        connect( url, fresh: true ).tap { |c| c.pid = pid }
+    end
+
+    # @note Will also kill all Instances started by the Scheduler.
+    #
+    # @param    [String]    url URL of the Scheduler to kill.
+    def kill( url )
+        scheduler = connect( url )
+        scheduler.clear
+        scheduler.running.each do |id, instance|
+            Manager.kill instance['pid']
+        end
+        Manager.kill scheduler.pid
+    rescue => e
+        #ap e
+        #ap e.backtrace
+        nil
+    ensure
+        @list.delete( url )
+        @clients.delete( url ).close
+    end
+
+    # Kills all {Queues #list}.
+    def killall
+        @list.dup.each do |url|
+            kill url
+        end
+    end
+
+    def self.method_missing( sym, *args, &block )
+        if instance.respond_to?( sym )
+            instance.send( sym, *args, &block )
+        else
+            super( sym, *args, &block )
+        end
+    end
+
+    def self.respond_to?( m )
+        super( m ) || instance.respond_to?( m )
+    end
+
+end
+
+end
+end

data/lib/cuboid/report.rb

@@ -0,0 +1,220 @@
+require_relative 'rpc/serializer'
+
+module Cuboid
+
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+class Report
+    include Utilities
+
+    EXTENSION = 'crf'
+    
+    INTEGER_SIZE = 4
+
+    UNPACK = 'N'
+    
+    # @return    [String]
+    #   {Cuboid::VERSION}
+    attr_accessor :version
+
+    attr_accessor :application
+
+    # @return    [Symbol]
+    attr_accessor :status
+
+    # @return    [String]
+    #   Scan seed.
+    attr_accessor :seed
+
+    # @return    [Hash]
+    #   {Options#to_h}
+    attr_accessor :options
+
+    # Arbitrary data from the Application.
+    attr_accessor :data
+
+    # @return    [Time]
+    #   The date and time when the scan started.
+    attr_accessor :start_datetime
+
+    # @return    [Time]
+    #   The date and time when the scan finished.
+    attr_accessor :finish_datetime
+
+    def initialize( options = {} )
+        options.each { |k, v| send( "#{k}=", v ) }
+
+        @version ||= Cuboid::VERSION
+
+        @start_datetime  ||= Time.now
+        @finish_datetime ||= Time.now
+    end
+
+    # @note If no {#finish_datetime} has been provided, it will use `Time.now`.
+    #
+    # @return   [String]
+    #   `{#start_datetime} - {#finish_datetime}` in `00:00:00`
+    #   (`hours:minutes:seconds`) format.
+    def delta_time
+        seconds_to_hms( (@finish_datetime || Time.now) - @start_datetime )
+    end
+
+    # @param    [String]    report
+    #   Location of the report.
+    #
+    # @return   [Hash]
+    #   {#summary} associated with the given report.
+    def self.read_summary( report )
+        File.open( report ) do |f|
+            f.seek -INTEGER_SIZE, IO::SEEK_END
+            summary_size = f.read( INTEGER_SIZE ).unpack( 'N' ).first
+
+            f.seek -summary_size-INTEGER_SIZE, IO::SEEK_END
+            summary = RPC::Serializer.load( f.read( summary_size ) ).my_symbolize_keys
+            summary[:application] = ObjectSpace.const_get( summary[:application].to_sym )
+            summary
+        end
+    end
+
+    # Loads and a {#save saved} {Report} object from file.
+    #
+    # @param    [String]    file
+    #   File created by {#save}.
+    #
+    # @return    [Report]
+    #   Loaded instance.
+    def self.load( file )
+        File.open( file, 'rb' ) do |f|
+            from_rpc_data RPC::Serializer.load( self.crf_without_summary( f ) )
+        end
+    end
+
+    # @param    [String]    location
+    #   Location for the {#to_crf dumped} report file.
+    #
+    # @return   [String]
+    #   Absolute location of the report.
+    def save( location = nil )
+        if !location
+            location = default_filename
+        elsif File.directory? location
+            location += "/#{default_filename}"
+        end
+
+        # We do it this way to prevent FS watchers from grabbing files that
+        # are in the process of being written and thus partial.
+        tmp = "#{Options.paths.tmpdir}/#{Utilities.generate_token}"
+        IO.binwrite( tmp, to_crf )
+        FileUtils.mv tmp, location
+
+        File.expand_path( location )
+    end
+
+    # @return   [String]
+    #   Report serialized in the Cuboid Report format.
+    def to_crf
+        crf = RPC::Serializer.dump( self )
+
+        sum = summary
+        sum[:application] = sum[:application].to_s
+        # Append metadata to the end of the dump.
+        metadata = RPC::Serializer.dump( sum )
+        crf << [metadata, metadata.size].pack( "a*#{UNPACK}" )
+
+        crf
+    end
+
+    def self.from_crf( data )
+        from_rpc_data RPC::Serializer.load(
+          self.crf_without_summary( StringIO.new( data ) )
+        )
+    end
+
+    # @return   [Hash]
+    #   Hash representation of `self`.
+    def to_h
+        h = {
+            application:     @application,
+            version:         @version,
+            status:          @status,
+            seed:            @seed,
+            data:            @data,
+            options:         Cuboid::Options.hash_to_rpc_data( @options ),
+            start_datetime:  @start_datetime.to_s,
+            finish_datetime: @finish_datetime.to_s,
+            delta_time:      delta_time
+        }
+    end
+    alias :to_hash :to_h
+
+    # @return   [Hash]
+    #   Summary data of the report.
+    def summary
+        {
+            application:     @application,
+            version:         @version,
+            status:          @status,
+            seed:            @seed,
+            start_datetime:  @start_datetime.to_s,
+            finish_datetime: @finish_datetime.to_s,
+            delta_time:      delta_time
+        }
+    end
+
+    # @return   [Hash]
+    #   Data representing this instance that are suitable the RPC transmission.
+    def to_rpc_data
+        data = {}
+        instance_variables.each do |ivar|
+            data[ivar.to_s.gsub('@','')] = instance_variable_get( ivar )
+        end
+
+        data['application'] = data['application'].to_s
+        data['data']        = @application.serializer.dump( data['data'] )
+        data['options']     = @application.serializer.dump( data['options'] )
+
+        data['start_datetime']  = data['start_datetime'].to_s
+        data['finish_datetime'] = data['finish_datetime'].to_s
+        data
+    end
+
+    # @param    [Hash]  data    {#to_rpc_data}
+    # @return   [DOM]
+    def self.from_rpc_data( data )
+        data['start_datetime']  = Time.parse( data['start_datetime'] )
+        data['finish_datetime'] = Time.parse( data['finish_datetime'] )
+
+        data['application'] = ObjectSpace.const_get( data['application'] )
+        data['data']        = data['application'].serializer.load( data['data'] )
+        data['options']     = data['application'].serializer.load( data['options'] )
+
+        new data
+    end
+
+    def ==( other )
+        hash == other.hash
+    end
+
+    def hash
+        h = to_hash
+        [:start_datetime, :finish_datetime, :delta_datetime].each do |k|
+            h.delete k
+        end
+        h.hash
+    end
+
+    private
+
+    def self.crf_without_summary( io )
+        io.seek -INTEGER_SIZE, IO::SEEK_END
+        summary_size = io.read( INTEGER_SIZE ).unpack( UNPACK ).first
+
+        io.rewind
+        io.read( io.size - summary_size - INTEGER_SIZE )
+    end
+
+    def default_filename
+        "Cuboid #{@finish_datetime.to_s.gsub( ':', '_' )}.#{EXTENSION}"
+    end
+
+end
+end