rhoconnect 3.0.6 → 3.1.0.beta1

data/bench/lib/bench/aws_utils.rb

@@ -0,0 +1,326 @@
+$:.unshift File.join(File.dirname(__FILE__), '..', '..','..','lib')
+
+require 'rubygems'
+require 'yaml'
+
+require 'readline'
+require 'xmlsimple'
+
+module Bench
+  module AWSUtils
+    extend self
+    
+    module Constants
+      RC_VERSION      = Rhoconnect::VERSION
+      REGION          = 'us-west-1'
+      TEMPLATE_URL    = 'http://s3.amazonaws.com/rhoconnect-bench/packages/cloud-formation/ec2-autostack.txt'
+      CLIENTS_GROUP_LOGICAL_ID = 'BenchClientsGroup'
+      WAIT_FOR_SSH = 120
+      HOME_DIR        = `echo ~/`.strip.chomp("/")
+    end
+    
+    class ClientsGroup
+      attr_accessor :stack_name, :client_instances, :auto_scaling_group
+      
+      def initialize(stack_name)
+        @stack_name = stack_name
+        auto_scaling = Fog::AWS::AutoScaling.new(
+          :region => Bench::AWSUtils.aws_region,
+          :aws_access_key_id     => Bench::AWSUtils.aws_access_key_id,
+          :aws_secret_access_key => Bench::AWSUtils.aws_secret_access_key
+        )
+        group_resources = Bench::AWSUtils.cloud_formation.describe_stack_resources({'StackName' => stack_name, 
+          'LogicalResourceId' => Bench::AWSUtils::Constants::CLIENTS_GROUP_LOGICAL_ID}).body
+
+        @auto_scaling_group = auto_scaling.groups.get(group_resources['StackResources'].first['PhysicalResourceId'])      
+        @client_instances = []
+        @auto_scaling_group.instances.each do |instance|
+          next if instance.auto_scaling_group_name != @auto_scaling_group.id
+          ec2_instance = Bench::AWSUtils.fog_connection.servers.get(instance.id)
+          @client_instances << ec2_instance.dns_name 
+        end
+      end
+    end
+    
+    class << self
+      attr_accessor :fog_connection, :cloud_formation, :aws_access_key_id, :aws_secret_access_key, :aws_region
+      attr_accessor :aws_key_pair_name, :aws_ssh_pem_file
+    end
+    
+    def validate_presense_of_file(fname)
+      return File.file?(File.expand_path(fname.to_s))
+    end
+
+    def init_connection(settings_file)
+      unless Bench::gem_installed?('net-ssh-multi')
+        puts "In order to run distributed benchmark you need to have 'net-ssh-multi' gem installed"
+        puts "Install it by using : '[sudo] gem install net-ssh-multi'"
+        raise "Gem 'net-ssh-multi' is missing"    
+      end
+      unless Bench::gem_installed?('fog')
+        puts "In order to run distributed benchmark you need to have 'fog' gem installed"
+        puts "Install it by using : '[sudo] gem install fog'"
+        raise "Gem 'fog' is missing"                
+      end
+      
+      require 'net/ssh/multi'
+      require 'fog'
+      
+      fog_conf_file = ENV['HOME'] + '/.fog'
+      settings_file ||= fog_conf_file
+      
+      if validate_presense_of_file(settings_file)
+        # Read Fog ~/.fog configuration file
+        # :default:
+        #   :aws_access_key_id:     AKIAI...
+        #   :aws_secret_access_key: 9l2ruLeCINbilik...
+        #   :region:                us-west-1
+        puts "Using AWS settings from #{settings_file} file" 
+
+        settings = YAML::load(File.open(settings_file))
+        if not settings or settings[:default].nil?
+          raise "ERROR : AWS Settings file '#{settings_file}' doesn't have the mandatoty 'default' section"
+        end
+        config = settings[:default]
+        
+        @aws_ssh_pem_file = config[:aws_ssh_pem_file]
+        unless validate_presense_of_file(aws_ssh_pem_file)
+          raise "ERROR : Can not locate SSH Access Pem File '#{aws_ssh_pem_file}'\nMake sure you set :aws_ssh_pem_file properly in the AWS Settings file"
+        end
+        @aws_access_key_id = config[:aws_access_key_id]
+        @aws_secret_access_key = config[:aws_secret_access_key]
+        @aws_region = config[:region] || Constants::REGION
+        @aws_key_pair_name = config[:aws_key_pair_name]
+          
+      else
+        raise "ERROR : Can not locate AWS Settings file '#{settings_file}'\nYou must have this file in order to run the Distributed Benchmark Test"
+      end
+      
+      make_fog
+      make_cloud_formation
+    end
+    
+    # get_access_keys
+    # Retrieves the access key and secret access key from the above specified file.
+    def get_access_keys(fname)
+      return true if aws_access_key_id and aws_secret_access_key
+            
+      lines = IO.readlines(fname)
+      @aws_access_key_id = lines.first.strip.split("=")[1]
+      @aws_secret_access_key = lines.last.strip.split("=")[1]
+    end
+
+    # make_fog
+    # Generates the Fog object used to create the new ec2 instance.
+    def make_fog
+      @fog_connection ||= Fog::Compute.new(
+          :provider              => 'AWS',
+          :region                => aws_region,
+          :aws_access_key_id     => aws_access_key_id,
+          :aws_secret_access_key => aws_secret_access_key
+      )
+    end #make_fog
+
+    def make_cloud_formation
+      @cloud_formation ||= Fog::AWS::CloudFormation.new(
+        :region => aws_region,
+        :aws_access_key_id     => aws_access_key_id,
+        :aws_secret_access_key => aws_secret_access_key
+      )
+    end
+    
+    def get_template_data(template_url)
+      template_data = ''
+      begin
+        uri = URI.parse(template_url)
+        unless uri.scheme
+          File.open(uri.path) { |f| template_data << f.read }
+        else
+          response = Net::HTTP.get_response(uri) 
+          template_data = response.body if response.code == '200'
+        end      
+      rescue Exception => e
+        puts "ERROR: Can not obtain CloudFormation template from '#{template_url}'"
+        puts e.message
+      end
+      template_data
+    end
+    
+    # Creates new CloudFormation stack based upon template
+    def create_cf_stack
+      puts ""
+      puts " Creating new AWS CloudFormation stack at '#{aws_region}' region"
+      puts " using '#{Constants::TEMPLATE_URL}' template ..."
+      puts "    This may take several minutes, please be patient ..."
+      puts ""
+
+      stack_name = nil
+      stack_created = false
+      
+      begin
+        template_data = get_template_data(Constants::TEMPLATE_URL)
+        cloud_formation.validate_template('TemplateBody' => template_data)
+       
+        template_params = {}
+        template_params['SecurityKeyPair'] = aws_key_pair_name.to_s
+        options = {'TemplateBody' => template_data,
+                    'Parameters' => template_params}
+        stack_name = "BenchStack" + Time.now.strftime("%Y%m%d%H%M%S")
+        result = cloud_formation.create_stack(stack_name, options)
+        
+        event_counter = 0
+        in_progress = true
+        stack_created = false
+        while in_progress
+          events = cloud_formation.describe_stack_events(stack_name).body['StackEvents']
+          events.reverse[event_counter..-1].each do |event|
+            puts "Timestamp: #{event['Timestamp']}"
+            puts "LogicalResourceId: #{event['LogicalResourceId']}"
+            puts "ResourceType: #{event['ResourceType']}"
+            puts "ResourceStatus: #{event['ResourceStatus']}"
+            puts "ResourceStatusReason: #{event['ResourceStatusReason']}" if event['ResourceStatusReason']
+            puts "--"
+          
+            # track creation of the stack
+            if event['LogicalResourceId'] == stack_name
+              case event['ResourceStatus']
+              when 'CREATE_COMPLETE'
+                stack_created = true
+                in_progress = false
+              when /ROLLBACK/
+                stack_created = false
+                in_progress = false
+              when /DELETE/
+                stack_created = false
+                in_progress = false
+              when /FAILED/
+                stack_created = false
+                in_progress = false
+                break
+              end
+            end
+          end
+          event_counter += events.size - event_counter
+          sleep(2)
+        end
+      rescue Excon::Errors::BadRequest => excon_error
+        error_str = XmlSimple.xml_in(excon_error.response.body)['Error'][0]['Message'][0]
+        puts "ERROR: Cannot create AWS CloudFormation stack  : #{error_str}"
+        stack_created = false
+      rescue Excon::Errors::Forbidden => excon_error
+        error_str = XmlSimple.xml_in(excon_error.response.body)['Error'][0]['Message'][0]
+        puts "ERROR: Cannot create AWS CloudFormation stack  : #{error_str}"
+        stack_created = false
+      rescue Exception => e
+        puts "ERROR: Cannot create AWS CloudFormation stack  : #{e.class.name}: #{e.message}"
+        stack_created = false
+      end
+      
+      clients_group = nil
+      if stack_created
+        clients_group = get_clients_group(stack_name)
+         # wait until the SSH service is up and running
+        stack_created = establish_ssh_connection(clients_group)
+      end
+      
+      unless stack_created
+        delete_cf_stack(stack_name)
+        clients_group = nil
+        stack_name = nil
+      end
+      
+      clients_group
+    end
+    
+    # Creates new CloudFormation stack based upon template
+    def delete_cf_stack(stack_name)
+      return unless stack_name
+      puts ""
+      puts "Destroying AWS CloudFormation stack '#{stack_name}' at '#{ aws_region}' region"
+      puts " NOTE: this command doesn't ensure deletion of the stack. "
+      puts "       It is advised to check later that the stack has been really destroyed"
+      puts ""
+      
+      begin
+        cloud_formation.delete_stack(stack_name)
+      rescue Excon::Errors::BadRequest => excon_error
+        error_str = XmlSimple.xml_in(excon_error.response.body)['Error'][0]['Message'][0]
+        puts "ERROR: Cannot delete the stack '#{stack_name}' : #{error_str}"
+      rescue Excon::Errors::Forbidden => excon_error
+        error_str = XmlSimple.xml_in(excon_error.response.body)['Error'][0]['Message'][0]
+        puts "ERROR: Cannot delete the stack '#{stack_name}' : #{error_str}"
+      rescue Exception => e
+        puts "ERROR: Cannot delete the stack '#{stack_name}' : #{e.class.name}: #{e.message}"
+      end
+    end
+    
+    def get_clients_group(stack_name)
+      ClientsGroup.new(stack_name)
+    end
+    
+    def establish_ssh_connection(clients_group)
+      STDOUT.sync = true
+      ssh_established = false
+      begin 
+        start_timestamp = Time.now
+        sess_options = {:keys => [aws_ssh_pem_file]}
+        # just some simple command
+        command = 'pwd 1>/dev/null'
+        
+        # clean-up outdated info (sometimes DNS names are re-used
+        # so we need to clean-up SSH known hosts file)
+        clients_group.client_instances.each do |hostname|
+          system("ssh-keygen -R #{hostname} 1>/dev/null 2>&1")
+        end
+          
+        puts ""
+        puts " Stack '#{clients_group.stack_name}' is created. Waiting for SSH services to start-up..."
+        while not ssh_established
+          begin
+            run_stack_ssh_command(clients_group.client_instances, command)
+            # if we are here - SSH command has executed succesfully
+            puts " Done."
+            ssh_established = true
+            break
+          rescue Interrupt => i
+            raise "User Interruption"
+          rescue Net::SSH::AuthenticationFailed => e
+            raise e
+          rescue OpenSSL::PKey::PKeyError => e
+            raise e
+          rescue Errno::ECONNREFUSED => e
+            # service is not yet started - wait more
+          end
+        
+          # try for 60 seconds maximum
+          if (Time.now.to_i - start_timestamp.to_i) > Constants::WAIT_FOR_SSH
+            puts " Failed!"
+            puts "ERROR: Cannot establish SSH session with the stack's EC2 instances..."
+            puts ""
+            break
+          end
+          
+          sleep(10)
+          print '. '
+        end
+      rescue Exception => e
+        puts " Failed!"
+        puts "ERROR: Cannot establish SSH session with the stack's EC2 instances : #{e.class.name} : #{e.message}"
+        puts ""
+      end
+      
+      ssh_established
+    end
+    
+    def run_stack_ssh_command(ec2_clients, command)
+      sess_options = {:keys => [aws_ssh_pem_file]}
+      Net::SSH::Multi.start({:default_user => 'ec2-user'}) do |session|
+        # define the servers we want to use
+        session.use(sess_options) { ec2_clients }
+
+        # execute commands on all servers
+        session.exec command
+      end
+    end
+  end
+end

data/bench/lib/bench/bench_result_processor.rb

@@ -1,90 +1,283 @@
 #!/usr/bin/ruby
 require 'rubygems'
-require 'yaml'
-require 'gruff'
-$LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', '..','lib'))
-require 'bench'
 
-# 1) load all result YAML files
-current_dir = Dir.pwd
-results_dir = ARGV[0]
-if results_dir.nil?
-  puts "Bench Result Processor: No result directory is provided - Skipping ..."
-  exit(1)
-end
-Dir.chdir results_dir
-results_dir = Dir.pwd
-
-Dir.chdir current_dir
-output_dir = ARGV[1]
-output_dir ||= 'images'
-begin
-  Dir.mkdir output_dir
-rescue
-end 
-Dir.chdir output_dir
-output_dir = Dir.pwd
+module Bench
+  def self.gem_installed?(gem_name)
+    (Gem::Specification.respond_to?(:find_by_name) ?
+     Gem::Specification.find_by_name(gem_name) : Gem.source_index.find_name(gem_name).last) != nil
+  rescue Exception => e
+    false
+  end
 
-# load meta.yaml
-meta_hash = YAML.load_file("#{results_dir}/meta.yml") if File.exists?("#{results_dir}/meta.yml")
-if meta_hash.nil?
-  puts "Bench Result Processor: No valid meta.yml file is found in the result directory - Skipping ..."
-  exit(1)
-end
+  module PostProcessing
+    @plugins = ['RhoSpreadSheet','RhoGruff']
+    
+    
+    def self.execute(res_dir)
+      return unless res_dir
+      
+      puts ""
+      puts "Starting Benchmark Post-Processing ..."
+      puts ""
+      
+      @plugins.each do |plugin|
+        plugin_class = eval(plugin)
+        available = plugin_class.has?
+        if available
+          available = plugin_class.load_myself
+        end
+        
+        # print the message
+        if not available
+          puts ""
+          plugin_class.what_is_needed?
+          puts ""
+          next
+        end
+        
+        plugin_instance = plugin_class.new
+        plugin_instance.process res_dir
+      end
+    end
+    
+    # this post-processor creates EXCEL spreadsheets
+    class RhoSpreadSheet
+      def self.has?
+        Bench::gem_installed?('spreadsheet')
+      end  
+      
+      def self.what_is_needed?
+        puts "In order to run SpreadSheet post-processor - you need to have SpreadSheet gem installed"
+        puts "Install it by using : '[sudo] gem install spreadsheet'"
+      end 
+    
+      def self.load_myself
+        require 'yaml'
+        require 'spreadsheet'
+        true
+      end
+          
+      def process(res_dir)
+        current_dir = Dir.pwd
+        begin
+          puts "Starting SpreadSheet post-processor..."
+          # 1) Create images dir
+          Dir.chdir res_dir
+          @results_dir = Dir.pwd
+          output_dir = Bench.create_subdir 'spreadsheet'
+          Dir.chdir output_dir
+          @output_dir = Dir.pwd
+          
+          _load_meta_hash
+          _init
+          _process_res_files
+          _write
+          Dir.chdir current_dir
+        rescue Exception => e
+          Dir.chdir current_dir
+          raise e
+        end 
+      end
+      
+      def _load_meta_hash
+        # load meta.yaml
+        @meta_hash = YAML.load_file(File.join(@results_dir,'raw_data','meta.yml')) if File.exists?(File.join(@results_dir,'raw_data','meta.yml'))
+        if @meta_hash.nil?
+          raise "SpreadSheet Result Processor: No valid meta.yml file is found in the result directory - Skipping ..."
+        end
 
-@metrics = meta_hash[:metrics]
-if @metrics.nil?
-  puts "Bench Result Processor: No valid metrics are found in the result directory - Skipping ..."
-  exit(1)
-end
+        @metrics = @meta_hash[:metrics]
+        if @metrics.nil?
+          raise "SpreadSheet Result Processor: No valid metrics are found in the result directory - Skipping ..."
+        end
 
-if meta_hash[:x_keys].nil?
-  puts "Bench Result Processor: No valid x_keys are found in the result directory - Skipping ..."
-  exit(1)
-end
-@x_keys = meta_hash[:x_keys].keys
-@x_keys = @x_keys.sort_by(&Bench.sort_natural_order)
+        if @meta_hash[:x_keys].nil?
+          raise "SpreadSheet Result Processor: No valid x_keys are found in the result directory - Skipping ..."
+        end
+        @x_keys = @meta_hash[:x_keys].keys
+        @x_keys = @x_keys.sort_by(&Bench.sort_natural_order)
+      end
+      
+      def _init
+        # initialize graphs for each metric
+        # row 0 - payload labels
+        # col 0 - X keys  
+        @title = @meta_hash[:label]
+        @book = Spreadsheet::Workbook.new(@title)
+        @sheets = {}
+        axis_format = Spreadsheet::Format.new :color => :blue,
+                                             :weight => :bold,
+                                             :size => 18
+        @metrics.each do |name,index|
+          sheet = @book.create_worksheet({:name => "#{name} (#{@title})"})
+          @sheets[index] = sheet
+          sheet.column(0).default_format = axis_format
+          sheet.row(0).default_format = axis_format             
+          @meta_hash[:x_keys].each do |key,key_index|
+            sheet[key_index + 1, 0] = "#{key}"
+          end
+        end 
+      end
+      
+      def _process_res_files
+        # load all result files
+        res_files = Dir.entries(File.join(@results_dir,'raw_data')).collect { |entry| entry if entry =~ /bench.*result/ }
+        res_files.compact!
+        res_files = res_files.sort_by(&Bench.sort_natural_order)
+        
+        res_files.each_with_index do |entry, entry_index|
+          begin
+            res_hash = YAML.load_file(File.join(@results_dir,'raw_data',entry))
+            next if res_hash.nil? or res_hash.empty?
 
-# initialize graphs for each metric
-graphs = {}
-title = meta_hash[:label]
-@metrics.each do |name,index|
-  g = Gruff::Line.new
-  g.title = "#{title} (#{name})"
-  g.labels = meta_hash[:x_keys].invert
-  graphs[index] = g
-end
+            marker = entry.split('.').last.to_s
+            
+            @sheets.each do |index,sheet|
+              sheet[0, entry_index + 1] = "#{marker}"                        
+            end
 
-# load all result files
-Dir.foreach(results_dir) do |entry|
-  begin
-    next unless entry =~ /bench.*result/
-    
-    res_hash = YAML.load_file(File.join(results_dir, entry))
-    next if res_hash.nil? or res_hash.empty?
-    
-    marker = entry.split('.').last.to_s
-    
-    g_data = Array.new(@metrics.size) { Array.new }
-    @x_keys.each do |x_key|
-      results = res_hash[x_key]
-      results ||= Array.new(@metrics.size, 0.0)
-      results.each_with_index do |res, index|
-        g_data[index] << ("%0.4f" % res).to_f
+            g_data = Array.new(@metrics.size) { Array.new }
+            @x_keys.each do |x_key|
+              row_idx = @meta_hash[:x_keys][x_key] + 1
+              results = res_hash[x_key]
+              results ||= Array.new(@metrics.size, 0.0)
+              results.each_with_index do |res, index|
+                col_idx = entry_index + 1
+                @sheets[index][row_idx,col_idx] = ("%0.4f" % res).to_f
+              end
+            end
+          rescue Exception => e
+            raise "SpreadSheet processing resulted in Error : #{e.message} " + e.backtrace.join("\n")
+          end
+        end 
+      end
+      
+      def _write
+        image_fname = File.join(@output_dir,"bench_results.xls")
+        puts "Spreadsheet processor: writing #{image_fname}"
+        @book.write image_fname
       end
     end
     
-    graphs.each do |index, graph|
-      graph.data("#{marker}", g_data[index])
+    # this post-processor creates PNG graph files
+    class RhoGruff
+      def self.has?
+        Bench::gem_installed?('gruff')
+      end 
+      
+      def self.what_is_needed?
+        puts "In order to run Gruff post-processor - you need to have Gruff gem installed"
+        puts "Install it by using : '[sudo] gem install gruff'"
+        puts "You may also need to install additional components - please check Gruff documentation for details"
+      end  
+    
+      def self.load_myself
+        res = true
+        begin
+          require 'yaml'
+          require 'gruff'
+        rescue Exception => e
+          puts " Can not run Gruff post-processor : #{e.message}"
+          res = false
+        end
+        res
+      end
+          
+      def process(res_dir)
+        current_dir = Dir.pwd
+        begin
+          puts "Starting Gruff post-processor..."
+          # 1) Create images dir
+          Dir.chdir res_dir
+          @results_dir = Dir.pwd
+          output_dir = Bench.create_subdir 'images'
+          Dir.chdir output_dir
+          @output_dir = Dir.pwd
+          
+          _load_meta_hash
+          _init_graphs
+          _process_res_files
+          _write_graphs
+          Dir.chdir current_dir
+        rescue Exception => e
+          Dir.chdir current_dir
+          raise e
+        end 
+      end
+      
+      def _load_meta_hash
+        # load meta.yaml
+        @meta_hash = YAML.load_file(File.join(@results_dir,'raw_data','meta.yml')) if File.exists?(File.join(@results_dir,'raw_data','meta.yml'))
+        if @meta_hash.nil?
+          raise "Gruff Result Processor: No valid meta.yml file is found in the result directory - Skipping ..."
+        end
+
+        @metrics = @meta_hash[:metrics]
+        if @metrics.nil?
+          raise "Gruff Result Processor: No valid metrics are found in the result directory - Skipping ..."
+        end
+
+        if @meta_hash[:x_keys].nil?
+          raise "Gruff Result Processor: No valid x_keys are found in the result directory - Skipping ..."
+        end
+        @x_keys = @meta_hash[:x_keys].keys
+        @x_keys = @x_keys.sort_by(&Bench.sort_natural_order)
+      end
+      
+      def _init_graphs
+        # initialize graphs for each metric
+        @graphs = {}
+        @title = @meta_hash[:label]
+        @metrics.each do |name,index|
+          g = Gruff::Line.new
+          g.title = "#{@title} (#{name})"
+          g.labels = @meta_hash[:x_keys].invert
+          @graphs[index] = g
+        end 
+      end
+      
+      def _process_res_files
+        # load all result files
+        res_files = Dir.entries(File.join(@results_dir,'raw_data')).collect { |entry| entry if entry =~ /bench.*result/ }
+        res_files.compact!
+        res_files = res_files.sort_by(&Bench.sort_natural_order)
+        
+        # we can only create 7 unique lines
+        # per graph
+        for entry in res_files.last(7) do
+          begin
+            res_hash = YAML.load_file(File.join(@results_dir,'raw_data',entry))
+            next if res_hash.nil? or res_hash.empty?
+
+            marker = entry.split('.').last.to_s
+
+            g_data = Array.new(@metrics.size) { Array.new }
+            @x_keys.each do |x_key|
+              results = res_hash[x_key]
+              results ||= Array.new(@metrics.size, 0.0)
+              results.each_with_index do |res, index|
+                g_data[index] << ("%0.4f" % res).to_f
+              end
+            end
+
+            @graphs.each do |index, graph|
+              graph.data("#{marker}", g_data[index])
+            end
+          rescue Exception => e
+            raise "Gruff processing resulted in Error : " + e.backtrace.join("\n")
+          end
+        end 
+      end
+      
+      def _write_graphs
+        # write out resulting graphs
+        @metrics.each do |name, index|
+          image_fname = File.join(@output_dir,"#{name}.png")
+          puts "Gruff processor: writing #{image_fname}"
+          @graphs[index].write image_fname
+        end 
+      end
     end
-  rescue Exception => e
-    puts " Benchmark processing resulted in Error : " + e.backtrace.join("\n")
-    throw e
   end
 end
 
-# write out resulting graphs
-@metrics.each do |name, index|
-  puts "writing #{output_dir}/#{name}.png"
-  graphs[index].write "#{output_dir}/#{name}.png"
-end

data/bench/lib/bench/cli.rb

@@ -20,6 +20,7 @@ module Bench
       Statistics.new(Bench.concurrency,Bench.iterations,
         Bench.total_time,Bench.start_time,Bench.end_time,Bench.sessions).process.print_stats.save_results
       bench_log "Bench completed..."
+      Bench.reset_app
     end
   end